1 /* 2 * linux/mm/page_alloc.c 3 * 4 * Manages the free list, the system allocates free pages here. 5 * Note that kmalloc() lives in slab.c 6 * 7 * Copyright (C) 1991, 1992, 1993, 1994 Linus Torvalds 8 * Swap reorganised 29.12.95, Stephen Tweedie 9 * Support of BIGMEM added by Gerhard Wichert, Siemens AG, July 1999 10 * Reshaped it to be a zoned allocator, Ingo Molnar, Red Hat, 1999 11 * Discontiguous memory support, Kanoj Sarcar, SGI, Nov 1999 12 * Zone balancing, Kanoj Sarcar, SGI, Jan 2000 13 * Per cpu hot/cold page lists, bulk allocation, Martin J. Bligh, Sept 2002 14 * (lots of bits borrowed from Ingo Molnar & Andrew Morton) 15 */ 16 17 #include <linux/stddef.h> 18 #include <linux/mm.h> 19 #include <linux/swap.h> 20 #include <linux/interrupt.h> 21 #include <linux/pagemap.h> 22 #include <linux/jiffies.h> 23 #include <linux/bootmem.h> 24 #include <linux/memblock.h> 25 #include <linux/compiler.h> 26 #include <linux/kernel.h> 27 #include <linux/kmemcheck.h> 28 #include <linux/kasan.h> 29 #include <linux/module.h> 30 #include <linux/suspend.h> 31 #include <linux/pagevec.h> 32 #include <linux/blkdev.h> 33 #include <linux/slab.h> 34 #include <linux/ratelimit.h> 35 #include <linux/oom.h> 36 #include <linux/notifier.h> 37 #include <linux/topology.h> 38 #include <linux/sysctl.h> 39 #include <linux/cpu.h> 40 #include <linux/cpuset.h> 41 #include <linux/memory_hotplug.h> 42 #include <linux/nodemask.h> 43 #include <linux/vmalloc.h> 44 #include <linux/vmstat.h> 45 #include <linux/mempolicy.h> 46 #include <linux/memremap.h> 47 #include <linux/stop_machine.h> 48 #include <linux/sort.h> 49 #include <linux/pfn.h> 50 #include <linux/backing-dev.h> 51 #include <linux/fault-inject.h> 52 #include <linux/page-isolation.h> 53 #include <linux/page_ext.h> 54 #include <linux/debugobjects.h> 55 #include <linux/kmemleak.h> 56 #include <linux/compaction.h> 57 #include <trace/events/kmem.h> 58 #include <linux/prefetch.h> 59 #include <linux/mm_inline.h> 60 #include <linux/migrate.h> 61 #include <linux/page_ext.h> 62 #include <linux/hugetlb.h> 63 #include <linux/sched/rt.h> 64 #include <linux/page_owner.h> 65 #include <linux/kthread.h> 66 #include <linux/memcontrol.h> 67 68 #include <asm/sections.h> 69 #include <asm/tlbflush.h> 70 #include <asm/div64.h> 71 #include "internal.h" 72 73 /* prevent >1 _updater_ of zone percpu pageset ->high and ->batch fields */ 74 static DEFINE_MUTEX(pcp_batch_high_lock); 75 #define MIN_PERCPU_PAGELIST_FRACTION (8) 76 77 #ifdef CONFIG_USE_PERCPU_NUMA_NODE_ID 78 DEFINE_PER_CPU(int, numa_node); 79 EXPORT_PER_CPU_SYMBOL(numa_node); 80 #endif 81 82 #ifdef CONFIG_HAVE_MEMORYLESS_NODES 83 /* 84 * N.B., Do NOT reference the '_numa_mem_' per cpu variable directly. 85 * It will not be defined when CONFIG_HAVE_MEMORYLESS_NODES is not defined. 86 * Use the accessor functions set_numa_mem(), numa_mem_id() and cpu_to_mem() 87 * defined in <linux/topology.h>. 88 */ 89 DEFINE_PER_CPU(int, _numa_mem_); /* Kernel "local memory" node */ 90 EXPORT_PER_CPU_SYMBOL(_numa_mem_); 91 int _node_numa_mem_[MAX_NUMNODES]; 92 #endif 93 94 /* 95 * Array of node states. 96 */ 97 nodemask_t node_states[NR_NODE_STATES] __read_mostly = { 98 [N_POSSIBLE] = NODE_MASK_ALL, 99 [N_ONLINE] = { { [0] = 1UL } }, 100 #ifndef CONFIG_NUMA 101 [N_NORMAL_MEMORY] = { { [0] = 1UL } }, 102 #ifdef CONFIG_HIGHMEM 103 [N_HIGH_MEMORY] = { { [0] = 1UL } }, 104 #endif 105 #ifdef CONFIG_MOVABLE_NODE 106 [N_MEMORY] = { { [0] = 1UL } }, 107 #endif 108 [N_CPU] = { { [0] = 1UL } }, 109 #endif /* NUMA */ 110 }; 111 EXPORT_SYMBOL(node_states); 112 113 /* Protect totalram_pages and zone->managed_pages */ 114 static DEFINE_SPINLOCK(managed_page_count_lock); 115 116 unsigned long totalram_pages __read_mostly; 117 unsigned long totalreserve_pages __read_mostly; 118 unsigned long totalcma_pages __read_mostly; 119 120 int percpu_pagelist_fraction; 121 gfp_t gfp_allowed_mask __read_mostly = GFP_BOOT_MASK; 122 123 /* 124 * A cached value of the page's pageblock's migratetype, used when the page is 125 * put on a pcplist. Used to avoid the pageblock migratetype lookup when 126 * freeing from pcplists in most cases, at the cost of possibly becoming stale. 127 * Also the migratetype set in the page does not necessarily match the pcplist 128 * index, e.g. page might have MIGRATE_CMA set but be on a pcplist with any 129 * other index - this ensures that it will be put on the correct CMA freelist. 130 */ 131 static inline int get_pcppage_migratetype(struct page *page) 132 { 133 return page->index; 134 } 135 136 static inline void set_pcppage_migratetype(struct page *page, int migratetype) 137 { 138 page->index = migratetype; 139 } 140 141 #ifdef CONFIG_PM_SLEEP 142 /* 143 * The following functions are used by the suspend/hibernate code to temporarily 144 * change gfp_allowed_mask in order to avoid using I/O during memory allocations 145 * while devices are suspended. To avoid races with the suspend/hibernate code, 146 * they should always be called with pm_mutex held (gfp_allowed_mask also should 147 * only be modified with pm_mutex held, unless the suspend/hibernate code is 148 * guaranteed not to run in parallel with that modification). 149 */ 150 151 static gfp_t saved_gfp_mask; 152 153 void pm_restore_gfp_mask(void) 154 { 155 WARN_ON(!mutex_is_locked(&pm_mutex)); 156 if (saved_gfp_mask) { 157 gfp_allowed_mask = saved_gfp_mask; 158 saved_gfp_mask = 0; 159 } 160 } 161 162 void pm_restrict_gfp_mask(void) 163 { 164 WARN_ON(!mutex_is_locked(&pm_mutex)); 165 WARN_ON(saved_gfp_mask); 166 saved_gfp_mask = gfp_allowed_mask; 167 gfp_allowed_mask &= ~(__GFP_IO | __GFP_FS); 168 } 169 170 bool pm_suspended_storage(void) 171 { 172 if ((gfp_allowed_mask & (__GFP_IO | __GFP_FS)) == (__GFP_IO | __GFP_FS)) 173 return false; 174 return true; 175 } 176 #endif /* CONFIG_PM_SLEEP */ 177 178 #ifdef CONFIG_HUGETLB_PAGE_SIZE_VARIABLE 179 unsigned int pageblock_order __read_mostly; 180 #endif 181 182 static void __free_pages_ok(struct page *page, unsigned int order); 183 184 /* 185 * results with 256, 32 in the lowmem_reserve sysctl: 186 * 1G machine -> (16M dma, 800M-16M normal, 1G-800M high) 187 * 1G machine -> (16M dma, 784M normal, 224M high) 188 * NORMAL allocation will leave 784M/256 of ram reserved in the ZONE_DMA 189 * HIGHMEM allocation will leave 224M/32 of ram reserved in ZONE_NORMAL 190 * HIGHMEM allocation will leave (224M+784M)/256 of ram reserved in ZONE_DMA 191 * 192 * TBD: should special case ZONE_DMA32 machines here - in those we normally 193 * don't need any ZONE_NORMAL reservation 194 */ 195 int sysctl_lowmem_reserve_ratio[MAX_NR_ZONES-1] = { 196 #ifdef CONFIG_ZONE_DMA 197 256, 198 #endif 199 #ifdef CONFIG_ZONE_DMA32 200 256, 201 #endif 202 #ifdef CONFIG_HIGHMEM 203 32, 204 #endif 205 32, 206 }; 207 208 EXPORT_SYMBOL(totalram_pages); 209 210 static char * const zone_names[MAX_NR_ZONES] = { 211 #ifdef CONFIG_ZONE_DMA 212 "DMA", 213 #endif 214 #ifdef CONFIG_ZONE_DMA32 215 "DMA32", 216 #endif 217 "Normal", 218 #ifdef CONFIG_HIGHMEM 219 "HighMem", 220 #endif 221 "Movable", 222 #ifdef CONFIG_ZONE_DEVICE 223 "Device", 224 #endif 225 }; 226 227 char * const migratetype_names[MIGRATE_TYPES] = { 228 "Unmovable", 229 "Movable", 230 "Reclaimable", 231 "HighAtomic", 232 #ifdef CONFIG_CMA 233 "CMA", 234 #endif 235 #ifdef CONFIG_MEMORY_ISOLATION 236 "Isolate", 237 #endif 238 }; 239 240 compound_page_dtor * const compound_page_dtors[] = { 241 NULL, 242 free_compound_page, 243 #ifdef CONFIG_HUGETLB_PAGE 244 free_huge_page, 245 #endif 246 #ifdef CONFIG_TRANSPARENT_HUGEPAGE 247 free_transhuge_page, 248 #endif 249 }; 250 251 int min_free_kbytes = 1024; 252 int user_min_free_kbytes = -1; 253 int watermark_scale_factor = 10; 254 255 static unsigned long __meminitdata nr_kernel_pages; 256 static unsigned long __meminitdata nr_all_pages; 257 static unsigned long __meminitdata dma_reserve; 258 259 #ifdef CONFIG_HAVE_MEMBLOCK_NODE_MAP 260 static unsigned long __meminitdata arch_zone_lowest_possible_pfn[MAX_NR_ZONES]; 261 static unsigned long __meminitdata arch_zone_highest_possible_pfn[MAX_NR_ZONES]; 262 static unsigned long __initdata required_kernelcore; 263 static unsigned long __initdata required_movablecore; 264 static unsigned long __meminitdata zone_movable_pfn[MAX_NUMNODES]; 265 static bool mirrored_kernelcore; 266 267 /* movable_zone is the "real" zone pages in ZONE_MOVABLE are taken from */ 268 int movable_zone; 269 EXPORT_SYMBOL(movable_zone); 270 #endif /* CONFIG_HAVE_MEMBLOCK_NODE_MAP */ 271 272 #if MAX_NUMNODES > 1 273 int nr_node_ids __read_mostly = MAX_NUMNODES; 274 int nr_online_nodes __read_mostly = 1; 275 EXPORT_SYMBOL(nr_node_ids); 276 EXPORT_SYMBOL(nr_online_nodes); 277 #endif 278 279 int page_group_by_mobility_disabled __read_mostly; 280 281 #ifdef CONFIG_DEFERRED_STRUCT_PAGE_INIT 282 static inline void reset_deferred_meminit(pg_data_t *pgdat) 283 { 284 pgdat->first_deferred_pfn = ULONG_MAX; 285 } 286 287 /* Returns true if the struct page for the pfn is uninitialised */ 288 static inline bool __meminit early_page_uninitialised(unsigned long pfn) 289 { 290 int nid = early_pfn_to_nid(pfn); 291 292 if (node_online(nid) && pfn >= NODE_DATA(nid)->first_deferred_pfn) 293 return true; 294 295 return false; 296 } 297 298 /* 299 * Returns false when the remaining initialisation should be deferred until 300 * later in the boot cycle when it can be parallelised. 301 */ 302 static inline bool update_defer_init(pg_data_t *pgdat, 303 unsigned long pfn, unsigned long zone_end, 304 unsigned long *nr_initialised) 305 { 306 unsigned long max_initialise; 307 308 /* Always populate low zones for address-contrained allocations */ 309 if (zone_end < pgdat_end_pfn(pgdat)) 310 return true; 311 /* 312 * Initialise at least 2G of a node but also take into account that 313 * two large system hashes that can take up 1GB for 0.25TB/node. 314 */ 315 max_initialise = max(2UL << (30 - PAGE_SHIFT), 316 (pgdat->node_spanned_pages >> 8)); 317 318 (*nr_initialised)++; 319 if ((*nr_initialised > max_initialise) && 320 (pfn & (PAGES_PER_SECTION - 1)) == 0) { 321 pgdat->first_deferred_pfn = pfn; 322 return false; 323 } 324 325 return true; 326 } 327 #else 328 static inline void reset_deferred_meminit(pg_data_t *pgdat) 329 { 330 } 331 332 static inline bool early_page_uninitialised(unsigned long pfn) 333 { 334 return false; 335 } 336 337 static inline bool update_defer_init(pg_data_t *pgdat, 338 unsigned long pfn, unsigned long zone_end, 339 unsigned long *nr_initialised) 340 { 341 return true; 342 } 343 #endif 344 345 /* Return a pointer to the bitmap storing bits affecting a block of pages */ 346 static inline unsigned long *get_pageblock_bitmap(struct page *page, 347 unsigned long pfn) 348 { 349 #ifdef CONFIG_SPARSEMEM 350 return __pfn_to_section(pfn)->pageblock_flags; 351 #else 352 return page_zone(page)->pageblock_flags; 353 #endif /* CONFIG_SPARSEMEM */ 354 } 355 356 static inline int pfn_to_bitidx(struct page *page, unsigned long pfn) 357 { 358 #ifdef CONFIG_SPARSEMEM 359 pfn &= (PAGES_PER_SECTION-1); 360 return (pfn >> pageblock_order) * NR_PAGEBLOCK_BITS; 361 #else 362 pfn = pfn - round_down(page_zone(page)->zone_start_pfn, pageblock_nr_pages); 363 return (pfn >> pageblock_order) * NR_PAGEBLOCK_BITS; 364 #endif /* CONFIG_SPARSEMEM */ 365 } 366 367 /** 368 * get_pfnblock_flags_mask - Return the requested group of flags for the pageblock_nr_pages block of pages 369 * @page: The page within the block of interest 370 * @pfn: The target page frame number 371 * @end_bitidx: The last bit of interest to retrieve 372 * @mask: mask of bits that the caller is interested in 373 * 374 * Return: pageblock_bits flags 375 */ 376 static __always_inline unsigned long __get_pfnblock_flags_mask(struct page *page, 377 unsigned long pfn, 378 unsigned long end_bitidx, 379 unsigned long mask) 380 { 381 unsigned long *bitmap; 382 unsigned long bitidx, word_bitidx; 383 unsigned long word; 384 385 bitmap = get_pageblock_bitmap(page, pfn); 386 bitidx = pfn_to_bitidx(page, pfn); 387 word_bitidx = bitidx / BITS_PER_LONG; 388 bitidx &= (BITS_PER_LONG-1); 389 390 word = bitmap[word_bitidx]; 391 bitidx += end_bitidx; 392 return (word >> (BITS_PER_LONG - bitidx - 1)) & mask; 393 } 394 395 unsigned long get_pfnblock_flags_mask(struct page *page, unsigned long pfn, 396 unsigned long end_bitidx, 397 unsigned long mask) 398 { 399 return __get_pfnblock_flags_mask(page, pfn, end_bitidx, mask); 400 } 401 402 static __always_inline int get_pfnblock_migratetype(struct page *page, unsigned long pfn) 403 { 404 return __get_pfnblock_flags_mask(page, pfn, PB_migrate_end, MIGRATETYPE_MASK); 405 } 406 407 /** 408 * set_pfnblock_flags_mask - Set the requested group of flags for a pageblock_nr_pages block of pages 409 * @page: The page within the block of interest 410 * @flags: The flags to set 411 * @pfn: The target page frame number 412 * @end_bitidx: The last bit of interest 413 * @mask: mask of bits that the caller is interested in 414 */ 415 void set_pfnblock_flags_mask(struct page *page, unsigned long flags, 416 unsigned long pfn, 417 unsigned long end_bitidx, 418 unsigned long mask) 419 { 420 unsigned long *bitmap; 421 unsigned long bitidx, word_bitidx; 422 unsigned long old_word, word; 423 424 BUILD_BUG_ON(NR_PAGEBLOCK_BITS != 4); 425 426 bitmap = get_pageblock_bitmap(page, pfn); 427 bitidx = pfn_to_bitidx(page, pfn); 428 word_bitidx = bitidx / BITS_PER_LONG; 429 bitidx &= (BITS_PER_LONG-1); 430 431 VM_BUG_ON_PAGE(!zone_spans_pfn(page_zone(page), pfn), page); 432 433 bitidx += end_bitidx; 434 mask <<= (BITS_PER_LONG - bitidx - 1); 435 flags <<= (BITS_PER_LONG - bitidx - 1); 436 437 word = READ_ONCE(bitmap[word_bitidx]); 438 for (;;) { 439 old_word = cmpxchg(&bitmap[word_bitidx], word, (word & ~mask) | flags); 440 if (word == old_word) 441 break; 442 word = old_word; 443 } 444 } 445 446 void set_pageblock_migratetype(struct page *page, int migratetype) 447 { 448 if (unlikely(page_group_by_mobility_disabled && 449 migratetype < MIGRATE_PCPTYPES)) 450 migratetype = MIGRATE_UNMOVABLE; 451 452 set_pageblock_flags_group(page, (unsigned long)migratetype, 453 PB_migrate, PB_migrate_end); 454 } 455 456 #ifdef CONFIG_DEBUG_VM 457 static int page_outside_zone_boundaries(struct zone *zone, struct page *page) 458 { 459 int ret = 0; 460 unsigned seq; 461 unsigned long pfn = page_to_pfn(page); 462 unsigned long sp, start_pfn; 463 464 do { 465 seq = zone_span_seqbegin(zone); 466 start_pfn = zone->zone_start_pfn; 467 sp = zone->spanned_pages; 468 if (!zone_spans_pfn(zone, pfn)) 469 ret = 1; 470 } while (zone_span_seqretry(zone, seq)); 471 472 if (ret) 473 pr_err("page 0x%lx outside node %d zone %s [ 0x%lx - 0x%lx ]\n", 474 pfn, zone_to_nid(zone), zone->name, 475 start_pfn, start_pfn + sp); 476 477 return ret; 478 } 479 480 static int page_is_consistent(struct zone *zone, struct page *page) 481 { 482 if (!pfn_valid_within(page_to_pfn(page))) 483 return 0; 484 if (zone != page_zone(page)) 485 return 0; 486 487 return 1; 488 } 489 /* 490 * Temporary debugging check for pages not lying within a given zone. 491 */ 492 static int bad_range(struct zone *zone, struct page *page) 493 { 494 if (page_outside_zone_boundaries(zone, page)) 495 return 1; 496 if (!page_is_consistent(zone, page)) 497 return 1; 498 499 return 0; 500 } 501 #else 502 static inline int bad_range(struct zone *zone, struct page *page) 503 { 504 return 0; 505 } 506 #endif 507 508 static void bad_page(struct page *page, const char *reason, 509 unsigned long bad_flags) 510 { 511 static unsigned long resume; 512 static unsigned long nr_shown; 513 static unsigned long nr_unshown; 514 515 /* 516 * Allow a burst of 60 reports, then keep quiet for that minute; 517 * or allow a steady drip of one report per second. 518 */ 519 if (nr_shown == 60) { 520 if (time_before(jiffies, resume)) { 521 nr_unshown++; 522 goto out; 523 } 524 if (nr_unshown) { 525 pr_alert( 526 "BUG: Bad page state: %lu messages suppressed\n", 527 nr_unshown); 528 nr_unshown = 0; 529 } 530 nr_shown = 0; 531 } 532 if (nr_shown++ == 0) 533 resume = jiffies + 60 * HZ; 534 535 pr_alert("BUG: Bad page state in process %s pfn:%05lx\n", 536 current->comm, page_to_pfn(page)); 537 __dump_page(page, reason); 538 bad_flags &= page->flags; 539 if (bad_flags) 540 pr_alert("bad because of flags: %#lx(%pGp)\n", 541 bad_flags, &bad_flags); 542 dump_page_owner(page); 543 544 print_modules(); 545 dump_stack(); 546 out: 547 /* Leave bad fields for debug, except PageBuddy could make trouble */ 548 page_mapcount_reset(page); /* remove PageBuddy */ 549 add_taint(TAINT_BAD_PAGE, LOCKDEP_NOW_UNRELIABLE); 550 } 551 552 /* 553 * Higher-order pages are called "compound pages". They are structured thusly: 554 * 555 * The first PAGE_SIZE page is called the "head page" and have PG_head set. 556 * 557 * The remaining PAGE_SIZE pages are called "tail pages". PageTail() is encoded 558 * in bit 0 of page->compound_head. The rest of bits is pointer to head page. 559 * 560 * The first tail page's ->compound_dtor holds the offset in array of compound 561 * page destructors. See compound_page_dtors. 562 * 563 * The first tail page's ->compound_order holds the order of allocation. 564 * This usage means that zero-order pages may not be compound. 565 */ 566 567 void free_compound_page(struct page *page) 568 { 569 __free_pages_ok(page, compound_order(page)); 570 } 571 572 void prep_compound_page(struct page *page, unsigned int order) 573 { 574 int i; 575 int nr_pages = 1 << order; 576 577 set_compound_page_dtor(page, COMPOUND_PAGE_DTOR); 578 set_compound_order(page, order); 579 __SetPageHead(page); 580 for (i = 1; i < nr_pages; i++) { 581 struct page *p = page + i; 582 set_page_count(p, 0); 583 p->mapping = TAIL_MAPPING; 584 set_compound_head(p, page); 585 } 586 atomic_set(compound_mapcount_ptr(page), -1); 587 } 588 589 #ifdef CONFIG_DEBUG_PAGEALLOC 590 unsigned int _debug_guardpage_minorder; 591 bool _debug_pagealloc_enabled __read_mostly 592 = IS_ENABLED(CONFIG_DEBUG_PAGEALLOC_ENABLE_DEFAULT); 593 EXPORT_SYMBOL(_debug_pagealloc_enabled); 594 bool _debug_guardpage_enabled __read_mostly; 595 596 static int __init early_debug_pagealloc(char *buf) 597 { 598 if (!buf) 599 return -EINVAL; 600 return kstrtobool(buf, &_debug_pagealloc_enabled); 601 } 602 early_param("debug_pagealloc", early_debug_pagealloc); 603 604 static bool need_debug_guardpage(void) 605 { 606 /* If we don't use debug_pagealloc, we don't need guard page */ 607 if (!debug_pagealloc_enabled()) 608 return false; 609 610 return true; 611 } 612 613 static void init_debug_guardpage(void) 614 { 615 if (!debug_pagealloc_enabled()) 616 return; 617 618 _debug_guardpage_enabled = true; 619 } 620 621 struct page_ext_operations debug_guardpage_ops = { 622 .need = need_debug_guardpage, 623 .init = init_debug_guardpage, 624 }; 625 626 static int __init debug_guardpage_minorder_setup(char *buf) 627 { 628 unsigned long res; 629 630 if (kstrtoul(buf, 10, &res) < 0 || res > MAX_ORDER / 2) { 631 pr_err("Bad debug_guardpage_minorder value\n"); 632 return 0; 633 } 634 _debug_guardpage_minorder = res; 635 pr_info("Setting debug_guardpage_minorder to %lu\n", res); 636 return 0; 637 } 638 __setup("debug_guardpage_minorder=", debug_guardpage_minorder_setup); 639 640 static inline void set_page_guard(struct zone *zone, struct page *page, 641 unsigned int order, int migratetype) 642 { 643 struct page_ext *page_ext; 644 645 if (!debug_guardpage_enabled()) 646 return; 647 648 page_ext = lookup_page_ext(page); 649 if (unlikely(!page_ext)) 650 return; 651 652 __set_bit(PAGE_EXT_DEBUG_GUARD, &page_ext->flags); 653 654 INIT_LIST_HEAD(&page->lru); 655 set_page_private(page, order); 656 /* Guard pages are not available for any usage */ 657 __mod_zone_freepage_state(zone, -(1 << order), migratetype); 658 } 659 660 static inline void clear_page_guard(struct zone *zone, struct page *page, 661 unsigned int order, int migratetype) 662 { 663 struct page_ext *page_ext; 664 665 if (!debug_guardpage_enabled()) 666 return; 667 668 page_ext = lookup_page_ext(page); 669 if (unlikely(!page_ext)) 670 return; 671 672 __clear_bit(PAGE_EXT_DEBUG_GUARD, &page_ext->flags); 673 674 set_page_private(page, 0); 675 if (!is_migrate_isolate(migratetype)) 676 __mod_zone_freepage_state(zone, (1 << order), migratetype); 677 } 678 #else 679 struct page_ext_operations debug_guardpage_ops = { NULL, }; 680 static inline void set_page_guard(struct zone *zone, struct page *page, 681 unsigned int order, int migratetype) {} 682 static inline void clear_page_guard(struct zone *zone, struct page *page, 683 unsigned int order, int migratetype) {} 684 #endif 685 686 static inline void set_page_order(struct page *page, unsigned int order) 687 { 688 set_page_private(page, order); 689 __SetPageBuddy(page); 690 } 691 692 static inline void rmv_page_order(struct page *page) 693 { 694 __ClearPageBuddy(page); 695 set_page_private(page, 0); 696 } 697 698 /* 699 * This function checks whether a page is free && is the buddy 700 * we can do coalesce a page and its buddy if 701 * (a) the buddy is not in a hole && 702 * (b) the buddy is in the buddy system && 703 * (c) a page and its buddy have the same order && 704 * (d) a page and its buddy are in the same zone. 705 * 706 * For recording whether a page is in the buddy system, we set ->_mapcount 707 * PAGE_BUDDY_MAPCOUNT_VALUE. 708 * Setting, clearing, and testing _mapcount PAGE_BUDDY_MAPCOUNT_VALUE is 709 * serialized by zone->lock. 710 * 711 * For recording page's order, we use page_private(page). 712 */ 713 static inline int page_is_buddy(struct page *page, struct page *buddy, 714 unsigned int order) 715 { 716 if (!pfn_valid_within(page_to_pfn(buddy))) 717 return 0; 718 719 if (page_is_guard(buddy) && page_order(buddy) == order) { 720 if (page_zone_id(page) != page_zone_id(buddy)) 721 return 0; 722 723 VM_BUG_ON_PAGE(page_count(buddy) != 0, buddy); 724 725 return 1; 726 } 727 728 if (PageBuddy(buddy) && page_order(buddy) == order) { 729 /* 730 * zone check is done late to avoid uselessly 731 * calculating zone/node ids for pages that could 732 * never merge. 733 */ 734 if (page_zone_id(page) != page_zone_id(buddy)) 735 return 0; 736 737 VM_BUG_ON_PAGE(page_count(buddy) != 0, buddy); 738 739 return 1; 740 } 741 return 0; 742 } 743 744 /* 745 * Freeing function for a buddy system allocator. 746 * 747 * The concept of a buddy system is to maintain direct-mapped table 748 * (containing bit values) for memory blocks of various "orders". 749 * The bottom level table contains the map for the smallest allocatable 750 * units of memory (here, pages), and each level above it describes 751 * pairs of units from the levels below, hence, "buddies". 752 * At a high level, all that happens here is marking the table entry 753 * at the bottom level available, and propagating the changes upward 754 * as necessary, plus some accounting needed to play nicely with other 755 * parts of the VM system. 756 * At each level, we keep a list of pages, which are heads of continuous 757 * free pages of length of (1 << order) and marked with _mapcount 758 * PAGE_BUDDY_MAPCOUNT_VALUE. Page's order is recorded in page_private(page) 759 * field. 760 * So when we are allocating or freeing one, we can derive the state of the 761 * other. That is, if we allocate a small block, and both were 762 * free, the remainder of the region must be split into blocks. 763 * If a block is freed, and its buddy is also free, then this 764 * triggers coalescing into a block of larger size. 765 * 766 * -- nyc 767 */ 768 769 static inline void __free_one_page(struct page *page, 770 unsigned long pfn, 771 struct zone *zone, unsigned int order, 772 int migratetype) 773 { 774 unsigned long page_idx; 775 unsigned long combined_idx; 776 unsigned long uninitialized_var(buddy_idx); 777 struct page *buddy; 778 unsigned int max_order; 779 780 max_order = min_t(unsigned int, MAX_ORDER, pageblock_order + 1); 781 782 VM_BUG_ON(!zone_is_initialized(zone)); 783 VM_BUG_ON_PAGE(page->flags & PAGE_FLAGS_CHECK_AT_PREP, page); 784 785 VM_BUG_ON(migratetype == -1); 786 if (likely(!is_migrate_isolate(migratetype))) 787 __mod_zone_freepage_state(zone, 1 << order, migratetype); 788 789 page_idx = pfn & ((1 << MAX_ORDER) - 1); 790 791 VM_BUG_ON_PAGE(page_idx & ((1 << order) - 1), page); 792 VM_BUG_ON_PAGE(bad_range(zone, page), page); 793 794 continue_merging: 795 while (order < max_order - 1) { 796 buddy_idx = __find_buddy_index(page_idx, order); 797 buddy = page + (buddy_idx - page_idx); 798 if (!page_is_buddy(page, buddy, order)) 799 goto done_merging; 800 /* 801 * Our buddy is free or it is CONFIG_DEBUG_PAGEALLOC guard page, 802 * merge with it and move up one order. 803 */ 804 if (page_is_guard(buddy)) { 805 clear_page_guard(zone, buddy, order, migratetype); 806 } else { 807 list_del(&buddy->lru); 808 zone->free_area[order].nr_free--; 809 rmv_page_order(buddy); 810 } 811 combined_idx = buddy_idx & page_idx; 812 page = page + (combined_idx - page_idx); 813 page_idx = combined_idx; 814 order++; 815 } 816 if (max_order < MAX_ORDER) { 817 /* If we are here, it means order is >= pageblock_order. 818 * We want to prevent merge between freepages on isolate 819 * pageblock and normal pageblock. Without this, pageblock 820 * isolation could cause incorrect freepage or CMA accounting. 821 * 822 * We don't want to hit this code for the more frequent 823 * low-order merging. 824 */ 825 if (unlikely(has_isolate_pageblock(zone))) { 826 int buddy_mt; 827 828 buddy_idx = __find_buddy_index(page_idx, order); 829 buddy = page + (buddy_idx - page_idx); 830 buddy_mt = get_pageblock_migratetype(buddy); 831 832 if (migratetype != buddy_mt 833 && (is_migrate_isolate(migratetype) || 834 is_migrate_isolate(buddy_mt))) 835 goto done_merging; 836 } 837 max_order++; 838 goto continue_merging; 839 } 840 841 done_merging: 842 set_page_order(page, order); 843 844 /* 845 * If this is not the largest possible page, check if the buddy 846 * of the next-highest order is free. If it is, it's possible 847 * that pages are being freed that will coalesce soon. In case, 848 * that is happening, add the free page to the tail of the list 849 * so it's less likely to be used soon and more likely to be merged 850 * as a higher order page 851 */ 852 if ((order < MAX_ORDER-2) && pfn_valid_within(page_to_pfn(buddy))) { 853 struct page *higher_page, *higher_buddy; 854 combined_idx = buddy_idx & page_idx; 855 higher_page = page + (combined_idx - page_idx); 856 buddy_idx = __find_buddy_index(combined_idx, order + 1); 857 higher_buddy = higher_page + (buddy_idx - combined_idx); 858 if (page_is_buddy(higher_page, higher_buddy, order + 1)) { 859 list_add_tail(&page->lru, 860 &zone->free_area[order].free_list[migratetype]); 861 goto out; 862 } 863 } 864 865 list_add(&page->lru, &zone->free_area[order].free_list[migratetype]); 866 out: 867 zone->free_area[order].nr_free++; 868 } 869 870 /* 871 * A bad page could be due to a number of fields. Instead of multiple branches, 872 * try and check multiple fields with one check. The caller must do a detailed 873 * check if necessary. 874 */ 875 static inline bool page_expected_state(struct page *page, 876 unsigned long check_flags) 877 { 878 if (unlikely(atomic_read(&page->_mapcount) != -1)) 879 return false; 880 881 if (unlikely((unsigned long)page->mapping | 882 page_ref_count(page) | 883 #ifdef CONFIG_MEMCG 884 (unsigned long)page->mem_cgroup | 885 #endif 886 (page->flags & check_flags))) 887 return false; 888 889 return true; 890 } 891 892 static void free_pages_check_bad(struct page *page) 893 { 894 const char *bad_reason; 895 unsigned long bad_flags; 896 897 bad_reason = NULL; 898 bad_flags = 0; 899 900 if (unlikely(atomic_read(&page->_mapcount) != -1)) 901 bad_reason = "nonzero mapcount"; 902 if (unlikely(page->mapping != NULL)) 903 bad_reason = "non-NULL mapping"; 904 if (unlikely(page_ref_count(page) != 0)) 905 bad_reason = "nonzero _refcount"; 906 if (unlikely(page->flags & PAGE_FLAGS_CHECK_AT_FREE)) { 907 bad_reason = "PAGE_FLAGS_CHECK_AT_FREE flag(s) set"; 908 bad_flags = PAGE_FLAGS_CHECK_AT_FREE; 909 } 910 #ifdef CONFIG_MEMCG 911 if (unlikely(page->mem_cgroup)) 912 bad_reason = "page still charged to cgroup"; 913 #endif 914 bad_page(page, bad_reason, bad_flags); 915 } 916 917 static inline int free_pages_check(struct page *page) 918 { 919 if (likely(page_expected_state(page, PAGE_FLAGS_CHECK_AT_FREE))) 920 return 0; 921 922 /* Something has gone sideways, find it */ 923 free_pages_check_bad(page); 924 return 1; 925 } 926 927 static int free_tail_pages_check(struct page *head_page, struct page *page) 928 { 929 int ret = 1; 930 931 /* 932 * We rely page->lru.next never has bit 0 set, unless the page 933 * is PageTail(). Let's make sure that's true even for poisoned ->lru. 934 */ 935 BUILD_BUG_ON((unsigned long)LIST_POISON1 & 1); 936 937 if (!IS_ENABLED(CONFIG_DEBUG_VM)) { 938 ret = 0; 939 goto out; 940 } 941 switch (page - head_page) { 942 case 1: 943 /* the first tail page: ->mapping is compound_mapcount() */ 944 if (unlikely(compound_mapcount(page))) { 945 bad_page(page, "nonzero compound_mapcount", 0); 946 goto out; 947 } 948 break; 949 case 2: 950 /* 951 * the second tail page: ->mapping is 952 * page_deferred_list().next -- ignore value. 953 */ 954 break; 955 default: 956 if (page->mapping != TAIL_MAPPING) { 957 bad_page(page, "corrupted mapping in tail page", 0); 958 goto out; 959 } 960 break; 961 } 962 if (unlikely(!PageTail(page))) { 963 bad_page(page, "PageTail not set", 0); 964 goto out; 965 } 966 if (unlikely(compound_head(page) != head_page)) { 967 bad_page(page, "compound_head not consistent", 0); 968 goto out; 969 } 970 ret = 0; 971 out: 972 page->mapping = NULL; 973 clear_compound_head(page); 974 return ret; 975 } 976 977 static __always_inline bool free_pages_prepare(struct page *page, 978 unsigned int order, bool check_free) 979 { 980 int bad = 0; 981 982 VM_BUG_ON_PAGE(PageTail(page), page); 983 984 trace_mm_page_free(page, order); 985 kmemcheck_free_shadow(page, order); 986 987 /* 988 * Check tail pages before head page information is cleared to 989 * avoid checking PageCompound for order-0 pages. 990 */ 991 if (unlikely(order)) { 992 bool compound = PageCompound(page); 993 int i; 994 995 VM_BUG_ON_PAGE(compound && compound_order(page) != order, page); 996 997 if (compound) 998 ClearPageDoubleMap(page); 999 for (i = 1; i < (1 << order); i++) { 1000 if (compound) 1001 bad += free_tail_pages_check(page, page + i); 1002 if (unlikely(free_pages_check(page + i))) { 1003 bad++; 1004 continue; 1005 } 1006 (page + i)->flags &= ~PAGE_FLAGS_CHECK_AT_PREP; 1007 } 1008 } 1009 if (PageMappingFlags(page)) 1010 page->mapping = NULL; 1011 if (memcg_kmem_enabled() && PageKmemcg(page)) 1012 memcg_kmem_uncharge(page, order); 1013 if (check_free) 1014 bad += free_pages_check(page); 1015 if (bad) 1016 return false; 1017 1018 page_cpupid_reset_last(page); 1019 page->flags &= ~PAGE_FLAGS_CHECK_AT_PREP; 1020 reset_page_owner(page, order); 1021 1022 if (!PageHighMem(page)) { 1023 debug_check_no_locks_freed(page_address(page), 1024 PAGE_SIZE << order); 1025 debug_check_no_obj_freed(page_address(page), 1026 PAGE_SIZE << order); 1027 } 1028 arch_free_page(page, order); 1029 kernel_poison_pages(page, 1 << order, 0); 1030 kernel_map_pages(page, 1 << order, 0); 1031 kasan_free_pages(page, order); 1032 1033 return true; 1034 } 1035 1036 #ifdef CONFIG_DEBUG_VM 1037 static inline bool free_pcp_prepare(struct page *page) 1038 { 1039 return free_pages_prepare(page, 0, true); 1040 } 1041 1042 static inline bool bulkfree_pcp_prepare(struct page *page) 1043 { 1044 return false; 1045 } 1046 #else 1047 static bool free_pcp_prepare(struct page *page) 1048 { 1049 return free_pages_prepare(page, 0, false); 1050 } 1051 1052 static bool bulkfree_pcp_prepare(struct page *page) 1053 { 1054 return free_pages_check(page); 1055 } 1056 #endif /* CONFIG_DEBUG_VM */ 1057 1058 /* 1059 * Frees a number of pages from the PCP lists 1060 * Assumes all pages on list are in same zone, and of same order. 1061 * count is the number of pages to free. 1062 * 1063 * If the zone was previously in an "all pages pinned" state then look to 1064 * see if this freeing clears that state. 1065 * 1066 * And clear the zone's pages_scanned counter, to hold off the "all pages are 1067 * pinned" detection logic. 1068 */ 1069 static void free_pcppages_bulk(struct zone *zone, int count, 1070 struct per_cpu_pages *pcp) 1071 { 1072 int migratetype = 0; 1073 int batch_free = 0; 1074 unsigned long nr_scanned; 1075 bool isolated_pageblocks; 1076 1077 spin_lock(&zone->lock); 1078 isolated_pageblocks = has_isolate_pageblock(zone); 1079 nr_scanned = node_page_state(zone->zone_pgdat, NR_PAGES_SCANNED); 1080 if (nr_scanned) 1081 __mod_node_page_state(zone->zone_pgdat, NR_PAGES_SCANNED, -nr_scanned); 1082 1083 while (count) { 1084 struct page *page; 1085 struct list_head *list; 1086 1087 /* 1088 * Remove pages from lists in a round-robin fashion. A 1089 * batch_free count is maintained that is incremented when an 1090 * empty list is encountered. This is so more pages are freed 1091 * off fuller lists instead of spinning excessively around empty 1092 * lists 1093 */ 1094 do { 1095 batch_free++; 1096 if (++migratetype == MIGRATE_PCPTYPES) 1097 migratetype = 0; 1098 list = &pcp->lists[migratetype]; 1099 } while (list_empty(list)); 1100 1101 /* This is the only non-empty list. Free them all. */ 1102 if (batch_free == MIGRATE_PCPTYPES) 1103 batch_free = count; 1104 1105 do { 1106 int mt; /* migratetype of the to-be-freed page */ 1107 1108 page = list_last_entry(list, struct page, lru); 1109 /* must delete as __free_one_page list manipulates */ 1110 list_del(&page->lru); 1111 1112 mt = get_pcppage_migratetype(page); 1113 /* MIGRATE_ISOLATE page should not go to pcplists */ 1114 VM_BUG_ON_PAGE(is_migrate_isolate(mt), page); 1115 /* Pageblock could have been isolated meanwhile */ 1116 if (unlikely(isolated_pageblocks)) 1117 mt = get_pageblock_migratetype(page); 1118 1119 if (bulkfree_pcp_prepare(page)) 1120 continue; 1121 1122 __free_one_page(page, page_to_pfn(page), zone, 0, mt); 1123 trace_mm_page_pcpu_drain(page, 0, mt); 1124 } while (--count && --batch_free && !list_empty(list)); 1125 } 1126 spin_unlock(&zone->lock); 1127 } 1128 1129 static void free_one_page(struct zone *zone, 1130 struct page *page, unsigned long pfn, 1131 unsigned int order, 1132 int migratetype) 1133 { 1134 unsigned long nr_scanned; 1135 spin_lock(&zone->lock); 1136 nr_scanned = node_page_state(zone->zone_pgdat, NR_PAGES_SCANNED); 1137 if (nr_scanned) 1138 __mod_node_page_state(zone->zone_pgdat, NR_PAGES_SCANNED, -nr_scanned); 1139 1140 if (unlikely(has_isolate_pageblock(zone) || 1141 is_migrate_isolate(migratetype))) { 1142 migratetype = get_pfnblock_migratetype(page, pfn); 1143 } 1144 __free_one_page(page, pfn, zone, order, migratetype); 1145 spin_unlock(&zone->lock); 1146 } 1147 1148 static void __meminit __init_single_page(struct page *page, unsigned long pfn, 1149 unsigned long zone, int nid) 1150 { 1151 set_page_links(page, zone, nid, pfn); 1152 init_page_count(page); 1153 page_mapcount_reset(page); 1154 page_cpupid_reset_last(page); 1155 1156 INIT_LIST_HEAD(&page->lru); 1157 #ifdef WANT_PAGE_VIRTUAL 1158 /* The shift won't overflow because ZONE_NORMAL is below 4G. */ 1159 if (!is_highmem_idx(zone)) 1160 set_page_address(page, __va(pfn << PAGE_SHIFT)); 1161 #endif 1162 } 1163 1164 static void __meminit __init_single_pfn(unsigned long pfn, unsigned long zone, 1165 int nid) 1166 { 1167 return __init_single_page(pfn_to_page(pfn), pfn, zone, nid); 1168 } 1169 1170 #ifdef CONFIG_DEFERRED_STRUCT_PAGE_INIT 1171 static void init_reserved_page(unsigned long pfn) 1172 { 1173 pg_data_t *pgdat; 1174 int nid, zid; 1175 1176 if (!early_page_uninitialised(pfn)) 1177 return; 1178 1179 nid = early_pfn_to_nid(pfn); 1180 pgdat = NODE_DATA(nid); 1181 1182 for (zid = 0; zid < MAX_NR_ZONES; zid++) { 1183 struct zone *zone = &pgdat->node_zones[zid]; 1184 1185 if (pfn >= zone->zone_start_pfn && pfn < zone_end_pfn(zone)) 1186 break; 1187 } 1188 __init_single_pfn(pfn, zid, nid); 1189 } 1190 #else 1191 static inline void init_reserved_page(unsigned long pfn) 1192 { 1193 } 1194 #endif /* CONFIG_DEFERRED_STRUCT_PAGE_INIT */ 1195 1196 /* 1197 * Initialised pages do not have PageReserved set. This function is 1198 * called for each range allocated by the bootmem allocator and 1199 * marks the pages PageReserved. The remaining valid pages are later 1200 * sent to the buddy page allocator. 1201 */ 1202 void __meminit reserve_bootmem_region(phys_addr_t start, phys_addr_t end) 1203 { 1204 unsigned long start_pfn = PFN_DOWN(start); 1205 unsigned long end_pfn = PFN_UP(end); 1206 1207 for (; start_pfn < end_pfn; start_pfn++) { 1208 if (pfn_valid(start_pfn)) { 1209 struct page *page = pfn_to_page(start_pfn); 1210 1211 init_reserved_page(start_pfn); 1212 1213 /* Avoid false-positive PageTail() */ 1214 INIT_LIST_HEAD(&page->lru); 1215 1216 SetPageReserved(page); 1217 } 1218 } 1219 } 1220 1221 static void __free_pages_ok(struct page *page, unsigned int order) 1222 { 1223 unsigned long flags; 1224 int migratetype; 1225 unsigned long pfn = page_to_pfn(page); 1226 1227 if (!free_pages_prepare(page, order, true)) 1228 return; 1229 1230 migratetype = get_pfnblock_migratetype(page, pfn); 1231 local_irq_save(flags); 1232 __count_vm_events(PGFREE, 1 << order); 1233 free_one_page(page_zone(page), page, pfn, order, migratetype); 1234 local_irq_restore(flags); 1235 } 1236 1237 static void __init __free_pages_boot_core(struct page *page, unsigned int order) 1238 { 1239 unsigned int nr_pages = 1 << order; 1240 struct page *p = page; 1241 unsigned int loop; 1242 1243 prefetchw(p); 1244 for (loop = 0; loop < (nr_pages - 1); loop++, p++) { 1245 prefetchw(p + 1); 1246 __ClearPageReserved(p); 1247 set_page_count(p, 0); 1248 } 1249 __ClearPageReserved(p); 1250 set_page_count(p, 0); 1251 1252 page_zone(page)->managed_pages += nr_pages; 1253 set_page_refcounted(page); 1254 __free_pages(page, order); 1255 } 1256 1257 #if defined(CONFIG_HAVE_ARCH_EARLY_PFN_TO_NID) || \ 1258 defined(CONFIG_HAVE_MEMBLOCK_NODE_MAP) 1259 1260 static struct mminit_pfnnid_cache early_pfnnid_cache __meminitdata; 1261 1262 int __meminit early_pfn_to_nid(unsigned long pfn) 1263 { 1264 static DEFINE_SPINLOCK(early_pfn_lock); 1265 int nid; 1266 1267 spin_lock(&early_pfn_lock); 1268 nid = __early_pfn_to_nid(pfn, &early_pfnnid_cache); 1269 if (nid < 0) 1270 nid = first_online_node; 1271 spin_unlock(&early_pfn_lock); 1272 1273 return nid; 1274 } 1275 #endif 1276 1277 #ifdef CONFIG_NODES_SPAN_OTHER_NODES 1278 static inline bool __meminit meminit_pfn_in_nid(unsigned long pfn, int node, 1279 struct mminit_pfnnid_cache *state) 1280 { 1281 int nid; 1282 1283 nid = __early_pfn_to_nid(pfn, state); 1284 if (nid >= 0 && nid != node) 1285 return false; 1286 return true; 1287 } 1288 1289 /* Only safe to use early in boot when initialisation is single-threaded */ 1290 static inline bool __meminit early_pfn_in_nid(unsigned long pfn, int node) 1291 { 1292 return meminit_pfn_in_nid(pfn, node, &early_pfnnid_cache); 1293 } 1294 1295 #else 1296 1297 static inline bool __meminit early_pfn_in_nid(unsigned long pfn, int node) 1298 { 1299 return true; 1300 } 1301 static inline bool __meminit meminit_pfn_in_nid(unsigned long pfn, int node, 1302 struct mminit_pfnnid_cache *state) 1303 { 1304 return true; 1305 } 1306 #endif 1307 1308 1309 void __init __free_pages_bootmem(struct page *page, unsigned long pfn, 1310 unsigned int order) 1311 { 1312 if (early_page_uninitialised(pfn)) 1313 return; 1314 return __free_pages_boot_core(page, order); 1315 } 1316 1317 /* 1318 * Check that the whole (or subset of) a pageblock given by the interval of 1319 * [start_pfn, end_pfn) is valid and within the same zone, before scanning it 1320 * with the migration of free compaction scanner. The scanners then need to 1321 * use only pfn_valid_within() check for arches that allow holes within 1322 * pageblocks. 1323 * 1324 * Return struct page pointer of start_pfn, or NULL if checks were not passed. 1325 * 1326 * It's possible on some configurations to have a setup like node0 node1 node0 1327 * i.e. it's possible that all pages within a zones range of pages do not 1328 * belong to a single zone. We assume that a border between node0 and node1 1329 * can occur within a single pageblock, but not a node0 node1 node0 1330 * interleaving within a single pageblock. It is therefore sufficient to check 1331 * the first and last page of a pageblock and avoid checking each individual 1332 * page in a pageblock. 1333 */ 1334 struct page *__pageblock_pfn_to_page(unsigned long start_pfn, 1335 unsigned long end_pfn, struct zone *zone) 1336 { 1337 struct page *start_page; 1338 struct page *end_page; 1339 1340 /* end_pfn is one past the range we are checking */ 1341 end_pfn--; 1342 1343 if (!pfn_valid(start_pfn) || !pfn_valid(end_pfn)) 1344 return NULL; 1345 1346 start_page = pfn_to_page(start_pfn); 1347 1348 if (page_zone(start_page) != zone) 1349 return NULL; 1350 1351 end_page = pfn_to_page(end_pfn); 1352 1353 /* This gives a shorter code than deriving page_zone(end_page) */ 1354 if (page_zone_id(start_page) != page_zone_id(end_page)) 1355 return NULL; 1356 1357 return start_page; 1358 } 1359 1360 void set_zone_contiguous(struct zone *zone) 1361 { 1362 unsigned long block_start_pfn = zone->zone_start_pfn; 1363 unsigned long block_end_pfn; 1364 1365 block_end_pfn = ALIGN(block_start_pfn + 1, pageblock_nr_pages); 1366 for (; block_start_pfn < zone_end_pfn(zone); 1367 block_start_pfn = block_end_pfn, 1368 block_end_pfn += pageblock_nr_pages) { 1369 1370 block_end_pfn = min(block_end_pfn, zone_end_pfn(zone)); 1371 1372 if (!__pageblock_pfn_to_page(block_start_pfn, 1373 block_end_pfn, zone)) 1374 return; 1375 } 1376 1377 /* We confirm that there is no hole */ 1378 zone->contiguous = true; 1379 } 1380 1381 void clear_zone_contiguous(struct zone *zone) 1382 { 1383 zone->contiguous = false; 1384 } 1385 1386 #ifdef CONFIG_DEFERRED_STRUCT_PAGE_INIT 1387 static void __init deferred_free_range(struct page *page, 1388 unsigned long pfn, int nr_pages) 1389 { 1390 int i; 1391 1392 if (!page) 1393 return; 1394 1395 /* Free a large naturally-aligned chunk if possible */ 1396 if (nr_pages == MAX_ORDER_NR_PAGES && 1397 (pfn & (MAX_ORDER_NR_PAGES-1)) == 0) { 1398 set_pageblock_migratetype(page, MIGRATE_MOVABLE); 1399 __free_pages_boot_core(page, MAX_ORDER-1); 1400 return; 1401 } 1402 1403 for (i = 0; i < nr_pages; i++, page++) 1404 __free_pages_boot_core(page, 0); 1405 } 1406 1407 /* Completion tracking for deferred_init_memmap() threads */ 1408 static atomic_t pgdat_init_n_undone __initdata; 1409 static __initdata DECLARE_COMPLETION(pgdat_init_all_done_comp); 1410 1411 static inline void __init pgdat_init_report_one_done(void) 1412 { 1413 if (atomic_dec_and_test(&pgdat_init_n_undone)) 1414 complete(&pgdat_init_all_done_comp); 1415 } 1416 1417 /* Initialise remaining memory on a node */ 1418 static int __init deferred_init_memmap(void *data) 1419 { 1420 pg_data_t *pgdat = data; 1421 int nid = pgdat->node_id; 1422 struct mminit_pfnnid_cache nid_init_state = { }; 1423 unsigned long start = jiffies; 1424 unsigned long nr_pages = 0; 1425 unsigned long walk_start, walk_end; 1426 int i, zid; 1427 struct zone *zone; 1428 unsigned long first_init_pfn = pgdat->first_deferred_pfn; 1429 const struct cpumask *cpumask = cpumask_of_node(pgdat->node_id); 1430 1431 if (first_init_pfn == ULONG_MAX) { 1432 pgdat_init_report_one_done(); 1433 return 0; 1434 } 1435 1436 /* Bind memory initialisation thread to a local node if possible */ 1437 if (!cpumask_empty(cpumask)) 1438 set_cpus_allowed_ptr(current, cpumask); 1439 1440 /* Sanity check boundaries */ 1441 BUG_ON(pgdat->first_deferred_pfn < pgdat->node_start_pfn); 1442 BUG_ON(pgdat->first_deferred_pfn > pgdat_end_pfn(pgdat)); 1443 pgdat->first_deferred_pfn = ULONG_MAX; 1444 1445 /* Only the highest zone is deferred so find it */ 1446 for (zid = 0; zid < MAX_NR_ZONES; zid++) { 1447 zone = pgdat->node_zones + zid; 1448 if (first_init_pfn < zone_end_pfn(zone)) 1449 break; 1450 } 1451 1452 for_each_mem_pfn_range(i, nid, &walk_start, &walk_end, NULL) { 1453 unsigned long pfn, end_pfn; 1454 struct page *page = NULL; 1455 struct page *free_base_page = NULL; 1456 unsigned long free_base_pfn = 0; 1457 int nr_to_free = 0; 1458 1459 end_pfn = min(walk_end, zone_end_pfn(zone)); 1460 pfn = first_init_pfn; 1461 if (pfn < walk_start) 1462 pfn = walk_start; 1463 if (pfn < zone->zone_start_pfn) 1464 pfn = zone->zone_start_pfn; 1465 1466 for (; pfn < end_pfn; pfn++) { 1467 if (!pfn_valid_within(pfn)) 1468 goto free_range; 1469 1470 /* 1471 * Ensure pfn_valid is checked every 1472 * MAX_ORDER_NR_PAGES for memory holes 1473 */ 1474 if ((pfn & (MAX_ORDER_NR_PAGES - 1)) == 0) { 1475 if (!pfn_valid(pfn)) { 1476 page = NULL; 1477 goto free_range; 1478 } 1479 } 1480 1481 if (!meminit_pfn_in_nid(pfn, nid, &nid_init_state)) { 1482 page = NULL; 1483 goto free_range; 1484 } 1485 1486 /* Minimise pfn page lookups and scheduler checks */ 1487 if (page && (pfn & (MAX_ORDER_NR_PAGES - 1)) != 0) { 1488 page++; 1489 } else { 1490 nr_pages += nr_to_free; 1491 deferred_free_range(free_base_page, 1492 free_base_pfn, nr_to_free); 1493 free_base_page = NULL; 1494 free_base_pfn = nr_to_free = 0; 1495 1496 page = pfn_to_page(pfn); 1497 cond_resched(); 1498 } 1499 1500 if (page->flags) { 1501 VM_BUG_ON(page_zone(page) != zone); 1502 goto free_range; 1503 } 1504 1505 __init_single_page(page, pfn, zid, nid); 1506 if (!free_base_page) { 1507 free_base_page = page; 1508 free_base_pfn = pfn; 1509 nr_to_free = 0; 1510 } 1511 nr_to_free++; 1512 1513 /* Where possible, batch up pages for a single free */ 1514 continue; 1515 free_range: 1516 /* Free the current block of pages to allocator */ 1517 nr_pages += nr_to_free; 1518 deferred_free_range(free_base_page, free_base_pfn, 1519 nr_to_free); 1520 free_base_page = NULL; 1521 free_base_pfn = nr_to_free = 0; 1522 } 1523 1524 first_init_pfn = max(end_pfn, first_init_pfn); 1525 } 1526 1527 /* Sanity check that the next zone really is unpopulated */ 1528 WARN_ON(++zid < MAX_NR_ZONES && populated_zone(++zone)); 1529 1530 pr_info("node %d initialised, %lu pages in %ums\n", nid, nr_pages, 1531 jiffies_to_msecs(jiffies - start)); 1532 1533 pgdat_init_report_one_done(); 1534 return 0; 1535 } 1536 #endif /* CONFIG_DEFERRED_STRUCT_PAGE_INIT */ 1537 1538 void __init page_alloc_init_late(void) 1539 { 1540 struct zone *zone; 1541 1542 #ifdef CONFIG_DEFERRED_STRUCT_PAGE_INIT 1543 int nid; 1544 1545 /* There will be num_node_state(N_MEMORY) threads */ 1546 atomic_set(&pgdat_init_n_undone, num_node_state(N_MEMORY)); 1547 for_each_node_state(nid, N_MEMORY) { 1548 kthread_run(deferred_init_memmap, NODE_DATA(nid), "pgdatinit%d", nid); 1549 } 1550 1551 /* Block until all are initialised */ 1552 wait_for_completion(&pgdat_init_all_done_comp); 1553 1554 /* Reinit limits that are based on free pages after the kernel is up */ 1555 files_maxfiles_init(); 1556 #endif 1557 1558 for_each_populated_zone(zone) 1559 set_zone_contiguous(zone); 1560 } 1561 1562 #ifdef CONFIG_CMA 1563 /* Free whole pageblock and set its migration type to MIGRATE_CMA. */ 1564 void __init init_cma_reserved_pageblock(struct page *page) 1565 { 1566 unsigned i = pageblock_nr_pages; 1567 struct page *p = page; 1568 1569 do { 1570 __ClearPageReserved(p); 1571 set_page_count(p, 0); 1572 } while (++p, --i); 1573 1574 set_pageblock_migratetype(page, MIGRATE_CMA); 1575 1576 if (pageblock_order >= MAX_ORDER) { 1577 i = pageblock_nr_pages; 1578 p = page; 1579 do { 1580 set_page_refcounted(p); 1581 __free_pages(p, MAX_ORDER - 1); 1582 p += MAX_ORDER_NR_PAGES; 1583 } while (i -= MAX_ORDER_NR_PAGES); 1584 } else { 1585 set_page_refcounted(page); 1586 __free_pages(page, pageblock_order); 1587 } 1588 1589 adjust_managed_page_count(page, pageblock_nr_pages); 1590 } 1591 #endif 1592 1593 /* 1594 * The order of subdivision here is critical for the IO subsystem. 1595 * Please do not alter this order without good reasons and regression 1596 * testing. Specifically, as large blocks of memory are subdivided, 1597 * the order in which smaller blocks are delivered depends on the order 1598 * they're subdivided in this function. This is the primary factor 1599 * influencing the order in which pages are delivered to the IO 1600 * subsystem according to empirical testing, and this is also justified 1601 * by considering the behavior of a buddy system containing a single 1602 * large block of memory acted on by a series of small allocations. 1603 * This behavior is a critical factor in sglist merging's success. 1604 * 1605 * -- nyc 1606 */ 1607 static inline void expand(struct zone *zone, struct page *page, 1608 int low, int high, struct free_area *area, 1609 int migratetype) 1610 { 1611 unsigned long size = 1 << high; 1612 1613 while (high > low) { 1614 area--; 1615 high--; 1616 size >>= 1; 1617 VM_BUG_ON_PAGE(bad_range(zone, &page[size]), &page[size]); 1618 1619 if (IS_ENABLED(CONFIG_DEBUG_PAGEALLOC) && 1620 debug_guardpage_enabled() && 1621 high < debug_guardpage_minorder()) { 1622 /* 1623 * Mark as guard pages (or page), that will allow to 1624 * merge back to allocator when buddy will be freed. 1625 * Corresponding page table entries will not be touched, 1626 * pages will stay not present in virtual address space 1627 */ 1628 set_page_guard(zone, &page[size], high, migratetype); 1629 continue; 1630 } 1631 list_add(&page[size].lru, &area->free_list[migratetype]); 1632 area->nr_free++; 1633 set_page_order(&page[size], high); 1634 } 1635 } 1636 1637 static void check_new_page_bad(struct page *page) 1638 { 1639 const char *bad_reason = NULL; 1640 unsigned long bad_flags = 0; 1641 1642 if (unlikely(atomic_read(&page->_mapcount) != -1)) 1643 bad_reason = "nonzero mapcount"; 1644 if (unlikely(page->mapping != NULL)) 1645 bad_reason = "non-NULL mapping"; 1646 if (unlikely(page_ref_count(page) != 0)) 1647 bad_reason = "nonzero _count"; 1648 if (unlikely(page->flags & __PG_HWPOISON)) { 1649 bad_reason = "HWPoisoned (hardware-corrupted)"; 1650 bad_flags = __PG_HWPOISON; 1651 /* Don't complain about hwpoisoned pages */ 1652 page_mapcount_reset(page); /* remove PageBuddy */ 1653 return; 1654 } 1655 if (unlikely(page->flags & PAGE_FLAGS_CHECK_AT_PREP)) { 1656 bad_reason = "PAGE_FLAGS_CHECK_AT_PREP flag set"; 1657 bad_flags = PAGE_FLAGS_CHECK_AT_PREP; 1658 } 1659 #ifdef CONFIG_MEMCG 1660 if (unlikely(page->mem_cgroup)) 1661 bad_reason = "page still charged to cgroup"; 1662 #endif 1663 bad_page(page, bad_reason, bad_flags); 1664 } 1665 1666 /* 1667 * This page is about to be returned from the page allocator 1668 */ 1669 static inline int check_new_page(struct page *page) 1670 { 1671 if (likely(page_expected_state(page, 1672 PAGE_FLAGS_CHECK_AT_PREP|__PG_HWPOISON))) 1673 return 0; 1674 1675 check_new_page_bad(page); 1676 return 1; 1677 } 1678 1679 static inline bool free_pages_prezeroed(bool poisoned) 1680 { 1681 return IS_ENABLED(CONFIG_PAGE_POISONING_ZERO) && 1682 page_poisoning_enabled() && poisoned; 1683 } 1684 1685 #ifdef CONFIG_DEBUG_VM 1686 static bool check_pcp_refill(struct page *page) 1687 { 1688 return false; 1689 } 1690 1691 static bool check_new_pcp(struct page *page) 1692 { 1693 return check_new_page(page); 1694 } 1695 #else 1696 static bool check_pcp_refill(struct page *page) 1697 { 1698 return check_new_page(page); 1699 } 1700 static bool check_new_pcp(struct page *page) 1701 { 1702 return false; 1703 } 1704 #endif /* CONFIG_DEBUG_VM */ 1705 1706 static bool check_new_pages(struct page *page, unsigned int order) 1707 { 1708 int i; 1709 for (i = 0; i < (1 << order); i++) { 1710 struct page *p = page + i; 1711 1712 if (unlikely(check_new_page(p))) 1713 return true; 1714 } 1715 1716 return false; 1717 } 1718 1719 inline void post_alloc_hook(struct page *page, unsigned int order, 1720 gfp_t gfp_flags) 1721 { 1722 set_page_private(page, 0); 1723 set_page_refcounted(page); 1724 1725 arch_alloc_page(page, order); 1726 kernel_map_pages(page, 1 << order, 1); 1727 kernel_poison_pages(page, 1 << order, 1); 1728 kasan_alloc_pages(page, order); 1729 set_page_owner(page, order, gfp_flags); 1730 } 1731 1732 static void prep_new_page(struct page *page, unsigned int order, gfp_t gfp_flags, 1733 unsigned int alloc_flags) 1734 { 1735 int i; 1736 bool poisoned = true; 1737 1738 for (i = 0; i < (1 << order); i++) { 1739 struct page *p = page + i; 1740 if (poisoned) 1741 poisoned &= page_is_poisoned(p); 1742 } 1743 1744 post_alloc_hook(page, order, gfp_flags); 1745 1746 if (!free_pages_prezeroed(poisoned) && (gfp_flags & __GFP_ZERO)) 1747 for (i = 0; i < (1 << order); i++) 1748 clear_highpage(page + i); 1749 1750 if (order && (gfp_flags & __GFP_COMP)) 1751 prep_compound_page(page, order); 1752 1753 /* 1754 * page is set pfmemalloc when ALLOC_NO_WATERMARKS was necessary to 1755 * allocate the page. The expectation is that the caller is taking 1756 * steps that will free more memory. The caller should avoid the page 1757 * being used for !PFMEMALLOC purposes. 1758 */ 1759 if (alloc_flags & ALLOC_NO_WATERMARKS) 1760 set_page_pfmemalloc(page); 1761 else 1762 clear_page_pfmemalloc(page); 1763 } 1764 1765 /* 1766 * Go through the free lists for the given migratetype and remove 1767 * the smallest available page from the freelists 1768 */ 1769 static inline 1770 struct page *__rmqueue_smallest(struct zone *zone, unsigned int order, 1771 int migratetype) 1772 { 1773 unsigned int current_order; 1774 struct free_area *area; 1775 struct page *page; 1776 1777 /* Find a page of the appropriate size in the preferred list */ 1778 for (current_order = order; current_order < MAX_ORDER; ++current_order) { 1779 area = &(zone->free_area[current_order]); 1780 page = list_first_entry_or_null(&area->free_list[migratetype], 1781 struct page, lru); 1782 if (!page) 1783 continue; 1784 list_del(&page->lru); 1785 rmv_page_order(page); 1786 area->nr_free--; 1787 expand(zone, page, order, current_order, area, migratetype); 1788 set_pcppage_migratetype(page, migratetype); 1789 return page; 1790 } 1791 1792 return NULL; 1793 } 1794 1795 1796 /* 1797 * This array describes the order lists are fallen back to when 1798 * the free lists for the desirable migrate type are depleted 1799 */ 1800 static int fallbacks[MIGRATE_TYPES][4] = { 1801 [MIGRATE_UNMOVABLE] = { MIGRATE_RECLAIMABLE, MIGRATE_MOVABLE, MIGRATE_TYPES }, 1802 [MIGRATE_RECLAIMABLE] = { MIGRATE_UNMOVABLE, MIGRATE_MOVABLE, MIGRATE_TYPES }, 1803 [MIGRATE_MOVABLE] = { MIGRATE_RECLAIMABLE, MIGRATE_UNMOVABLE, MIGRATE_TYPES }, 1804 #ifdef CONFIG_CMA 1805 [MIGRATE_CMA] = { MIGRATE_TYPES }, /* Never used */ 1806 #endif 1807 #ifdef CONFIG_MEMORY_ISOLATION 1808 [MIGRATE_ISOLATE] = { MIGRATE_TYPES }, /* Never used */ 1809 #endif 1810 }; 1811 1812 #ifdef CONFIG_CMA 1813 static struct page *__rmqueue_cma_fallback(struct zone *zone, 1814 unsigned int order) 1815 { 1816 return __rmqueue_smallest(zone, order, MIGRATE_CMA); 1817 } 1818 #else 1819 static inline struct page *__rmqueue_cma_fallback(struct zone *zone, 1820 unsigned int order) { return NULL; } 1821 #endif 1822 1823 /* 1824 * Move the free pages in a range to the free lists of the requested type. 1825 * Note that start_page and end_pages are not aligned on a pageblock 1826 * boundary. If alignment is required, use move_freepages_block() 1827 */ 1828 int move_freepages(struct zone *zone, 1829 struct page *start_page, struct page *end_page, 1830 int migratetype) 1831 { 1832 struct page *page; 1833 unsigned int order; 1834 int pages_moved = 0; 1835 1836 #ifndef CONFIG_HOLES_IN_ZONE 1837 /* 1838 * page_zone is not safe to call in this context when 1839 * CONFIG_HOLES_IN_ZONE is set. This bug check is probably redundant 1840 * anyway as we check zone boundaries in move_freepages_block(). 1841 * Remove at a later date when no bug reports exist related to 1842 * grouping pages by mobility 1843 */ 1844 VM_BUG_ON(page_zone(start_page) != page_zone(end_page)); 1845 #endif 1846 1847 for (page = start_page; page <= end_page;) { 1848 /* Make sure we are not inadvertently changing nodes */ 1849 VM_BUG_ON_PAGE(page_to_nid(page) != zone_to_nid(zone), page); 1850 1851 if (!pfn_valid_within(page_to_pfn(page))) { 1852 page++; 1853 continue; 1854 } 1855 1856 if (!PageBuddy(page)) { 1857 page++; 1858 continue; 1859 } 1860 1861 order = page_order(page); 1862 list_move(&page->lru, 1863 &zone->free_area[order].free_list[migratetype]); 1864 page += 1 << order; 1865 pages_moved += 1 << order; 1866 } 1867 1868 return pages_moved; 1869 } 1870 1871 int move_freepages_block(struct zone *zone, struct page *page, 1872 int migratetype) 1873 { 1874 unsigned long start_pfn, end_pfn; 1875 struct page *start_page, *end_page; 1876 1877 start_pfn = page_to_pfn(page); 1878 start_pfn = start_pfn & ~(pageblock_nr_pages-1); 1879 start_page = pfn_to_page(start_pfn); 1880 end_page = start_page + pageblock_nr_pages - 1; 1881 end_pfn = start_pfn + pageblock_nr_pages - 1; 1882 1883 /* Do not cross zone boundaries */ 1884 if (!zone_spans_pfn(zone, start_pfn)) 1885 start_page = page; 1886 if (!zone_spans_pfn(zone, end_pfn)) 1887 return 0; 1888 1889 return move_freepages(zone, start_page, end_page, migratetype); 1890 } 1891 1892 static void change_pageblock_range(struct page *pageblock_page, 1893 int start_order, int migratetype) 1894 { 1895 int nr_pageblocks = 1 << (start_order - pageblock_order); 1896 1897 while (nr_pageblocks--) { 1898 set_pageblock_migratetype(pageblock_page, migratetype); 1899 pageblock_page += pageblock_nr_pages; 1900 } 1901 } 1902 1903 /* 1904 * When we are falling back to another migratetype during allocation, try to 1905 * steal extra free pages from the same pageblocks to satisfy further 1906 * allocations, instead of polluting multiple pageblocks. 1907 * 1908 * If we are stealing a relatively large buddy page, it is likely there will 1909 * be more free pages in the pageblock, so try to steal them all. For 1910 * reclaimable and unmovable allocations, we steal regardless of page size, 1911 * as fragmentation caused by those allocations polluting movable pageblocks 1912 * is worse than movable allocations stealing from unmovable and reclaimable 1913 * pageblocks. 1914 */ 1915 static bool can_steal_fallback(unsigned int order, int start_mt) 1916 { 1917 /* 1918 * Leaving this order check is intended, although there is 1919 * relaxed order check in next check. The reason is that 1920 * we can actually steal whole pageblock if this condition met, 1921 * but, below check doesn't guarantee it and that is just heuristic 1922 * so could be changed anytime. 1923 */ 1924 if (order >= pageblock_order) 1925 return true; 1926 1927 if (order >= pageblock_order / 2 || 1928 start_mt == MIGRATE_RECLAIMABLE || 1929 start_mt == MIGRATE_UNMOVABLE || 1930 page_group_by_mobility_disabled) 1931 return true; 1932 1933 return false; 1934 } 1935 1936 /* 1937 * This function implements actual steal behaviour. If order is large enough, 1938 * we can steal whole pageblock. If not, we first move freepages in this 1939 * pageblock and check whether half of pages are moved or not. If half of 1940 * pages are moved, we can change migratetype of pageblock and permanently 1941 * use it's pages as requested migratetype in the future. 1942 */ 1943 static void steal_suitable_fallback(struct zone *zone, struct page *page, 1944 int start_type) 1945 { 1946 unsigned int current_order = page_order(page); 1947 int pages; 1948 1949 /* Take ownership for orders >= pageblock_order */ 1950 if (current_order >= pageblock_order) { 1951 change_pageblock_range(page, current_order, start_type); 1952 return; 1953 } 1954 1955 pages = move_freepages_block(zone, page, start_type); 1956 1957 /* Claim the whole block if over half of it is free */ 1958 if (pages >= (1 << (pageblock_order-1)) || 1959 page_group_by_mobility_disabled) 1960 set_pageblock_migratetype(page, start_type); 1961 } 1962 1963 /* 1964 * Check whether there is a suitable fallback freepage with requested order. 1965 * If only_stealable is true, this function returns fallback_mt only if 1966 * we can steal other freepages all together. This would help to reduce 1967 * fragmentation due to mixed migratetype pages in one pageblock. 1968 */ 1969 int find_suitable_fallback(struct free_area *area, unsigned int order, 1970 int migratetype, bool only_stealable, bool *can_steal) 1971 { 1972 int i; 1973 int fallback_mt; 1974 1975 if (area->nr_free == 0) 1976 return -1; 1977 1978 *can_steal = false; 1979 for (i = 0;; i++) { 1980 fallback_mt = fallbacks[migratetype][i]; 1981 if (fallback_mt == MIGRATE_TYPES) 1982 break; 1983 1984 if (list_empty(&area->free_list[fallback_mt])) 1985 continue; 1986 1987 if (can_steal_fallback(order, migratetype)) 1988 *can_steal = true; 1989 1990 if (!only_stealable) 1991 return fallback_mt; 1992 1993 if (*can_steal) 1994 return fallback_mt; 1995 } 1996 1997 return -1; 1998 } 1999 2000 /* 2001 * Reserve a pageblock for exclusive use of high-order atomic allocations if 2002 * there are no empty page blocks that contain a page with a suitable order 2003 */ 2004 static void reserve_highatomic_pageblock(struct page *page, struct zone *zone, 2005 unsigned int alloc_order) 2006 { 2007 int mt; 2008 unsigned long max_managed, flags; 2009 2010 /* 2011 * Limit the number reserved to 1 pageblock or roughly 1% of a zone. 2012 * Check is race-prone but harmless. 2013 */ 2014 max_managed = (zone->managed_pages / 100) + pageblock_nr_pages; 2015 if (zone->nr_reserved_highatomic >= max_managed) 2016 return; 2017 2018 spin_lock_irqsave(&zone->lock, flags); 2019 2020 /* Recheck the nr_reserved_highatomic limit under the lock */ 2021 if (zone->nr_reserved_highatomic >= max_managed) 2022 goto out_unlock; 2023 2024 /* Yoink! */ 2025 mt = get_pageblock_migratetype(page); 2026 if (mt != MIGRATE_HIGHATOMIC && 2027 !is_migrate_isolate(mt) && !is_migrate_cma(mt)) { 2028 zone->nr_reserved_highatomic += pageblock_nr_pages; 2029 set_pageblock_migratetype(page, MIGRATE_HIGHATOMIC); 2030 move_freepages_block(zone, page, MIGRATE_HIGHATOMIC); 2031 } 2032 2033 out_unlock: 2034 spin_unlock_irqrestore(&zone->lock, flags); 2035 } 2036 2037 /* 2038 * Used when an allocation is about to fail under memory pressure. This 2039 * potentially hurts the reliability of high-order allocations when under 2040 * intense memory pressure but failed atomic allocations should be easier 2041 * to recover from than an OOM. 2042 */ 2043 static void unreserve_highatomic_pageblock(const struct alloc_context *ac) 2044 { 2045 struct zonelist *zonelist = ac->zonelist; 2046 unsigned long flags; 2047 struct zoneref *z; 2048 struct zone *zone; 2049 struct page *page; 2050 int order; 2051 2052 for_each_zone_zonelist_nodemask(zone, z, zonelist, ac->high_zoneidx, 2053 ac->nodemask) { 2054 /* Preserve at least one pageblock */ 2055 if (zone->nr_reserved_highatomic <= pageblock_nr_pages) 2056 continue; 2057 2058 spin_lock_irqsave(&zone->lock, flags); 2059 for (order = 0; order < MAX_ORDER; order++) { 2060 struct free_area *area = &(zone->free_area[order]); 2061 2062 page = list_first_entry_or_null( 2063 &area->free_list[MIGRATE_HIGHATOMIC], 2064 struct page, lru); 2065 if (!page) 2066 continue; 2067 2068 /* 2069 * It should never happen but changes to locking could 2070 * inadvertently allow a per-cpu drain to add pages 2071 * to MIGRATE_HIGHATOMIC while unreserving so be safe 2072 * and watch for underflows. 2073 */ 2074 zone->nr_reserved_highatomic -= min(pageblock_nr_pages, 2075 zone->nr_reserved_highatomic); 2076 2077 /* 2078 * Convert to ac->migratetype and avoid the normal 2079 * pageblock stealing heuristics. Minimally, the caller 2080 * is doing the work and needs the pages. More 2081 * importantly, if the block was always converted to 2082 * MIGRATE_UNMOVABLE or another type then the number 2083 * of pageblocks that cannot be completely freed 2084 * may increase. 2085 */ 2086 set_pageblock_migratetype(page, ac->migratetype); 2087 move_freepages_block(zone, page, ac->migratetype); 2088 spin_unlock_irqrestore(&zone->lock, flags); 2089 return; 2090 } 2091 spin_unlock_irqrestore(&zone->lock, flags); 2092 } 2093 } 2094 2095 /* Remove an element from the buddy allocator from the fallback list */ 2096 static inline struct page * 2097 __rmqueue_fallback(struct zone *zone, unsigned int order, int start_migratetype) 2098 { 2099 struct free_area *area; 2100 unsigned int current_order; 2101 struct page *page; 2102 int fallback_mt; 2103 bool can_steal; 2104 2105 /* Find the largest possible block of pages in the other list */ 2106 for (current_order = MAX_ORDER-1; 2107 current_order >= order && current_order <= MAX_ORDER-1; 2108 --current_order) { 2109 area = &(zone->free_area[current_order]); 2110 fallback_mt = find_suitable_fallback(area, current_order, 2111 start_migratetype, false, &can_steal); 2112 if (fallback_mt == -1) 2113 continue; 2114 2115 page = list_first_entry(&area->free_list[fallback_mt], 2116 struct page, lru); 2117 if (can_steal) 2118 steal_suitable_fallback(zone, page, start_migratetype); 2119 2120 /* Remove the page from the freelists */ 2121 area->nr_free--; 2122 list_del(&page->lru); 2123 rmv_page_order(page); 2124 2125 expand(zone, page, order, current_order, area, 2126 start_migratetype); 2127 /* 2128 * The pcppage_migratetype may differ from pageblock's 2129 * migratetype depending on the decisions in 2130 * find_suitable_fallback(). This is OK as long as it does not 2131 * differ for MIGRATE_CMA pageblocks. Those can be used as 2132 * fallback only via special __rmqueue_cma_fallback() function 2133 */ 2134 set_pcppage_migratetype(page, start_migratetype); 2135 2136 trace_mm_page_alloc_extfrag(page, order, current_order, 2137 start_migratetype, fallback_mt); 2138 2139 return page; 2140 } 2141 2142 return NULL; 2143 } 2144 2145 /* 2146 * Do the hard work of removing an element from the buddy allocator. 2147 * Call me with the zone->lock already held. 2148 */ 2149 static struct page *__rmqueue(struct zone *zone, unsigned int order, 2150 int migratetype) 2151 { 2152 struct page *page; 2153 2154 page = __rmqueue_smallest(zone, order, migratetype); 2155 if (unlikely(!page)) { 2156 if (migratetype == MIGRATE_MOVABLE) 2157 page = __rmqueue_cma_fallback(zone, order); 2158 2159 if (!page) 2160 page = __rmqueue_fallback(zone, order, migratetype); 2161 } 2162 2163 trace_mm_page_alloc_zone_locked(page, order, migratetype); 2164 return page; 2165 } 2166 2167 /* 2168 * Obtain a specified number of elements from the buddy allocator, all under 2169 * a single hold of the lock, for efficiency. Add them to the supplied list. 2170 * Returns the number of new pages which were placed at *list. 2171 */ 2172 static int rmqueue_bulk(struct zone *zone, unsigned int order, 2173 unsigned long count, struct list_head *list, 2174 int migratetype, bool cold) 2175 { 2176 int i; 2177 2178 spin_lock(&zone->lock); 2179 for (i = 0; i < count; ++i) { 2180 struct page *page = __rmqueue(zone, order, migratetype); 2181 if (unlikely(page == NULL)) 2182 break; 2183 2184 if (unlikely(check_pcp_refill(page))) 2185 continue; 2186 2187 /* 2188 * Split buddy pages returned by expand() are received here 2189 * in physical page order. The page is added to the callers and 2190 * list and the list head then moves forward. From the callers 2191 * perspective, the linked list is ordered by page number in 2192 * some conditions. This is useful for IO devices that can 2193 * merge IO requests if the physical pages are ordered 2194 * properly. 2195 */ 2196 if (likely(!cold)) 2197 list_add(&page->lru, list); 2198 else 2199 list_add_tail(&page->lru, list); 2200 list = &page->lru; 2201 if (is_migrate_cma(get_pcppage_migratetype(page))) 2202 __mod_zone_page_state(zone, NR_FREE_CMA_PAGES, 2203 -(1 << order)); 2204 } 2205 __mod_zone_page_state(zone, NR_FREE_PAGES, -(i << order)); 2206 spin_unlock(&zone->lock); 2207 return i; 2208 } 2209 2210 #ifdef CONFIG_NUMA 2211 /* 2212 * Called from the vmstat counter updater to drain pagesets of this 2213 * currently executing processor on remote nodes after they have 2214 * expired. 2215 * 2216 * Note that this function must be called with the thread pinned to 2217 * a single processor. 2218 */ 2219 void drain_zone_pages(struct zone *zone, struct per_cpu_pages *pcp) 2220 { 2221 unsigned long flags; 2222 int to_drain, batch; 2223 2224 local_irq_save(flags); 2225 batch = READ_ONCE(pcp->batch); 2226 to_drain = min(pcp->count, batch); 2227 if (to_drain > 0) { 2228 free_pcppages_bulk(zone, to_drain, pcp); 2229 pcp->count -= to_drain; 2230 } 2231 local_irq_restore(flags); 2232 } 2233 #endif 2234 2235 /* 2236 * Drain pcplists of the indicated processor and zone. 2237 * 2238 * The processor must either be the current processor and the 2239 * thread pinned to the current processor or a processor that 2240 * is not online. 2241 */ 2242 static void drain_pages_zone(unsigned int cpu, struct zone *zone) 2243 { 2244 unsigned long flags; 2245 struct per_cpu_pageset *pset; 2246 struct per_cpu_pages *pcp; 2247 2248 local_irq_save(flags); 2249 pset = per_cpu_ptr(zone->pageset, cpu); 2250 2251 pcp = &pset->pcp; 2252 if (pcp->count) { 2253 free_pcppages_bulk(zone, pcp->count, pcp); 2254 pcp->count = 0; 2255 } 2256 local_irq_restore(flags); 2257 } 2258 2259 /* 2260 * Drain pcplists of all zones on the indicated processor. 2261 * 2262 * The processor must either be the current processor and the 2263 * thread pinned to the current processor or a processor that 2264 * is not online. 2265 */ 2266 static void drain_pages(unsigned int cpu) 2267 { 2268 struct zone *zone; 2269 2270 for_each_populated_zone(zone) { 2271 drain_pages_zone(cpu, zone); 2272 } 2273 } 2274 2275 /* 2276 * Spill all of this CPU's per-cpu pages back into the buddy allocator. 2277 * 2278 * The CPU has to be pinned. When zone parameter is non-NULL, spill just 2279 * the single zone's pages. 2280 */ 2281 void drain_local_pages(struct zone *zone) 2282 { 2283 int cpu = smp_processor_id(); 2284 2285 if (zone) 2286 drain_pages_zone(cpu, zone); 2287 else 2288 drain_pages(cpu); 2289 } 2290 2291 /* 2292 * Spill all the per-cpu pages from all CPUs back into the buddy allocator. 2293 * 2294 * When zone parameter is non-NULL, spill just the single zone's pages. 2295 * 2296 * Note that this code is protected against sending an IPI to an offline 2297 * CPU but does not guarantee sending an IPI to newly hotplugged CPUs: 2298 * on_each_cpu_mask() blocks hotplug and won't talk to offlined CPUs but 2299 * nothing keeps CPUs from showing up after we populated the cpumask and 2300 * before the call to on_each_cpu_mask(). 2301 */ 2302 void drain_all_pages(struct zone *zone) 2303 { 2304 int cpu; 2305 2306 /* 2307 * Allocate in the BSS so we wont require allocation in 2308 * direct reclaim path for CONFIG_CPUMASK_OFFSTACK=y 2309 */ 2310 static cpumask_t cpus_with_pcps; 2311 2312 /* 2313 * We don't care about racing with CPU hotplug event 2314 * as offline notification will cause the notified 2315 * cpu to drain that CPU pcps and on_each_cpu_mask 2316 * disables preemption as part of its processing 2317 */ 2318 for_each_online_cpu(cpu) { 2319 struct per_cpu_pageset *pcp; 2320 struct zone *z; 2321 bool has_pcps = false; 2322 2323 if (zone) { 2324 pcp = per_cpu_ptr(zone->pageset, cpu); 2325 if (pcp->pcp.count) 2326 has_pcps = true; 2327 } else { 2328 for_each_populated_zone(z) { 2329 pcp = per_cpu_ptr(z->pageset, cpu); 2330 if (pcp->pcp.count) { 2331 has_pcps = true; 2332 break; 2333 } 2334 } 2335 } 2336 2337 if (has_pcps) 2338 cpumask_set_cpu(cpu, &cpus_with_pcps); 2339 else 2340 cpumask_clear_cpu(cpu, &cpus_with_pcps); 2341 } 2342 on_each_cpu_mask(&cpus_with_pcps, (smp_call_func_t) drain_local_pages, 2343 zone, 1); 2344 } 2345 2346 #ifdef CONFIG_HIBERNATION 2347 2348 void mark_free_pages(struct zone *zone) 2349 { 2350 unsigned long pfn, max_zone_pfn; 2351 unsigned long flags; 2352 unsigned int order, t; 2353 struct page *page; 2354 2355 if (zone_is_empty(zone)) 2356 return; 2357 2358 spin_lock_irqsave(&zone->lock, flags); 2359 2360 max_zone_pfn = zone_end_pfn(zone); 2361 for (pfn = zone->zone_start_pfn; pfn < max_zone_pfn; pfn++) 2362 if (pfn_valid(pfn)) { 2363 page = pfn_to_page(pfn); 2364 2365 if (page_zone(page) != zone) 2366 continue; 2367 2368 if (!swsusp_page_is_forbidden(page)) 2369 swsusp_unset_page_free(page); 2370 } 2371 2372 for_each_migratetype_order(order, t) { 2373 list_for_each_entry(page, 2374 &zone->free_area[order].free_list[t], lru) { 2375 unsigned long i; 2376 2377 pfn = page_to_pfn(page); 2378 for (i = 0; i < (1UL << order); i++) 2379 swsusp_set_page_free(pfn_to_page(pfn + i)); 2380 } 2381 } 2382 spin_unlock_irqrestore(&zone->lock, flags); 2383 } 2384 #endif /* CONFIG_PM */ 2385 2386 /* 2387 * Free a 0-order page 2388 * cold == true ? free a cold page : free a hot page 2389 */ 2390 void free_hot_cold_page(struct page *page, bool cold) 2391 { 2392 struct zone *zone = page_zone(page); 2393 struct per_cpu_pages *pcp; 2394 unsigned long flags; 2395 unsigned long pfn = page_to_pfn(page); 2396 int migratetype; 2397 2398 if (!free_pcp_prepare(page)) 2399 return; 2400 2401 migratetype = get_pfnblock_migratetype(page, pfn); 2402 set_pcppage_migratetype(page, migratetype); 2403 local_irq_save(flags); 2404 __count_vm_event(PGFREE); 2405 2406 /* 2407 * We only track unmovable, reclaimable and movable on pcp lists. 2408 * Free ISOLATE pages back to the allocator because they are being 2409 * offlined but treat RESERVE as movable pages so we can get those 2410 * areas back if necessary. Otherwise, we may have to free 2411 * excessively into the page allocator 2412 */ 2413 if (migratetype >= MIGRATE_PCPTYPES) { 2414 if (unlikely(is_migrate_isolate(migratetype))) { 2415 free_one_page(zone, page, pfn, 0, migratetype); 2416 goto out; 2417 } 2418 migratetype = MIGRATE_MOVABLE; 2419 } 2420 2421 pcp = &this_cpu_ptr(zone->pageset)->pcp; 2422 if (!cold) 2423 list_add(&page->lru, &pcp->lists[migratetype]); 2424 else 2425 list_add_tail(&page->lru, &pcp->lists[migratetype]); 2426 pcp->count++; 2427 if (pcp->count >= pcp->high) { 2428 unsigned long batch = READ_ONCE(pcp->batch); 2429 free_pcppages_bulk(zone, batch, pcp); 2430 pcp->count -= batch; 2431 } 2432 2433 out: 2434 local_irq_restore(flags); 2435 } 2436 2437 /* 2438 * Free a list of 0-order pages 2439 */ 2440 void free_hot_cold_page_list(struct list_head *list, bool cold) 2441 { 2442 struct page *page, *next; 2443 2444 list_for_each_entry_safe(page, next, list, lru) { 2445 trace_mm_page_free_batched(page, cold); 2446 free_hot_cold_page(page, cold); 2447 } 2448 } 2449 2450 /* 2451 * split_page takes a non-compound higher-order page, and splits it into 2452 * n (1<<order) sub-pages: page[0..n] 2453 * Each sub-page must be freed individually. 2454 * 2455 * Note: this is probably too low level an operation for use in drivers. 2456 * Please consult with lkml before using this in your driver. 2457 */ 2458 void split_page(struct page *page, unsigned int order) 2459 { 2460 int i; 2461 2462 VM_BUG_ON_PAGE(PageCompound(page), page); 2463 VM_BUG_ON_PAGE(!page_count(page), page); 2464 2465 #ifdef CONFIG_KMEMCHECK 2466 /* 2467 * Split shadow pages too, because free(page[0]) would 2468 * otherwise free the whole shadow. 2469 */ 2470 if (kmemcheck_page_is_tracked(page)) 2471 split_page(virt_to_page(page[0].shadow), order); 2472 #endif 2473 2474 for (i = 1; i < (1 << order); i++) 2475 set_page_refcounted(page + i); 2476 split_page_owner(page, order); 2477 } 2478 EXPORT_SYMBOL_GPL(split_page); 2479 2480 int __isolate_free_page(struct page *page, unsigned int order) 2481 { 2482 unsigned long watermark; 2483 struct zone *zone; 2484 int mt; 2485 2486 BUG_ON(!PageBuddy(page)); 2487 2488 zone = page_zone(page); 2489 mt = get_pageblock_migratetype(page); 2490 2491 if (!is_migrate_isolate(mt)) { 2492 /* Obey watermarks as if the page was being allocated */ 2493 watermark = low_wmark_pages(zone) + (1 << order); 2494 if (!zone_watermark_ok(zone, 0, watermark, 0, 0)) 2495 return 0; 2496 2497 __mod_zone_freepage_state(zone, -(1UL << order), mt); 2498 } 2499 2500 /* Remove page from free list */ 2501 list_del(&page->lru); 2502 zone->free_area[order].nr_free--; 2503 rmv_page_order(page); 2504 2505 /* 2506 * Set the pageblock if the isolated page is at least half of a 2507 * pageblock 2508 */ 2509 if (order >= pageblock_order - 1) { 2510 struct page *endpage = page + (1 << order) - 1; 2511 for (; page < endpage; page += pageblock_nr_pages) { 2512 int mt = get_pageblock_migratetype(page); 2513 if (!is_migrate_isolate(mt) && !is_migrate_cma(mt)) 2514 set_pageblock_migratetype(page, 2515 MIGRATE_MOVABLE); 2516 } 2517 } 2518 2519 2520 return 1UL << order; 2521 } 2522 2523 /* 2524 * Update NUMA hit/miss statistics 2525 * 2526 * Must be called with interrupts disabled. 2527 * 2528 * When __GFP_OTHER_NODE is set assume the node of the preferred 2529 * zone is the local node. This is useful for daemons who allocate 2530 * memory on behalf of other processes. 2531 */ 2532 static inline void zone_statistics(struct zone *preferred_zone, struct zone *z, 2533 gfp_t flags) 2534 { 2535 #ifdef CONFIG_NUMA 2536 int local_nid = numa_node_id(); 2537 enum zone_stat_item local_stat = NUMA_LOCAL; 2538 2539 if (unlikely(flags & __GFP_OTHER_NODE)) { 2540 local_stat = NUMA_OTHER; 2541 local_nid = preferred_zone->node; 2542 } 2543 2544 if (z->node == local_nid) { 2545 __inc_zone_state(z, NUMA_HIT); 2546 __inc_zone_state(z, local_stat); 2547 } else { 2548 __inc_zone_state(z, NUMA_MISS); 2549 __inc_zone_state(preferred_zone, NUMA_FOREIGN); 2550 } 2551 #endif 2552 } 2553 2554 /* 2555 * Allocate a page from the given zone. Use pcplists for order-0 allocations. 2556 */ 2557 static inline 2558 struct page *buffered_rmqueue(struct zone *preferred_zone, 2559 struct zone *zone, unsigned int order, 2560 gfp_t gfp_flags, unsigned int alloc_flags, 2561 int migratetype) 2562 { 2563 unsigned long flags; 2564 struct page *page; 2565 bool cold = ((gfp_flags & __GFP_COLD) != 0); 2566 2567 if (likely(order == 0)) { 2568 struct per_cpu_pages *pcp; 2569 struct list_head *list; 2570 2571 local_irq_save(flags); 2572 do { 2573 pcp = &this_cpu_ptr(zone->pageset)->pcp; 2574 list = &pcp->lists[migratetype]; 2575 if (list_empty(list)) { 2576 pcp->count += rmqueue_bulk(zone, 0, 2577 pcp->batch, list, 2578 migratetype, cold); 2579 if (unlikely(list_empty(list))) 2580 goto failed; 2581 } 2582 2583 if (cold) 2584 page = list_last_entry(list, struct page, lru); 2585 else 2586 page = list_first_entry(list, struct page, lru); 2587 2588 list_del(&page->lru); 2589 pcp->count--; 2590 2591 } while (check_new_pcp(page)); 2592 } else { 2593 /* 2594 * We most definitely don't want callers attempting to 2595 * allocate greater than order-1 page units with __GFP_NOFAIL. 2596 */ 2597 WARN_ON_ONCE((gfp_flags & __GFP_NOFAIL) && (order > 1)); 2598 spin_lock_irqsave(&zone->lock, flags); 2599 2600 do { 2601 page = NULL; 2602 if (alloc_flags & ALLOC_HARDER) { 2603 page = __rmqueue_smallest(zone, order, MIGRATE_HIGHATOMIC); 2604 if (page) 2605 trace_mm_page_alloc_zone_locked(page, order, migratetype); 2606 } 2607 if (!page) 2608 page = __rmqueue(zone, order, migratetype); 2609 } while (page && check_new_pages(page, order)); 2610 spin_unlock(&zone->lock); 2611 if (!page) 2612 goto failed; 2613 __mod_zone_freepage_state(zone, -(1 << order), 2614 get_pcppage_migratetype(page)); 2615 } 2616 2617 __count_zid_vm_events(PGALLOC, page_zonenum(page), 1 << order); 2618 zone_statistics(preferred_zone, zone, gfp_flags); 2619 local_irq_restore(flags); 2620 2621 VM_BUG_ON_PAGE(bad_range(zone, page), page); 2622 return page; 2623 2624 failed: 2625 local_irq_restore(flags); 2626 return NULL; 2627 } 2628 2629 #ifdef CONFIG_FAIL_PAGE_ALLOC 2630 2631 static struct { 2632 struct fault_attr attr; 2633 2634 bool ignore_gfp_highmem; 2635 bool ignore_gfp_reclaim; 2636 u32 min_order; 2637 } fail_page_alloc = { 2638 .attr = FAULT_ATTR_INITIALIZER, 2639 .ignore_gfp_reclaim = true, 2640 .ignore_gfp_highmem = true, 2641 .min_order = 1, 2642 }; 2643 2644 static int __init setup_fail_page_alloc(char *str) 2645 { 2646 return setup_fault_attr(&fail_page_alloc.attr, str); 2647 } 2648 __setup("fail_page_alloc=", setup_fail_page_alloc); 2649 2650 static bool should_fail_alloc_page(gfp_t gfp_mask, unsigned int order) 2651 { 2652 if (order < fail_page_alloc.min_order) 2653 return false; 2654 if (gfp_mask & __GFP_NOFAIL) 2655 return false; 2656 if (fail_page_alloc.ignore_gfp_highmem && (gfp_mask & __GFP_HIGHMEM)) 2657 return false; 2658 if (fail_page_alloc.ignore_gfp_reclaim && 2659 (gfp_mask & __GFP_DIRECT_RECLAIM)) 2660 return false; 2661 2662 return should_fail(&fail_page_alloc.attr, 1 << order); 2663 } 2664 2665 #ifdef CONFIG_FAULT_INJECTION_DEBUG_FS 2666 2667 static int __init fail_page_alloc_debugfs(void) 2668 { 2669 umode_t mode = S_IFREG | S_IRUSR | S_IWUSR; 2670 struct dentry *dir; 2671 2672 dir = fault_create_debugfs_attr("fail_page_alloc", NULL, 2673 &fail_page_alloc.attr); 2674 if (IS_ERR(dir)) 2675 return PTR_ERR(dir); 2676 2677 if (!debugfs_create_bool("ignore-gfp-wait", mode, dir, 2678 &fail_page_alloc.ignore_gfp_reclaim)) 2679 goto fail; 2680 if (!debugfs_create_bool("ignore-gfp-highmem", mode, dir, 2681 &fail_page_alloc.ignore_gfp_highmem)) 2682 goto fail; 2683 if (!debugfs_create_u32("min-order", mode, dir, 2684 &fail_page_alloc.min_order)) 2685 goto fail; 2686 2687 return 0; 2688 fail: 2689 debugfs_remove_recursive(dir); 2690 2691 return -ENOMEM; 2692 } 2693 2694 late_initcall(fail_page_alloc_debugfs); 2695 2696 #endif /* CONFIG_FAULT_INJECTION_DEBUG_FS */ 2697 2698 #else /* CONFIG_FAIL_PAGE_ALLOC */ 2699 2700 static inline bool should_fail_alloc_page(gfp_t gfp_mask, unsigned int order) 2701 { 2702 return false; 2703 } 2704 2705 #endif /* CONFIG_FAIL_PAGE_ALLOC */ 2706 2707 /* 2708 * Return true if free base pages are above 'mark'. For high-order checks it 2709 * will return true of the order-0 watermark is reached and there is at least 2710 * one free page of a suitable size. Checking now avoids taking the zone lock 2711 * to check in the allocation paths if no pages are free. 2712 */ 2713 bool __zone_watermark_ok(struct zone *z, unsigned int order, unsigned long mark, 2714 int classzone_idx, unsigned int alloc_flags, 2715 long free_pages) 2716 { 2717 long min = mark; 2718 int o; 2719 const bool alloc_harder = (alloc_flags & ALLOC_HARDER); 2720 2721 /* free_pages may go negative - that's OK */ 2722 free_pages -= (1 << order) - 1; 2723 2724 if (alloc_flags & ALLOC_HIGH) 2725 min -= min / 2; 2726 2727 /* 2728 * If the caller does not have rights to ALLOC_HARDER then subtract 2729 * the high-atomic reserves. This will over-estimate the size of the 2730 * atomic reserve but it avoids a search. 2731 */ 2732 if (likely(!alloc_harder)) 2733 free_pages -= z->nr_reserved_highatomic; 2734 else 2735 min -= min / 4; 2736 2737 #ifdef CONFIG_CMA 2738 /* If allocation can't use CMA areas don't use free CMA pages */ 2739 if (!(alloc_flags & ALLOC_CMA)) 2740 free_pages -= zone_page_state(z, NR_FREE_CMA_PAGES); 2741 #endif 2742 2743 /* 2744 * Check watermarks for an order-0 allocation request. If these 2745 * are not met, then a high-order request also cannot go ahead 2746 * even if a suitable page happened to be free. 2747 */ 2748 if (free_pages <= min + z->lowmem_reserve[classzone_idx]) 2749 return false; 2750 2751 /* If this is an order-0 request then the watermark is fine */ 2752 if (!order) 2753 return true; 2754 2755 /* For a high-order request, check at least one suitable page is free */ 2756 for (o = order; o < MAX_ORDER; o++) { 2757 struct free_area *area = &z->free_area[o]; 2758 int mt; 2759 2760 if (!area->nr_free) 2761 continue; 2762 2763 if (alloc_harder) 2764 return true; 2765 2766 for (mt = 0; mt < MIGRATE_PCPTYPES; mt++) { 2767 if (!list_empty(&area->free_list[mt])) 2768 return true; 2769 } 2770 2771 #ifdef CONFIG_CMA 2772 if ((alloc_flags & ALLOC_CMA) && 2773 !list_empty(&area->free_list[MIGRATE_CMA])) { 2774 return true; 2775 } 2776 #endif 2777 } 2778 return false; 2779 } 2780 2781 bool zone_watermark_ok(struct zone *z, unsigned int order, unsigned long mark, 2782 int classzone_idx, unsigned int alloc_flags) 2783 { 2784 return __zone_watermark_ok(z, order, mark, classzone_idx, alloc_flags, 2785 zone_page_state(z, NR_FREE_PAGES)); 2786 } 2787 2788 static inline bool zone_watermark_fast(struct zone *z, unsigned int order, 2789 unsigned long mark, int classzone_idx, unsigned int alloc_flags) 2790 { 2791 long free_pages = zone_page_state(z, NR_FREE_PAGES); 2792 long cma_pages = 0; 2793 2794 #ifdef CONFIG_CMA 2795 /* If allocation can't use CMA areas don't use free CMA pages */ 2796 if (!(alloc_flags & ALLOC_CMA)) 2797 cma_pages = zone_page_state(z, NR_FREE_CMA_PAGES); 2798 #endif 2799 2800 /* 2801 * Fast check for order-0 only. If this fails then the reserves 2802 * need to be calculated. There is a corner case where the check 2803 * passes but only the high-order atomic reserve are free. If 2804 * the caller is !atomic then it'll uselessly search the free 2805 * list. That corner case is then slower but it is harmless. 2806 */ 2807 if (!order && (free_pages - cma_pages) > mark + z->lowmem_reserve[classzone_idx]) 2808 return true; 2809 2810 return __zone_watermark_ok(z, order, mark, classzone_idx, alloc_flags, 2811 free_pages); 2812 } 2813 2814 bool zone_watermark_ok_safe(struct zone *z, unsigned int order, 2815 unsigned long mark, int classzone_idx) 2816 { 2817 long free_pages = zone_page_state(z, NR_FREE_PAGES); 2818 2819 if (z->percpu_drift_mark && free_pages < z->percpu_drift_mark) 2820 free_pages = zone_page_state_snapshot(z, NR_FREE_PAGES); 2821 2822 return __zone_watermark_ok(z, order, mark, classzone_idx, 0, 2823 free_pages); 2824 } 2825 2826 #ifdef CONFIG_NUMA 2827 static bool zone_allows_reclaim(struct zone *local_zone, struct zone *zone) 2828 { 2829 return node_distance(zone_to_nid(local_zone), zone_to_nid(zone)) < 2830 RECLAIM_DISTANCE; 2831 } 2832 #else /* CONFIG_NUMA */ 2833 static bool zone_allows_reclaim(struct zone *local_zone, struct zone *zone) 2834 { 2835 return true; 2836 } 2837 #endif /* CONFIG_NUMA */ 2838 2839 /* 2840 * get_page_from_freelist goes through the zonelist trying to allocate 2841 * a page. 2842 */ 2843 static struct page * 2844 get_page_from_freelist(gfp_t gfp_mask, unsigned int order, int alloc_flags, 2845 const struct alloc_context *ac) 2846 { 2847 struct zoneref *z = ac->preferred_zoneref; 2848 struct zone *zone; 2849 struct pglist_data *last_pgdat_dirty_limit = NULL; 2850 2851 /* 2852 * Scan zonelist, looking for a zone with enough free. 2853 * See also __cpuset_node_allowed() comment in kernel/cpuset.c. 2854 */ 2855 for_next_zone_zonelist_nodemask(zone, z, ac->zonelist, ac->high_zoneidx, 2856 ac->nodemask) { 2857 struct page *page; 2858 unsigned long mark; 2859 2860 if (cpusets_enabled() && 2861 (alloc_flags & ALLOC_CPUSET) && 2862 !__cpuset_zone_allowed(zone, gfp_mask)) 2863 continue; 2864 /* 2865 * When allocating a page cache page for writing, we 2866 * want to get it from a node that is within its dirty 2867 * limit, such that no single node holds more than its 2868 * proportional share of globally allowed dirty pages. 2869 * The dirty limits take into account the node's 2870 * lowmem reserves and high watermark so that kswapd 2871 * should be able to balance it without having to 2872 * write pages from its LRU list. 2873 * 2874 * XXX: For now, allow allocations to potentially 2875 * exceed the per-node dirty limit in the slowpath 2876 * (spread_dirty_pages unset) before going into reclaim, 2877 * which is important when on a NUMA setup the allowed 2878 * nodes are together not big enough to reach the 2879 * global limit. The proper fix for these situations 2880 * will require awareness of nodes in the 2881 * dirty-throttling and the flusher threads. 2882 */ 2883 if (ac->spread_dirty_pages) { 2884 if (last_pgdat_dirty_limit == zone->zone_pgdat) 2885 continue; 2886 2887 if (!node_dirty_ok(zone->zone_pgdat)) { 2888 last_pgdat_dirty_limit = zone->zone_pgdat; 2889 continue; 2890 } 2891 } 2892 2893 mark = zone->watermark[alloc_flags & ALLOC_WMARK_MASK]; 2894 if (!zone_watermark_fast(zone, order, mark, 2895 ac_classzone_idx(ac), alloc_flags)) { 2896 int ret; 2897 2898 /* Checked here to keep the fast path fast */ 2899 BUILD_BUG_ON(ALLOC_NO_WATERMARKS < NR_WMARK); 2900 if (alloc_flags & ALLOC_NO_WATERMARKS) 2901 goto try_this_zone; 2902 2903 if (node_reclaim_mode == 0 || 2904 !zone_allows_reclaim(ac->preferred_zoneref->zone, zone)) 2905 continue; 2906 2907 ret = node_reclaim(zone->zone_pgdat, gfp_mask, order); 2908 switch (ret) { 2909 case NODE_RECLAIM_NOSCAN: 2910 /* did not scan */ 2911 continue; 2912 case NODE_RECLAIM_FULL: 2913 /* scanned but unreclaimable */ 2914 continue; 2915 default: 2916 /* did we reclaim enough */ 2917 if (zone_watermark_ok(zone, order, mark, 2918 ac_classzone_idx(ac), alloc_flags)) 2919 goto try_this_zone; 2920 2921 continue; 2922 } 2923 } 2924 2925 try_this_zone: 2926 page = buffered_rmqueue(ac->preferred_zoneref->zone, zone, order, 2927 gfp_mask, alloc_flags, ac->migratetype); 2928 if (page) { 2929 prep_new_page(page, order, gfp_mask, alloc_flags); 2930 2931 /* 2932 * If this is a high-order atomic allocation then check 2933 * if the pageblock should be reserved for the future 2934 */ 2935 if (unlikely(order && (alloc_flags & ALLOC_HARDER))) 2936 reserve_highatomic_pageblock(page, zone, order); 2937 2938 return page; 2939 } 2940 } 2941 2942 return NULL; 2943 } 2944 2945 /* 2946 * Large machines with many possible nodes should not always dump per-node 2947 * meminfo in irq context. 2948 */ 2949 static inline bool should_suppress_show_mem(void) 2950 { 2951 bool ret = false; 2952 2953 #if NODES_SHIFT > 8 2954 ret = in_interrupt(); 2955 #endif 2956 return ret; 2957 } 2958 2959 static DEFINE_RATELIMIT_STATE(nopage_rs, 2960 DEFAULT_RATELIMIT_INTERVAL, 2961 DEFAULT_RATELIMIT_BURST); 2962 2963 void warn_alloc_failed(gfp_t gfp_mask, unsigned int order, const char *fmt, ...) 2964 { 2965 unsigned int filter = SHOW_MEM_FILTER_NODES; 2966 2967 if ((gfp_mask & __GFP_NOWARN) || !__ratelimit(&nopage_rs) || 2968 debug_guardpage_minorder() > 0) 2969 return; 2970 2971 /* 2972 * This documents exceptions given to allocations in certain 2973 * contexts that are allowed to allocate outside current's set 2974 * of allowed nodes. 2975 */ 2976 if (!(gfp_mask & __GFP_NOMEMALLOC)) 2977 if (test_thread_flag(TIF_MEMDIE) || 2978 (current->flags & (PF_MEMALLOC | PF_EXITING))) 2979 filter &= ~SHOW_MEM_FILTER_NODES; 2980 if (in_interrupt() || !(gfp_mask & __GFP_DIRECT_RECLAIM)) 2981 filter &= ~SHOW_MEM_FILTER_NODES; 2982 2983 if (fmt) { 2984 struct va_format vaf; 2985 va_list args; 2986 2987 va_start(args, fmt); 2988 2989 vaf.fmt = fmt; 2990 vaf.va = &args; 2991 2992 pr_warn("%pV", &vaf); 2993 2994 va_end(args); 2995 } 2996 2997 pr_warn("%s: page allocation failure: order:%u, mode:%#x(%pGg)\n", 2998 current->comm, order, gfp_mask, &gfp_mask); 2999 dump_stack(); 3000 if (!should_suppress_show_mem()) 3001 show_mem(filter); 3002 } 3003 3004 static inline struct page * 3005 __alloc_pages_may_oom(gfp_t gfp_mask, unsigned int order, 3006 const struct alloc_context *ac, unsigned long *did_some_progress) 3007 { 3008 struct oom_control oc = { 3009 .zonelist = ac->zonelist, 3010 .nodemask = ac->nodemask, 3011 .memcg = NULL, 3012 .gfp_mask = gfp_mask, 3013 .order = order, 3014 }; 3015 struct page *page; 3016 3017 *did_some_progress = 0; 3018 3019 /* 3020 * Acquire the oom lock. If that fails, somebody else is 3021 * making progress for us. 3022 */ 3023 if (!mutex_trylock(&oom_lock)) { 3024 *did_some_progress = 1; 3025 schedule_timeout_uninterruptible(1); 3026 return NULL; 3027 } 3028 3029 /* 3030 * Go through the zonelist yet one more time, keep very high watermark 3031 * here, this is only to catch a parallel oom killing, we must fail if 3032 * we're still under heavy pressure. 3033 */ 3034 page = get_page_from_freelist(gfp_mask | __GFP_HARDWALL, order, 3035 ALLOC_WMARK_HIGH|ALLOC_CPUSET, ac); 3036 if (page) 3037 goto out; 3038 3039 if (!(gfp_mask & __GFP_NOFAIL)) { 3040 /* Coredumps can quickly deplete all memory reserves */ 3041 if (current->flags & PF_DUMPCORE) 3042 goto out; 3043 /* The OOM killer will not help higher order allocs */ 3044 if (order > PAGE_ALLOC_COSTLY_ORDER) 3045 goto out; 3046 /* The OOM killer does not needlessly kill tasks for lowmem */ 3047 if (ac->high_zoneidx < ZONE_NORMAL) 3048 goto out; 3049 if (pm_suspended_storage()) 3050 goto out; 3051 /* 3052 * XXX: GFP_NOFS allocations should rather fail than rely on 3053 * other request to make a forward progress. 3054 * We are in an unfortunate situation where out_of_memory cannot 3055 * do much for this context but let's try it to at least get 3056 * access to memory reserved if the current task is killed (see 3057 * out_of_memory). Once filesystems are ready to handle allocation 3058 * failures more gracefully we should just bail out here. 3059 */ 3060 3061 /* The OOM killer may not free memory on a specific node */ 3062 if (gfp_mask & __GFP_THISNODE) 3063 goto out; 3064 } 3065 /* Exhausted what can be done so it's blamo time */ 3066 if (out_of_memory(&oc) || WARN_ON_ONCE(gfp_mask & __GFP_NOFAIL)) { 3067 *did_some_progress = 1; 3068 3069 if (gfp_mask & __GFP_NOFAIL) { 3070 page = get_page_from_freelist(gfp_mask, order, 3071 ALLOC_NO_WATERMARKS|ALLOC_CPUSET, ac); 3072 /* 3073 * fallback to ignore cpuset restriction if our nodes 3074 * are depleted 3075 */ 3076 if (!page) 3077 page = get_page_from_freelist(gfp_mask, order, 3078 ALLOC_NO_WATERMARKS, ac); 3079 } 3080 } 3081 out: 3082 mutex_unlock(&oom_lock); 3083 return page; 3084 } 3085 3086 /* 3087 * Maximum number of compaction retries wit a progress before OOM 3088 * killer is consider as the only way to move forward. 3089 */ 3090 #define MAX_COMPACT_RETRIES 16 3091 3092 #ifdef CONFIG_COMPACTION 3093 /* Try memory compaction for high-order allocations before reclaim */ 3094 static struct page * 3095 __alloc_pages_direct_compact(gfp_t gfp_mask, unsigned int order, 3096 unsigned int alloc_flags, const struct alloc_context *ac, 3097 enum compact_priority prio, enum compact_result *compact_result) 3098 { 3099 struct page *page; 3100 3101 if (!order) 3102 return NULL; 3103 3104 current->flags |= PF_MEMALLOC; 3105 *compact_result = try_to_compact_pages(gfp_mask, order, alloc_flags, ac, 3106 prio); 3107 current->flags &= ~PF_MEMALLOC; 3108 3109 if (*compact_result <= COMPACT_INACTIVE) 3110 return NULL; 3111 3112 /* 3113 * At least in one zone compaction wasn't deferred or skipped, so let's 3114 * count a compaction stall 3115 */ 3116 count_vm_event(COMPACTSTALL); 3117 3118 page = get_page_from_freelist(gfp_mask, order, alloc_flags, ac); 3119 3120 if (page) { 3121 struct zone *zone = page_zone(page); 3122 3123 zone->compact_blockskip_flush = false; 3124 compaction_defer_reset(zone, order, true); 3125 count_vm_event(COMPACTSUCCESS); 3126 return page; 3127 } 3128 3129 /* 3130 * It's bad if compaction run occurs and fails. The most likely reason 3131 * is that pages exist, but not enough to satisfy watermarks. 3132 */ 3133 count_vm_event(COMPACTFAIL); 3134 3135 cond_resched(); 3136 3137 return NULL; 3138 } 3139 3140 #else 3141 static inline struct page * 3142 __alloc_pages_direct_compact(gfp_t gfp_mask, unsigned int order, 3143 unsigned int alloc_flags, const struct alloc_context *ac, 3144 enum compact_priority prio, enum compact_result *compact_result) 3145 { 3146 *compact_result = COMPACT_SKIPPED; 3147 return NULL; 3148 } 3149 3150 #endif /* CONFIG_COMPACTION */ 3151 3152 static inline bool 3153 should_compact_retry(struct alloc_context *ac, unsigned int order, int alloc_flags, 3154 enum compact_result compact_result, 3155 enum compact_priority *compact_priority, 3156 int compaction_retries) 3157 { 3158 struct zone *zone; 3159 struct zoneref *z; 3160 3161 if (!order || order > PAGE_ALLOC_COSTLY_ORDER) 3162 return false; 3163 3164 /* 3165 * There are setups with compaction disabled which would prefer to loop 3166 * inside the allocator rather than hit the oom killer prematurely. 3167 * Let's give them a good hope and keep retrying while the order-0 3168 * watermarks are OK. 3169 */ 3170 for_each_zone_zonelist_nodemask(zone, z, ac->zonelist, ac->high_zoneidx, 3171 ac->nodemask) { 3172 if (zone_watermark_ok(zone, 0, min_wmark_pages(zone), 3173 ac_classzone_idx(ac), alloc_flags)) 3174 return true; 3175 } 3176 return false; 3177 } 3178 3179 /* Perform direct synchronous page reclaim */ 3180 static int 3181 __perform_reclaim(gfp_t gfp_mask, unsigned int order, 3182 const struct alloc_context *ac) 3183 { 3184 struct reclaim_state reclaim_state; 3185 int progress; 3186 3187 cond_resched(); 3188 3189 /* We now go into synchronous reclaim */ 3190 cpuset_memory_pressure_bump(); 3191 current->flags |= PF_MEMALLOC; 3192 lockdep_set_current_reclaim_state(gfp_mask); 3193 reclaim_state.reclaimed_slab = 0; 3194 current->reclaim_state = &reclaim_state; 3195 3196 progress = try_to_free_pages(ac->zonelist, order, gfp_mask, 3197 ac->nodemask); 3198 3199 current->reclaim_state = NULL; 3200 lockdep_clear_current_reclaim_state(); 3201 current->flags &= ~PF_MEMALLOC; 3202 3203 cond_resched(); 3204 3205 return progress; 3206 } 3207 3208 /* The really slow allocator path where we enter direct reclaim */ 3209 static inline struct page * 3210 __alloc_pages_direct_reclaim(gfp_t gfp_mask, unsigned int order, 3211 unsigned int alloc_flags, const struct alloc_context *ac, 3212 unsigned long *did_some_progress) 3213 { 3214 struct page *page = NULL; 3215 bool drained = false; 3216 3217 *did_some_progress = __perform_reclaim(gfp_mask, order, ac); 3218 if (unlikely(!(*did_some_progress))) 3219 return NULL; 3220 3221 retry: 3222 page = get_page_from_freelist(gfp_mask, order, alloc_flags, ac); 3223 3224 /* 3225 * If an allocation failed after direct reclaim, it could be because 3226 * pages are pinned on the per-cpu lists or in high alloc reserves. 3227 * Shrink them them and try again 3228 */ 3229 if (!page && !drained) { 3230 unreserve_highatomic_pageblock(ac); 3231 drain_all_pages(NULL); 3232 drained = true; 3233 goto retry; 3234 } 3235 3236 return page; 3237 } 3238 3239 static void wake_all_kswapds(unsigned int order, const struct alloc_context *ac) 3240 { 3241 struct zoneref *z; 3242 struct zone *zone; 3243 pg_data_t *last_pgdat = NULL; 3244 3245 for_each_zone_zonelist_nodemask(zone, z, ac->zonelist, 3246 ac->high_zoneidx, ac->nodemask) { 3247 if (last_pgdat != zone->zone_pgdat) 3248 wakeup_kswapd(zone, order, ac->high_zoneidx); 3249 last_pgdat = zone->zone_pgdat; 3250 } 3251 } 3252 3253 static inline unsigned int 3254 gfp_to_alloc_flags(gfp_t gfp_mask) 3255 { 3256 unsigned int alloc_flags = ALLOC_WMARK_MIN | ALLOC_CPUSET; 3257 3258 /* __GFP_HIGH is assumed to be the same as ALLOC_HIGH to save a branch. */ 3259 BUILD_BUG_ON(__GFP_HIGH != (__force gfp_t) ALLOC_HIGH); 3260 3261 /* 3262 * The caller may dip into page reserves a bit more if the caller 3263 * cannot run direct reclaim, or if the caller has realtime scheduling 3264 * policy or is asking for __GFP_HIGH memory. GFP_ATOMIC requests will 3265 * set both ALLOC_HARDER (__GFP_ATOMIC) and ALLOC_HIGH (__GFP_HIGH). 3266 */ 3267 alloc_flags |= (__force int) (gfp_mask & __GFP_HIGH); 3268 3269 if (gfp_mask & __GFP_ATOMIC) { 3270 /* 3271 * Not worth trying to allocate harder for __GFP_NOMEMALLOC even 3272 * if it can't schedule. 3273 */ 3274 if (!(gfp_mask & __GFP_NOMEMALLOC)) 3275 alloc_flags |= ALLOC_HARDER; 3276 /* 3277 * Ignore cpuset mems for GFP_ATOMIC rather than fail, see the 3278 * comment for __cpuset_node_allowed(). 3279 */ 3280 alloc_flags &= ~ALLOC_CPUSET; 3281 } else if (unlikely(rt_task(current)) && !in_interrupt()) 3282 alloc_flags |= ALLOC_HARDER; 3283 3284 #ifdef CONFIG_CMA 3285 if (gfpflags_to_migratetype(gfp_mask) == MIGRATE_MOVABLE) 3286 alloc_flags |= ALLOC_CMA; 3287 #endif 3288 return alloc_flags; 3289 } 3290 3291 bool gfp_pfmemalloc_allowed(gfp_t gfp_mask) 3292 { 3293 if (unlikely(gfp_mask & __GFP_NOMEMALLOC)) 3294 return false; 3295 3296 if (gfp_mask & __GFP_MEMALLOC) 3297 return true; 3298 if (in_serving_softirq() && (current->flags & PF_MEMALLOC)) 3299 return true; 3300 if (!in_interrupt() && 3301 ((current->flags & PF_MEMALLOC) || 3302 unlikely(test_thread_flag(TIF_MEMDIE)))) 3303 return true; 3304 3305 return false; 3306 } 3307 3308 /* 3309 * Maximum number of reclaim retries without any progress before OOM killer 3310 * is consider as the only way to move forward. 3311 */ 3312 #define MAX_RECLAIM_RETRIES 16 3313 3314 /* 3315 * Checks whether it makes sense to retry the reclaim to make a forward progress 3316 * for the given allocation request. 3317 * The reclaim feedback represented by did_some_progress (any progress during 3318 * the last reclaim round) and no_progress_loops (number of reclaim rounds without 3319 * any progress in a row) is considered as well as the reclaimable pages on the 3320 * applicable zone list (with a backoff mechanism which is a function of 3321 * no_progress_loops). 3322 * 3323 * Returns true if a retry is viable or false to enter the oom path. 3324 */ 3325 static inline bool 3326 should_reclaim_retry(gfp_t gfp_mask, unsigned order, 3327 struct alloc_context *ac, int alloc_flags, 3328 bool did_some_progress, int no_progress_loops) 3329 { 3330 struct zone *zone; 3331 struct zoneref *z; 3332 3333 /* 3334 * Make sure we converge to OOM if we cannot make any progress 3335 * several times in the row. 3336 */ 3337 if (no_progress_loops > MAX_RECLAIM_RETRIES) 3338 return false; 3339 3340 /* 3341 * Keep reclaiming pages while there is a chance this will lead 3342 * somewhere. If none of the target zones can satisfy our allocation 3343 * request even if all reclaimable pages are considered then we are 3344 * screwed and have to go OOM. 3345 */ 3346 for_each_zone_zonelist_nodemask(zone, z, ac->zonelist, ac->high_zoneidx, 3347 ac->nodemask) { 3348 unsigned long available; 3349 unsigned long reclaimable; 3350 3351 available = reclaimable = zone_reclaimable_pages(zone); 3352 available -= DIV_ROUND_UP(no_progress_loops * available, 3353 MAX_RECLAIM_RETRIES); 3354 available += zone_page_state_snapshot(zone, NR_FREE_PAGES); 3355 3356 /* 3357 * Would the allocation succeed if we reclaimed the whole 3358 * available? 3359 */ 3360 if (__zone_watermark_ok(zone, order, min_wmark_pages(zone), 3361 ac_classzone_idx(ac), alloc_flags, available)) { 3362 /* 3363 * If we didn't make any progress and have a lot of 3364 * dirty + writeback pages then we should wait for 3365 * an IO to complete to slow down the reclaim and 3366 * prevent from pre mature OOM 3367 */ 3368 if (!did_some_progress) { 3369 unsigned long write_pending; 3370 3371 write_pending = zone_page_state_snapshot(zone, 3372 NR_ZONE_WRITE_PENDING); 3373 3374 if (2 * write_pending > reclaimable) { 3375 congestion_wait(BLK_RW_ASYNC, HZ/10); 3376 return true; 3377 } 3378 } 3379 3380 /* 3381 * Memory allocation/reclaim might be called from a WQ 3382 * context and the current implementation of the WQ 3383 * concurrency control doesn't recognize that 3384 * a particular WQ is congested if the worker thread is 3385 * looping without ever sleeping. Therefore we have to 3386 * do a short sleep here rather than calling 3387 * cond_resched(). 3388 */ 3389 if (current->flags & PF_WQ_WORKER) 3390 schedule_timeout_uninterruptible(1); 3391 else 3392 cond_resched(); 3393 3394 return true; 3395 } 3396 } 3397 3398 return false; 3399 } 3400 3401 static inline struct page * 3402 __alloc_pages_slowpath(gfp_t gfp_mask, unsigned int order, 3403 struct alloc_context *ac) 3404 { 3405 bool can_direct_reclaim = gfp_mask & __GFP_DIRECT_RECLAIM; 3406 struct page *page = NULL; 3407 unsigned int alloc_flags; 3408 unsigned long did_some_progress; 3409 enum compact_priority compact_priority = DEF_COMPACT_PRIORITY; 3410 enum compact_result compact_result; 3411 int compaction_retries = 0; 3412 int no_progress_loops = 0; 3413 3414 /* 3415 * In the slowpath, we sanity check order to avoid ever trying to 3416 * reclaim >= MAX_ORDER areas which will never succeed. Callers may 3417 * be using allocators in order of preference for an area that is 3418 * too large. 3419 */ 3420 if (order >= MAX_ORDER) { 3421 WARN_ON_ONCE(!(gfp_mask & __GFP_NOWARN)); 3422 return NULL; 3423 } 3424 3425 /* 3426 * We also sanity check to catch abuse of atomic reserves being used by 3427 * callers that are not in atomic context. 3428 */ 3429 if (WARN_ON_ONCE((gfp_mask & (__GFP_ATOMIC|__GFP_DIRECT_RECLAIM)) == 3430 (__GFP_ATOMIC|__GFP_DIRECT_RECLAIM))) 3431 gfp_mask &= ~__GFP_ATOMIC; 3432 3433 /* 3434 * The fast path uses conservative alloc_flags to succeed only until 3435 * kswapd needs to be woken up, and to avoid the cost of setting up 3436 * alloc_flags precisely. So we do that now. 3437 */ 3438 alloc_flags = gfp_to_alloc_flags(gfp_mask); 3439 3440 if (gfp_mask & __GFP_KSWAPD_RECLAIM) 3441 wake_all_kswapds(order, ac); 3442 3443 /* 3444 * The adjusted alloc_flags might result in immediate success, so try 3445 * that first 3446 */ 3447 page = get_page_from_freelist(gfp_mask, order, alloc_flags, ac); 3448 if (page) 3449 goto got_pg; 3450 3451 /* 3452 * For costly allocations, try direct compaction first, as it's likely 3453 * that we have enough base pages and don't need to reclaim. Don't try 3454 * that for allocations that are allowed to ignore watermarks, as the 3455 * ALLOC_NO_WATERMARKS attempt didn't yet happen. 3456 */ 3457 if (can_direct_reclaim && order > PAGE_ALLOC_COSTLY_ORDER && 3458 !gfp_pfmemalloc_allowed(gfp_mask)) { 3459 page = __alloc_pages_direct_compact(gfp_mask, order, 3460 alloc_flags, ac, 3461 INIT_COMPACT_PRIORITY, 3462 &compact_result); 3463 if (page) 3464 goto got_pg; 3465 3466 /* 3467 * Checks for costly allocations with __GFP_NORETRY, which 3468 * includes THP page fault allocations 3469 */ 3470 if (gfp_mask & __GFP_NORETRY) { 3471 /* 3472 * If compaction is deferred for high-order allocations, 3473 * it is because sync compaction recently failed. If 3474 * this is the case and the caller requested a THP 3475 * allocation, we do not want to heavily disrupt the 3476 * system, so we fail the allocation instead of entering 3477 * direct reclaim. 3478 */ 3479 if (compact_result == COMPACT_DEFERRED) 3480 goto nopage; 3481 3482 /* 3483 * Looks like reclaim/compaction is worth trying, but 3484 * sync compaction could be very expensive, so keep 3485 * using async compaction. 3486 */ 3487 compact_priority = INIT_COMPACT_PRIORITY; 3488 } 3489 } 3490 3491 retry: 3492 /* Ensure kswapd doesn't accidentally go to sleep as long as we loop */ 3493 if (gfp_mask & __GFP_KSWAPD_RECLAIM) 3494 wake_all_kswapds(order, ac); 3495 3496 if (gfp_pfmemalloc_allowed(gfp_mask)) 3497 alloc_flags = ALLOC_NO_WATERMARKS; 3498 3499 /* 3500 * Reset the zonelist iterators if memory policies can be ignored. 3501 * These allocations are high priority and system rather than user 3502 * orientated. 3503 */ 3504 if (!(alloc_flags & ALLOC_CPUSET) || (alloc_flags & ALLOC_NO_WATERMARKS)) { 3505 ac->zonelist = node_zonelist(numa_node_id(), gfp_mask); 3506 ac->preferred_zoneref = first_zones_zonelist(ac->zonelist, 3507 ac->high_zoneidx, ac->nodemask); 3508 } 3509 3510 /* Attempt with potentially adjusted zonelist and alloc_flags */ 3511 page = get_page_from_freelist(gfp_mask, order, alloc_flags, ac); 3512 if (page) 3513 goto got_pg; 3514 3515 /* Caller is not willing to reclaim, we can't balance anything */ 3516 if (!can_direct_reclaim) { 3517 /* 3518 * All existing users of the __GFP_NOFAIL are blockable, so warn 3519 * of any new users that actually allow this type of allocation 3520 * to fail. 3521 */ 3522 WARN_ON_ONCE(gfp_mask & __GFP_NOFAIL); 3523 goto nopage; 3524 } 3525 3526 /* Avoid recursion of direct reclaim */ 3527 if (current->flags & PF_MEMALLOC) { 3528 /* 3529 * __GFP_NOFAIL request from this context is rather bizarre 3530 * because we cannot reclaim anything and only can loop waiting 3531 * for somebody to do a work for us. 3532 */ 3533 if (WARN_ON_ONCE(gfp_mask & __GFP_NOFAIL)) { 3534 cond_resched(); 3535 goto retry; 3536 } 3537 goto nopage; 3538 } 3539 3540 /* Avoid allocations with no watermarks from looping endlessly */ 3541 if (test_thread_flag(TIF_MEMDIE) && !(gfp_mask & __GFP_NOFAIL)) 3542 goto nopage; 3543 3544 3545 /* Try direct reclaim and then allocating */ 3546 page = __alloc_pages_direct_reclaim(gfp_mask, order, alloc_flags, ac, 3547 &did_some_progress); 3548 if (page) 3549 goto got_pg; 3550 3551 /* Try direct compaction and then allocating */ 3552 page = __alloc_pages_direct_compact(gfp_mask, order, alloc_flags, ac, 3553 compact_priority, &compact_result); 3554 if (page) 3555 goto got_pg; 3556 3557 if (order && compaction_made_progress(compact_result)) 3558 compaction_retries++; 3559 3560 /* Do not loop if specifically requested */ 3561 if (gfp_mask & __GFP_NORETRY) 3562 goto nopage; 3563 3564 /* 3565 * Do not retry costly high order allocations unless they are 3566 * __GFP_REPEAT 3567 */ 3568 if (order > PAGE_ALLOC_COSTLY_ORDER && !(gfp_mask & __GFP_REPEAT)) 3569 goto nopage; 3570 3571 /* 3572 * Costly allocations might have made a progress but this doesn't mean 3573 * their order will become available due to high fragmentation so 3574 * always increment the no progress counter for them 3575 */ 3576 if (did_some_progress && order <= PAGE_ALLOC_COSTLY_ORDER) 3577 no_progress_loops = 0; 3578 else 3579 no_progress_loops++; 3580 3581 if (should_reclaim_retry(gfp_mask, order, ac, alloc_flags, 3582 did_some_progress > 0, no_progress_loops)) 3583 goto retry; 3584 3585 /* 3586 * It doesn't make any sense to retry for the compaction if the order-0 3587 * reclaim is not able to make any progress because the current 3588 * implementation of the compaction depends on the sufficient amount 3589 * of free memory (see __compaction_suitable) 3590 */ 3591 if (did_some_progress > 0 && 3592 should_compact_retry(ac, order, alloc_flags, 3593 compact_result, &compact_priority, 3594 compaction_retries)) 3595 goto retry; 3596 3597 /* Reclaim has failed us, start killing things */ 3598 page = __alloc_pages_may_oom(gfp_mask, order, ac, &did_some_progress); 3599 if (page) 3600 goto got_pg; 3601 3602 /* Retry as long as the OOM killer is making progress */ 3603 if (did_some_progress) { 3604 no_progress_loops = 0; 3605 goto retry; 3606 } 3607 3608 nopage: 3609 warn_alloc_failed(gfp_mask, order, NULL); 3610 got_pg: 3611 return page; 3612 } 3613 3614 /* 3615 * This is the 'heart' of the zoned buddy allocator. 3616 */ 3617 struct page * 3618 __alloc_pages_nodemask(gfp_t gfp_mask, unsigned int order, 3619 struct zonelist *zonelist, nodemask_t *nodemask) 3620 { 3621 struct page *page; 3622 unsigned int cpuset_mems_cookie; 3623 unsigned int alloc_flags = ALLOC_WMARK_LOW; 3624 gfp_t alloc_mask = gfp_mask; /* The gfp_t that was actually used for allocation */ 3625 struct alloc_context ac = { 3626 .high_zoneidx = gfp_zone(gfp_mask), 3627 .zonelist = zonelist, 3628 .nodemask = nodemask, 3629 .migratetype = gfpflags_to_migratetype(gfp_mask), 3630 }; 3631 3632 if (cpusets_enabled()) { 3633 alloc_mask |= __GFP_HARDWALL; 3634 alloc_flags |= ALLOC_CPUSET; 3635 if (!ac.nodemask) 3636 ac.nodemask = &cpuset_current_mems_allowed; 3637 } 3638 3639 gfp_mask &= gfp_allowed_mask; 3640 3641 lockdep_trace_alloc(gfp_mask); 3642 3643 might_sleep_if(gfp_mask & __GFP_DIRECT_RECLAIM); 3644 3645 if (should_fail_alloc_page(gfp_mask, order)) 3646 return NULL; 3647 3648 /* 3649 * Check the zones suitable for the gfp_mask contain at least one 3650 * valid zone. It's possible to have an empty zonelist as a result 3651 * of __GFP_THISNODE and a memoryless node 3652 */ 3653 if (unlikely(!zonelist->_zonerefs->zone)) 3654 return NULL; 3655 3656 if (IS_ENABLED(CONFIG_CMA) && ac.migratetype == MIGRATE_MOVABLE) 3657 alloc_flags |= ALLOC_CMA; 3658 3659 retry_cpuset: 3660 cpuset_mems_cookie = read_mems_allowed_begin(); 3661 3662 /* Dirty zone balancing only done in the fast path */ 3663 ac.spread_dirty_pages = (gfp_mask & __GFP_WRITE); 3664 3665 /* 3666 * The preferred zone is used for statistics but crucially it is 3667 * also used as the starting point for the zonelist iterator. It 3668 * may get reset for allocations that ignore memory policies. 3669 */ 3670 ac.preferred_zoneref = first_zones_zonelist(ac.zonelist, 3671 ac.high_zoneidx, ac.nodemask); 3672 if (!ac.preferred_zoneref) { 3673 page = NULL; 3674 goto no_zone; 3675 } 3676 3677 /* First allocation attempt */ 3678 page = get_page_from_freelist(alloc_mask, order, alloc_flags, &ac); 3679 if (likely(page)) 3680 goto out; 3681 3682 /* 3683 * Runtime PM, block IO and its error handling path can deadlock 3684 * because I/O on the device might not complete. 3685 */ 3686 alloc_mask = memalloc_noio_flags(gfp_mask); 3687 ac.spread_dirty_pages = false; 3688 3689 /* 3690 * Restore the original nodemask if it was potentially replaced with 3691 * &cpuset_current_mems_allowed to optimize the fast-path attempt. 3692 */ 3693 if (cpusets_enabled()) 3694 ac.nodemask = nodemask; 3695 page = __alloc_pages_slowpath(alloc_mask, order, &ac); 3696 3697 no_zone: 3698 /* 3699 * When updating a task's mems_allowed, it is possible to race with 3700 * parallel threads in such a way that an allocation can fail while 3701 * the mask is being updated. If a page allocation is about to fail, 3702 * check if the cpuset changed during allocation and if so, retry. 3703 */ 3704 if (unlikely(!page && read_mems_allowed_retry(cpuset_mems_cookie))) { 3705 alloc_mask = gfp_mask; 3706 goto retry_cpuset; 3707 } 3708 3709 out: 3710 if (memcg_kmem_enabled() && (gfp_mask & __GFP_ACCOUNT) && page && 3711 unlikely(memcg_kmem_charge(page, gfp_mask, order) != 0)) { 3712 __free_pages(page, order); 3713 page = NULL; 3714 } 3715 3716 if (kmemcheck_enabled && page) 3717 kmemcheck_pagealloc_alloc(page, order, gfp_mask); 3718 3719 trace_mm_page_alloc(page, order, alloc_mask, ac.migratetype); 3720 3721 return page; 3722 } 3723 EXPORT_SYMBOL(__alloc_pages_nodemask); 3724 3725 /* 3726 * Common helper functions. 3727 */ 3728 unsigned long __get_free_pages(gfp_t gfp_mask, unsigned int order) 3729 { 3730 struct page *page; 3731 3732 /* 3733 * __get_free_pages() returns a 32-bit address, which cannot represent 3734 * a highmem page 3735 */ 3736 VM_BUG_ON((gfp_mask & __GFP_HIGHMEM) != 0); 3737 3738 page = alloc_pages(gfp_mask, order); 3739 if (!page) 3740 return 0; 3741 return (unsigned long) page_address(page); 3742 } 3743 EXPORT_SYMBOL(__get_free_pages); 3744 3745 unsigned long get_zeroed_page(gfp_t gfp_mask) 3746 { 3747 return __get_free_pages(gfp_mask | __GFP_ZERO, 0); 3748 } 3749 EXPORT_SYMBOL(get_zeroed_page); 3750 3751 void __free_pages(struct page *page, unsigned int order) 3752 { 3753 if (put_page_testzero(page)) { 3754 if (order == 0) 3755 free_hot_cold_page(page, false); 3756 else 3757 __free_pages_ok(page, order); 3758 } 3759 } 3760 3761 EXPORT_SYMBOL(__free_pages); 3762 3763 void free_pages(unsigned long addr, unsigned int order) 3764 { 3765 if (addr != 0) { 3766 VM_BUG_ON(!virt_addr_valid((void *)addr)); 3767 __free_pages(virt_to_page((void *)addr), order); 3768 } 3769 } 3770 3771 EXPORT_SYMBOL(free_pages); 3772 3773 /* 3774 * Page Fragment: 3775 * An arbitrary-length arbitrary-offset area of memory which resides 3776 * within a 0 or higher order page. Multiple fragments within that page 3777 * are individually refcounted, in the page's reference counter. 3778 * 3779 * The page_frag functions below provide a simple allocation framework for 3780 * page fragments. This is used by the network stack and network device 3781 * drivers to provide a backing region of memory for use as either an 3782 * sk_buff->head, or to be used in the "frags" portion of skb_shared_info. 3783 */ 3784 static struct page *__page_frag_refill(struct page_frag_cache *nc, 3785 gfp_t gfp_mask) 3786 { 3787 struct page *page = NULL; 3788 gfp_t gfp = gfp_mask; 3789 3790 #if (PAGE_SIZE < PAGE_FRAG_CACHE_MAX_SIZE) 3791 gfp_mask |= __GFP_COMP | __GFP_NOWARN | __GFP_NORETRY | 3792 __GFP_NOMEMALLOC; 3793 page = alloc_pages_node(NUMA_NO_NODE, gfp_mask, 3794 PAGE_FRAG_CACHE_MAX_ORDER); 3795 nc->size = page ? PAGE_FRAG_CACHE_MAX_SIZE : PAGE_SIZE; 3796 #endif 3797 if (unlikely(!page)) 3798 page = alloc_pages_node(NUMA_NO_NODE, gfp, 0); 3799 3800 nc->va = page ? page_address(page) : NULL; 3801 3802 return page; 3803 } 3804 3805 void *__alloc_page_frag(struct page_frag_cache *nc, 3806 unsigned int fragsz, gfp_t gfp_mask) 3807 { 3808 unsigned int size = PAGE_SIZE; 3809 struct page *page; 3810 int offset; 3811 3812 if (unlikely(!nc->va)) { 3813 refill: 3814 page = __page_frag_refill(nc, gfp_mask); 3815 if (!page) 3816 return NULL; 3817 3818 #if (PAGE_SIZE < PAGE_FRAG_CACHE_MAX_SIZE) 3819 /* if size can vary use size else just use PAGE_SIZE */ 3820 size = nc->size; 3821 #endif 3822 /* Even if we own the page, we do not use atomic_set(). 3823 * This would break get_page_unless_zero() users. 3824 */ 3825 page_ref_add(page, size - 1); 3826 3827 /* reset page count bias and offset to start of new frag */ 3828 nc->pfmemalloc = page_is_pfmemalloc(page); 3829 nc->pagecnt_bias = size; 3830 nc->offset = size; 3831 } 3832 3833 offset = nc->offset - fragsz; 3834 if (unlikely(offset < 0)) { 3835 page = virt_to_page(nc->va); 3836 3837 if (!page_ref_sub_and_test(page, nc->pagecnt_bias)) 3838 goto refill; 3839 3840 #if (PAGE_SIZE < PAGE_FRAG_CACHE_MAX_SIZE) 3841 /* if size can vary use size else just use PAGE_SIZE */ 3842 size = nc->size; 3843 #endif 3844 /* OK, page count is 0, we can safely set it */ 3845 set_page_count(page, size); 3846 3847 /* reset page count bias and offset to start of new frag */ 3848 nc->pagecnt_bias = size; 3849 offset = size - fragsz; 3850 } 3851 3852 nc->pagecnt_bias--; 3853 nc->offset = offset; 3854 3855 return nc->va + offset; 3856 } 3857 EXPORT_SYMBOL(__alloc_page_frag); 3858 3859 /* 3860 * Frees a page fragment allocated out of either a compound or order 0 page. 3861 */ 3862 void __free_page_frag(void *addr) 3863 { 3864 struct page *page = virt_to_head_page(addr); 3865 3866 if (unlikely(put_page_testzero(page))) 3867 __free_pages_ok(page, compound_order(page)); 3868 } 3869 EXPORT_SYMBOL(__free_page_frag); 3870 3871 static void *make_alloc_exact(unsigned long addr, unsigned int order, 3872 size_t size) 3873 { 3874 if (addr) { 3875 unsigned long alloc_end = addr + (PAGE_SIZE << order); 3876 unsigned long used = addr + PAGE_ALIGN(size); 3877 3878 split_page(virt_to_page((void *)addr), order); 3879 while (used < alloc_end) { 3880 free_page(used); 3881 used += PAGE_SIZE; 3882 } 3883 } 3884 return (void *)addr; 3885 } 3886 3887 /** 3888 * alloc_pages_exact - allocate an exact number physically-contiguous pages. 3889 * @size: the number of bytes to allocate 3890 * @gfp_mask: GFP flags for the allocation 3891 * 3892 * This function is similar to alloc_pages(), except that it allocates the 3893 * minimum number of pages to satisfy the request. alloc_pages() can only 3894 * allocate memory in power-of-two pages. 3895 * 3896 * This function is also limited by MAX_ORDER. 3897 * 3898 * Memory allocated by this function must be released by free_pages_exact(). 3899 */ 3900 void *alloc_pages_exact(size_t size, gfp_t gfp_mask) 3901 { 3902 unsigned int order = get_order(size); 3903 unsigned long addr; 3904 3905 addr = __get_free_pages(gfp_mask, order); 3906 return make_alloc_exact(addr, order, size); 3907 } 3908 EXPORT_SYMBOL(alloc_pages_exact); 3909 3910 /** 3911 * alloc_pages_exact_nid - allocate an exact number of physically-contiguous 3912 * pages on a node. 3913 * @nid: the preferred node ID where memory should be allocated 3914 * @size: the number of bytes to allocate 3915 * @gfp_mask: GFP flags for the allocation 3916 * 3917 * Like alloc_pages_exact(), but try to allocate on node nid first before falling 3918 * back. 3919 */ 3920 void * __meminit alloc_pages_exact_nid(int nid, size_t size, gfp_t gfp_mask) 3921 { 3922 unsigned int order = get_order(size); 3923 struct page *p = alloc_pages_node(nid, gfp_mask, order); 3924 if (!p) 3925 return NULL; 3926 return make_alloc_exact((unsigned long)page_address(p), order, size); 3927 } 3928 3929 /** 3930 * free_pages_exact - release memory allocated via alloc_pages_exact() 3931 * @virt: the value returned by alloc_pages_exact. 3932 * @size: size of allocation, same value as passed to alloc_pages_exact(). 3933 * 3934 * Release the memory allocated by a previous call to alloc_pages_exact. 3935 */ 3936 void free_pages_exact(void *virt, size_t size) 3937 { 3938 unsigned long addr = (unsigned long)virt; 3939 unsigned long end = addr + PAGE_ALIGN(size); 3940 3941 while (addr < end) { 3942 free_page(addr); 3943 addr += PAGE_SIZE; 3944 } 3945 } 3946 EXPORT_SYMBOL(free_pages_exact); 3947 3948 /** 3949 * nr_free_zone_pages - count number of pages beyond high watermark 3950 * @offset: The zone index of the highest zone 3951 * 3952 * nr_free_zone_pages() counts the number of counts pages which are beyond the 3953 * high watermark within all zones at or below a given zone index. For each 3954 * zone, the number of pages is calculated as: 3955 * managed_pages - high_pages 3956 */ 3957 static unsigned long nr_free_zone_pages(int offset) 3958 { 3959 struct zoneref *z; 3960 struct zone *zone; 3961 3962 /* Just pick one node, since fallback list is circular */ 3963 unsigned long sum = 0; 3964 3965 struct zonelist *zonelist = node_zonelist(numa_node_id(), GFP_KERNEL); 3966 3967 for_each_zone_zonelist(zone, z, zonelist, offset) { 3968 unsigned long size = zone->managed_pages; 3969 unsigned long high = high_wmark_pages(zone); 3970 if (size > high) 3971 sum += size - high; 3972 } 3973 3974 return sum; 3975 } 3976 3977 /** 3978 * nr_free_buffer_pages - count number of pages beyond high watermark 3979 * 3980 * nr_free_buffer_pages() counts the number of pages which are beyond the high 3981 * watermark within ZONE_DMA and ZONE_NORMAL. 3982 */ 3983 unsigned long nr_free_buffer_pages(void) 3984 { 3985 return nr_free_zone_pages(gfp_zone(GFP_USER)); 3986 } 3987 EXPORT_SYMBOL_GPL(nr_free_buffer_pages); 3988 3989 /** 3990 * nr_free_pagecache_pages - count number of pages beyond high watermark 3991 * 3992 * nr_free_pagecache_pages() counts the number of pages which are beyond the 3993 * high watermark within all zones. 3994 */ 3995 unsigned long nr_free_pagecache_pages(void) 3996 { 3997 return nr_free_zone_pages(gfp_zone(GFP_HIGHUSER_MOVABLE)); 3998 } 3999 4000 static inline void show_node(struct zone *zone) 4001 { 4002 if (IS_ENABLED(CONFIG_NUMA)) 4003 printk("Node %d ", zone_to_nid(zone)); 4004 } 4005 4006 long si_mem_available(void) 4007 { 4008 long available; 4009 unsigned long pagecache; 4010 unsigned long wmark_low = 0; 4011 unsigned long pages[NR_LRU_LISTS]; 4012 struct zone *zone; 4013 int lru; 4014 4015 for (lru = LRU_BASE; lru < NR_LRU_LISTS; lru++) 4016 pages[lru] = global_node_page_state(NR_LRU_BASE + lru); 4017 4018 for_each_zone(zone) 4019 wmark_low += zone->watermark[WMARK_LOW]; 4020 4021 /* 4022 * Estimate the amount of memory available for userspace allocations, 4023 * without causing swapping. 4024 */ 4025 available = global_page_state(NR_FREE_PAGES) - totalreserve_pages; 4026 4027 /* 4028 * Not all the page cache can be freed, otherwise the system will 4029 * start swapping. Assume at least half of the page cache, or the 4030 * low watermark worth of cache, needs to stay. 4031 */ 4032 pagecache = pages[LRU_ACTIVE_FILE] + pages[LRU_INACTIVE_FILE]; 4033 pagecache -= min(pagecache / 2, wmark_low); 4034 available += pagecache; 4035 4036 /* 4037 * Part of the reclaimable slab consists of items that are in use, 4038 * and cannot be freed. Cap this estimate at the low watermark. 4039 */ 4040 available += global_page_state(NR_SLAB_RECLAIMABLE) - 4041 min(global_page_state(NR_SLAB_RECLAIMABLE) / 2, wmark_low); 4042 4043 if (available < 0) 4044 available = 0; 4045 return available; 4046 } 4047 EXPORT_SYMBOL_GPL(si_mem_available); 4048 4049 void si_meminfo(struct sysinfo *val) 4050 { 4051 val->totalram = totalram_pages; 4052 val->sharedram = global_node_page_state(NR_SHMEM); 4053 val->freeram = global_page_state(NR_FREE_PAGES); 4054 val->bufferram = nr_blockdev_pages(); 4055 val->totalhigh = totalhigh_pages; 4056 val->freehigh = nr_free_highpages(); 4057 val->mem_unit = PAGE_SIZE; 4058 } 4059 4060 EXPORT_SYMBOL(si_meminfo); 4061 4062 #ifdef CONFIG_NUMA 4063 void si_meminfo_node(struct sysinfo *val, int nid) 4064 { 4065 int zone_type; /* needs to be signed */ 4066 unsigned long managed_pages = 0; 4067 unsigned long managed_highpages = 0; 4068 unsigned long free_highpages = 0; 4069 pg_data_t *pgdat = NODE_DATA(nid); 4070 4071 for (zone_type = 0; zone_type < MAX_NR_ZONES; zone_type++) 4072 managed_pages += pgdat->node_zones[zone_type].managed_pages; 4073 val->totalram = managed_pages; 4074 val->sharedram = node_page_state(pgdat, NR_SHMEM); 4075 val->freeram = sum_zone_node_page_state(nid, NR_FREE_PAGES); 4076 #ifdef CONFIG_HIGHMEM 4077 for (zone_type = 0; zone_type < MAX_NR_ZONES; zone_type++) { 4078 struct zone *zone = &pgdat->node_zones[zone_type]; 4079 4080 if (is_highmem(zone)) { 4081 managed_highpages += zone->managed_pages; 4082 free_highpages += zone_page_state(zone, NR_FREE_PAGES); 4083 } 4084 } 4085 val->totalhigh = managed_highpages; 4086 val->freehigh = free_highpages; 4087 #else 4088 val->totalhigh = managed_highpages; 4089 val->freehigh = free_highpages; 4090 #endif 4091 val->mem_unit = PAGE_SIZE; 4092 } 4093 #endif 4094 4095 /* 4096 * Determine whether the node should be displayed or not, depending on whether 4097 * SHOW_MEM_FILTER_NODES was passed to show_free_areas(). 4098 */ 4099 bool skip_free_areas_node(unsigned int flags, int nid) 4100 { 4101 bool ret = false; 4102 unsigned int cpuset_mems_cookie; 4103 4104 if (!(flags & SHOW_MEM_FILTER_NODES)) 4105 goto out; 4106 4107 do { 4108 cpuset_mems_cookie = read_mems_allowed_begin(); 4109 ret = !node_isset(nid, cpuset_current_mems_allowed); 4110 } while (read_mems_allowed_retry(cpuset_mems_cookie)); 4111 out: 4112 return ret; 4113 } 4114 4115 #define K(x) ((x) << (PAGE_SHIFT-10)) 4116 4117 static void show_migration_types(unsigned char type) 4118 { 4119 static const char types[MIGRATE_TYPES] = { 4120 [MIGRATE_UNMOVABLE] = 'U', 4121 [MIGRATE_MOVABLE] = 'M', 4122 [MIGRATE_RECLAIMABLE] = 'E', 4123 [MIGRATE_HIGHATOMIC] = 'H', 4124 #ifdef CONFIG_CMA 4125 [MIGRATE_CMA] = 'C', 4126 #endif 4127 #ifdef CONFIG_MEMORY_ISOLATION 4128 [MIGRATE_ISOLATE] = 'I', 4129 #endif 4130 }; 4131 char tmp[MIGRATE_TYPES + 1]; 4132 char *p = tmp; 4133 int i; 4134 4135 for (i = 0; i < MIGRATE_TYPES; i++) { 4136 if (type & (1 << i)) 4137 *p++ = types[i]; 4138 } 4139 4140 *p = '\0'; 4141 printk("(%s) ", tmp); 4142 } 4143 4144 /* 4145 * Show free area list (used inside shift_scroll-lock stuff) 4146 * We also calculate the percentage fragmentation. We do this by counting the 4147 * memory on each free list with the exception of the first item on the list. 4148 * 4149 * Bits in @filter: 4150 * SHOW_MEM_FILTER_NODES: suppress nodes that are not allowed by current's 4151 * cpuset. 4152 */ 4153 void show_free_areas(unsigned int filter) 4154 { 4155 unsigned long free_pcp = 0; 4156 int cpu; 4157 struct zone *zone; 4158 pg_data_t *pgdat; 4159 4160 for_each_populated_zone(zone) { 4161 if (skip_free_areas_node(filter, zone_to_nid(zone))) 4162 continue; 4163 4164 for_each_online_cpu(cpu) 4165 free_pcp += per_cpu_ptr(zone->pageset, cpu)->pcp.count; 4166 } 4167 4168 printk("active_anon:%lu inactive_anon:%lu isolated_anon:%lu\n" 4169 " active_file:%lu inactive_file:%lu isolated_file:%lu\n" 4170 " unevictable:%lu dirty:%lu writeback:%lu unstable:%lu\n" 4171 " slab_reclaimable:%lu slab_unreclaimable:%lu\n" 4172 " mapped:%lu shmem:%lu pagetables:%lu bounce:%lu\n" 4173 " free:%lu free_pcp:%lu free_cma:%lu\n", 4174 global_node_page_state(NR_ACTIVE_ANON), 4175 global_node_page_state(NR_INACTIVE_ANON), 4176 global_node_page_state(NR_ISOLATED_ANON), 4177 global_node_page_state(NR_ACTIVE_FILE), 4178 global_node_page_state(NR_INACTIVE_FILE), 4179 global_node_page_state(NR_ISOLATED_FILE), 4180 global_node_page_state(NR_UNEVICTABLE), 4181 global_node_page_state(NR_FILE_DIRTY), 4182 global_node_page_state(NR_WRITEBACK), 4183 global_node_page_state(NR_UNSTABLE_NFS), 4184 global_page_state(NR_SLAB_RECLAIMABLE), 4185 global_page_state(NR_SLAB_UNRECLAIMABLE), 4186 global_node_page_state(NR_FILE_MAPPED), 4187 global_node_page_state(NR_SHMEM), 4188 global_page_state(NR_PAGETABLE), 4189 global_page_state(NR_BOUNCE), 4190 global_page_state(NR_FREE_PAGES), 4191 free_pcp, 4192 global_page_state(NR_FREE_CMA_PAGES)); 4193 4194 for_each_online_pgdat(pgdat) { 4195 printk("Node %d" 4196 " active_anon:%lukB" 4197 " inactive_anon:%lukB" 4198 " active_file:%lukB" 4199 " inactive_file:%lukB" 4200 " unevictable:%lukB" 4201 " isolated(anon):%lukB" 4202 " isolated(file):%lukB" 4203 " mapped:%lukB" 4204 " dirty:%lukB" 4205 " writeback:%lukB" 4206 " shmem:%lukB" 4207 #ifdef CONFIG_TRANSPARENT_HUGEPAGE 4208 " shmem_thp: %lukB" 4209 " shmem_pmdmapped: %lukB" 4210 " anon_thp: %lukB" 4211 #endif 4212 " writeback_tmp:%lukB" 4213 " unstable:%lukB" 4214 " pages_scanned:%lu" 4215 " all_unreclaimable? %s" 4216 "\n", 4217 pgdat->node_id, 4218 K(node_page_state(pgdat, NR_ACTIVE_ANON)), 4219 K(node_page_state(pgdat, NR_INACTIVE_ANON)), 4220 K(node_page_state(pgdat, NR_ACTIVE_FILE)), 4221 K(node_page_state(pgdat, NR_INACTIVE_FILE)), 4222 K(node_page_state(pgdat, NR_UNEVICTABLE)), 4223 K(node_page_state(pgdat, NR_ISOLATED_ANON)), 4224 K(node_page_state(pgdat, NR_ISOLATED_FILE)), 4225 K(node_page_state(pgdat, NR_FILE_MAPPED)), 4226 K(node_page_state(pgdat, NR_FILE_DIRTY)), 4227 K(node_page_state(pgdat, NR_WRITEBACK)), 4228 #ifdef CONFIG_TRANSPARENT_HUGEPAGE 4229 K(node_page_state(pgdat, NR_SHMEM_THPS) * HPAGE_PMD_NR), 4230 K(node_page_state(pgdat, NR_SHMEM_PMDMAPPED) 4231 * HPAGE_PMD_NR), 4232 K(node_page_state(pgdat, NR_ANON_THPS) * HPAGE_PMD_NR), 4233 #endif 4234 K(node_page_state(pgdat, NR_SHMEM)), 4235 K(node_page_state(pgdat, NR_WRITEBACK_TEMP)), 4236 K(node_page_state(pgdat, NR_UNSTABLE_NFS)), 4237 node_page_state(pgdat, NR_PAGES_SCANNED), 4238 !pgdat_reclaimable(pgdat) ? "yes" : "no"); 4239 } 4240 4241 for_each_populated_zone(zone) { 4242 int i; 4243 4244 if (skip_free_areas_node(filter, zone_to_nid(zone))) 4245 continue; 4246 4247 free_pcp = 0; 4248 for_each_online_cpu(cpu) 4249 free_pcp += per_cpu_ptr(zone->pageset, cpu)->pcp.count; 4250 4251 show_node(zone); 4252 printk("%s" 4253 " free:%lukB" 4254 " min:%lukB" 4255 " low:%lukB" 4256 " high:%lukB" 4257 " active_anon:%lukB" 4258 " inactive_anon:%lukB" 4259 " active_file:%lukB" 4260 " inactive_file:%lukB" 4261 " unevictable:%lukB" 4262 " writepending:%lukB" 4263 " present:%lukB" 4264 " managed:%lukB" 4265 " mlocked:%lukB" 4266 " slab_reclaimable:%lukB" 4267 " slab_unreclaimable:%lukB" 4268 " kernel_stack:%lukB" 4269 " pagetables:%lukB" 4270 " bounce:%lukB" 4271 " free_pcp:%lukB" 4272 " local_pcp:%ukB" 4273 " free_cma:%lukB" 4274 "\n", 4275 zone->name, 4276 K(zone_page_state(zone, NR_FREE_PAGES)), 4277 K(min_wmark_pages(zone)), 4278 K(low_wmark_pages(zone)), 4279 K(high_wmark_pages(zone)), 4280 K(zone_page_state(zone, NR_ZONE_ACTIVE_ANON)), 4281 K(zone_page_state(zone, NR_ZONE_INACTIVE_ANON)), 4282 K(zone_page_state(zone, NR_ZONE_ACTIVE_FILE)), 4283 K(zone_page_state(zone, NR_ZONE_INACTIVE_FILE)), 4284 K(zone_page_state(zone, NR_ZONE_UNEVICTABLE)), 4285 K(zone_page_state(zone, NR_ZONE_WRITE_PENDING)), 4286 K(zone->present_pages), 4287 K(zone->managed_pages), 4288 K(zone_page_state(zone, NR_MLOCK)), 4289 K(zone_page_state(zone, NR_SLAB_RECLAIMABLE)), 4290 K(zone_page_state(zone, NR_SLAB_UNRECLAIMABLE)), 4291 zone_page_state(zone, NR_KERNEL_STACK_KB), 4292 K(zone_page_state(zone, NR_PAGETABLE)), 4293 K(zone_page_state(zone, NR_BOUNCE)), 4294 K(free_pcp), 4295 K(this_cpu_read(zone->pageset->pcp.count)), 4296 K(zone_page_state(zone, NR_FREE_CMA_PAGES))); 4297 printk("lowmem_reserve[]:"); 4298 for (i = 0; i < MAX_NR_ZONES; i++) 4299 printk(" %ld", zone->lowmem_reserve[i]); 4300 printk("\n"); 4301 } 4302 4303 for_each_populated_zone(zone) { 4304 unsigned int order; 4305 unsigned long nr[MAX_ORDER], flags, total = 0; 4306 unsigned char types[MAX_ORDER]; 4307 4308 if (skip_free_areas_node(filter, zone_to_nid(zone))) 4309 continue; 4310 show_node(zone); 4311 printk("%s: ", zone->name); 4312 4313 spin_lock_irqsave(&zone->lock, flags); 4314 for (order = 0; order < MAX_ORDER; order++) { 4315 struct free_area *area = &zone->free_area[order]; 4316 int type; 4317 4318 nr[order] = area->nr_free; 4319 total += nr[order] << order; 4320 4321 types[order] = 0; 4322 for (type = 0; type < MIGRATE_TYPES; type++) { 4323 if (!list_empty(&area->free_list[type])) 4324 types[order] |= 1 << type; 4325 } 4326 } 4327 spin_unlock_irqrestore(&zone->lock, flags); 4328 for (order = 0; order < MAX_ORDER; order++) { 4329 printk("%lu*%lukB ", nr[order], K(1UL) << order); 4330 if (nr[order]) 4331 show_migration_types(types[order]); 4332 } 4333 printk("= %lukB\n", K(total)); 4334 } 4335 4336 hugetlb_show_meminfo(); 4337 4338 printk("%ld total pagecache pages\n", global_node_page_state(NR_FILE_PAGES)); 4339 4340 show_swap_cache_info(); 4341 } 4342 4343 static void zoneref_set_zone(struct zone *zone, struct zoneref *zoneref) 4344 { 4345 zoneref->zone = zone; 4346 zoneref->zone_idx = zone_idx(zone); 4347 } 4348 4349 /* 4350 * Builds allocation fallback zone lists. 4351 * 4352 * Add all populated zones of a node to the zonelist. 4353 */ 4354 static int build_zonelists_node(pg_data_t *pgdat, struct zonelist *zonelist, 4355 int nr_zones) 4356 { 4357 struct zone *zone; 4358 enum zone_type zone_type = MAX_NR_ZONES; 4359 4360 do { 4361 zone_type--; 4362 zone = pgdat->node_zones + zone_type; 4363 if (managed_zone(zone)) { 4364 zoneref_set_zone(zone, 4365 &zonelist->_zonerefs[nr_zones++]); 4366 check_highest_zone(zone_type); 4367 } 4368 } while (zone_type); 4369 4370 return nr_zones; 4371 } 4372 4373 4374 /* 4375 * zonelist_order: 4376 * 0 = automatic detection of better ordering. 4377 * 1 = order by ([node] distance, -zonetype) 4378 * 2 = order by (-zonetype, [node] distance) 4379 * 4380 * If not NUMA, ZONELIST_ORDER_ZONE and ZONELIST_ORDER_NODE will create 4381 * the same zonelist. So only NUMA can configure this param. 4382 */ 4383 #define ZONELIST_ORDER_DEFAULT 0 4384 #define ZONELIST_ORDER_NODE 1 4385 #define ZONELIST_ORDER_ZONE 2 4386 4387 /* zonelist order in the kernel. 4388 * set_zonelist_order() will set this to NODE or ZONE. 4389 */ 4390 static int current_zonelist_order = ZONELIST_ORDER_DEFAULT; 4391 static char zonelist_order_name[3][8] = {"Default", "Node", "Zone"}; 4392 4393 4394 #ifdef CONFIG_NUMA 4395 /* The value user specified ....changed by config */ 4396 static int user_zonelist_order = ZONELIST_ORDER_DEFAULT; 4397 /* string for sysctl */ 4398 #define NUMA_ZONELIST_ORDER_LEN 16 4399 char numa_zonelist_order[16] = "default"; 4400 4401 /* 4402 * interface for configure zonelist ordering. 4403 * command line option "numa_zonelist_order" 4404 * = "[dD]efault - default, automatic configuration. 4405 * = "[nN]ode - order by node locality, then by zone within node 4406 * = "[zZ]one - order by zone, then by locality within zone 4407 */ 4408 4409 static int __parse_numa_zonelist_order(char *s) 4410 { 4411 if (*s == 'd' || *s == 'D') { 4412 user_zonelist_order = ZONELIST_ORDER_DEFAULT; 4413 } else if (*s == 'n' || *s == 'N') { 4414 user_zonelist_order = ZONELIST_ORDER_NODE; 4415 } else if (*s == 'z' || *s == 'Z') { 4416 user_zonelist_order = ZONELIST_ORDER_ZONE; 4417 } else { 4418 pr_warn("Ignoring invalid numa_zonelist_order value: %s\n", s); 4419 return -EINVAL; 4420 } 4421 return 0; 4422 } 4423 4424 static __init int setup_numa_zonelist_order(char *s) 4425 { 4426 int ret; 4427 4428 if (!s) 4429 return 0; 4430 4431 ret = __parse_numa_zonelist_order(s); 4432 if (ret == 0) 4433 strlcpy(numa_zonelist_order, s, NUMA_ZONELIST_ORDER_LEN); 4434 4435 return ret; 4436 } 4437 early_param("numa_zonelist_order", setup_numa_zonelist_order); 4438 4439 /* 4440 * sysctl handler for numa_zonelist_order 4441 */ 4442 int numa_zonelist_order_handler(struct ctl_table *table, int write, 4443 void __user *buffer, size_t *length, 4444 loff_t *ppos) 4445 { 4446 char saved_string[NUMA_ZONELIST_ORDER_LEN]; 4447 int ret; 4448 static DEFINE_MUTEX(zl_order_mutex); 4449 4450 mutex_lock(&zl_order_mutex); 4451 if (write) { 4452 if (strlen((char *)table->data) >= NUMA_ZONELIST_ORDER_LEN) { 4453 ret = -EINVAL; 4454 goto out; 4455 } 4456 strcpy(saved_string, (char *)table->data); 4457 } 4458 ret = proc_dostring(table, write, buffer, length, ppos); 4459 if (ret) 4460 goto out; 4461 if (write) { 4462 int oldval = user_zonelist_order; 4463 4464 ret = __parse_numa_zonelist_order((char *)table->data); 4465 if (ret) { 4466 /* 4467 * bogus value. restore saved string 4468 */ 4469 strncpy((char *)table->data, saved_string, 4470 NUMA_ZONELIST_ORDER_LEN); 4471 user_zonelist_order = oldval; 4472 } else if (oldval != user_zonelist_order) { 4473 mutex_lock(&zonelists_mutex); 4474 build_all_zonelists(NULL, NULL); 4475 mutex_unlock(&zonelists_mutex); 4476 } 4477 } 4478 out: 4479 mutex_unlock(&zl_order_mutex); 4480 return ret; 4481 } 4482 4483 4484 #define MAX_NODE_LOAD (nr_online_nodes) 4485 static int node_load[MAX_NUMNODES]; 4486 4487 /** 4488 * find_next_best_node - find the next node that should appear in a given node's fallback list 4489 * @node: node whose fallback list we're appending 4490 * @used_node_mask: nodemask_t of already used nodes 4491 * 4492 * We use a number of factors to determine which is the next node that should 4493 * appear on a given node's fallback list. The node should not have appeared 4494 * already in @node's fallback list, and it should be the next closest node 4495 * according to the distance array (which contains arbitrary distance values 4496 * from each node to each node in the system), and should also prefer nodes 4497 * with no CPUs, since presumably they'll have very little allocation pressure 4498 * on them otherwise. 4499 * It returns -1 if no node is found. 4500 */ 4501 static int find_next_best_node(int node, nodemask_t *used_node_mask) 4502 { 4503 int n, val; 4504 int min_val = INT_MAX; 4505 int best_node = NUMA_NO_NODE; 4506 const struct cpumask *tmp = cpumask_of_node(0); 4507 4508 /* Use the local node if we haven't already */ 4509 if (!node_isset(node, *used_node_mask)) { 4510 node_set(node, *used_node_mask); 4511 return node; 4512 } 4513 4514 for_each_node_state(n, N_MEMORY) { 4515 4516 /* Don't want a node to appear more than once */ 4517 if (node_isset(n, *used_node_mask)) 4518 continue; 4519 4520 /* Use the distance array to find the distance */ 4521 val = node_distance(node, n); 4522 4523 /* Penalize nodes under us ("prefer the next node") */ 4524 val += (n < node); 4525 4526 /* Give preference to headless and unused nodes */ 4527 tmp = cpumask_of_node(n); 4528 if (!cpumask_empty(tmp)) 4529 val += PENALTY_FOR_NODE_WITH_CPUS; 4530 4531 /* Slight preference for less loaded node */ 4532 val *= (MAX_NODE_LOAD*MAX_NUMNODES); 4533 val += node_load[n]; 4534 4535 if (val < min_val) { 4536 min_val = val; 4537 best_node = n; 4538 } 4539 } 4540 4541 if (best_node >= 0) 4542 node_set(best_node, *used_node_mask); 4543 4544 return best_node; 4545 } 4546 4547 4548 /* 4549 * Build zonelists ordered by node and zones within node. 4550 * This results in maximum locality--normal zone overflows into local 4551 * DMA zone, if any--but risks exhausting DMA zone. 4552 */ 4553 static void build_zonelists_in_node_order(pg_data_t *pgdat, int node) 4554 { 4555 int j; 4556 struct zonelist *zonelist; 4557 4558 zonelist = &pgdat->node_zonelists[0]; 4559 for (j = 0; zonelist->_zonerefs[j].zone != NULL; j++) 4560 ; 4561 j = build_zonelists_node(NODE_DATA(node), zonelist, j); 4562 zonelist->_zonerefs[j].zone = NULL; 4563 zonelist->_zonerefs[j].zone_idx = 0; 4564 } 4565 4566 /* 4567 * Build gfp_thisnode zonelists 4568 */ 4569 static void build_thisnode_zonelists(pg_data_t *pgdat) 4570 { 4571 int j; 4572 struct zonelist *zonelist; 4573 4574 zonelist = &pgdat->node_zonelists[1]; 4575 j = build_zonelists_node(pgdat, zonelist, 0); 4576 zonelist->_zonerefs[j].zone = NULL; 4577 zonelist->_zonerefs[j].zone_idx = 0; 4578 } 4579 4580 /* 4581 * Build zonelists ordered by zone and nodes within zones. 4582 * This results in conserving DMA zone[s] until all Normal memory is 4583 * exhausted, but results in overflowing to remote node while memory 4584 * may still exist in local DMA zone. 4585 */ 4586 static int node_order[MAX_NUMNODES]; 4587 4588 static void build_zonelists_in_zone_order(pg_data_t *pgdat, int nr_nodes) 4589 { 4590 int pos, j, node; 4591 int zone_type; /* needs to be signed */ 4592 struct zone *z; 4593 struct zonelist *zonelist; 4594 4595 zonelist = &pgdat->node_zonelists[0]; 4596 pos = 0; 4597 for (zone_type = MAX_NR_ZONES - 1; zone_type >= 0; zone_type--) { 4598 for (j = 0; j < nr_nodes; j++) { 4599 node = node_order[j]; 4600 z = &NODE_DATA(node)->node_zones[zone_type]; 4601 if (managed_zone(z)) { 4602 zoneref_set_zone(z, 4603 &zonelist->_zonerefs[pos++]); 4604 check_highest_zone(zone_type); 4605 } 4606 } 4607 } 4608 zonelist->_zonerefs[pos].zone = NULL; 4609 zonelist->_zonerefs[pos].zone_idx = 0; 4610 } 4611 4612 #if defined(CONFIG_64BIT) 4613 /* 4614 * Devices that require DMA32/DMA are relatively rare and do not justify a 4615 * penalty to every machine in case the specialised case applies. Default 4616 * to Node-ordering on 64-bit NUMA machines 4617 */ 4618 static int default_zonelist_order(void) 4619 { 4620 return ZONELIST_ORDER_NODE; 4621 } 4622 #else 4623 /* 4624 * On 32-bit, the Normal zone needs to be preserved for allocations accessible 4625 * by the kernel. If processes running on node 0 deplete the low memory zone 4626 * then reclaim will occur more frequency increasing stalls and potentially 4627 * be easier to OOM if a large percentage of the zone is under writeback or 4628 * dirty. The problem is significantly worse if CONFIG_HIGHPTE is not set. 4629 * Hence, default to zone ordering on 32-bit. 4630 */ 4631 static int default_zonelist_order(void) 4632 { 4633 return ZONELIST_ORDER_ZONE; 4634 } 4635 #endif /* CONFIG_64BIT */ 4636 4637 static void set_zonelist_order(void) 4638 { 4639 if (user_zonelist_order == ZONELIST_ORDER_DEFAULT) 4640 current_zonelist_order = default_zonelist_order(); 4641 else 4642 current_zonelist_order = user_zonelist_order; 4643 } 4644 4645 static void build_zonelists(pg_data_t *pgdat) 4646 { 4647 int i, node, load; 4648 nodemask_t used_mask; 4649 int local_node, prev_node; 4650 struct zonelist *zonelist; 4651 unsigned int order = current_zonelist_order; 4652 4653 /* initialize zonelists */ 4654 for (i = 0; i < MAX_ZONELISTS; i++) { 4655 zonelist = pgdat->node_zonelists + i; 4656 zonelist->_zonerefs[0].zone = NULL; 4657 zonelist->_zonerefs[0].zone_idx = 0; 4658 } 4659 4660 /* NUMA-aware ordering of nodes */ 4661 local_node = pgdat->node_id; 4662 load = nr_online_nodes; 4663 prev_node = local_node; 4664 nodes_clear(used_mask); 4665 4666 memset(node_order, 0, sizeof(node_order)); 4667 i = 0; 4668 4669 while ((node = find_next_best_node(local_node, &used_mask)) >= 0) { 4670 /* 4671 * We don't want to pressure a particular node. 4672 * So adding penalty to the first node in same 4673 * distance group to make it round-robin. 4674 */ 4675 if (node_distance(local_node, node) != 4676 node_distance(local_node, prev_node)) 4677 node_load[node] = load; 4678 4679 prev_node = node; 4680 load--; 4681 if (order == ZONELIST_ORDER_NODE) 4682 build_zonelists_in_node_order(pgdat, node); 4683 else 4684 node_order[i++] = node; /* remember order */ 4685 } 4686 4687 if (order == ZONELIST_ORDER_ZONE) { 4688 /* calculate node order -- i.e., DMA last! */ 4689 build_zonelists_in_zone_order(pgdat, i); 4690 } 4691 4692 build_thisnode_zonelists(pgdat); 4693 } 4694 4695 #ifdef CONFIG_HAVE_MEMORYLESS_NODES 4696 /* 4697 * Return node id of node used for "local" allocations. 4698 * I.e., first node id of first zone in arg node's generic zonelist. 4699 * Used for initializing percpu 'numa_mem', which is used primarily 4700 * for kernel allocations, so use GFP_KERNEL flags to locate zonelist. 4701 */ 4702 int local_memory_node(int node) 4703 { 4704 struct zoneref *z; 4705 4706 z = first_zones_zonelist(node_zonelist(node, GFP_KERNEL), 4707 gfp_zone(GFP_KERNEL), 4708 NULL); 4709 return z->zone->node; 4710 } 4711 #endif 4712 4713 static void setup_min_unmapped_ratio(void); 4714 static void setup_min_slab_ratio(void); 4715 #else /* CONFIG_NUMA */ 4716 4717 static void set_zonelist_order(void) 4718 { 4719 current_zonelist_order = ZONELIST_ORDER_ZONE; 4720 } 4721 4722 static void build_zonelists(pg_data_t *pgdat) 4723 { 4724 int node, local_node; 4725 enum zone_type j; 4726 struct zonelist *zonelist; 4727 4728 local_node = pgdat->node_id; 4729 4730 zonelist = &pgdat->node_zonelists[0]; 4731 j = build_zonelists_node(pgdat, zonelist, 0); 4732 4733 /* 4734 * Now we build the zonelist so that it contains the zones 4735 * of all the other nodes. 4736 * We don't want to pressure a particular node, so when 4737 * building the zones for node N, we make sure that the 4738 * zones coming right after the local ones are those from 4739 * node N+1 (modulo N) 4740 */ 4741 for (node = local_node + 1; node < MAX_NUMNODES; node++) { 4742 if (!node_online(node)) 4743 continue; 4744 j = build_zonelists_node(NODE_DATA(node), zonelist, j); 4745 } 4746 for (node = 0; node < local_node; node++) { 4747 if (!node_online(node)) 4748 continue; 4749 j = build_zonelists_node(NODE_DATA(node), zonelist, j); 4750 } 4751 4752 zonelist->_zonerefs[j].zone = NULL; 4753 zonelist->_zonerefs[j].zone_idx = 0; 4754 } 4755 4756 #endif /* CONFIG_NUMA */ 4757 4758 /* 4759 * Boot pageset table. One per cpu which is going to be used for all 4760 * zones and all nodes. The parameters will be set in such a way 4761 * that an item put on a list will immediately be handed over to 4762 * the buddy list. This is safe since pageset manipulation is done 4763 * with interrupts disabled. 4764 * 4765 * The boot_pagesets must be kept even after bootup is complete for 4766 * unused processors and/or zones. They do play a role for bootstrapping 4767 * hotplugged processors. 4768 * 4769 * zoneinfo_show() and maybe other functions do 4770 * not check if the processor is online before following the pageset pointer. 4771 * Other parts of the kernel may not check if the zone is available. 4772 */ 4773 static void setup_pageset(struct per_cpu_pageset *p, unsigned long batch); 4774 static DEFINE_PER_CPU(struct per_cpu_pageset, boot_pageset); 4775 static void setup_zone_pageset(struct zone *zone); 4776 4777 /* 4778 * Global mutex to protect against size modification of zonelists 4779 * as well as to serialize pageset setup for the new populated zone. 4780 */ 4781 DEFINE_MUTEX(zonelists_mutex); 4782 4783 /* return values int ....just for stop_machine() */ 4784 static int __build_all_zonelists(void *data) 4785 { 4786 int nid; 4787 int cpu; 4788 pg_data_t *self = data; 4789 4790 #ifdef CONFIG_NUMA 4791 memset(node_load, 0, sizeof(node_load)); 4792 #endif 4793 4794 if (self && !node_online(self->node_id)) { 4795 build_zonelists(self); 4796 } 4797 4798 for_each_online_node(nid) { 4799 pg_data_t *pgdat = NODE_DATA(nid); 4800 4801 build_zonelists(pgdat); 4802 } 4803 4804 /* 4805 * Initialize the boot_pagesets that are going to be used 4806 * for bootstrapping processors. The real pagesets for 4807 * each zone will be allocated later when the per cpu 4808 * allocator is available. 4809 * 4810 * boot_pagesets are used also for bootstrapping offline 4811 * cpus if the system is already booted because the pagesets 4812 * are needed to initialize allocators on a specific cpu too. 4813 * F.e. the percpu allocator needs the page allocator which 4814 * needs the percpu allocator in order to allocate its pagesets 4815 * (a chicken-egg dilemma). 4816 */ 4817 for_each_possible_cpu(cpu) { 4818 setup_pageset(&per_cpu(boot_pageset, cpu), 0); 4819 4820 #ifdef CONFIG_HAVE_MEMORYLESS_NODES 4821 /* 4822 * We now know the "local memory node" for each node-- 4823 * i.e., the node of the first zone in the generic zonelist. 4824 * Set up numa_mem percpu variable for on-line cpus. During 4825 * boot, only the boot cpu should be on-line; we'll init the 4826 * secondary cpus' numa_mem as they come on-line. During 4827 * node/memory hotplug, we'll fixup all on-line cpus. 4828 */ 4829 if (cpu_online(cpu)) 4830 set_cpu_numa_mem(cpu, local_memory_node(cpu_to_node(cpu))); 4831 #endif 4832 } 4833 4834 return 0; 4835 } 4836 4837 static noinline void __init 4838 build_all_zonelists_init(void) 4839 { 4840 __build_all_zonelists(NULL); 4841 mminit_verify_zonelist(); 4842 cpuset_init_current_mems_allowed(); 4843 } 4844 4845 /* 4846 * Called with zonelists_mutex held always 4847 * unless system_state == SYSTEM_BOOTING. 4848 * 4849 * __ref due to (1) call of __meminit annotated setup_zone_pageset 4850 * [we're only called with non-NULL zone through __meminit paths] and 4851 * (2) call of __init annotated helper build_all_zonelists_init 4852 * [protected by SYSTEM_BOOTING]. 4853 */ 4854 void __ref build_all_zonelists(pg_data_t *pgdat, struct zone *zone) 4855 { 4856 set_zonelist_order(); 4857 4858 if (system_state == SYSTEM_BOOTING) { 4859 build_all_zonelists_init(); 4860 } else { 4861 #ifdef CONFIG_MEMORY_HOTPLUG 4862 if (zone) 4863 setup_zone_pageset(zone); 4864 #endif 4865 /* we have to stop all cpus to guarantee there is no user 4866 of zonelist */ 4867 stop_machine(__build_all_zonelists, pgdat, NULL); 4868 /* cpuset refresh routine should be here */ 4869 } 4870 vm_total_pages = nr_free_pagecache_pages(); 4871 /* 4872 * Disable grouping by mobility if the number of pages in the 4873 * system is too low to allow the mechanism to work. It would be 4874 * more accurate, but expensive to check per-zone. This check is 4875 * made on memory-hotadd so a system can start with mobility 4876 * disabled and enable it later 4877 */ 4878 if (vm_total_pages < (pageblock_nr_pages * MIGRATE_TYPES)) 4879 page_group_by_mobility_disabled = 1; 4880 else 4881 page_group_by_mobility_disabled = 0; 4882 4883 pr_info("Built %i zonelists in %s order, mobility grouping %s. Total pages: %ld\n", 4884 nr_online_nodes, 4885 zonelist_order_name[current_zonelist_order], 4886 page_group_by_mobility_disabled ? "off" : "on", 4887 vm_total_pages); 4888 #ifdef CONFIG_NUMA 4889 pr_info("Policy zone: %s\n", zone_names[policy_zone]); 4890 #endif 4891 } 4892 4893 /* 4894 * Helper functions to size the waitqueue hash table. 4895 * Essentially these want to choose hash table sizes sufficiently 4896 * large so that collisions trying to wait on pages are rare. 4897 * But in fact, the number of active page waitqueues on typical 4898 * systems is ridiculously low, less than 200. So this is even 4899 * conservative, even though it seems large. 4900 * 4901 * The constant PAGES_PER_WAITQUEUE specifies the ratio of pages to 4902 * waitqueues, i.e. the size of the waitq table given the number of pages. 4903 */ 4904 #define PAGES_PER_WAITQUEUE 256 4905 4906 #ifndef CONFIG_MEMORY_HOTPLUG 4907 static inline unsigned long wait_table_hash_nr_entries(unsigned long pages) 4908 { 4909 unsigned long size = 1; 4910 4911 pages /= PAGES_PER_WAITQUEUE; 4912 4913 while (size < pages) 4914 size <<= 1; 4915 4916 /* 4917 * Once we have dozens or even hundreds of threads sleeping 4918 * on IO we've got bigger problems than wait queue collision. 4919 * Limit the size of the wait table to a reasonable size. 4920 */ 4921 size = min(size, 4096UL); 4922 4923 return max(size, 4UL); 4924 } 4925 #else 4926 /* 4927 * A zone's size might be changed by hot-add, so it is not possible to determine 4928 * a suitable size for its wait_table. So we use the maximum size now. 4929 * 4930 * The max wait table size = 4096 x sizeof(wait_queue_head_t). ie: 4931 * 4932 * i386 (preemption config) : 4096 x 16 = 64Kbyte. 4933 * ia64, x86-64 (no preemption): 4096 x 20 = 80Kbyte. 4934 * ia64, x86-64 (preemption) : 4096 x 24 = 96Kbyte. 4935 * 4936 * The maximum entries are prepared when a zone's memory is (512K + 256) pages 4937 * or more by the traditional way. (See above). It equals: 4938 * 4939 * i386, x86-64, powerpc(4K page size) : = ( 2G + 1M)byte. 4940 * ia64(16K page size) : = ( 8G + 4M)byte. 4941 * powerpc (64K page size) : = (32G +16M)byte. 4942 */ 4943 static inline unsigned long wait_table_hash_nr_entries(unsigned long pages) 4944 { 4945 return 4096UL; 4946 } 4947 #endif 4948 4949 /* 4950 * This is an integer logarithm so that shifts can be used later 4951 * to extract the more random high bits from the multiplicative 4952 * hash function before the remainder is taken. 4953 */ 4954 static inline unsigned long wait_table_bits(unsigned long size) 4955 { 4956 return ffz(~size); 4957 } 4958 4959 /* 4960 * Initially all pages are reserved - free ones are freed 4961 * up by free_all_bootmem() once the early boot process is 4962 * done. Non-atomic initialization, single-pass. 4963 */ 4964 void __meminit memmap_init_zone(unsigned long size, int nid, unsigned long zone, 4965 unsigned long start_pfn, enum memmap_context context) 4966 { 4967 struct vmem_altmap *altmap = to_vmem_altmap(__pfn_to_phys(start_pfn)); 4968 unsigned long end_pfn = start_pfn + size; 4969 pg_data_t *pgdat = NODE_DATA(nid); 4970 unsigned long pfn; 4971 unsigned long nr_initialised = 0; 4972 #ifdef CONFIG_HAVE_MEMBLOCK_NODE_MAP 4973 struct memblock_region *r = NULL, *tmp; 4974 #endif 4975 4976 if (highest_memmap_pfn < end_pfn - 1) 4977 highest_memmap_pfn = end_pfn - 1; 4978 4979 /* 4980 * Honor reservation requested by the driver for this ZONE_DEVICE 4981 * memory 4982 */ 4983 if (altmap && start_pfn == altmap->base_pfn) 4984 start_pfn += altmap->reserve; 4985 4986 for (pfn = start_pfn; pfn < end_pfn; pfn++) { 4987 /* 4988 * There can be holes in boot-time mem_map[]s handed to this 4989 * function. They do not exist on hotplugged memory. 4990 */ 4991 if (context != MEMMAP_EARLY) 4992 goto not_early; 4993 4994 if (!early_pfn_valid(pfn)) 4995 continue; 4996 if (!early_pfn_in_nid(pfn, nid)) 4997 continue; 4998 if (!update_defer_init(pgdat, pfn, end_pfn, &nr_initialised)) 4999 break; 5000 5001 #ifdef CONFIG_HAVE_MEMBLOCK_NODE_MAP 5002 /* 5003 * If not mirrored_kernelcore and ZONE_MOVABLE exists, range 5004 * from zone_movable_pfn[nid] to end of each node should be 5005 * ZONE_MOVABLE not ZONE_NORMAL. skip it. 5006 */ 5007 if (!mirrored_kernelcore && zone_movable_pfn[nid]) 5008 if (zone == ZONE_NORMAL && pfn >= zone_movable_pfn[nid]) 5009 continue; 5010 5011 /* 5012 * Check given memblock attribute by firmware which can affect 5013 * kernel memory layout. If zone==ZONE_MOVABLE but memory is 5014 * mirrored, it's an overlapped memmap init. skip it. 5015 */ 5016 if (mirrored_kernelcore && zone == ZONE_MOVABLE) { 5017 if (!r || pfn >= memblock_region_memory_end_pfn(r)) { 5018 for_each_memblock(memory, tmp) 5019 if (pfn < memblock_region_memory_end_pfn(tmp)) 5020 break; 5021 r = tmp; 5022 } 5023 if (pfn >= memblock_region_memory_base_pfn(r) && 5024 memblock_is_mirror(r)) { 5025 /* already initialized as NORMAL */ 5026 pfn = memblock_region_memory_end_pfn(r); 5027 continue; 5028 } 5029 } 5030 #endif 5031 5032 not_early: 5033 /* 5034 * Mark the block movable so that blocks are reserved for 5035 * movable at startup. This will force kernel allocations 5036 * to reserve their blocks rather than leaking throughout 5037 * the address space during boot when many long-lived 5038 * kernel allocations are made. 5039 * 5040 * bitmap is created for zone's valid pfn range. but memmap 5041 * can be created for invalid pages (for alignment) 5042 * check here not to call set_pageblock_migratetype() against 5043 * pfn out of zone. 5044 */ 5045 if (!(pfn & (pageblock_nr_pages - 1))) { 5046 struct page *page = pfn_to_page(pfn); 5047 5048 __init_single_page(page, pfn, zone, nid); 5049 set_pageblock_migratetype(page, MIGRATE_MOVABLE); 5050 } else { 5051 __init_single_pfn(pfn, zone, nid); 5052 } 5053 } 5054 } 5055 5056 static void __meminit zone_init_free_lists(struct zone *zone) 5057 { 5058 unsigned int order, t; 5059 for_each_migratetype_order(order, t) { 5060 INIT_LIST_HEAD(&zone->free_area[order].free_list[t]); 5061 zone->free_area[order].nr_free = 0; 5062 } 5063 } 5064 5065 #ifndef __HAVE_ARCH_MEMMAP_INIT 5066 #define memmap_init(size, nid, zone, start_pfn) \ 5067 memmap_init_zone((size), (nid), (zone), (start_pfn), MEMMAP_EARLY) 5068 #endif 5069 5070 static int zone_batchsize(struct zone *zone) 5071 { 5072 #ifdef CONFIG_MMU 5073 int batch; 5074 5075 /* 5076 * The per-cpu-pages pools are set to around 1000th of the 5077 * size of the zone. But no more than 1/2 of a meg. 5078 * 5079 * OK, so we don't know how big the cache is. So guess. 5080 */ 5081 batch = zone->managed_pages / 1024; 5082 if (batch * PAGE_SIZE > 512 * 1024) 5083 batch = (512 * 1024) / PAGE_SIZE; 5084 batch /= 4; /* We effectively *= 4 below */ 5085 if (batch < 1) 5086 batch = 1; 5087 5088 /* 5089 * Clamp the batch to a 2^n - 1 value. Having a power 5090 * of 2 value was found to be more likely to have 5091 * suboptimal cache aliasing properties in some cases. 5092 * 5093 * For example if 2 tasks are alternately allocating 5094 * batches of pages, one task can end up with a lot 5095 * of pages of one half of the possible page colors 5096 * and the other with pages of the other colors. 5097 */ 5098 batch = rounddown_pow_of_two(batch + batch/2) - 1; 5099 5100 return batch; 5101 5102 #else 5103 /* The deferral and batching of frees should be suppressed under NOMMU 5104 * conditions. 5105 * 5106 * The problem is that NOMMU needs to be able to allocate large chunks 5107 * of contiguous memory as there's no hardware page translation to 5108 * assemble apparent contiguous memory from discontiguous pages. 5109 * 5110 * Queueing large contiguous runs of pages for batching, however, 5111 * causes the pages to actually be freed in smaller chunks. As there 5112 * can be a significant delay between the individual batches being 5113 * recycled, this leads to the once large chunks of space being 5114 * fragmented and becoming unavailable for high-order allocations. 5115 */ 5116 return 0; 5117 #endif 5118 } 5119 5120 /* 5121 * pcp->high and pcp->batch values are related and dependent on one another: 5122 * ->batch must never be higher then ->high. 5123 * The following function updates them in a safe manner without read side 5124 * locking. 5125 * 5126 * Any new users of pcp->batch and pcp->high should ensure they can cope with 5127 * those fields changing asynchronously (acording the the above rule). 5128 * 5129 * mutex_is_locked(&pcp_batch_high_lock) required when calling this function 5130 * outside of boot time (or some other assurance that no concurrent updaters 5131 * exist). 5132 */ 5133 static void pageset_update(struct per_cpu_pages *pcp, unsigned long high, 5134 unsigned long batch) 5135 { 5136 /* start with a fail safe value for batch */ 5137 pcp->batch = 1; 5138 smp_wmb(); 5139 5140 /* Update high, then batch, in order */ 5141 pcp->high = high; 5142 smp_wmb(); 5143 5144 pcp->batch = batch; 5145 } 5146 5147 /* a companion to pageset_set_high() */ 5148 static void pageset_set_batch(struct per_cpu_pageset *p, unsigned long batch) 5149 { 5150 pageset_update(&p->pcp, 6 * batch, max(1UL, 1 * batch)); 5151 } 5152 5153 static void pageset_init(struct per_cpu_pageset *p) 5154 { 5155 struct per_cpu_pages *pcp; 5156 int migratetype; 5157 5158 memset(p, 0, sizeof(*p)); 5159 5160 pcp = &p->pcp; 5161 pcp->count = 0; 5162 for (migratetype = 0; migratetype < MIGRATE_PCPTYPES; migratetype++) 5163 INIT_LIST_HEAD(&pcp->lists[migratetype]); 5164 } 5165 5166 static void setup_pageset(struct per_cpu_pageset *p, unsigned long batch) 5167 { 5168 pageset_init(p); 5169 pageset_set_batch(p, batch); 5170 } 5171 5172 /* 5173 * pageset_set_high() sets the high water mark for hot per_cpu_pagelist 5174 * to the value high for the pageset p. 5175 */ 5176 static void pageset_set_high(struct per_cpu_pageset *p, 5177 unsigned long high) 5178 { 5179 unsigned long batch = max(1UL, high / 4); 5180 if ((high / 4) > (PAGE_SHIFT * 8)) 5181 batch = PAGE_SHIFT * 8; 5182 5183 pageset_update(&p->pcp, high, batch); 5184 } 5185 5186 static void pageset_set_high_and_batch(struct zone *zone, 5187 struct per_cpu_pageset *pcp) 5188 { 5189 if (percpu_pagelist_fraction) 5190 pageset_set_high(pcp, 5191 (zone->managed_pages / 5192 percpu_pagelist_fraction)); 5193 else 5194 pageset_set_batch(pcp, zone_batchsize(zone)); 5195 } 5196 5197 static void __meminit zone_pageset_init(struct zone *zone, int cpu) 5198 { 5199 struct per_cpu_pageset *pcp = per_cpu_ptr(zone->pageset, cpu); 5200 5201 pageset_init(pcp); 5202 pageset_set_high_and_batch(zone, pcp); 5203 } 5204 5205 static void __meminit setup_zone_pageset(struct zone *zone) 5206 { 5207 int cpu; 5208 zone->pageset = alloc_percpu(struct per_cpu_pageset); 5209 for_each_possible_cpu(cpu) 5210 zone_pageset_init(zone, cpu); 5211 } 5212 5213 /* 5214 * Allocate per cpu pagesets and initialize them. 5215 * Before this call only boot pagesets were available. 5216 */ 5217 void __init setup_per_cpu_pageset(void) 5218 { 5219 struct pglist_data *pgdat; 5220 struct zone *zone; 5221 5222 for_each_populated_zone(zone) 5223 setup_zone_pageset(zone); 5224 5225 for_each_online_pgdat(pgdat) 5226 pgdat->per_cpu_nodestats = 5227 alloc_percpu(struct per_cpu_nodestat); 5228 } 5229 5230 static noinline __ref 5231 int zone_wait_table_init(struct zone *zone, unsigned long zone_size_pages) 5232 { 5233 int i; 5234 size_t alloc_size; 5235 5236 /* 5237 * The per-page waitqueue mechanism uses hashed waitqueues 5238 * per zone. 5239 */ 5240 zone->wait_table_hash_nr_entries = 5241 wait_table_hash_nr_entries(zone_size_pages); 5242 zone->wait_table_bits = 5243 wait_table_bits(zone->wait_table_hash_nr_entries); 5244 alloc_size = zone->wait_table_hash_nr_entries 5245 * sizeof(wait_queue_head_t); 5246 5247 if (!slab_is_available()) { 5248 zone->wait_table = (wait_queue_head_t *) 5249 memblock_virt_alloc_node_nopanic( 5250 alloc_size, zone->zone_pgdat->node_id); 5251 } else { 5252 /* 5253 * This case means that a zone whose size was 0 gets new memory 5254 * via memory hot-add. 5255 * But it may be the case that a new node was hot-added. In 5256 * this case vmalloc() will not be able to use this new node's 5257 * memory - this wait_table must be initialized to use this new 5258 * node itself as well. 5259 * To use this new node's memory, further consideration will be 5260 * necessary. 5261 */ 5262 zone->wait_table = vmalloc(alloc_size); 5263 } 5264 if (!zone->wait_table) 5265 return -ENOMEM; 5266 5267 for (i = 0; i < zone->wait_table_hash_nr_entries; ++i) 5268 init_waitqueue_head(zone->wait_table + i); 5269 5270 return 0; 5271 } 5272 5273 static __meminit void zone_pcp_init(struct zone *zone) 5274 { 5275 /* 5276 * per cpu subsystem is not up at this point. The following code 5277 * relies on the ability of the linker to provide the 5278 * offset of a (static) per cpu variable into the per cpu area. 5279 */ 5280 zone->pageset = &boot_pageset; 5281 5282 if (populated_zone(zone)) 5283 printk(KERN_DEBUG " %s zone: %lu pages, LIFO batch:%u\n", 5284 zone->name, zone->present_pages, 5285 zone_batchsize(zone)); 5286 } 5287 5288 int __meminit init_currently_empty_zone(struct zone *zone, 5289 unsigned long zone_start_pfn, 5290 unsigned long size) 5291 { 5292 struct pglist_data *pgdat = zone->zone_pgdat; 5293 int ret; 5294 ret = zone_wait_table_init(zone, size); 5295 if (ret) 5296 return ret; 5297 pgdat->nr_zones = zone_idx(zone) + 1; 5298 5299 zone->zone_start_pfn = zone_start_pfn; 5300 5301 mminit_dprintk(MMINIT_TRACE, "memmap_init", 5302 "Initialising map node %d zone %lu pfns %lu -> %lu\n", 5303 pgdat->node_id, 5304 (unsigned long)zone_idx(zone), 5305 zone_start_pfn, (zone_start_pfn + size)); 5306 5307 zone_init_free_lists(zone); 5308 5309 return 0; 5310 } 5311 5312 #ifdef CONFIG_HAVE_MEMBLOCK_NODE_MAP 5313 #ifndef CONFIG_HAVE_ARCH_EARLY_PFN_TO_NID 5314 5315 /* 5316 * Required by SPARSEMEM. Given a PFN, return what node the PFN is on. 5317 */ 5318 int __meminit __early_pfn_to_nid(unsigned long pfn, 5319 struct mminit_pfnnid_cache *state) 5320 { 5321 unsigned long start_pfn, end_pfn; 5322 int nid; 5323 5324 if (state->last_start <= pfn && pfn < state->last_end) 5325 return state->last_nid; 5326 5327 nid = memblock_search_pfn_nid(pfn, &start_pfn, &end_pfn); 5328 if (nid != -1) { 5329 state->last_start = start_pfn; 5330 state->last_end = end_pfn; 5331 state->last_nid = nid; 5332 } 5333 5334 return nid; 5335 } 5336 #endif /* CONFIG_HAVE_ARCH_EARLY_PFN_TO_NID */ 5337 5338 /** 5339 * free_bootmem_with_active_regions - Call memblock_free_early_nid for each active range 5340 * @nid: The node to free memory on. If MAX_NUMNODES, all nodes are freed. 5341 * @max_low_pfn: The highest PFN that will be passed to memblock_free_early_nid 5342 * 5343 * If an architecture guarantees that all ranges registered contain no holes 5344 * and may be freed, this this function may be used instead of calling 5345 * memblock_free_early_nid() manually. 5346 */ 5347 void __init free_bootmem_with_active_regions(int nid, unsigned long max_low_pfn) 5348 { 5349 unsigned long start_pfn, end_pfn; 5350 int i, this_nid; 5351 5352 for_each_mem_pfn_range(i, nid, &start_pfn, &end_pfn, &this_nid) { 5353 start_pfn = min(start_pfn, max_low_pfn); 5354 end_pfn = min(end_pfn, max_low_pfn); 5355 5356 if (start_pfn < end_pfn) 5357 memblock_free_early_nid(PFN_PHYS(start_pfn), 5358 (end_pfn - start_pfn) << PAGE_SHIFT, 5359 this_nid); 5360 } 5361 } 5362 5363 /** 5364 * sparse_memory_present_with_active_regions - Call memory_present for each active range 5365 * @nid: The node to call memory_present for. If MAX_NUMNODES, all nodes will be used. 5366 * 5367 * If an architecture guarantees that all ranges registered contain no holes and may 5368 * be freed, this function may be used instead of calling memory_present() manually. 5369 */ 5370 void __init sparse_memory_present_with_active_regions(int nid) 5371 { 5372 unsigned long start_pfn, end_pfn; 5373 int i, this_nid; 5374 5375 for_each_mem_pfn_range(i, nid, &start_pfn, &end_pfn, &this_nid) 5376 memory_present(this_nid, start_pfn, end_pfn); 5377 } 5378 5379 /** 5380 * get_pfn_range_for_nid - Return the start and end page frames for a node 5381 * @nid: The nid to return the range for. If MAX_NUMNODES, the min and max PFN are returned. 5382 * @start_pfn: Passed by reference. On return, it will have the node start_pfn. 5383 * @end_pfn: Passed by reference. On return, it will have the node end_pfn. 5384 * 5385 * It returns the start and end page frame of a node based on information 5386 * provided by memblock_set_node(). If called for a node 5387 * with no available memory, a warning is printed and the start and end 5388 * PFNs will be 0. 5389 */ 5390 void __meminit get_pfn_range_for_nid(unsigned int nid, 5391 unsigned long *start_pfn, unsigned long *end_pfn) 5392 { 5393 unsigned long this_start_pfn, this_end_pfn; 5394 int i; 5395 5396 *start_pfn = -1UL; 5397 *end_pfn = 0; 5398 5399 for_each_mem_pfn_range(i, nid, &this_start_pfn, &this_end_pfn, NULL) { 5400 *start_pfn = min(*start_pfn, this_start_pfn); 5401 *end_pfn = max(*end_pfn, this_end_pfn); 5402 } 5403 5404 if (*start_pfn == -1UL) 5405 *start_pfn = 0; 5406 } 5407 5408 /* 5409 * This finds a zone that can be used for ZONE_MOVABLE pages. The 5410 * assumption is made that zones within a node are ordered in monotonic 5411 * increasing memory addresses so that the "highest" populated zone is used 5412 */ 5413 static void __init find_usable_zone_for_movable(void) 5414 { 5415 int zone_index; 5416 for (zone_index = MAX_NR_ZONES - 1; zone_index >= 0; zone_index--) { 5417 if (zone_index == ZONE_MOVABLE) 5418 continue; 5419 5420 if (arch_zone_highest_possible_pfn[zone_index] > 5421 arch_zone_lowest_possible_pfn[zone_index]) 5422 break; 5423 } 5424 5425 VM_BUG_ON(zone_index == -1); 5426 movable_zone = zone_index; 5427 } 5428 5429 /* 5430 * The zone ranges provided by the architecture do not include ZONE_MOVABLE 5431 * because it is sized independent of architecture. Unlike the other zones, 5432 * the starting point for ZONE_MOVABLE is not fixed. It may be different 5433 * in each node depending on the size of each node and how evenly kernelcore 5434 * is distributed. This helper function adjusts the zone ranges 5435 * provided by the architecture for a given node by using the end of the 5436 * highest usable zone for ZONE_MOVABLE. This preserves the assumption that 5437 * zones within a node are in order of monotonic increases memory addresses 5438 */ 5439 static void __meminit adjust_zone_range_for_zone_movable(int nid, 5440 unsigned long zone_type, 5441 unsigned long node_start_pfn, 5442 unsigned long node_end_pfn, 5443 unsigned long *zone_start_pfn, 5444 unsigned long *zone_end_pfn) 5445 { 5446 /* Only adjust if ZONE_MOVABLE is on this node */ 5447 if (zone_movable_pfn[nid]) { 5448 /* Size ZONE_MOVABLE */ 5449 if (zone_type == ZONE_MOVABLE) { 5450 *zone_start_pfn = zone_movable_pfn[nid]; 5451 *zone_end_pfn = min(node_end_pfn, 5452 arch_zone_highest_possible_pfn[movable_zone]); 5453 5454 /* Check if this whole range is within ZONE_MOVABLE */ 5455 } else if (*zone_start_pfn >= zone_movable_pfn[nid]) 5456 *zone_start_pfn = *zone_end_pfn; 5457 } 5458 } 5459 5460 /* 5461 * Return the number of pages a zone spans in a node, including holes 5462 * present_pages = zone_spanned_pages_in_node() - zone_absent_pages_in_node() 5463 */ 5464 static unsigned long __meminit zone_spanned_pages_in_node(int nid, 5465 unsigned long zone_type, 5466 unsigned long node_start_pfn, 5467 unsigned long node_end_pfn, 5468 unsigned long *zone_start_pfn, 5469 unsigned long *zone_end_pfn, 5470 unsigned long *ignored) 5471 { 5472 /* When hotadd a new node from cpu_up(), the node should be empty */ 5473 if (!node_start_pfn && !node_end_pfn) 5474 return 0; 5475 5476 /* Get the start and end of the zone */ 5477 *zone_start_pfn = arch_zone_lowest_possible_pfn[zone_type]; 5478 *zone_end_pfn = arch_zone_highest_possible_pfn[zone_type]; 5479 adjust_zone_range_for_zone_movable(nid, zone_type, 5480 node_start_pfn, node_end_pfn, 5481 zone_start_pfn, zone_end_pfn); 5482 5483 /* Check that this node has pages within the zone's required range */ 5484 if (*zone_end_pfn < node_start_pfn || *zone_start_pfn > node_end_pfn) 5485 return 0; 5486 5487 /* Move the zone boundaries inside the node if necessary */ 5488 *zone_end_pfn = min(*zone_end_pfn, node_end_pfn); 5489 *zone_start_pfn = max(*zone_start_pfn, node_start_pfn); 5490 5491 /* Return the spanned pages */ 5492 return *zone_end_pfn - *zone_start_pfn; 5493 } 5494 5495 /* 5496 * Return the number of holes in a range on a node. If nid is MAX_NUMNODES, 5497 * then all holes in the requested range will be accounted for. 5498 */ 5499 unsigned long __meminit __absent_pages_in_range(int nid, 5500 unsigned long range_start_pfn, 5501 unsigned long range_end_pfn) 5502 { 5503 unsigned long nr_absent = range_end_pfn - range_start_pfn; 5504 unsigned long start_pfn, end_pfn; 5505 int i; 5506 5507 for_each_mem_pfn_range(i, nid, &start_pfn, &end_pfn, NULL) { 5508 start_pfn = clamp(start_pfn, range_start_pfn, range_end_pfn); 5509 end_pfn = clamp(end_pfn, range_start_pfn, range_end_pfn); 5510 nr_absent -= end_pfn - start_pfn; 5511 } 5512 return nr_absent; 5513 } 5514 5515 /** 5516 * absent_pages_in_range - Return number of page frames in holes within a range 5517 * @start_pfn: The start PFN to start searching for holes 5518 * @end_pfn: The end PFN to stop searching for holes 5519 * 5520 * It returns the number of pages frames in memory holes within a range. 5521 */ 5522 unsigned long __init absent_pages_in_range(unsigned long start_pfn, 5523 unsigned long end_pfn) 5524 { 5525 return __absent_pages_in_range(MAX_NUMNODES, start_pfn, end_pfn); 5526 } 5527 5528 /* Return the number of page frames in holes in a zone on a node */ 5529 static unsigned long __meminit zone_absent_pages_in_node(int nid, 5530 unsigned long zone_type, 5531 unsigned long node_start_pfn, 5532 unsigned long node_end_pfn, 5533 unsigned long *ignored) 5534 { 5535 unsigned long zone_low = arch_zone_lowest_possible_pfn[zone_type]; 5536 unsigned long zone_high = arch_zone_highest_possible_pfn[zone_type]; 5537 unsigned long zone_start_pfn, zone_end_pfn; 5538 unsigned long nr_absent; 5539 5540 /* When hotadd a new node from cpu_up(), the node should be empty */ 5541 if (!node_start_pfn && !node_end_pfn) 5542 return 0; 5543 5544 zone_start_pfn = clamp(node_start_pfn, zone_low, zone_high); 5545 zone_end_pfn = clamp(node_end_pfn, zone_low, zone_high); 5546 5547 adjust_zone_range_for_zone_movable(nid, zone_type, 5548 node_start_pfn, node_end_pfn, 5549 &zone_start_pfn, &zone_end_pfn); 5550 nr_absent = __absent_pages_in_range(nid, zone_start_pfn, zone_end_pfn); 5551 5552 /* 5553 * ZONE_MOVABLE handling. 5554 * Treat pages to be ZONE_MOVABLE in ZONE_NORMAL as absent pages 5555 * and vice versa. 5556 */ 5557 if (zone_movable_pfn[nid]) { 5558 if (mirrored_kernelcore) { 5559 unsigned long start_pfn, end_pfn; 5560 struct memblock_region *r; 5561 5562 for_each_memblock(memory, r) { 5563 start_pfn = clamp(memblock_region_memory_base_pfn(r), 5564 zone_start_pfn, zone_end_pfn); 5565 end_pfn = clamp(memblock_region_memory_end_pfn(r), 5566 zone_start_pfn, zone_end_pfn); 5567 5568 if (zone_type == ZONE_MOVABLE && 5569 memblock_is_mirror(r)) 5570 nr_absent += end_pfn - start_pfn; 5571 5572 if (zone_type == ZONE_NORMAL && 5573 !memblock_is_mirror(r)) 5574 nr_absent += end_pfn - start_pfn; 5575 } 5576 } else { 5577 if (zone_type == ZONE_NORMAL) 5578 nr_absent += node_end_pfn - zone_movable_pfn[nid]; 5579 } 5580 } 5581 5582 return nr_absent; 5583 } 5584 5585 #else /* CONFIG_HAVE_MEMBLOCK_NODE_MAP */ 5586 static inline unsigned long __meminit zone_spanned_pages_in_node(int nid, 5587 unsigned long zone_type, 5588 unsigned long node_start_pfn, 5589 unsigned long node_end_pfn, 5590 unsigned long *zone_start_pfn, 5591 unsigned long *zone_end_pfn, 5592 unsigned long *zones_size) 5593 { 5594 unsigned int zone; 5595 5596 *zone_start_pfn = node_start_pfn; 5597 for (zone = 0; zone < zone_type; zone++) 5598 *zone_start_pfn += zones_size[zone]; 5599 5600 *zone_end_pfn = *zone_start_pfn + zones_size[zone_type]; 5601 5602 return zones_size[zone_type]; 5603 } 5604 5605 static inline unsigned long __meminit zone_absent_pages_in_node(int nid, 5606 unsigned long zone_type, 5607 unsigned long node_start_pfn, 5608 unsigned long node_end_pfn, 5609 unsigned long *zholes_size) 5610 { 5611 if (!zholes_size) 5612 return 0; 5613 5614 return zholes_size[zone_type]; 5615 } 5616 5617 #endif /* CONFIG_HAVE_MEMBLOCK_NODE_MAP */ 5618 5619 static void __meminit calculate_node_totalpages(struct pglist_data *pgdat, 5620 unsigned long node_start_pfn, 5621 unsigned long node_end_pfn, 5622 unsigned long *zones_size, 5623 unsigned long *zholes_size) 5624 { 5625 unsigned long realtotalpages = 0, totalpages = 0; 5626 enum zone_type i; 5627 5628 for (i = 0; i < MAX_NR_ZONES; i++) { 5629 struct zone *zone = pgdat->node_zones + i; 5630 unsigned long zone_start_pfn, zone_end_pfn; 5631 unsigned long size, real_size; 5632 5633 size = zone_spanned_pages_in_node(pgdat->node_id, i, 5634 node_start_pfn, 5635 node_end_pfn, 5636 &zone_start_pfn, 5637 &zone_end_pfn, 5638 zones_size); 5639 real_size = size - zone_absent_pages_in_node(pgdat->node_id, i, 5640 node_start_pfn, node_end_pfn, 5641 zholes_size); 5642 if (size) 5643 zone->zone_start_pfn = zone_start_pfn; 5644 else 5645 zone->zone_start_pfn = 0; 5646 zone->spanned_pages = size; 5647 zone->present_pages = real_size; 5648 5649 totalpages += size; 5650 realtotalpages += real_size; 5651 } 5652 5653 pgdat->node_spanned_pages = totalpages; 5654 pgdat->node_present_pages = realtotalpages; 5655 printk(KERN_DEBUG "On node %d totalpages: %lu\n", pgdat->node_id, 5656 realtotalpages); 5657 } 5658 5659 #ifndef CONFIG_SPARSEMEM 5660 /* 5661 * Calculate the size of the zone->blockflags rounded to an unsigned long 5662 * Start by making sure zonesize is a multiple of pageblock_order by rounding 5663 * up. Then use 1 NR_PAGEBLOCK_BITS worth of bits per pageblock, finally 5664 * round what is now in bits to nearest long in bits, then return it in 5665 * bytes. 5666 */ 5667 static unsigned long __init usemap_size(unsigned long zone_start_pfn, unsigned long zonesize) 5668 { 5669 unsigned long usemapsize; 5670 5671 zonesize += zone_start_pfn & (pageblock_nr_pages-1); 5672 usemapsize = roundup(zonesize, pageblock_nr_pages); 5673 usemapsize = usemapsize >> pageblock_order; 5674 usemapsize *= NR_PAGEBLOCK_BITS; 5675 usemapsize = roundup(usemapsize, 8 * sizeof(unsigned long)); 5676 5677 return usemapsize / 8; 5678 } 5679 5680 static void __init setup_usemap(struct pglist_data *pgdat, 5681 struct zone *zone, 5682 unsigned long zone_start_pfn, 5683 unsigned long zonesize) 5684 { 5685 unsigned long usemapsize = usemap_size(zone_start_pfn, zonesize); 5686 zone->pageblock_flags = NULL; 5687 if (usemapsize) 5688 zone->pageblock_flags = 5689 memblock_virt_alloc_node_nopanic(usemapsize, 5690 pgdat->node_id); 5691 } 5692 #else 5693 static inline void setup_usemap(struct pglist_data *pgdat, struct zone *zone, 5694 unsigned long zone_start_pfn, unsigned long zonesize) {} 5695 #endif /* CONFIG_SPARSEMEM */ 5696 5697 #ifdef CONFIG_HUGETLB_PAGE_SIZE_VARIABLE 5698 5699 /* Initialise the number of pages represented by NR_PAGEBLOCK_BITS */ 5700 void __paginginit set_pageblock_order(void) 5701 { 5702 unsigned int order; 5703 5704 /* Check that pageblock_nr_pages has not already been setup */ 5705 if (pageblock_order) 5706 return; 5707 5708 if (HPAGE_SHIFT > PAGE_SHIFT) 5709 order = HUGETLB_PAGE_ORDER; 5710 else 5711 order = MAX_ORDER - 1; 5712 5713 /* 5714 * Assume the largest contiguous order of interest is a huge page. 5715 * This value may be variable depending on boot parameters on IA64 and 5716 * powerpc. 5717 */ 5718 pageblock_order = order; 5719 } 5720 #else /* CONFIG_HUGETLB_PAGE_SIZE_VARIABLE */ 5721 5722 /* 5723 * When CONFIG_HUGETLB_PAGE_SIZE_VARIABLE is not set, set_pageblock_order() 5724 * is unused as pageblock_order is set at compile-time. See 5725 * include/linux/pageblock-flags.h for the values of pageblock_order based on 5726 * the kernel config 5727 */ 5728 void __paginginit set_pageblock_order(void) 5729 { 5730 } 5731 5732 #endif /* CONFIG_HUGETLB_PAGE_SIZE_VARIABLE */ 5733 5734 static unsigned long __paginginit calc_memmap_size(unsigned long spanned_pages, 5735 unsigned long present_pages) 5736 { 5737 unsigned long pages = spanned_pages; 5738 5739 /* 5740 * Provide a more accurate estimation if there are holes within 5741 * the zone and SPARSEMEM is in use. If there are holes within the 5742 * zone, each populated memory region may cost us one or two extra 5743 * memmap pages due to alignment because memmap pages for each 5744 * populated regions may not naturally algined on page boundary. 5745 * So the (present_pages >> 4) heuristic is a tradeoff for that. 5746 */ 5747 if (spanned_pages > present_pages + (present_pages >> 4) && 5748 IS_ENABLED(CONFIG_SPARSEMEM)) 5749 pages = present_pages; 5750 5751 return PAGE_ALIGN(pages * sizeof(struct page)) >> PAGE_SHIFT; 5752 } 5753 5754 /* 5755 * Set up the zone data structures: 5756 * - mark all pages reserved 5757 * - mark all memory queues empty 5758 * - clear the memory bitmaps 5759 * 5760 * NOTE: pgdat should get zeroed by caller. 5761 */ 5762 static void __paginginit free_area_init_core(struct pglist_data *pgdat) 5763 { 5764 enum zone_type j; 5765 int nid = pgdat->node_id; 5766 int ret; 5767 5768 pgdat_resize_init(pgdat); 5769 #ifdef CONFIG_NUMA_BALANCING 5770 spin_lock_init(&pgdat->numabalancing_migrate_lock); 5771 pgdat->numabalancing_migrate_nr_pages = 0; 5772 pgdat->numabalancing_migrate_next_window = jiffies; 5773 #endif 5774 #ifdef CONFIG_TRANSPARENT_HUGEPAGE 5775 spin_lock_init(&pgdat->split_queue_lock); 5776 INIT_LIST_HEAD(&pgdat->split_queue); 5777 pgdat->split_queue_len = 0; 5778 #endif 5779 init_waitqueue_head(&pgdat->kswapd_wait); 5780 init_waitqueue_head(&pgdat->pfmemalloc_wait); 5781 #ifdef CONFIG_COMPACTION 5782 init_waitqueue_head(&pgdat->kcompactd_wait); 5783 #endif 5784 pgdat_page_ext_init(pgdat); 5785 spin_lock_init(&pgdat->lru_lock); 5786 lruvec_init(node_lruvec(pgdat)); 5787 5788 for (j = 0; j < MAX_NR_ZONES; j++) { 5789 struct zone *zone = pgdat->node_zones + j; 5790 unsigned long size, realsize, freesize, memmap_pages; 5791 unsigned long zone_start_pfn = zone->zone_start_pfn; 5792 5793 size = zone->spanned_pages; 5794 realsize = freesize = zone->present_pages; 5795 5796 /* 5797 * Adjust freesize so that it accounts for how much memory 5798 * is used by this zone for memmap. This affects the watermark 5799 * and per-cpu initialisations 5800 */ 5801 memmap_pages = calc_memmap_size(size, realsize); 5802 if (!is_highmem_idx(j)) { 5803 if (freesize >= memmap_pages) { 5804 freesize -= memmap_pages; 5805 if (memmap_pages) 5806 printk(KERN_DEBUG 5807 " %s zone: %lu pages used for memmap\n", 5808 zone_names[j], memmap_pages); 5809 } else 5810 pr_warn(" %s zone: %lu pages exceeds freesize %lu\n", 5811 zone_names[j], memmap_pages, freesize); 5812 } 5813 5814 /* Account for reserved pages */ 5815 if (j == 0 && freesize > dma_reserve) { 5816 freesize -= dma_reserve; 5817 printk(KERN_DEBUG " %s zone: %lu pages reserved\n", 5818 zone_names[0], dma_reserve); 5819 } 5820 5821 if (!is_highmem_idx(j)) 5822 nr_kernel_pages += freesize; 5823 /* Charge for highmem memmap if there are enough kernel pages */ 5824 else if (nr_kernel_pages > memmap_pages * 2) 5825 nr_kernel_pages -= memmap_pages; 5826 nr_all_pages += freesize; 5827 5828 /* 5829 * Set an approximate value for lowmem here, it will be adjusted 5830 * when the bootmem allocator frees pages into the buddy system. 5831 * And all highmem pages will be managed by the buddy system. 5832 */ 5833 zone->managed_pages = is_highmem_idx(j) ? realsize : freesize; 5834 #ifdef CONFIG_NUMA 5835 zone->node = nid; 5836 #endif 5837 zone->name = zone_names[j]; 5838 zone->zone_pgdat = pgdat; 5839 spin_lock_init(&zone->lock); 5840 zone_seqlock_init(zone); 5841 zone_pcp_init(zone); 5842 5843 if (!size) 5844 continue; 5845 5846 set_pageblock_order(); 5847 setup_usemap(pgdat, zone, zone_start_pfn, size); 5848 ret = init_currently_empty_zone(zone, zone_start_pfn, size); 5849 BUG_ON(ret); 5850 memmap_init(size, nid, j, zone_start_pfn); 5851 } 5852 } 5853 5854 static void __ref alloc_node_mem_map(struct pglist_data *pgdat) 5855 { 5856 unsigned long __maybe_unused start = 0; 5857 unsigned long __maybe_unused offset = 0; 5858 5859 /* Skip empty nodes */ 5860 if (!pgdat->node_spanned_pages) 5861 return; 5862 5863 #ifdef CONFIG_FLAT_NODE_MEM_MAP 5864 start = pgdat->node_start_pfn & ~(MAX_ORDER_NR_PAGES - 1); 5865 offset = pgdat->node_start_pfn - start; 5866 /* ia64 gets its own node_mem_map, before this, without bootmem */ 5867 if (!pgdat->node_mem_map) { 5868 unsigned long size, end; 5869 struct page *map; 5870 5871 /* 5872 * The zone's endpoints aren't required to be MAX_ORDER 5873 * aligned but the node_mem_map endpoints must be in order 5874 * for the buddy allocator to function correctly. 5875 */ 5876 end = pgdat_end_pfn(pgdat); 5877 end = ALIGN(end, MAX_ORDER_NR_PAGES); 5878 size = (end - start) * sizeof(struct page); 5879 map = alloc_remap(pgdat->node_id, size); 5880 if (!map) 5881 map = memblock_virt_alloc_node_nopanic(size, 5882 pgdat->node_id); 5883 pgdat->node_mem_map = map + offset; 5884 } 5885 #ifndef CONFIG_NEED_MULTIPLE_NODES 5886 /* 5887 * With no DISCONTIG, the global mem_map is just set as node 0's 5888 */ 5889 if (pgdat == NODE_DATA(0)) { 5890 mem_map = NODE_DATA(0)->node_mem_map; 5891 #if defined(CONFIG_HAVE_MEMBLOCK_NODE_MAP) || defined(CONFIG_FLATMEM) 5892 if (page_to_pfn(mem_map) != pgdat->node_start_pfn) 5893 mem_map -= offset; 5894 #endif /* CONFIG_HAVE_MEMBLOCK_NODE_MAP */ 5895 } 5896 #endif 5897 #endif /* CONFIG_FLAT_NODE_MEM_MAP */ 5898 } 5899 5900 void __paginginit free_area_init_node(int nid, unsigned long *zones_size, 5901 unsigned long node_start_pfn, unsigned long *zholes_size) 5902 { 5903 pg_data_t *pgdat = NODE_DATA(nid); 5904 unsigned long start_pfn = 0; 5905 unsigned long end_pfn = 0; 5906 5907 /* pg_data_t should be reset to zero when it's allocated */ 5908 WARN_ON(pgdat->nr_zones || pgdat->kswapd_classzone_idx); 5909 5910 reset_deferred_meminit(pgdat); 5911 pgdat->node_id = nid; 5912 pgdat->node_start_pfn = node_start_pfn; 5913 pgdat->per_cpu_nodestats = NULL; 5914 #ifdef CONFIG_HAVE_MEMBLOCK_NODE_MAP 5915 get_pfn_range_for_nid(nid, &start_pfn, &end_pfn); 5916 pr_info("Initmem setup node %d [mem %#018Lx-%#018Lx]\n", nid, 5917 (u64)start_pfn << PAGE_SHIFT, 5918 end_pfn ? ((u64)end_pfn << PAGE_SHIFT) - 1 : 0); 5919 #else 5920 start_pfn = node_start_pfn; 5921 #endif 5922 calculate_node_totalpages(pgdat, start_pfn, end_pfn, 5923 zones_size, zholes_size); 5924 5925 alloc_node_mem_map(pgdat); 5926 #ifdef CONFIG_FLAT_NODE_MEM_MAP 5927 printk(KERN_DEBUG "free_area_init_node: node %d, pgdat %08lx, node_mem_map %08lx\n", 5928 nid, (unsigned long)pgdat, 5929 (unsigned long)pgdat->node_mem_map); 5930 #endif 5931 5932 free_area_init_core(pgdat); 5933 } 5934 5935 #ifdef CONFIG_HAVE_MEMBLOCK_NODE_MAP 5936 5937 #if MAX_NUMNODES > 1 5938 /* 5939 * Figure out the number of possible node ids. 5940 */ 5941 void __init setup_nr_node_ids(void) 5942 { 5943 unsigned int highest; 5944 5945 highest = find_last_bit(node_possible_map.bits, MAX_NUMNODES); 5946 nr_node_ids = highest + 1; 5947 } 5948 #endif 5949 5950 /** 5951 * node_map_pfn_alignment - determine the maximum internode alignment 5952 * 5953 * This function should be called after node map is populated and sorted. 5954 * It calculates the maximum power of two alignment which can distinguish 5955 * all the nodes. 5956 * 5957 * For example, if all nodes are 1GiB and aligned to 1GiB, the return value 5958 * would indicate 1GiB alignment with (1 << (30 - PAGE_SHIFT)). If the 5959 * nodes are shifted by 256MiB, 256MiB. Note that if only the last node is 5960 * shifted, 1GiB is enough and this function will indicate so. 5961 * 5962 * This is used to test whether pfn -> nid mapping of the chosen memory 5963 * model has fine enough granularity to avoid incorrect mapping for the 5964 * populated node map. 5965 * 5966 * Returns the determined alignment in pfn's. 0 if there is no alignment 5967 * requirement (single node). 5968 */ 5969 unsigned long __init node_map_pfn_alignment(void) 5970 { 5971 unsigned long accl_mask = 0, last_end = 0; 5972 unsigned long start, end, mask; 5973 int last_nid = -1; 5974 int i, nid; 5975 5976 for_each_mem_pfn_range(i, MAX_NUMNODES, &start, &end, &nid) { 5977 if (!start || last_nid < 0 || last_nid == nid) { 5978 last_nid = nid; 5979 last_end = end; 5980 continue; 5981 } 5982 5983 /* 5984 * Start with a mask granular enough to pin-point to the 5985 * start pfn and tick off bits one-by-one until it becomes 5986 * too coarse to separate the current node from the last. 5987 */ 5988 mask = ~((1 << __ffs(start)) - 1); 5989 while (mask && last_end <= (start & (mask << 1))) 5990 mask <<= 1; 5991 5992 /* accumulate all internode masks */ 5993 accl_mask |= mask; 5994 } 5995 5996 /* convert mask to number of pages */ 5997 return ~accl_mask + 1; 5998 } 5999 6000 /* Find the lowest pfn for a node */ 6001 static unsigned long __init find_min_pfn_for_node(int nid) 6002 { 6003 unsigned long min_pfn = ULONG_MAX; 6004 unsigned long start_pfn; 6005 int i; 6006 6007 for_each_mem_pfn_range(i, nid, &start_pfn, NULL, NULL) 6008 min_pfn = min(min_pfn, start_pfn); 6009 6010 if (min_pfn == ULONG_MAX) { 6011 pr_warn("Could not find start_pfn for node %d\n", nid); 6012 return 0; 6013 } 6014 6015 return min_pfn; 6016 } 6017 6018 /** 6019 * find_min_pfn_with_active_regions - Find the minimum PFN registered 6020 * 6021 * It returns the minimum PFN based on information provided via 6022 * memblock_set_node(). 6023 */ 6024 unsigned long __init find_min_pfn_with_active_regions(void) 6025 { 6026 return find_min_pfn_for_node(MAX_NUMNODES); 6027 } 6028 6029 /* 6030 * early_calculate_totalpages() 6031 * Sum pages in active regions for movable zone. 6032 * Populate N_MEMORY for calculating usable_nodes. 6033 */ 6034 static unsigned long __init early_calculate_totalpages(void) 6035 { 6036 unsigned long totalpages = 0; 6037 unsigned long start_pfn, end_pfn; 6038 int i, nid; 6039 6040 for_each_mem_pfn_range(i, MAX_NUMNODES, &start_pfn, &end_pfn, &nid) { 6041 unsigned long pages = end_pfn - start_pfn; 6042 6043 totalpages += pages; 6044 if (pages) 6045 node_set_state(nid, N_MEMORY); 6046 } 6047 return totalpages; 6048 } 6049 6050 /* 6051 * Find the PFN the Movable zone begins in each node. Kernel memory 6052 * is spread evenly between nodes as long as the nodes have enough 6053 * memory. When they don't, some nodes will have more kernelcore than 6054 * others 6055 */ 6056 static void __init find_zone_movable_pfns_for_nodes(void) 6057 { 6058 int i, nid; 6059 unsigned long usable_startpfn; 6060 unsigned long kernelcore_node, kernelcore_remaining; 6061 /* save the state before borrow the nodemask */ 6062 nodemask_t saved_node_state = node_states[N_MEMORY]; 6063 unsigned long totalpages = early_calculate_totalpages(); 6064 int usable_nodes = nodes_weight(node_states[N_MEMORY]); 6065 struct memblock_region *r; 6066 6067 /* Need to find movable_zone earlier when movable_node is specified. */ 6068 find_usable_zone_for_movable(); 6069 6070 /* 6071 * If movable_node is specified, ignore kernelcore and movablecore 6072 * options. 6073 */ 6074 if (movable_node_is_enabled()) { 6075 for_each_memblock(memory, r) { 6076 if (!memblock_is_hotpluggable(r)) 6077 continue; 6078 6079 nid = r->nid; 6080 6081 usable_startpfn = PFN_DOWN(r->base); 6082 zone_movable_pfn[nid] = zone_movable_pfn[nid] ? 6083 min(usable_startpfn, zone_movable_pfn[nid]) : 6084 usable_startpfn; 6085 } 6086 6087 goto out2; 6088 } 6089 6090 /* 6091 * If kernelcore=mirror is specified, ignore movablecore option 6092 */ 6093 if (mirrored_kernelcore) { 6094 bool mem_below_4gb_not_mirrored = false; 6095 6096 for_each_memblock(memory, r) { 6097 if (memblock_is_mirror(r)) 6098 continue; 6099 6100 nid = r->nid; 6101 6102 usable_startpfn = memblock_region_memory_base_pfn(r); 6103 6104 if (usable_startpfn < 0x100000) { 6105 mem_below_4gb_not_mirrored = true; 6106 continue; 6107 } 6108 6109 zone_movable_pfn[nid] = zone_movable_pfn[nid] ? 6110 min(usable_startpfn, zone_movable_pfn[nid]) : 6111 usable_startpfn; 6112 } 6113 6114 if (mem_below_4gb_not_mirrored) 6115 pr_warn("This configuration results in unmirrored kernel memory."); 6116 6117 goto out2; 6118 } 6119 6120 /* 6121 * If movablecore=nn[KMG] was specified, calculate what size of 6122 * kernelcore that corresponds so that memory usable for 6123 * any allocation type is evenly spread. If both kernelcore 6124 * and movablecore are specified, then the value of kernelcore 6125 * will be used for required_kernelcore if it's greater than 6126 * what movablecore would have allowed. 6127 */ 6128 if (required_movablecore) { 6129 unsigned long corepages; 6130 6131 /* 6132 * Round-up so that ZONE_MOVABLE is at least as large as what 6133 * was requested by the user 6134 */ 6135 required_movablecore = 6136 roundup(required_movablecore, MAX_ORDER_NR_PAGES); 6137 required_movablecore = min(totalpages, required_movablecore); 6138 corepages = totalpages - required_movablecore; 6139 6140 required_kernelcore = max(required_kernelcore, corepages); 6141 } 6142 6143 /* 6144 * If kernelcore was not specified or kernelcore size is larger 6145 * than totalpages, there is no ZONE_MOVABLE. 6146 */ 6147 if (!required_kernelcore || required_kernelcore >= totalpages) 6148 goto out; 6149 6150 /* usable_startpfn is the lowest possible pfn ZONE_MOVABLE can be at */ 6151 usable_startpfn = arch_zone_lowest_possible_pfn[movable_zone]; 6152 6153 restart: 6154 /* Spread kernelcore memory as evenly as possible throughout nodes */ 6155 kernelcore_node = required_kernelcore / usable_nodes; 6156 for_each_node_state(nid, N_MEMORY) { 6157 unsigned long start_pfn, end_pfn; 6158 6159 /* 6160 * Recalculate kernelcore_node if the division per node 6161 * now exceeds what is necessary to satisfy the requested 6162 * amount of memory for the kernel 6163 */ 6164 if (required_kernelcore < kernelcore_node) 6165 kernelcore_node = required_kernelcore / usable_nodes; 6166 6167 /* 6168 * As the map is walked, we track how much memory is usable 6169 * by the kernel using kernelcore_remaining. When it is 6170 * 0, the rest of the node is usable by ZONE_MOVABLE 6171 */ 6172 kernelcore_remaining = kernelcore_node; 6173 6174 /* Go through each range of PFNs within this node */ 6175 for_each_mem_pfn_range(i, nid, &start_pfn, &end_pfn, NULL) { 6176 unsigned long size_pages; 6177 6178 start_pfn = max(start_pfn, zone_movable_pfn[nid]); 6179 if (start_pfn >= end_pfn) 6180 continue; 6181 6182 /* Account for what is only usable for kernelcore */ 6183 if (start_pfn < usable_startpfn) { 6184 unsigned long kernel_pages; 6185 kernel_pages = min(end_pfn, usable_startpfn) 6186 - start_pfn; 6187 6188 kernelcore_remaining -= min(kernel_pages, 6189 kernelcore_remaining); 6190 required_kernelcore -= min(kernel_pages, 6191 required_kernelcore); 6192 6193 /* Continue if range is now fully accounted */ 6194 if (end_pfn <= usable_startpfn) { 6195 6196 /* 6197 * Push zone_movable_pfn to the end so 6198 * that if we have to rebalance 6199 * kernelcore across nodes, we will 6200 * not double account here 6201 */ 6202 zone_movable_pfn[nid] = end_pfn; 6203 continue; 6204 } 6205 start_pfn = usable_startpfn; 6206 } 6207 6208 /* 6209 * The usable PFN range for ZONE_MOVABLE is from 6210 * start_pfn->end_pfn. Calculate size_pages as the 6211 * number of pages used as kernelcore 6212 */ 6213 size_pages = end_pfn - start_pfn; 6214 if (size_pages > kernelcore_remaining) 6215 size_pages = kernelcore_remaining; 6216 zone_movable_pfn[nid] = start_pfn + size_pages; 6217 6218 /* 6219 * Some kernelcore has been met, update counts and 6220 * break if the kernelcore for this node has been 6221 * satisfied 6222 */ 6223 required_kernelcore -= min(required_kernelcore, 6224 size_pages); 6225 kernelcore_remaining -= size_pages; 6226 if (!kernelcore_remaining) 6227 break; 6228 } 6229 } 6230 6231 /* 6232 * If there is still required_kernelcore, we do another pass with one 6233 * less node in the count. This will push zone_movable_pfn[nid] further 6234 * along on the nodes that still have memory until kernelcore is 6235 * satisfied 6236 */ 6237 usable_nodes--; 6238 if (usable_nodes && required_kernelcore > usable_nodes) 6239 goto restart; 6240 6241 out2: 6242 /* Align start of ZONE_MOVABLE on all nids to MAX_ORDER_NR_PAGES */ 6243 for (nid = 0; nid < MAX_NUMNODES; nid++) 6244 zone_movable_pfn[nid] = 6245 roundup(zone_movable_pfn[nid], MAX_ORDER_NR_PAGES); 6246 6247 out: 6248 /* restore the node_state */ 6249 node_states[N_MEMORY] = saved_node_state; 6250 } 6251 6252 /* Any regular or high memory on that node ? */ 6253 static void check_for_memory(pg_data_t *pgdat, int nid) 6254 { 6255 enum zone_type zone_type; 6256 6257 if (N_MEMORY == N_NORMAL_MEMORY) 6258 return; 6259 6260 for (zone_type = 0; zone_type <= ZONE_MOVABLE - 1; zone_type++) { 6261 struct zone *zone = &pgdat->node_zones[zone_type]; 6262 if (populated_zone(zone)) { 6263 node_set_state(nid, N_HIGH_MEMORY); 6264 if (N_NORMAL_MEMORY != N_HIGH_MEMORY && 6265 zone_type <= ZONE_NORMAL) 6266 node_set_state(nid, N_NORMAL_MEMORY); 6267 break; 6268 } 6269 } 6270 } 6271 6272 /** 6273 * free_area_init_nodes - Initialise all pg_data_t and zone data 6274 * @max_zone_pfn: an array of max PFNs for each zone 6275 * 6276 * This will call free_area_init_node() for each active node in the system. 6277 * Using the page ranges provided by memblock_set_node(), the size of each 6278 * zone in each node and their holes is calculated. If the maximum PFN 6279 * between two adjacent zones match, it is assumed that the zone is empty. 6280 * For example, if arch_max_dma_pfn == arch_max_dma32_pfn, it is assumed 6281 * that arch_max_dma32_pfn has no pages. It is also assumed that a zone 6282 * starts where the previous one ended. For example, ZONE_DMA32 starts 6283 * at arch_max_dma_pfn. 6284 */ 6285 void __init free_area_init_nodes(unsigned long *max_zone_pfn) 6286 { 6287 unsigned long start_pfn, end_pfn; 6288 int i, nid; 6289 6290 /* Record where the zone boundaries are */ 6291 memset(arch_zone_lowest_possible_pfn, 0, 6292 sizeof(arch_zone_lowest_possible_pfn)); 6293 memset(arch_zone_highest_possible_pfn, 0, 6294 sizeof(arch_zone_highest_possible_pfn)); 6295 6296 start_pfn = find_min_pfn_with_active_regions(); 6297 6298 for (i = 0; i < MAX_NR_ZONES; i++) { 6299 if (i == ZONE_MOVABLE) 6300 continue; 6301 6302 end_pfn = max(max_zone_pfn[i], start_pfn); 6303 arch_zone_lowest_possible_pfn[i] = start_pfn; 6304 arch_zone_highest_possible_pfn[i] = end_pfn; 6305 6306 start_pfn = end_pfn; 6307 } 6308 arch_zone_lowest_possible_pfn[ZONE_MOVABLE] = 0; 6309 arch_zone_highest_possible_pfn[ZONE_MOVABLE] = 0; 6310 6311 /* Find the PFNs that ZONE_MOVABLE begins at in each node */ 6312 memset(zone_movable_pfn, 0, sizeof(zone_movable_pfn)); 6313 find_zone_movable_pfns_for_nodes(); 6314 6315 /* Print out the zone ranges */ 6316 pr_info("Zone ranges:\n"); 6317 for (i = 0; i < MAX_NR_ZONES; i++) { 6318 if (i == ZONE_MOVABLE) 6319 continue; 6320 pr_info(" %-8s ", zone_names[i]); 6321 if (arch_zone_lowest_possible_pfn[i] == 6322 arch_zone_highest_possible_pfn[i]) 6323 pr_cont("empty\n"); 6324 else 6325 pr_cont("[mem %#018Lx-%#018Lx]\n", 6326 (u64)arch_zone_lowest_possible_pfn[i] 6327 << PAGE_SHIFT, 6328 ((u64)arch_zone_highest_possible_pfn[i] 6329 << PAGE_SHIFT) - 1); 6330 } 6331 6332 /* Print out the PFNs ZONE_MOVABLE begins at in each node */ 6333 pr_info("Movable zone start for each node\n"); 6334 for (i = 0; i < MAX_NUMNODES; i++) { 6335 if (zone_movable_pfn[i]) 6336 pr_info(" Node %d: %#018Lx\n", i, 6337 (u64)zone_movable_pfn[i] << PAGE_SHIFT); 6338 } 6339 6340 /* Print out the early node map */ 6341 pr_info("Early memory node ranges\n"); 6342 for_each_mem_pfn_range(i, MAX_NUMNODES, &start_pfn, &end_pfn, &nid) 6343 pr_info(" node %3d: [mem %#018Lx-%#018Lx]\n", nid, 6344 (u64)start_pfn << PAGE_SHIFT, 6345 ((u64)end_pfn << PAGE_SHIFT) - 1); 6346 6347 /* Initialise every node */ 6348 mminit_verify_pageflags_layout(); 6349 setup_nr_node_ids(); 6350 for_each_online_node(nid) { 6351 pg_data_t *pgdat = NODE_DATA(nid); 6352 free_area_init_node(nid, NULL, 6353 find_min_pfn_for_node(nid), NULL); 6354 6355 /* Any memory on that node */ 6356 if (pgdat->node_present_pages) 6357 node_set_state(nid, N_MEMORY); 6358 check_for_memory(pgdat, nid); 6359 } 6360 } 6361 6362 static int __init cmdline_parse_core(char *p, unsigned long *core) 6363 { 6364 unsigned long long coremem; 6365 if (!p) 6366 return -EINVAL; 6367 6368 coremem = memparse(p, &p); 6369 *core = coremem >> PAGE_SHIFT; 6370 6371 /* Paranoid check that UL is enough for the coremem value */ 6372 WARN_ON((coremem >> PAGE_SHIFT) > ULONG_MAX); 6373 6374 return 0; 6375 } 6376 6377 /* 6378 * kernelcore=size sets the amount of memory for use for allocations that 6379 * cannot be reclaimed or migrated. 6380 */ 6381 static int __init cmdline_parse_kernelcore(char *p) 6382 { 6383 /* parse kernelcore=mirror */ 6384 if (parse_option_str(p, "mirror")) { 6385 mirrored_kernelcore = true; 6386 return 0; 6387 } 6388 6389 return cmdline_parse_core(p, &required_kernelcore); 6390 } 6391 6392 /* 6393 * movablecore=size sets the amount of memory for use for allocations that 6394 * can be reclaimed or migrated. 6395 */ 6396 static int __init cmdline_parse_movablecore(char *p) 6397 { 6398 return cmdline_parse_core(p, &required_movablecore); 6399 } 6400 6401 early_param("kernelcore", cmdline_parse_kernelcore); 6402 early_param("movablecore", cmdline_parse_movablecore); 6403 6404 #endif /* CONFIG_HAVE_MEMBLOCK_NODE_MAP */ 6405 6406 void adjust_managed_page_count(struct page *page, long count) 6407 { 6408 spin_lock(&managed_page_count_lock); 6409 page_zone(page)->managed_pages += count; 6410 totalram_pages += count; 6411 #ifdef CONFIG_HIGHMEM 6412 if (PageHighMem(page)) 6413 totalhigh_pages += count; 6414 #endif 6415 spin_unlock(&managed_page_count_lock); 6416 } 6417 EXPORT_SYMBOL(adjust_managed_page_count); 6418 6419 unsigned long free_reserved_area(void *start, void *end, int poison, char *s) 6420 { 6421 void *pos; 6422 unsigned long pages = 0; 6423 6424 start = (void *)PAGE_ALIGN((unsigned long)start); 6425 end = (void *)((unsigned long)end & PAGE_MASK); 6426 for (pos = start; pos < end; pos += PAGE_SIZE, pages++) { 6427 if ((unsigned int)poison <= 0xFF) 6428 memset(pos, poison, PAGE_SIZE); 6429 free_reserved_page(virt_to_page(pos)); 6430 } 6431 6432 if (pages && s) 6433 pr_info("Freeing %s memory: %ldK (%p - %p)\n", 6434 s, pages << (PAGE_SHIFT - 10), start, end); 6435 6436 return pages; 6437 } 6438 EXPORT_SYMBOL(free_reserved_area); 6439 6440 #ifdef CONFIG_HIGHMEM 6441 void free_highmem_page(struct page *page) 6442 { 6443 __free_reserved_page(page); 6444 totalram_pages++; 6445 page_zone(page)->managed_pages++; 6446 totalhigh_pages++; 6447 } 6448 #endif 6449 6450 6451 void __init mem_init_print_info(const char *str) 6452 { 6453 unsigned long physpages, codesize, datasize, rosize, bss_size; 6454 unsigned long init_code_size, init_data_size; 6455 6456 physpages = get_num_physpages(); 6457 codesize = _etext - _stext; 6458 datasize = _edata - _sdata; 6459 rosize = __end_rodata - __start_rodata; 6460 bss_size = __bss_stop - __bss_start; 6461 init_data_size = __init_end - __init_begin; 6462 init_code_size = _einittext - _sinittext; 6463 6464 /* 6465 * Detect special cases and adjust section sizes accordingly: 6466 * 1) .init.* may be embedded into .data sections 6467 * 2) .init.text.* may be out of [__init_begin, __init_end], 6468 * please refer to arch/tile/kernel/vmlinux.lds.S. 6469 * 3) .rodata.* may be embedded into .text or .data sections. 6470 */ 6471 #define adj_init_size(start, end, size, pos, adj) \ 6472 do { \ 6473 if (start <= pos && pos < end && size > adj) \ 6474 size -= adj; \ 6475 } while (0) 6476 6477 adj_init_size(__init_begin, __init_end, init_data_size, 6478 _sinittext, init_code_size); 6479 adj_init_size(_stext, _etext, codesize, _sinittext, init_code_size); 6480 adj_init_size(_sdata, _edata, datasize, __init_begin, init_data_size); 6481 adj_init_size(_stext, _etext, codesize, __start_rodata, rosize); 6482 adj_init_size(_sdata, _edata, datasize, __start_rodata, rosize); 6483 6484 #undef adj_init_size 6485 6486 pr_info("Memory: %luK/%luK available (%luK kernel code, %luK rwdata, %luK rodata, %luK init, %luK bss, %luK reserved, %luK cma-reserved" 6487 #ifdef CONFIG_HIGHMEM 6488 ", %luK highmem" 6489 #endif 6490 "%s%s)\n", 6491 nr_free_pages() << (PAGE_SHIFT - 10), 6492 physpages << (PAGE_SHIFT - 10), 6493 codesize >> 10, datasize >> 10, rosize >> 10, 6494 (init_data_size + init_code_size) >> 10, bss_size >> 10, 6495 (physpages - totalram_pages - totalcma_pages) << (PAGE_SHIFT - 10), 6496 totalcma_pages << (PAGE_SHIFT - 10), 6497 #ifdef CONFIG_HIGHMEM 6498 totalhigh_pages << (PAGE_SHIFT - 10), 6499 #endif 6500 str ? ", " : "", str ? str : ""); 6501 } 6502 6503 /** 6504 * set_dma_reserve - set the specified number of pages reserved in the first zone 6505 * @new_dma_reserve: The number of pages to mark reserved 6506 * 6507 * The per-cpu batchsize and zone watermarks are determined by managed_pages. 6508 * In the DMA zone, a significant percentage may be consumed by kernel image 6509 * and other unfreeable allocations which can skew the watermarks badly. This 6510 * function may optionally be used to account for unfreeable pages in the 6511 * first zone (e.g., ZONE_DMA). The effect will be lower watermarks and 6512 * smaller per-cpu batchsize. 6513 */ 6514 void __init set_dma_reserve(unsigned long new_dma_reserve) 6515 { 6516 dma_reserve = new_dma_reserve; 6517 } 6518 6519 void __init free_area_init(unsigned long *zones_size) 6520 { 6521 free_area_init_node(0, zones_size, 6522 __pa(PAGE_OFFSET) >> PAGE_SHIFT, NULL); 6523 } 6524 6525 static int page_alloc_cpu_notify(struct notifier_block *self, 6526 unsigned long action, void *hcpu) 6527 { 6528 int cpu = (unsigned long)hcpu; 6529 6530 if (action == CPU_DEAD || action == CPU_DEAD_FROZEN) { 6531 lru_add_drain_cpu(cpu); 6532 drain_pages(cpu); 6533 6534 /* 6535 * Spill the event counters of the dead processor 6536 * into the current processors event counters. 6537 * This artificially elevates the count of the current 6538 * processor. 6539 */ 6540 vm_events_fold_cpu(cpu); 6541 6542 /* 6543 * Zero the differential counters of the dead processor 6544 * so that the vm statistics are consistent. 6545 * 6546 * This is only okay since the processor is dead and cannot 6547 * race with what we are doing. 6548 */ 6549 cpu_vm_stats_fold(cpu); 6550 } 6551 return NOTIFY_OK; 6552 } 6553 6554 void __init page_alloc_init(void) 6555 { 6556 hotcpu_notifier(page_alloc_cpu_notify, 0); 6557 } 6558 6559 /* 6560 * calculate_totalreserve_pages - called when sysctl_lowmem_reserve_ratio 6561 * or min_free_kbytes changes. 6562 */ 6563 static void calculate_totalreserve_pages(void) 6564 { 6565 struct pglist_data *pgdat; 6566 unsigned long reserve_pages = 0; 6567 enum zone_type i, j; 6568 6569 for_each_online_pgdat(pgdat) { 6570 6571 pgdat->totalreserve_pages = 0; 6572 6573 for (i = 0; i < MAX_NR_ZONES; i++) { 6574 struct zone *zone = pgdat->node_zones + i; 6575 long max = 0; 6576 6577 /* Find valid and maximum lowmem_reserve in the zone */ 6578 for (j = i; j < MAX_NR_ZONES; j++) { 6579 if (zone->lowmem_reserve[j] > max) 6580 max = zone->lowmem_reserve[j]; 6581 } 6582 6583 /* we treat the high watermark as reserved pages. */ 6584 max += high_wmark_pages(zone); 6585 6586 if (max > zone->managed_pages) 6587 max = zone->managed_pages; 6588 6589 pgdat->totalreserve_pages += max; 6590 6591 reserve_pages += max; 6592 } 6593 } 6594 totalreserve_pages = reserve_pages; 6595 } 6596 6597 /* 6598 * setup_per_zone_lowmem_reserve - called whenever 6599 * sysctl_lowmem_reserve_ratio changes. Ensures that each zone 6600 * has a correct pages reserved value, so an adequate number of 6601 * pages are left in the zone after a successful __alloc_pages(). 6602 */ 6603 static void setup_per_zone_lowmem_reserve(void) 6604 { 6605 struct pglist_data *pgdat; 6606 enum zone_type j, idx; 6607 6608 for_each_online_pgdat(pgdat) { 6609 for (j = 0; j < MAX_NR_ZONES; j++) { 6610 struct zone *zone = pgdat->node_zones + j; 6611 unsigned long managed_pages = zone->managed_pages; 6612 6613 zone->lowmem_reserve[j] = 0; 6614 6615 idx = j; 6616 while (idx) { 6617 struct zone *lower_zone; 6618 6619 idx--; 6620 6621 if (sysctl_lowmem_reserve_ratio[idx] < 1) 6622 sysctl_lowmem_reserve_ratio[idx] = 1; 6623 6624 lower_zone = pgdat->node_zones + idx; 6625 lower_zone->lowmem_reserve[j] = managed_pages / 6626 sysctl_lowmem_reserve_ratio[idx]; 6627 managed_pages += lower_zone->managed_pages; 6628 } 6629 } 6630 } 6631 6632 /* update totalreserve_pages */ 6633 calculate_totalreserve_pages(); 6634 } 6635 6636 static void __setup_per_zone_wmarks(void) 6637 { 6638 unsigned long pages_min = min_free_kbytes >> (PAGE_SHIFT - 10); 6639 unsigned long lowmem_pages = 0; 6640 struct zone *zone; 6641 unsigned long flags; 6642 6643 /* Calculate total number of !ZONE_HIGHMEM pages */ 6644 for_each_zone(zone) { 6645 if (!is_highmem(zone)) 6646 lowmem_pages += zone->managed_pages; 6647 } 6648 6649 for_each_zone(zone) { 6650 u64 tmp; 6651 6652 spin_lock_irqsave(&zone->lock, flags); 6653 tmp = (u64)pages_min * zone->managed_pages; 6654 do_div(tmp, lowmem_pages); 6655 if (is_highmem(zone)) { 6656 /* 6657 * __GFP_HIGH and PF_MEMALLOC allocations usually don't 6658 * need highmem pages, so cap pages_min to a small 6659 * value here. 6660 * 6661 * The WMARK_HIGH-WMARK_LOW and (WMARK_LOW-WMARK_MIN) 6662 * deltas control asynch page reclaim, and so should 6663 * not be capped for highmem. 6664 */ 6665 unsigned long min_pages; 6666 6667 min_pages = zone->managed_pages / 1024; 6668 min_pages = clamp(min_pages, SWAP_CLUSTER_MAX, 128UL); 6669 zone->watermark[WMARK_MIN] = min_pages; 6670 } else { 6671 /* 6672 * If it's a lowmem zone, reserve a number of pages 6673 * proportionate to the zone's size. 6674 */ 6675 zone->watermark[WMARK_MIN] = tmp; 6676 } 6677 6678 /* 6679 * Set the kswapd watermarks distance according to the 6680 * scale factor in proportion to available memory, but 6681 * ensure a minimum size on small systems. 6682 */ 6683 tmp = max_t(u64, tmp >> 2, 6684 mult_frac(zone->managed_pages, 6685 watermark_scale_factor, 10000)); 6686 6687 zone->watermark[WMARK_LOW] = min_wmark_pages(zone) + tmp; 6688 zone->watermark[WMARK_HIGH] = min_wmark_pages(zone) + tmp * 2; 6689 6690 spin_unlock_irqrestore(&zone->lock, flags); 6691 } 6692 6693 /* update totalreserve_pages */ 6694 calculate_totalreserve_pages(); 6695 } 6696 6697 /** 6698 * setup_per_zone_wmarks - called when min_free_kbytes changes 6699 * or when memory is hot-{added|removed} 6700 * 6701 * Ensures that the watermark[min,low,high] values for each zone are set 6702 * correctly with respect to min_free_kbytes. 6703 */ 6704 void setup_per_zone_wmarks(void) 6705 { 6706 mutex_lock(&zonelists_mutex); 6707 __setup_per_zone_wmarks(); 6708 mutex_unlock(&zonelists_mutex); 6709 } 6710 6711 /* 6712 * Initialise min_free_kbytes. 6713 * 6714 * For small machines we want it small (128k min). For large machines 6715 * we want it large (64MB max). But it is not linear, because network 6716 * bandwidth does not increase linearly with machine size. We use 6717 * 6718 * min_free_kbytes = 4 * sqrt(lowmem_kbytes), for better accuracy: 6719 * min_free_kbytes = sqrt(lowmem_kbytes * 16) 6720 * 6721 * which yields 6722 * 6723 * 16MB: 512k 6724 * 32MB: 724k 6725 * 64MB: 1024k 6726 * 128MB: 1448k 6727 * 256MB: 2048k 6728 * 512MB: 2896k 6729 * 1024MB: 4096k 6730 * 2048MB: 5792k 6731 * 4096MB: 8192k 6732 * 8192MB: 11584k 6733 * 16384MB: 16384k 6734 */ 6735 int __meminit init_per_zone_wmark_min(void) 6736 { 6737 unsigned long lowmem_kbytes; 6738 int new_min_free_kbytes; 6739 6740 lowmem_kbytes = nr_free_buffer_pages() * (PAGE_SIZE >> 10); 6741 new_min_free_kbytes = int_sqrt(lowmem_kbytes * 16); 6742 6743 if (new_min_free_kbytes > user_min_free_kbytes) { 6744 min_free_kbytes = new_min_free_kbytes; 6745 if (min_free_kbytes < 128) 6746 min_free_kbytes = 128; 6747 if (min_free_kbytes > 65536) 6748 min_free_kbytes = 65536; 6749 } else { 6750 pr_warn("min_free_kbytes is not updated to %d because user defined value %d is preferred\n", 6751 new_min_free_kbytes, user_min_free_kbytes); 6752 } 6753 setup_per_zone_wmarks(); 6754 refresh_zone_stat_thresholds(); 6755 setup_per_zone_lowmem_reserve(); 6756 6757 #ifdef CONFIG_NUMA 6758 setup_min_unmapped_ratio(); 6759 setup_min_slab_ratio(); 6760 #endif 6761 6762 return 0; 6763 } 6764 core_initcall(init_per_zone_wmark_min) 6765 6766 /* 6767 * min_free_kbytes_sysctl_handler - just a wrapper around proc_dointvec() so 6768 * that we can call two helper functions whenever min_free_kbytes 6769 * changes. 6770 */ 6771 int min_free_kbytes_sysctl_handler(struct ctl_table *table, int write, 6772 void __user *buffer, size_t *length, loff_t *ppos) 6773 { 6774 int rc; 6775 6776 rc = proc_dointvec_minmax(table, write, buffer, length, ppos); 6777 if (rc) 6778 return rc; 6779 6780 if (write) { 6781 user_min_free_kbytes = min_free_kbytes; 6782 setup_per_zone_wmarks(); 6783 } 6784 return 0; 6785 } 6786 6787 int watermark_scale_factor_sysctl_handler(struct ctl_table *table, int write, 6788 void __user *buffer, size_t *length, loff_t *ppos) 6789 { 6790 int rc; 6791 6792 rc = proc_dointvec_minmax(table, write, buffer, length, ppos); 6793 if (rc) 6794 return rc; 6795 6796 if (write) 6797 setup_per_zone_wmarks(); 6798 6799 return 0; 6800 } 6801 6802 #ifdef CONFIG_NUMA 6803 static void setup_min_unmapped_ratio(void) 6804 { 6805 pg_data_t *pgdat; 6806 struct zone *zone; 6807 6808 for_each_online_pgdat(pgdat) 6809 pgdat->min_unmapped_pages = 0; 6810 6811 for_each_zone(zone) 6812 zone->zone_pgdat->min_unmapped_pages += (zone->managed_pages * 6813 sysctl_min_unmapped_ratio) / 100; 6814 } 6815 6816 6817 int sysctl_min_unmapped_ratio_sysctl_handler(struct ctl_table *table, int write, 6818 void __user *buffer, size_t *length, loff_t *ppos) 6819 { 6820 int rc; 6821 6822 rc = proc_dointvec_minmax(table, write, buffer, length, ppos); 6823 if (rc) 6824 return rc; 6825 6826 setup_min_unmapped_ratio(); 6827 6828 return 0; 6829 } 6830 6831 static void setup_min_slab_ratio(void) 6832 { 6833 pg_data_t *pgdat; 6834 struct zone *zone; 6835 6836 for_each_online_pgdat(pgdat) 6837 pgdat->min_slab_pages = 0; 6838 6839 for_each_zone(zone) 6840 zone->zone_pgdat->min_slab_pages += (zone->managed_pages * 6841 sysctl_min_slab_ratio) / 100; 6842 } 6843 6844 int sysctl_min_slab_ratio_sysctl_handler(struct ctl_table *table, int write, 6845 void __user *buffer, size_t *length, loff_t *ppos) 6846 { 6847 int rc; 6848 6849 rc = proc_dointvec_minmax(table, write, buffer, length, ppos); 6850 if (rc) 6851 return rc; 6852 6853 setup_min_slab_ratio(); 6854 6855 return 0; 6856 } 6857 #endif 6858 6859 /* 6860 * lowmem_reserve_ratio_sysctl_handler - just a wrapper around 6861 * proc_dointvec() so that we can call setup_per_zone_lowmem_reserve() 6862 * whenever sysctl_lowmem_reserve_ratio changes. 6863 * 6864 * The reserve ratio obviously has absolutely no relation with the 6865 * minimum watermarks. The lowmem reserve ratio can only make sense 6866 * if in function of the boot time zone sizes. 6867 */ 6868 int lowmem_reserve_ratio_sysctl_handler(struct ctl_table *table, int write, 6869 void __user *buffer, size_t *length, loff_t *ppos) 6870 { 6871 proc_dointvec_minmax(table, write, buffer, length, ppos); 6872 setup_per_zone_lowmem_reserve(); 6873 return 0; 6874 } 6875 6876 /* 6877 * percpu_pagelist_fraction - changes the pcp->high for each zone on each 6878 * cpu. It is the fraction of total pages in each zone that a hot per cpu 6879 * pagelist can have before it gets flushed back to buddy allocator. 6880 */ 6881 int percpu_pagelist_fraction_sysctl_handler(struct ctl_table *table, int write, 6882 void __user *buffer, size_t *length, loff_t *ppos) 6883 { 6884 struct zone *zone; 6885 int old_percpu_pagelist_fraction; 6886 int ret; 6887 6888 mutex_lock(&pcp_batch_high_lock); 6889 old_percpu_pagelist_fraction = percpu_pagelist_fraction; 6890 6891 ret = proc_dointvec_minmax(table, write, buffer, length, ppos); 6892 if (!write || ret < 0) 6893 goto out; 6894 6895 /* Sanity checking to avoid pcp imbalance */ 6896 if (percpu_pagelist_fraction && 6897 percpu_pagelist_fraction < MIN_PERCPU_PAGELIST_FRACTION) { 6898 percpu_pagelist_fraction = old_percpu_pagelist_fraction; 6899 ret = -EINVAL; 6900 goto out; 6901 } 6902 6903 /* No change? */ 6904 if (percpu_pagelist_fraction == old_percpu_pagelist_fraction) 6905 goto out; 6906 6907 for_each_populated_zone(zone) { 6908 unsigned int cpu; 6909 6910 for_each_possible_cpu(cpu) 6911 pageset_set_high_and_batch(zone, 6912 per_cpu_ptr(zone->pageset, cpu)); 6913 } 6914 out: 6915 mutex_unlock(&pcp_batch_high_lock); 6916 return ret; 6917 } 6918 6919 #ifdef CONFIG_NUMA 6920 int hashdist = HASHDIST_DEFAULT; 6921 6922 static int __init set_hashdist(char *str) 6923 { 6924 if (!str) 6925 return 0; 6926 hashdist = simple_strtoul(str, &str, 0); 6927 return 1; 6928 } 6929 __setup("hashdist=", set_hashdist); 6930 #endif 6931 6932 /* 6933 * allocate a large system hash table from bootmem 6934 * - it is assumed that the hash table must contain an exact power-of-2 6935 * quantity of entries 6936 * - limit is the number of hash buckets, not the total allocation size 6937 */ 6938 void *__init alloc_large_system_hash(const char *tablename, 6939 unsigned long bucketsize, 6940 unsigned long numentries, 6941 int scale, 6942 int flags, 6943 unsigned int *_hash_shift, 6944 unsigned int *_hash_mask, 6945 unsigned long low_limit, 6946 unsigned long high_limit) 6947 { 6948 unsigned long long max = high_limit; 6949 unsigned long log2qty, size; 6950 void *table = NULL; 6951 6952 /* allow the kernel cmdline to have a say */ 6953 if (!numentries) { 6954 /* round applicable memory size up to nearest megabyte */ 6955 numentries = nr_kernel_pages; 6956 6957 /* It isn't necessary when PAGE_SIZE >= 1MB */ 6958 if (PAGE_SHIFT < 20) 6959 numentries = round_up(numentries, (1<<20)/PAGE_SIZE); 6960 6961 /* limit to 1 bucket per 2^scale bytes of low memory */ 6962 if (scale > PAGE_SHIFT) 6963 numentries >>= (scale - PAGE_SHIFT); 6964 else 6965 numentries <<= (PAGE_SHIFT - scale); 6966 6967 /* Make sure we've got at least a 0-order allocation.. */ 6968 if (unlikely(flags & HASH_SMALL)) { 6969 /* Makes no sense without HASH_EARLY */ 6970 WARN_ON(!(flags & HASH_EARLY)); 6971 if (!(numentries >> *_hash_shift)) { 6972 numentries = 1UL << *_hash_shift; 6973 BUG_ON(!numentries); 6974 } 6975 } else if (unlikely((numentries * bucketsize) < PAGE_SIZE)) 6976 numentries = PAGE_SIZE / bucketsize; 6977 } 6978 numentries = roundup_pow_of_two(numentries); 6979 6980 /* limit allocation size to 1/16 total memory by default */ 6981 if (max == 0) { 6982 max = ((unsigned long long)nr_all_pages << PAGE_SHIFT) >> 4; 6983 do_div(max, bucketsize); 6984 } 6985 max = min(max, 0x80000000ULL); 6986 6987 if (numentries < low_limit) 6988 numentries = low_limit; 6989 if (numentries > max) 6990 numentries = max; 6991 6992 log2qty = ilog2(numentries); 6993 6994 do { 6995 size = bucketsize << log2qty; 6996 if (flags & HASH_EARLY) 6997 table = memblock_virt_alloc_nopanic(size, 0); 6998 else if (hashdist) 6999 table = __vmalloc(size, GFP_ATOMIC, PAGE_KERNEL); 7000 else { 7001 /* 7002 * If bucketsize is not a power-of-two, we may free 7003 * some pages at the end of hash table which 7004 * alloc_pages_exact() automatically does 7005 */ 7006 if (get_order(size) < MAX_ORDER) { 7007 table = alloc_pages_exact(size, GFP_ATOMIC); 7008 kmemleak_alloc(table, size, 1, GFP_ATOMIC); 7009 } 7010 } 7011 } while (!table && size > PAGE_SIZE && --log2qty); 7012 7013 if (!table) 7014 panic("Failed to allocate %s hash table\n", tablename); 7015 7016 pr_info("%s hash table entries: %ld (order: %d, %lu bytes)\n", 7017 tablename, 1UL << log2qty, ilog2(size) - PAGE_SHIFT, size); 7018 7019 if (_hash_shift) 7020 *_hash_shift = log2qty; 7021 if (_hash_mask) 7022 *_hash_mask = (1 << log2qty) - 1; 7023 7024 return table; 7025 } 7026 7027 /* 7028 * This function checks whether pageblock includes unmovable pages or not. 7029 * If @count is not zero, it is okay to include less @count unmovable pages 7030 * 7031 * PageLRU check without isolation or lru_lock could race so that 7032 * MIGRATE_MOVABLE block might include unmovable pages. It means you can't 7033 * expect this function should be exact. 7034 */ 7035 bool has_unmovable_pages(struct zone *zone, struct page *page, int count, 7036 bool skip_hwpoisoned_pages) 7037 { 7038 unsigned long pfn, iter, found; 7039 int mt; 7040 7041 /* 7042 * For avoiding noise data, lru_add_drain_all() should be called 7043 * If ZONE_MOVABLE, the zone never contains unmovable pages 7044 */ 7045 if (zone_idx(zone) == ZONE_MOVABLE) 7046 return false; 7047 mt = get_pageblock_migratetype(page); 7048 if (mt == MIGRATE_MOVABLE || is_migrate_cma(mt)) 7049 return false; 7050 7051 pfn = page_to_pfn(page); 7052 for (found = 0, iter = 0; iter < pageblock_nr_pages; iter++) { 7053 unsigned long check = pfn + iter; 7054 7055 if (!pfn_valid_within(check)) 7056 continue; 7057 7058 page = pfn_to_page(check); 7059 7060 /* 7061 * Hugepages are not in LRU lists, but they're movable. 7062 * We need not scan over tail pages bacause we don't 7063 * handle each tail page individually in migration. 7064 */ 7065 if (PageHuge(page)) { 7066 iter = round_up(iter + 1, 1<<compound_order(page)) - 1; 7067 continue; 7068 } 7069 7070 /* 7071 * We can't use page_count without pin a page 7072 * because another CPU can free compound page. 7073 * This check already skips compound tails of THP 7074 * because their page->_refcount is zero at all time. 7075 */ 7076 if (!page_ref_count(page)) { 7077 if (PageBuddy(page)) 7078 iter += (1 << page_order(page)) - 1; 7079 continue; 7080 } 7081 7082 /* 7083 * The HWPoisoned page may be not in buddy system, and 7084 * page_count() is not 0. 7085 */ 7086 if (skip_hwpoisoned_pages && PageHWPoison(page)) 7087 continue; 7088 7089 if (!PageLRU(page)) 7090 found++; 7091 /* 7092 * If there are RECLAIMABLE pages, we need to check 7093 * it. But now, memory offline itself doesn't call 7094 * shrink_node_slabs() and it still to be fixed. 7095 */ 7096 /* 7097 * If the page is not RAM, page_count()should be 0. 7098 * we don't need more check. This is an _used_ not-movable page. 7099 * 7100 * The problematic thing here is PG_reserved pages. PG_reserved 7101 * is set to both of a memory hole page and a _used_ kernel 7102 * page at boot. 7103 */ 7104 if (found > count) 7105 return true; 7106 } 7107 return false; 7108 } 7109 7110 bool is_pageblock_removable_nolock(struct page *page) 7111 { 7112 struct zone *zone; 7113 unsigned long pfn; 7114 7115 /* 7116 * We have to be careful here because we are iterating over memory 7117 * sections which are not zone aware so we might end up outside of 7118 * the zone but still within the section. 7119 * We have to take care about the node as well. If the node is offline 7120 * its NODE_DATA will be NULL - see page_zone. 7121 */ 7122 if (!node_online(page_to_nid(page))) 7123 return false; 7124 7125 zone = page_zone(page); 7126 pfn = page_to_pfn(page); 7127 if (!zone_spans_pfn(zone, pfn)) 7128 return false; 7129 7130 return !has_unmovable_pages(zone, page, 0, true); 7131 } 7132 7133 #if (defined(CONFIG_MEMORY_ISOLATION) && defined(CONFIG_COMPACTION)) || defined(CONFIG_CMA) 7134 7135 static unsigned long pfn_max_align_down(unsigned long pfn) 7136 { 7137 return pfn & ~(max_t(unsigned long, MAX_ORDER_NR_PAGES, 7138 pageblock_nr_pages) - 1); 7139 } 7140 7141 static unsigned long pfn_max_align_up(unsigned long pfn) 7142 { 7143 return ALIGN(pfn, max_t(unsigned long, MAX_ORDER_NR_PAGES, 7144 pageblock_nr_pages)); 7145 } 7146 7147 /* [start, end) must belong to a single zone. */ 7148 static int __alloc_contig_migrate_range(struct compact_control *cc, 7149 unsigned long start, unsigned long end) 7150 { 7151 /* This function is based on compact_zone() from compaction.c. */ 7152 unsigned long nr_reclaimed; 7153 unsigned long pfn = start; 7154 unsigned int tries = 0; 7155 int ret = 0; 7156 7157 migrate_prep(); 7158 7159 while (pfn < end || !list_empty(&cc->migratepages)) { 7160 if (fatal_signal_pending(current)) { 7161 ret = -EINTR; 7162 break; 7163 } 7164 7165 if (list_empty(&cc->migratepages)) { 7166 cc->nr_migratepages = 0; 7167 pfn = isolate_migratepages_range(cc, pfn, end); 7168 if (!pfn) { 7169 ret = -EINTR; 7170 break; 7171 } 7172 tries = 0; 7173 } else if (++tries == 5) { 7174 ret = ret < 0 ? ret : -EBUSY; 7175 break; 7176 } 7177 7178 nr_reclaimed = reclaim_clean_pages_from_list(cc->zone, 7179 &cc->migratepages); 7180 cc->nr_migratepages -= nr_reclaimed; 7181 7182 ret = migrate_pages(&cc->migratepages, alloc_migrate_target, 7183 NULL, 0, cc->mode, MR_CMA); 7184 } 7185 if (ret < 0) { 7186 putback_movable_pages(&cc->migratepages); 7187 return ret; 7188 } 7189 return 0; 7190 } 7191 7192 /** 7193 * alloc_contig_range() -- tries to allocate given range of pages 7194 * @start: start PFN to allocate 7195 * @end: one-past-the-last PFN to allocate 7196 * @migratetype: migratetype of the underlaying pageblocks (either 7197 * #MIGRATE_MOVABLE or #MIGRATE_CMA). All pageblocks 7198 * in range must have the same migratetype and it must 7199 * be either of the two. 7200 * 7201 * The PFN range does not have to be pageblock or MAX_ORDER_NR_PAGES 7202 * aligned, however it's the caller's responsibility to guarantee that 7203 * we are the only thread that changes migrate type of pageblocks the 7204 * pages fall in. 7205 * 7206 * The PFN range must belong to a single zone. 7207 * 7208 * Returns zero on success or negative error code. On success all 7209 * pages which PFN is in [start, end) are allocated for the caller and 7210 * need to be freed with free_contig_range(). 7211 */ 7212 int alloc_contig_range(unsigned long start, unsigned long end, 7213 unsigned migratetype) 7214 { 7215 unsigned long outer_start, outer_end; 7216 unsigned int order; 7217 int ret = 0; 7218 7219 struct compact_control cc = { 7220 .nr_migratepages = 0, 7221 .order = -1, 7222 .zone = page_zone(pfn_to_page(start)), 7223 .mode = MIGRATE_SYNC, 7224 .ignore_skip_hint = true, 7225 }; 7226 INIT_LIST_HEAD(&cc.migratepages); 7227 7228 /* 7229 * What we do here is we mark all pageblocks in range as 7230 * MIGRATE_ISOLATE. Because pageblock and max order pages may 7231 * have different sizes, and due to the way page allocator 7232 * work, we align the range to biggest of the two pages so 7233 * that page allocator won't try to merge buddies from 7234 * different pageblocks and change MIGRATE_ISOLATE to some 7235 * other migration type. 7236 * 7237 * Once the pageblocks are marked as MIGRATE_ISOLATE, we 7238 * migrate the pages from an unaligned range (ie. pages that 7239 * we are interested in). This will put all the pages in 7240 * range back to page allocator as MIGRATE_ISOLATE. 7241 * 7242 * When this is done, we take the pages in range from page 7243 * allocator removing them from the buddy system. This way 7244 * page allocator will never consider using them. 7245 * 7246 * This lets us mark the pageblocks back as 7247 * MIGRATE_CMA/MIGRATE_MOVABLE so that free pages in the 7248 * aligned range but not in the unaligned, original range are 7249 * put back to page allocator so that buddy can use them. 7250 */ 7251 7252 ret = start_isolate_page_range(pfn_max_align_down(start), 7253 pfn_max_align_up(end), migratetype, 7254 false); 7255 if (ret) 7256 return ret; 7257 7258 /* 7259 * In case of -EBUSY, we'd like to know which page causes problem. 7260 * So, just fall through. We will check it in test_pages_isolated(). 7261 */ 7262 ret = __alloc_contig_migrate_range(&cc, start, end); 7263 if (ret && ret != -EBUSY) 7264 goto done; 7265 7266 /* 7267 * Pages from [start, end) are within a MAX_ORDER_NR_PAGES 7268 * aligned blocks that are marked as MIGRATE_ISOLATE. What's 7269 * more, all pages in [start, end) are free in page allocator. 7270 * What we are going to do is to allocate all pages from 7271 * [start, end) (that is remove them from page allocator). 7272 * 7273 * The only problem is that pages at the beginning and at the 7274 * end of interesting range may be not aligned with pages that 7275 * page allocator holds, ie. they can be part of higher order 7276 * pages. Because of this, we reserve the bigger range and 7277 * once this is done free the pages we are not interested in. 7278 * 7279 * We don't have to hold zone->lock here because the pages are 7280 * isolated thus they won't get removed from buddy. 7281 */ 7282 7283 lru_add_drain_all(); 7284 drain_all_pages(cc.zone); 7285 7286 order = 0; 7287 outer_start = start; 7288 while (!PageBuddy(pfn_to_page(outer_start))) { 7289 if (++order >= MAX_ORDER) { 7290 outer_start = start; 7291 break; 7292 } 7293 outer_start &= ~0UL << order; 7294 } 7295 7296 if (outer_start != start) { 7297 order = page_order(pfn_to_page(outer_start)); 7298 7299 /* 7300 * outer_start page could be small order buddy page and 7301 * it doesn't include start page. Adjust outer_start 7302 * in this case to report failed page properly 7303 * on tracepoint in test_pages_isolated() 7304 */ 7305 if (outer_start + (1UL << order) <= start) 7306 outer_start = start; 7307 } 7308 7309 /* Make sure the range is really isolated. */ 7310 if (test_pages_isolated(outer_start, end, false)) { 7311 pr_info("%s: [%lx, %lx) PFNs busy\n", 7312 __func__, outer_start, end); 7313 ret = -EBUSY; 7314 goto done; 7315 } 7316 7317 /* Grab isolated pages from freelists. */ 7318 outer_end = isolate_freepages_range(&cc, outer_start, end); 7319 if (!outer_end) { 7320 ret = -EBUSY; 7321 goto done; 7322 } 7323 7324 /* Free head and tail (if any) */ 7325 if (start != outer_start) 7326 free_contig_range(outer_start, start - outer_start); 7327 if (end != outer_end) 7328 free_contig_range(end, outer_end - end); 7329 7330 done: 7331 undo_isolate_page_range(pfn_max_align_down(start), 7332 pfn_max_align_up(end), migratetype); 7333 return ret; 7334 } 7335 7336 void free_contig_range(unsigned long pfn, unsigned nr_pages) 7337 { 7338 unsigned int count = 0; 7339 7340 for (; nr_pages--; pfn++) { 7341 struct page *page = pfn_to_page(pfn); 7342 7343 count += page_count(page) != 1; 7344 __free_page(page); 7345 } 7346 WARN(count != 0, "%d pages are still in use!\n", count); 7347 } 7348 #endif 7349 7350 #ifdef CONFIG_MEMORY_HOTPLUG 7351 /* 7352 * The zone indicated has a new number of managed_pages; batch sizes and percpu 7353 * page high values need to be recalulated. 7354 */ 7355 void __meminit zone_pcp_update(struct zone *zone) 7356 { 7357 unsigned cpu; 7358 mutex_lock(&pcp_batch_high_lock); 7359 for_each_possible_cpu(cpu) 7360 pageset_set_high_and_batch(zone, 7361 per_cpu_ptr(zone->pageset, cpu)); 7362 mutex_unlock(&pcp_batch_high_lock); 7363 } 7364 #endif 7365 7366 void zone_pcp_reset(struct zone *zone) 7367 { 7368 unsigned long flags; 7369 int cpu; 7370 struct per_cpu_pageset *pset; 7371 7372 /* avoid races with drain_pages() */ 7373 local_irq_save(flags); 7374 if (zone->pageset != &boot_pageset) { 7375 for_each_online_cpu(cpu) { 7376 pset = per_cpu_ptr(zone->pageset, cpu); 7377 drain_zonestat(zone, pset); 7378 } 7379 free_percpu(zone->pageset); 7380 zone->pageset = &boot_pageset; 7381 } 7382 local_irq_restore(flags); 7383 } 7384 7385 #ifdef CONFIG_MEMORY_HOTREMOVE 7386 /* 7387 * All pages in the range must be in a single zone and isolated 7388 * before calling this. 7389 */ 7390 void 7391 __offline_isolated_pages(unsigned long start_pfn, unsigned long end_pfn) 7392 { 7393 struct page *page; 7394 struct zone *zone; 7395 unsigned int order, i; 7396 unsigned long pfn; 7397 unsigned long flags; 7398 /* find the first valid pfn */ 7399 for (pfn = start_pfn; pfn < end_pfn; pfn++) 7400 if (pfn_valid(pfn)) 7401 break; 7402 if (pfn == end_pfn) 7403 return; 7404 zone = page_zone(pfn_to_page(pfn)); 7405 spin_lock_irqsave(&zone->lock, flags); 7406 pfn = start_pfn; 7407 while (pfn < end_pfn) { 7408 if (!pfn_valid(pfn)) { 7409 pfn++; 7410 continue; 7411 } 7412 page = pfn_to_page(pfn); 7413 /* 7414 * The HWPoisoned page may be not in buddy system, and 7415 * page_count() is not 0. 7416 */ 7417 if (unlikely(!PageBuddy(page) && PageHWPoison(page))) { 7418 pfn++; 7419 SetPageReserved(page); 7420 continue; 7421 } 7422 7423 BUG_ON(page_count(page)); 7424 BUG_ON(!PageBuddy(page)); 7425 order = page_order(page); 7426 #ifdef CONFIG_DEBUG_VM 7427 pr_info("remove from free list %lx %d %lx\n", 7428 pfn, 1 << order, end_pfn); 7429 #endif 7430 list_del(&page->lru); 7431 rmv_page_order(page); 7432 zone->free_area[order].nr_free--; 7433 for (i = 0; i < (1 << order); i++) 7434 SetPageReserved((page+i)); 7435 pfn += (1 << order); 7436 } 7437 spin_unlock_irqrestore(&zone->lock, flags); 7438 } 7439 #endif 7440 7441 bool is_free_buddy_page(struct page *page) 7442 { 7443 struct zone *zone = page_zone(page); 7444 unsigned long pfn = page_to_pfn(page); 7445 unsigned long flags; 7446 unsigned int order; 7447 7448 spin_lock_irqsave(&zone->lock, flags); 7449 for (order = 0; order < MAX_ORDER; order++) { 7450 struct page *page_head = page - (pfn & ((1 << order) - 1)); 7451 7452 if (PageBuddy(page_head) && page_order(page_head) >= order) 7453 break; 7454 } 7455 spin_unlock_irqrestore(&zone->lock, flags); 7456 7457 return order < MAX_ORDER; 7458 } 7459