1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * mm_init.c - Memory initialisation verification and debugging 4 * 5 * Copyright 2008 IBM Corporation, 2008 6 * Author Mel Gorman <mel@csn.ul.ie> 7 * 8 */ 9 #include <linux/kernel.h> 10 #include <linux/init.h> 11 #include <linux/kobject.h> 12 #include <linux/export.h> 13 #include <linux/memory.h> 14 #include <linux/notifier.h> 15 #include <linux/sched.h> 16 #include <linux/mman.h> 17 #include <linux/memblock.h> 18 #include <linux/page-isolation.h> 19 #include <linux/padata.h> 20 #include <linux/nmi.h> 21 #include <linux/buffer_head.h> 22 #include <linux/kmemleak.h> 23 #include <linux/kfence.h> 24 #include <linux/page_ext.h> 25 #include <linux/pti.h> 26 #include <linux/pgtable.h> 27 #include <linux/swap.h> 28 #include <linux/cma.h> 29 #include "internal.h" 30 #include "slab.h" 31 #include "shuffle.h" 32 33 #include <asm/setup.h> 34 35 #ifdef CONFIG_DEBUG_MEMORY_INIT 36 int __meminitdata mminit_loglevel; 37 38 /* The zonelists are simply reported, validation is manual. */ 39 void __init mminit_verify_zonelist(void) 40 { 41 int nid; 42 43 if (mminit_loglevel < MMINIT_VERIFY) 44 return; 45 46 for_each_online_node(nid) { 47 pg_data_t *pgdat = NODE_DATA(nid); 48 struct zone *zone; 49 struct zoneref *z; 50 struct zonelist *zonelist; 51 int i, listid, zoneid; 52 53 BUILD_BUG_ON(MAX_ZONELISTS > 2); 54 for (i = 0; i < MAX_ZONELISTS * MAX_NR_ZONES; i++) { 55 56 /* Identify the zone and nodelist */ 57 zoneid = i % MAX_NR_ZONES; 58 listid = i / MAX_NR_ZONES; 59 zonelist = &pgdat->node_zonelists[listid]; 60 zone = &pgdat->node_zones[zoneid]; 61 if (!populated_zone(zone)) 62 continue; 63 64 /* Print information about the zonelist */ 65 printk(KERN_DEBUG "mminit::zonelist %s %d:%s = ", 66 listid > 0 ? "thisnode" : "general", nid, 67 zone->name); 68 69 /* Iterate the zonelist */ 70 for_each_zone_zonelist(zone, z, zonelist, zoneid) 71 pr_cont("%d:%s ", zone_to_nid(zone), zone->name); 72 pr_cont("\n"); 73 } 74 } 75 } 76 77 void __init mminit_verify_pageflags_layout(void) 78 { 79 int shift, width; 80 unsigned long or_mask, add_mask; 81 82 shift = 8 * sizeof(unsigned long); 83 width = shift - SECTIONS_WIDTH - NODES_WIDTH - ZONES_WIDTH 84 - LAST_CPUPID_SHIFT - KASAN_TAG_WIDTH - LRU_GEN_WIDTH - LRU_REFS_WIDTH; 85 mminit_dprintk(MMINIT_TRACE, "pageflags_layout_widths", 86 "Section %d Node %d Zone %d Lastcpupid %d Kasantag %d Gen %d Tier %d Flags %d\n", 87 SECTIONS_WIDTH, 88 NODES_WIDTH, 89 ZONES_WIDTH, 90 LAST_CPUPID_WIDTH, 91 KASAN_TAG_WIDTH, 92 LRU_GEN_WIDTH, 93 LRU_REFS_WIDTH, 94 NR_PAGEFLAGS); 95 mminit_dprintk(MMINIT_TRACE, "pageflags_layout_shifts", 96 "Section %d Node %d Zone %d Lastcpupid %d Kasantag %d\n", 97 SECTIONS_SHIFT, 98 NODES_SHIFT, 99 ZONES_SHIFT, 100 LAST_CPUPID_SHIFT, 101 KASAN_TAG_WIDTH); 102 mminit_dprintk(MMINIT_TRACE, "pageflags_layout_pgshifts", 103 "Section %lu Node %lu Zone %lu Lastcpupid %lu Kasantag %lu\n", 104 (unsigned long)SECTIONS_PGSHIFT, 105 (unsigned long)NODES_PGSHIFT, 106 (unsigned long)ZONES_PGSHIFT, 107 (unsigned long)LAST_CPUPID_PGSHIFT, 108 (unsigned long)KASAN_TAG_PGSHIFT); 109 mminit_dprintk(MMINIT_TRACE, "pageflags_layout_nodezoneid", 110 "Node/Zone ID: %lu -> %lu\n", 111 (unsigned long)(ZONEID_PGOFF + ZONEID_SHIFT), 112 (unsigned long)ZONEID_PGOFF); 113 mminit_dprintk(MMINIT_TRACE, "pageflags_layout_usage", 114 "location: %d -> %d layout %d -> %d unused %d -> %d page-flags\n", 115 shift, width, width, NR_PAGEFLAGS, NR_PAGEFLAGS, 0); 116 #ifdef NODE_NOT_IN_PAGE_FLAGS 117 mminit_dprintk(MMINIT_TRACE, "pageflags_layout_nodeflags", 118 "Node not in page flags"); 119 #endif 120 #ifdef LAST_CPUPID_NOT_IN_PAGE_FLAGS 121 mminit_dprintk(MMINIT_TRACE, "pageflags_layout_nodeflags", 122 "Last cpupid not in page flags"); 123 #endif 124 125 if (SECTIONS_WIDTH) { 126 shift -= SECTIONS_WIDTH; 127 BUG_ON(shift != SECTIONS_PGSHIFT); 128 } 129 if (NODES_WIDTH) { 130 shift -= NODES_WIDTH; 131 BUG_ON(shift != NODES_PGSHIFT); 132 } 133 if (ZONES_WIDTH) { 134 shift -= ZONES_WIDTH; 135 BUG_ON(shift != ZONES_PGSHIFT); 136 } 137 138 /* Check for bitmask overlaps */ 139 or_mask = (ZONES_MASK << ZONES_PGSHIFT) | 140 (NODES_MASK << NODES_PGSHIFT) | 141 (SECTIONS_MASK << SECTIONS_PGSHIFT); 142 add_mask = (ZONES_MASK << ZONES_PGSHIFT) + 143 (NODES_MASK << NODES_PGSHIFT) + 144 (SECTIONS_MASK << SECTIONS_PGSHIFT); 145 BUG_ON(or_mask != add_mask); 146 } 147 148 static __init int set_mminit_loglevel(char *str) 149 { 150 get_option(&str, &mminit_loglevel); 151 return 0; 152 } 153 early_param("mminit_loglevel", set_mminit_loglevel); 154 #endif /* CONFIG_DEBUG_MEMORY_INIT */ 155 156 struct kobject *mm_kobj; 157 EXPORT_SYMBOL_GPL(mm_kobj); 158 159 #ifdef CONFIG_SMP 160 s32 vm_committed_as_batch = 32; 161 162 void mm_compute_batch(int overcommit_policy) 163 { 164 u64 memsized_batch; 165 s32 nr = num_present_cpus(); 166 s32 batch = max_t(s32, nr*2, 32); 167 unsigned long ram_pages = totalram_pages(); 168 169 /* 170 * For policy OVERCOMMIT_NEVER, set batch size to 0.4% of 171 * (total memory/#cpus), and lift it to 25% for other policies 172 * to easy the possible lock contention for percpu_counter 173 * vm_committed_as, while the max limit is INT_MAX 174 */ 175 if (overcommit_policy == OVERCOMMIT_NEVER) 176 memsized_batch = min_t(u64, ram_pages/nr/256, INT_MAX); 177 else 178 memsized_batch = min_t(u64, ram_pages/nr/4, INT_MAX); 179 180 vm_committed_as_batch = max_t(s32, memsized_batch, batch); 181 } 182 183 static int __meminit mm_compute_batch_notifier(struct notifier_block *self, 184 unsigned long action, void *arg) 185 { 186 switch (action) { 187 case MEM_ONLINE: 188 case MEM_OFFLINE: 189 mm_compute_batch(sysctl_overcommit_memory); 190 break; 191 default: 192 break; 193 } 194 return NOTIFY_OK; 195 } 196 197 static int __init mm_compute_batch_init(void) 198 { 199 mm_compute_batch(sysctl_overcommit_memory); 200 hotplug_memory_notifier(mm_compute_batch_notifier, MM_COMPUTE_BATCH_PRI); 201 return 0; 202 } 203 204 __initcall(mm_compute_batch_init); 205 206 #endif 207 208 static int __init mm_sysfs_init(void) 209 { 210 mm_kobj = kobject_create_and_add("mm", kernel_kobj); 211 if (!mm_kobj) 212 return -ENOMEM; 213 214 return 0; 215 } 216 postcore_initcall(mm_sysfs_init); 217 218 static unsigned long arch_zone_lowest_possible_pfn[MAX_NR_ZONES] __initdata; 219 static unsigned long arch_zone_highest_possible_pfn[MAX_NR_ZONES] __initdata; 220 static unsigned long zone_movable_pfn[MAX_NUMNODES] __initdata; 221 222 static unsigned long required_kernelcore __initdata; 223 static unsigned long required_kernelcore_percent __initdata; 224 static unsigned long required_movablecore __initdata; 225 static unsigned long required_movablecore_percent __initdata; 226 227 static unsigned long nr_kernel_pages __initdata; 228 static unsigned long nr_all_pages __initdata; 229 static unsigned long dma_reserve __initdata; 230 231 static bool deferred_struct_pages __meminitdata; 232 233 static DEFINE_PER_CPU(struct per_cpu_nodestat, boot_nodestats); 234 235 static int __init cmdline_parse_core(char *p, unsigned long *core, 236 unsigned long *percent) 237 { 238 unsigned long long coremem; 239 char *endptr; 240 241 if (!p) 242 return -EINVAL; 243 244 /* Value may be a percentage of total memory, otherwise bytes */ 245 coremem = simple_strtoull(p, &endptr, 0); 246 if (*endptr == '%') { 247 /* Paranoid check for percent values greater than 100 */ 248 WARN_ON(coremem > 100); 249 250 *percent = coremem; 251 } else { 252 coremem = memparse(p, &p); 253 /* Paranoid check that UL is enough for the coremem value */ 254 WARN_ON((coremem >> PAGE_SHIFT) > ULONG_MAX); 255 256 *core = coremem >> PAGE_SHIFT; 257 *percent = 0UL; 258 } 259 return 0; 260 } 261 262 bool mirrored_kernelcore __initdata_memblock; 263 264 /* 265 * kernelcore=size sets the amount of memory for use for allocations that 266 * cannot be reclaimed or migrated. 267 */ 268 static int __init cmdline_parse_kernelcore(char *p) 269 { 270 /* parse kernelcore=mirror */ 271 if (parse_option_str(p, "mirror")) { 272 mirrored_kernelcore = true; 273 return 0; 274 } 275 276 return cmdline_parse_core(p, &required_kernelcore, 277 &required_kernelcore_percent); 278 } 279 early_param("kernelcore", cmdline_parse_kernelcore); 280 281 /* 282 * movablecore=size sets the amount of memory for use for allocations that 283 * can be reclaimed or migrated. 284 */ 285 static int __init cmdline_parse_movablecore(char *p) 286 { 287 return cmdline_parse_core(p, &required_movablecore, 288 &required_movablecore_percent); 289 } 290 early_param("movablecore", cmdline_parse_movablecore); 291 292 /* 293 * early_calculate_totalpages() 294 * Sum pages in active regions for movable zone. 295 * Populate N_MEMORY for calculating usable_nodes. 296 */ 297 static unsigned long __init early_calculate_totalpages(void) 298 { 299 unsigned long totalpages = 0; 300 unsigned long start_pfn, end_pfn; 301 int i, nid; 302 303 for_each_mem_pfn_range(i, MAX_NUMNODES, &start_pfn, &end_pfn, &nid) { 304 unsigned long pages = end_pfn - start_pfn; 305 306 totalpages += pages; 307 if (pages) 308 node_set_state(nid, N_MEMORY); 309 } 310 return totalpages; 311 } 312 313 /* 314 * This finds a zone that can be used for ZONE_MOVABLE pages. The 315 * assumption is made that zones within a node are ordered in monotonic 316 * increasing memory addresses so that the "highest" populated zone is used 317 */ 318 static void __init find_usable_zone_for_movable(void) 319 { 320 int zone_index; 321 for (zone_index = MAX_NR_ZONES - 1; zone_index >= 0; zone_index--) { 322 if (zone_index == ZONE_MOVABLE) 323 continue; 324 325 if (arch_zone_highest_possible_pfn[zone_index] > 326 arch_zone_lowest_possible_pfn[zone_index]) 327 break; 328 } 329 330 VM_BUG_ON(zone_index == -1); 331 movable_zone = zone_index; 332 } 333 334 /* 335 * Find the PFN the Movable zone begins in each node. Kernel memory 336 * is spread evenly between nodes as long as the nodes have enough 337 * memory. When they don't, some nodes will have more kernelcore than 338 * others 339 */ 340 static void __init find_zone_movable_pfns_for_nodes(void) 341 { 342 int i, nid; 343 unsigned long usable_startpfn; 344 unsigned long kernelcore_node, kernelcore_remaining; 345 /* save the state before borrow the nodemask */ 346 nodemask_t saved_node_state = node_states[N_MEMORY]; 347 unsigned long totalpages = early_calculate_totalpages(); 348 int usable_nodes = nodes_weight(node_states[N_MEMORY]); 349 struct memblock_region *r; 350 351 /* Need to find movable_zone earlier when movable_node is specified. */ 352 find_usable_zone_for_movable(); 353 354 /* 355 * If movable_node is specified, ignore kernelcore and movablecore 356 * options. 357 */ 358 if (movable_node_is_enabled()) { 359 for_each_mem_region(r) { 360 if (!memblock_is_hotpluggable(r)) 361 continue; 362 363 nid = memblock_get_region_node(r); 364 365 usable_startpfn = PFN_DOWN(r->base); 366 zone_movable_pfn[nid] = zone_movable_pfn[nid] ? 367 min(usable_startpfn, zone_movable_pfn[nid]) : 368 usable_startpfn; 369 } 370 371 goto out2; 372 } 373 374 /* 375 * If kernelcore=mirror is specified, ignore movablecore option 376 */ 377 if (mirrored_kernelcore) { 378 bool mem_below_4gb_not_mirrored = false; 379 380 for_each_mem_region(r) { 381 if (memblock_is_mirror(r)) 382 continue; 383 384 nid = memblock_get_region_node(r); 385 386 usable_startpfn = memblock_region_memory_base_pfn(r); 387 388 if (usable_startpfn < PHYS_PFN(SZ_4G)) { 389 mem_below_4gb_not_mirrored = true; 390 continue; 391 } 392 393 zone_movable_pfn[nid] = zone_movable_pfn[nid] ? 394 min(usable_startpfn, zone_movable_pfn[nid]) : 395 usable_startpfn; 396 } 397 398 if (mem_below_4gb_not_mirrored) 399 pr_warn("This configuration results in unmirrored kernel memory.\n"); 400 401 goto out2; 402 } 403 404 /* 405 * If kernelcore=nn% or movablecore=nn% was specified, calculate the 406 * amount of necessary memory. 407 */ 408 if (required_kernelcore_percent) 409 required_kernelcore = (totalpages * 100 * required_kernelcore_percent) / 410 10000UL; 411 if (required_movablecore_percent) 412 required_movablecore = (totalpages * 100 * required_movablecore_percent) / 413 10000UL; 414 415 /* 416 * If movablecore= was specified, calculate what size of 417 * kernelcore that corresponds so that memory usable for 418 * any allocation type is evenly spread. If both kernelcore 419 * and movablecore are specified, then the value of kernelcore 420 * will be used for required_kernelcore if it's greater than 421 * what movablecore would have allowed. 422 */ 423 if (required_movablecore) { 424 unsigned long corepages; 425 426 /* 427 * Round-up so that ZONE_MOVABLE is at least as large as what 428 * was requested by the user 429 */ 430 required_movablecore = 431 roundup(required_movablecore, MAX_ORDER_NR_PAGES); 432 required_movablecore = min(totalpages, required_movablecore); 433 corepages = totalpages - required_movablecore; 434 435 required_kernelcore = max(required_kernelcore, corepages); 436 } 437 438 /* 439 * If kernelcore was not specified or kernelcore size is larger 440 * than totalpages, there is no ZONE_MOVABLE. 441 */ 442 if (!required_kernelcore || required_kernelcore >= totalpages) 443 goto out; 444 445 /* usable_startpfn is the lowest possible pfn ZONE_MOVABLE can be at */ 446 usable_startpfn = arch_zone_lowest_possible_pfn[movable_zone]; 447 448 restart: 449 /* Spread kernelcore memory as evenly as possible throughout nodes */ 450 kernelcore_node = required_kernelcore / usable_nodes; 451 for_each_node_state(nid, N_MEMORY) { 452 unsigned long start_pfn, end_pfn; 453 454 /* 455 * Recalculate kernelcore_node if the division per node 456 * now exceeds what is necessary to satisfy the requested 457 * amount of memory for the kernel 458 */ 459 if (required_kernelcore < kernelcore_node) 460 kernelcore_node = required_kernelcore / usable_nodes; 461 462 /* 463 * As the map is walked, we track how much memory is usable 464 * by the kernel using kernelcore_remaining. When it is 465 * 0, the rest of the node is usable by ZONE_MOVABLE 466 */ 467 kernelcore_remaining = kernelcore_node; 468 469 /* Go through each range of PFNs within this node */ 470 for_each_mem_pfn_range(i, nid, &start_pfn, &end_pfn, NULL) { 471 unsigned long size_pages; 472 473 start_pfn = max(start_pfn, zone_movable_pfn[nid]); 474 if (start_pfn >= end_pfn) 475 continue; 476 477 /* Account for what is only usable for kernelcore */ 478 if (start_pfn < usable_startpfn) { 479 unsigned long kernel_pages; 480 kernel_pages = min(end_pfn, usable_startpfn) 481 - start_pfn; 482 483 kernelcore_remaining -= min(kernel_pages, 484 kernelcore_remaining); 485 required_kernelcore -= min(kernel_pages, 486 required_kernelcore); 487 488 /* Continue if range is now fully accounted */ 489 if (end_pfn <= usable_startpfn) { 490 491 /* 492 * Push zone_movable_pfn to the end so 493 * that if we have to rebalance 494 * kernelcore across nodes, we will 495 * not double account here 496 */ 497 zone_movable_pfn[nid] = end_pfn; 498 continue; 499 } 500 start_pfn = usable_startpfn; 501 } 502 503 /* 504 * The usable PFN range for ZONE_MOVABLE is from 505 * start_pfn->end_pfn. Calculate size_pages as the 506 * number of pages used as kernelcore 507 */ 508 size_pages = end_pfn - start_pfn; 509 if (size_pages > kernelcore_remaining) 510 size_pages = kernelcore_remaining; 511 zone_movable_pfn[nid] = start_pfn + size_pages; 512 513 /* 514 * Some kernelcore has been met, update counts and 515 * break if the kernelcore for this node has been 516 * satisfied 517 */ 518 required_kernelcore -= min(required_kernelcore, 519 size_pages); 520 kernelcore_remaining -= size_pages; 521 if (!kernelcore_remaining) 522 break; 523 } 524 } 525 526 /* 527 * If there is still required_kernelcore, we do another pass with one 528 * less node in the count. This will push zone_movable_pfn[nid] further 529 * along on the nodes that still have memory until kernelcore is 530 * satisfied 531 */ 532 usable_nodes--; 533 if (usable_nodes && required_kernelcore > usable_nodes) 534 goto restart; 535 536 out2: 537 /* Align start of ZONE_MOVABLE on all nids to MAX_ORDER_NR_PAGES */ 538 for (nid = 0; nid < MAX_NUMNODES; nid++) { 539 unsigned long start_pfn, end_pfn; 540 541 zone_movable_pfn[nid] = 542 roundup(zone_movable_pfn[nid], MAX_ORDER_NR_PAGES); 543 544 get_pfn_range_for_nid(nid, &start_pfn, &end_pfn); 545 if (zone_movable_pfn[nid] >= end_pfn) 546 zone_movable_pfn[nid] = 0; 547 } 548 549 out: 550 /* restore the node_state */ 551 node_states[N_MEMORY] = saved_node_state; 552 } 553 554 static void __meminit __init_single_page(struct page *page, unsigned long pfn, 555 unsigned long zone, int nid) 556 { 557 mm_zero_struct_page(page); 558 set_page_links(page, zone, nid, pfn); 559 init_page_count(page); 560 page_mapcount_reset(page); 561 page_cpupid_reset_last(page); 562 page_kasan_tag_reset(page); 563 564 INIT_LIST_HEAD(&page->lru); 565 #ifdef WANT_PAGE_VIRTUAL 566 /* The shift won't overflow because ZONE_NORMAL is below 4G. */ 567 if (!is_highmem_idx(zone)) 568 set_page_address(page, __va(pfn << PAGE_SHIFT)); 569 #endif 570 } 571 572 #ifdef CONFIG_NUMA 573 /* 574 * During memory init memblocks map pfns to nids. The search is expensive and 575 * this caches recent lookups. The implementation of __early_pfn_to_nid 576 * treats start/end as pfns. 577 */ 578 struct mminit_pfnnid_cache { 579 unsigned long last_start; 580 unsigned long last_end; 581 int last_nid; 582 }; 583 584 static struct mminit_pfnnid_cache early_pfnnid_cache __meminitdata; 585 586 /* 587 * Required by SPARSEMEM. Given a PFN, return what node the PFN is on. 588 */ 589 static int __meminit __early_pfn_to_nid(unsigned long pfn, 590 struct mminit_pfnnid_cache *state) 591 { 592 unsigned long start_pfn, end_pfn; 593 int nid; 594 595 if (state->last_start <= pfn && pfn < state->last_end) 596 return state->last_nid; 597 598 nid = memblock_search_pfn_nid(pfn, &start_pfn, &end_pfn); 599 if (nid != NUMA_NO_NODE) { 600 state->last_start = start_pfn; 601 state->last_end = end_pfn; 602 state->last_nid = nid; 603 } 604 605 return nid; 606 } 607 608 int __meminit early_pfn_to_nid(unsigned long pfn) 609 { 610 static DEFINE_SPINLOCK(early_pfn_lock); 611 int nid; 612 613 spin_lock(&early_pfn_lock); 614 nid = __early_pfn_to_nid(pfn, &early_pfnnid_cache); 615 if (nid < 0) 616 nid = first_online_node; 617 spin_unlock(&early_pfn_lock); 618 619 return nid; 620 } 621 622 int hashdist = HASHDIST_DEFAULT; 623 624 static int __init set_hashdist(char *str) 625 { 626 if (!str) 627 return 0; 628 hashdist = simple_strtoul(str, &str, 0); 629 return 1; 630 } 631 __setup("hashdist=", set_hashdist); 632 633 static inline void fixup_hashdist(void) 634 { 635 if (num_node_state(N_MEMORY) == 1) 636 hashdist = 0; 637 } 638 #else 639 static inline void fixup_hashdist(void) {} 640 #endif /* CONFIG_NUMA */ 641 642 #ifdef CONFIG_DEFERRED_STRUCT_PAGE_INIT 643 static inline void pgdat_set_deferred_range(pg_data_t *pgdat) 644 { 645 pgdat->first_deferred_pfn = ULONG_MAX; 646 } 647 648 /* Returns true if the struct page for the pfn is initialised */ 649 static inline bool __meminit early_page_initialised(unsigned long pfn, int nid) 650 { 651 if (node_online(nid) && pfn >= NODE_DATA(nid)->first_deferred_pfn) 652 return false; 653 654 return true; 655 } 656 657 /* 658 * Returns true when the remaining initialisation should be deferred until 659 * later in the boot cycle when it can be parallelised. 660 */ 661 static bool __meminit 662 defer_init(int nid, unsigned long pfn, unsigned long end_pfn) 663 { 664 static unsigned long prev_end_pfn, nr_initialised; 665 666 if (early_page_ext_enabled()) 667 return false; 668 /* 669 * prev_end_pfn static that contains the end of previous zone 670 * No need to protect because called very early in boot before smp_init. 671 */ 672 if (prev_end_pfn != end_pfn) { 673 prev_end_pfn = end_pfn; 674 nr_initialised = 0; 675 } 676 677 /* Always populate low zones for address-constrained allocations */ 678 if (end_pfn < pgdat_end_pfn(NODE_DATA(nid))) 679 return false; 680 681 if (NODE_DATA(nid)->first_deferred_pfn != ULONG_MAX) 682 return true; 683 /* 684 * We start only with one section of pages, more pages are added as 685 * needed until the rest of deferred pages are initialized. 686 */ 687 nr_initialised++; 688 if ((nr_initialised > PAGES_PER_SECTION) && 689 (pfn & (PAGES_PER_SECTION - 1)) == 0) { 690 NODE_DATA(nid)->first_deferred_pfn = pfn; 691 return true; 692 } 693 return false; 694 } 695 696 static void __meminit init_reserved_page(unsigned long pfn, int nid) 697 { 698 pg_data_t *pgdat; 699 int zid; 700 701 if (early_page_initialised(pfn, nid)) 702 return; 703 704 pgdat = NODE_DATA(nid); 705 706 for (zid = 0; zid < MAX_NR_ZONES; zid++) { 707 struct zone *zone = &pgdat->node_zones[zid]; 708 709 if (zone_spans_pfn(zone, pfn)) 710 break; 711 } 712 __init_single_page(pfn_to_page(pfn), pfn, zid, nid); 713 } 714 #else 715 static inline void pgdat_set_deferred_range(pg_data_t *pgdat) {} 716 717 static inline bool early_page_initialised(unsigned long pfn, int nid) 718 { 719 return true; 720 } 721 722 static inline bool defer_init(int nid, unsigned long pfn, unsigned long end_pfn) 723 { 724 return false; 725 } 726 727 static inline void init_reserved_page(unsigned long pfn, int nid) 728 { 729 } 730 #endif /* CONFIG_DEFERRED_STRUCT_PAGE_INIT */ 731 732 /* 733 * Initialised pages do not have PageReserved set. This function is 734 * called for each range allocated by the bootmem allocator and 735 * marks the pages PageReserved. The remaining valid pages are later 736 * sent to the buddy page allocator. 737 */ 738 void __meminit reserve_bootmem_region(phys_addr_t start, 739 phys_addr_t end, int nid) 740 { 741 unsigned long start_pfn = PFN_DOWN(start); 742 unsigned long end_pfn = PFN_UP(end); 743 744 for (; start_pfn < end_pfn; start_pfn++) { 745 if (pfn_valid(start_pfn)) { 746 struct page *page = pfn_to_page(start_pfn); 747 748 init_reserved_page(start_pfn, nid); 749 750 /* Avoid false-positive PageTail() */ 751 INIT_LIST_HEAD(&page->lru); 752 753 /* 754 * no need for atomic set_bit because the struct 755 * page is not visible yet so nobody should 756 * access it yet. 757 */ 758 __SetPageReserved(page); 759 } 760 } 761 } 762 763 /* If zone is ZONE_MOVABLE but memory is mirrored, it is an overlapped init */ 764 static bool __meminit 765 overlap_memmap_init(unsigned long zone, unsigned long *pfn) 766 { 767 static struct memblock_region *r; 768 769 if (mirrored_kernelcore && zone == ZONE_MOVABLE) { 770 if (!r || *pfn >= memblock_region_memory_end_pfn(r)) { 771 for_each_mem_region(r) { 772 if (*pfn < memblock_region_memory_end_pfn(r)) 773 break; 774 } 775 } 776 if (*pfn >= memblock_region_memory_base_pfn(r) && 777 memblock_is_mirror(r)) { 778 *pfn = memblock_region_memory_end_pfn(r); 779 return true; 780 } 781 } 782 return false; 783 } 784 785 /* 786 * Only struct pages that correspond to ranges defined by memblock.memory 787 * are zeroed and initialized by going through __init_single_page() during 788 * memmap_init_zone_range(). 789 * 790 * But, there could be struct pages that correspond to holes in 791 * memblock.memory. This can happen because of the following reasons: 792 * - physical memory bank size is not necessarily the exact multiple of the 793 * arbitrary section size 794 * - early reserved memory may not be listed in memblock.memory 795 * - memory layouts defined with memmap= kernel parameter may not align 796 * nicely with memmap sections 797 * 798 * Explicitly initialize those struct pages so that: 799 * - PG_Reserved is set 800 * - zone and node links point to zone and node that span the page if the 801 * hole is in the middle of a zone 802 * - zone and node links point to adjacent zone/node if the hole falls on 803 * the zone boundary; the pages in such holes will be prepended to the 804 * zone/node above the hole except for the trailing pages in the last 805 * section that will be appended to the zone/node below. 806 */ 807 static void __init init_unavailable_range(unsigned long spfn, 808 unsigned long epfn, 809 int zone, int node) 810 { 811 unsigned long pfn; 812 u64 pgcnt = 0; 813 814 for (pfn = spfn; pfn < epfn; pfn++) { 815 if (!pfn_valid(pageblock_start_pfn(pfn))) { 816 pfn = pageblock_end_pfn(pfn) - 1; 817 continue; 818 } 819 __init_single_page(pfn_to_page(pfn), pfn, zone, node); 820 __SetPageReserved(pfn_to_page(pfn)); 821 pgcnt++; 822 } 823 824 if (pgcnt) 825 pr_info("On node %d, zone %s: %lld pages in unavailable ranges", 826 node, zone_names[zone], pgcnt); 827 } 828 829 /* 830 * Initially all pages are reserved - free ones are freed 831 * up by memblock_free_all() once the early boot process is 832 * done. Non-atomic initialization, single-pass. 833 * 834 * All aligned pageblocks are initialized to the specified migratetype 835 * (usually MIGRATE_MOVABLE). Besides setting the migratetype, no related 836 * zone stats (e.g., nr_isolate_pageblock) are touched. 837 */ 838 void __meminit memmap_init_range(unsigned long size, int nid, unsigned long zone, 839 unsigned long start_pfn, unsigned long zone_end_pfn, 840 enum meminit_context context, 841 struct vmem_altmap *altmap, int migratetype) 842 { 843 unsigned long pfn, end_pfn = start_pfn + size; 844 struct page *page; 845 846 if (highest_memmap_pfn < end_pfn - 1) 847 highest_memmap_pfn = end_pfn - 1; 848 849 #ifdef CONFIG_ZONE_DEVICE 850 /* 851 * Honor reservation requested by the driver for this ZONE_DEVICE 852 * memory. We limit the total number of pages to initialize to just 853 * those that might contain the memory mapping. We will defer the 854 * ZONE_DEVICE page initialization until after we have released 855 * the hotplug lock. 856 */ 857 if (zone == ZONE_DEVICE) { 858 if (!altmap) 859 return; 860 861 if (start_pfn == altmap->base_pfn) 862 start_pfn += altmap->reserve; 863 end_pfn = altmap->base_pfn + vmem_altmap_offset(altmap); 864 } 865 #endif 866 867 for (pfn = start_pfn; pfn < end_pfn; ) { 868 /* 869 * There can be holes in boot-time mem_map[]s handed to this 870 * function. They do not exist on hotplugged memory. 871 */ 872 if (context == MEMINIT_EARLY) { 873 if (overlap_memmap_init(zone, &pfn)) 874 continue; 875 if (defer_init(nid, pfn, zone_end_pfn)) { 876 deferred_struct_pages = true; 877 break; 878 } 879 } 880 881 page = pfn_to_page(pfn); 882 __init_single_page(page, pfn, zone, nid); 883 if (context == MEMINIT_HOTPLUG) 884 __SetPageReserved(page); 885 886 /* 887 * Usually, we want to mark the pageblock MIGRATE_MOVABLE, 888 * such that unmovable allocations won't be scattered all 889 * over the place during system boot. 890 */ 891 if (pageblock_aligned(pfn)) { 892 set_pageblock_migratetype(page, migratetype); 893 cond_resched(); 894 } 895 pfn++; 896 } 897 } 898 899 static void __init memmap_init_zone_range(struct zone *zone, 900 unsigned long start_pfn, 901 unsigned long end_pfn, 902 unsigned long *hole_pfn) 903 { 904 unsigned long zone_start_pfn = zone->zone_start_pfn; 905 unsigned long zone_end_pfn = zone_start_pfn + zone->spanned_pages; 906 int nid = zone_to_nid(zone), zone_id = zone_idx(zone); 907 908 start_pfn = clamp(start_pfn, zone_start_pfn, zone_end_pfn); 909 end_pfn = clamp(end_pfn, zone_start_pfn, zone_end_pfn); 910 911 if (start_pfn >= end_pfn) 912 return; 913 914 memmap_init_range(end_pfn - start_pfn, nid, zone_id, start_pfn, 915 zone_end_pfn, MEMINIT_EARLY, NULL, MIGRATE_MOVABLE); 916 917 if (*hole_pfn < start_pfn) 918 init_unavailable_range(*hole_pfn, start_pfn, zone_id, nid); 919 920 *hole_pfn = end_pfn; 921 } 922 923 static void __init memmap_init(void) 924 { 925 unsigned long start_pfn, end_pfn; 926 unsigned long hole_pfn = 0; 927 int i, j, zone_id = 0, nid; 928 929 for_each_mem_pfn_range(i, MAX_NUMNODES, &start_pfn, &end_pfn, &nid) { 930 struct pglist_data *node = NODE_DATA(nid); 931 932 for (j = 0; j < MAX_NR_ZONES; j++) { 933 struct zone *zone = node->node_zones + j; 934 935 if (!populated_zone(zone)) 936 continue; 937 938 memmap_init_zone_range(zone, start_pfn, end_pfn, 939 &hole_pfn); 940 zone_id = j; 941 } 942 } 943 944 #ifdef CONFIG_SPARSEMEM 945 /* 946 * Initialize the memory map for hole in the range [memory_end, 947 * section_end]. 948 * Append the pages in this hole to the highest zone in the last 949 * node. 950 * The call to init_unavailable_range() is outside the ifdef to 951 * silence the compiler warining about zone_id set but not used; 952 * for FLATMEM it is a nop anyway 953 */ 954 end_pfn = round_up(end_pfn, PAGES_PER_SECTION); 955 if (hole_pfn < end_pfn) 956 #endif 957 init_unavailable_range(hole_pfn, end_pfn, zone_id, nid); 958 } 959 960 #ifdef CONFIG_ZONE_DEVICE 961 static void __ref __init_zone_device_page(struct page *page, unsigned long pfn, 962 unsigned long zone_idx, int nid, 963 struct dev_pagemap *pgmap) 964 { 965 966 __init_single_page(page, pfn, zone_idx, nid); 967 968 /* 969 * Mark page reserved as it will need to wait for onlining 970 * phase for it to be fully associated with a zone. 971 * 972 * We can use the non-atomic __set_bit operation for setting 973 * the flag as we are still initializing the pages. 974 */ 975 __SetPageReserved(page); 976 977 /* 978 * ZONE_DEVICE pages union ->lru with a ->pgmap back pointer 979 * and zone_device_data. It is a bug if a ZONE_DEVICE page is 980 * ever freed or placed on a driver-private list. 981 */ 982 page->pgmap = pgmap; 983 page->zone_device_data = NULL; 984 985 /* 986 * Mark the block movable so that blocks are reserved for 987 * movable at startup. This will force kernel allocations 988 * to reserve their blocks rather than leaking throughout 989 * the address space during boot when many long-lived 990 * kernel allocations are made. 991 * 992 * Please note that MEMINIT_HOTPLUG path doesn't clear memmap 993 * because this is done early in section_activate() 994 */ 995 if (pageblock_aligned(pfn)) { 996 set_pageblock_migratetype(page, MIGRATE_MOVABLE); 997 cond_resched(); 998 } 999 1000 /* 1001 * ZONE_DEVICE pages are released directly to the driver page allocator 1002 * which will set the page count to 1 when allocating the page. 1003 */ 1004 if (pgmap->type == MEMORY_DEVICE_PRIVATE || 1005 pgmap->type == MEMORY_DEVICE_COHERENT) 1006 set_page_count(page, 0); 1007 } 1008 1009 /* 1010 * With compound page geometry and when struct pages are stored in ram most 1011 * tail pages are reused. Consequently, the amount of unique struct pages to 1012 * initialize is a lot smaller that the total amount of struct pages being 1013 * mapped. This is a paired / mild layering violation with explicit knowledge 1014 * of how the sparse_vmemmap internals handle compound pages in the lack 1015 * of an altmap. See vmemmap_populate_compound_pages(). 1016 */ 1017 static inline unsigned long compound_nr_pages(struct vmem_altmap *altmap, 1018 struct dev_pagemap *pgmap) 1019 { 1020 if (!vmemmap_can_optimize(altmap, pgmap)) 1021 return pgmap_vmemmap_nr(pgmap); 1022 1023 return 2 * (PAGE_SIZE / sizeof(struct page)); 1024 } 1025 1026 static void __ref memmap_init_compound(struct page *head, 1027 unsigned long head_pfn, 1028 unsigned long zone_idx, int nid, 1029 struct dev_pagemap *pgmap, 1030 unsigned long nr_pages) 1031 { 1032 unsigned long pfn, end_pfn = head_pfn + nr_pages; 1033 unsigned int order = pgmap->vmemmap_shift; 1034 1035 __SetPageHead(head); 1036 for (pfn = head_pfn + 1; pfn < end_pfn; pfn++) { 1037 struct page *page = pfn_to_page(pfn); 1038 1039 __init_zone_device_page(page, pfn, zone_idx, nid, pgmap); 1040 prep_compound_tail(head, pfn - head_pfn); 1041 set_page_count(page, 0); 1042 1043 /* 1044 * The first tail page stores important compound page info. 1045 * Call prep_compound_head() after the first tail page has 1046 * been initialized, to not have the data overwritten. 1047 */ 1048 if (pfn == head_pfn + 1) 1049 prep_compound_head(head, order); 1050 } 1051 } 1052 1053 void __ref memmap_init_zone_device(struct zone *zone, 1054 unsigned long start_pfn, 1055 unsigned long nr_pages, 1056 struct dev_pagemap *pgmap) 1057 { 1058 unsigned long pfn, end_pfn = start_pfn + nr_pages; 1059 struct pglist_data *pgdat = zone->zone_pgdat; 1060 struct vmem_altmap *altmap = pgmap_altmap(pgmap); 1061 unsigned int pfns_per_compound = pgmap_vmemmap_nr(pgmap); 1062 unsigned long zone_idx = zone_idx(zone); 1063 unsigned long start = jiffies; 1064 int nid = pgdat->node_id; 1065 1066 if (WARN_ON_ONCE(!pgmap || zone_idx != ZONE_DEVICE)) 1067 return; 1068 1069 /* 1070 * The call to memmap_init should have already taken care 1071 * of the pages reserved for the memmap, so we can just jump to 1072 * the end of that region and start processing the device pages. 1073 */ 1074 if (altmap) { 1075 start_pfn = altmap->base_pfn + vmem_altmap_offset(altmap); 1076 nr_pages = end_pfn - start_pfn; 1077 } 1078 1079 for (pfn = start_pfn; pfn < end_pfn; pfn += pfns_per_compound) { 1080 struct page *page = pfn_to_page(pfn); 1081 1082 __init_zone_device_page(page, pfn, zone_idx, nid, pgmap); 1083 1084 if (pfns_per_compound == 1) 1085 continue; 1086 1087 memmap_init_compound(page, pfn, zone_idx, nid, pgmap, 1088 compound_nr_pages(altmap, pgmap)); 1089 } 1090 1091 pr_debug("%s initialised %lu pages in %ums\n", __func__, 1092 nr_pages, jiffies_to_msecs(jiffies - start)); 1093 } 1094 #endif 1095 1096 /* 1097 * The zone ranges provided by the architecture do not include ZONE_MOVABLE 1098 * because it is sized independent of architecture. Unlike the other zones, 1099 * the starting point for ZONE_MOVABLE is not fixed. It may be different 1100 * in each node depending on the size of each node and how evenly kernelcore 1101 * is distributed. This helper function adjusts the zone ranges 1102 * provided by the architecture for a given node by using the end of the 1103 * highest usable zone for ZONE_MOVABLE. This preserves the assumption that 1104 * zones within a node are in order of monotonic increases memory addresses 1105 */ 1106 static void __init adjust_zone_range_for_zone_movable(int nid, 1107 unsigned long zone_type, 1108 unsigned long node_start_pfn, 1109 unsigned long node_end_pfn, 1110 unsigned long *zone_start_pfn, 1111 unsigned long *zone_end_pfn) 1112 { 1113 /* Only adjust if ZONE_MOVABLE is on this node */ 1114 if (zone_movable_pfn[nid]) { 1115 /* Size ZONE_MOVABLE */ 1116 if (zone_type == ZONE_MOVABLE) { 1117 *zone_start_pfn = zone_movable_pfn[nid]; 1118 *zone_end_pfn = min(node_end_pfn, 1119 arch_zone_highest_possible_pfn[movable_zone]); 1120 1121 /* Adjust for ZONE_MOVABLE starting within this range */ 1122 } else if (!mirrored_kernelcore && 1123 *zone_start_pfn < zone_movable_pfn[nid] && 1124 *zone_end_pfn > zone_movable_pfn[nid]) { 1125 *zone_end_pfn = zone_movable_pfn[nid]; 1126 1127 /* Check if this whole range is within ZONE_MOVABLE */ 1128 } else if (*zone_start_pfn >= zone_movable_pfn[nid]) 1129 *zone_start_pfn = *zone_end_pfn; 1130 } 1131 } 1132 1133 /* 1134 * Return the number of holes in a range on a node. If nid is MAX_NUMNODES, 1135 * then all holes in the requested range will be accounted for. 1136 */ 1137 unsigned long __init __absent_pages_in_range(int nid, 1138 unsigned long range_start_pfn, 1139 unsigned long range_end_pfn) 1140 { 1141 unsigned long nr_absent = range_end_pfn - range_start_pfn; 1142 unsigned long start_pfn, end_pfn; 1143 int i; 1144 1145 for_each_mem_pfn_range(i, nid, &start_pfn, &end_pfn, NULL) { 1146 start_pfn = clamp(start_pfn, range_start_pfn, range_end_pfn); 1147 end_pfn = clamp(end_pfn, range_start_pfn, range_end_pfn); 1148 nr_absent -= end_pfn - start_pfn; 1149 } 1150 return nr_absent; 1151 } 1152 1153 /** 1154 * absent_pages_in_range - Return number of page frames in holes within a range 1155 * @start_pfn: The start PFN to start searching for holes 1156 * @end_pfn: The end PFN to stop searching for holes 1157 * 1158 * Return: the number of pages frames in memory holes within a range. 1159 */ 1160 unsigned long __init absent_pages_in_range(unsigned long start_pfn, 1161 unsigned long end_pfn) 1162 { 1163 return __absent_pages_in_range(MAX_NUMNODES, start_pfn, end_pfn); 1164 } 1165 1166 /* Return the number of page frames in holes in a zone on a node */ 1167 static unsigned long __init zone_absent_pages_in_node(int nid, 1168 unsigned long zone_type, 1169 unsigned long zone_start_pfn, 1170 unsigned long zone_end_pfn) 1171 { 1172 unsigned long nr_absent; 1173 1174 /* zone is empty, we don't have any absent pages */ 1175 if (zone_start_pfn == zone_end_pfn) 1176 return 0; 1177 1178 nr_absent = __absent_pages_in_range(nid, zone_start_pfn, zone_end_pfn); 1179 1180 /* 1181 * ZONE_MOVABLE handling. 1182 * Treat pages to be ZONE_MOVABLE in ZONE_NORMAL as absent pages 1183 * and vice versa. 1184 */ 1185 if (mirrored_kernelcore && zone_movable_pfn[nid]) { 1186 unsigned long start_pfn, end_pfn; 1187 struct memblock_region *r; 1188 1189 for_each_mem_region(r) { 1190 start_pfn = clamp(memblock_region_memory_base_pfn(r), 1191 zone_start_pfn, zone_end_pfn); 1192 end_pfn = clamp(memblock_region_memory_end_pfn(r), 1193 zone_start_pfn, zone_end_pfn); 1194 1195 if (zone_type == ZONE_MOVABLE && 1196 memblock_is_mirror(r)) 1197 nr_absent += end_pfn - start_pfn; 1198 1199 if (zone_type == ZONE_NORMAL && 1200 !memblock_is_mirror(r)) 1201 nr_absent += end_pfn - start_pfn; 1202 } 1203 } 1204 1205 return nr_absent; 1206 } 1207 1208 /* 1209 * Return the number of pages a zone spans in a node, including holes 1210 * present_pages = zone_spanned_pages_in_node() - zone_absent_pages_in_node() 1211 */ 1212 static unsigned long __init zone_spanned_pages_in_node(int nid, 1213 unsigned long zone_type, 1214 unsigned long node_start_pfn, 1215 unsigned long node_end_pfn, 1216 unsigned long *zone_start_pfn, 1217 unsigned long *zone_end_pfn) 1218 { 1219 unsigned long zone_low = arch_zone_lowest_possible_pfn[zone_type]; 1220 unsigned long zone_high = arch_zone_highest_possible_pfn[zone_type]; 1221 1222 /* Get the start and end of the zone */ 1223 *zone_start_pfn = clamp(node_start_pfn, zone_low, zone_high); 1224 *zone_end_pfn = clamp(node_end_pfn, zone_low, zone_high); 1225 adjust_zone_range_for_zone_movable(nid, zone_type, 1226 node_start_pfn, node_end_pfn, 1227 zone_start_pfn, zone_end_pfn); 1228 1229 /* Check that this node has pages within the zone's required range */ 1230 if (*zone_end_pfn < node_start_pfn || *zone_start_pfn > node_end_pfn) 1231 return 0; 1232 1233 /* Move the zone boundaries inside the node if necessary */ 1234 *zone_end_pfn = min(*zone_end_pfn, node_end_pfn); 1235 *zone_start_pfn = max(*zone_start_pfn, node_start_pfn); 1236 1237 /* Return the spanned pages */ 1238 return *zone_end_pfn - *zone_start_pfn; 1239 } 1240 1241 static void __init reset_memoryless_node_totalpages(struct pglist_data *pgdat) 1242 { 1243 struct zone *z; 1244 1245 for (z = pgdat->node_zones; z < pgdat->node_zones + MAX_NR_ZONES; z++) { 1246 z->zone_start_pfn = 0; 1247 z->spanned_pages = 0; 1248 z->present_pages = 0; 1249 #if defined(CONFIG_MEMORY_HOTPLUG) 1250 z->present_early_pages = 0; 1251 #endif 1252 } 1253 1254 pgdat->node_spanned_pages = 0; 1255 pgdat->node_present_pages = 0; 1256 pr_debug("On node %d totalpages: 0\n", pgdat->node_id); 1257 } 1258 1259 static void __init calculate_node_totalpages(struct pglist_data *pgdat, 1260 unsigned long node_start_pfn, 1261 unsigned long node_end_pfn) 1262 { 1263 unsigned long realtotalpages = 0, totalpages = 0; 1264 enum zone_type i; 1265 1266 for (i = 0; i < MAX_NR_ZONES; i++) { 1267 struct zone *zone = pgdat->node_zones + i; 1268 unsigned long zone_start_pfn, zone_end_pfn; 1269 unsigned long spanned, absent; 1270 unsigned long real_size; 1271 1272 spanned = zone_spanned_pages_in_node(pgdat->node_id, i, 1273 node_start_pfn, 1274 node_end_pfn, 1275 &zone_start_pfn, 1276 &zone_end_pfn); 1277 absent = zone_absent_pages_in_node(pgdat->node_id, i, 1278 zone_start_pfn, 1279 zone_end_pfn); 1280 1281 real_size = spanned - absent; 1282 1283 if (spanned) 1284 zone->zone_start_pfn = zone_start_pfn; 1285 else 1286 zone->zone_start_pfn = 0; 1287 zone->spanned_pages = spanned; 1288 zone->present_pages = real_size; 1289 #if defined(CONFIG_MEMORY_HOTPLUG) 1290 zone->present_early_pages = real_size; 1291 #endif 1292 1293 totalpages += spanned; 1294 realtotalpages += real_size; 1295 } 1296 1297 pgdat->node_spanned_pages = totalpages; 1298 pgdat->node_present_pages = realtotalpages; 1299 pr_debug("On node %d totalpages: %lu\n", pgdat->node_id, realtotalpages); 1300 } 1301 1302 static unsigned long __init calc_memmap_size(unsigned long spanned_pages, 1303 unsigned long present_pages) 1304 { 1305 unsigned long pages = spanned_pages; 1306 1307 /* 1308 * Provide a more accurate estimation if there are holes within 1309 * the zone and SPARSEMEM is in use. If there are holes within the 1310 * zone, each populated memory region may cost us one or two extra 1311 * memmap pages due to alignment because memmap pages for each 1312 * populated regions may not be naturally aligned on page boundary. 1313 * So the (present_pages >> 4) heuristic is a tradeoff for that. 1314 */ 1315 if (spanned_pages > present_pages + (present_pages >> 4) && 1316 IS_ENABLED(CONFIG_SPARSEMEM)) 1317 pages = present_pages; 1318 1319 return PAGE_ALIGN(pages * sizeof(struct page)) >> PAGE_SHIFT; 1320 } 1321 1322 #ifdef CONFIG_TRANSPARENT_HUGEPAGE 1323 static void pgdat_init_split_queue(struct pglist_data *pgdat) 1324 { 1325 struct deferred_split *ds_queue = &pgdat->deferred_split_queue; 1326 1327 spin_lock_init(&ds_queue->split_queue_lock); 1328 INIT_LIST_HEAD(&ds_queue->split_queue); 1329 ds_queue->split_queue_len = 0; 1330 } 1331 #else 1332 static void pgdat_init_split_queue(struct pglist_data *pgdat) {} 1333 #endif 1334 1335 #ifdef CONFIG_COMPACTION 1336 static void pgdat_init_kcompactd(struct pglist_data *pgdat) 1337 { 1338 init_waitqueue_head(&pgdat->kcompactd_wait); 1339 } 1340 #else 1341 static void pgdat_init_kcompactd(struct pglist_data *pgdat) {} 1342 #endif 1343 1344 static void __meminit pgdat_init_internals(struct pglist_data *pgdat) 1345 { 1346 int i; 1347 1348 pgdat_resize_init(pgdat); 1349 pgdat_kswapd_lock_init(pgdat); 1350 1351 pgdat_init_split_queue(pgdat); 1352 pgdat_init_kcompactd(pgdat); 1353 1354 init_waitqueue_head(&pgdat->kswapd_wait); 1355 init_waitqueue_head(&pgdat->pfmemalloc_wait); 1356 1357 for (i = 0; i < NR_VMSCAN_THROTTLE; i++) 1358 init_waitqueue_head(&pgdat->reclaim_wait[i]); 1359 1360 pgdat_page_ext_init(pgdat); 1361 lruvec_init(&pgdat->__lruvec); 1362 } 1363 1364 static void __meminit zone_init_internals(struct zone *zone, enum zone_type idx, int nid, 1365 unsigned long remaining_pages) 1366 { 1367 atomic_long_set(&zone->managed_pages, remaining_pages); 1368 zone_set_nid(zone, nid); 1369 zone->name = zone_names[idx]; 1370 zone->zone_pgdat = NODE_DATA(nid); 1371 spin_lock_init(&zone->lock); 1372 zone_seqlock_init(zone); 1373 zone_pcp_init(zone); 1374 } 1375 1376 static void __meminit zone_init_free_lists(struct zone *zone) 1377 { 1378 unsigned int order, t; 1379 for_each_migratetype_order(order, t) { 1380 INIT_LIST_HEAD(&zone->free_area[order].free_list[t]); 1381 zone->free_area[order].nr_free = 0; 1382 } 1383 } 1384 1385 void __meminit init_currently_empty_zone(struct zone *zone, 1386 unsigned long zone_start_pfn, 1387 unsigned long size) 1388 { 1389 struct pglist_data *pgdat = zone->zone_pgdat; 1390 int zone_idx = zone_idx(zone) + 1; 1391 1392 if (zone_idx > pgdat->nr_zones) 1393 pgdat->nr_zones = zone_idx; 1394 1395 zone->zone_start_pfn = zone_start_pfn; 1396 1397 mminit_dprintk(MMINIT_TRACE, "memmap_init", 1398 "Initialising map node %d zone %lu pfns %lu -> %lu\n", 1399 pgdat->node_id, 1400 (unsigned long)zone_idx(zone), 1401 zone_start_pfn, (zone_start_pfn + size)); 1402 1403 zone_init_free_lists(zone); 1404 zone->initialized = 1; 1405 } 1406 1407 #ifndef CONFIG_SPARSEMEM 1408 /* 1409 * Calculate the size of the zone->blockflags rounded to an unsigned long 1410 * Start by making sure zonesize is a multiple of pageblock_order by rounding 1411 * up. Then use 1 NR_PAGEBLOCK_BITS worth of bits per pageblock, finally 1412 * round what is now in bits to nearest long in bits, then return it in 1413 * bytes. 1414 */ 1415 static unsigned long __init usemap_size(unsigned long zone_start_pfn, unsigned long zonesize) 1416 { 1417 unsigned long usemapsize; 1418 1419 zonesize += zone_start_pfn & (pageblock_nr_pages-1); 1420 usemapsize = roundup(zonesize, pageblock_nr_pages); 1421 usemapsize = usemapsize >> pageblock_order; 1422 usemapsize *= NR_PAGEBLOCK_BITS; 1423 usemapsize = roundup(usemapsize, 8 * sizeof(unsigned long)); 1424 1425 return usemapsize / 8; 1426 } 1427 1428 static void __ref setup_usemap(struct zone *zone) 1429 { 1430 unsigned long usemapsize = usemap_size(zone->zone_start_pfn, 1431 zone->spanned_pages); 1432 zone->pageblock_flags = NULL; 1433 if (usemapsize) { 1434 zone->pageblock_flags = 1435 memblock_alloc_node(usemapsize, SMP_CACHE_BYTES, 1436 zone_to_nid(zone)); 1437 if (!zone->pageblock_flags) 1438 panic("Failed to allocate %ld bytes for zone %s pageblock flags on node %d\n", 1439 usemapsize, zone->name, zone_to_nid(zone)); 1440 } 1441 } 1442 #else 1443 static inline void setup_usemap(struct zone *zone) {} 1444 #endif /* CONFIG_SPARSEMEM */ 1445 1446 #ifdef CONFIG_HUGETLB_PAGE_SIZE_VARIABLE 1447 1448 /* Initialise the number of pages represented by NR_PAGEBLOCK_BITS */ 1449 void __init set_pageblock_order(void) 1450 { 1451 unsigned int order = MAX_ORDER; 1452 1453 /* Check that pageblock_nr_pages has not already been setup */ 1454 if (pageblock_order) 1455 return; 1456 1457 /* Don't let pageblocks exceed the maximum allocation granularity. */ 1458 if (HPAGE_SHIFT > PAGE_SHIFT && HUGETLB_PAGE_ORDER < order) 1459 order = HUGETLB_PAGE_ORDER; 1460 1461 /* 1462 * Assume the largest contiguous order of interest is a huge page. 1463 * This value may be variable depending on boot parameters on IA64 and 1464 * powerpc. 1465 */ 1466 pageblock_order = order; 1467 } 1468 #else /* CONFIG_HUGETLB_PAGE_SIZE_VARIABLE */ 1469 1470 /* 1471 * When CONFIG_HUGETLB_PAGE_SIZE_VARIABLE is not set, set_pageblock_order() 1472 * is unused as pageblock_order is set at compile-time. See 1473 * include/linux/pageblock-flags.h for the values of pageblock_order based on 1474 * the kernel config 1475 */ 1476 void __init set_pageblock_order(void) 1477 { 1478 } 1479 1480 #endif /* CONFIG_HUGETLB_PAGE_SIZE_VARIABLE */ 1481 1482 /* 1483 * Set up the zone data structures 1484 * - init pgdat internals 1485 * - init all zones belonging to this node 1486 * 1487 * NOTE: this function is only called during memory hotplug 1488 */ 1489 #ifdef CONFIG_MEMORY_HOTPLUG 1490 void __ref free_area_init_core_hotplug(struct pglist_data *pgdat) 1491 { 1492 int nid = pgdat->node_id; 1493 enum zone_type z; 1494 int cpu; 1495 1496 pgdat_init_internals(pgdat); 1497 1498 if (pgdat->per_cpu_nodestats == &boot_nodestats) 1499 pgdat->per_cpu_nodestats = alloc_percpu(struct per_cpu_nodestat); 1500 1501 /* 1502 * Reset the nr_zones, order and highest_zoneidx before reuse. 1503 * Note that kswapd will init kswapd_highest_zoneidx properly 1504 * when it starts in the near future. 1505 */ 1506 pgdat->nr_zones = 0; 1507 pgdat->kswapd_order = 0; 1508 pgdat->kswapd_highest_zoneidx = 0; 1509 pgdat->node_start_pfn = 0; 1510 pgdat->node_present_pages = 0; 1511 1512 for_each_online_cpu(cpu) { 1513 struct per_cpu_nodestat *p; 1514 1515 p = per_cpu_ptr(pgdat->per_cpu_nodestats, cpu); 1516 memset(p, 0, sizeof(*p)); 1517 } 1518 1519 /* 1520 * When memory is hot-added, all the memory is in offline state. So 1521 * clear all zones' present_pages and managed_pages because they will 1522 * be updated in online_pages() and offline_pages(). 1523 */ 1524 for (z = 0; z < MAX_NR_ZONES; z++) { 1525 struct zone *zone = pgdat->node_zones + z; 1526 1527 zone->present_pages = 0; 1528 zone_init_internals(zone, z, nid, 0); 1529 } 1530 } 1531 #endif 1532 1533 /* 1534 * Set up the zone data structures: 1535 * - mark all pages reserved 1536 * - mark all memory queues empty 1537 * - clear the memory bitmaps 1538 * 1539 * NOTE: pgdat should get zeroed by caller. 1540 * NOTE: this function is only called during early init. 1541 */ 1542 static void __init free_area_init_core(struct pglist_data *pgdat) 1543 { 1544 enum zone_type j; 1545 int nid = pgdat->node_id; 1546 1547 pgdat_init_internals(pgdat); 1548 pgdat->per_cpu_nodestats = &boot_nodestats; 1549 1550 for (j = 0; j < MAX_NR_ZONES; j++) { 1551 struct zone *zone = pgdat->node_zones + j; 1552 unsigned long size, freesize, memmap_pages; 1553 1554 size = zone->spanned_pages; 1555 freesize = zone->present_pages; 1556 1557 /* 1558 * Adjust freesize so that it accounts for how much memory 1559 * is used by this zone for memmap. This affects the watermark 1560 * and per-cpu initialisations 1561 */ 1562 memmap_pages = calc_memmap_size(size, freesize); 1563 if (!is_highmem_idx(j)) { 1564 if (freesize >= memmap_pages) { 1565 freesize -= memmap_pages; 1566 if (memmap_pages) 1567 pr_debug(" %s zone: %lu pages used for memmap\n", 1568 zone_names[j], memmap_pages); 1569 } else 1570 pr_warn(" %s zone: %lu memmap pages exceeds freesize %lu\n", 1571 zone_names[j], memmap_pages, freesize); 1572 } 1573 1574 /* Account for reserved pages */ 1575 if (j == 0 && freesize > dma_reserve) { 1576 freesize -= dma_reserve; 1577 pr_debug(" %s zone: %lu pages reserved\n", zone_names[0], dma_reserve); 1578 } 1579 1580 if (!is_highmem_idx(j)) 1581 nr_kernel_pages += freesize; 1582 /* Charge for highmem memmap if there are enough kernel pages */ 1583 else if (nr_kernel_pages > memmap_pages * 2) 1584 nr_kernel_pages -= memmap_pages; 1585 nr_all_pages += freesize; 1586 1587 /* 1588 * Set an approximate value for lowmem here, it will be adjusted 1589 * when the bootmem allocator frees pages into the buddy system. 1590 * And all highmem pages will be managed by the buddy system. 1591 */ 1592 zone_init_internals(zone, j, nid, freesize); 1593 1594 if (!size) 1595 continue; 1596 1597 setup_usemap(zone); 1598 init_currently_empty_zone(zone, zone->zone_start_pfn, size); 1599 } 1600 } 1601 1602 void __init *memmap_alloc(phys_addr_t size, phys_addr_t align, 1603 phys_addr_t min_addr, int nid, bool exact_nid) 1604 { 1605 void *ptr; 1606 1607 if (exact_nid) 1608 ptr = memblock_alloc_exact_nid_raw(size, align, min_addr, 1609 MEMBLOCK_ALLOC_ACCESSIBLE, 1610 nid); 1611 else 1612 ptr = memblock_alloc_try_nid_raw(size, align, min_addr, 1613 MEMBLOCK_ALLOC_ACCESSIBLE, 1614 nid); 1615 1616 if (ptr && size > 0) 1617 page_init_poison(ptr, size); 1618 1619 return ptr; 1620 } 1621 1622 #ifdef CONFIG_FLATMEM 1623 static void __init alloc_node_mem_map(struct pglist_data *pgdat) 1624 { 1625 unsigned long __maybe_unused start = 0; 1626 unsigned long __maybe_unused offset = 0; 1627 1628 /* Skip empty nodes */ 1629 if (!pgdat->node_spanned_pages) 1630 return; 1631 1632 start = pgdat->node_start_pfn & ~(MAX_ORDER_NR_PAGES - 1); 1633 offset = pgdat->node_start_pfn - start; 1634 /* ia64 gets its own node_mem_map, before this, without bootmem */ 1635 if (!pgdat->node_mem_map) { 1636 unsigned long size, end; 1637 struct page *map; 1638 1639 /* 1640 * The zone's endpoints aren't required to be MAX_ORDER 1641 * aligned but the node_mem_map endpoints must be in order 1642 * for the buddy allocator to function correctly. 1643 */ 1644 end = pgdat_end_pfn(pgdat); 1645 end = ALIGN(end, MAX_ORDER_NR_PAGES); 1646 size = (end - start) * sizeof(struct page); 1647 map = memmap_alloc(size, SMP_CACHE_BYTES, MEMBLOCK_LOW_LIMIT, 1648 pgdat->node_id, false); 1649 if (!map) 1650 panic("Failed to allocate %ld bytes for node %d memory map\n", 1651 size, pgdat->node_id); 1652 pgdat->node_mem_map = map + offset; 1653 } 1654 pr_debug("%s: node %d, pgdat %08lx, node_mem_map %08lx\n", 1655 __func__, pgdat->node_id, (unsigned long)pgdat, 1656 (unsigned long)pgdat->node_mem_map); 1657 #ifndef CONFIG_NUMA 1658 /* 1659 * With no DISCONTIG, the global mem_map is just set as node 0's 1660 */ 1661 if (pgdat == NODE_DATA(0)) { 1662 mem_map = NODE_DATA(0)->node_mem_map; 1663 if (page_to_pfn(mem_map) != pgdat->node_start_pfn) 1664 mem_map -= offset; 1665 } 1666 #endif 1667 } 1668 #else 1669 static inline void alloc_node_mem_map(struct pglist_data *pgdat) { } 1670 #endif /* CONFIG_FLATMEM */ 1671 1672 /** 1673 * get_pfn_range_for_nid - Return the start and end page frames for a node 1674 * @nid: The nid to return the range for. If MAX_NUMNODES, the min and max PFN are returned. 1675 * @start_pfn: Passed by reference. On return, it will have the node start_pfn. 1676 * @end_pfn: Passed by reference. On return, it will have the node end_pfn. 1677 * 1678 * It returns the start and end page frame of a node based on information 1679 * provided by memblock_set_node(). If called for a node 1680 * with no available memory, a warning is printed and the start and end 1681 * PFNs will be 0. 1682 */ 1683 void __init get_pfn_range_for_nid(unsigned int nid, 1684 unsigned long *start_pfn, unsigned long *end_pfn) 1685 { 1686 unsigned long this_start_pfn, this_end_pfn; 1687 int i; 1688 1689 *start_pfn = -1UL; 1690 *end_pfn = 0; 1691 1692 for_each_mem_pfn_range(i, nid, &this_start_pfn, &this_end_pfn, NULL) { 1693 *start_pfn = min(*start_pfn, this_start_pfn); 1694 *end_pfn = max(*end_pfn, this_end_pfn); 1695 } 1696 1697 if (*start_pfn == -1UL) 1698 *start_pfn = 0; 1699 } 1700 1701 static void __init free_area_init_node(int nid) 1702 { 1703 pg_data_t *pgdat = NODE_DATA(nid); 1704 unsigned long start_pfn = 0; 1705 unsigned long end_pfn = 0; 1706 1707 /* pg_data_t should be reset to zero when it's allocated */ 1708 WARN_ON(pgdat->nr_zones || pgdat->kswapd_highest_zoneidx); 1709 1710 get_pfn_range_for_nid(nid, &start_pfn, &end_pfn); 1711 1712 pgdat->node_id = nid; 1713 pgdat->node_start_pfn = start_pfn; 1714 pgdat->per_cpu_nodestats = NULL; 1715 1716 if (start_pfn != end_pfn) { 1717 pr_info("Initmem setup node %d [mem %#018Lx-%#018Lx]\n", nid, 1718 (u64)start_pfn << PAGE_SHIFT, 1719 end_pfn ? ((u64)end_pfn << PAGE_SHIFT) - 1 : 0); 1720 1721 calculate_node_totalpages(pgdat, start_pfn, end_pfn); 1722 } else { 1723 pr_info("Initmem setup node %d as memoryless\n", nid); 1724 1725 reset_memoryless_node_totalpages(pgdat); 1726 } 1727 1728 alloc_node_mem_map(pgdat); 1729 pgdat_set_deferred_range(pgdat); 1730 1731 free_area_init_core(pgdat); 1732 lru_gen_init_pgdat(pgdat); 1733 } 1734 1735 /* Any regular or high memory on that node ? */ 1736 static void check_for_memory(pg_data_t *pgdat) 1737 { 1738 enum zone_type zone_type; 1739 1740 for (zone_type = 0; zone_type <= ZONE_MOVABLE - 1; zone_type++) { 1741 struct zone *zone = &pgdat->node_zones[zone_type]; 1742 if (populated_zone(zone)) { 1743 if (IS_ENABLED(CONFIG_HIGHMEM)) 1744 node_set_state(pgdat->node_id, N_HIGH_MEMORY); 1745 if (zone_type <= ZONE_NORMAL) 1746 node_set_state(pgdat->node_id, N_NORMAL_MEMORY); 1747 break; 1748 } 1749 } 1750 } 1751 1752 #if MAX_NUMNODES > 1 1753 /* 1754 * Figure out the number of possible node ids. 1755 */ 1756 void __init setup_nr_node_ids(void) 1757 { 1758 unsigned int highest; 1759 1760 highest = find_last_bit(node_possible_map.bits, MAX_NUMNODES); 1761 nr_node_ids = highest + 1; 1762 } 1763 #endif 1764 1765 /* 1766 * Some architectures, e.g. ARC may have ZONE_HIGHMEM below ZONE_NORMAL. For 1767 * such cases we allow max_zone_pfn sorted in the descending order 1768 */ 1769 static bool arch_has_descending_max_zone_pfns(void) 1770 { 1771 return IS_ENABLED(CONFIG_ARC) && !IS_ENABLED(CONFIG_ARC_HAS_PAE40); 1772 } 1773 1774 /** 1775 * free_area_init - Initialise all pg_data_t and zone data 1776 * @max_zone_pfn: an array of max PFNs for each zone 1777 * 1778 * This will call free_area_init_node() for each active node in the system. 1779 * Using the page ranges provided by memblock_set_node(), the size of each 1780 * zone in each node and their holes is calculated. If the maximum PFN 1781 * between two adjacent zones match, it is assumed that the zone is empty. 1782 * For example, if arch_max_dma_pfn == arch_max_dma32_pfn, it is assumed 1783 * that arch_max_dma32_pfn has no pages. It is also assumed that a zone 1784 * starts where the previous one ended. For example, ZONE_DMA32 starts 1785 * at arch_max_dma_pfn. 1786 */ 1787 void __init free_area_init(unsigned long *max_zone_pfn) 1788 { 1789 unsigned long start_pfn, end_pfn; 1790 int i, nid, zone; 1791 bool descending; 1792 1793 /* Record where the zone boundaries are */ 1794 memset(arch_zone_lowest_possible_pfn, 0, 1795 sizeof(arch_zone_lowest_possible_pfn)); 1796 memset(arch_zone_highest_possible_pfn, 0, 1797 sizeof(arch_zone_highest_possible_pfn)); 1798 1799 start_pfn = PHYS_PFN(memblock_start_of_DRAM()); 1800 descending = arch_has_descending_max_zone_pfns(); 1801 1802 for (i = 0; i < MAX_NR_ZONES; i++) { 1803 if (descending) 1804 zone = MAX_NR_ZONES - i - 1; 1805 else 1806 zone = i; 1807 1808 if (zone == ZONE_MOVABLE) 1809 continue; 1810 1811 end_pfn = max(max_zone_pfn[zone], start_pfn); 1812 arch_zone_lowest_possible_pfn[zone] = start_pfn; 1813 arch_zone_highest_possible_pfn[zone] = end_pfn; 1814 1815 start_pfn = end_pfn; 1816 } 1817 1818 /* Find the PFNs that ZONE_MOVABLE begins at in each node */ 1819 memset(zone_movable_pfn, 0, sizeof(zone_movable_pfn)); 1820 find_zone_movable_pfns_for_nodes(); 1821 1822 /* Print out the zone ranges */ 1823 pr_info("Zone ranges:\n"); 1824 for (i = 0; i < MAX_NR_ZONES; i++) { 1825 if (i == ZONE_MOVABLE) 1826 continue; 1827 pr_info(" %-8s ", zone_names[i]); 1828 if (arch_zone_lowest_possible_pfn[i] == 1829 arch_zone_highest_possible_pfn[i]) 1830 pr_cont("empty\n"); 1831 else 1832 pr_cont("[mem %#018Lx-%#018Lx]\n", 1833 (u64)arch_zone_lowest_possible_pfn[i] 1834 << PAGE_SHIFT, 1835 ((u64)arch_zone_highest_possible_pfn[i] 1836 << PAGE_SHIFT) - 1); 1837 } 1838 1839 /* Print out the PFNs ZONE_MOVABLE begins at in each node */ 1840 pr_info("Movable zone start for each node\n"); 1841 for (i = 0; i < MAX_NUMNODES; i++) { 1842 if (zone_movable_pfn[i]) 1843 pr_info(" Node %d: %#018Lx\n", i, 1844 (u64)zone_movable_pfn[i] << PAGE_SHIFT); 1845 } 1846 1847 /* 1848 * Print out the early node map, and initialize the 1849 * subsection-map relative to active online memory ranges to 1850 * enable future "sub-section" extensions of the memory map. 1851 */ 1852 pr_info("Early memory node ranges\n"); 1853 for_each_mem_pfn_range(i, MAX_NUMNODES, &start_pfn, &end_pfn, &nid) { 1854 pr_info(" node %3d: [mem %#018Lx-%#018Lx]\n", nid, 1855 (u64)start_pfn << PAGE_SHIFT, 1856 ((u64)end_pfn << PAGE_SHIFT) - 1); 1857 subsection_map_init(start_pfn, end_pfn - start_pfn); 1858 } 1859 1860 /* Initialise every node */ 1861 mminit_verify_pageflags_layout(); 1862 setup_nr_node_ids(); 1863 set_pageblock_order(); 1864 1865 for_each_node(nid) { 1866 pg_data_t *pgdat; 1867 1868 if (!node_online(nid)) { 1869 pr_info("Initializing node %d as memoryless\n", nid); 1870 1871 /* Allocator not initialized yet */ 1872 pgdat = arch_alloc_nodedata(nid); 1873 if (!pgdat) 1874 panic("Cannot allocate %zuB for node %d.\n", 1875 sizeof(*pgdat), nid); 1876 arch_refresh_nodedata(nid, pgdat); 1877 free_area_init_node(nid); 1878 1879 /* 1880 * We do not want to confuse userspace by sysfs 1881 * files/directories for node without any memory 1882 * attached to it, so this node is not marked as 1883 * N_MEMORY and not marked online so that no sysfs 1884 * hierarchy will be created via register_one_node for 1885 * it. The pgdat will get fully initialized by 1886 * hotadd_init_pgdat() when memory is hotplugged into 1887 * this node. 1888 */ 1889 continue; 1890 } 1891 1892 pgdat = NODE_DATA(nid); 1893 free_area_init_node(nid); 1894 1895 /* Any memory on that node */ 1896 if (pgdat->node_present_pages) 1897 node_set_state(nid, N_MEMORY); 1898 check_for_memory(pgdat); 1899 } 1900 1901 memmap_init(); 1902 1903 /* disable hash distribution for systems with a single node */ 1904 fixup_hashdist(); 1905 } 1906 1907 /** 1908 * node_map_pfn_alignment - determine the maximum internode alignment 1909 * 1910 * This function should be called after node map is populated and sorted. 1911 * It calculates the maximum power of two alignment which can distinguish 1912 * all the nodes. 1913 * 1914 * For example, if all nodes are 1GiB and aligned to 1GiB, the return value 1915 * would indicate 1GiB alignment with (1 << (30 - PAGE_SHIFT)). If the 1916 * nodes are shifted by 256MiB, 256MiB. Note that if only the last node is 1917 * shifted, 1GiB is enough and this function will indicate so. 1918 * 1919 * This is used to test whether pfn -> nid mapping of the chosen memory 1920 * model has fine enough granularity to avoid incorrect mapping for the 1921 * populated node map. 1922 * 1923 * Return: the determined alignment in pfn's. 0 if there is no alignment 1924 * requirement (single node). 1925 */ 1926 unsigned long __init node_map_pfn_alignment(void) 1927 { 1928 unsigned long accl_mask = 0, last_end = 0; 1929 unsigned long start, end, mask; 1930 int last_nid = NUMA_NO_NODE; 1931 int i, nid; 1932 1933 for_each_mem_pfn_range(i, MAX_NUMNODES, &start, &end, &nid) { 1934 if (!start || last_nid < 0 || last_nid == nid) { 1935 last_nid = nid; 1936 last_end = end; 1937 continue; 1938 } 1939 1940 /* 1941 * Start with a mask granular enough to pin-point to the 1942 * start pfn and tick off bits one-by-one until it becomes 1943 * too coarse to separate the current node from the last. 1944 */ 1945 mask = ~((1 << __ffs(start)) - 1); 1946 while (mask && last_end <= (start & (mask << 1))) 1947 mask <<= 1; 1948 1949 /* accumulate all internode masks */ 1950 accl_mask |= mask; 1951 } 1952 1953 /* convert mask to number of pages */ 1954 return ~accl_mask + 1; 1955 } 1956 1957 #ifdef CONFIG_DEFERRED_STRUCT_PAGE_INIT 1958 static void __init deferred_free_range(unsigned long pfn, 1959 unsigned long nr_pages) 1960 { 1961 struct page *page; 1962 unsigned long i; 1963 1964 if (!nr_pages) 1965 return; 1966 1967 page = pfn_to_page(pfn); 1968 1969 /* Free a large naturally-aligned chunk if possible */ 1970 if (nr_pages == MAX_ORDER_NR_PAGES && IS_MAX_ORDER_ALIGNED(pfn)) { 1971 for (i = 0; i < nr_pages; i += pageblock_nr_pages) 1972 set_pageblock_migratetype(page + i, MIGRATE_MOVABLE); 1973 __free_pages_core(page, MAX_ORDER); 1974 return; 1975 } 1976 1977 for (i = 0; i < nr_pages; i++, page++, pfn++) { 1978 if (pageblock_aligned(pfn)) 1979 set_pageblock_migratetype(page, MIGRATE_MOVABLE); 1980 __free_pages_core(page, 0); 1981 } 1982 } 1983 1984 /* Completion tracking for deferred_init_memmap() threads */ 1985 static atomic_t pgdat_init_n_undone __initdata; 1986 static __initdata DECLARE_COMPLETION(pgdat_init_all_done_comp); 1987 1988 static inline void __init pgdat_init_report_one_done(void) 1989 { 1990 if (atomic_dec_and_test(&pgdat_init_n_undone)) 1991 complete(&pgdat_init_all_done_comp); 1992 } 1993 1994 /* 1995 * Returns true if page needs to be initialized or freed to buddy allocator. 1996 * 1997 * We check if a current MAX_ORDER block is valid by only checking the validity 1998 * of the head pfn. 1999 */ 2000 static inline bool __init deferred_pfn_valid(unsigned long pfn) 2001 { 2002 if (IS_MAX_ORDER_ALIGNED(pfn) && !pfn_valid(pfn)) 2003 return false; 2004 return true; 2005 } 2006 2007 /* 2008 * Free pages to buddy allocator. Try to free aligned pages in 2009 * MAX_ORDER_NR_PAGES sizes. 2010 */ 2011 static void __init deferred_free_pages(unsigned long pfn, 2012 unsigned long end_pfn) 2013 { 2014 unsigned long nr_free = 0; 2015 2016 for (; pfn < end_pfn; pfn++) { 2017 if (!deferred_pfn_valid(pfn)) { 2018 deferred_free_range(pfn - nr_free, nr_free); 2019 nr_free = 0; 2020 } else if (IS_MAX_ORDER_ALIGNED(pfn)) { 2021 deferred_free_range(pfn - nr_free, nr_free); 2022 nr_free = 1; 2023 } else { 2024 nr_free++; 2025 } 2026 } 2027 /* Free the last block of pages to allocator */ 2028 deferred_free_range(pfn - nr_free, nr_free); 2029 } 2030 2031 /* 2032 * Initialize struct pages. We minimize pfn page lookups and scheduler checks 2033 * by performing it only once every MAX_ORDER_NR_PAGES. 2034 * Return number of pages initialized. 2035 */ 2036 static unsigned long __init deferred_init_pages(struct zone *zone, 2037 unsigned long pfn, 2038 unsigned long end_pfn) 2039 { 2040 int nid = zone_to_nid(zone); 2041 unsigned long nr_pages = 0; 2042 int zid = zone_idx(zone); 2043 struct page *page = NULL; 2044 2045 for (; pfn < end_pfn; pfn++) { 2046 if (!deferred_pfn_valid(pfn)) { 2047 page = NULL; 2048 continue; 2049 } else if (!page || IS_MAX_ORDER_ALIGNED(pfn)) { 2050 page = pfn_to_page(pfn); 2051 } else { 2052 page++; 2053 } 2054 __init_single_page(page, pfn, zid, nid); 2055 nr_pages++; 2056 } 2057 return (nr_pages); 2058 } 2059 2060 /* 2061 * This function is meant to pre-load the iterator for the zone init. 2062 * Specifically it walks through the ranges until we are caught up to the 2063 * first_init_pfn value and exits there. If we never encounter the value we 2064 * return false indicating there are no valid ranges left. 2065 */ 2066 static bool __init 2067 deferred_init_mem_pfn_range_in_zone(u64 *i, struct zone *zone, 2068 unsigned long *spfn, unsigned long *epfn, 2069 unsigned long first_init_pfn) 2070 { 2071 u64 j; 2072 2073 /* 2074 * Start out by walking through the ranges in this zone that have 2075 * already been initialized. We don't need to do anything with them 2076 * so we just need to flush them out of the system. 2077 */ 2078 for_each_free_mem_pfn_range_in_zone(j, zone, spfn, epfn) { 2079 if (*epfn <= first_init_pfn) 2080 continue; 2081 if (*spfn < first_init_pfn) 2082 *spfn = first_init_pfn; 2083 *i = j; 2084 return true; 2085 } 2086 2087 return false; 2088 } 2089 2090 /* 2091 * Initialize and free pages. We do it in two loops: first we initialize 2092 * struct page, then free to buddy allocator, because while we are 2093 * freeing pages we can access pages that are ahead (computing buddy 2094 * page in __free_one_page()). 2095 * 2096 * In order to try and keep some memory in the cache we have the loop 2097 * broken along max page order boundaries. This way we will not cause 2098 * any issues with the buddy page computation. 2099 */ 2100 static unsigned long __init 2101 deferred_init_maxorder(u64 *i, struct zone *zone, unsigned long *start_pfn, 2102 unsigned long *end_pfn) 2103 { 2104 unsigned long mo_pfn = ALIGN(*start_pfn + 1, MAX_ORDER_NR_PAGES); 2105 unsigned long spfn = *start_pfn, epfn = *end_pfn; 2106 unsigned long nr_pages = 0; 2107 u64 j = *i; 2108 2109 /* First we loop through and initialize the page values */ 2110 for_each_free_mem_pfn_range_in_zone_from(j, zone, start_pfn, end_pfn) { 2111 unsigned long t; 2112 2113 if (mo_pfn <= *start_pfn) 2114 break; 2115 2116 t = min(mo_pfn, *end_pfn); 2117 nr_pages += deferred_init_pages(zone, *start_pfn, t); 2118 2119 if (mo_pfn < *end_pfn) { 2120 *start_pfn = mo_pfn; 2121 break; 2122 } 2123 } 2124 2125 /* Reset values and now loop through freeing pages as needed */ 2126 swap(j, *i); 2127 2128 for_each_free_mem_pfn_range_in_zone_from(j, zone, &spfn, &epfn) { 2129 unsigned long t; 2130 2131 if (mo_pfn <= spfn) 2132 break; 2133 2134 t = min(mo_pfn, epfn); 2135 deferred_free_pages(spfn, t); 2136 2137 if (mo_pfn <= epfn) 2138 break; 2139 } 2140 2141 return nr_pages; 2142 } 2143 2144 static void __init 2145 deferred_init_memmap_chunk(unsigned long start_pfn, unsigned long end_pfn, 2146 void *arg) 2147 { 2148 unsigned long spfn, epfn; 2149 struct zone *zone = arg; 2150 u64 i; 2151 2152 deferred_init_mem_pfn_range_in_zone(&i, zone, &spfn, &epfn, start_pfn); 2153 2154 /* 2155 * Initialize and free pages in MAX_ORDER sized increments so that we 2156 * can avoid introducing any issues with the buddy allocator. 2157 */ 2158 while (spfn < end_pfn) { 2159 deferred_init_maxorder(&i, zone, &spfn, &epfn); 2160 cond_resched(); 2161 } 2162 } 2163 2164 /* An arch may override for more concurrency. */ 2165 __weak int __init 2166 deferred_page_init_max_threads(const struct cpumask *node_cpumask) 2167 { 2168 return 1; 2169 } 2170 2171 /* Initialise remaining memory on a node */ 2172 static int __init deferred_init_memmap(void *data) 2173 { 2174 pg_data_t *pgdat = data; 2175 const struct cpumask *cpumask = cpumask_of_node(pgdat->node_id); 2176 unsigned long spfn = 0, epfn = 0; 2177 unsigned long first_init_pfn, flags; 2178 unsigned long start = jiffies; 2179 struct zone *zone; 2180 int zid, max_threads; 2181 u64 i; 2182 2183 /* Bind memory initialisation thread to a local node if possible */ 2184 if (!cpumask_empty(cpumask)) 2185 set_cpus_allowed_ptr(current, cpumask); 2186 2187 pgdat_resize_lock(pgdat, &flags); 2188 first_init_pfn = pgdat->first_deferred_pfn; 2189 if (first_init_pfn == ULONG_MAX) { 2190 pgdat_resize_unlock(pgdat, &flags); 2191 pgdat_init_report_one_done(); 2192 return 0; 2193 } 2194 2195 /* Sanity check boundaries */ 2196 BUG_ON(pgdat->first_deferred_pfn < pgdat->node_start_pfn); 2197 BUG_ON(pgdat->first_deferred_pfn > pgdat_end_pfn(pgdat)); 2198 pgdat->first_deferred_pfn = ULONG_MAX; 2199 2200 /* 2201 * Once we unlock here, the zone cannot be grown anymore, thus if an 2202 * interrupt thread must allocate this early in boot, zone must be 2203 * pre-grown prior to start of deferred page initialization. 2204 */ 2205 pgdat_resize_unlock(pgdat, &flags); 2206 2207 /* Only the highest zone is deferred so find it */ 2208 for (zid = 0; zid < MAX_NR_ZONES; zid++) { 2209 zone = pgdat->node_zones + zid; 2210 if (first_init_pfn < zone_end_pfn(zone)) 2211 break; 2212 } 2213 2214 /* If the zone is empty somebody else may have cleared out the zone */ 2215 if (!deferred_init_mem_pfn_range_in_zone(&i, zone, &spfn, &epfn, 2216 first_init_pfn)) 2217 goto zone_empty; 2218 2219 max_threads = deferred_page_init_max_threads(cpumask); 2220 2221 while (spfn < epfn) { 2222 unsigned long epfn_align = ALIGN(epfn, PAGES_PER_SECTION); 2223 struct padata_mt_job job = { 2224 .thread_fn = deferred_init_memmap_chunk, 2225 .fn_arg = zone, 2226 .start = spfn, 2227 .size = epfn_align - spfn, 2228 .align = PAGES_PER_SECTION, 2229 .min_chunk = PAGES_PER_SECTION, 2230 .max_threads = max_threads, 2231 }; 2232 2233 padata_do_multithreaded(&job); 2234 deferred_init_mem_pfn_range_in_zone(&i, zone, &spfn, &epfn, 2235 epfn_align); 2236 } 2237 zone_empty: 2238 /* Sanity check that the next zone really is unpopulated */ 2239 WARN_ON(++zid < MAX_NR_ZONES && populated_zone(++zone)); 2240 2241 pr_info("node %d deferred pages initialised in %ums\n", 2242 pgdat->node_id, jiffies_to_msecs(jiffies - start)); 2243 2244 pgdat_init_report_one_done(); 2245 return 0; 2246 } 2247 2248 /* 2249 * If this zone has deferred pages, try to grow it by initializing enough 2250 * deferred pages to satisfy the allocation specified by order, rounded up to 2251 * the nearest PAGES_PER_SECTION boundary. So we're adding memory in increments 2252 * of SECTION_SIZE bytes by initializing struct pages in increments of 2253 * PAGES_PER_SECTION * sizeof(struct page) bytes. 2254 * 2255 * Return true when zone was grown, otherwise return false. We return true even 2256 * when we grow less than requested, to let the caller decide if there are 2257 * enough pages to satisfy the allocation. 2258 * 2259 * Note: We use noinline because this function is needed only during boot, and 2260 * it is called from a __ref function _deferred_grow_zone. This way we are 2261 * making sure that it is not inlined into permanent text section. 2262 */ 2263 bool __init deferred_grow_zone(struct zone *zone, unsigned int order) 2264 { 2265 unsigned long nr_pages_needed = ALIGN(1 << order, PAGES_PER_SECTION); 2266 pg_data_t *pgdat = zone->zone_pgdat; 2267 unsigned long first_deferred_pfn = pgdat->first_deferred_pfn; 2268 unsigned long spfn, epfn, flags; 2269 unsigned long nr_pages = 0; 2270 u64 i; 2271 2272 /* Only the last zone may have deferred pages */ 2273 if (zone_end_pfn(zone) != pgdat_end_pfn(pgdat)) 2274 return false; 2275 2276 pgdat_resize_lock(pgdat, &flags); 2277 2278 /* 2279 * If someone grew this zone while we were waiting for spinlock, return 2280 * true, as there might be enough pages already. 2281 */ 2282 if (first_deferred_pfn != pgdat->first_deferred_pfn) { 2283 pgdat_resize_unlock(pgdat, &flags); 2284 return true; 2285 } 2286 2287 /* If the zone is empty somebody else may have cleared out the zone */ 2288 if (!deferred_init_mem_pfn_range_in_zone(&i, zone, &spfn, &epfn, 2289 first_deferred_pfn)) { 2290 pgdat->first_deferred_pfn = ULONG_MAX; 2291 pgdat_resize_unlock(pgdat, &flags); 2292 /* Retry only once. */ 2293 return first_deferred_pfn != ULONG_MAX; 2294 } 2295 2296 /* 2297 * Initialize and free pages in MAX_ORDER sized increments so 2298 * that we can avoid introducing any issues with the buddy 2299 * allocator. 2300 */ 2301 while (spfn < epfn) { 2302 /* update our first deferred PFN for this section */ 2303 first_deferred_pfn = spfn; 2304 2305 nr_pages += deferred_init_maxorder(&i, zone, &spfn, &epfn); 2306 touch_nmi_watchdog(); 2307 2308 /* We should only stop along section boundaries */ 2309 if ((first_deferred_pfn ^ spfn) < PAGES_PER_SECTION) 2310 continue; 2311 2312 /* If our quota has been met we can stop here */ 2313 if (nr_pages >= nr_pages_needed) 2314 break; 2315 } 2316 2317 pgdat->first_deferred_pfn = spfn; 2318 pgdat_resize_unlock(pgdat, &flags); 2319 2320 return nr_pages > 0; 2321 } 2322 2323 #endif /* CONFIG_DEFERRED_STRUCT_PAGE_INIT */ 2324 2325 #ifdef CONFIG_CMA 2326 void __init init_cma_reserved_pageblock(struct page *page) 2327 { 2328 unsigned i = pageblock_nr_pages; 2329 struct page *p = page; 2330 2331 do { 2332 __ClearPageReserved(p); 2333 set_page_count(p, 0); 2334 } while (++p, --i); 2335 2336 set_pageblock_migratetype(page, MIGRATE_CMA); 2337 set_page_refcounted(page); 2338 __free_pages(page, pageblock_order); 2339 2340 adjust_managed_page_count(page, pageblock_nr_pages); 2341 page_zone(page)->cma_pages += pageblock_nr_pages; 2342 } 2343 #endif 2344 2345 void set_zone_contiguous(struct zone *zone) 2346 { 2347 unsigned long block_start_pfn = zone->zone_start_pfn; 2348 unsigned long block_end_pfn; 2349 2350 block_end_pfn = pageblock_end_pfn(block_start_pfn); 2351 for (; block_start_pfn < zone_end_pfn(zone); 2352 block_start_pfn = block_end_pfn, 2353 block_end_pfn += pageblock_nr_pages) { 2354 2355 block_end_pfn = min(block_end_pfn, zone_end_pfn(zone)); 2356 2357 if (!__pageblock_pfn_to_page(block_start_pfn, 2358 block_end_pfn, zone)) 2359 return; 2360 cond_resched(); 2361 } 2362 2363 /* We confirm that there is no hole */ 2364 zone->contiguous = true; 2365 } 2366 2367 void __init page_alloc_init_late(void) 2368 { 2369 struct zone *zone; 2370 int nid; 2371 2372 #ifdef CONFIG_DEFERRED_STRUCT_PAGE_INIT 2373 2374 /* There will be num_node_state(N_MEMORY) threads */ 2375 atomic_set(&pgdat_init_n_undone, num_node_state(N_MEMORY)); 2376 for_each_node_state(nid, N_MEMORY) { 2377 kthread_run(deferred_init_memmap, NODE_DATA(nid), "pgdatinit%d", nid); 2378 } 2379 2380 /* Block until all are initialised */ 2381 wait_for_completion(&pgdat_init_all_done_comp); 2382 2383 /* 2384 * We initialized the rest of the deferred pages. Permanently disable 2385 * on-demand struct page initialization. 2386 */ 2387 static_branch_disable(&deferred_pages); 2388 2389 /* Reinit limits that are based on free pages after the kernel is up */ 2390 files_maxfiles_init(); 2391 #endif 2392 2393 buffer_init(); 2394 2395 /* Discard memblock private memory */ 2396 memblock_discard(); 2397 2398 for_each_node_state(nid, N_MEMORY) 2399 shuffle_free_memory(NODE_DATA(nid)); 2400 2401 for_each_populated_zone(zone) 2402 set_zone_contiguous(zone); 2403 2404 /* Initialize page ext after all struct pages are initialized. */ 2405 if (deferred_struct_pages) 2406 page_ext_init(); 2407 2408 page_alloc_sysctl_init(); 2409 } 2410 2411 #ifndef __HAVE_ARCH_RESERVED_KERNEL_PAGES 2412 /* 2413 * Returns the number of pages that arch has reserved but 2414 * is not known to alloc_large_system_hash(). 2415 */ 2416 static unsigned long __init arch_reserved_kernel_pages(void) 2417 { 2418 return 0; 2419 } 2420 #endif 2421 2422 /* 2423 * Adaptive scale is meant to reduce sizes of hash tables on large memory 2424 * machines. As memory size is increased the scale is also increased but at 2425 * slower pace. Starting from ADAPT_SCALE_BASE (64G), every time memory 2426 * quadruples the scale is increased by one, which means the size of hash table 2427 * only doubles, instead of quadrupling as well. 2428 * Because 32-bit systems cannot have large physical memory, where this scaling 2429 * makes sense, it is disabled on such platforms. 2430 */ 2431 #if __BITS_PER_LONG > 32 2432 #define ADAPT_SCALE_BASE (64ul << 30) 2433 #define ADAPT_SCALE_SHIFT 2 2434 #define ADAPT_SCALE_NPAGES (ADAPT_SCALE_BASE >> PAGE_SHIFT) 2435 #endif 2436 2437 /* 2438 * allocate a large system hash table from bootmem 2439 * - it is assumed that the hash table must contain an exact power-of-2 2440 * quantity of entries 2441 * - limit is the number of hash buckets, not the total allocation size 2442 */ 2443 void *__init alloc_large_system_hash(const char *tablename, 2444 unsigned long bucketsize, 2445 unsigned long numentries, 2446 int scale, 2447 int flags, 2448 unsigned int *_hash_shift, 2449 unsigned int *_hash_mask, 2450 unsigned long low_limit, 2451 unsigned long high_limit) 2452 { 2453 unsigned long long max = high_limit; 2454 unsigned long log2qty, size; 2455 void *table; 2456 gfp_t gfp_flags; 2457 bool virt; 2458 bool huge; 2459 2460 /* allow the kernel cmdline to have a say */ 2461 if (!numentries) { 2462 /* round applicable memory size up to nearest megabyte */ 2463 numentries = nr_kernel_pages; 2464 numentries -= arch_reserved_kernel_pages(); 2465 2466 /* It isn't necessary when PAGE_SIZE >= 1MB */ 2467 if (PAGE_SIZE < SZ_1M) 2468 numentries = round_up(numentries, SZ_1M / PAGE_SIZE); 2469 2470 #if __BITS_PER_LONG > 32 2471 if (!high_limit) { 2472 unsigned long adapt; 2473 2474 for (adapt = ADAPT_SCALE_NPAGES; adapt < numentries; 2475 adapt <<= ADAPT_SCALE_SHIFT) 2476 scale++; 2477 } 2478 #endif 2479 2480 /* limit to 1 bucket per 2^scale bytes of low memory */ 2481 if (scale > PAGE_SHIFT) 2482 numentries >>= (scale - PAGE_SHIFT); 2483 else 2484 numentries <<= (PAGE_SHIFT - scale); 2485 2486 /* Make sure we've got at least a 0-order allocation.. */ 2487 if (unlikely(flags & HASH_SMALL)) { 2488 /* Makes no sense without HASH_EARLY */ 2489 WARN_ON(!(flags & HASH_EARLY)); 2490 if (!(numentries >> *_hash_shift)) { 2491 numentries = 1UL << *_hash_shift; 2492 BUG_ON(!numentries); 2493 } 2494 } else if (unlikely((numentries * bucketsize) < PAGE_SIZE)) 2495 numentries = PAGE_SIZE / bucketsize; 2496 } 2497 numentries = roundup_pow_of_two(numentries); 2498 2499 /* limit allocation size to 1/16 total memory by default */ 2500 if (max == 0) { 2501 max = ((unsigned long long)nr_all_pages << PAGE_SHIFT) >> 4; 2502 do_div(max, bucketsize); 2503 } 2504 max = min(max, 0x80000000ULL); 2505 2506 if (numentries < low_limit) 2507 numentries = low_limit; 2508 if (numentries > max) 2509 numentries = max; 2510 2511 log2qty = ilog2(numentries); 2512 2513 gfp_flags = (flags & HASH_ZERO) ? GFP_ATOMIC | __GFP_ZERO : GFP_ATOMIC; 2514 do { 2515 virt = false; 2516 size = bucketsize << log2qty; 2517 if (flags & HASH_EARLY) { 2518 if (flags & HASH_ZERO) 2519 table = memblock_alloc(size, SMP_CACHE_BYTES); 2520 else 2521 table = memblock_alloc_raw(size, 2522 SMP_CACHE_BYTES); 2523 } else if (get_order(size) > MAX_ORDER || hashdist) { 2524 table = vmalloc_huge(size, gfp_flags); 2525 virt = true; 2526 if (table) 2527 huge = is_vm_area_hugepages(table); 2528 } else { 2529 /* 2530 * If bucketsize is not a power-of-two, we may free 2531 * some pages at the end of hash table which 2532 * alloc_pages_exact() automatically does 2533 */ 2534 table = alloc_pages_exact(size, gfp_flags); 2535 kmemleak_alloc(table, size, 1, gfp_flags); 2536 } 2537 } while (!table && size > PAGE_SIZE && --log2qty); 2538 2539 if (!table) 2540 panic("Failed to allocate %s hash table\n", tablename); 2541 2542 pr_info("%s hash table entries: %ld (order: %d, %lu bytes, %s)\n", 2543 tablename, 1UL << log2qty, ilog2(size) - PAGE_SHIFT, size, 2544 virt ? (huge ? "vmalloc hugepage" : "vmalloc") : "linear"); 2545 2546 if (_hash_shift) 2547 *_hash_shift = log2qty; 2548 if (_hash_mask) 2549 *_hash_mask = (1 << log2qty) - 1; 2550 2551 return table; 2552 } 2553 2554 /** 2555 * set_dma_reserve - set the specified number of pages reserved in the first zone 2556 * @new_dma_reserve: The number of pages to mark reserved 2557 * 2558 * The per-cpu batchsize and zone watermarks are determined by managed_pages. 2559 * In the DMA zone, a significant percentage may be consumed by kernel image 2560 * and other unfreeable allocations which can skew the watermarks badly. This 2561 * function may optionally be used to account for unfreeable pages in the 2562 * first zone (e.g., ZONE_DMA). The effect will be lower watermarks and 2563 * smaller per-cpu batchsize. 2564 */ 2565 void __init set_dma_reserve(unsigned long new_dma_reserve) 2566 { 2567 dma_reserve = new_dma_reserve; 2568 } 2569 2570 void __init memblock_free_pages(struct page *page, unsigned long pfn, 2571 unsigned int order) 2572 { 2573 2574 if (IS_ENABLED(CONFIG_DEFERRED_STRUCT_PAGE_INIT)) { 2575 int nid = early_pfn_to_nid(pfn); 2576 2577 if (!early_page_initialised(pfn, nid)) 2578 return; 2579 } 2580 2581 if (!kmsan_memblock_free_pages(page, order)) { 2582 /* KMSAN will take care of these pages. */ 2583 return; 2584 } 2585 __free_pages_core(page, order); 2586 } 2587 2588 DEFINE_STATIC_KEY_MAYBE(CONFIG_INIT_ON_ALLOC_DEFAULT_ON, init_on_alloc); 2589 EXPORT_SYMBOL(init_on_alloc); 2590 2591 DEFINE_STATIC_KEY_MAYBE(CONFIG_INIT_ON_FREE_DEFAULT_ON, init_on_free); 2592 EXPORT_SYMBOL(init_on_free); 2593 2594 static bool _init_on_alloc_enabled_early __read_mostly 2595 = IS_ENABLED(CONFIG_INIT_ON_ALLOC_DEFAULT_ON); 2596 static int __init early_init_on_alloc(char *buf) 2597 { 2598 2599 return kstrtobool(buf, &_init_on_alloc_enabled_early); 2600 } 2601 early_param("init_on_alloc", early_init_on_alloc); 2602 2603 static bool _init_on_free_enabled_early __read_mostly 2604 = IS_ENABLED(CONFIG_INIT_ON_FREE_DEFAULT_ON); 2605 static int __init early_init_on_free(char *buf) 2606 { 2607 return kstrtobool(buf, &_init_on_free_enabled_early); 2608 } 2609 early_param("init_on_free", early_init_on_free); 2610 2611 DEFINE_STATIC_KEY_MAYBE(CONFIG_DEBUG_VM, check_pages_enabled); 2612 2613 /* 2614 * Enable static keys related to various memory debugging and hardening options. 2615 * Some override others, and depend on early params that are evaluated in the 2616 * order of appearance. So we need to first gather the full picture of what was 2617 * enabled, and then make decisions. 2618 */ 2619 static void __init mem_debugging_and_hardening_init(void) 2620 { 2621 bool page_poisoning_requested = false; 2622 bool want_check_pages = false; 2623 2624 #ifdef CONFIG_PAGE_POISONING 2625 /* 2626 * Page poisoning is debug page alloc for some arches. If 2627 * either of those options are enabled, enable poisoning. 2628 */ 2629 if (page_poisoning_enabled() || 2630 (!IS_ENABLED(CONFIG_ARCH_SUPPORTS_DEBUG_PAGEALLOC) && 2631 debug_pagealloc_enabled())) { 2632 static_branch_enable(&_page_poisoning_enabled); 2633 page_poisoning_requested = true; 2634 want_check_pages = true; 2635 } 2636 #endif 2637 2638 if ((_init_on_alloc_enabled_early || _init_on_free_enabled_early) && 2639 page_poisoning_requested) { 2640 pr_info("mem auto-init: CONFIG_PAGE_POISONING is on, " 2641 "will take precedence over init_on_alloc and init_on_free\n"); 2642 _init_on_alloc_enabled_early = false; 2643 _init_on_free_enabled_early = false; 2644 } 2645 2646 if (_init_on_alloc_enabled_early) { 2647 want_check_pages = true; 2648 static_branch_enable(&init_on_alloc); 2649 } else { 2650 static_branch_disable(&init_on_alloc); 2651 } 2652 2653 if (_init_on_free_enabled_early) { 2654 want_check_pages = true; 2655 static_branch_enable(&init_on_free); 2656 } else { 2657 static_branch_disable(&init_on_free); 2658 } 2659 2660 if (IS_ENABLED(CONFIG_KMSAN) && 2661 (_init_on_alloc_enabled_early || _init_on_free_enabled_early)) 2662 pr_info("mem auto-init: please make sure init_on_alloc and init_on_free are disabled when running KMSAN\n"); 2663 2664 #ifdef CONFIG_DEBUG_PAGEALLOC 2665 if (debug_pagealloc_enabled()) { 2666 want_check_pages = true; 2667 static_branch_enable(&_debug_pagealloc_enabled); 2668 2669 if (debug_guardpage_minorder()) 2670 static_branch_enable(&_debug_guardpage_enabled); 2671 } 2672 #endif 2673 2674 /* 2675 * Any page debugging or hardening option also enables sanity checking 2676 * of struct pages being allocated or freed. With CONFIG_DEBUG_VM it's 2677 * enabled already. 2678 */ 2679 if (!IS_ENABLED(CONFIG_DEBUG_VM) && want_check_pages) 2680 static_branch_enable(&check_pages_enabled); 2681 } 2682 2683 /* Report memory auto-initialization states for this boot. */ 2684 static void __init report_meminit(void) 2685 { 2686 const char *stack; 2687 2688 if (IS_ENABLED(CONFIG_INIT_STACK_ALL_PATTERN)) 2689 stack = "all(pattern)"; 2690 else if (IS_ENABLED(CONFIG_INIT_STACK_ALL_ZERO)) 2691 stack = "all(zero)"; 2692 else if (IS_ENABLED(CONFIG_GCC_PLUGIN_STRUCTLEAK_BYREF_ALL)) 2693 stack = "byref_all(zero)"; 2694 else if (IS_ENABLED(CONFIG_GCC_PLUGIN_STRUCTLEAK_BYREF)) 2695 stack = "byref(zero)"; 2696 else if (IS_ENABLED(CONFIG_GCC_PLUGIN_STRUCTLEAK_USER)) 2697 stack = "__user(zero)"; 2698 else 2699 stack = "off"; 2700 2701 pr_info("mem auto-init: stack:%s, heap alloc:%s, heap free:%s\n", 2702 stack, want_init_on_alloc(GFP_KERNEL) ? "on" : "off", 2703 want_init_on_free() ? "on" : "off"); 2704 if (want_init_on_free()) 2705 pr_info("mem auto-init: clearing system memory may take some time...\n"); 2706 } 2707 2708 static void __init mem_init_print_info(void) 2709 { 2710 unsigned long physpages, codesize, datasize, rosize, bss_size; 2711 unsigned long init_code_size, init_data_size; 2712 2713 physpages = get_num_physpages(); 2714 codesize = _etext - _stext; 2715 datasize = _edata - _sdata; 2716 rosize = __end_rodata - __start_rodata; 2717 bss_size = __bss_stop - __bss_start; 2718 init_data_size = __init_end - __init_begin; 2719 init_code_size = _einittext - _sinittext; 2720 2721 /* 2722 * Detect special cases and adjust section sizes accordingly: 2723 * 1) .init.* may be embedded into .data sections 2724 * 2) .init.text.* may be out of [__init_begin, __init_end], 2725 * please refer to arch/tile/kernel/vmlinux.lds.S. 2726 * 3) .rodata.* may be embedded into .text or .data sections. 2727 */ 2728 #define adj_init_size(start, end, size, pos, adj) \ 2729 do { \ 2730 if (&start[0] <= &pos[0] && &pos[0] < &end[0] && size > adj) \ 2731 size -= adj; \ 2732 } while (0) 2733 2734 adj_init_size(__init_begin, __init_end, init_data_size, 2735 _sinittext, init_code_size); 2736 adj_init_size(_stext, _etext, codesize, _sinittext, init_code_size); 2737 adj_init_size(_sdata, _edata, datasize, __init_begin, init_data_size); 2738 adj_init_size(_stext, _etext, codesize, __start_rodata, rosize); 2739 adj_init_size(_sdata, _edata, datasize, __start_rodata, rosize); 2740 2741 #undef adj_init_size 2742 2743 pr_info("Memory: %luK/%luK available (%luK kernel code, %luK rwdata, %luK rodata, %luK init, %luK bss, %luK reserved, %luK cma-reserved" 2744 #ifdef CONFIG_HIGHMEM 2745 ", %luK highmem" 2746 #endif 2747 ")\n", 2748 K(nr_free_pages()), K(physpages), 2749 codesize / SZ_1K, datasize / SZ_1K, rosize / SZ_1K, 2750 (init_data_size + init_code_size) / SZ_1K, bss_size / SZ_1K, 2751 K(physpages - totalram_pages() - totalcma_pages), 2752 K(totalcma_pages) 2753 #ifdef CONFIG_HIGHMEM 2754 , K(totalhigh_pages()) 2755 #endif 2756 ); 2757 } 2758 2759 /* 2760 * Set up kernel memory allocators 2761 */ 2762 void __init mm_core_init(void) 2763 { 2764 /* Initializations relying on SMP setup */ 2765 build_all_zonelists(NULL); 2766 page_alloc_init_cpuhp(); 2767 2768 /* 2769 * page_ext requires contiguous pages, 2770 * bigger than MAX_ORDER unless SPARSEMEM. 2771 */ 2772 page_ext_init_flatmem(); 2773 mem_debugging_and_hardening_init(); 2774 kfence_alloc_pool(); 2775 report_meminit(); 2776 kmsan_init_shadow(); 2777 stack_depot_early_init(); 2778 mem_init(); 2779 mem_init_print_info(); 2780 kmem_cache_init(); 2781 /* 2782 * page_owner must be initialized after buddy is ready, and also after 2783 * slab is ready so that stack_depot_init() works properly 2784 */ 2785 page_ext_init_flatmem_late(); 2786 kmemleak_init(); 2787 ptlock_cache_init(); 2788 pgtable_cache_init(); 2789 debug_objects_mem_init(); 2790 vmalloc_init(); 2791 /* If no deferred init page_ext now, as vmap is fully initialized */ 2792 if (!deferred_struct_pages) 2793 page_ext_init(); 2794 /* Should be run before the first non-init thread is created */ 2795 init_espfix_bsp(); 2796 /* Should be run after espfix64 is set up. */ 2797 pti_init(); 2798 kmsan_init_runtime(); 2799 mm_cache_init(); 2800 } 2801