1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * mm_init.c - Memory initialisation verification and debugging 4 * 5 * Copyright 2008 IBM Corporation, 2008 6 * Author Mel Gorman <mel@csn.ul.ie> 7 * 8 */ 9 #include <linux/kernel.h> 10 #include <linux/init.h> 11 #include <linux/kobject.h> 12 #include <linux/export.h> 13 #include <linux/memory.h> 14 #include <linux/notifier.h> 15 #include <linux/sched.h> 16 #include <linux/mman.h> 17 #include <linux/memblock.h> 18 #include <linux/page-isolation.h> 19 #include <linux/padata.h> 20 #include <linux/nmi.h> 21 #include <linux/buffer_head.h> 22 #include <linux/kmemleak.h> 23 #include <linux/kfence.h> 24 #include <linux/page_ext.h> 25 #include <linux/pti.h> 26 #include <linux/pgtable.h> 27 #include <linux/swap.h> 28 #include <linux/cma.h> 29 #include "internal.h" 30 #include "slab.h" 31 #include "shuffle.h" 32 33 #include <asm/setup.h> 34 35 #ifdef CONFIG_DEBUG_MEMORY_INIT 36 int __meminitdata mminit_loglevel; 37 38 /* The zonelists are simply reported, validation is manual. */ 39 void __init mminit_verify_zonelist(void) 40 { 41 int nid; 42 43 if (mminit_loglevel < MMINIT_VERIFY) 44 return; 45 46 for_each_online_node(nid) { 47 pg_data_t *pgdat = NODE_DATA(nid); 48 struct zone *zone; 49 struct zoneref *z; 50 struct zonelist *zonelist; 51 int i, listid, zoneid; 52 53 BUILD_BUG_ON(MAX_ZONELISTS > 2); 54 for (i = 0; i < MAX_ZONELISTS * MAX_NR_ZONES; i++) { 55 56 /* Identify the zone and nodelist */ 57 zoneid = i % MAX_NR_ZONES; 58 listid = i / MAX_NR_ZONES; 59 zonelist = &pgdat->node_zonelists[listid]; 60 zone = &pgdat->node_zones[zoneid]; 61 if (!populated_zone(zone)) 62 continue; 63 64 /* Print information about the zonelist */ 65 printk(KERN_DEBUG "mminit::zonelist %s %d:%s = ", 66 listid > 0 ? "thisnode" : "general", nid, 67 zone->name); 68 69 /* Iterate the zonelist */ 70 for_each_zone_zonelist(zone, z, zonelist, zoneid) 71 pr_cont("%d:%s ", zone_to_nid(zone), zone->name); 72 pr_cont("\n"); 73 } 74 } 75 } 76 77 void __init mminit_verify_pageflags_layout(void) 78 { 79 int shift, width; 80 unsigned long or_mask, add_mask; 81 82 shift = 8 * sizeof(unsigned long); 83 width = shift - SECTIONS_WIDTH - NODES_WIDTH - ZONES_WIDTH 84 - LAST_CPUPID_SHIFT - KASAN_TAG_WIDTH - LRU_GEN_WIDTH - LRU_REFS_WIDTH; 85 mminit_dprintk(MMINIT_TRACE, "pageflags_layout_widths", 86 "Section %d Node %d Zone %d Lastcpupid %d Kasantag %d Gen %d Tier %d Flags %d\n", 87 SECTIONS_WIDTH, 88 NODES_WIDTH, 89 ZONES_WIDTH, 90 LAST_CPUPID_WIDTH, 91 KASAN_TAG_WIDTH, 92 LRU_GEN_WIDTH, 93 LRU_REFS_WIDTH, 94 NR_PAGEFLAGS); 95 mminit_dprintk(MMINIT_TRACE, "pageflags_layout_shifts", 96 "Section %d Node %d Zone %d Lastcpupid %d Kasantag %d\n", 97 SECTIONS_SHIFT, 98 NODES_SHIFT, 99 ZONES_SHIFT, 100 LAST_CPUPID_SHIFT, 101 KASAN_TAG_WIDTH); 102 mminit_dprintk(MMINIT_TRACE, "pageflags_layout_pgshifts", 103 "Section %lu Node %lu Zone %lu Lastcpupid %lu Kasantag %lu\n", 104 (unsigned long)SECTIONS_PGSHIFT, 105 (unsigned long)NODES_PGSHIFT, 106 (unsigned long)ZONES_PGSHIFT, 107 (unsigned long)LAST_CPUPID_PGSHIFT, 108 (unsigned long)KASAN_TAG_PGSHIFT); 109 mminit_dprintk(MMINIT_TRACE, "pageflags_layout_nodezoneid", 110 "Node/Zone ID: %lu -> %lu\n", 111 (unsigned long)(ZONEID_PGOFF + ZONEID_SHIFT), 112 (unsigned long)ZONEID_PGOFF); 113 mminit_dprintk(MMINIT_TRACE, "pageflags_layout_usage", 114 "location: %d -> %d layout %d -> %d unused %d -> %d page-flags\n", 115 shift, width, width, NR_PAGEFLAGS, NR_PAGEFLAGS, 0); 116 #ifdef NODE_NOT_IN_PAGE_FLAGS 117 mminit_dprintk(MMINIT_TRACE, "pageflags_layout_nodeflags", 118 "Node not in page flags"); 119 #endif 120 #ifdef LAST_CPUPID_NOT_IN_PAGE_FLAGS 121 mminit_dprintk(MMINIT_TRACE, "pageflags_layout_nodeflags", 122 "Last cpupid not in page flags"); 123 #endif 124 125 if (SECTIONS_WIDTH) { 126 shift -= SECTIONS_WIDTH; 127 BUG_ON(shift != SECTIONS_PGSHIFT); 128 } 129 if (NODES_WIDTH) { 130 shift -= NODES_WIDTH; 131 BUG_ON(shift != NODES_PGSHIFT); 132 } 133 if (ZONES_WIDTH) { 134 shift -= ZONES_WIDTH; 135 BUG_ON(shift != ZONES_PGSHIFT); 136 } 137 138 /* Check for bitmask overlaps */ 139 or_mask = (ZONES_MASK << ZONES_PGSHIFT) | 140 (NODES_MASK << NODES_PGSHIFT) | 141 (SECTIONS_MASK << SECTIONS_PGSHIFT); 142 add_mask = (ZONES_MASK << ZONES_PGSHIFT) + 143 (NODES_MASK << NODES_PGSHIFT) + 144 (SECTIONS_MASK << SECTIONS_PGSHIFT); 145 BUG_ON(or_mask != add_mask); 146 } 147 148 static __init int set_mminit_loglevel(char *str) 149 { 150 get_option(&str, &mminit_loglevel); 151 return 0; 152 } 153 early_param("mminit_loglevel", set_mminit_loglevel); 154 #endif /* CONFIG_DEBUG_MEMORY_INIT */ 155 156 struct kobject *mm_kobj; 157 EXPORT_SYMBOL_GPL(mm_kobj); 158 159 #ifdef CONFIG_SMP 160 s32 vm_committed_as_batch = 32; 161 162 void mm_compute_batch(int overcommit_policy) 163 { 164 u64 memsized_batch; 165 s32 nr = num_present_cpus(); 166 s32 batch = max_t(s32, nr*2, 32); 167 unsigned long ram_pages = totalram_pages(); 168 169 /* 170 * For policy OVERCOMMIT_NEVER, set batch size to 0.4% of 171 * (total memory/#cpus), and lift it to 25% for other policies 172 * to easy the possible lock contention for percpu_counter 173 * vm_committed_as, while the max limit is INT_MAX 174 */ 175 if (overcommit_policy == OVERCOMMIT_NEVER) 176 memsized_batch = min_t(u64, ram_pages/nr/256, INT_MAX); 177 else 178 memsized_batch = min_t(u64, ram_pages/nr/4, INT_MAX); 179 180 vm_committed_as_batch = max_t(s32, memsized_batch, batch); 181 } 182 183 static int __meminit mm_compute_batch_notifier(struct notifier_block *self, 184 unsigned long action, void *arg) 185 { 186 switch (action) { 187 case MEM_ONLINE: 188 case MEM_OFFLINE: 189 mm_compute_batch(sysctl_overcommit_memory); 190 break; 191 default: 192 break; 193 } 194 return NOTIFY_OK; 195 } 196 197 static int __init mm_compute_batch_init(void) 198 { 199 mm_compute_batch(sysctl_overcommit_memory); 200 hotplug_memory_notifier(mm_compute_batch_notifier, MM_COMPUTE_BATCH_PRI); 201 return 0; 202 } 203 204 __initcall(mm_compute_batch_init); 205 206 #endif 207 208 static int __init mm_sysfs_init(void) 209 { 210 mm_kobj = kobject_create_and_add("mm", kernel_kobj); 211 if (!mm_kobj) 212 return -ENOMEM; 213 214 return 0; 215 } 216 postcore_initcall(mm_sysfs_init); 217 218 static unsigned long arch_zone_lowest_possible_pfn[MAX_NR_ZONES] __initdata; 219 static unsigned long arch_zone_highest_possible_pfn[MAX_NR_ZONES] __initdata; 220 static unsigned long zone_movable_pfn[MAX_NUMNODES] __initdata; 221 222 static unsigned long required_kernelcore __initdata; 223 static unsigned long required_kernelcore_percent __initdata; 224 static unsigned long required_movablecore __initdata; 225 static unsigned long required_movablecore_percent __initdata; 226 227 static unsigned long nr_kernel_pages __initdata; 228 static unsigned long nr_all_pages __initdata; 229 static unsigned long dma_reserve __initdata; 230 231 static bool deferred_struct_pages __meminitdata; 232 233 static DEFINE_PER_CPU(struct per_cpu_nodestat, boot_nodestats); 234 235 static int __init cmdline_parse_core(char *p, unsigned long *core, 236 unsigned long *percent) 237 { 238 unsigned long long coremem; 239 char *endptr; 240 241 if (!p) 242 return -EINVAL; 243 244 /* Value may be a percentage of total memory, otherwise bytes */ 245 coremem = simple_strtoull(p, &endptr, 0); 246 if (*endptr == '%') { 247 /* Paranoid check for percent values greater than 100 */ 248 WARN_ON(coremem > 100); 249 250 *percent = coremem; 251 } else { 252 coremem = memparse(p, &p); 253 /* Paranoid check that UL is enough for the coremem value */ 254 WARN_ON((coremem >> PAGE_SHIFT) > ULONG_MAX); 255 256 *core = coremem >> PAGE_SHIFT; 257 *percent = 0UL; 258 } 259 return 0; 260 } 261 262 bool mirrored_kernelcore __initdata_memblock; 263 264 /* 265 * kernelcore=size sets the amount of memory for use for allocations that 266 * cannot be reclaimed or migrated. 267 */ 268 static int __init cmdline_parse_kernelcore(char *p) 269 { 270 /* parse kernelcore=mirror */ 271 if (parse_option_str(p, "mirror")) { 272 mirrored_kernelcore = true; 273 return 0; 274 } 275 276 return cmdline_parse_core(p, &required_kernelcore, 277 &required_kernelcore_percent); 278 } 279 early_param("kernelcore", cmdline_parse_kernelcore); 280 281 /* 282 * movablecore=size sets the amount of memory for use for allocations that 283 * can be reclaimed or migrated. 284 */ 285 static int __init cmdline_parse_movablecore(char *p) 286 { 287 return cmdline_parse_core(p, &required_movablecore, 288 &required_movablecore_percent); 289 } 290 early_param("movablecore", cmdline_parse_movablecore); 291 292 /* 293 * early_calculate_totalpages() 294 * Sum pages in active regions for movable zone. 295 * Populate N_MEMORY for calculating usable_nodes. 296 */ 297 static unsigned long __init early_calculate_totalpages(void) 298 { 299 unsigned long totalpages = 0; 300 unsigned long start_pfn, end_pfn; 301 int i, nid; 302 303 for_each_mem_pfn_range(i, MAX_NUMNODES, &start_pfn, &end_pfn, &nid) { 304 unsigned long pages = end_pfn - start_pfn; 305 306 totalpages += pages; 307 if (pages) 308 node_set_state(nid, N_MEMORY); 309 } 310 return totalpages; 311 } 312 313 /* 314 * This finds a zone that can be used for ZONE_MOVABLE pages. The 315 * assumption is made that zones within a node are ordered in monotonic 316 * increasing memory addresses so that the "highest" populated zone is used 317 */ 318 static void __init find_usable_zone_for_movable(void) 319 { 320 int zone_index; 321 for (zone_index = MAX_NR_ZONES - 1; zone_index >= 0; zone_index--) { 322 if (zone_index == ZONE_MOVABLE) 323 continue; 324 325 if (arch_zone_highest_possible_pfn[zone_index] > 326 arch_zone_lowest_possible_pfn[zone_index]) 327 break; 328 } 329 330 VM_BUG_ON(zone_index == -1); 331 movable_zone = zone_index; 332 } 333 334 /* 335 * Find the PFN the Movable zone begins in each node. Kernel memory 336 * is spread evenly between nodes as long as the nodes have enough 337 * memory. When they don't, some nodes will have more kernelcore than 338 * others 339 */ 340 static void __init find_zone_movable_pfns_for_nodes(void) 341 { 342 int i, nid; 343 unsigned long usable_startpfn; 344 unsigned long kernelcore_node, kernelcore_remaining; 345 /* save the state before borrow the nodemask */ 346 nodemask_t saved_node_state = node_states[N_MEMORY]; 347 unsigned long totalpages = early_calculate_totalpages(); 348 int usable_nodes = nodes_weight(node_states[N_MEMORY]); 349 struct memblock_region *r; 350 351 /* Need to find movable_zone earlier when movable_node is specified. */ 352 find_usable_zone_for_movable(); 353 354 /* 355 * If movable_node is specified, ignore kernelcore and movablecore 356 * options. 357 */ 358 if (movable_node_is_enabled()) { 359 for_each_mem_region(r) { 360 if (!memblock_is_hotpluggable(r)) 361 continue; 362 363 nid = memblock_get_region_node(r); 364 365 usable_startpfn = PFN_DOWN(r->base); 366 zone_movable_pfn[nid] = zone_movable_pfn[nid] ? 367 min(usable_startpfn, zone_movable_pfn[nid]) : 368 usable_startpfn; 369 } 370 371 goto out2; 372 } 373 374 /* 375 * If kernelcore=mirror is specified, ignore movablecore option 376 */ 377 if (mirrored_kernelcore) { 378 bool mem_below_4gb_not_mirrored = false; 379 380 for_each_mem_region(r) { 381 if (memblock_is_mirror(r)) 382 continue; 383 384 nid = memblock_get_region_node(r); 385 386 usable_startpfn = memblock_region_memory_base_pfn(r); 387 388 if (usable_startpfn < PHYS_PFN(SZ_4G)) { 389 mem_below_4gb_not_mirrored = true; 390 continue; 391 } 392 393 zone_movable_pfn[nid] = zone_movable_pfn[nid] ? 394 min(usable_startpfn, zone_movable_pfn[nid]) : 395 usable_startpfn; 396 } 397 398 if (mem_below_4gb_not_mirrored) 399 pr_warn("This configuration results in unmirrored kernel memory.\n"); 400 401 goto out2; 402 } 403 404 /* 405 * If kernelcore=nn% or movablecore=nn% was specified, calculate the 406 * amount of necessary memory. 407 */ 408 if (required_kernelcore_percent) 409 required_kernelcore = (totalpages * 100 * required_kernelcore_percent) / 410 10000UL; 411 if (required_movablecore_percent) 412 required_movablecore = (totalpages * 100 * required_movablecore_percent) / 413 10000UL; 414 415 /* 416 * If movablecore= was specified, calculate what size of 417 * kernelcore that corresponds so that memory usable for 418 * any allocation type is evenly spread. If both kernelcore 419 * and movablecore are specified, then the value of kernelcore 420 * will be used for required_kernelcore if it's greater than 421 * what movablecore would have allowed. 422 */ 423 if (required_movablecore) { 424 unsigned long corepages; 425 426 /* 427 * Round-up so that ZONE_MOVABLE is at least as large as what 428 * was requested by the user 429 */ 430 required_movablecore = 431 roundup(required_movablecore, MAX_ORDER_NR_PAGES); 432 required_movablecore = min(totalpages, required_movablecore); 433 corepages = totalpages - required_movablecore; 434 435 required_kernelcore = max(required_kernelcore, corepages); 436 } 437 438 /* 439 * If kernelcore was not specified or kernelcore size is larger 440 * than totalpages, there is no ZONE_MOVABLE. 441 */ 442 if (!required_kernelcore || required_kernelcore >= totalpages) 443 goto out; 444 445 /* usable_startpfn is the lowest possible pfn ZONE_MOVABLE can be at */ 446 usable_startpfn = arch_zone_lowest_possible_pfn[movable_zone]; 447 448 restart: 449 /* Spread kernelcore memory as evenly as possible throughout nodes */ 450 kernelcore_node = required_kernelcore / usable_nodes; 451 for_each_node_state(nid, N_MEMORY) { 452 unsigned long start_pfn, end_pfn; 453 454 /* 455 * Recalculate kernelcore_node if the division per node 456 * now exceeds what is necessary to satisfy the requested 457 * amount of memory for the kernel 458 */ 459 if (required_kernelcore < kernelcore_node) 460 kernelcore_node = required_kernelcore / usable_nodes; 461 462 /* 463 * As the map is walked, we track how much memory is usable 464 * by the kernel using kernelcore_remaining. When it is 465 * 0, the rest of the node is usable by ZONE_MOVABLE 466 */ 467 kernelcore_remaining = kernelcore_node; 468 469 /* Go through each range of PFNs within this node */ 470 for_each_mem_pfn_range(i, nid, &start_pfn, &end_pfn, NULL) { 471 unsigned long size_pages; 472 473 start_pfn = max(start_pfn, zone_movable_pfn[nid]); 474 if (start_pfn >= end_pfn) 475 continue; 476 477 /* Account for what is only usable for kernelcore */ 478 if (start_pfn < usable_startpfn) { 479 unsigned long kernel_pages; 480 kernel_pages = min(end_pfn, usable_startpfn) 481 - start_pfn; 482 483 kernelcore_remaining -= min(kernel_pages, 484 kernelcore_remaining); 485 required_kernelcore -= min(kernel_pages, 486 required_kernelcore); 487 488 /* Continue if range is now fully accounted */ 489 if (end_pfn <= usable_startpfn) { 490 491 /* 492 * Push zone_movable_pfn to the end so 493 * that if we have to rebalance 494 * kernelcore across nodes, we will 495 * not double account here 496 */ 497 zone_movable_pfn[nid] = end_pfn; 498 continue; 499 } 500 start_pfn = usable_startpfn; 501 } 502 503 /* 504 * The usable PFN range for ZONE_MOVABLE is from 505 * start_pfn->end_pfn. Calculate size_pages as the 506 * number of pages used as kernelcore 507 */ 508 size_pages = end_pfn - start_pfn; 509 if (size_pages > kernelcore_remaining) 510 size_pages = kernelcore_remaining; 511 zone_movable_pfn[nid] = start_pfn + size_pages; 512 513 /* 514 * Some kernelcore has been met, update counts and 515 * break if the kernelcore for this node has been 516 * satisfied 517 */ 518 required_kernelcore -= min(required_kernelcore, 519 size_pages); 520 kernelcore_remaining -= size_pages; 521 if (!kernelcore_remaining) 522 break; 523 } 524 } 525 526 /* 527 * If there is still required_kernelcore, we do another pass with one 528 * less node in the count. This will push zone_movable_pfn[nid] further 529 * along on the nodes that still have memory until kernelcore is 530 * satisfied 531 */ 532 usable_nodes--; 533 if (usable_nodes && required_kernelcore > usable_nodes) 534 goto restart; 535 536 out2: 537 /* Align start of ZONE_MOVABLE on all nids to MAX_ORDER_NR_PAGES */ 538 for (nid = 0; nid < MAX_NUMNODES; nid++) { 539 unsigned long start_pfn, end_pfn; 540 541 zone_movable_pfn[nid] = 542 roundup(zone_movable_pfn[nid], MAX_ORDER_NR_PAGES); 543 544 get_pfn_range_for_nid(nid, &start_pfn, &end_pfn); 545 if (zone_movable_pfn[nid] >= end_pfn) 546 zone_movable_pfn[nid] = 0; 547 } 548 549 out: 550 /* restore the node_state */ 551 node_states[N_MEMORY] = saved_node_state; 552 } 553 554 static void __meminit __init_single_page(struct page *page, unsigned long pfn, 555 unsigned long zone, int nid) 556 { 557 mm_zero_struct_page(page); 558 set_page_links(page, zone, nid, pfn); 559 init_page_count(page); 560 page_mapcount_reset(page); 561 page_cpupid_reset_last(page); 562 page_kasan_tag_reset(page); 563 564 INIT_LIST_HEAD(&page->lru); 565 #ifdef WANT_PAGE_VIRTUAL 566 /* The shift won't overflow because ZONE_NORMAL is below 4G. */ 567 if (!is_highmem_idx(zone)) 568 set_page_address(page, __va(pfn << PAGE_SHIFT)); 569 #endif 570 } 571 572 #ifdef CONFIG_NUMA 573 /* 574 * During memory init memblocks map pfns to nids. The search is expensive and 575 * this caches recent lookups. The implementation of __early_pfn_to_nid 576 * treats start/end as pfns. 577 */ 578 struct mminit_pfnnid_cache { 579 unsigned long last_start; 580 unsigned long last_end; 581 int last_nid; 582 }; 583 584 static struct mminit_pfnnid_cache early_pfnnid_cache __meminitdata; 585 586 /* 587 * Required by SPARSEMEM. Given a PFN, return what node the PFN is on. 588 */ 589 static int __meminit __early_pfn_to_nid(unsigned long pfn, 590 struct mminit_pfnnid_cache *state) 591 { 592 unsigned long start_pfn, end_pfn; 593 int nid; 594 595 if (state->last_start <= pfn && pfn < state->last_end) 596 return state->last_nid; 597 598 nid = memblock_search_pfn_nid(pfn, &start_pfn, &end_pfn); 599 if (nid != NUMA_NO_NODE) { 600 state->last_start = start_pfn; 601 state->last_end = end_pfn; 602 state->last_nid = nid; 603 } 604 605 return nid; 606 } 607 608 int __meminit early_pfn_to_nid(unsigned long pfn) 609 { 610 static DEFINE_SPINLOCK(early_pfn_lock); 611 int nid; 612 613 spin_lock(&early_pfn_lock); 614 nid = __early_pfn_to_nid(pfn, &early_pfnnid_cache); 615 if (nid < 0) 616 nid = first_online_node; 617 spin_unlock(&early_pfn_lock); 618 619 return nid; 620 } 621 622 int hashdist = HASHDIST_DEFAULT; 623 624 static int __init set_hashdist(char *str) 625 { 626 if (!str) 627 return 0; 628 hashdist = simple_strtoul(str, &str, 0); 629 return 1; 630 } 631 __setup("hashdist=", set_hashdist); 632 633 static inline void fixup_hashdist(void) 634 { 635 if (num_node_state(N_MEMORY) == 1) 636 hashdist = 0; 637 } 638 #else 639 static inline void fixup_hashdist(void) {} 640 #endif /* CONFIG_NUMA */ 641 642 #ifdef CONFIG_DEFERRED_STRUCT_PAGE_INIT 643 static inline void pgdat_set_deferred_range(pg_data_t *pgdat) 644 { 645 pgdat->first_deferred_pfn = ULONG_MAX; 646 } 647 648 /* Returns true if the struct page for the pfn is initialised */ 649 static inline bool __meminit early_page_initialised(unsigned long pfn, int nid) 650 { 651 if (node_online(nid) && pfn >= NODE_DATA(nid)->first_deferred_pfn) 652 return false; 653 654 return true; 655 } 656 657 /* 658 * Returns true when the remaining initialisation should be deferred until 659 * later in the boot cycle when it can be parallelised. 660 */ 661 static bool __meminit 662 defer_init(int nid, unsigned long pfn, unsigned long end_pfn) 663 { 664 static unsigned long prev_end_pfn, nr_initialised; 665 666 if (early_page_ext_enabled()) 667 return false; 668 /* 669 * prev_end_pfn static that contains the end of previous zone 670 * No need to protect because called very early in boot before smp_init. 671 */ 672 if (prev_end_pfn != end_pfn) { 673 prev_end_pfn = end_pfn; 674 nr_initialised = 0; 675 } 676 677 /* Always populate low zones for address-constrained allocations */ 678 if (end_pfn < pgdat_end_pfn(NODE_DATA(nid))) 679 return false; 680 681 if (NODE_DATA(nid)->first_deferred_pfn != ULONG_MAX) 682 return true; 683 /* 684 * We start only with one section of pages, more pages are added as 685 * needed until the rest of deferred pages are initialized. 686 */ 687 nr_initialised++; 688 if ((nr_initialised > PAGES_PER_SECTION) && 689 (pfn & (PAGES_PER_SECTION - 1)) == 0) { 690 NODE_DATA(nid)->first_deferred_pfn = pfn; 691 return true; 692 } 693 return false; 694 } 695 696 static void __meminit init_reserved_page(unsigned long pfn, int nid) 697 { 698 pg_data_t *pgdat; 699 int zid; 700 701 if (early_page_initialised(pfn, nid)) 702 return; 703 704 pgdat = NODE_DATA(nid); 705 706 for (zid = 0; zid < MAX_NR_ZONES; zid++) { 707 struct zone *zone = &pgdat->node_zones[zid]; 708 709 if (zone_spans_pfn(zone, pfn)) 710 break; 711 } 712 __init_single_page(pfn_to_page(pfn), pfn, zid, nid); 713 } 714 #else 715 static inline void pgdat_set_deferred_range(pg_data_t *pgdat) {} 716 717 static inline bool early_page_initialised(unsigned long pfn, int nid) 718 { 719 return true; 720 } 721 722 static inline bool defer_init(int nid, unsigned long pfn, unsigned long end_pfn) 723 { 724 return false; 725 } 726 727 static inline void init_reserved_page(unsigned long pfn, int nid) 728 { 729 } 730 #endif /* CONFIG_DEFERRED_STRUCT_PAGE_INIT */ 731 732 /* 733 * Initialised pages do not have PageReserved set. This function is 734 * called for each range allocated by the bootmem allocator and 735 * marks the pages PageReserved. The remaining valid pages are later 736 * sent to the buddy page allocator. 737 */ 738 void __meminit reserve_bootmem_region(phys_addr_t start, 739 phys_addr_t end, int nid) 740 { 741 unsigned long start_pfn = PFN_DOWN(start); 742 unsigned long end_pfn = PFN_UP(end); 743 744 for (; start_pfn < end_pfn; start_pfn++) { 745 if (pfn_valid(start_pfn)) { 746 struct page *page = pfn_to_page(start_pfn); 747 748 init_reserved_page(start_pfn, nid); 749 750 /* Avoid false-positive PageTail() */ 751 INIT_LIST_HEAD(&page->lru); 752 753 /* 754 * no need for atomic set_bit because the struct 755 * page is not visible yet so nobody should 756 * access it yet. 757 */ 758 __SetPageReserved(page); 759 } 760 } 761 } 762 763 /* If zone is ZONE_MOVABLE but memory is mirrored, it is an overlapped init */ 764 static bool __meminit 765 overlap_memmap_init(unsigned long zone, unsigned long *pfn) 766 { 767 static struct memblock_region *r; 768 769 if (mirrored_kernelcore && zone == ZONE_MOVABLE) { 770 if (!r || *pfn >= memblock_region_memory_end_pfn(r)) { 771 for_each_mem_region(r) { 772 if (*pfn < memblock_region_memory_end_pfn(r)) 773 break; 774 } 775 } 776 if (*pfn >= memblock_region_memory_base_pfn(r) && 777 memblock_is_mirror(r)) { 778 *pfn = memblock_region_memory_end_pfn(r); 779 return true; 780 } 781 } 782 return false; 783 } 784 785 /* 786 * Only struct pages that correspond to ranges defined by memblock.memory 787 * are zeroed and initialized by going through __init_single_page() during 788 * memmap_init_zone_range(). 789 * 790 * But, there could be struct pages that correspond to holes in 791 * memblock.memory. This can happen because of the following reasons: 792 * - physical memory bank size is not necessarily the exact multiple of the 793 * arbitrary section size 794 * - early reserved memory may not be listed in memblock.memory 795 * - memory layouts defined with memmap= kernel parameter may not align 796 * nicely with memmap sections 797 * 798 * Explicitly initialize those struct pages so that: 799 * - PG_Reserved is set 800 * - zone and node links point to zone and node that span the page if the 801 * hole is in the middle of a zone 802 * - zone and node links point to adjacent zone/node if the hole falls on 803 * the zone boundary; the pages in such holes will be prepended to the 804 * zone/node above the hole except for the trailing pages in the last 805 * section that will be appended to the zone/node below. 806 */ 807 static void __init init_unavailable_range(unsigned long spfn, 808 unsigned long epfn, 809 int zone, int node) 810 { 811 unsigned long pfn; 812 u64 pgcnt = 0; 813 814 for (pfn = spfn; pfn < epfn; pfn++) { 815 if (!pfn_valid(pageblock_start_pfn(pfn))) { 816 pfn = pageblock_end_pfn(pfn) - 1; 817 continue; 818 } 819 __init_single_page(pfn_to_page(pfn), pfn, zone, node); 820 __SetPageReserved(pfn_to_page(pfn)); 821 pgcnt++; 822 } 823 824 if (pgcnt) 825 pr_info("On node %d, zone %s: %lld pages in unavailable ranges", 826 node, zone_names[zone], pgcnt); 827 } 828 829 /* 830 * Initially all pages are reserved - free ones are freed 831 * up by memblock_free_all() once the early boot process is 832 * done. Non-atomic initialization, single-pass. 833 * 834 * All aligned pageblocks are initialized to the specified migratetype 835 * (usually MIGRATE_MOVABLE). Besides setting the migratetype, no related 836 * zone stats (e.g., nr_isolate_pageblock) are touched. 837 */ 838 void __meminit memmap_init_range(unsigned long size, int nid, unsigned long zone, 839 unsigned long start_pfn, unsigned long zone_end_pfn, 840 enum meminit_context context, 841 struct vmem_altmap *altmap, int migratetype) 842 { 843 unsigned long pfn, end_pfn = start_pfn + size; 844 struct page *page; 845 846 if (highest_memmap_pfn < end_pfn - 1) 847 highest_memmap_pfn = end_pfn - 1; 848 849 #ifdef CONFIG_ZONE_DEVICE 850 /* 851 * Honor reservation requested by the driver for this ZONE_DEVICE 852 * memory. We limit the total number of pages to initialize to just 853 * those that might contain the memory mapping. We will defer the 854 * ZONE_DEVICE page initialization until after we have released 855 * the hotplug lock. 856 */ 857 if (zone == ZONE_DEVICE) { 858 if (!altmap) 859 return; 860 861 if (start_pfn == altmap->base_pfn) 862 start_pfn += altmap->reserve; 863 end_pfn = altmap->base_pfn + vmem_altmap_offset(altmap); 864 } 865 #endif 866 867 for (pfn = start_pfn; pfn < end_pfn; ) { 868 /* 869 * There can be holes in boot-time mem_map[]s handed to this 870 * function. They do not exist on hotplugged memory. 871 */ 872 if (context == MEMINIT_EARLY) { 873 if (overlap_memmap_init(zone, &pfn)) 874 continue; 875 if (defer_init(nid, pfn, zone_end_pfn)) { 876 deferred_struct_pages = true; 877 break; 878 } 879 } 880 881 page = pfn_to_page(pfn); 882 __init_single_page(page, pfn, zone, nid); 883 if (context == MEMINIT_HOTPLUG) 884 __SetPageReserved(page); 885 886 /* 887 * Usually, we want to mark the pageblock MIGRATE_MOVABLE, 888 * such that unmovable allocations won't be scattered all 889 * over the place during system boot. 890 */ 891 if (pageblock_aligned(pfn)) { 892 set_pageblock_migratetype(page, migratetype); 893 cond_resched(); 894 } 895 pfn++; 896 } 897 } 898 899 static void __init memmap_init_zone_range(struct zone *zone, 900 unsigned long start_pfn, 901 unsigned long end_pfn, 902 unsigned long *hole_pfn) 903 { 904 unsigned long zone_start_pfn = zone->zone_start_pfn; 905 unsigned long zone_end_pfn = zone_start_pfn + zone->spanned_pages; 906 int nid = zone_to_nid(zone), zone_id = zone_idx(zone); 907 908 start_pfn = clamp(start_pfn, zone_start_pfn, zone_end_pfn); 909 end_pfn = clamp(end_pfn, zone_start_pfn, zone_end_pfn); 910 911 if (start_pfn >= end_pfn) 912 return; 913 914 memmap_init_range(end_pfn - start_pfn, nid, zone_id, start_pfn, 915 zone_end_pfn, MEMINIT_EARLY, NULL, MIGRATE_MOVABLE); 916 917 if (*hole_pfn < start_pfn) 918 init_unavailable_range(*hole_pfn, start_pfn, zone_id, nid); 919 920 *hole_pfn = end_pfn; 921 } 922 923 static void __init memmap_init(void) 924 { 925 unsigned long start_pfn, end_pfn; 926 unsigned long hole_pfn = 0; 927 int i, j, zone_id = 0, nid; 928 929 for_each_mem_pfn_range(i, MAX_NUMNODES, &start_pfn, &end_pfn, &nid) { 930 struct pglist_data *node = NODE_DATA(nid); 931 932 for (j = 0; j < MAX_NR_ZONES; j++) { 933 struct zone *zone = node->node_zones + j; 934 935 if (!populated_zone(zone)) 936 continue; 937 938 memmap_init_zone_range(zone, start_pfn, end_pfn, 939 &hole_pfn); 940 zone_id = j; 941 } 942 } 943 944 #ifdef CONFIG_SPARSEMEM 945 /* 946 * Initialize the memory map for hole in the range [memory_end, 947 * section_end]. 948 * Append the pages in this hole to the highest zone in the last 949 * node. 950 * The call to init_unavailable_range() is outside the ifdef to 951 * silence the compiler warining about zone_id set but not used; 952 * for FLATMEM it is a nop anyway 953 */ 954 end_pfn = round_up(end_pfn, PAGES_PER_SECTION); 955 if (hole_pfn < end_pfn) 956 #endif 957 init_unavailable_range(hole_pfn, end_pfn, zone_id, nid); 958 } 959 960 #ifdef CONFIG_ZONE_DEVICE 961 static void __ref __init_zone_device_page(struct page *page, unsigned long pfn, 962 unsigned long zone_idx, int nid, 963 struct dev_pagemap *pgmap) 964 { 965 966 __init_single_page(page, pfn, zone_idx, nid); 967 968 /* 969 * Mark page reserved as it will need to wait for onlining 970 * phase for it to be fully associated with a zone. 971 * 972 * We can use the non-atomic __set_bit operation for setting 973 * the flag as we are still initializing the pages. 974 */ 975 __SetPageReserved(page); 976 977 /* 978 * ZONE_DEVICE pages union ->lru with a ->pgmap back pointer 979 * and zone_device_data. It is a bug if a ZONE_DEVICE page is 980 * ever freed or placed on a driver-private list. 981 */ 982 page->pgmap = pgmap; 983 page->zone_device_data = NULL; 984 985 /* 986 * Mark the block movable so that blocks are reserved for 987 * movable at startup. This will force kernel allocations 988 * to reserve their blocks rather than leaking throughout 989 * the address space during boot when many long-lived 990 * kernel allocations are made. 991 * 992 * Please note that MEMINIT_HOTPLUG path doesn't clear memmap 993 * because this is done early in section_activate() 994 */ 995 if (pageblock_aligned(pfn)) { 996 set_pageblock_migratetype(page, MIGRATE_MOVABLE); 997 cond_resched(); 998 } 999 1000 /* 1001 * ZONE_DEVICE pages are released directly to the driver page allocator 1002 * which will set the page count to 1 when allocating the page. 1003 */ 1004 if (pgmap->type == MEMORY_DEVICE_PRIVATE || 1005 pgmap->type == MEMORY_DEVICE_COHERENT) 1006 set_page_count(page, 0); 1007 } 1008 1009 /* 1010 * With compound page geometry and when struct pages are stored in ram most 1011 * tail pages are reused. Consequently, the amount of unique struct pages to 1012 * initialize is a lot smaller that the total amount of struct pages being 1013 * mapped. This is a paired / mild layering violation with explicit knowledge 1014 * of how the sparse_vmemmap internals handle compound pages in the lack 1015 * of an altmap. See vmemmap_populate_compound_pages(). 1016 */ 1017 static inline unsigned long compound_nr_pages(struct vmem_altmap *altmap, 1018 struct dev_pagemap *pgmap) 1019 { 1020 if (!vmemmap_can_optimize(altmap, pgmap)) 1021 return pgmap_vmemmap_nr(pgmap); 1022 1023 return 2 * (PAGE_SIZE / sizeof(struct page)); 1024 } 1025 1026 static void __ref memmap_init_compound(struct page *head, 1027 unsigned long head_pfn, 1028 unsigned long zone_idx, int nid, 1029 struct dev_pagemap *pgmap, 1030 unsigned long nr_pages) 1031 { 1032 unsigned long pfn, end_pfn = head_pfn + nr_pages; 1033 unsigned int order = pgmap->vmemmap_shift; 1034 1035 __SetPageHead(head); 1036 for (pfn = head_pfn + 1; pfn < end_pfn; pfn++) { 1037 struct page *page = pfn_to_page(pfn); 1038 1039 __init_zone_device_page(page, pfn, zone_idx, nid, pgmap); 1040 prep_compound_tail(head, pfn - head_pfn); 1041 set_page_count(page, 0); 1042 1043 /* 1044 * The first tail page stores important compound page info. 1045 * Call prep_compound_head() after the first tail page has 1046 * been initialized, to not have the data overwritten. 1047 */ 1048 if (pfn == head_pfn + 1) 1049 prep_compound_head(head, order); 1050 } 1051 } 1052 1053 void __ref memmap_init_zone_device(struct zone *zone, 1054 unsigned long start_pfn, 1055 unsigned long nr_pages, 1056 struct dev_pagemap *pgmap) 1057 { 1058 unsigned long pfn, end_pfn = start_pfn + nr_pages; 1059 struct pglist_data *pgdat = zone->zone_pgdat; 1060 struct vmem_altmap *altmap = pgmap_altmap(pgmap); 1061 unsigned int pfns_per_compound = pgmap_vmemmap_nr(pgmap); 1062 unsigned long zone_idx = zone_idx(zone); 1063 unsigned long start = jiffies; 1064 int nid = pgdat->node_id; 1065 1066 if (WARN_ON_ONCE(!pgmap || zone_idx != ZONE_DEVICE)) 1067 return; 1068 1069 /* 1070 * The call to memmap_init should have already taken care 1071 * of the pages reserved for the memmap, so we can just jump to 1072 * the end of that region and start processing the device pages. 1073 */ 1074 if (altmap) { 1075 start_pfn = altmap->base_pfn + vmem_altmap_offset(altmap); 1076 nr_pages = end_pfn - start_pfn; 1077 } 1078 1079 for (pfn = start_pfn; pfn < end_pfn; pfn += pfns_per_compound) { 1080 struct page *page = pfn_to_page(pfn); 1081 1082 __init_zone_device_page(page, pfn, zone_idx, nid, pgmap); 1083 1084 if (pfns_per_compound == 1) 1085 continue; 1086 1087 memmap_init_compound(page, pfn, zone_idx, nid, pgmap, 1088 compound_nr_pages(altmap, pgmap)); 1089 } 1090 1091 pr_debug("%s initialised %lu pages in %ums\n", __func__, 1092 nr_pages, jiffies_to_msecs(jiffies - start)); 1093 } 1094 #endif 1095 1096 /* 1097 * The zone ranges provided by the architecture do not include ZONE_MOVABLE 1098 * because it is sized independent of architecture. Unlike the other zones, 1099 * the starting point for ZONE_MOVABLE is not fixed. It may be different 1100 * in each node depending on the size of each node and how evenly kernelcore 1101 * is distributed. This helper function adjusts the zone ranges 1102 * provided by the architecture for a given node by using the end of the 1103 * highest usable zone for ZONE_MOVABLE. This preserves the assumption that 1104 * zones within a node are in order of monotonic increases memory addresses 1105 */ 1106 static void __init adjust_zone_range_for_zone_movable(int nid, 1107 unsigned long zone_type, 1108 unsigned long node_start_pfn, 1109 unsigned long node_end_pfn, 1110 unsigned long *zone_start_pfn, 1111 unsigned long *zone_end_pfn) 1112 { 1113 /* Only adjust if ZONE_MOVABLE is on this node */ 1114 if (zone_movable_pfn[nid]) { 1115 /* Size ZONE_MOVABLE */ 1116 if (zone_type == ZONE_MOVABLE) { 1117 *zone_start_pfn = zone_movable_pfn[nid]; 1118 *zone_end_pfn = min(node_end_pfn, 1119 arch_zone_highest_possible_pfn[movable_zone]); 1120 1121 /* Adjust for ZONE_MOVABLE starting within this range */ 1122 } else if (!mirrored_kernelcore && 1123 *zone_start_pfn < zone_movable_pfn[nid] && 1124 *zone_end_pfn > zone_movable_pfn[nid]) { 1125 *zone_end_pfn = zone_movable_pfn[nid]; 1126 1127 /* Check if this whole range is within ZONE_MOVABLE */ 1128 } else if (*zone_start_pfn >= zone_movable_pfn[nid]) 1129 *zone_start_pfn = *zone_end_pfn; 1130 } 1131 } 1132 1133 /* 1134 * Return the number of holes in a range on a node. If nid is MAX_NUMNODES, 1135 * then all holes in the requested range will be accounted for. 1136 */ 1137 unsigned long __init __absent_pages_in_range(int nid, 1138 unsigned long range_start_pfn, 1139 unsigned long range_end_pfn) 1140 { 1141 unsigned long nr_absent = range_end_pfn - range_start_pfn; 1142 unsigned long start_pfn, end_pfn; 1143 int i; 1144 1145 for_each_mem_pfn_range(i, nid, &start_pfn, &end_pfn, NULL) { 1146 start_pfn = clamp(start_pfn, range_start_pfn, range_end_pfn); 1147 end_pfn = clamp(end_pfn, range_start_pfn, range_end_pfn); 1148 nr_absent -= end_pfn - start_pfn; 1149 } 1150 return nr_absent; 1151 } 1152 1153 /** 1154 * absent_pages_in_range - Return number of page frames in holes within a range 1155 * @start_pfn: The start PFN to start searching for holes 1156 * @end_pfn: The end PFN to stop searching for holes 1157 * 1158 * Return: the number of pages frames in memory holes within a range. 1159 */ 1160 unsigned long __init absent_pages_in_range(unsigned long start_pfn, 1161 unsigned long end_pfn) 1162 { 1163 return __absent_pages_in_range(MAX_NUMNODES, start_pfn, end_pfn); 1164 } 1165 1166 /* Return the number of page frames in holes in a zone on a node */ 1167 static unsigned long __init zone_absent_pages_in_node(int nid, 1168 unsigned long zone_type, 1169 unsigned long zone_start_pfn, 1170 unsigned long zone_end_pfn) 1171 { 1172 unsigned long nr_absent; 1173 1174 /* zone is empty, we don't have any absent pages */ 1175 if (zone_start_pfn == zone_end_pfn) 1176 return 0; 1177 1178 nr_absent = __absent_pages_in_range(nid, zone_start_pfn, zone_end_pfn); 1179 1180 /* 1181 * ZONE_MOVABLE handling. 1182 * Treat pages to be ZONE_MOVABLE in ZONE_NORMAL as absent pages 1183 * and vice versa. 1184 */ 1185 if (mirrored_kernelcore && zone_movable_pfn[nid]) { 1186 unsigned long start_pfn, end_pfn; 1187 struct memblock_region *r; 1188 1189 for_each_mem_region(r) { 1190 start_pfn = clamp(memblock_region_memory_base_pfn(r), 1191 zone_start_pfn, zone_end_pfn); 1192 end_pfn = clamp(memblock_region_memory_end_pfn(r), 1193 zone_start_pfn, zone_end_pfn); 1194 1195 if (zone_type == ZONE_MOVABLE && 1196 memblock_is_mirror(r)) 1197 nr_absent += end_pfn - start_pfn; 1198 1199 if (zone_type == ZONE_NORMAL && 1200 !memblock_is_mirror(r)) 1201 nr_absent += end_pfn - start_pfn; 1202 } 1203 } 1204 1205 return nr_absent; 1206 } 1207 1208 /* 1209 * Return the number of pages a zone spans in a node, including holes 1210 * present_pages = zone_spanned_pages_in_node() - zone_absent_pages_in_node() 1211 */ 1212 static unsigned long __init zone_spanned_pages_in_node(int nid, 1213 unsigned long zone_type, 1214 unsigned long node_start_pfn, 1215 unsigned long node_end_pfn, 1216 unsigned long *zone_start_pfn, 1217 unsigned long *zone_end_pfn) 1218 { 1219 unsigned long zone_low = arch_zone_lowest_possible_pfn[zone_type]; 1220 unsigned long zone_high = arch_zone_highest_possible_pfn[zone_type]; 1221 1222 /* Get the start and end of the zone */ 1223 *zone_start_pfn = clamp(node_start_pfn, zone_low, zone_high); 1224 *zone_end_pfn = clamp(node_end_pfn, zone_low, zone_high); 1225 adjust_zone_range_for_zone_movable(nid, zone_type, 1226 node_start_pfn, node_end_pfn, 1227 zone_start_pfn, zone_end_pfn); 1228 1229 /* Check that this node has pages within the zone's required range */ 1230 if (*zone_end_pfn < node_start_pfn || *zone_start_pfn > node_end_pfn) 1231 return 0; 1232 1233 /* Move the zone boundaries inside the node if necessary */ 1234 *zone_end_pfn = min(*zone_end_pfn, node_end_pfn); 1235 *zone_start_pfn = max(*zone_start_pfn, node_start_pfn); 1236 1237 /* Return the spanned pages */ 1238 return *zone_end_pfn - *zone_start_pfn; 1239 } 1240 1241 static void __init reset_memoryless_node_totalpages(struct pglist_data *pgdat) 1242 { 1243 struct zone *z; 1244 1245 for (z = pgdat->node_zones; z < pgdat->node_zones + MAX_NR_ZONES; z++) { 1246 z->zone_start_pfn = 0; 1247 z->spanned_pages = 0; 1248 z->present_pages = 0; 1249 #if defined(CONFIG_MEMORY_HOTPLUG) 1250 z->present_early_pages = 0; 1251 #endif 1252 } 1253 1254 pgdat->node_spanned_pages = 0; 1255 pgdat->node_present_pages = 0; 1256 pr_debug("On node %d totalpages: 0\n", pgdat->node_id); 1257 } 1258 1259 static void __init calculate_node_totalpages(struct pglist_data *pgdat, 1260 unsigned long node_start_pfn, 1261 unsigned long node_end_pfn) 1262 { 1263 unsigned long realtotalpages = 0, totalpages = 0; 1264 enum zone_type i; 1265 1266 for (i = 0; i < MAX_NR_ZONES; i++) { 1267 struct zone *zone = pgdat->node_zones + i; 1268 unsigned long zone_start_pfn, zone_end_pfn; 1269 unsigned long spanned, absent; 1270 unsigned long real_size; 1271 1272 spanned = zone_spanned_pages_in_node(pgdat->node_id, i, 1273 node_start_pfn, 1274 node_end_pfn, 1275 &zone_start_pfn, 1276 &zone_end_pfn); 1277 absent = zone_absent_pages_in_node(pgdat->node_id, i, 1278 zone_start_pfn, 1279 zone_end_pfn); 1280 1281 real_size = spanned - absent; 1282 1283 if (spanned) 1284 zone->zone_start_pfn = zone_start_pfn; 1285 else 1286 zone->zone_start_pfn = 0; 1287 zone->spanned_pages = spanned; 1288 zone->present_pages = real_size; 1289 #if defined(CONFIG_MEMORY_HOTPLUG) 1290 zone->present_early_pages = real_size; 1291 #endif 1292 1293 totalpages += spanned; 1294 realtotalpages += real_size; 1295 } 1296 1297 pgdat->node_spanned_pages = totalpages; 1298 pgdat->node_present_pages = realtotalpages; 1299 pr_debug("On node %d totalpages: %lu\n", pgdat->node_id, realtotalpages); 1300 } 1301 1302 static unsigned long __init calc_memmap_size(unsigned long spanned_pages, 1303 unsigned long present_pages) 1304 { 1305 unsigned long pages = spanned_pages; 1306 1307 /* 1308 * Provide a more accurate estimation if there are holes within 1309 * the zone and SPARSEMEM is in use. If there are holes within the 1310 * zone, each populated memory region may cost us one or two extra 1311 * memmap pages due to alignment because memmap pages for each 1312 * populated regions may not be naturally aligned on page boundary. 1313 * So the (present_pages >> 4) heuristic is a tradeoff for that. 1314 */ 1315 if (spanned_pages > present_pages + (present_pages >> 4) && 1316 IS_ENABLED(CONFIG_SPARSEMEM)) 1317 pages = present_pages; 1318 1319 return PAGE_ALIGN(pages * sizeof(struct page)) >> PAGE_SHIFT; 1320 } 1321 1322 #ifdef CONFIG_TRANSPARENT_HUGEPAGE 1323 static void pgdat_init_split_queue(struct pglist_data *pgdat) 1324 { 1325 struct deferred_split *ds_queue = &pgdat->deferred_split_queue; 1326 1327 spin_lock_init(&ds_queue->split_queue_lock); 1328 INIT_LIST_HEAD(&ds_queue->split_queue); 1329 ds_queue->split_queue_len = 0; 1330 } 1331 #else 1332 static void pgdat_init_split_queue(struct pglist_data *pgdat) {} 1333 #endif 1334 1335 #ifdef CONFIG_COMPACTION 1336 static void pgdat_init_kcompactd(struct pglist_data *pgdat) 1337 { 1338 init_waitqueue_head(&pgdat->kcompactd_wait); 1339 } 1340 #else 1341 static void pgdat_init_kcompactd(struct pglist_data *pgdat) {} 1342 #endif 1343 1344 static void __meminit pgdat_init_internals(struct pglist_data *pgdat) 1345 { 1346 int i; 1347 1348 pgdat_resize_init(pgdat); 1349 pgdat_kswapd_lock_init(pgdat); 1350 1351 pgdat_init_split_queue(pgdat); 1352 pgdat_init_kcompactd(pgdat); 1353 1354 init_waitqueue_head(&pgdat->kswapd_wait); 1355 init_waitqueue_head(&pgdat->pfmemalloc_wait); 1356 1357 for (i = 0; i < NR_VMSCAN_THROTTLE; i++) 1358 init_waitqueue_head(&pgdat->reclaim_wait[i]); 1359 1360 pgdat_page_ext_init(pgdat); 1361 lruvec_init(&pgdat->__lruvec); 1362 } 1363 1364 static void __meminit zone_init_internals(struct zone *zone, enum zone_type idx, int nid, 1365 unsigned long remaining_pages) 1366 { 1367 atomic_long_set(&zone->managed_pages, remaining_pages); 1368 zone_set_nid(zone, nid); 1369 zone->name = zone_names[idx]; 1370 zone->zone_pgdat = NODE_DATA(nid); 1371 spin_lock_init(&zone->lock); 1372 zone_seqlock_init(zone); 1373 zone_pcp_init(zone); 1374 } 1375 1376 static void __meminit zone_init_free_lists(struct zone *zone) 1377 { 1378 unsigned int order, t; 1379 for_each_migratetype_order(order, t) { 1380 INIT_LIST_HEAD(&zone->free_area[order].free_list[t]); 1381 zone->free_area[order].nr_free = 0; 1382 } 1383 1384 #ifdef CONFIG_UNACCEPTED_MEMORY 1385 INIT_LIST_HEAD(&zone->unaccepted_pages); 1386 #endif 1387 } 1388 1389 void __meminit init_currently_empty_zone(struct zone *zone, 1390 unsigned long zone_start_pfn, 1391 unsigned long size) 1392 { 1393 struct pglist_data *pgdat = zone->zone_pgdat; 1394 int zone_idx = zone_idx(zone) + 1; 1395 1396 if (zone_idx > pgdat->nr_zones) 1397 pgdat->nr_zones = zone_idx; 1398 1399 zone->zone_start_pfn = zone_start_pfn; 1400 1401 mminit_dprintk(MMINIT_TRACE, "memmap_init", 1402 "Initialising map node %d zone %lu pfns %lu -> %lu\n", 1403 pgdat->node_id, 1404 (unsigned long)zone_idx(zone), 1405 zone_start_pfn, (zone_start_pfn + size)); 1406 1407 zone_init_free_lists(zone); 1408 zone->initialized = 1; 1409 } 1410 1411 #ifndef CONFIG_SPARSEMEM 1412 /* 1413 * Calculate the size of the zone->blockflags rounded to an unsigned long 1414 * Start by making sure zonesize is a multiple of pageblock_order by rounding 1415 * up. Then use 1 NR_PAGEBLOCK_BITS worth of bits per pageblock, finally 1416 * round what is now in bits to nearest long in bits, then return it in 1417 * bytes. 1418 */ 1419 static unsigned long __init usemap_size(unsigned long zone_start_pfn, unsigned long zonesize) 1420 { 1421 unsigned long usemapsize; 1422 1423 zonesize += zone_start_pfn & (pageblock_nr_pages-1); 1424 usemapsize = roundup(zonesize, pageblock_nr_pages); 1425 usemapsize = usemapsize >> pageblock_order; 1426 usemapsize *= NR_PAGEBLOCK_BITS; 1427 usemapsize = roundup(usemapsize, 8 * sizeof(unsigned long)); 1428 1429 return usemapsize / 8; 1430 } 1431 1432 static void __ref setup_usemap(struct zone *zone) 1433 { 1434 unsigned long usemapsize = usemap_size(zone->zone_start_pfn, 1435 zone->spanned_pages); 1436 zone->pageblock_flags = NULL; 1437 if (usemapsize) { 1438 zone->pageblock_flags = 1439 memblock_alloc_node(usemapsize, SMP_CACHE_BYTES, 1440 zone_to_nid(zone)); 1441 if (!zone->pageblock_flags) 1442 panic("Failed to allocate %ld bytes for zone %s pageblock flags on node %d\n", 1443 usemapsize, zone->name, zone_to_nid(zone)); 1444 } 1445 } 1446 #else 1447 static inline void setup_usemap(struct zone *zone) {} 1448 #endif /* CONFIG_SPARSEMEM */ 1449 1450 #ifdef CONFIG_HUGETLB_PAGE_SIZE_VARIABLE 1451 1452 /* Initialise the number of pages represented by NR_PAGEBLOCK_BITS */ 1453 void __init set_pageblock_order(void) 1454 { 1455 unsigned int order = MAX_ORDER; 1456 1457 /* Check that pageblock_nr_pages has not already been setup */ 1458 if (pageblock_order) 1459 return; 1460 1461 /* Don't let pageblocks exceed the maximum allocation granularity. */ 1462 if (HPAGE_SHIFT > PAGE_SHIFT && HUGETLB_PAGE_ORDER < order) 1463 order = HUGETLB_PAGE_ORDER; 1464 1465 /* 1466 * Assume the largest contiguous order of interest is a huge page. 1467 * This value may be variable depending on boot parameters on IA64 and 1468 * powerpc. 1469 */ 1470 pageblock_order = order; 1471 } 1472 #else /* CONFIG_HUGETLB_PAGE_SIZE_VARIABLE */ 1473 1474 /* 1475 * When CONFIG_HUGETLB_PAGE_SIZE_VARIABLE is not set, set_pageblock_order() 1476 * is unused as pageblock_order is set at compile-time. See 1477 * include/linux/pageblock-flags.h for the values of pageblock_order based on 1478 * the kernel config 1479 */ 1480 void __init set_pageblock_order(void) 1481 { 1482 } 1483 1484 #endif /* CONFIG_HUGETLB_PAGE_SIZE_VARIABLE */ 1485 1486 /* 1487 * Set up the zone data structures 1488 * - init pgdat internals 1489 * - init all zones belonging to this node 1490 * 1491 * NOTE: this function is only called during memory hotplug 1492 */ 1493 #ifdef CONFIG_MEMORY_HOTPLUG 1494 void __ref free_area_init_core_hotplug(struct pglist_data *pgdat) 1495 { 1496 int nid = pgdat->node_id; 1497 enum zone_type z; 1498 int cpu; 1499 1500 pgdat_init_internals(pgdat); 1501 1502 if (pgdat->per_cpu_nodestats == &boot_nodestats) 1503 pgdat->per_cpu_nodestats = alloc_percpu(struct per_cpu_nodestat); 1504 1505 /* 1506 * Reset the nr_zones, order and highest_zoneidx before reuse. 1507 * Note that kswapd will init kswapd_highest_zoneidx properly 1508 * when it starts in the near future. 1509 */ 1510 pgdat->nr_zones = 0; 1511 pgdat->kswapd_order = 0; 1512 pgdat->kswapd_highest_zoneidx = 0; 1513 pgdat->node_start_pfn = 0; 1514 pgdat->node_present_pages = 0; 1515 1516 for_each_online_cpu(cpu) { 1517 struct per_cpu_nodestat *p; 1518 1519 p = per_cpu_ptr(pgdat->per_cpu_nodestats, cpu); 1520 memset(p, 0, sizeof(*p)); 1521 } 1522 1523 /* 1524 * When memory is hot-added, all the memory is in offline state. So 1525 * clear all zones' present_pages and managed_pages because they will 1526 * be updated in online_pages() and offline_pages(). 1527 */ 1528 for (z = 0; z < MAX_NR_ZONES; z++) { 1529 struct zone *zone = pgdat->node_zones + z; 1530 1531 zone->present_pages = 0; 1532 zone_init_internals(zone, z, nid, 0); 1533 } 1534 } 1535 #endif 1536 1537 /* 1538 * Set up the zone data structures: 1539 * - mark all pages reserved 1540 * - mark all memory queues empty 1541 * - clear the memory bitmaps 1542 * 1543 * NOTE: pgdat should get zeroed by caller. 1544 * NOTE: this function is only called during early init. 1545 */ 1546 static void __init free_area_init_core(struct pglist_data *pgdat) 1547 { 1548 enum zone_type j; 1549 int nid = pgdat->node_id; 1550 1551 pgdat_init_internals(pgdat); 1552 pgdat->per_cpu_nodestats = &boot_nodestats; 1553 1554 for (j = 0; j < MAX_NR_ZONES; j++) { 1555 struct zone *zone = pgdat->node_zones + j; 1556 unsigned long size, freesize, memmap_pages; 1557 1558 size = zone->spanned_pages; 1559 freesize = zone->present_pages; 1560 1561 /* 1562 * Adjust freesize so that it accounts for how much memory 1563 * is used by this zone for memmap. This affects the watermark 1564 * and per-cpu initialisations 1565 */ 1566 memmap_pages = calc_memmap_size(size, freesize); 1567 if (!is_highmem_idx(j)) { 1568 if (freesize >= memmap_pages) { 1569 freesize -= memmap_pages; 1570 if (memmap_pages) 1571 pr_debug(" %s zone: %lu pages used for memmap\n", 1572 zone_names[j], memmap_pages); 1573 } else 1574 pr_warn(" %s zone: %lu memmap pages exceeds freesize %lu\n", 1575 zone_names[j], memmap_pages, freesize); 1576 } 1577 1578 /* Account for reserved pages */ 1579 if (j == 0 && freesize > dma_reserve) { 1580 freesize -= dma_reserve; 1581 pr_debug(" %s zone: %lu pages reserved\n", zone_names[0], dma_reserve); 1582 } 1583 1584 if (!is_highmem_idx(j)) 1585 nr_kernel_pages += freesize; 1586 /* Charge for highmem memmap if there are enough kernel pages */ 1587 else if (nr_kernel_pages > memmap_pages * 2) 1588 nr_kernel_pages -= memmap_pages; 1589 nr_all_pages += freesize; 1590 1591 /* 1592 * Set an approximate value for lowmem here, it will be adjusted 1593 * when the bootmem allocator frees pages into the buddy system. 1594 * And all highmem pages will be managed by the buddy system. 1595 */ 1596 zone_init_internals(zone, j, nid, freesize); 1597 1598 if (!size) 1599 continue; 1600 1601 setup_usemap(zone); 1602 init_currently_empty_zone(zone, zone->zone_start_pfn, size); 1603 } 1604 } 1605 1606 void __init *memmap_alloc(phys_addr_t size, phys_addr_t align, 1607 phys_addr_t min_addr, int nid, bool exact_nid) 1608 { 1609 void *ptr; 1610 1611 if (exact_nid) 1612 ptr = memblock_alloc_exact_nid_raw(size, align, min_addr, 1613 MEMBLOCK_ALLOC_ACCESSIBLE, 1614 nid); 1615 else 1616 ptr = memblock_alloc_try_nid_raw(size, align, min_addr, 1617 MEMBLOCK_ALLOC_ACCESSIBLE, 1618 nid); 1619 1620 if (ptr && size > 0) 1621 page_init_poison(ptr, size); 1622 1623 return ptr; 1624 } 1625 1626 #ifdef CONFIG_FLATMEM 1627 static void __init alloc_node_mem_map(struct pglist_data *pgdat) 1628 { 1629 unsigned long __maybe_unused start = 0; 1630 unsigned long __maybe_unused offset = 0; 1631 1632 /* Skip empty nodes */ 1633 if (!pgdat->node_spanned_pages) 1634 return; 1635 1636 start = pgdat->node_start_pfn & ~(MAX_ORDER_NR_PAGES - 1); 1637 offset = pgdat->node_start_pfn - start; 1638 /* ia64 gets its own node_mem_map, before this, without bootmem */ 1639 if (!pgdat->node_mem_map) { 1640 unsigned long size, end; 1641 struct page *map; 1642 1643 /* 1644 * The zone's endpoints aren't required to be MAX_ORDER 1645 * aligned but the node_mem_map endpoints must be in order 1646 * for the buddy allocator to function correctly. 1647 */ 1648 end = pgdat_end_pfn(pgdat); 1649 end = ALIGN(end, MAX_ORDER_NR_PAGES); 1650 size = (end - start) * sizeof(struct page); 1651 map = memmap_alloc(size, SMP_CACHE_BYTES, MEMBLOCK_LOW_LIMIT, 1652 pgdat->node_id, false); 1653 if (!map) 1654 panic("Failed to allocate %ld bytes for node %d memory map\n", 1655 size, pgdat->node_id); 1656 pgdat->node_mem_map = map + offset; 1657 } 1658 pr_debug("%s: node %d, pgdat %08lx, node_mem_map %08lx\n", 1659 __func__, pgdat->node_id, (unsigned long)pgdat, 1660 (unsigned long)pgdat->node_mem_map); 1661 #ifndef CONFIG_NUMA 1662 /* 1663 * With no DISCONTIG, the global mem_map is just set as node 0's 1664 */ 1665 if (pgdat == NODE_DATA(0)) { 1666 mem_map = NODE_DATA(0)->node_mem_map; 1667 if (page_to_pfn(mem_map) != pgdat->node_start_pfn) 1668 mem_map -= offset; 1669 } 1670 #endif 1671 } 1672 #else 1673 static inline void alloc_node_mem_map(struct pglist_data *pgdat) { } 1674 #endif /* CONFIG_FLATMEM */ 1675 1676 /** 1677 * get_pfn_range_for_nid - Return the start and end page frames for a node 1678 * @nid: The nid to return the range for. If MAX_NUMNODES, the min and max PFN are returned. 1679 * @start_pfn: Passed by reference. On return, it will have the node start_pfn. 1680 * @end_pfn: Passed by reference. On return, it will have the node end_pfn. 1681 * 1682 * It returns the start and end page frame of a node based on information 1683 * provided by memblock_set_node(). If called for a node 1684 * with no available memory, a warning is printed and the start and end 1685 * PFNs will be 0. 1686 */ 1687 void __init get_pfn_range_for_nid(unsigned int nid, 1688 unsigned long *start_pfn, unsigned long *end_pfn) 1689 { 1690 unsigned long this_start_pfn, this_end_pfn; 1691 int i; 1692 1693 *start_pfn = -1UL; 1694 *end_pfn = 0; 1695 1696 for_each_mem_pfn_range(i, nid, &this_start_pfn, &this_end_pfn, NULL) { 1697 *start_pfn = min(*start_pfn, this_start_pfn); 1698 *end_pfn = max(*end_pfn, this_end_pfn); 1699 } 1700 1701 if (*start_pfn == -1UL) 1702 *start_pfn = 0; 1703 } 1704 1705 static void __init free_area_init_node(int nid) 1706 { 1707 pg_data_t *pgdat = NODE_DATA(nid); 1708 unsigned long start_pfn = 0; 1709 unsigned long end_pfn = 0; 1710 1711 /* pg_data_t should be reset to zero when it's allocated */ 1712 WARN_ON(pgdat->nr_zones || pgdat->kswapd_highest_zoneidx); 1713 1714 get_pfn_range_for_nid(nid, &start_pfn, &end_pfn); 1715 1716 pgdat->node_id = nid; 1717 pgdat->node_start_pfn = start_pfn; 1718 pgdat->per_cpu_nodestats = NULL; 1719 1720 if (start_pfn != end_pfn) { 1721 pr_info("Initmem setup node %d [mem %#018Lx-%#018Lx]\n", nid, 1722 (u64)start_pfn << PAGE_SHIFT, 1723 end_pfn ? ((u64)end_pfn << PAGE_SHIFT) - 1 : 0); 1724 1725 calculate_node_totalpages(pgdat, start_pfn, end_pfn); 1726 } else { 1727 pr_info("Initmem setup node %d as memoryless\n", nid); 1728 1729 reset_memoryless_node_totalpages(pgdat); 1730 } 1731 1732 alloc_node_mem_map(pgdat); 1733 pgdat_set_deferred_range(pgdat); 1734 1735 free_area_init_core(pgdat); 1736 lru_gen_init_pgdat(pgdat); 1737 } 1738 1739 /* Any regular or high memory on that node ? */ 1740 static void check_for_memory(pg_data_t *pgdat) 1741 { 1742 enum zone_type zone_type; 1743 1744 for (zone_type = 0; zone_type <= ZONE_MOVABLE - 1; zone_type++) { 1745 struct zone *zone = &pgdat->node_zones[zone_type]; 1746 if (populated_zone(zone)) { 1747 if (IS_ENABLED(CONFIG_HIGHMEM)) 1748 node_set_state(pgdat->node_id, N_HIGH_MEMORY); 1749 if (zone_type <= ZONE_NORMAL) 1750 node_set_state(pgdat->node_id, N_NORMAL_MEMORY); 1751 break; 1752 } 1753 } 1754 } 1755 1756 #if MAX_NUMNODES > 1 1757 /* 1758 * Figure out the number of possible node ids. 1759 */ 1760 void __init setup_nr_node_ids(void) 1761 { 1762 unsigned int highest; 1763 1764 highest = find_last_bit(node_possible_map.bits, MAX_NUMNODES); 1765 nr_node_ids = highest + 1; 1766 } 1767 #endif 1768 1769 /* 1770 * Some architectures, e.g. ARC may have ZONE_HIGHMEM below ZONE_NORMAL. For 1771 * such cases we allow max_zone_pfn sorted in the descending order 1772 */ 1773 static bool arch_has_descending_max_zone_pfns(void) 1774 { 1775 return IS_ENABLED(CONFIG_ARC) && !IS_ENABLED(CONFIG_ARC_HAS_PAE40); 1776 } 1777 1778 /** 1779 * free_area_init - Initialise all pg_data_t and zone data 1780 * @max_zone_pfn: an array of max PFNs for each zone 1781 * 1782 * This will call free_area_init_node() for each active node in the system. 1783 * Using the page ranges provided by memblock_set_node(), the size of each 1784 * zone in each node and their holes is calculated. If the maximum PFN 1785 * between two adjacent zones match, it is assumed that the zone is empty. 1786 * For example, if arch_max_dma_pfn == arch_max_dma32_pfn, it is assumed 1787 * that arch_max_dma32_pfn has no pages. It is also assumed that a zone 1788 * starts where the previous one ended. For example, ZONE_DMA32 starts 1789 * at arch_max_dma_pfn. 1790 */ 1791 void __init free_area_init(unsigned long *max_zone_pfn) 1792 { 1793 unsigned long start_pfn, end_pfn; 1794 int i, nid, zone; 1795 bool descending; 1796 1797 /* Record where the zone boundaries are */ 1798 memset(arch_zone_lowest_possible_pfn, 0, 1799 sizeof(arch_zone_lowest_possible_pfn)); 1800 memset(arch_zone_highest_possible_pfn, 0, 1801 sizeof(arch_zone_highest_possible_pfn)); 1802 1803 start_pfn = PHYS_PFN(memblock_start_of_DRAM()); 1804 descending = arch_has_descending_max_zone_pfns(); 1805 1806 for (i = 0; i < MAX_NR_ZONES; i++) { 1807 if (descending) 1808 zone = MAX_NR_ZONES - i - 1; 1809 else 1810 zone = i; 1811 1812 if (zone == ZONE_MOVABLE) 1813 continue; 1814 1815 end_pfn = max(max_zone_pfn[zone], start_pfn); 1816 arch_zone_lowest_possible_pfn[zone] = start_pfn; 1817 arch_zone_highest_possible_pfn[zone] = end_pfn; 1818 1819 start_pfn = end_pfn; 1820 } 1821 1822 /* Find the PFNs that ZONE_MOVABLE begins at in each node */ 1823 memset(zone_movable_pfn, 0, sizeof(zone_movable_pfn)); 1824 find_zone_movable_pfns_for_nodes(); 1825 1826 /* Print out the zone ranges */ 1827 pr_info("Zone ranges:\n"); 1828 for (i = 0; i < MAX_NR_ZONES; i++) { 1829 if (i == ZONE_MOVABLE) 1830 continue; 1831 pr_info(" %-8s ", zone_names[i]); 1832 if (arch_zone_lowest_possible_pfn[i] == 1833 arch_zone_highest_possible_pfn[i]) 1834 pr_cont("empty\n"); 1835 else 1836 pr_cont("[mem %#018Lx-%#018Lx]\n", 1837 (u64)arch_zone_lowest_possible_pfn[i] 1838 << PAGE_SHIFT, 1839 ((u64)arch_zone_highest_possible_pfn[i] 1840 << PAGE_SHIFT) - 1); 1841 } 1842 1843 /* Print out the PFNs ZONE_MOVABLE begins at in each node */ 1844 pr_info("Movable zone start for each node\n"); 1845 for (i = 0; i < MAX_NUMNODES; i++) { 1846 if (zone_movable_pfn[i]) 1847 pr_info(" Node %d: %#018Lx\n", i, 1848 (u64)zone_movable_pfn[i] << PAGE_SHIFT); 1849 } 1850 1851 /* 1852 * Print out the early node map, and initialize the 1853 * subsection-map relative to active online memory ranges to 1854 * enable future "sub-section" extensions of the memory map. 1855 */ 1856 pr_info("Early memory node ranges\n"); 1857 for_each_mem_pfn_range(i, MAX_NUMNODES, &start_pfn, &end_pfn, &nid) { 1858 pr_info(" node %3d: [mem %#018Lx-%#018Lx]\n", nid, 1859 (u64)start_pfn << PAGE_SHIFT, 1860 ((u64)end_pfn << PAGE_SHIFT) - 1); 1861 subsection_map_init(start_pfn, end_pfn - start_pfn); 1862 } 1863 1864 /* Initialise every node */ 1865 mminit_verify_pageflags_layout(); 1866 setup_nr_node_ids(); 1867 set_pageblock_order(); 1868 1869 for_each_node(nid) { 1870 pg_data_t *pgdat; 1871 1872 if (!node_online(nid)) { 1873 pr_info("Initializing node %d as memoryless\n", nid); 1874 1875 /* Allocator not initialized yet */ 1876 pgdat = arch_alloc_nodedata(nid); 1877 if (!pgdat) 1878 panic("Cannot allocate %zuB for node %d.\n", 1879 sizeof(*pgdat), nid); 1880 arch_refresh_nodedata(nid, pgdat); 1881 free_area_init_node(nid); 1882 1883 /* 1884 * We do not want to confuse userspace by sysfs 1885 * files/directories for node without any memory 1886 * attached to it, so this node is not marked as 1887 * N_MEMORY and not marked online so that no sysfs 1888 * hierarchy will be created via register_one_node for 1889 * it. The pgdat will get fully initialized by 1890 * hotadd_init_pgdat() when memory is hotplugged into 1891 * this node. 1892 */ 1893 continue; 1894 } 1895 1896 pgdat = NODE_DATA(nid); 1897 free_area_init_node(nid); 1898 1899 /* Any memory on that node */ 1900 if (pgdat->node_present_pages) 1901 node_set_state(nid, N_MEMORY); 1902 check_for_memory(pgdat); 1903 } 1904 1905 memmap_init(); 1906 1907 /* disable hash distribution for systems with a single node */ 1908 fixup_hashdist(); 1909 } 1910 1911 /** 1912 * node_map_pfn_alignment - determine the maximum internode alignment 1913 * 1914 * This function should be called after node map is populated and sorted. 1915 * It calculates the maximum power of two alignment which can distinguish 1916 * all the nodes. 1917 * 1918 * For example, if all nodes are 1GiB and aligned to 1GiB, the return value 1919 * would indicate 1GiB alignment with (1 << (30 - PAGE_SHIFT)). If the 1920 * nodes are shifted by 256MiB, 256MiB. Note that if only the last node is 1921 * shifted, 1GiB is enough and this function will indicate so. 1922 * 1923 * This is used to test whether pfn -> nid mapping of the chosen memory 1924 * model has fine enough granularity to avoid incorrect mapping for the 1925 * populated node map. 1926 * 1927 * Return: the determined alignment in pfn's. 0 if there is no alignment 1928 * requirement (single node). 1929 */ 1930 unsigned long __init node_map_pfn_alignment(void) 1931 { 1932 unsigned long accl_mask = 0, last_end = 0; 1933 unsigned long start, end, mask; 1934 int last_nid = NUMA_NO_NODE; 1935 int i, nid; 1936 1937 for_each_mem_pfn_range(i, MAX_NUMNODES, &start, &end, &nid) { 1938 if (!start || last_nid < 0 || last_nid == nid) { 1939 last_nid = nid; 1940 last_end = end; 1941 continue; 1942 } 1943 1944 /* 1945 * Start with a mask granular enough to pin-point to the 1946 * start pfn and tick off bits one-by-one until it becomes 1947 * too coarse to separate the current node from the last. 1948 */ 1949 mask = ~((1 << __ffs(start)) - 1); 1950 while (mask && last_end <= (start & (mask << 1))) 1951 mask <<= 1; 1952 1953 /* accumulate all internode masks */ 1954 accl_mask |= mask; 1955 } 1956 1957 /* convert mask to number of pages */ 1958 return ~accl_mask + 1; 1959 } 1960 1961 #ifdef CONFIG_DEFERRED_STRUCT_PAGE_INIT 1962 static void __init deferred_free_range(unsigned long pfn, 1963 unsigned long nr_pages) 1964 { 1965 struct page *page; 1966 unsigned long i; 1967 1968 if (!nr_pages) 1969 return; 1970 1971 page = pfn_to_page(pfn); 1972 1973 /* Free a large naturally-aligned chunk if possible */ 1974 if (nr_pages == MAX_ORDER_NR_PAGES && IS_MAX_ORDER_ALIGNED(pfn)) { 1975 for (i = 0; i < nr_pages; i += pageblock_nr_pages) 1976 set_pageblock_migratetype(page + i, MIGRATE_MOVABLE); 1977 __free_pages_core(page, MAX_ORDER); 1978 return; 1979 } 1980 1981 /* Accept chunks smaller than MAX_ORDER upfront */ 1982 accept_memory(PFN_PHYS(pfn), PFN_PHYS(pfn + nr_pages)); 1983 1984 for (i = 0; i < nr_pages; i++, page++, pfn++) { 1985 if (pageblock_aligned(pfn)) 1986 set_pageblock_migratetype(page, MIGRATE_MOVABLE); 1987 __free_pages_core(page, 0); 1988 } 1989 } 1990 1991 /* Completion tracking for deferred_init_memmap() threads */ 1992 static atomic_t pgdat_init_n_undone __initdata; 1993 static __initdata DECLARE_COMPLETION(pgdat_init_all_done_comp); 1994 1995 static inline void __init pgdat_init_report_one_done(void) 1996 { 1997 if (atomic_dec_and_test(&pgdat_init_n_undone)) 1998 complete(&pgdat_init_all_done_comp); 1999 } 2000 2001 /* 2002 * Returns true if page needs to be initialized or freed to buddy allocator. 2003 * 2004 * We check if a current MAX_ORDER block is valid by only checking the validity 2005 * of the head pfn. 2006 */ 2007 static inline bool __init deferred_pfn_valid(unsigned long pfn) 2008 { 2009 if (IS_MAX_ORDER_ALIGNED(pfn) && !pfn_valid(pfn)) 2010 return false; 2011 return true; 2012 } 2013 2014 /* 2015 * Free pages to buddy allocator. Try to free aligned pages in 2016 * MAX_ORDER_NR_PAGES sizes. 2017 */ 2018 static void __init deferred_free_pages(unsigned long pfn, 2019 unsigned long end_pfn) 2020 { 2021 unsigned long nr_free = 0; 2022 2023 for (; pfn < end_pfn; pfn++) { 2024 if (!deferred_pfn_valid(pfn)) { 2025 deferred_free_range(pfn - nr_free, nr_free); 2026 nr_free = 0; 2027 } else if (IS_MAX_ORDER_ALIGNED(pfn)) { 2028 deferred_free_range(pfn - nr_free, nr_free); 2029 nr_free = 1; 2030 } else { 2031 nr_free++; 2032 } 2033 } 2034 /* Free the last block of pages to allocator */ 2035 deferred_free_range(pfn - nr_free, nr_free); 2036 } 2037 2038 /* 2039 * Initialize struct pages. We minimize pfn page lookups and scheduler checks 2040 * by performing it only once every MAX_ORDER_NR_PAGES. 2041 * Return number of pages initialized. 2042 */ 2043 static unsigned long __init deferred_init_pages(struct zone *zone, 2044 unsigned long pfn, 2045 unsigned long end_pfn) 2046 { 2047 int nid = zone_to_nid(zone); 2048 unsigned long nr_pages = 0; 2049 int zid = zone_idx(zone); 2050 struct page *page = NULL; 2051 2052 for (; pfn < end_pfn; pfn++) { 2053 if (!deferred_pfn_valid(pfn)) { 2054 page = NULL; 2055 continue; 2056 } else if (!page || IS_MAX_ORDER_ALIGNED(pfn)) { 2057 page = pfn_to_page(pfn); 2058 } else { 2059 page++; 2060 } 2061 __init_single_page(page, pfn, zid, nid); 2062 nr_pages++; 2063 } 2064 return (nr_pages); 2065 } 2066 2067 /* 2068 * This function is meant to pre-load the iterator for the zone init. 2069 * Specifically it walks through the ranges until we are caught up to the 2070 * first_init_pfn value and exits there. If we never encounter the value we 2071 * return false indicating there are no valid ranges left. 2072 */ 2073 static bool __init 2074 deferred_init_mem_pfn_range_in_zone(u64 *i, struct zone *zone, 2075 unsigned long *spfn, unsigned long *epfn, 2076 unsigned long first_init_pfn) 2077 { 2078 u64 j; 2079 2080 /* 2081 * Start out by walking through the ranges in this zone that have 2082 * already been initialized. We don't need to do anything with them 2083 * so we just need to flush them out of the system. 2084 */ 2085 for_each_free_mem_pfn_range_in_zone(j, zone, spfn, epfn) { 2086 if (*epfn <= first_init_pfn) 2087 continue; 2088 if (*spfn < first_init_pfn) 2089 *spfn = first_init_pfn; 2090 *i = j; 2091 return true; 2092 } 2093 2094 return false; 2095 } 2096 2097 /* 2098 * Initialize and free pages. We do it in two loops: first we initialize 2099 * struct page, then free to buddy allocator, because while we are 2100 * freeing pages we can access pages that are ahead (computing buddy 2101 * page in __free_one_page()). 2102 * 2103 * In order to try and keep some memory in the cache we have the loop 2104 * broken along max page order boundaries. This way we will not cause 2105 * any issues with the buddy page computation. 2106 */ 2107 static unsigned long __init 2108 deferred_init_maxorder(u64 *i, struct zone *zone, unsigned long *start_pfn, 2109 unsigned long *end_pfn) 2110 { 2111 unsigned long mo_pfn = ALIGN(*start_pfn + 1, MAX_ORDER_NR_PAGES); 2112 unsigned long spfn = *start_pfn, epfn = *end_pfn; 2113 unsigned long nr_pages = 0; 2114 u64 j = *i; 2115 2116 /* First we loop through and initialize the page values */ 2117 for_each_free_mem_pfn_range_in_zone_from(j, zone, start_pfn, end_pfn) { 2118 unsigned long t; 2119 2120 if (mo_pfn <= *start_pfn) 2121 break; 2122 2123 t = min(mo_pfn, *end_pfn); 2124 nr_pages += deferred_init_pages(zone, *start_pfn, t); 2125 2126 if (mo_pfn < *end_pfn) { 2127 *start_pfn = mo_pfn; 2128 break; 2129 } 2130 } 2131 2132 /* Reset values and now loop through freeing pages as needed */ 2133 swap(j, *i); 2134 2135 for_each_free_mem_pfn_range_in_zone_from(j, zone, &spfn, &epfn) { 2136 unsigned long t; 2137 2138 if (mo_pfn <= spfn) 2139 break; 2140 2141 t = min(mo_pfn, epfn); 2142 deferred_free_pages(spfn, t); 2143 2144 if (mo_pfn <= epfn) 2145 break; 2146 } 2147 2148 return nr_pages; 2149 } 2150 2151 static void __init 2152 deferred_init_memmap_chunk(unsigned long start_pfn, unsigned long end_pfn, 2153 void *arg) 2154 { 2155 unsigned long spfn, epfn; 2156 struct zone *zone = arg; 2157 u64 i; 2158 2159 deferred_init_mem_pfn_range_in_zone(&i, zone, &spfn, &epfn, start_pfn); 2160 2161 /* 2162 * Initialize and free pages in MAX_ORDER sized increments so that we 2163 * can avoid introducing any issues with the buddy allocator. 2164 */ 2165 while (spfn < end_pfn) { 2166 deferred_init_maxorder(&i, zone, &spfn, &epfn); 2167 cond_resched(); 2168 } 2169 } 2170 2171 /* An arch may override for more concurrency. */ 2172 __weak int __init 2173 deferred_page_init_max_threads(const struct cpumask *node_cpumask) 2174 { 2175 return 1; 2176 } 2177 2178 /* Initialise remaining memory on a node */ 2179 static int __init deferred_init_memmap(void *data) 2180 { 2181 pg_data_t *pgdat = data; 2182 const struct cpumask *cpumask = cpumask_of_node(pgdat->node_id); 2183 unsigned long spfn = 0, epfn = 0; 2184 unsigned long first_init_pfn, flags; 2185 unsigned long start = jiffies; 2186 struct zone *zone; 2187 int zid, max_threads; 2188 u64 i; 2189 2190 /* Bind memory initialisation thread to a local node if possible */ 2191 if (!cpumask_empty(cpumask)) 2192 set_cpus_allowed_ptr(current, cpumask); 2193 2194 pgdat_resize_lock(pgdat, &flags); 2195 first_init_pfn = pgdat->first_deferred_pfn; 2196 if (first_init_pfn == ULONG_MAX) { 2197 pgdat_resize_unlock(pgdat, &flags); 2198 pgdat_init_report_one_done(); 2199 return 0; 2200 } 2201 2202 /* Sanity check boundaries */ 2203 BUG_ON(pgdat->first_deferred_pfn < pgdat->node_start_pfn); 2204 BUG_ON(pgdat->first_deferred_pfn > pgdat_end_pfn(pgdat)); 2205 pgdat->first_deferred_pfn = ULONG_MAX; 2206 2207 /* 2208 * Once we unlock here, the zone cannot be grown anymore, thus if an 2209 * interrupt thread must allocate this early in boot, zone must be 2210 * pre-grown prior to start of deferred page initialization. 2211 */ 2212 pgdat_resize_unlock(pgdat, &flags); 2213 2214 /* Only the highest zone is deferred so find it */ 2215 for (zid = 0; zid < MAX_NR_ZONES; zid++) { 2216 zone = pgdat->node_zones + zid; 2217 if (first_init_pfn < zone_end_pfn(zone)) 2218 break; 2219 } 2220 2221 /* If the zone is empty somebody else may have cleared out the zone */ 2222 if (!deferred_init_mem_pfn_range_in_zone(&i, zone, &spfn, &epfn, 2223 first_init_pfn)) 2224 goto zone_empty; 2225 2226 max_threads = deferred_page_init_max_threads(cpumask); 2227 2228 while (spfn < epfn) { 2229 unsigned long epfn_align = ALIGN(epfn, PAGES_PER_SECTION); 2230 struct padata_mt_job job = { 2231 .thread_fn = deferred_init_memmap_chunk, 2232 .fn_arg = zone, 2233 .start = spfn, 2234 .size = epfn_align - spfn, 2235 .align = PAGES_PER_SECTION, 2236 .min_chunk = PAGES_PER_SECTION, 2237 .max_threads = max_threads, 2238 }; 2239 2240 padata_do_multithreaded(&job); 2241 deferred_init_mem_pfn_range_in_zone(&i, zone, &spfn, &epfn, 2242 epfn_align); 2243 } 2244 zone_empty: 2245 /* Sanity check that the next zone really is unpopulated */ 2246 WARN_ON(++zid < MAX_NR_ZONES && populated_zone(++zone)); 2247 2248 pr_info("node %d deferred pages initialised in %ums\n", 2249 pgdat->node_id, jiffies_to_msecs(jiffies - start)); 2250 2251 pgdat_init_report_one_done(); 2252 return 0; 2253 } 2254 2255 /* 2256 * If this zone has deferred pages, try to grow it by initializing enough 2257 * deferred pages to satisfy the allocation specified by order, rounded up to 2258 * the nearest PAGES_PER_SECTION boundary. So we're adding memory in increments 2259 * of SECTION_SIZE bytes by initializing struct pages in increments of 2260 * PAGES_PER_SECTION * sizeof(struct page) bytes. 2261 * 2262 * Return true when zone was grown, otherwise return false. We return true even 2263 * when we grow less than requested, to let the caller decide if there are 2264 * enough pages to satisfy the allocation. 2265 * 2266 * Note: We use noinline because this function is needed only during boot, and 2267 * it is called from a __ref function _deferred_grow_zone. This way we are 2268 * making sure that it is not inlined into permanent text section. 2269 */ 2270 bool __init deferred_grow_zone(struct zone *zone, unsigned int order) 2271 { 2272 unsigned long nr_pages_needed = ALIGN(1 << order, PAGES_PER_SECTION); 2273 pg_data_t *pgdat = zone->zone_pgdat; 2274 unsigned long first_deferred_pfn = pgdat->first_deferred_pfn; 2275 unsigned long spfn, epfn, flags; 2276 unsigned long nr_pages = 0; 2277 u64 i; 2278 2279 /* Only the last zone may have deferred pages */ 2280 if (zone_end_pfn(zone) != pgdat_end_pfn(pgdat)) 2281 return false; 2282 2283 pgdat_resize_lock(pgdat, &flags); 2284 2285 /* 2286 * If someone grew this zone while we were waiting for spinlock, return 2287 * true, as there might be enough pages already. 2288 */ 2289 if (first_deferred_pfn != pgdat->first_deferred_pfn) { 2290 pgdat_resize_unlock(pgdat, &flags); 2291 return true; 2292 } 2293 2294 /* If the zone is empty somebody else may have cleared out the zone */ 2295 if (!deferred_init_mem_pfn_range_in_zone(&i, zone, &spfn, &epfn, 2296 first_deferred_pfn)) { 2297 pgdat->first_deferred_pfn = ULONG_MAX; 2298 pgdat_resize_unlock(pgdat, &flags); 2299 /* Retry only once. */ 2300 return first_deferred_pfn != ULONG_MAX; 2301 } 2302 2303 /* 2304 * Initialize and free pages in MAX_ORDER sized increments so 2305 * that we can avoid introducing any issues with the buddy 2306 * allocator. 2307 */ 2308 while (spfn < epfn) { 2309 /* update our first deferred PFN for this section */ 2310 first_deferred_pfn = spfn; 2311 2312 nr_pages += deferred_init_maxorder(&i, zone, &spfn, &epfn); 2313 touch_nmi_watchdog(); 2314 2315 /* We should only stop along section boundaries */ 2316 if ((first_deferred_pfn ^ spfn) < PAGES_PER_SECTION) 2317 continue; 2318 2319 /* If our quota has been met we can stop here */ 2320 if (nr_pages >= nr_pages_needed) 2321 break; 2322 } 2323 2324 pgdat->first_deferred_pfn = spfn; 2325 pgdat_resize_unlock(pgdat, &flags); 2326 2327 return nr_pages > 0; 2328 } 2329 2330 #endif /* CONFIG_DEFERRED_STRUCT_PAGE_INIT */ 2331 2332 #ifdef CONFIG_CMA 2333 void __init init_cma_reserved_pageblock(struct page *page) 2334 { 2335 unsigned i = pageblock_nr_pages; 2336 struct page *p = page; 2337 2338 do { 2339 __ClearPageReserved(p); 2340 set_page_count(p, 0); 2341 } while (++p, --i); 2342 2343 set_pageblock_migratetype(page, MIGRATE_CMA); 2344 set_page_refcounted(page); 2345 __free_pages(page, pageblock_order); 2346 2347 adjust_managed_page_count(page, pageblock_nr_pages); 2348 page_zone(page)->cma_pages += pageblock_nr_pages; 2349 } 2350 #endif 2351 2352 void set_zone_contiguous(struct zone *zone) 2353 { 2354 unsigned long block_start_pfn = zone->zone_start_pfn; 2355 unsigned long block_end_pfn; 2356 2357 block_end_pfn = pageblock_end_pfn(block_start_pfn); 2358 for (; block_start_pfn < zone_end_pfn(zone); 2359 block_start_pfn = block_end_pfn, 2360 block_end_pfn += pageblock_nr_pages) { 2361 2362 block_end_pfn = min(block_end_pfn, zone_end_pfn(zone)); 2363 2364 if (!__pageblock_pfn_to_page(block_start_pfn, 2365 block_end_pfn, zone)) 2366 return; 2367 cond_resched(); 2368 } 2369 2370 /* We confirm that there is no hole */ 2371 zone->contiguous = true; 2372 } 2373 2374 void __init page_alloc_init_late(void) 2375 { 2376 struct zone *zone; 2377 int nid; 2378 2379 #ifdef CONFIG_DEFERRED_STRUCT_PAGE_INIT 2380 2381 /* There will be num_node_state(N_MEMORY) threads */ 2382 atomic_set(&pgdat_init_n_undone, num_node_state(N_MEMORY)); 2383 for_each_node_state(nid, N_MEMORY) { 2384 kthread_run(deferred_init_memmap, NODE_DATA(nid), "pgdatinit%d", nid); 2385 } 2386 2387 /* Block until all are initialised */ 2388 wait_for_completion(&pgdat_init_all_done_comp); 2389 2390 /* 2391 * We initialized the rest of the deferred pages. Permanently disable 2392 * on-demand struct page initialization. 2393 */ 2394 static_branch_disable(&deferred_pages); 2395 2396 /* Reinit limits that are based on free pages after the kernel is up */ 2397 files_maxfiles_init(); 2398 #endif 2399 2400 buffer_init(); 2401 2402 /* Discard memblock private memory */ 2403 memblock_discard(); 2404 2405 for_each_node_state(nid, N_MEMORY) 2406 shuffle_free_memory(NODE_DATA(nid)); 2407 2408 for_each_populated_zone(zone) 2409 set_zone_contiguous(zone); 2410 2411 /* Initialize page ext after all struct pages are initialized. */ 2412 if (deferred_struct_pages) 2413 page_ext_init(); 2414 2415 page_alloc_sysctl_init(); 2416 } 2417 2418 #ifndef __HAVE_ARCH_RESERVED_KERNEL_PAGES 2419 /* 2420 * Returns the number of pages that arch has reserved but 2421 * is not known to alloc_large_system_hash(). 2422 */ 2423 static unsigned long __init arch_reserved_kernel_pages(void) 2424 { 2425 return 0; 2426 } 2427 #endif 2428 2429 /* 2430 * Adaptive scale is meant to reduce sizes of hash tables on large memory 2431 * machines. As memory size is increased the scale is also increased but at 2432 * slower pace. Starting from ADAPT_SCALE_BASE (64G), every time memory 2433 * quadruples the scale is increased by one, which means the size of hash table 2434 * only doubles, instead of quadrupling as well. 2435 * Because 32-bit systems cannot have large physical memory, where this scaling 2436 * makes sense, it is disabled on such platforms. 2437 */ 2438 #if __BITS_PER_LONG > 32 2439 #define ADAPT_SCALE_BASE (64ul << 30) 2440 #define ADAPT_SCALE_SHIFT 2 2441 #define ADAPT_SCALE_NPAGES (ADAPT_SCALE_BASE >> PAGE_SHIFT) 2442 #endif 2443 2444 /* 2445 * allocate a large system hash table from bootmem 2446 * - it is assumed that the hash table must contain an exact power-of-2 2447 * quantity of entries 2448 * - limit is the number of hash buckets, not the total allocation size 2449 */ 2450 void *__init alloc_large_system_hash(const char *tablename, 2451 unsigned long bucketsize, 2452 unsigned long numentries, 2453 int scale, 2454 int flags, 2455 unsigned int *_hash_shift, 2456 unsigned int *_hash_mask, 2457 unsigned long low_limit, 2458 unsigned long high_limit) 2459 { 2460 unsigned long long max = high_limit; 2461 unsigned long log2qty, size; 2462 void *table; 2463 gfp_t gfp_flags; 2464 bool virt; 2465 bool huge; 2466 2467 /* allow the kernel cmdline to have a say */ 2468 if (!numentries) { 2469 /* round applicable memory size up to nearest megabyte */ 2470 numentries = nr_kernel_pages; 2471 numentries -= arch_reserved_kernel_pages(); 2472 2473 /* It isn't necessary when PAGE_SIZE >= 1MB */ 2474 if (PAGE_SIZE < SZ_1M) 2475 numentries = round_up(numentries, SZ_1M / PAGE_SIZE); 2476 2477 #if __BITS_PER_LONG > 32 2478 if (!high_limit) { 2479 unsigned long adapt; 2480 2481 for (adapt = ADAPT_SCALE_NPAGES; adapt < numentries; 2482 adapt <<= ADAPT_SCALE_SHIFT) 2483 scale++; 2484 } 2485 #endif 2486 2487 /* limit to 1 bucket per 2^scale bytes of low memory */ 2488 if (scale > PAGE_SHIFT) 2489 numentries >>= (scale - PAGE_SHIFT); 2490 else 2491 numentries <<= (PAGE_SHIFT - scale); 2492 2493 /* Make sure we've got at least a 0-order allocation.. */ 2494 if (unlikely(flags & HASH_SMALL)) { 2495 /* Makes no sense without HASH_EARLY */ 2496 WARN_ON(!(flags & HASH_EARLY)); 2497 if (!(numentries >> *_hash_shift)) { 2498 numentries = 1UL << *_hash_shift; 2499 BUG_ON(!numentries); 2500 } 2501 } else if (unlikely((numentries * bucketsize) < PAGE_SIZE)) 2502 numentries = PAGE_SIZE / bucketsize; 2503 } 2504 numentries = roundup_pow_of_two(numentries); 2505 2506 /* limit allocation size to 1/16 total memory by default */ 2507 if (max == 0) { 2508 max = ((unsigned long long)nr_all_pages << PAGE_SHIFT) >> 4; 2509 do_div(max, bucketsize); 2510 } 2511 max = min(max, 0x80000000ULL); 2512 2513 if (numentries < low_limit) 2514 numentries = low_limit; 2515 if (numentries > max) 2516 numentries = max; 2517 2518 log2qty = ilog2(numentries); 2519 2520 gfp_flags = (flags & HASH_ZERO) ? GFP_ATOMIC | __GFP_ZERO : GFP_ATOMIC; 2521 do { 2522 virt = false; 2523 size = bucketsize << log2qty; 2524 if (flags & HASH_EARLY) { 2525 if (flags & HASH_ZERO) 2526 table = memblock_alloc(size, SMP_CACHE_BYTES); 2527 else 2528 table = memblock_alloc_raw(size, 2529 SMP_CACHE_BYTES); 2530 } else if (get_order(size) > MAX_ORDER || hashdist) { 2531 table = vmalloc_huge(size, gfp_flags); 2532 virt = true; 2533 if (table) 2534 huge = is_vm_area_hugepages(table); 2535 } else { 2536 /* 2537 * If bucketsize is not a power-of-two, we may free 2538 * some pages at the end of hash table which 2539 * alloc_pages_exact() automatically does 2540 */ 2541 table = alloc_pages_exact(size, gfp_flags); 2542 kmemleak_alloc(table, size, 1, gfp_flags); 2543 } 2544 } while (!table && size > PAGE_SIZE && --log2qty); 2545 2546 if (!table) 2547 panic("Failed to allocate %s hash table\n", tablename); 2548 2549 pr_info("%s hash table entries: %ld (order: %d, %lu bytes, %s)\n", 2550 tablename, 1UL << log2qty, ilog2(size) - PAGE_SHIFT, size, 2551 virt ? (huge ? "vmalloc hugepage" : "vmalloc") : "linear"); 2552 2553 if (_hash_shift) 2554 *_hash_shift = log2qty; 2555 if (_hash_mask) 2556 *_hash_mask = (1 << log2qty) - 1; 2557 2558 return table; 2559 } 2560 2561 /** 2562 * set_dma_reserve - set the specified number of pages reserved in the first zone 2563 * @new_dma_reserve: The number of pages to mark reserved 2564 * 2565 * The per-cpu batchsize and zone watermarks are determined by managed_pages. 2566 * In the DMA zone, a significant percentage may be consumed by kernel image 2567 * and other unfreeable allocations which can skew the watermarks badly. This 2568 * function may optionally be used to account for unfreeable pages in the 2569 * first zone (e.g., ZONE_DMA). The effect will be lower watermarks and 2570 * smaller per-cpu batchsize. 2571 */ 2572 void __init set_dma_reserve(unsigned long new_dma_reserve) 2573 { 2574 dma_reserve = new_dma_reserve; 2575 } 2576 2577 void __init memblock_free_pages(struct page *page, unsigned long pfn, 2578 unsigned int order) 2579 { 2580 2581 if (IS_ENABLED(CONFIG_DEFERRED_STRUCT_PAGE_INIT)) { 2582 int nid = early_pfn_to_nid(pfn); 2583 2584 if (!early_page_initialised(pfn, nid)) 2585 return; 2586 } 2587 2588 if (!kmsan_memblock_free_pages(page, order)) { 2589 /* KMSAN will take care of these pages. */ 2590 return; 2591 } 2592 __free_pages_core(page, order); 2593 } 2594 2595 DEFINE_STATIC_KEY_MAYBE(CONFIG_INIT_ON_ALLOC_DEFAULT_ON, init_on_alloc); 2596 EXPORT_SYMBOL(init_on_alloc); 2597 2598 DEFINE_STATIC_KEY_MAYBE(CONFIG_INIT_ON_FREE_DEFAULT_ON, init_on_free); 2599 EXPORT_SYMBOL(init_on_free); 2600 2601 static bool _init_on_alloc_enabled_early __read_mostly 2602 = IS_ENABLED(CONFIG_INIT_ON_ALLOC_DEFAULT_ON); 2603 static int __init early_init_on_alloc(char *buf) 2604 { 2605 2606 return kstrtobool(buf, &_init_on_alloc_enabled_early); 2607 } 2608 early_param("init_on_alloc", early_init_on_alloc); 2609 2610 static bool _init_on_free_enabled_early __read_mostly 2611 = IS_ENABLED(CONFIG_INIT_ON_FREE_DEFAULT_ON); 2612 static int __init early_init_on_free(char *buf) 2613 { 2614 return kstrtobool(buf, &_init_on_free_enabled_early); 2615 } 2616 early_param("init_on_free", early_init_on_free); 2617 2618 DEFINE_STATIC_KEY_MAYBE(CONFIG_DEBUG_VM, check_pages_enabled); 2619 2620 /* 2621 * Enable static keys related to various memory debugging and hardening options. 2622 * Some override others, and depend on early params that are evaluated in the 2623 * order of appearance. So we need to first gather the full picture of what was 2624 * enabled, and then make decisions. 2625 */ 2626 static void __init mem_debugging_and_hardening_init(void) 2627 { 2628 bool page_poisoning_requested = false; 2629 bool want_check_pages = false; 2630 2631 #ifdef CONFIG_PAGE_POISONING 2632 /* 2633 * Page poisoning is debug page alloc for some arches. If 2634 * either of those options are enabled, enable poisoning. 2635 */ 2636 if (page_poisoning_enabled() || 2637 (!IS_ENABLED(CONFIG_ARCH_SUPPORTS_DEBUG_PAGEALLOC) && 2638 debug_pagealloc_enabled())) { 2639 static_branch_enable(&_page_poisoning_enabled); 2640 page_poisoning_requested = true; 2641 want_check_pages = true; 2642 } 2643 #endif 2644 2645 if ((_init_on_alloc_enabled_early || _init_on_free_enabled_early) && 2646 page_poisoning_requested) { 2647 pr_info("mem auto-init: CONFIG_PAGE_POISONING is on, " 2648 "will take precedence over init_on_alloc and init_on_free\n"); 2649 _init_on_alloc_enabled_early = false; 2650 _init_on_free_enabled_early = false; 2651 } 2652 2653 if (_init_on_alloc_enabled_early) { 2654 want_check_pages = true; 2655 static_branch_enable(&init_on_alloc); 2656 } else { 2657 static_branch_disable(&init_on_alloc); 2658 } 2659 2660 if (_init_on_free_enabled_early) { 2661 want_check_pages = true; 2662 static_branch_enable(&init_on_free); 2663 } else { 2664 static_branch_disable(&init_on_free); 2665 } 2666 2667 if (IS_ENABLED(CONFIG_KMSAN) && 2668 (_init_on_alloc_enabled_early || _init_on_free_enabled_early)) 2669 pr_info("mem auto-init: please make sure init_on_alloc and init_on_free are disabled when running KMSAN\n"); 2670 2671 #ifdef CONFIG_DEBUG_PAGEALLOC 2672 if (debug_pagealloc_enabled()) { 2673 want_check_pages = true; 2674 static_branch_enable(&_debug_pagealloc_enabled); 2675 2676 if (debug_guardpage_minorder()) 2677 static_branch_enable(&_debug_guardpage_enabled); 2678 } 2679 #endif 2680 2681 /* 2682 * Any page debugging or hardening option also enables sanity checking 2683 * of struct pages being allocated or freed. With CONFIG_DEBUG_VM it's 2684 * enabled already. 2685 */ 2686 if (!IS_ENABLED(CONFIG_DEBUG_VM) && want_check_pages) 2687 static_branch_enable(&check_pages_enabled); 2688 } 2689 2690 /* Report memory auto-initialization states for this boot. */ 2691 static void __init report_meminit(void) 2692 { 2693 const char *stack; 2694 2695 if (IS_ENABLED(CONFIG_INIT_STACK_ALL_PATTERN)) 2696 stack = "all(pattern)"; 2697 else if (IS_ENABLED(CONFIG_INIT_STACK_ALL_ZERO)) 2698 stack = "all(zero)"; 2699 else if (IS_ENABLED(CONFIG_GCC_PLUGIN_STRUCTLEAK_BYREF_ALL)) 2700 stack = "byref_all(zero)"; 2701 else if (IS_ENABLED(CONFIG_GCC_PLUGIN_STRUCTLEAK_BYREF)) 2702 stack = "byref(zero)"; 2703 else if (IS_ENABLED(CONFIG_GCC_PLUGIN_STRUCTLEAK_USER)) 2704 stack = "__user(zero)"; 2705 else 2706 stack = "off"; 2707 2708 pr_info("mem auto-init: stack:%s, heap alloc:%s, heap free:%s\n", 2709 stack, want_init_on_alloc(GFP_KERNEL) ? "on" : "off", 2710 want_init_on_free() ? "on" : "off"); 2711 if (want_init_on_free()) 2712 pr_info("mem auto-init: clearing system memory may take some time...\n"); 2713 } 2714 2715 static void __init mem_init_print_info(void) 2716 { 2717 unsigned long physpages, codesize, datasize, rosize, bss_size; 2718 unsigned long init_code_size, init_data_size; 2719 2720 physpages = get_num_physpages(); 2721 codesize = _etext - _stext; 2722 datasize = _edata - _sdata; 2723 rosize = __end_rodata - __start_rodata; 2724 bss_size = __bss_stop - __bss_start; 2725 init_data_size = __init_end - __init_begin; 2726 init_code_size = _einittext - _sinittext; 2727 2728 /* 2729 * Detect special cases and adjust section sizes accordingly: 2730 * 1) .init.* may be embedded into .data sections 2731 * 2) .init.text.* may be out of [__init_begin, __init_end], 2732 * please refer to arch/tile/kernel/vmlinux.lds.S. 2733 * 3) .rodata.* may be embedded into .text or .data sections. 2734 */ 2735 #define adj_init_size(start, end, size, pos, adj) \ 2736 do { \ 2737 if (&start[0] <= &pos[0] && &pos[0] < &end[0] && size > adj) \ 2738 size -= adj; \ 2739 } while (0) 2740 2741 adj_init_size(__init_begin, __init_end, init_data_size, 2742 _sinittext, init_code_size); 2743 adj_init_size(_stext, _etext, codesize, _sinittext, init_code_size); 2744 adj_init_size(_sdata, _edata, datasize, __init_begin, init_data_size); 2745 adj_init_size(_stext, _etext, codesize, __start_rodata, rosize); 2746 adj_init_size(_sdata, _edata, datasize, __start_rodata, rosize); 2747 2748 #undef adj_init_size 2749 2750 pr_info("Memory: %luK/%luK available (%luK kernel code, %luK rwdata, %luK rodata, %luK init, %luK bss, %luK reserved, %luK cma-reserved" 2751 #ifdef CONFIG_HIGHMEM 2752 ", %luK highmem" 2753 #endif 2754 ")\n", 2755 K(nr_free_pages()), K(physpages), 2756 codesize / SZ_1K, datasize / SZ_1K, rosize / SZ_1K, 2757 (init_data_size + init_code_size) / SZ_1K, bss_size / SZ_1K, 2758 K(physpages - totalram_pages() - totalcma_pages), 2759 K(totalcma_pages) 2760 #ifdef CONFIG_HIGHMEM 2761 , K(totalhigh_pages()) 2762 #endif 2763 ); 2764 } 2765 2766 /* 2767 * Set up kernel memory allocators 2768 */ 2769 void __init mm_core_init(void) 2770 { 2771 /* Initializations relying on SMP setup */ 2772 build_all_zonelists(NULL); 2773 page_alloc_init_cpuhp(); 2774 2775 /* 2776 * page_ext requires contiguous pages, 2777 * bigger than MAX_ORDER unless SPARSEMEM. 2778 */ 2779 page_ext_init_flatmem(); 2780 mem_debugging_and_hardening_init(); 2781 kfence_alloc_pool(); 2782 report_meminit(); 2783 kmsan_init_shadow(); 2784 stack_depot_early_init(); 2785 mem_init(); 2786 mem_init_print_info(); 2787 kmem_cache_init(); 2788 /* 2789 * page_owner must be initialized after buddy is ready, and also after 2790 * slab is ready so that stack_depot_init() works properly 2791 */ 2792 page_ext_init_flatmem_late(); 2793 kmemleak_init(); 2794 ptlock_cache_init(); 2795 pgtable_cache_init(); 2796 debug_objects_mem_init(); 2797 vmalloc_init(); 2798 /* If no deferred init page_ext now, as vmap is fully initialized */ 2799 if (!deferred_struct_pages) 2800 page_ext_init(); 2801 /* Should be run before the first non-init thread is created */ 2802 init_espfix_bsp(); 2803 /* Should be run after espfix64 is set up. */ 2804 pti_init(); 2805 kmsan_init_runtime(); 2806 mm_cache_init(); 2807 } 2808