1 // SPDX-License-Identifier: GPL-2.0-only 2 /* Common code for 32 and 64-bit NUMA */ 3 #include <linux/acpi.h> 4 #include <linux/kernel.h> 5 #include <linux/mm.h> 6 #include <linux/string.h> 7 #include <linux/init.h> 8 #include <linux/memblock.h> 9 #include <linux/mmzone.h> 10 #include <linux/ctype.h> 11 #include <linux/nodemask.h> 12 #include <linux/sched.h> 13 #include <linux/topology.h> 14 15 #include <asm/e820/api.h> 16 #include <asm/proto.h> 17 #include <asm/dma.h> 18 #include <asm/amd_nb.h> 19 20 #include "numa_internal.h" 21 22 int numa_off; 23 nodemask_t numa_nodes_parsed __initdata; 24 25 struct pglist_data *node_data[MAX_NUMNODES] __read_mostly; 26 EXPORT_SYMBOL(node_data); 27 28 static struct numa_meminfo numa_meminfo __initdata_or_meminfo; 29 static struct numa_meminfo numa_reserved_meminfo __initdata_or_meminfo; 30 31 static int numa_distance_cnt; 32 static u8 *numa_distance; 33 34 static __init int numa_setup(char *opt) 35 { 36 if (!opt) 37 return -EINVAL; 38 if (!strncmp(opt, "off", 3)) 39 numa_off = 1; 40 #ifdef CONFIG_NUMA_EMU 41 if (!strncmp(opt, "fake=", 5)) 42 numa_emu_cmdline(opt + 5); 43 #endif 44 #ifdef CONFIG_ACPI_NUMA 45 if (!strncmp(opt, "noacpi", 6)) 46 acpi_numa = -1; 47 #endif 48 return 0; 49 } 50 early_param("numa", numa_setup); 51 52 /* 53 * apicid, cpu, node mappings 54 */ 55 s16 __apicid_to_node[MAX_LOCAL_APIC] = { 56 [0 ... MAX_LOCAL_APIC-1] = NUMA_NO_NODE 57 }; 58 59 int numa_cpu_node(int cpu) 60 { 61 int apicid = early_per_cpu(x86_cpu_to_apicid, cpu); 62 63 if (apicid != BAD_APICID) 64 return __apicid_to_node[apicid]; 65 return NUMA_NO_NODE; 66 } 67 68 cpumask_var_t node_to_cpumask_map[MAX_NUMNODES]; 69 EXPORT_SYMBOL(node_to_cpumask_map); 70 71 /* 72 * Map cpu index to node index 73 */ 74 DEFINE_EARLY_PER_CPU(int, x86_cpu_to_node_map, NUMA_NO_NODE); 75 EXPORT_EARLY_PER_CPU_SYMBOL(x86_cpu_to_node_map); 76 77 void numa_set_node(int cpu, int node) 78 { 79 int *cpu_to_node_map = early_per_cpu_ptr(x86_cpu_to_node_map); 80 81 /* early setting, no percpu area yet */ 82 if (cpu_to_node_map) { 83 cpu_to_node_map[cpu] = node; 84 return; 85 } 86 87 #ifdef CONFIG_DEBUG_PER_CPU_MAPS 88 if (cpu >= nr_cpu_ids || !cpu_possible(cpu)) { 89 printk(KERN_ERR "numa_set_node: invalid cpu# (%d)\n", cpu); 90 dump_stack(); 91 return; 92 } 93 #endif 94 per_cpu(x86_cpu_to_node_map, cpu) = node; 95 96 set_cpu_numa_node(cpu, node); 97 } 98 99 void numa_clear_node(int cpu) 100 { 101 numa_set_node(cpu, NUMA_NO_NODE); 102 } 103 104 /* 105 * Allocate node_to_cpumask_map based on number of available nodes 106 * Requires node_possible_map to be valid. 107 * 108 * Note: cpumask_of_node() is not valid until after this is done. 109 * (Use CONFIG_DEBUG_PER_CPU_MAPS to check this.) 110 */ 111 void __init setup_node_to_cpumask_map(void) 112 { 113 unsigned int node; 114 115 /* setup nr_node_ids if not done yet */ 116 if (nr_node_ids == MAX_NUMNODES) 117 setup_nr_node_ids(); 118 119 /* allocate the map */ 120 for (node = 0; node < nr_node_ids; node++) 121 alloc_bootmem_cpumask_var(&node_to_cpumask_map[node]); 122 123 /* cpumask_of_node() will now work */ 124 pr_debug("Node to cpumask map for %u nodes\n", nr_node_ids); 125 } 126 127 static int __init numa_add_memblk_to(int nid, u64 start, u64 end, 128 struct numa_meminfo *mi) 129 { 130 /* ignore zero length blks */ 131 if (start == end) 132 return 0; 133 134 /* whine about and ignore invalid blks */ 135 if (start > end || nid < 0 || nid >= MAX_NUMNODES) { 136 pr_warn("Warning: invalid memblk node %d [mem %#010Lx-%#010Lx]\n", 137 nid, start, end - 1); 138 return 0; 139 } 140 141 if (mi->nr_blks >= NR_NODE_MEMBLKS) { 142 pr_err("too many memblk ranges\n"); 143 return -EINVAL; 144 } 145 146 mi->blk[mi->nr_blks].start = start; 147 mi->blk[mi->nr_blks].end = end; 148 mi->blk[mi->nr_blks].nid = nid; 149 mi->nr_blks++; 150 return 0; 151 } 152 153 /** 154 * numa_remove_memblk_from - Remove one numa_memblk from a numa_meminfo 155 * @idx: Index of memblk to remove 156 * @mi: numa_meminfo to remove memblk from 157 * 158 * Remove @idx'th numa_memblk from @mi by shifting @mi->blk[] and 159 * decrementing @mi->nr_blks. 160 */ 161 void __init numa_remove_memblk_from(int idx, struct numa_meminfo *mi) 162 { 163 mi->nr_blks--; 164 memmove(&mi->blk[idx], &mi->blk[idx + 1], 165 (mi->nr_blks - idx) * sizeof(mi->blk[0])); 166 } 167 168 /** 169 * numa_move_tail_memblk - Move a numa_memblk from one numa_meminfo to another 170 * @dst: numa_meminfo to append block to 171 * @idx: Index of memblk to remove 172 * @src: numa_meminfo to remove memblk from 173 */ 174 static void __init numa_move_tail_memblk(struct numa_meminfo *dst, int idx, 175 struct numa_meminfo *src) 176 { 177 dst->blk[dst->nr_blks++] = src->blk[idx]; 178 numa_remove_memblk_from(idx, src); 179 } 180 181 /** 182 * numa_add_memblk - Add one numa_memblk to numa_meminfo 183 * @nid: NUMA node ID of the new memblk 184 * @start: Start address of the new memblk 185 * @end: End address of the new memblk 186 * 187 * Add a new memblk to the default numa_meminfo. 188 * 189 * RETURNS: 190 * 0 on success, -errno on failure. 191 */ 192 int __init numa_add_memblk(int nid, u64 start, u64 end) 193 { 194 return numa_add_memblk_to(nid, start, end, &numa_meminfo); 195 } 196 197 /* Allocate NODE_DATA for a node on the local memory */ 198 static void __init alloc_node_data(int nid) 199 { 200 const size_t nd_size = roundup(sizeof(pg_data_t), PAGE_SIZE); 201 u64 nd_pa; 202 void *nd; 203 int tnid; 204 205 /* 206 * Allocate node data. Try node-local memory and then any node. 207 * Never allocate in DMA zone. 208 */ 209 nd_pa = memblock_phys_alloc_try_nid(nd_size, SMP_CACHE_BYTES, nid); 210 if (!nd_pa) { 211 pr_err("Cannot find %zu bytes in any node (initial node: %d)\n", 212 nd_size, nid); 213 return; 214 } 215 nd = __va(nd_pa); 216 217 /* report and initialize */ 218 printk(KERN_INFO "NODE_DATA(%d) allocated [mem %#010Lx-%#010Lx]\n", nid, 219 nd_pa, nd_pa + nd_size - 1); 220 tnid = early_pfn_to_nid(nd_pa >> PAGE_SHIFT); 221 if (tnid != nid) 222 printk(KERN_INFO " NODE_DATA(%d) on node %d\n", nid, tnid); 223 224 node_data[nid] = nd; 225 memset(NODE_DATA(nid), 0, sizeof(pg_data_t)); 226 227 node_set_online(nid); 228 } 229 230 /** 231 * numa_cleanup_meminfo - Cleanup a numa_meminfo 232 * @mi: numa_meminfo to clean up 233 * 234 * Sanitize @mi by merging and removing unnecessary memblks. Also check for 235 * conflicts and clear unused memblks. 236 * 237 * RETURNS: 238 * 0 on success, -errno on failure. 239 */ 240 int __init numa_cleanup_meminfo(struct numa_meminfo *mi) 241 { 242 const u64 low = 0; 243 const u64 high = PFN_PHYS(max_pfn); 244 int i, j, k; 245 246 /* first, trim all entries */ 247 for (i = 0; i < mi->nr_blks; i++) { 248 struct numa_memblk *bi = &mi->blk[i]; 249 250 /* move / save reserved memory ranges */ 251 if (!memblock_overlaps_region(&memblock.memory, 252 bi->start, bi->end - bi->start)) { 253 numa_move_tail_memblk(&numa_reserved_meminfo, i--, mi); 254 continue; 255 } 256 257 /* make sure all non-reserved blocks are inside the limits */ 258 bi->start = max(bi->start, low); 259 bi->end = min(bi->end, high); 260 261 /* and there's no empty block */ 262 if (bi->start >= bi->end) 263 numa_remove_memblk_from(i--, mi); 264 } 265 266 /* merge neighboring / overlapping entries */ 267 for (i = 0; i < mi->nr_blks; i++) { 268 struct numa_memblk *bi = &mi->blk[i]; 269 270 for (j = i + 1; j < mi->nr_blks; j++) { 271 struct numa_memblk *bj = &mi->blk[j]; 272 u64 start, end; 273 274 /* 275 * See whether there are overlapping blocks. Whine 276 * about but allow overlaps of the same nid. They 277 * will be merged below. 278 */ 279 if (bi->end > bj->start && bi->start < bj->end) { 280 if (bi->nid != bj->nid) { 281 pr_err("node %d [mem %#010Lx-%#010Lx] overlaps with node %d [mem %#010Lx-%#010Lx]\n", 282 bi->nid, bi->start, bi->end - 1, 283 bj->nid, bj->start, bj->end - 1); 284 return -EINVAL; 285 } 286 pr_warn("Warning: node %d [mem %#010Lx-%#010Lx] overlaps with itself [mem %#010Lx-%#010Lx]\n", 287 bi->nid, bi->start, bi->end - 1, 288 bj->start, bj->end - 1); 289 } 290 291 /* 292 * Join together blocks on the same node, holes 293 * between which don't overlap with memory on other 294 * nodes. 295 */ 296 if (bi->nid != bj->nid) 297 continue; 298 start = min(bi->start, bj->start); 299 end = max(bi->end, bj->end); 300 for (k = 0; k < mi->nr_blks; k++) { 301 struct numa_memblk *bk = &mi->blk[k]; 302 303 if (bi->nid == bk->nid) 304 continue; 305 if (start < bk->end && end > bk->start) 306 break; 307 } 308 if (k < mi->nr_blks) 309 continue; 310 printk(KERN_INFO "NUMA: Node %d [mem %#010Lx-%#010Lx] + [mem %#010Lx-%#010Lx] -> [mem %#010Lx-%#010Lx]\n", 311 bi->nid, bi->start, bi->end - 1, bj->start, 312 bj->end - 1, start, end - 1); 313 bi->start = start; 314 bi->end = end; 315 numa_remove_memblk_from(j--, mi); 316 } 317 } 318 319 /* clear unused ones */ 320 for (i = mi->nr_blks; i < ARRAY_SIZE(mi->blk); i++) { 321 mi->blk[i].start = mi->blk[i].end = 0; 322 mi->blk[i].nid = NUMA_NO_NODE; 323 } 324 325 return 0; 326 } 327 328 /* 329 * Set nodes, which have memory in @mi, in *@nodemask. 330 */ 331 static void __init numa_nodemask_from_meminfo(nodemask_t *nodemask, 332 const struct numa_meminfo *mi) 333 { 334 int i; 335 336 for (i = 0; i < ARRAY_SIZE(mi->blk); i++) 337 if (mi->blk[i].start != mi->blk[i].end && 338 mi->blk[i].nid != NUMA_NO_NODE) 339 node_set(mi->blk[i].nid, *nodemask); 340 } 341 342 /** 343 * numa_reset_distance - Reset NUMA distance table 344 * 345 * The current table is freed. The next numa_set_distance() call will 346 * create a new one. 347 */ 348 void __init numa_reset_distance(void) 349 { 350 size_t size = numa_distance_cnt * numa_distance_cnt * sizeof(numa_distance[0]); 351 352 /* numa_distance could be 1LU marking allocation failure, test cnt */ 353 if (numa_distance_cnt) 354 memblock_free(__pa(numa_distance), size); 355 numa_distance_cnt = 0; 356 numa_distance = NULL; /* enable table creation */ 357 } 358 359 static int __init numa_alloc_distance(void) 360 { 361 nodemask_t nodes_parsed; 362 size_t size; 363 int i, j, cnt = 0; 364 u64 phys; 365 366 /* size the new table and allocate it */ 367 nodes_parsed = numa_nodes_parsed; 368 numa_nodemask_from_meminfo(&nodes_parsed, &numa_meminfo); 369 370 for_each_node_mask(i, nodes_parsed) 371 cnt = i; 372 cnt++; 373 size = cnt * cnt * sizeof(numa_distance[0]); 374 375 phys = memblock_find_in_range(0, PFN_PHYS(max_pfn_mapped), 376 size, PAGE_SIZE); 377 if (!phys) { 378 pr_warn("Warning: can't allocate distance table!\n"); 379 /* don't retry until explicitly reset */ 380 numa_distance = (void *)1LU; 381 return -ENOMEM; 382 } 383 memblock_reserve(phys, size); 384 385 numa_distance = __va(phys); 386 numa_distance_cnt = cnt; 387 388 /* fill with the default distances */ 389 for (i = 0; i < cnt; i++) 390 for (j = 0; j < cnt; j++) 391 numa_distance[i * cnt + j] = i == j ? 392 LOCAL_DISTANCE : REMOTE_DISTANCE; 393 printk(KERN_DEBUG "NUMA: Initialized distance table, cnt=%d\n", cnt); 394 395 return 0; 396 } 397 398 /** 399 * numa_set_distance - Set NUMA distance from one NUMA to another 400 * @from: the 'from' node to set distance 401 * @to: the 'to' node to set distance 402 * @distance: NUMA distance 403 * 404 * Set the distance from node @from to @to to @distance. If distance table 405 * doesn't exist, one which is large enough to accommodate all the currently 406 * known nodes will be created. 407 * 408 * If such table cannot be allocated, a warning is printed and further 409 * calls are ignored until the distance table is reset with 410 * numa_reset_distance(). 411 * 412 * If @from or @to is higher than the highest known node or lower than zero 413 * at the time of table creation or @distance doesn't make sense, the call 414 * is ignored. 415 * This is to allow simplification of specific NUMA config implementations. 416 */ 417 void __init numa_set_distance(int from, int to, int distance) 418 { 419 if (!numa_distance && numa_alloc_distance() < 0) 420 return; 421 422 if (from >= numa_distance_cnt || to >= numa_distance_cnt || 423 from < 0 || to < 0) { 424 pr_warn_once("Warning: node ids are out of bound, from=%d to=%d distance=%d\n", 425 from, to, distance); 426 return; 427 } 428 429 if ((u8)distance != distance || 430 (from == to && distance != LOCAL_DISTANCE)) { 431 pr_warn_once("Warning: invalid distance parameter, from=%d to=%d distance=%d\n", 432 from, to, distance); 433 return; 434 } 435 436 numa_distance[from * numa_distance_cnt + to] = distance; 437 } 438 439 int __node_distance(int from, int to) 440 { 441 if (from >= numa_distance_cnt || to >= numa_distance_cnt) 442 return from == to ? LOCAL_DISTANCE : REMOTE_DISTANCE; 443 return numa_distance[from * numa_distance_cnt + to]; 444 } 445 EXPORT_SYMBOL(__node_distance); 446 447 /* 448 * Sanity check to catch more bad NUMA configurations (they are amazingly 449 * common). Make sure the nodes cover all memory. 450 */ 451 static bool __init numa_meminfo_cover_memory(const struct numa_meminfo *mi) 452 { 453 u64 numaram, e820ram; 454 int i; 455 456 numaram = 0; 457 for (i = 0; i < mi->nr_blks; i++) { 458 u64 s = mi->blk[i].start >> PAGE_SHIFT; 459 u64 e = mi->blk[i].end >> PAGE_SHIFT; 460 numaram += e - s; 461 numaram -= __absent_pages_in_range(mi->blk[i].nid, s, e); 462 if ((s64)numaram < 0) 463 numaram = 0; 464 } 465 466 e820ram = max_pfn - absent_pages_in_range(0, max_pfn); 467 468 /* We seem to lose 3 pages somewhere. Allow 1M of slack. */ 469 if ((s64)(e820ram - numaram) >= (1 << (20 - PAGE_SHIFT))) { 470 printk(KERN_ERR "NUMA: nodes only cover %LuMB of your %LuMB e820 RAM. Not used.\n", 471 (numaram << PAGE_SHIFT) >> 20, 472 (e820ram << PAGE_SHIFT) >> 20); 473 return false; 474 } 475 return true; 476 } 477 478 /* 479 * Mark all currently memblock-reserved physical memory (which covers the 480 * kernel's own memory ranges) as hot-unswappable. 481 */ 482 static void __init numa_clear_kernel_node_hotplug(void) 483 { 484 nodemask_t reserved_nodemask = NODE_MASK_NONE; 485 struct memblock_region *mb_region; 486 int i; 487 488 /* 489 * We have to do some preprocessing of memblock regions, to 490 * make them suitable for reservation. 491 * 492 * At this time, all memory regions reserved by memblock are 493 * used by the kernel, but those regions are not split up 494 * along node boundaries yet, and don't necessarily have their 495 * node ID set yet either. 496 * 497 * So iterate over all memory known to the x86 architecture, 498 * and use those ranges to set the nid in memblock.reserved. 499 * This will split up the memblock regions along node 500 * boundaries and will set the node IDs as well. 501 */ 502 for (i = 0; i < numa_meminfo.nr_blks; i++) { 503 struct numa_memblk *mb = numa_meminfo.blk + i; 504 int ret; 505 506 ret = memblock_set_node(mb->start, mb->end - mb->start, &memblock.reserved, mb->nid); 507 WARN_ON_ONCE(ret); 508 } 509 510 /* 511 * Now go over all reserved memblock regions, to construct a 512 * node mask of all kernel reserved memory areas. 513 * 514 * [ Note, when booting with mem=nn[kMG] or in a kdump kernel, 515 * numa_meminfo might not include all memblock.reserved 516 * memory ranges, because quirks such as trim_snb_memory() 517 * reserve specific pages for Sandy Bridge graphics. ] 518 */ 519 for_each_memblock(reserved, mb_region) { 520 int nid = memblock_get_region_node(mb_region); 521 522 if (nid != MAX_NUMNODES) 523 node_set(nid, reserved_nodemask); 524 } 525 526 /* 527 * Finally, clear the MEMBLOCK_HOTPLUG flag for all memory 528 * belonging to the reserved node mask. 529 * 530 * Note that this will include memory regions that reside 531 * on nodes that contain kernel memory - entire nodes 532 * become hot-unpluggable: 533 */ 534 for (i = 0; i < numa_meminfo.nr_blks; i++) { 535 struct numa_memblk *mb = numa_meminfo.blk + i; 536 537 if (!node_isset(mb->nid, reserved_nodemask)) 538 continue; 539 540 memblock_clear_hotplug(mb->start, mb->end - mb->start); 541 } 542 } 543 544 static int __init numa_register_memblks(struct numa_meminfo *mi) 545 { 546 int i, nid; 547 548 /* Account for nodes with cpus and no memory */ 549 node_possible_map = numa_nodes_parsed; 550 numa_nodemask_from_meminfo(&node_possible_map, mi); 551 if (WARN_ON(nodes_empty(node_possible_map))) 552 return -EINVAL; 553 554 for (i = 0; i < mi->nr_blks; i++) { 555 struct numa_memblk *mb = &mi->blk[i]; 556 memblock_set_node(mb->start, mb->end - mb->start, 557 &memblock.memory, mb->nid); 558 } 559 560 /* 561 * At very early time, the kernel have to use some memory such as 562 * loading the kernel image. We cannot prevent this anyway. So any 563 * node the kernel resides in should be un-hotpluggable. 564 * 565 * And when we come here, alloc node data won't fail. 566 */ 567 numa_clear_kernel_node_hotplug(); 568 569 /* 570 * If sections array is gonna be used for pfn -> nid mapping, check 571 * whether its granularity is fine enough. 572 */ 573 if (IS_ENABLED(NODE_NOT_IN_PAGE_FLAGS)) { 574 unsigned long pfn_align = node_map_pfn_alignment(); 575 576 if (pfn_align && pfn_align < PAGES_PER_SECTION) { 577 pr_warn("Node alignment %LuMB < min %LuMB, rejecting NUMA config\n", 578 PFN_PHYS(pfn_align) >> 20, 579 PFN_PHYS(PAGES_PER_SECTION) >> 20); 580 return -EINVAL; 581 } 582 } 583 if (!numa_meminfo_cover_memory(mi)) 584 return -EINVAL; 585 586 /* Finally register nodes. */ 587 for_each_node_mask(nid, node_possible_map) { 588 u64 start = PFN_PHYS(max_pfn); 589 u64 end = 0; 590 591 for (i = 0; i < mi->nr_blks; i++) { 592 if (nid != mi->blk[i].nid) 593 continue; 594 start = min(mi->blk[i].start, start); 595 end = max(mi->blk[i].end, end); 596 } 597 598 if (start >= end) 599 continue; 600 601 /* 602 * Don't confuse VM with a node that doesn't have the 603 * minimum amount of memory: 604 */ 605 if (end && (end - start) < NODE_MIN_SIZE) 606 continue; 607 608 alloc_node_data(nid); 609 } 610 611 /* Dump memblock with node info and return. */ 612 memblock_dump_all(); 613 return 0; 614 } 615 616 /* 617 * There are unfortunately some poorly designed mainboards around that 618 * only connect memory to a single CPU. This breaks the 1:1 cpu->node 619 * mapping. To avoid this fill in the mapping for all possible CPUs, 620 * as the number of CPUs is not known yet. We round robin the existing 621 * nodes. 622 */ 623 static void __init numa_init_array(void) 624 { 625 int rr, i; 626 627 rr = first_node(node_online_map); 628 for (i = 0; i < nr_cpu_ids; i++) { 629 if (early_cpu_to_node(i) != NUMA_NO_NODE) 630 continue; 631 numa_set_node(i, rr); 632 rr = next_node_in(rr, node_online_map); 633 } 634 } 635 636 static int __init numa_init(int (*init_func)(void)) 637 { 638 int i; 639 int ret; 640 641 for (i = 0; i < MAX_LOCAL_APIC; i++) 642 set_apicid_to_node(i, NUMA_NO_NODE); 643 644 nodes_clear(numa_nodes_parsed); 645 nodes_clear(node_possible_map); 646 nodes_clear(node_online_map); 647 memset(&numa_meminfo, 0, sizeof(numa_meminfo)); 648 WARN_ON(memblock_set_node(0, ULLONG_MAX, &memblock.memory, 649 MAX_NUMNODES)); 650 WARN_ON(memblock_set_node(0, ULLONG_MAX, &memblock.reserved, 651 MAX_NUMNODES)); 652 /* In case that parsing SRAT failed. */ 653 WARN_ON(memblock_clear_hotplug(0, ULLONG_MAX)); 654 numa_reset_distance(); 655 656 ret = init_func(); 657 if (ret < 0) 658 return ret; 659 660 /* 661 * We reset memblock back to the top-down direction 662 * here because if we configured ACPI_NUMA, we have 663 * parsed SRAT in init_func(). It is ok to have the 664 * reset here even if we did't configure ACPI_NUMA 665 * or acpi numa init fails and fallbacks to dummy 666 * numa init. 667 */ 668 memblock_set_bottom_up(false); 669 670 ret = numa_cleanup_meminfo(&numa_meminfo); 671 if (ret < 0) 672 return ret; 673 674 numa_emulation(&numa_meminfo, numa_distance_cnt); 675 676 ret = numa_register_memblks(&numa_meminfo); 677 if (ret < 0) 678 return ret; 679 680 for (i = 0; i < nr_cpu_ids; i++) { 681 int nid = early_cpu_to_node(i); 682 683 if (nid == NUMA_NO_NODE) 684 continue; 685 if (!node_online(nid)) 686 numa_clear_node(i); 687 } 688 numa_init_array(); 689 690 return 0; 691 } 692 693 /** 694 * dummy_numa_init - Fallback dummy NUMA init 695 * 696 * Used if there's no underlying NUMA architecture, NUMA initialization 697 * fails, or NUMA is disabled on the command line. 698 * 699 * Must online at least one node and add memory blocks that cover all 700 * allowed memory. This function must not fail. 701 */ 702 static int __init dummy_numa_init(void) 703 { 704 printk(KERN_INFO "%s\n", 705 numa_off ? "NUMA turned off" : "No NUMA configuration found"); 706 printk(KERN_INFO "Faking a node at [mem %#018Lx-%#018Lx]\n", 707 0LLU, PFN_PHYS(max_pfn) - 1); 708 709 node_set(0, numa_nodes_parsed); 710 numa_add_memblk(0, 0, PFN_PHYS(max_pfn)); 711 712 return 0; 713 } 714 715 /** 716 * x86_numa_init - Initialize NUMA 717 * 718 * Try each configured NUMA initialization method until one succeeds. The 719 * last fallback is dummy single node config encompassing whole memory and 720 * never fails. 721 */ 722 void __init x86_numa_init(void) 723 { 724 if (!numa_off) { 725 #ifdef CONFIG_ACPI_NUMA 726 if (!numa_init(x86_acpi_numa_init)) 727 return; 728 #endif 729 #ifdef CONFIG_AMD_NUMA 730 if (!numa_init(amd_numa_init)) 731 return; 732 #endif 733 } 734 735 numa_init(dummy_numa_init); 736 } 737 738 static void __init init_memory_less_node(int nid) 739 { 740 /* Allocate and initialize node data. Memory-less node is now online.*/ 741 alloc_node_data(nid); 742 free_area_init_memoryless_node(nid); 743 744 /* 745 * All zonelists will be built later in start_kernel() after per cpu 746 * areas are initialized. 747 */ 748 } 749 750 /* 751 * Setup early cpu_to_node. 752 * 753 * Populate cpu_to_node[] only if x86_cpu_to_apicid[], 754 * and apicid_to_node[] tables have valid entries for a CPU. 755 * This means we skip cpu_to_node[] initialisation for NUMA 756 * emulation and faking node case (when running a kernel compiled 757 * for NUMA on a non NUMA box), which is OK as cpu_to_node[] 758 * is already initialized in a round robin manner at numa_init_array, 759 * prior to this call, and this initialization is good enough 760 * for the fake NUMA cases. 761 * 762 * Called before the per_cpu areas are setup. 763 */ 764 void __init init_cpu_to_node(void) 765 { 766 int cpu; 767 u16 *cpu_to_apicid = early_per_cpu_ptr(x86_cpu_to_apicid); 768 769 BUG_ON(cpu_to_apicid == NULL); 770 771 for_each_possible_cpu(cpu) { 772 int node = numa_cpu_node(cpu); 773 774 if (node == NUMA_NO_NODE) 775 continue; 776 777 if (!node_online(node)) 778 init_memory_less_node(node); 779 780 numa_set_node(cpu, node); 781 } 782 } 783 784 #ifndef CONFIG_DEBUG_PER_CPU_MAPS 785 786 # ifndef CONFIG_NUMA_EMU 787 void numa_add_cpu(int cpu) 788 { 789 cpumask_set_cpu(cpu, node_to_cpumask_map[early_cpu_to_node(cpu)]); 790 } 791 792 void numa_remove_cpu(int cpu) 793 { 794 cpumask_clear_cpu(cpu, node_to_cpumask_map[early_cpu_to_node(cpu)]); 795 } 796 # endif /* !CONFIG_NUMA_EMU */ 797 798 #else /* !CONFIG_DEBUG_PER_CPU_MAPS */ 799 800 int __cpu_to_node(int cpu) 801 { 802 if (early_per_cpu_ptr(x86_cpu_to_node_map)) { 803 printk(KERN_WARNING 804 "cpu_to_node(%d): usage too early!\n", cpu); 805 dump_stack(); 806 return early_per_cpu_ptr(x86_cpu_to_node_map)[cpu]; 807 } 808 return per_cpu(x86_cpu_to_node_map, cpu); 809 } 810 EXPORT_SYMBOL(__cpu_to_node); 811 812 /* 813 * Same function as cpu_to_node() but used if called before the 814 * per_cpu areas are setup. 815 */ 816 int early_cpu_to_node(int cpu) 817 { 818 if (early_per_cpu_ptr(x86_cpu_to_node_map)) 819 return early_per_cpu_ptr(x86_cpu_to_node_map)[cpu]; 820 821 if (!cpu_possible(cpu)) { 822 printk(KERN_WARNING 823 "early_cpu_to_node(%d): no per_cpu area!\n", cpu); 824 dump_stack(); 825 return NUMA_NO_NODE; 826 } 827 return per_cpu(x86_cpu_to_node_map, cpu); 828 } 829 830 void debug_cpumask_set_cpu(int cpu, int node, bool enable) 831 { 832 struct cpumask *mask; 833 834 if (node == NUMA_NO_NODE) { 835 /* early_cpu_to_node() already emits a warning and trace */ 836 return; 837 } 838 mask = node_to_cpumask_map[node]; 839 if (!mask) { 840 pr_err("node_to_cpumask_map[%i] NULL\n", node); 841 dump_stack(); 842 return; 843 } 844 845 if (enable) 846 cpumask_set_cpu(cpu, mask); 847 else 848 cpumask_clear_cpu(cpu, mask); 849 850 printk(KERN_DEBUG "%s cpu %d node %d: mask now %*pbl\n", 851 enable ? "numa_add_cpu" : "numa_remove_cpu", 852 cpu, node, cpumask_pr_args(mask)); 853 return; 854 } 855 856 # ifndef CONFIG_NUMA_EMU 857 static void numa_set_cpumask(int cpu, bool enable) 858 { 859 debug_cpumask_set_cpu(cpu, early_cpu_to_node(cpu), enable); 860 } 861 862 void numa_add_cpu(int cpu) 863 { 864 numa_set_cpumask(cpu, true); 865 } 866 867 void numa_remove_cpu(int cpu) 868 { 869 numa_set_cpumask(cpu, false); 870 } 871 # endif /* !CONFIG_NUMA_EMU */ 872 873 /* 874 * Returns a pointer to the bitmask of CPUs on Node 'node'. 875 */ 876 const struct cpumask *cpumask_of_node(int node) 877 { 878 if ((unsigned)node >= nr_node_ids) { 879 printk(KERN_WARNING 880 "cpumask_of_node(%d): (unsigned)node >= nr_node_ids(%u)\n", 881 node, nr_node_ids); 882 dump_stack(); 883 return cpu_none_mask; 884 } 885 if (node_to_cpumask_map[node] == NULL) { 886 printk(KERN_WARNING 887 "cpumask_of_node(%d): no node_to_cpumask_map!\n", 888 node); 889 dump_stack(); 890 return cpu_online_mask; 891 } 892 return node_to_cpumask_map[node]; 893 } 894 EXPORT_SYMBOL(cpumask_of_node); 895 896 #endif /* !CONFIG_DEBUG_PER_CPU_MAPS */ 897 898 #ifdef CONFIG_NUMA_KEEP_MEMINFO 899 static int meminfo_to_nid(struct numa_meminfo *mi, u64 start) 900 { 901 int i; 902 903 for (i = 0; i < mi->nr_blks; i++) 904 if (mi->blk[i].start <= start && mi->blk[i].end > start) 905 return mi->blk[i].nid; 906 return NUMA_NO_NODE; 907 } 908 909 int phys_to_target_node(phys_addr_t start) 910 { 911 int nid = meminfo_to_nid(&numa_meminfo, start); 912 913 /* 914 * Prefer online nodes, but if reserved memory might be 915 * hot-added continue the search with reserved ranges. 916 */ 917 if (nid != NUMA_NO_NODE) 918 return nid; 919 920 return meminfo_to_nid(&numa_reserved_meminfo, start); 921 } 922 EXPORT_SYMBOL_GPL(phys_to_target_node); 923 924 int memory_add_physaddr_to_nid(u64 start) 925 { 926 int nid = meminfo_to_nid(&numa_meminfo, start); 927 928 if (nid == NUMA_NO_NODE) 929 nid = numa_meminfo.blk[0].nid; 930 return nid; 931 } 932 #endif 933