1 /* 2 * pSeries NUMA support 3 * 4 * Copyright (C) 2002 Anton Blanchard <anton@au.ibm.com>, IBM 5 * 6 * This program is free software; you can redistribute it and/or 7 * modify it under the terms of the GNU General Public License 8 * as published by the Free Software Foundation; either version 9 * 2 of the License, or (at your option) any later version. 10 */ 11 #define pr_fmt(fmt) "numa: " fmt 12 13 #include <linux/threads.h> 14 #include <linux/memblock.h> 15 #include <linux/init.h> 16 #include <linux/mm.h> 17 #include <linux/mmzone.h> 18 #include <linux/export.h> 19 #include <linux/nodemask.h> 20 #include <linux/cpu.h> 21 #include <linux/notifier.h> 22 #include <linux/of.h> 23 #include <linux/pfn.h> 24 #include <linux/cpuset.h> 25 #include <linux/node.h> 26 #include <linux/stop_machine.h> 27 #include <linux/proc_fs.h> 28 #include <linux/seq_file.h> 29 #include <linux/uaccess.h> 30 #include <linux/slab.h> 31 #include <asm/cputhreads.h> 32 #include <asm/sparsemem.h> 33 #include <asm/prom.h> 34 #include <asm/smp.h> 35 #include <asm/topology.h> 36 #include <asm/firmware.h> 37 #include <asm/paca.h> 38 #include <asm/hvcall.h> 39 #include <asm/setup.h> 40 #include <asm/vdso.h> 41 #include <asm/drmem.h> 42 43 static int numa_enabled = 1; 44 45 static char *cmdline __initdata; 46 47 static int numa_debug; 48 #define dbg(args...) if (numa_debug) { printk(KERN_INFO args); } 49 50 int numa_cpu_lookup_table[NR_CPUS]; 51 cpumask_var_t node_to_cpumask_map[MAX_NUMNODES]; 52 struct pglist_data *node_data[MAX_NUMNODES]; 53 54 EXPORT_SYMBOL(numa_cpu_lookup_table); 55 EXPORT_SYMBOL(node_to_cpumask_map); 56 EXPORT_SYMBOL(node_data); 57 58 static int min_common_depth; 59 static int n_mem_addr_cells, n_mem_size_cells; 60 static int form1_affinity; 61 62 #define MAX_DISTANCE_REF_POINTS 4 63 static int distance_ref_points_depth; 64 static const __be32 *distance_ref_points; 65 static int distance_lookup_table[MAX_NUMNODES][MAX_DISTANCE_REF_POINTS]; 66 67 /* 68 * Allocate node_to_cpumask_map based on number of available nodes 69 * Requires node_possible_map to be valid. 70 * 71 * Note: cpumask_of_node() is not valid until after this is done. 72 */ 73 static void __init setup_node_to_cpumask_map(void) 74 { 75 unsigned int node; 76 77 /* setup nr_node_ids if not done yet */ 78 if (nr_node_ids == MAX_NUMNODES) 79 setup_nr_node_ids(); 80 81 /* allocate the map */ 82 for_each_node(node) 83 alloc_bootmem_cpumask_var(&node_to_cpumask_map[node]); 84 85 /* cpumask_of_node() will now work */ 86 dbg("Node to cpumask map for %u nodes\n", nr_node_ids); 87 } 88 89 static int __init fake_numa_create_new_node(unsigned long end_pfn, 90 unsigned int *nid) 91 { 92 unsigned long long mem; 93 char *p = cmdline; 94 static unsigned int fake_nid; 95 static unsigned long long curr_boundary; 96 97 /* 98 * Modify node id, iff we started creating NUMA nodes 99 * We want to continue from where we left of the last time 100 */ 101 if (fake_nid) 102 *nid = fake_nid; 103 /* 104 * In case there are no more arguments to parse, the 105 * node_id should be the same as the last fake node id 106 * (we've handled this above). 107 */ 108 if (!p) 109 return 0; 110 111 mem = memparse(p, &p); 112 if (!mem) 113 return 0; 114 115 if (mem < curr_boundary) 116 return 0; 117 118 curr_boundary = mem; 119 120 if ((end_pfn << PAGE_SHIFT) > mem) { 121 /* 122 * Skip commas and spaces 123 */ 124 while (*p == ',' || *p == ' ' || *p == '\t') 125 p++; 126 127 cmdline = p; 128 fake_nid++; 129 *nid = fake_nid; 130 dbg("created new fake_node with id %d\n", fake_nid); 131 return 1; 132 } 133 return 0; 134 } 135 136 static void reset_numa_cpu_lookup_table(void) 137 { 138 unsigned int cpu; 139 140 for_each_possible_cpu(cpu) 141 numa_cpu_lookup_table[cpu] = -1; 142 } 143 144 static void map_cpu_to_node(int cpu, int node) 145 { 146 update_numa_cpu_lookup_table(cpu, node); 147 148 dbg("adding cpu %d to node %d\n", cpu, node); 149 150 if (!(cpumask_test_cpu(cpu, node_to_cpumask_map[node]))) 151 cpumask_set_cpu(cpu, node_to_cpumask_map[node]); 152 } 153 154 #if defined(CONFIG_HOTPLUG_CPU) || defined(CONFIG_PPC_SPLPAR) 155 static void unmap_cpu_from_node(unsigned long cpu) 156 { 157 int node = numa_cpu_lookup_table[cpu]; 158 159 dbg("removing cpu %lu from node %d\n", cpu, node); 160 161 if (cpumask_test_cpu(cpu, node_to_cpumask_map[node])) { 162 cpumask_clear_cpu(cpu, node_to_cpumask_map[node]); 163 } else { 164 printk(KERN_ERR "WARNING: cpu %lu not found in node %d\n", 165 cpu, node); 166 } 167 } 168 #endif /* CONFIG_HOTPLUG_CPU || CONFIG_PPC_SPLPAR */ 169 170 /* must hold reference to node during call */ 171 static const __be32 *of_get_associativity(struct device_node *dev) 172 { 173 return of_get_property(dev, "ibm,associativity", NULL); 174 } 175 176 int __node_distance(int a, int b) 177 { 178 int i; 179 int distance = LOCAL_DISTANCE; 180 181 if (!form1_affinity) 182 return ((a == b) ? LOCAL_DISTANCE : REMOTE_DISTANCE); 183 184 for (i = 0; i < distance_ref_points_depth; i++) { 185 if (distance_lookup_table[a][i] == distance_lookup_table[b][i]) 186 break; 187 188 /* Double the distance for each NUMA level */ 189 distance *= 2; 190 } 191 192 return distance; 193 } 194 EXPORT_SYMBOL(__node_distance); 195 196 static void initialize_distance_lookup_table(int nid, 197 const __be32 *associativity) 198 { 199 int i; 200 201 if (!form1_affinity) 202 return; 203 204 for (i = 0; i < distance_ref_points_depth; i++) { 205 const __be32 *entry; 206 207 entry = &associativity[be32_to_cpu(distance_ref_points[i]) - 1]; 208 distance_lookup_table[nid][i] = of_read_number(entry, 1); 209 } 210 } 211 212 /* Returns nid in the range [0..MAX_NUMNODES-1], or -1 if no useful numa 213 * info is found. 214 */ 215 static int associativity_to_nid(const __be32 *associativity) 216 { 217 int nid = NUMA_NO_NODE; 218 219 if (min_common_depth == -1) 220 goto out; 221 222 if (of_read_number(associativity, 1) >= min_common_depth) 223 nid = of_read_number(&associativity[min_common_depth], 1); 224 225 /* POWER4 LPAR uses 0xffff as invalid node */ 226 if (nid == 0xffff || nid >= MAX_NUMNODES) 227 nid = NUMA_NO_NODE; 228 229 if (nid > 0 && 230 of_read_number(associativity, 1) >= distance_ref_points_depth) { 231 /* 232 * Skip the length field and send start of associativity array 233 */ 234 initialize_distance_lookup_table(nid, associativity + 1); 235 } 236 237 out: 238 return nid; 239 } 240 241 /* Returns the nid associated with the given device tree node, 242 * or -1 if not found. 243 */ 244 static int of_node_to_nid_single(struct device_node *device) 245 { 246 int nid = NUMA_NO_NODE; 247 const __be32 *tmp; 248 249 tmp = of_get_associativity(device); 250 if (tmp) 251 nid = associativity_to_nid(tmp); 252 return nid; 253 } 254 255 /* Walk the device tree upwards, looking for an associativity id */ 256 int of_node_to_nid(struct device_node *device) 257 { 258 int nid = NUMA_NO_NODE; 259 260 of_node_get(device); 261 while (device) { 262 nid = of_node_to_nid_single(device); 263 if (nid != -1) 264 break; 265 266 device = of_get_next_parent(device); 267 } 268 of_node_put(device); 269 270 return nid; 271 } 272 EXPORT_SYMBOL(of_node_to_nid); 273 274 static int __init find_min_common_depth(void) 275 { 276 int depth; 277 struct device_node *root; 278 279 if (firmware_has_feature(FW_FEATURE_OPAL)) 280 root = of_find_node_by_path("/ibm,opal"); 281 else 282 root = of_find_node_by_path("/rtas"); 283 if (!root) 284 root = of_find_node_by_path("/"); 285 286 /* 287 * This property is a set of 32-bit integers, each representing 288 * an index into the ibm,associativity nodes. 289 * 290 * With form 0 affinity the first integer is for an SMP configuration 291 * (should be all 0's) and the second is for a normal NUMA 292 * configuration. We have only one level of NUMA. 293 * 294 * With form 1 affinity the first integer is the most significant 295 * NUMA boundary and the following are progressively less significant 296 * boundaries. There can be more than one level of NUMA. 297 */ 298 distance_ref_points = of_get_property(root, 299 "ibm,associativity-reference-points", 300 &distance_ref_points_depth); 301 302 if (!distance_ref_points) { 303 dbg("NUMA: ibm,associativity-reference-points not found.\n"); 304 goto err; 305 } 306 307 distance_ref_points_depth /= sizeof(int); 308 309 if (firmware_has_feature(FW_FEATURE_OPAL) || 310 firmware_has_feature(FW_FEATURE_TYPE1_AFFINITY)) { 311 dbg("Using form 1 affinity\n"); 312 form1_affinity = 1; 313 } 314 315 if (form1_affinity) { 316 depth = of_read_number(distance_ref_points, 1); 317 } else { 318 if (distance_ref_points_depth < 2) { 319 printk(KERN_WARNING "NUMA: " 320 "short ibm,associativity-reference-points\n"); 321 goto err; 322 } 323 324 depth = of_read_number(&distance_ref_points[1], 1); 325 } 326 327 /* 328 * Warn and cap if the hardware supports more than 329 * MAX_DISTANCE_REF_POINTS domains. 330 */ 331 if (distance_ref_points_depth > MAX_DISTANCE_REF_POINTS) { 332 printk(KERN_WARNING "NUMA: distance array capped at " 333 "%d entries\n", MAX_DISTANCE_REF_POINTS); 334 distance_ref_points_depth = MAX_DISTANCE_REF_POINTS; 335 } 336 337 of_node_put(root); 338 return depth; 339 340 err: 341 of_node_put(root); 342 return -1; 343 } 344 345 static void __init get_n_mem_cells(int *n_addr_cells, int *n_size_cells) 346 { 347 struct device_node *memory = NULL; 348 349 memory = of_find_node_by_type(memory, "memory"); 350 if (!memory) 351 panic("numa.c: No memory nodes found!"); 352 353 *n_addr_cells = of_n_addr_cells(memory); 354 *n_size_cells = of_n_size_cells(memory); 355 of_node_put(memory); 356 } 357 358 static unsigned long read_n_cells(int n, const __be32 **buf) 359 { 360 unsigned long result = 0; 361 362 while (n--) { 363 result = (result << 32) | of_read_number(*buf, 1); 364 (*buf)++; 365 } 366 return result; 367 } 368 369 struct assoc_arrays { 370 u32 n_arrays; 371 u32 array_sz; 372 const __be32 *arrays; 373 }; 374 375 /* 376 * Retrieve and validate the list of associativity arrays for drconf 377 * memory from the ibm,associativity-lookup-arrays property of the 378 * device tree.. 379 * 380 * The layout of the ibm,associativity-lookup-arrays property is a number N 381 * indicating the number of associativity arrays, followed by a number M 382 * indicating the size of each associativity array, followed by a list 383 * of N associativity arrays. 384 */ 385 static int of_get_assoc_arrays(struct assoc_arrays *aa) 386 { 387 struct device_node *memory; 388 const __be32 *prop; 389 u32 len; 390 391 memory = of_find_node_by_path("/ibm,dynamic-reconfiguration-memory"); 392 if (!memory) 393 return -1; 394 395 prop = of_get_property(memory, "ibm,associativity-lookup-arrays", &len); 396 if (!prop || len < 2 * sizeof(unsigned int)) { 397 of_node_put(memory); 398 return -1; 399 } 400 401 aa->n_arrays = of_read_number(prop++, 1); 402 aa->array_sz = of_read_number(prop++, 1); 403 404 of_node_put(memory); 405 406 /* Now that we know the number of arrays and size of each array, 407 * revalidate the size of the property read in. 408 */ 409 if (len < (aa->n_arrays * aa->array_sz + 2) * sizeof(unsigned int)) 410 return -1; 411 412 aa->arrays = prop; 413 return 0; 414 } 415 416 /* 417 * This is like of_node_to_nid_single() for memory represented in the 418 * ibm,dynamic-reconfiguration-memory node. 419 */ 420 static int of_drconf_to_nid_single(struct drmem_lmb *lmb) 421 { 422 struct assoc_arrays aa = { .arrays = NULL }; 423 int default_nid = 0; 424 int nid = default_nid; 425 int rc, index; 426 427 rc = of_get_assoc_arrays(&aa); 428 if (rc) 429 return default_nid; 430 431 if (min_common_depth > 0 && min_common_depth <= aa.array_sz && 432 !(lmb->flags & DRCONF_MEM_AI_INVALID) && 433 lmb->aa_index < aa.n_arrays) { 434 index = lmb->aa_index * aa.array_sz + min_common_depth - 1; 435 nid = of_read_number(&aa.arrays[index], 1); 436 437 if (nid == 0xffff || nid >= MAX_NUMNODES) 438 nid = default_nid; 439 440 if (nid > 0) { 441 index = lmb->aa_index * aa.array_sz; 442 initialize_distance_lookup_table(nid, 443 &aa.arrays[index]); 444 } 445 } 446 447 return nid; 448 } 449 450 /* 451 * Figure out to which domain a cpu belongs and stick it there. 452 * Return the id of the domain used. 453 */ 454 static int numa_setup_cpu(unsigned long lcpu) 455 { 456 int nid = NUMA_NO_NODE; 457 struct device_node *cpu; 458 459 /* 460 * If a valid cpu-to-node mapping is already available, use it 461 * directly instead of querying the firmware, since it represents 462 * the most recent mapping notified to us by the platform (eg: VPHN). 463 */ 464 if ((nid = numa_cpu_lookup_table[lcpu]) >= 0) { 465 map_cpu_to_node(lcpu, nid); 466 return nid; 467 } 468 469 cpu = of_get_cpu_node(lcpu, NULL); 470 471 if (!cpu) { 472 WARN_ON(1); 473 if (cpu_present(lcpu)) 474 goto out_present; 475 else 476 goto out; 477 } 478 479 nid = of_node_to_nid_single(cpu); 480 481 out_present: 482 if (nid < 0 || !node_possible(nid)) 483 nid = first_online_node; 484 485 map_cpu_to_node(lcpu, nid); 486 of_node_put(cpu); 487 out: 488 return nid; 489 } 490 491 static void verify_cpu_node_mapping(int cpu, int node) 492 { 493 int base, sibling, i; 494 495 /* Verify that all the threads in the core belong to the same node */ 496 base = cpu_first_thread_sibling(cpu); 497 498 for (i = 0; i < threads_per_core; i++) { 499 sibling = base + i; 500 501 if (sibling == cpu || cpu_is_offline(sibling)) 502 continue; 503 504 if (cpu_to_node(sibling) != node) { 505 WARN(1, "CPU thread siblings %d and %d don't belong" 506 " to the same node!\n", cpu, sibling); 507 break; 508 } 509 } 510 } 511 512 /* Must run before sched domains notifier. */ 513 static int ppc_numa_cpu_prepare(unsigned int cpu) 514 { 515 int nid; 516 517 nid = numa_setup_cpu(cpu); 518 verify_cpu_node_mapping(cpu, nid); 519 return 0; 520 } 521 522 static int ppc_numa_cpu_dead(unsigned int cpu) 523 { 524 #ifdef CONFIG_HOTPLUG_CPU 525 unmap_cpu_from_node(cpu); 526 #endif 527 return 0; 528 } 529 530 /* 531 * Check and possibly modify a memory region to enforce the memory limit. 532 * 533 * Returns the size the region should have to enforce the memory limit. 534 * This will either be the original value of size, a truncated value, 535 * or zero. If the returned value of size is 0 the region should be 536 * discarded as it lies wholly above the memory limit. 537 */ 538 static unsigned long __init numa_enforce_memory_limit(unsigned long start, 539 unsigned long size) 540 { 541 /* 542 * We use memblock_end_of_DRAM() in here instead of memory_limit because 543 * we've already adjusted it for the limit and it takes care of 544 * having memory holes below the limit. Also, in the case of 545 * iommu_is_off, memory_limit is not set but is implicitly enforced. 546 */ 547 548 if (start + size <= memblock_end_of_DRAM()) 549 return size; 550 551 if (start >= memblock_end_of_DRAM()) 552 return 0; 553 554 return memblock_end_of_DRAM() - start; 555 } 556 557 /* 558 * Reads the counter for a given entry in 559 * linux,drconf-usable-memory property 560 */ 561 static inline int __init read_usm_ranges(const __be32 **usm) 562 { 563 /* 564 * For each lmb in ibm,dynamic-memory a corresponding 565 * entry in linux,drconf-usable-memory property contains 566 * a counter followed by that many (base, size) duple. 567 * read the counter from linux,drconf-usable-memory 568 */ 569 return read_n_cells(n_mem_size_cells, usm); 570 } 571 572 /* 573 * Extract NUMA information from the ibm,dynamic-reconfiguration-memory 574 * node. This assumes n_mem_{addr,size}_cells have been set. 575 */ 576 static void __init numa_setup_drmem_lmb(struct drmem_lmb *lmb, 577 const __be32 **usm) 578 { 579 unsigned int ranges, is_kexec_kdump = 0; 580 unsigned long base, size, sz; 581 int nid; 582 583 /* 584 * Skip this block if the reserved bit is set in flags (0x80) 585 * or if the block is not assigned to this partition (0x8) 586 */ 587 if ((lmb->flags & DRCONF_MEM_RESERVED) 588 || !(lmb->flags & DRCONF_MEM_ASSIGNED)) 589 return; 590 591 if (*usm) 592 is_kexec_kdump = 1; 593 594 base = lmb->base_addr; 595 size = drmem_lmb_size(); 596 ranges = 1; 597 598 if (is_kexec_kdump) { 599 ranges = read_usm_ranges(usm); 600 if (!ranges) /* there are no (base, size) duple */ 601 return; 602 } 603 604 do { 605 if (is_kexec_kdump) { 606 base = read_n_cells(n_mem_addr_cells, usm); 607 size = read_n_cells(n_mem_size_cells, usm); 608 } 609 610 nid = of_drconf_to_nid_single(lmb); 611 fake_numa_create_new_node(((base + size) >> PAGE_SHIFT), 612 &nid); 613 node_set_online(nid); 614 sz = numa_enforce_memory_limit(base, size); 615 if (sz) 616 memblock_set_node(base, sz, &memblock.memory, nid); 617 } while (--ranges); 618 } 619 620 static int __init parse_numa_properties(void) 621 { 622 struct device_node *memory; 623 int default_nid = 0; 624 unsigned long i; 625 626 if (numa_enabled == 0) { 627 printk(KERN_WARNING "NUMA disabled by user\n"); 628 return -1; 629 } 630 631 min_common_depth = find_min_common_depth(); 632 633 if (min_common_depth < 0) 634 return min_common_depth; 635 636 dbg("NUMA associativity depth for CPU/Memory: %d\n", min_common_depth); 637 638 /* 639 * Even though we connect cpus to numa domains later in SMP 640 * init, we need to know the node ids now. This is because 641 * each node to be onlined must have NODE_DATA etc backing it. 642 */ 643 for_each_present_cpu(i) { 644 struct device_node *cpu; 645 int nid; 646 647 cpu = of_get_cpu_node(i, NULL); 648 BUG_ON(!cpu); 649 nid = of_node_to_nid_single(cpu); 650 of_node_put(cpu); 651 652 /* 653 * Don't fall back to default_nid yet -- we will plug 654 * cpus into nodes once the memory scan has discovered 655 * the topology. 656 */ 657 if (nid < 0) 658 continue; 659 node_set_online(nid); 660 } 661 662 get_n_mem_cells(&n_mem_addr_cells, &n_mem_size_cells); 663 664 for_each_node_by_type(memory, "memory") { 665 unsigned long start; 666 unsigned long size; 667 int nid; 668 int ranges; 669 const __be32 *memcell_buf; 670 unsigned int len; 671 672 memcell_buf = of_get_property(memory, 673 "linux,usable-memory", &len); 674 if (!memcell_buf || len <= 0) 675 memcell_buf = of_get_property(memory, "reg", &len); 676 if (!memcell_buf || len <= 0) 677 continue; 678 679 /* ranges in cell */ 680 ranges = (len >> 2) / (n_mem_addr_cells + n_mem_size_cells); 681 new_range: 682 /* these are order-sensitive, and modify the buffer pointer */ 683 start = read_n_cells(n_mem_addr_cells, &memcell_buf); 684 size = read_n_cells(n_mem_size_cells, &memcell_buf); 685 686 /* 687 * Assumption: either all memory nodes or none will 688 * have associativity properties. If none, then 689 * everything goes to default_nid. 690 */ 691 nid = of_node_to_nid_single(memory); 692 if (nid < 0) 693 nid = default_nid; 694 695 fake_numa_create_new_node(((start + size) >> PAGE_SHIFT), &nid); 696 node_set_online(nid); 697 698 size = numa_enforce_memory_limit(start, size); 699 if (size) 700 memblock_set_node(start, size, &memblock.memory, nid); 701 702 if (--ranges) 703 goto new_range; 704 } 705 706 /* 707 * Now do the same thing for each MEMBLOCK listed in the 708 * ibm,dynamic-memory property in the 709 * ibm,dynamic-reconfiguration-memory node. 710 */ 711 memory = of_find_node_by_path("/ibm,dynamic-reconfiguration-memory"); 712 if (memory) { 713 walk_drmem_lmbs(memory, numa_setup_drmem_lmb); 714 of_node_put(memory); 715 } 716 717 return 0; 718 } 719 720 static void __init setup_nonnuma(void) 721 { 722 unsigned long top_of_ram = memblock_end_of_DRAM(); 723 unsigned long total_ram = memblock_phys_mem_size(); 724 unsigned long start_pfn, end_pfn; 725 unsigned int nid = 0; 726 struct memblock_region *reg; 727 728 printk(KERN_DEBUG "Top of RAM: 0x%lx, Total RAM: 0x%lx\n", 729 top_of_ram, total_ram); 730 printk(KERN_DEBUG "Memory hole size: %ldMB\n", 731 (top_of_ram - total_ram) >> 20); 732 733 for_each_memblock(memory, reg) { 734 start_pfn = memblock_region_memory_base_pfn(reg); 735 end_pfn = memblock_region_memory_end_pfn(reg); 736 737 fake_numa_create_new_node(end_pfn, &nid); 738 memblock_set_node(PFN_PHYS(start_pfn), 739 PFN_PHYS(end_pfn - start_pfn), 740 &memblock.memory, nid); 741 node_set_online(nid); 742 } 743 } 744 745 void __init dump_numa_cpu_topology(void) 746 { 747 unsigned int node; 748 unsigned int cpu, count; 749 750 if (min_common_depth == -1 || !numa_enabled) 751 return; 752 753 for_each_online_node(node) { 754 pr_info("Node %d CPUs:", node); 755 756 count = 0; 757 /* 758 * If we used a CPU iterator here we would miss printing 759 * the holes in the cpumap. 760 */ 761 for (cpu = 0; cpu < nr_cpu_ids; cpu++) { 762 if (cpumask_test_cpu(cpu, 763 node_to_cpumask_map[node])) { 764 if (count == 0) 765 pr_cont(" %u", cpu); 766 ++count; 767 } else { 768 if (count > 1) 769 pr_cont("-%u", cpu - 1); 770 count = 0; 771 } 772 } 773 774 if (count > 1) 775 pr_cont("-%u", nr_cpu_ids - 1); 776 pr_cont("\n"); 777 } 778 } 779 780 /* Initialize NODE_DATA for a node on the local memory */ 781 static void __init setup_node_data(int nid, u64 start_pfn, u64 end_pfn) 782 { 783 u64 spanned_pages = end_pfn - start_pfn; 784 const size_t nd_size = roundup(sizeof(pg_data_t), SMP_CACHE_BYTES); 785 u64 nd_pa; 786 void *nd; 787 int tnid; 788 789 nd_pa = memblock_phys_alloc_try_nid(nd_size, SMP_CACHE_BYTES, nid); 790 if (!nd_pa) 791 panic("Cannot allocate %zu bytes for node %d data\n", 792 nd_size, nid); 793 794 nd = __va(nd_pa); 795 796 /* report and initialize */ 797 pr_info(" NODE_DATA [mem %#010Lx-%#010Lx]\n", 798 nd_pa, nd_pa + nd_size - 1); 799 tnid = early_pfn_to_nid(nd_pa >> PAGE_SHIFT); 800 if (tnid != nid) 801 pr_info(" NODE_DATA(%d) on node %d\n", nid, tnid); 802 803 node_data[nid] = nd; 804 memset(NODE_DATA(nid), 0, sizeof(pg_data_t)); 805 NODE_DATA(nid)->node_id = nid; 806 NODE_DATA(nid)->node_start_pfn = start_pfn; 807 NODE_DATA(nid)->node_spanned_pages = spanned_pages; 808 } 809 810 static void __init find_possible_nodes(void) 811 { 812 struct device_node *rtas; 813 u32 numnodes, i; 814 815 if (min_common_depth <= 0) 816 return; 817 818 rtas = of_find_node_by_path("/rtas"); 819 if (!rtas) 820 return; 821 822 if (of_property_read_u32_index(rtas, 823 "ibm,max-associativity-domains", 824 min_common_depth, &numnodes)) 825 goto out; 826 827 for (i = 0; i < numnodes; i++) { 828 if (!node_possible(i)) 829 node_set(i, node_possible_map); 830 } 831 832 out: 833 of_node_put(rtas); 834 } 835 836 void __init mem_topology_setup(void) 837 { 838 int cpu; 839 840 if (parse_numa_properties()) 841 setup_nonnuma(); 842 843 /* 844 * Modify the set of possible NUMA nodes to reflect information 845 * available about the set of online nodes, and the set of nodes 846 * that we expect to make use of for this platform's affinity 847 * calculations. 848 */ 849 nodes_and(node_possible_map, node_possible_map, node_online_map); 850 851 find_possible_nodes(); 852 853 setup_node_to_cpumask_map(); 854 855 reset_numa_cpu_lookup_table(); 856 857 for_each_present_cpu(cpu) 858 numa_setup_cpu(cpu); 859 } 860 861 void __init initmem_init(void) 862 { 863 int nid; 864 865 max_low_pfn = memblock_end_of_DRAM() >> PAGE_SHIFT; 866 max_pfn = max_low_pfn; 867 868 memblock_dump_all(); 869 870 for_each_online_node(nid) { 871 unsigned long start_pfn, end_pfn; 872 873 get_pfn_range_for_nid(nid, &start_pfn, &end_pfn); 874 setup_node_data(nid, start_pfn, end_pfn); 875 sparse_memory_present_with_active_regions(nid); 876 } 877 878 sparse_init(); 879 880 /* 881 * We need the numa_cpu_lookup_table to be accurate for all CPUs, 882 * even before we online them, so that we can use cpu_to_{node,mem} 883 * early in boot, cf. smp_prepare_cpus(). 884 * _nocalls() + manual invocation is used because cpuhp is not yet 885 * initialized for the boot CPU. 886 */ 887 cpuhp_setup_state_nocalls(CPUHP_POWER_NUMA_PREPARE, "powerpc/numa:prepare", 888 ppc_numa_cpu_prepare, ppc_numa_cpu_dead); 889 } 890 891 static int __init early_numa(char *p) 892 { 893 if (!p) 894 return 0; 895 896 if (strstr(p, "off")) 897 numa_enabled = 0; 898 899 if (strstr(p, "debug")) 900 numa_debug = 1; 901 902 p = strstr(p, "fake="); 903 if (p) 904 cmdline = p + strlen("fake="); 905 906 return 0; 907 } 908 early_param("numa", early_numa); 909 910 /* 911 * The platform can inform us through one of several mechanisms 912 * (post-migration device tree updates, PRRN or VPHN) that the NUMA 913 * assignment of a resource has changed. This controls whether we act 914 * on that. Disabled by default. 915 */ 916 static bool topology_updates_enabled; 917 918 static int __init early_topology_updates(char *p) 919 { 920 if (!p) 921 return 0; 922 923 if (!strcmp(p, "on")) { 924 pr_warn("Caution: enabling topology updates\n"); 925 topology_updates_enabled = true; 926 } 927 928 return 0; 929 } 930 early_param("topology_updates", early_topology_updates); 931 932 #ifdef CONFIG_MEMORY_HOTPLUG 933 /* 934 * Find the node associated with a hot added memory section for 935 * memory represented in the device tree by the property 936 * ibm,dynamic-reconfiguration-memory/ibm,dynamic-memory. 937 */ 938 static int hot_add_drconf_scn_to_nid(unsigned long scn_addr) 939 { 940 struct drmem_lmb *lmb; 941 unsigned long lmb_size; 942 int nid = NUMA_NO_NODE; 943 944 lmb_size = drmem_lmb_size(); 945 946 for_each_drmem_lmb(lmb) { 947 /* skip this block if it is reserved or not assigned to 948 * this partition */ 949 if ((lmb->flags & DRCONF_MEM_RESERVED) 950 || !(lmb->flags & DRCONF_MEM_ASSIGNED)) 951 continue; 952 953 if ((scn_addr < lmb->base_addr) 954 || (scn_addr >= (lmb->base_addr + lmb_size))) 955 continue; 956 957 nid = of_drconf_to_nid_single(lmb); 958 break; 959 } 960 961 return nid; 962 } 963 964 /* 965 * Find the node associated with a hot added memory section for memory 966 * represented in the device tree as a node (i.e. memory@XXXX) for 967 * each memblock. 968 */ 969 static int hot_add_node_scn_to_nid(unsigned long scn_addr) 970 { 971 struct device_node *memory; 972 int nid = NUMA_NO_NODE; 973 974 for_each_node_by_type(memory, "memory") { 975 unsigned long start, size; 976 int ranges; 977 const __be32 *memcell_buf; 978 unsigned int len; 979 980 memcell_buf = of_get_property(memory, "reg", &len); 981 if (!memcell_buf || len <= 0) 982 continue; 983 984 /* ranges in cell */ 985 ranges = (len >> 2) / (n_mem_addr_cells + n_mem_size_cells); 986 987 while (ranges--) { 988 start = read_n_cells(n_mem_addr_cells, &memcell_buf); 989 size = read_n_cells(n_mem_size_cells, &memcell_buf); 990 991 if ((scn_addr < start) || (scn_addr >= (start + size))) 992 continue; 993 994 nid = of_node_to_nid_single(memory); 995 break; 996 } 997 998 if (nid >= 0) 999 break; 1000 } 1001 1002 of_node_put(memory); 1003 1004 return nid; 1005 } 1006 1007 /* 1008 * Find the node associated with a hot added memory section. Section 1009 * corresponds to a SPARSEMEM section, not an MEMBLOCK. It is assumed that 1010 * sections are fully contained within a single MEMBLOCK. 1011 */ 1012 int hot_add_scn_to_nid(unsigned long scn_addr) 1013 { 1014 struct device_node *memory = NULL; 1015 int nid; 1016 1017 if (!numa_enabled || (min_common_depth < 0)) 1018 return first_online_node; 1019 1020 memory = of_find_node_by_path("/ibm,dynamic-reconfiguration-memory"); 1021 if (memory) { 1022 nid = hot_add_drconf_scn_to_nid(scn_addr); 1023 of_node_put(memory); 1024 } else { 1025 nid = hot_add_node_scn_to_nid(scn_addr); 1026 } 1027 1028 if (nid < 0 || !node_possible(nid)) 1029 nid = first_online_node; 1030 1031 return nid; 1032 } 1033 1034 static u64 hot_add_drconf_memory_max(void) 1035 { 1036 struct device_node *memory = NULL; 1037 struct device_node *dn = NULL; 1038 const __be64 *lrdr = NULL; 1039 1040 dn = of_find_node_by_path("/rtas"); 1041 if (dn) { 1042 lrdr = of_get_property(dn, "ibm,lrdr-capacity", NULL); 1043 of_node_put(dn); 1044 if (lrdr) 1045 return be64_to_cpup(lrdr); 1046 } 1047 1048 memory = of_find_node_by_path("/ibm,dynamic-reconfiguration-memory"); 1049 if (memory) { 1050 of_node_put(memory); 1051 return drmem_lmb_memory_max(); 1052 } 1053 return 0; 1054 } 1055 1056 /* 1057 * memory_hotplug_max - return max address of memory that may be added 1058 * 1059 * This is currently only used on systems that support drconfig memory 1060 * hotplug. 1061 */ 1062 u64 memory_hotplug_max(void) 1063 { 1064 return max(hot_add_drconf_memory_max(), memblock_end_of_DRAM()); 1065 } 1066 #endif /* CONFIG_MEMORY_HOTPLUG */ 1067 1068 /* Virtual Processor Home Node (VPHN) support */ 1069 #ifdef CONFIG_PPC_SPLPAR 1070 struct topology_update_data { 1071 struct topology_update_data *next; 1072 unsigned int cpu; 1073 int old_nid; 1074 int new_nid; 1075 }; 1076 1077 #define TOPOLOGY_DEF_TIMER_SECS 60 1078 1079 static u8 vphn_cpu_change_counts[NR_CPUS][MAX_DISTANCE_REF_POINTS]; 1080 static cpumask_t cpu_associativity_changes_mask; 1081 static int vphn_enabled; 1082 static int prrn_enabled; 1083 static void reset_topology_timer(void); 1084 static int topology_timer_secs = 1; 1085 static int topology_inited; 1086 1087 /* 1088 * Change polling interval for associativity changes. 1089 */ 1090 int timed_topology_update(int nsecs) 1091 { 1092 if (vphn_enabled) { 1093 if (nsecs > 0) 1094 topology_timer_secs = nsecs; 1095 else 1096 topology_timer_secs = TOPOLOGY_DEF_TIMER_SECS; 1097 1098 reset_topology_timer(); 1099 } 1100 1101 return 0; 1102 } 1103 1104 /* 1105 * Store the current values of the associativity change counters in the 1106 * hypervisor. 1107 */ 1108 static void setup_cpu_associativity_change_counters(void) 1109 { 1110 int cpu; 1111 1112 /* The VPHN feature supports a maximum of 8 reference points */ 1113 BUILD_BUG_ON(MAX_DISTANCE_REF_POINTS > 8); 1114 1115 for_each_possible_cpu(cpu) { 1116 int i; 1117 u8 *counts = vphn_cpu_change_counts[cpu]; 1118 volatile u8 *hypervisor_counts = lppaca_of(cpu).vphn_assoc_counts; 1119 1120 for (i = 0; i < distance_ref_points_depth; i++) 1121 counts[i] = hypervisor_counts[i]; 1122 } 1123 } 1124 1125 /* 1126 * The hypervisor maintains a set of 8 associativity change counters in 1127 * the VPA of each cpu that correspond to the associativity levels in the 1128 * ibm,associativity-reference-points property. When an associativity 1129 * level changes, the corresponding counter is incremented. 1130 * 1131 * Set a bit in cpu_associativity_changes_mask for each cpu whose home 1132 * node associativity levels have changed. 1133 * 1134 * Returns the number of cpus with unhandled associativity changes. 1135 */ 1136 static int update_cpu_associativity_changes_mask(void) 1137 { 1138 int cpu; 1139 cpumask_t *changes = &cpu_associativity_changes_mask; 1140 1141 for_each_possible_cpu(cpu) { 1142 int i, changed = 0; 1143 u8 *counts = vphn_cpu_change_counts[cpu]; 1144 volatile u8 *hypervisor_counts = lppaca_of(cpu).vphn_assoc_counts; 1145 1146 for (i = 0; i < distance_ref_points_depth; i++) { 1147 if (hypervisor_counts[i] != counts[i]) { 1148 counts[i] = hypervisor_counts[i]; 1149 changed = 1; 1150 } 1151 } 1152 if (changed) { 1153 cpumask_or(changes, changes, cpu_sibling_mask(cpu)); 1154 cpu = cpu_last_thread_sibling(cpu); 1155 } 1156 } 1157 1158 return cpumask_weight(changes); 1159 } 1160 1161 /* 1162 * Retrieve the new associativity information for a virtual processor's 1163 * home node. 1164 */ 1165 static long vphn_get_associativity(unsigned long cpu, 1166 __be32 *associativity) 1167 { 1168 long rc; 1169 1170 rc = hcall_vphn(get_hard_smp_processor_id(cpu), 1171 VPHN_FLAG_VCPU, associativity); 1172 1173 switch (rc) { 1174 case H_FUNCTION: 1175 printk_once(KERN_INFO 1176 "VPHN is not supported. Disabling polling...\n"); 1177 stop_topology_update(); 1178 break; 1179 case H_HARDWARE: 1180 printk(KERN_ERR 1181 "hcall_vphn() experienced a hardware fault " 1182 "preventing VPHN. Disabling polling...\n"); 1183 stop_topology_update(); 1184 break; 1185 case H_SUCCESS: 1186 dbg("VPHN hcall succeeded. Reset polling...\n"); 1187 timed_topology_update(0); 1188 break; 1189 } 1190 1191 return rc; 1192 } 1193 1194 int find_and_online_cpu_nid(int cpu) 1195 { 1196 __be32 associativity[VPHN_ASSOC_BUFSIZE] = {0}; 1197 int new_nid; 1198 1199 /* Use associativity from first thread for all siblings */ 1200 if (vphn_get_associativity(cpu, associativity)) 1201 return cpu_to_node(cpu); 1202 1203 new_nid = associativity_to_nid(associativity); 1204 if (new_nid < 0 || !node_possible(new_nid)) 1205 new_nid = first_online_node; 1206 1207 if (NODE_DATA(new_nid) == NULL) { 1208 #ifdef CONFIG_MEMORY_HOTPLUG 1209 /* 1210 * Need to ensure that NODE_DATA is initialized for a node from 1211 * available memory (see memblock_alloc_try_nid). If unable to 1212 * init the node, then default to nearest node that has memory 1213 * installed. Skip onlining a node if the subsystems are not 1214 * yet initialized. 1215 */ 1216 if (!topology_inited || try_online_node(new_nid)) 1217 new_nid = first_online_node; 1218 #else 1219 /* 1220 * Default to using the nearest node that has memory installed. 1221 * Otherwise, it would be necessary to patch the kernel MM code 1222 * to deal with more memoryless-node error conditions. 1223 */ 1224 new_nid = first_online_node; 1225 #endif 1226 } 1227 1228 pr_debug("%s:%d cpu %d nid %d\n", __FUNCTION__, __LINE__, 1229 cpu, new_nid); 1230 return new_nid; 1231 } 1232 1233 /* 1234 * Update the CPU maps and sysfs entries for a single CPU when its NUMA 1235 * characteristics change. This function doesn't perform any locking and is 1236 * only safe to call from stop_machine(). 1237 */ 1238 static int update_cpu_topology(void *data) 1239 { 1240 struct topology_update_data *update; 1241 unsigned long cpu; 1242 1243 if (!data) 1244 return -EINVAL; 1245 1246 cpu = smp_processor_id(); 1247 1248 for (update = data; update; update = update->next) { 1249 int new_nid = update->new_nid; 1250 if (cpu != update->cpu) 1251 continue; 1252 1253 unmap_cpu_from_node(cpu); 1254 map_cpu_to_node(cpu, new_nid); 1255 set_cpu_numa_node(cpu, new_nid); 1256 set_cpu_numa_mem(cpu, local_memory_node(new_nid)); 1257 vdso_getcpu_init(); 1258 } 1259 1260 return 0; 1261 } 1262 1263 static int update_lookup_table(void *data) 1264 { 1265 struct topology_update_data *update; 1266 1267 if (!data) 1268 return -EINVAL; 1269 1270 /* 1271 * Upon topology update, the numa-cpu lookup table needs to be updated 1272 * for all threads in the core, including offline CPUs, to ensure that 1273 * future hotplug operations respect the cpu-to-node associativity 1274 * properly. 1275 */ 1276 for (update = data; update; update = update->next) { 1277 int nid, base, j; 1278 1279 nid = update->new_nid; 1280 base = cpu_first_thread_sibling(update->cpu); 1281 1282 for (j = 0; j < threads_per_core; j++) { 1283 update_numa_cpu_lookup_table(base + j, nid); 1284 } 1285 } 1286 1287 return 0; 1288 } 1289 1290 /* 1291 * Update the node maps and sysfs entries for each cpu whose home node 1292 * has changed. Returns 1 when the topology has changed, and 0 otherwise. 1293 * 1294 * cpus_locked says whether we already hold cpu_hotplug_lock. 1295 */ 1296 int numa_update_cpu_topology(bool cpus_locked) 1297 { 1298 unsigned int cpu, sibling, changed = 0; 1299 struct topology_update_data *updates, *ud; 1300 cpumask_t updated_cpus; 1301 struct device *dev; 1302 int weight, new_nid, i = 0; 1303 1304 if (!prrn_enabled && !vphn_enabled && topology_inited) 1305 return 0; 1306 1307 weight = cpumask_weight(&cpu_associativity_changes_mask); 1308 if (!weight) 1309 return 0; 1310 1311 updates = kcalloc(weight, sizeof(*updates), GFP_KERNEL); 1312 if (!updates) 1313 return 0; 1314 1315 cpumask_clear(&updated_cpus); 1316 1317 for_each_cpu(cpu, &cpu_associativity_changes_mask) { 1318 /* 1319 * If siblings aren't flagged for changes, updates list 1320 * will be too short. Skip on this update and set for next 1321 * update. 1322 */ 1323 if (!cpumask_subset(cpu_sibling_mask(cpu), 1324 &cpu_associativity_changes_mask)) { 1325 pr_info("Sibling bits not set for associativity " 1326 "change, cpu%d\n", cpu); 1327 cpumask_or(&cpu_associativity_changes_mask, 1328 &cpu_associativity_changes_mask, 1329 cpu_sibling_mask(cpu)); 1330 cpu = cpu_last_thread_sibling(cpu); 1331 continue; 1332 } 1333 1334 new_nid = find_and_online_cpu_nid(cpu); 1335 1336 if (new_nid == numa_cpu_lookup_table[cpu]) { 1337 cpumask_andnot(&cpu_associativity_changes_mask, 1338 &cpu_associativity_changes_mask, 1339 cpu_sibling_mask(cpu)); 1340 dbg("Assoc chg gives same node %d for cpu%d\n", 1341 new_nid, cpu); 1342 cpu = cpu_last_thread_sibling(cpu); 1343 continue; 1344 } 1345 1346 for_each_cpu(sibling, cpu_sibling_mask(cpu)) { 1347 ud = &updates[i++]; 1348 ud->next = &updates[i]; 1349 ud->cpu = sibling; 1350 ud->new_nid = new_nid; 1351 ud->old_nid = numa_cpu_lookup_table[sibling]; 1352 cpumask_set_cpu(sibling, &updated_cpus); 1353 } 1354 cpu = cpu_last_thread_sibling(cpu); 1355 } 1356 1357 /* 1358 * Prevent processing of 'updates' from overflowing array 1359 * where last entry filled in a 'next' pointer. 1360 */ 1361 if (i) 1362 updates[i-1].next = NULL; 1363 1364 pr_debug("Topology update for the following CPUs:\n"); 1365 if (cpumask_weight(&updated_cpus)) { 1366 for (ud = &updates[0]; ud; ud = ud->next) { 1367 pr_debug("cpu %d moving from node %d " 1368 "to %d\n", ud->cpu, 1369 ud->old_nid, ud->new_nid); 1370 } 1371 } 1372 1373 /* 1374 * In cases where we have nothing to update (because the updates list 1375 * is too short or because the new topology is same as the old one), 1376 * skip invoking update_cpu_topology() via stop-machine(). This is 1377 * necessary (and not just a fast-path optimization) since stop-machine 1378 * can end up electing a random CPU to run update_cpu_topology(), and 1379 * thus trick us into setting up incorrect cpu-node mappings (since 1380 * 'updates' is kzalloc()'ed). 1381 * 1382 * And for the similar reason, we will skip all the following updating. 1383 */ 1384 if (!cpumask_weight(&updated_cpus)) 1385 goto out; 1386 1387 if (cpus_locked) 1388 stop_machine_cpuslocked(update_cpu_topology, &updates[0], 1389 &updated_cpus); 1390 else 1391 stop_machine(update_cpu_topology, &updates[0], &updated_cpus); 1392 1393 /* 1394 * Update the numa-cpu lookup table with the new mappings, even for 1395 * offline CPUs. It is best to perform this update from the stop- 1396 * machine context. 1397 */ 1398 if (cpus_locked) 1399 stop_machine_cpuslocked(update_lookup_table, &updates[0], 1400 cpumask_of(raw_smp_processor_id())); 1401 else 1402 stop_machine(update_lookup_table, &updates[0], 1403 cpumask_of(raw_smp_processor_id())); 1404 1405 for (ud = &updates[0]; ud; ud = ud->next) { 1406 unregister_cpu_under_node(ud->cpu, ud->old_nid); 1407 register_cpu_under_node(ud->cpu, ud->new_nid); 1408 1409 dev = get_cpu_device(ud->cpu); 1410 if (dev) 1411 kobject_uevent(&dev->kobj, KOBJ_CHANGE); 1412 cpumask_clear_cpu(ud->cpu, &cpu_associativity_changes_mask); 1413 changed = 1; 1414 } 1415 1416 out: 1417 kfree(updates); 1418 return changed; 1419 } 1420 1421 int arch_update_cpu_topology(void) 1422 { 1423 return numa_update_cpu_topology(true); 1424 } 1425 1426 static void topology_work_fn(struct work_struct *work) 1427 { 1428 rebuild_sched_domains(); 1429 } 1430 static DECLARE_WORK(topology_work, topology_work_fn); 1431 1432 static void topology_schedule_update(void) 1433 { 1434 schedule_work(&topology_work); 1435 } 1436 1437 static void topology_timer_fn(struct timer_list *unused) 1438 { 1439 if (prrn_enabled && cpumask_weight(&cpu_associativity_changes_mask)) 1440 topology_schedule_update(); 1441 else if (vphn_enabled) { 1442 if (update_cpu_associativity_changes_mask() > 0) 1443 topology_schedule_update(); 1444 reset_topology_timer(); 1445 } 1446 } 1447 static struct timer_list topology_timer; 1448 1449 static void reset_topology_timer(void) 1450 { 1451 if (vphn_enabled) 1452 mod_timer(&topology_timer, jiffies + topology_timer_secs * HZ); 1453 } 1454 1455 #ifdef CONFIG_SMP 1456 1457 static int dt_update_callback(struct notifier_block *nb, 1458 unsigned long action, void *data) 1459 { 1460 struct of_reconfig_data *update = data; 1461 int rc = NOTIFY_DONE; 1462 1463 switch (action) { 1464 case OF_RECONFIG_UPDATE_PROPERTY: 1465 if (of_node_is_type(update->dn, "cpu") && 1466 !of_prop_cmp(update->prop->name, "ibm,associativity")) { 1467 u32 core_id; 1468 of_property_read_u32(update->dn, "reg", &core_id); 1469 rc = dlpar_cpu_readd(core_id); 1470 rc = NOTIFY_OK; 1471 } 1472 break; 1473 } 1474 1475 return rc; 1476 } 1477 1478 static struct notifier_block dt_update_nb = { 1479 .notifier_call = dt_update_callback, 1480 }; 1481 1482 #endif 1483 1484 /* 1485 * Start polling for associativity changes. 1486 */ 1487 int start_topology_update(void) 1488 { 1489 int rc = 0; 1490 1491 if (!topology_updates_enabled) 1492 return 0; 1493 1494 if (firmware_has_feature(FW_FEATURE_PRRN)) { 1495 if (!prrn_enabled) { 1496 prrn_enabled = 1; 1497 #ifdef CONFIG_SMP 1498 rc = of_reconfig_notifier_register(&dt_update_nb); 1499 #endif 1500 } 1501 } 1502 if (firmware_has_feature(FW_FEATURE_VPHN) && 1503 lppaca_shared_proc(get_lppaca())) { 1504 if (!vphn_enabled) { 1505 vphn_enabled = 1; 1506 setup_cpu_associativity_change_counters(); 1507 timer_setup(&topology_timer, topology_timer_fn, 1508 TIMER_DEFERRABLE); 1509 reset_topology_timer(); 1510 } 1511 } 1512 1513 pr_info("Starting topology update%s%s\n", 1514 (prrn_enabled ? " prrn_enabled" : ""), 1515 (vphn_enabled ? " vphn_enabled" : "")); 1516 1517 return rc; 1518 } 1519 1520 /* 1521 * Disable polling for VPHN associativity changes. 1522 */ 1523 int stop_topology_update(void) 1524 { 1525 int rc = 0; 1526 1527 if (!topology_updates_enabled) 1528 return 0; 1529 1530 if (prrn_enabled) { 1531 prrn_enabled = 0; 1532 #ifdef CONFIG_SMP 1533 rc = of_reconfig_notifier_unregister(&dt_update_nb); 1534 #endif 1535 } 1536 if (vphn_enabled) { 1537 vphn_enabled = 0; 1538 rc = del_timer_sync(&topology_timer); 1539 } 1540 1541 pr_info("Stopping topology update\n"); 1542 1543 return rc; 1544 } 1545 1546 int prrn_is_enabled(void) 1547 { 1548 return prrn_enabled; 1549 } 1550 1551 void __init shared_proc_topology_init(void) 1552 { 1553 if (lppaca_shared_proc(get_lppaca())) { 1554 bitmap_fill(cpumask_bits(&cpu_associativity_changes_mask), 1555 nr_cpumask_bits); 1556 numa_update_cpu_topology(false); 1557 } 1558 } 1559 1560 static int topology_read(struct seq_file *file, void *v) 1561 { 1562 if (vphn_enabled || prrn_enabled) 1563 seq_puts(file, "on\n"); 1564 else 1565 seq_puts(file, "off\n"); 1566 1567 return 0; 1568 } 1569 1570 static int topology_open(struct inode *inode, struct file *file) 1571 { 1572 return single_open(file, topology_read, NULL); 1573 } 1574 1575 static ssize_t topology_write(struct file *file, const char __user *buf, 1576 size_t count, loff_t *off) 1577 { 1578 char kbuf[4]; /* "on" or "off" plus null. */ 1579 int read_len; 1580 1581 read_len = count < 3 ? count : 3; 1582 if (copy_from_user(kbuf, buf, read_len)) 1583 return -EINVAL; 1584 1585 kbuf[read_len] = '\0'; 1586 1587 if (!strncmp(kbuf, "on", 2)) { 1588 topology_updates_enabled = true; 1589 start_topology_update(); 1590 } else if (!strncmp(kbuf, "off", 3)) { 1591 stop_topology_update(); 1592 topology_updates_enabled = false; 1593 } else 1594 return -EINVAL; 1595 1596 return count; 1597 } 1598 1599 static const struct file_operations topology_ops = { 1600 .read = seq_read, 1601 .write = topology_write, 1602 .open = topology_open, 1603 .release = single_release 1604 }; 1605 1606 static int topology_update_init(void) 1607 { 1608 start_topology_update(); 1609 1610 if (vphn_enabled) 1611 topology_schedule_update(); 1612 1613 if (!proc_create("powerpc/topology_updates", 0644, NULL, &topology_ops)) 1614 return -ENOMEM; 1615 1616 topology_inited = 1; 1617 return 0; 1618 } 1619 device_initcall(topology_update_init); 1620 #endif /* CONFIG_PPC_SPLPAR */ 1621