xref: /openbmc/linux/arch/powerpc/mm/numa.c (revision 4a075bd4)
1 /*
2  * pSeries NUMA support
3  *
4  * Copyright (C) 2002 Anton Blanchard <anton@au.ibm.com>, IBM
5  *
6  * This program is free software; you can redistribute it and/or
7  * modify it under the terms of the GNU General Public License
8  * as published by the Free Software Foundation; either version
9  * 2 of the License, or (at your option) any later version.
10  */
11 #define pr_fmt(fmt) "numa: " fmt
12 
13 #include <linux/threads.h>
14 #include <linux/memblock.h>
15 #include <linux/init.h>
16 #include <linux/mm.h>
17 #include <linux/mmzone.h>
18 #include <linux/export.h>
19 #include <linux/nodemask.h>
20 #include <linux/cpu.h>
21 #include <linux/notifier.h>
22 #include <linux/of.h>
23 #include <linux/pfn.h>
24 #include <linux/cpuset.h>
25 #include <linux/node.h>
26 #include <linux/stop_machine.h>
27 #include <linux/proc_fs.h>
28 #include <linux/seq_file.h>
29 #include <linux/uaccess.h>
30 #include <linux/slab.h>
31 #include <asm/cputhreads.h>
32 #include <asm/sparsemem.h>
33 #include <asm/prom.h>
34 #include <asm/smp.h>
35 #include <asm/topology.h>
36 #include <asm/firmware.h>
37 #include <asm/paca.h>
38 #include <asm/hvcall.h>
39 #include <asm/setup.h>
40 #include <asm/vdso.h>
41 #include <asm/drmem.h>
42 
43 static int numa_enabled = 1;
44 
45 static char *cmdline __initdata;
46 
47 static int numa_debug;
48 #define dbg(args...) if (numa_debug) { printk(KERN_INFO args); }
49 
50 int numa_cpu_lookup_table[NR_CPUS];
51 cpumask_var_t node_to_cpumask_map[MAX_NUMNODES];
52 struct pglist_data *node_data[MAX_NUMNODES];
53 
54 EXPORT_SYMBOL(numa_cpu_lookup_table);
55 EXPORT_SYMBOL(node_to_cpumask_map);
56 EXPORT_SYMBOL(node_data);
57 
58 static int min_common_depth;
59 static int n_mem_addr_cells, n_mem_size_cells;
60 static int form1_affinity;
61 
62 #define MAX_DISTANCE_REF_POINTS 4
63 static int distance_ref_points_depth;
64 static const __be32 *distance_ref_points;
65 static int distance_lookup_table[MAX_NUMNODES][MAX_DISTANCE_REF_POINTS];
66 
67 /*
68  * Allocate node_to_cpumask_map based on number of available nodes
69  * Requires node_possible_map to be valid.
70  *
71  * Note: cpumask_of_node() is not valid until after this is done.
72  */
73 static void __init setup_node_to_cpumask_map(void)
74 {
75 	unsigned int node;
76 
77 	/* setup nr_node_ids if not done yet */
78 	if (nr_node_ids == MAX_NUMNODES)
79 		setup_nr_node_ids();
80 
81 	/* allocate the map */
82 	for_each_node(node)
83 		alloc_bootmem_cpumask_var(&node_to_cpumask_map[node]);
84 
85 	/* cpumask_of_node() will now work */
86 	dbg("Node to cpumask map for %u nodes\n", nr_node_ids);
87 }
88 
89 static int __init fake_numa_create_new_node(unsigned long end_pfn,
90 						unsigned int *nid)
91 {
92 	unsigned long long mem;
93 	char *p = cmdline;
94 	static unsigned int fake_nid;
95 	static unsigned long long curr_boundary;
96 
97 	/*
98 	 * Modify node id, iff we started creating NUMA nodes
99 	 * We want to continue from where we left of the last time
100 	 */
101 	if (fake_nid)
102 		*nid = fake_nid;
103 	/*
104 	 * In case there are no more arguments to parse, the
105 	 * node_id should be the same as the last fake node id
106 	 * (we've handled this above).
107 	 */
108 	if (!p)
109 		return 0;
110 
111 	mem = memparse(p, &p);
112 	if (!mem)
113 		return 0;
114 
115 	if (mem < curr_boundary)
116 		return 0;
117 
118 	curr_boundary = mem;
119 
120 	if ((end_pfn << PAGE_SHIFT) > mem) {
121 		/*
122 		 * Skip commas and spaces
123 		 */
124 		while (*p == ',' || *p == ' ' || *p == '\t')
125 			p++;
126 
127 		cmdline = p;
128 		fake_nid++;
129 		*nid = fake_nid;
130 		dbg("created new fake_node with id %d\n", fake_nid);
131 		return 1;
132 	}
133 	return 0;
134 }
135 
136 static void reset_numa_cpu_lookup_table(void)
137 {
138 	unsigned int cpu;
139 
140 	for_each_possible_cpu(cpu)
141 		numa_cpu_lookup_table[cpu] = -1;
142 }
143 
144 static void map_cpu_to_node(int cpu, int node)
145 {
146 	update_numa_cpu_lookup_table(cpu, node);
147 
148 	dbg("adding cpu %d to node %d\n", cpu, node);
149 
150 	if (!(cpumask_test_cpu(cpu, node_to_cpumask_map[node])))
151 		cpumask_set_cpu(cpu, node_to_cpumask_map[node]);
152 }
153 
154 #if defined(CONFIG_HOTPLUG_CPU) || defined(CONFIG_PPC_SPLPAR)
155 static void unmap_cpu_from_node(unsigned long cpu)
156 {
157 	int node = numa_cpu_lookup_table[cpu];
158 
159 	dbg("removing cpu %lu from node %d\n", cpu, node);
160 
161 	if (cpumask_test_cpu(cpu, node_to_cpumask_map[node])) {
162 		cpumask_clear_cpu(cpu, node_to_cpumask_map[node]);
163 	} else {
164 		printk(KERN_ERR "WARNING: cpu %lu not found in node %d\n",
165 		       cpu, node);
166 	}
167 }
168 #endif /* CONFIG_HOTPLUG_CPU || CONFIG_PPC_SPLPAR */
169 
170 /* must hold reference to node during call */
171 static const __be32 *of_get_associativity(struct device_node *dev)
172 {
173 	return of_get_property(dev, "ibm,associativity", NULL);
174 }
175 
176 int __node_distance(int a, int b)
177 {
178 	int i;
179 	int distance = LOCAL_DISTANCE;
180 
181 	if (!form1_affinity)
182 		return ((a == b) ? LOCAL_DISTANCE : REMOTE_DISTANCE);
183 
184 	for (i = 0; i < distance_ref_points_depth; i++) {
185 		if (distance_lookup_table[a][i] == distance_lookup_table[b][i])
186 			break;
187 
188 		/* Double the distance for each NUMA level */
189 		distance *= 2;
190 	}
191 
192 	return distance;
193 }
194 EXPORT_SYMBOL(__node_distance);
195 
196 static void initialize_distance_lookup_table(int nid,
197 		const __be32 *associativity)
198 {
199 	int i;
200 
201 	if (!form1_affinity)
202 		return;
203 
204 	for (i = 0; i < distance_ref_points_depth; i++) {
205 		const __be32 *entry;
206 
207 		entry = &associativity[be32_to_cpu(distance_ref_points[i]) - 1];
208 		distance_lookup_table[nid][i] = of_read_number(entry, 1);
209 	}
210 }
211 
212 /* Returns nid in the range [0..MAX_NUMNODES-1], or -1 if no useful numa
213  * info is found.
214  */
215 static int associativity_to_nid(const __be32 *associativity)
216 {
217 	int nid = NUMA_NO_NODE;
218 
219 	if (min_common_depth == -1)
220 		goto out;
221 
222 	if (of_read_number(associativity, 1) >= min_common_depth)
223 		nid = of_read_number(&associativity[min_common_depth], 1);
224 
225 	/* POWER4 LPAR uses 0xffff as invalid node */
226 	if (nid == 0xffff || nid >= MAX_NUMNODES)
227 		nid = NUMA_NO_NODE;
228 
229 	if (nid > 0 &&
230 		of_read_number(associativity, 1) >= distance_ref_points_depth) {
231 		/*
232 		 * Skip the length field and send start of associativity array
233 		 */
234 		initialize_distance_lookup_table(nid, associativity + 1);
235 	}
236 
237 out:
238 	return nid;
239 }
240 
241 /* Returns the nid associated with the given device tree node,
242  * or -1 if not found.
243  */
244 static int of_node_to_nid_single(struct device_node *device)
245 {
246 	int nid = NUMA_NO_NODE;
247 	const __be32 *tmp;
248 
249 	tmp = of_get_associativity(device);
250 	if (tmp)
251 		nid = associativity_to_nid(tmp);
252 	return nid;
253 }
254 
255 /* Walk the device tree upwards, looking for an associativity id */
256 int of_node_to_nid(struct device_node *device)
257 {
258 	int nid = NUMA_NO_NODE;
259 
260 	of_node_get(device);
261 	while (device) {
262 		nid = of_node_to_nid_single(device);
263 		if (nid != -1)
264 			break;
265 
266 		device = of_get_next_parent(device);
267 	}
268 	of_node_put(device);
269 
270 	return nid;
271 }
272 EXPORT_SYMBOL(of_node_to_nid);
273 
274 static int __init find_min_common_depth(void)
275 {
276 	int depth;
277 	struct device_node *root;
278 
279 	if (firmware_has_feature(FW_FEATURE_OPAL))
280 		root = of_find_node_by_path("/ibm,opal");
281 	else
282 		root = of_find_node_by_path("/rtas");
283 	if (!root)
284 		root = of_find_node_by_path("/");
285 
286 	/*
287 	 * This property is a set of 32-bit integers, each representing
288 	 * an index into the ibm,associativity nodes.
289 	 *
290 	 * With form 0 affinity the first integer is for an SMP configuration
291 	 * (should be all 0's) and the second is for a normal NUMA
292 	 * configuration. We have only one level of NUMA.
293 	 *
294 	 * With form 1 affinity the first integer is the most significant
295 	 * NUMA boundary and the following are progressively less significant
296 	 * boundaries. There can be more than one level of NUMA.
297 	 */
298 	distance_ref_points = of_get_property(root,
299 					"ibm,associativity-reference-points",
300 					&distance_ref_points_depth);
301 
302 	if (!distance_ref_points) {
303 		dbg("NUMA: ibm,associativity-reference-points not found.\n");
304 		goto err;
305 	}
306 
307 	distance_ref_points_depth /= sizeof(int);
308 
309 	if (firmware_has_feature(FW_FEATURE_OPAL) ||
310 	    firmware_has_feature(FW_FEATURE_TYPE1_AFFINITY)) {
311 		dbg("Using form 1 affinity\n");
312 		form1_affinity = 1;
313 	}
314 
315 	if (form1_affinity) {
316 		depth = of_read_number(distance_ref_points, 1);
317 	} else {
318 		if (distance_ref_points_depth < 2) {
319 			printk(KERN_WARNING "NUMA: "
320 				"short ibm,associativity-reference-points\n");
321 			goto err;
322 		}
323 
324 		depth = of_read_number(&distance_ref_points[1], 1);
325 	}
326 
327 	/*
328 	 * Warn and cap if the hardware supports more than
329 	 * MAX_DISTANCE_REF_POINTS domains.
330 	 */
331 	if (distance_ref_points_depth > MAX_DISTANCE_REF_POINTS) {
332 		printk(KERN_WARNING "NUMA: distance array capped at "
333 			"%d entries\n", MAX_DISTANCE_REF_POINTS);
334 		distance_ref_points_depth = MAX_DISTANCE_REF_POINTS;
335 	}
336 
337 	of_node_put(root);
338 	return depth;
339 
340 err:
341 	of_node_put(root);
342 	return -1;
343 }
344 
345 static void __init get_n_mem_cells(int *n_addr_cells, int *n_size_cells)
346 {
347 	struct device_node *memory = NULL;
348 
349 	memory = of_find_node_by_type(memory, "memory");
350 	if (!memory)
351 		panic("numa.c: No memory nodes found!");
352 
353 	*n_addr_cells = of_n_addr_cells(memory);
354 	*n_size_cells = of_n_size_cells(memory);
355 	of_node_put(memory);
356 }
357 
358 static unsigned long read_n_cells(int n, const __be32 **buf)
359 {
360 	unsigned long result = 0;
361 
362 	while (n--) {
363 		result = (result << 32) | of_read_number(*buf, 1);
364 		(*buf)++;
365 	}
366 	return result;
367 }
368 
369 struct assoc_arrays {
370 	u32	n_arrays;
371 	u32	array_sz;
372 	const __be32 *arrays;
373 };
374 
375 /*
376  * Retrieve and validate the list of associativity arrays for drconf
377  * memory from the ibm,associativity-lookup-arrays property of the
378  * device tree..
379  *
380  * The layout of the ibm,associativity-lookup-arrays property is a number N
381  * indicating the number of associativity arrays, followed by a number M
382  * indicating the size of each associativity array, followed by a list
383  * of N associativity arrays.
384  */
385 static int of_get_assoc_arrays(struct assoc_arrays *aa)
386 {
387 	struct device_node *memory;
388 	const __be32 *prop;
389 	u32 len;
390 
391 	memory = of_find_node_by_path("/ibm,dynamic-reconfiguration-memory");
392 	if (!memory)
393 		return -1;
394 
395 	prop = of_get_property(memory, "ibm,associativity-lookup-arrays", &len);
396 	if (!prop || len < 2 * sizeof(unsigned int)) {
397 		of_node_put(memory);
398 		return -1;
399 	}
400 
401 	aa->n_arrays = of_read_number(prop++, 1);
402 	aa->array_sz = of_read_number(prop++, 1);
403 
404 	of_node_put(memory);
405 
406 	/* Now that we know the number of arrays and size of each array,
407 	 * revalidate the size of the property read in.
408 	 */
409 	if (len < (aa->n_arrays * aa->array_sz + 2) * sizeof(unsigned int))
410 		return -1;
411 
412 	aa->arrays = prop;
413 	return 0;
414 }
415 
416 /*
417  * This is like of_node_to_nid_single() for memory represented in the
418  * ibm,dynamic-reconfiguration-memory node.
419  */
420 static int of_drconf_to_nid_single(struct drmem_lmb *lmb)
421 {
422 	struct assoc_arrays aa = { .arrays = NULL };
423 	int default_nid = 0;
424 	int nid = default_nid;
425 	int rc, index;
426 
427 	rc = of_get_assoc_arrays(&aa);
428 	if (rc)
429 		return default_nid;
430 
431 	if (min_common_depth > 0 && min_common_depth <= aa.array_sz &&
432 	    !(lmb->flags & DRCONF_MEM_AI_INVALID) &&
433 	    lmb->aa_index < aa.n_arrays) {
434 		index = lmb->aa_index * aa.array_sz + min_common_depth - 1;
435 		nid = of_read_number(&aa.arrays[index], 1);
436 
437 		if (nid == 0xffff || nid >= MAX_NUMNODES)
438 			nid = default_nid;
439 
440 		if (nid > 0) {
441 			index = lmb->aa_index * aa.array_sz;
442 			initialize_distance_lookup_table(nid,
443 							&aa.arrays[index]);
444 		}
445 	}
446 
447 	return nid;
448 }
449 
450 /*
451  * Figure out to which domain a cpu belongs and stick it there.
452  * Return the id of the domain used.
453  */
454 static int numa_setup_cpu(unsigned long lcpu)
455 {
456 	int nid = NUMA_NO_NODE;
457 	struct device_node *cpu;
458 
459 	/*
460 	 * If a valid cpu-to-node mapping is already available, use it
461 	 * directly instead of querying the firmware, since it represents
462 	 * the most recent mapping notified to us by the platform (eg: VPHN).
463 	 */
464 	if ((nid = numa_cpu_lookup_table[lcpu]) >= 0) {
465 		map_cpu_to_node(lcpu, nid);
466 		return nid;
467 	}
468 
469 	cpu = of_get_cpu_node(lcpu, NULL);
470 
471 	if (!cpu) {
472 		WARN_ON(1);
473 		if (cpu_present(lcpu))
474 			goto out_present;
475 		else
476 			goto out;
477 	}
478 
479 	nid = of_node_to_nid_single(cpu);
480 
481 out_present:
482 	if (nid < 0 || !node_possible(nid))
483 		nid = first_online_node;
484 
485 	map_cpu_to_node(lcpu, nid);
486 	of_node_put(cpu);
487 out:
488 	return nid;
489 }
490 
491 static void verify_cpu_node_mapping(int cpu, int node)
492 {
493 	int base, sibling, i;
494 
495 	/* Verify that all the threads in the core belong to the same node */
496 	base = cpu_first_thread_sibling(cpu);
497 
498 	for (i = 0; i < threads_per_core; i++) {
499 		sibling = base + i;
500 
501 		if (sibling == cpu || cpu_is_offline(sibling))
502 			continue;
503 
504 		if (cpu_to_node(sibling) != node) {
505 			WARN(1, "CPU thread siblings %d and %d don't belong"
506 				" to the same node!\n", cpu, sibling);
507 			break;
508 		}
509 	}
510 }
511 
512 /* Must run before sched domains notifier. */
513 static int ppc_numa_cpu_prepare(unsigned int cpu)
514 {
515 	int nid;
516 
517 	nid = numa_setup_cpu(cpu);
518 	verify_cpu_node_mapping(cpu, nid);
519 	return 0;
520 }
521 
522 static int ppc_numa_cpu_dead(unsigned int cpu)
523 {
524 #ifdef CONFIG_HOTPLUG_CPU
525 	unmap_cpu_from_node(cpu);
526 #endif
527 	return 0;
528 }
529 
530 /*
531  * Check and possibly modify a memory region to enforce the memory limit.
532  *
533  * Returns the size the region should have to enforce the memory limit.
534  * This will either be the original value of size, a truncated value,
535  * or zero. If the returned value of size is 0 the region should be
536  * discarded as it lies wholly above the memory limit.
537  */
538 static unsigned long __init numa_enforce_memory_limit(unsigned long start,
539 						      unsigned long size)
540 {
541 	/*
542 	 * We use memblock_end_of_DRAM() in here instead of memory_limit because
543 	 * we've already adjusted it for the limit and it takes care of
544 	 * having memory holes below the limit.  Also, in the case of
545 	 * iommu_is_off, memory_limit is not set but is implicitly enforced.
546 	 */
547 
548 	if (start + size <= memblock_end_of_DRAM())
549 		return size;
550 
551 	if (start >= memblock_end_of_DRAM())
552 		return 0;
553 
554 	return memblock_end_of_DRAM() - start;
555 }
556 
557 /*
558  * Reads the counter for a given entry in
559  * linux,drconf-usable-memory property
560  */
561 static inline int __init read_usm_ranges(const __be32 **usm)
562 {
563 	/*
564 	 * For each lmb in ibm,dynamic-memory a corresponding
565 	 * entry in linux,drconf-usable-memory property contains
566 	 * a counter followed by that many (base, size) duple.
567 	 * read the counter from linux,drconf-usable-memory
568 	 */
569 	return read_n_cells(n_mem_size_cells, usm);
570 }
571 
572 /*
573  * Extract NUMA information from the ibm,dynamic-reconfiguration-memory
574  * node.  This assumes n_mem_{addr,size}_cells have been set.
575  */
576 static void __init numa_setup_drmem_lmb(struct drmem_lmb *lmb,
577 					const __be32 **usm)
578 {
579 	unsigned int ranges, is_kexec_kdump = 0;
580 	unsigned long base, size, sz;
581 	int nid;
582 
583 	/*
584 	 * Skip this block if the reserved bit is set in flags (0x80)
585 	 * or if the block is not assigned to this partition (0x8)
586 	 */
587 	if ((lmb->flags & DRCONF_MEM_RESERVED)
588 	    || !(lmb->flags & DRCONF_MEM_ASSIGNED))
589 		return;
590 
591 	if (*usm)
592 		is_kexec_kdump = 1;
593 
594 	base = lmb->base_addr;
595 	size = drmem_lmb_size();
596 	ranges = 1;
597 
598 	if (is_kexec_kdump) {
599 		ranges = read_usm_ranges(usm);
600 		if (!ranges) /* there are no (base, size) duple */
601 			return;
602 	}
603 
604 	do {
605 		if (is_kexec_kdump) {
606 			base = read_n_cells(n_mem_addr_cells, usm);
607 			size = read_n_cells(n_mem_size_cells, usm);
608 		}
609 
610 		nid = of_drconf_to_nid_single(lmb);
611 		fake_numa_create_new_node(((base + size) >> PAGE_SHIFT),
612 					  &nid);
613 		node_set_online(nid);
614 		sz = numa_enforce_memory_limit(base, size);
615 		if (sz)
616 			memblock_set_node(base, sz, &memblock.memory, nid);
617 	} while (--ranges);
618 }
619 
620 static int __init parse_numa_properties(void)
621 {
622 	struct device_node *memory;
623 	int default_nid = 0;
624 	unsigned long i;
625 
626 	if (numa_enabled == 0) {
627 		printk(KERN_WARNING "NUMA disabled by user\n");
628 		return -1;
629 	}
630 
631 	min_common_depth = find_min_common_depth();
632 
633 	if (min_common_depth < 0)
634 		return min_common_depth;
635 
636 	dbg("NUMA associativity depth for CPU/Memory: %d\n", min_common_depth);
637 
638 	/*
639 	 * Even though we connect cpus to numa domains later in SMP
640 	 * init, we need to know the node ids now. This is because
641 	 * each node to be onlined must have NODE_DATA etc backing it.
642 	 */
643 	for_each_present_cpu(i) {
644 		struct device_node *cpu;
645 		int nid;
646 
647 		cpu = of_get_cpu_node(i, NULL);
648 		BUG_ON(!cpu);
649 		nid = of_node_to_nid_single(cpu);
650 		of_node_put(cpu);
651 
652 		/*
653 		 * Don't fall back to default_nid yet -- we will plug
654 		 * cpus into nodes once the memory scan has discovered
655 		 * the topology.
656 		 */
657 		if (nid < 0)
658 			continue;
659 		node_set_online(nid);
660 	}
661 
662 	get_n_mem_cells(&n_mem_addr_cells, &n_mem_size_cells);
663 
664 	for_each_node_by_type(memory, "memory") {
665 		unsigned long start;
666 		unsigned long size;
667 		int nid;
668 		int ranges;
669 		const __be32 *memcell_buf;
670 		unsigned int len;
671 
672 		memcell_buf = of_get_property(memory,
673 			"linux,usable-memory", &len);
674 		if (!memcell_buf || len <= 0)
675 			memcell_buf = of_get_property(memory, "reg", &len);
676 		if (!memcell_buf || len <= 0)
677 			continue;
678 
679 		/* ranges in cell */
680 		ranges = (len >> 2) / (n_mem_addr_cells + n_mem_size_cells);
681 new_range:
682 		/* these are order-sensitive, and modify the buffer pointer */
683 		start = read_n_cells(n_mem_addr_cells, &memcell_buf);
684 		size = read_n_cells(n_mem_size_cells, &memcell_buf);
685 
686 		/*
687 		 * Assumption: either all memory nodes or none will
688 		 * have associativity properties.  If none, then
689 		 * everything goes to default_nid.
690 		 */
691 		nid = of_node_to_nid_single(memory);
692 		if (nid < 0)
693 			nid = default_nid;
694 
695 		fake_numa_create_new_node(((start + size) >> PAGE_SHIFT), &nid);
696 		node_set_online(nid);
697 
698 		size = numa_enforce_memory_limit(start, size);
699 		if (size)
700 			memblock_set_node(start, size, &memblock.memory, nid);
701 
702 		if (--ranges)
703 			goto new_range;
704 	}
705 
706 	/*
707 	 * Now do the same thing for each MEMBLOCK listed in the
708 	 * ibm,dynamic-memory property in the
709 	 * ibm,dynamic-reconfiguration-memory node.
710 	 */
711 	memory = of_find_node_by_path("/ibm,dynamic-reconfiguration-memory");
712 	if (memory) {
713 		walk_drmem_lmbs(memory, numa_setup_drmem_lmb);
714 		of_node_put(memory);
715 	}
716 
717 	return 0;
718 }
719 
720 static void __init setup_nonnuma(void)
721 {
722 	unsigned long top_of_ram = memblock_end_of_DRAM();
723 	unsigned long total_ram = memblock_phys_mem_size();
724 	unsigned long start_pfn, end_pfn;
725 	unsigned int nid = 0;
726 	struct memblock_region *reg;
727 
728 	printk(KERN_DEBUG "Top of RAM: 0x%lx, Total RAM: 0x%lx\n",
729 	       top_of_ram, total_ram);
730 	printk(KERN_DEBUG "Memory hole size: %ldMB\n",
731 	       (top_of_ram - total_ram) >> 20);
732 
733 	for_each_memblock(memory, reg) {
734 		start_pfn = memblock_region_memory_base_pfn(reg);
735 		end_pfn = memblock_region_memory_end_pfn(reg);
736 
737 		fake_numa_create_new_node(end_pfn, &nid);
738 		memblock_set_node(PFN_PHYS(start_pfn),
739 				  PFN_PHYS(end_pfn - start_pfn),
740 				  &memblock.memory, nid);
741 		node_set_online(nid);
742 	}
743 }
744 
745 void __init dump_numa_cpu_topology(void)
746 {
747 	unsigned int node;
748 	unsigned int cpu, count;
749 
750 	if (min_common_depth == -1 || !numa_enabled)
751 		return;
752 
753 	for_each_online_node(node) {
754 		pr_info("Node %d CPUs:", node);
755 
756 		count = 0;
757 		/*
758 		 * If we used a CPU iterator here we would miss printing
759 		 * the holes in the cpumap.
760 		 */
761 		for (cpu = 0; cpu < nr_cpu_ids; cpu++) {
762 			if (cpumask_test_cpu(cpu,
763 					node_to_cpumask_map[node])) {
764 				if (count == 0)
765 					pr_cont(" %u", cpu);
766 				++count;
767 			} else {
768 				if (count > 1)
769 					pr_cont("-%u", cpu - 1);
770 				count = 0;
771 			}
772 		}
773 
774 		if (count > 1)
775 			pr_cont("-%u", nr_cpu_ids - 1);
776 		pr_cont("\n");
777 	}
778 }
779 
780 /* Initialize NODE_DATA for a node on the local memory */
781 static void __init setup_node_data(int nid, u64 start_pfn, u64 end_pfn)
782 {
783 	u64 spanned_pages = end_pfn - start_pfn;
784 	const size_t nd_size = roundup(sizeof(pg_data_t), SMP_CACHE_BYTES);
785 	u64 nd_pa;
786 	void *nd;
787 	int tnid;
788 
789 	nd_pa = memblock_phys_alloc_try_nid(nd_size, SMP_CACHE_BYTES, nid);
790 	if (!nd_pa)
791 		panic("Cannot allocate %zu bytes for node %d data\n",
792 		      nd_size, nid);
793 
794 	nd = __va(nd_pa);
795 
796 	/* report and initialize */
797 	pr_info("  NODE_DATA [mem %#010Lx-%#010Lx]\n",
798 		nd_pa, nd_pa + nd_size - 1);
799 	tnid = early_pfn_to_nid(nd_pa >> PAGE_SHIFT);
800 	if (tnid != nid)
801 		pr_info("    NODE_DATA(%d) on node %d\n", nid, tnid);
802 
803 	node_data[nid] = nd;
804 	memset(NODE_DATA(nid), 0, sizeof(pg_data_t));
805 	NODE_DATA(nid)->node_id = nid;
806 	NODE_DATA(nid)->node_start_pfn = start_pfn;
807 	NODE_DATA(nid)->node_spanned_pages = spanned_pages;
808 }
809 
810 static void __init find_possible_nodes(void)
811 {
812 	struct device_node *rtas;
813 	u32 numnodes, i;
814 
815 	if (min_common_depth <= 0)
816 		return;
817 
818 	rtas = of_find_node_by_path("/rtas");
819 	if (!rtas)
820 		return;
821 
822 	if (of_property_read_u32_index(rtas,
823 				"ibm,max-associativity-domains",
824 				min_common_depth, &numnodes))
825 		goto out;
826 
827 	for (i = 0; i < numnodes; i++) {
828 		if (!node_possible(i))
829 			node_set(i, node_possible_map);
830 	}
831 
832 out:
833 	of_node_put(rtas);
834 }
835 
836 void __init mem_topology_setup(void)
837 {
838 	int cpu;
839 
840 	if (parse_numa_properties())
841 		setup_nonnuma();
842 
843 	/*
844 	 * Modify the set of possible NUMA nodes to reflect information
845 	 * available about the set of online nodes, and the set of nodes
846 	 * that we expect to make use of for this platform's affinity
847 	 * calculations.
848 	 */
849 	nodes_and(node_possible_map, node_possible_map, node_online_map);
850 
851 	find_possible_nodes();
852 
853 	setup_node_to_cpumask_map();
854 
855 	reset_numa_cpu_lookup_table();
856 
857 	for_each_present_cpu(cpu)
858 		numa_setup_cpu(cpu);
859 }
860 
861 void __init initmem_init(void)
862 {
863 	int nid;
864 
865 	max_low_pfn = memblock_end_of_DRAM() >> PAGE_SHIFT;
866 	max_pfn = max_low_pfn;
867 
868 	memblock_dump_all();
869 
870 	for_each_online_node(nid) {
871 		unsigned long start_pfn, end_pfn;
872 
873 		get_pfn_range_for_nid(nid, &start_pfn, &end_pfn);
874 		setup_node_data(nid, start_pfn, end_pfn);
875 		sparse_memory_present_with_active_regions(nid);
876 	}
877 
878 	sparse_init();
879 
880 	/*
881 	 * We need the numa_cpu_lookup_table to be accurate for all CPUs,
882 	 * even before we online them, so that we can use cpu_to_{node,mem}
883 	 * early in boot, cf. smp_prepare_cpus().
884 	 * _nocalls() + manual invocation is used because cpuhp is not yet
885 	 * initialized for the boot CPU.
886 	 */
887 	cpuhp_setup_state_nocalls(CPUHP_POWER_NUMA_PREPARE, "powerpc/numa:prepare",
888 				  ppc_numa_cpu_prepare, ppc_numa_cpu_dead);
889 }
890 
891 static int __init early_numa(char *p)
892 {
893 	if (!p)
894 		return 0;
895 
896 	if (strstr(p, "off"))
897 		numa_enabled = 0;
898 
899 	if (strstr(p, "debug"))
900 		numa_debug = 1;
901 
902 	p = strstr(p, "fake=");
903 	if (p)
904 		cmdline = p + strlen("fake=");
905 
906 	return 0;
907 }
908 early_param("numa", early_numa);
909 
910 /*
911  * The platform can inform us through one of several mechanisms
912  * (post-migration device tree updates, PRRN or VPHN) that the NUMA
913  * assignment of a resource has changed. This controls whether we act
914  * on that. Disabled by default.
915  */
916 static bool topology_updates_enabled;
917 
918 static int __init early_topology_updates(char *p)
919 {
920 	if (!p)
921 		return 0;
922 
923 	if (!strcmp(p, "on")) {
924 		pr_warn("Caution: enabling topology updates\n");
925 		topology_updates_enabled = true;
926 	}
927 
928 	return 0;
929 }
930 early_param("topology_updates", early_topology_updates);
931 
932 #ifdef CONFIG_MEMORY_HOTPLUG
933 /*
934  * Find the node associated with a hot added memory section for
935  * memory represented in the device tree by the property
936  * ibm,dynamic-reconfiguration-memory/ibm,dynamic-memory.
937  */
938 static int hot_add_drconf_scn_to_nid(unsigned long scn_addr)
939 {
940 	struct drmem_lmb *lmb;
941 	unsigned long lmb_size;
942 	int nid = NUMA_NO_NODE;
943 
944 	lmb_size = drmem_lmb_size();
945 
946 	for_each_drmem_lmb(lmb) {
947 		/* skip this block if it is reserved or not assigned to
948 		 * this partition */
949 		if ((lmb->flags & DRCONF_MEM_RESERVED)
950 		    || !(lmb->flags & DRCONF_MEM_ASSIGNED))
951 			continue;
952 
953 		if ((scn_addr < lmb->base_addr)
954 		    || (scn_addr >= (lmb->base_addr + lmb_size)))
955 			continue;
956 
957 		nid = of_drconf_to_nid_single(lmb);
958 		break;
959 	}
960 
961 	return nid;
962 }
963 
964 /*
965  * Find the node associated with a hot added memory section for memory
966  * represented in the device tree as a node (i.e. memory@XXXX) for
967  * each memblock.
968  */
969 static int hot_add_node_scn_to_nid(unsigned long scn_addr)
970 {
971 	struct device_node *memory;
972 	int nid = NUMA_NO_NODE;
973 
974 	for_each_node_by_type(memory, "memory") {
975 		unsigned long start, size;
976 		int ranges;
977 		const __be32 *memcell_buf;
978 		unsigned int len;
979 
980 		memcell_buf = of_get_property(memory, "reg", &len);
981 		if (!memcell_buf || len <= 0)
982 			continue;
983 
984 		/* ranges in cell */
985 		ranges = (len >> 2) / (n_mem_addr_cells + n_mem_size_cells);
986 
987 		while (ranges--) {
988 			start = read_n_cells(n_mem_addr_cells, &memcell_buf);
989 			size = read_n_cells(n_mem_size_cells, &memcell_buf);
990 
991 			if ((scn_addr < start) || (scn_addr >= (start + size)))
992 				continue;
993 
994 			nid = of_node_to_nid_single(memory);
995 			break;
996 		}
997 
998 		if (nid >= 0)
999 			break;
1000 	}
1001 
1002 	of_node_put(memory);
1003 
1004 	return nid;
1005 }
1006 
1007 /*
1008  * Find the node associated with a hot added memory section.  Section
1009  * corresponds to a SPARSEMEM section, not an MEMBLOCK.  It is assumed that
1010  * sections are fully contained within a single MEMBLOCK.
1011  */
1012 int hot_add_scn_to_nid(unsigned long scn_addr)
1013 {
1014 	struct device_node *memory = NULL;
1015 	int nid;
1016 
1017 	if (!numa_enabled || (min_common_depth < 0))
1018 		return first_online_node;
1019 
1020 	memory = of_find_node_by_path("/ibm,dynamic-reconfiguration-memory");
1021 	if (memory) {
1022 		nid = hot_add_drconf_scn_to_nid(scn_addr);
1023 		of_node_put(memory);
1024 	} else {
1025 		nid = hot_add_node_scn_to_nid(scn_addr);
1026 	}
1027 
1028 	if (nid < 0 || !node_possible(nid))
1029 		nid = first_online_node;
1030 
1031 	return nid;
1032 }
1033 
1034 static u64 hot_add_drconf_memory_max(void)
1035 {
1036 	struct device_node *memory = NULL;
1037 	struct device_node *dn = NULL;
1038 	const __be64 *lrdr = NULL;
1039 
1040 	dn = of_find_node_by_path("/rtas");
1041 	if (dn) {
1042 		lrdr = of_get_property(dn, "ibm,lrdr-capacity", NULL);
1043 		of_node_put(dn);
1044 		if (lrdr)
1045 			return be64_to_cpup(lrdr);
1046 	}
1047 
1048 	memory = of_find_node_by_path("/ibm,dynamic-reconfiguration-memory");
1049 	if (memory) {
1050 		of_node_put(memory);
1051 		return drmem_lmb_memory_max();
1052 	}
1053 	return 0;
1054 }
1055 
1056 /*
1057  * memory_hotplug_max - return max address of memory that may be added
1058  *
1059  * This is currently only used on systems that support drconfig memory
1060  * hotplug.
1061  */
1062 u64 memory_hotplug_max(void)
1063 {
1064         return max(hot_add_drconf_memory_max(), memblock_end_of_DRAM());
1065 }
1066 #endif /* CONFIG_MEMORY_HOTPLUG */
1067 
1068 /* Virtual Processor Home Node (VPHN) support */
1069 #ifdef CONFIG_PPC_SPLPAR
1070 
1071 #include "book3s64/vphn.h"
1072 
1073 struct topology_update_data {
1074 	struct topology_update_data *next;
1075 	unsigned int cpu;
1076 	int old_nid;
1077 	int new_nid;
1078 };
1079 
1080 #define TOPOLOGY_DEF_TIMER_SECS	60
1081 
1082 static u8 vphn_cpu_change_counts[NR_CPUS][MAX_DISTANCE_REF_POINTS];
1083 static cpumask_t cpu_associativity_changes_mask;
1084 static int vphn_enabled;
1085 static int prrn_enabled;
1086 static void reset_topology_timer(void);
1087 static int topology_timer_secs = 1;
1088 static int topology_inited;
1089 
1090 /*
1091  * Change polling interval for associativity changes.
1092  */
1093 int timed_topology_update(int nsecs)
1094 {
1095 	if (vphn_enabled) {
1096 		if (nsecs > 0)
1097 			topology_timer_secs = nsecs;
1098 		else
1099 			topology_timer_secs = TOPOLOGY_DEF_TIMER_SECS;
1100 
1101 		reset_topology_timer();
1102 	}
1103 
1104 	return 0;
1105 }
1106 
1107 /*
1108  * Store the current values of the associativity change counters in the
1109  * hypervisor.
1110  */
1111 static void setup_cpu_associativity_change_counters(void)
1112 {
1113 	int cpu;
1114 
1115 	/* The VPHN feature supports a maximum of 8 reference points */
1116 	BUILD_BUG_ON(MAX_DISTANCE_REF_POINTS > 8);
1117 
1118 	for_each_possible_cpu(cpu) {
1119 		int i;
1120 		u8 *counts = vphn_cpu_change_counts[cpu];
1121 		volatile u8 *hypervisor_counts = lppaca_of(cpu).vphn_assoc_counts;
1122 
1123 		for (i = 0; i < distance_ref_points_depth; i++)
1124 			counts[i] = hypervisor_counts[i];
1125 	}
1126 }
1127 
1128 /*
1129  * The hypervisor maintains a set of 8 associativity change counters in
1130  * the VPA of each cpu that correspond to the associativity levels in the
1131  * ibm,associativity-reference-points property. When an associativity
1132  * level changes, the corresponding counter is incremented.
1133  *
1134  * Set a bit in cpu_associativity_changes_mask for each cpu whose home
1135  * node associativity levels have changed.
1136  *
1137  * Returns the number of cpus with unhandled associativity changes.
1138  */
1139 static int update_cpu_associativity_changes_mask(void)
1140 {
1141 	int cpu;
1142 	cpumask_t *changes = &cpu_associativity_changes_mask;
1143 
1144 	for_each_possible_cpu(cpu) {
1145 		int i, changed = 0;
1146 		u8 *counts = vphn_cpu_change_counts[cpu];
1147 		volatile u8 *hypervisor_counts = lppaca_of(cpu).vphn_assoc_counts;
1148 
1149 		for (i = 0; i < distance_ref_points_depth; i++) {
1150 			if (hypervisor_counts[i] != counts[i]) {
1151 				counts[i] = hypervisor_counts[i];
1152 				changed = 1;
1153 			}
1154 		}
1155 		if (changed) {
1156 			cpumask_or(changes, changes, cpu_sibling_mask(cpu));
1157 			cpu = cpu_last_thread_sibling(cpu);
1158 		}
1159 	}
1160 
1161 	return cpumask_weight(changes);
1162 }
1163 
1164 /*
1165  * Retrieve the new associativity information for a virtual processor's
1166  * home node.
1167  */
1168 static long hcall_vphn(unsigned long cpu, __be32 *associativity)
1169 {
1170 	long rc;
1171 	long retbuf[PLPAR_HCALL9_BUFSIZE] = {0};
1172 	u64 flags = 1;
1173 	int hwcpu = get_hard_smp_processor_id(cpu);
1174 
1175 	rc = plpar_hcall9(H_HOME_NODE_ASSOCIATIVITY, retbuf, flags, hwcpu);
1176 	vphn_unpack_associativity(retbuf, associativity);
1177 
1178 	return rc;
1179 }
1180 
1181 static long vphn_get_associativity(unsigned long cpu,
1182 					__be32 *associativity)
1183 {
1184 	long rc;
1185 
1186 	rc = hcall_vphn(cpu, associativity);
1187 
1188 	switch (rc) {
1189 	case H_FUNCTION:
1190 		printk_once(KERN_INFO
1191 			"VPHN is not supported. Disabling polling...\n");
1192 		stop_topology_update();
1193 		break;
1194 	case H_HARDWARE:
1195 		printk(KERN_ERR
1196 			"hcall_vphn() experienced a hardware fault "
1197 			"preventing VPHN. Disabling polling...\n");
1198 		stop_topology_update();
1199 		break;
1200 	case H_SUCCESS:
1201 		dbg("VPHN hcall succeeded. Reset polling...\n");
1202 		timed_topology_update(0);
1203 		break;
1204 	}
1205 
1206 	return rc;
1207 }
1208 
1209 int find_and_online_cpu_nid(int cpu)
1210 {
1211 	__be32 associativity[VPHN_ASSOC_BUFSIZE] = {0};
1212 	int new_nid;
1213 
1214 	/* Use associativity from first thread for all siblings */
1215 	if (vphn_get_associativity(cpu, associativity))
1216 		return cpu_to_node(cpu);
1217 
1218 	new_nid = associativity_to_nid(associativity);
1219 	if (new_nid < 0 || !node_possible(new_nid))
1220 		new_nid = first_online_node;
1221 
1222 	if (NODE_DATA(new_nid) == NULL) {
1223 #ifdef CONFIG_MEMORY_HOTPLUG
1224 		/*
1225 		 * Need to ensure that NODE_DATA is initialized for a node from
1226 		 * available memory (see memblock_alloc_try_nid). If unable to
1227 		 * init the node, then default to nearest node that has memory
1228 		 * installed. Skip onlining a node if the subsystems are not
1229 		 * yet initialized.
1230 		 */
1231 		if (!topology_inited || try_online_node(new_nid))
1232 			new_nid = first_online_node;
1233 #else
1234 		/*
1235 		 * Default to using the nearest node that has memory installed.
1236 		 * Otherwise, it would be necessary to patch the kernel MM code
1237 		 * to deal with more memoryless-node error conditions.
1238 		 */
1239 		new_nid = first_online_node;
1240 #endif
1241 	}
1242 
1243 	pr_debug("%s:%d cpu %d nid %d\n", __FUNCTION__, __LINE__,
1244 		cpu, new_nid);
1245 	return new_nid;
1246 }
1247 
1248 /*
1249  * Update the CPU maps and sysfs entries for a single CPU when its NUMA
1250  * characteristics change. This function doesn't perform any locking and is
1251  * only safe to call from stop_machine().
1252  */
1253 static int update_cpu_topology(void *data)
1254 {
1255 	struct topology_update_data *update;
1256 	unsigned long cpu;
1257 
1258 	if (!data)
1259 		return -EINVAL;
1260 
1261 	cpu = smp_processor_id();
1262 
1263 	for (update = data; update; update = update->next) {
1264 		int new_nid = update->new_nid;
1265 		if (cpu != update->cpu)
1266 			continue;
1267 
1268 		unmap_cpu_from_node(cpu);
1269 		map_cpu_to_node(cpu, new_nid);
1270 		set_cpu_numa_node(cpu, new_nid);
1271 		set_cpu_numa_mem(cpu, local_memory_node(new_nid));
1272 		vdso_getcpu_init();
1273 	}
1274 
1275 	return 0;
1276 }
1277 
1278 static int update_lookup_table(void *data)
1279 {
1280 	struct topology_update_data *update;
1281 
1282 	if (!data)
1283 		return -EINVAL;
1284 
1285 	/*
1286 	 * Upon topology update, the numa-cpu lookup table needs to be updated
1287 	 * for all threads in the core, including offline CPUs, to ensure that
1288 	 * future hotplug operations respect the cpu-to-node associativity
1289 	 * properly.
1290 	 */
1291 	for (update = data; update; update = update->next) {
1292 		int nid, base, j;
1293 
1294 		nid = update->new_nid;
1295 		base = cpu_first_thread_sibling(update->cpu);
1296 
1297 		for (j = 0; j < threads_per_core; j++) {
1298 			update_numa_cpu_lookup_table(base + j, nid);
1299 		}
1300 	}
1301 
1302 	return 0;
1303 }
1304 
1305 /*
1306  * Update the node maps and sysfs entries for each cpu whose home node
1307  * has changed. Returns 1 when the topology has changed, and 0 otherwise.
1308  *
1309  * cpus_locked says whether we already hold cpu_hotplug_lock.
1310  */
1311 int numa_update_cpu_topology(bool cpus_locked)
1312 {
1313 	unsigned int cpu, sibling, changed = 0;
1314 	struct topology_update_data *updates, *ud;
1315 	cpumask_t updated_cpus;
1316 	struct device *dev;
1317 	int weight, new_nid, i = 0;
1318 
1319 	if (!prrn_enabled && !vphn_enabled && topology_inited)
1320 		return 0;
1321 
1322 	weight = cpumask_weight(&cpu_associativity_changes_mask);
1323 	if (!weight)
1324 		return 0;
1325 
1326 	updates = kcalloc(weight, sizeof(*updates), GFP_KERNEL);
1327 	if (!updates)
1328 		return 0;
1329 
1330 	cpumask_clear(&updated_cpus);
1331 
1332 	for_each_cpu(cpu, &cpu_associativity_changes_mask) {
1333 		/*
1334 		 * If siblings aren't flagged for changes, updates list
1335 		 * will be too short. Skip on this update and set for next
1336 		 * update.
1337 		 */
1338 		if (!cpumask_subset(cpu_sibling_mask(cpu),
1339 					&cpu_associativity_changes_mask)) {
1340 			pr_info("Sibling bits not set for associativity "
1341 					"change, cpu%d\n", cpu);
1342 			cpumask_or(&cpu_associativity_changes_mask,
1343 					&cpu_associativity_changes_mask,
1344 					cpu_sibling_mask(cpu));
1345 			cpu = cpu_last_thread_sibling(cpu);
1346 			continue;
1347 		}
1348 
1349 		new_nid = find_and_online_cpu_nid(cpu);
1350 
1351 		if (new_nid == numa_cpu_lookup_table[cpu]) {
1352 			cpumask_andnot(&cpu_associativity_changes_mask,
1353 					&cpu_associativity_changes_mask,
1354 					cpu_sibling_mask(cpu));
1355 			dbg("Assoc chg gives same node %d for cpu%d\n",
1356 					new_nid, cpu);
1357 			cpu = cpu_last_thread_sibling(cpu);
1358 			continue;
1359 		}
1360 
1361 		for_each_cpu(sibling, cpu_sibling_mask(cpu)) {
1362 			ud = &updates[i++];
1363 			ud->next = &updates[i];
1364 			ud->cpu = sibling;
1365 			ud->new_nid = new_nid;
1366 			ud->old_nid = numa_cpu_lookup_table[sibling];
1367 			cpumask_set_cpu(sibling, &updated_cpus);
1368 		}
1369 		cpu = cpu_last_thread_sibling(cpu);
1370 	}
1371 
1372 	/*
1373 	 * Prevent processing of 'updates' from overflowing array
1374 	 * where last entry filled in a 'next' pointer.
1375 	 */
1376 	if (i)
1377 		updates[i-1].next = NULL;
1378 
1379 	pr_debug("Topology update for the following CPUs:\n");
1380 	if (cpumask_weight(&updated_cpus)) {
1381 		for (ud = &updates[0]; ud; ud = ud->next) {
1382 			pr_debug("cpu %d moving from node %d "
1383 					  "to %d\n", ud->cpu,
1384 					  ud->old_nid, ud->new_nid);
1385 		}
1386 	}
1387 
1388 	/*
1389 	 * In cases where we have nothing to update (because the updates list
1390 	 * is too short or because the new topology is same as the old one),
1391 	 * skip invoking update_cpu_topology() via stop-machine(). This is
1392 	 * necessary (and not just a fast-path optimization) since stop-machine
1393 	 * can end up electing a random CPU to run update_cpu_topology(), and
1394 	 * thus trick us into setting up incorrect cpu-node mappings (since
1395 	 * 'updates' is kzalloc()'ed).
1396 	 *
1397 	 * And for the similar reason, we will skip all the following updating.
1398 	 */
1399 	if (!cpumask_weight(&updated_cpus))
1400 		goto out;
1401 
1402 	if (cpus_locked)
1403 		stop_machine_cpuslocked(update_cpu_topology, &updates[0],
1404 					&updated_cpus);
1405 	else
1406 		stop_machine(update_cpu_topology, &updates[0], &updated_cpus);
1407 
1408 	/*
1409 	 * Update the numa-cpu lookup table with the new mappings, even for
1410 	 * offline CPUs. It is best to perform this update from the stop-
1411 	 * machine context.
1412 	 */
1413 	if (cpus_locked)
1414 		stop_machine_cpuslocked(update_lookup_table, &updates[0],
1415 					cpumask_of(raw_smp_processor_id()));
1416 	else
1417 		stop_machine(update_lookup_table, &updates[0],
1418 			     cpumask_of(raw_smp_processor_id()));
1419 
1420 	for (ud = &updates[0]; ud; ud = ud->next) {
1421 		unregister_cpu_under_node(ud->cpu, ud->old_nid);
1422 		register_cpu_under_node(ud->cpu, ud->new_nid);
1423 
1424 		dev = get_cpu_device(ud->cpu);
1425 		if (dev)
1426 			kobject_uevent(&dev->kobj, KOBJ_CHANGE);
1427 		cpumask_clear_cpu(ud->cpu, &cpu_associativity_changes_mask);
1428 		changed = 1;
1429 	}
1430 
1431 out:
1432 	kfree(updates);
1433 	return changed;
1434 }
1435 
1436 int arch_update_cpu_topology(void)
1437 {
1438 	return numa_update_cpu_topology(true);
1439 }
1440 
1441 static void topology_work_fn(struct work_struct *work)
1442 {
1443 	rebuild_sched_domains();
1444 }
1445 static DECLARE_WORK(topology_work, topology_work_fn);
1446 
1447 static void topology_schedule_update(void)
1448 {
1449 	schedule_work(&topology_work);
1450 }
1451 
1452 static void topology_timer_fn(struct timer_list *unused)
1453 {
1454 	if (prrn_enabled && cpumask_weight(&cpu_associativity_changes_mask))
1455 		topology_schedule_update();
1456 	else if (vphn_enabled) {
1457 		if (update_cpu_associativity_changes_mask() > 0)
1458 			topology_schedule_update();
1459 		reset_topology_timer();
1460 	}
1461 }
1462 static struct timer_list topology_timer;
1463 
1464 static void reset_topology_timer(void)
1465 {
1466 	if (vphn_enabled)
1467 		mod_timer(&topology_timer, jiffies + topology_timer_secs * HZ);
1468 }
1469 
1470 #ifdef CONFIG_SMP
1471 
1472 static int dt_update_callback(struct notifier_block *nb,
1473 				unsigned long action, void *data)
1474 {
1475 	struct of_reconfig_data *update = data;
1476 	int rc = NOTIFY_DONE;
1477 
1478 	switch (action) {
1479 	case OF_RECONFIG_UPDATE_PROPERTY:
1480 		if (of_node_is_type(update->dn, "cpu") &&
1481 		    !of_prop_cmp(update->prop->name, "ibm,associativity")) {
1482 			u32 core_id;
1483 			of_property_read_u32(update->dn, "reg", &core_id);
1484 			rc = dlpar_cpu_readd(core_id);
1485 			rc = NOTIFY_OK;
1486 		}
1487 		break;
1488 	}
1489 
1490 	return rc;
1491 }
1492 
1493 static struct notifier_block dt_update_nb = {
1494 	.notifier_call = dt_update_callback,
1495 };
1496 
1497 #endif
1498 
1499 /*
1500  * Start polling for associativity changes.
1501  */
1502 int start_topology_update(void)
1503 {
1504 	int rc = 0;
1505 
1506 	if (!topology_updates_enabled)
1507 		return 0;
1508 
1509 	if (firmware_has_feature(FW_FEATURE_PRRN)) {
1510 		if (!prrn_enabled) {
1511 			prrn_enabled = 1;
1512 #ifdef CONFIG_SMP
1513 			rc = of_reconfig_notifier_register(&dt_update_nb);
1514 #endif
1515 		}
1516 	}
1517 	if (firmware_has_feature(FW_FEATURE_VPHN) &&
1518 		   lppaca_shared_proc(get_lppaca())) {
1519 		if (!vphn_enabled) {
1520 			vphn_enabled = 1;
1521 			setup_cpu_associativity_change_counters();
1522 			timer_setup(&topology_timer, topology_timer_fn,
1523 				    TIMER_DEFERRABLE);
1524 			reset_topology_timer();
1525 		}
1526 	}
1527 
1528 	pr_info("Starting topology update%s%s\n",
1529 		(prrn_enabled ? " prrn_enabled" : ""),
1530 		(vphn_enabled ? " vphn_enabled" : ""));
1531 
1532 	return rc;
1533 }
1534 
1535 /*
1536  * Disable polling for VPHN associativity changes.
1537  */
1538 int stop_topology_update(void)
1539 {
1540 	int rc = 0;
1541 
1542 	if (!topology_updates_enabled)
1543 		return 0;
1544 
1545 	if (prrn_enabled) {
1546 		prrn_enabled = 0;
1547 #ifdef CONFIG_SMP
1548 		rc = of_reconfig_notifier_unregister(&dt_update_nb);
1549 #endif
1550 	}
1551 	if (vphn_enabled) {
1552 		vphn_enabled = 0;
1553 		rc = del_timer_sync(&topology_timer);
1554 	}
1555 
1556 	pr_info("Stopping topology update\n");
1557 
1558 	return rc;
1559 }
1560 
1561 int prrn_is_enabled(void)
1562 {
1563 	return prrn_enabled;
1564 }
1565 
1566 void __init shared_proc_topology_init(void)
1567 {
1568 	if (lppaca_shared_proc(get_lppaca())) {
1569 		bitmap_fill(cpumask_bits(&cpu_associativity_changes_mask),
1570 			    nr_cpumask_bits);
1571 		numa_update_cpu_topology(false);
1572 	}
1573 }
1574 
1575 static int topology_read(struct seq_file *file, void *v)
1576 {
1577 	if (vphn_enabled || prrn_enabled)
1578 		seq_puts(file, "on\n");
1579 	else
1580 		seq_puts(file, "off\n");
1581 
1582 	return 0;
1583 }
1584 
1585 static int topology_open(struct inode *inode, struct file *file)
1586 {
1587 	return single_open(file, topology_read, NULL);
1588 }
1589 
1590 static ssize_t topology_write(struct file *file, const char __user *buf,
1591 			      size_t count, loff_t *off)
1592 {
1593 	char kbuf[4]; /* "on" or "off" plus null. */
1594 	int read_len;
1595 
1596 	read_len = count < 3 ? count : 3;
1597 	if (copy_from_user(kbuf, buf, read_len))
1598 		return -EINVAL;
1599 
1600 	kbuf[read_len] = '\0';
1601 
1602 	if (!strncmp(kbuf, "on", 2)) {
1603 		topology_updates_enabled = true;
1604 		start_topology_update();
1605 	} else if (!strncmp(kbuf, "off", 3)) {
1606 		stop_topology_update();
1607 		topology_updates_enabled = false;
1608 	} else
1609 		return -EINVAL;
1610 
1611 	return count;
1612 }
1613 
1614 static const struct file_operations topology_ops = {
1615 	.read = seq_read,
1616 	.write = topology_write,
1617 	.open = topology_open,
1618 	.release = single_release
1619 };
1620 
1621 static int topology_update_init(void)
1622 {
1623 	start_topology_update();
1624 
1625 	if (vphn_enabled)
1626 		topology_schedule_update();
1627 
1628 	if (!proc_create("powerpc/topology_updates", 0644, NULL, &topology_ops))
1629 		return -ENOMEM;
1630 
1631 	topology_inited = 1;
1632 	return 0;
1633 }
1634 device_initcall(topology_update_init);
1635 #endif /* CONFIG_PPC_SPLPAR */
1636