1 // SPDX-License-Identifier: GPL-2.0-or-later
2 /*
3  * pseries CPU Hotplug infrastructure.
4  *
5  * Split out from arch/powerpc/platforms/pseries/setup.c
6  *  arch/powerpc/kernel/rtas.c, and arch/powerpc/platforms/pseries/smp.c
7  *
8  * Peter Bergner, IBM	March 2001.
9  * Copyright (C) 2001 IBM.
10  * Dave Engebretsen, Peter Bergner, and
11  * Mike Corrigan {engebret|bergner|mikec}@us.ibm.com
12  * Plus various changes from other IBM teams...
13  *
14  * Copyright (C) 2006 Michael Ellerman, IBM Corporation
15  */
16 
17 #define pr_fmt(fmt)     "pseries-hotplug-cpu: " fmt
18 
19 #include <linux/kernel.h>
20 #include <linux/interrupt.h>
21 #include <linux/delay.h>
22 #include <linux/sched.h>	/* for idle_task_exit */
23 #include <linux/sched/hotplug.h>
24 #include <linux/cpu.h>
25 #include <linux/of.h>
26 #include <linux/slab.h>
27 #include <asm/prom.h>
28 #include <asm/rtas.h>
29 #include <asm/firmware.h>
30 #include <asm/machdep.h>
31 #include <asm/vdso_datapage.h>
32 #include <asm/xics.h>
33 #include <asm/xive.h>
34 #include <asm/plpar_wrappers.h>
35 #include <asm/topology.h>
36 
37 #include "pseries.h"
38 #include "offline_states.h"
39 
40 /* This version can't take the spinlock, because it never returns */
41 static int rtas_stop_self_token = RTAS_UNKNOWN_SERVICE;
42 
43 static DEFINE_PER_CPU(enum cpu_state_vals, preferred_offline_state) =
44 							CPU_STATE_OFFLINE;
45 static DEFINE_PER_CPU(enum cpu_state_vals, current_state) = CPU_STATE_OFFLINE;
46 
47 static enum cpu_state_vals default_offline_state = CPU_STATE_OFFLINE;
48 
49 static bool cede_offline_enabled __read_mostly = true;
50 
51 /*
52  * Enable/disable cede_offline when available.
53  */
54 static int __init setup_cede_offline(char *str)
55 {
56 	return (kstrtobool(str, &cede_offline_enabled) == 0);
57 }
58 
59 __setup("cede_offline=", setup_cede_offline);
60 
61 enum cpu_state_vals get_cpu_current_state(int cpu)
62 {
63 	return per_cpu(current_state, cpu);
64 }
65 
66 void set_cpu_current_state(int cpu, enum cpu_state_vals state)
67 {
68 	per_cpu(current_state, cpu) = state;
69 }
70 
71 enum cpu_state_vals get_preferred_offline_state(int cpu)
72 {
73 	return per_cpu(preferred_offline_state, cpu);
74 }
75 
76 void set_preferred_offline_state(int cpu, enum cpu_state_vals state)
77 {
78 	per_cpu(preferred_offline_state, cpu) = state;
79 }
80 
81 void set_default_offline_state(int cpu)
82 {
83 	per_cpu(preferred_offline_state, cpu) = default_offline_state;
84 }
85 
86 static void rtas_stop_self(void)
87 {
88 	static struct rtas_args args;
89 
90 	local_irq_disable();
91 
92 	BUG_ON(rtas_stop_self_token == RTAS_UNKNOWN_SERVICE);
93 
94 	printk("cpu %u (hwid %u) Ready to die...\n",
95 	       smp_processor_id(), hard_smp_processor_id());
96 
97 	rtas_call_unlocked(&args, rtas_stop_self_token, 0, 1, NULL);
98 
99 	panic("Alas, I survived.\n");
100 }
101 
102 static void pseries_mach_cpu_die(void)
103 {
104 	unsigned int cpu = smp_processor_id();
105 	unsigned int hwcpu = hard_smp_processor_id();
106 	u8 cede_latency_hint = 0;
107 
108 	local_irq_disable();
109 	idle_task_exit();
110 	if (xive_enabled())
111 		xive_teardown_cpu();
112 	else
113 		xics_teardown_cpu();
114 
115 	if (get_preferred_offline_state(cpu) == CPU_STATE_INACTIVE) {
116 		set_cpu_current_state(cpu, CPU_STATE_INACTIVE);
117 		if (ppc_md.suspend_disable_cpu)
118 			ppc_md.suspend_disable_cpu();
119 
120 		cede_latency_hint = 2;
121 
122 		get_lppaca()->idle = 1;
123 		if (!lppaca_shared_proc(get_lppaca()))
124 			get_lppaca()->donate_dedicated_cpu = 1;
125 
126 		while (get_preferred_offline_state(cpu) == CPU_STATE_INACTIVE) {
127 			while (!prep_irq_for_idle()) {
128 				local_irq_enable();
129 				local_irq_disable();
130 			}
131 
132 			extended_cede_processor(cede_latency_hint);
133 		}
134 
135 		local_irq_disable();
136 
137 		if (!lppaca_shared_proc(get_lppaca()))
138 			get_lppaca()->donate_dedicated_cpu = 0;
139 		get_lppaca()->idle = 0;
140 
141 		if (get_preferred_offline_state(cpu) == CPU_STATE_ONLINE) {
142 			unregister_slb_shadow(hwcpu);
143 
144 			hard_irq_disable();
145 			/*
146 			 * Call to start_secondary_resume() will not return.
147 			 * Kernel stack will be reset and start_secondary()
148 			 * will be called to continue the online operation.
149 			 */
150 			start_secondary_resume();
151 		}
152 	}
153 
154 	/* Requested state is CPU_STATE_OFFLINE at this point */
155 	WARN_ON(get_preferred_offline_state(cpu) != CPU_STATE_OFFLINE);
156 
157 	set_cpu_current_state(cpu, CPU_STATE_OFFLINE);
158 	unregister_slb_shadow(hwcpu);
159 	rtas_stop_self();
160 
161 	/* Should never get here... */
162 	BUG();
163 	for(;;);
164 }
165 
166 static int pseries_cpu_disable(void)
167 {
168 	int cpu = smp_processor_id();
169 
170 	set_cpu_online(cpu, false);
171 	vdso_data->processorCount--;
172 
173 	/*fix boot_cpuid here*/
174 	if (cpu == boot_cpuid)
175 		boot_cpuid = cpumask_any(cpu_online_mask);
176 
177 	/* FIXME: abstract this to not be platform specific later on */
178 	if (xive_enabled())
179 		xive_smp_disable_cpu();
180 	else
181 		xics_migrate_irqs_away();
182 	return 0;
183 }
184 
185 /*
186  * pseries_cpu_die: Wait for the cpu to die.
187  * @cpu: logical processor id of the CPU whose death we're awaiting.
188  *
189  * This function is called from the context of the thread which is performing
190  * the cpu-offline. Here we wait for long enough to allow the cpu in question
191  * to self-destroy so that the cpu-offline thread can send the CPU_DEAD
192  * notifications.
193  *
194  * OTOH, pseries_mach_cpu_die() is called by the @cpu when it wants to
195  * self-destruct.
196  */
197 static void pseries_cpu_die(unsigned int cpu)
198 {
199 	int tries;
200 	int cpu_status = 1;
201 	unsigned int pcpu = get_hard_smp_processor_id(cpu);
202 
203 	if (get_preferred_offline_state(cpu) == CPU_STATE_INACTIVE) {
204 		cpu_status = 1;
205 		for (tries = 0; tries < 5000; tries++) {
206 			if (get_cpu_current_state(cpu) == CPU_STATE_INACTIVE) {
207 				cpu_status = 0;
208 				break;
209 			}
210 			msleep(1);
211 		}
212 	} else if (get_preferred_offline_state(cpu) == CPU_STATE_OFFLINE) {
213 
214 		for (tries = 0; tries < 25; tries++) {
215 			cpu_status = smp_query_cpu_stopped(pcpu);
216 			if (cpu_status == QCSS_STOPPED ||
217 			    cpu_status == QCSS_HARDWARE_ERROR)
218 				break;
219 			cpu_relax();
220 		}
221 	}
222 
223 	if (cpu_status != 0) {
224 		printk("Querying DEAD? cpu %i (%i) shows %i\n",
225 		       cpu, pcpu, cpu_status);
226 	}
227 
228 	/* Isolation and deallocation are definitely done by
229 	 * drslot_chrp_cpu.  If they were not they would be
230 	 * done here.  Change isolate state to Isolate and
231 	 * change allocation-state to Unusable.
232 	 */
233 	paca_ptrs[cpu]->cpu_start = 0;
234 }
235 
236 /*
237  * Update cpu_present_mask and paca(s) for a new cpu node.  The wrinkle
238  * here is that a cpu device node may represent up to two logical cpus
239  * in the SMT case.  We must honor the assumption in other code that
240  * the logical ids for sibling SMT threads x and y are adjacent, such
241  * that x^1 == y and y^1 == x.
242  */
243 static int pseries_add_processor(struct device_node *np)
244 {
245 	unsigned int cpu;
246 	cpumask_var_t candidate_mask, tmp;
247 	int err = -ENOSPC, len, nthreads, i;
248 	const __be32 *intserv;
249 
250 	intserv = of_get_property(np, "ibm,ppc-interrupt-server#s", &len);
251 	if (!intserv)
252 		return 0;
253 
254 	zalloc_cpumask_var(&candidate_mask, GFP_KERNEL);
255 	zalloc_cpumask_var(&tmp, GFP_KERNEL);
256 
257 	nthreads = len / sizeof(u32);
258 	for (i = 0; i < nthreads; i++)
259 		cpumask_set_cpu(i, tmp);
260 
261 	cpu_maps_update_begin();
262 
263 	BUG_ON(!cpumask_subset(cpu_present_mask, cpu_possible_mask));
264 
265 	/* Get a bitmap of unoccupied slots. */
266 	cpumask_xor(candidate_mask, cpu_possible_mask, cpu_present_mask);
267 	if (cpumask_empty(candidate_mask)) {
268 		/* If we get here, it most likely means that NR_CPUS is
269 		 * less than the partition's max processors setting.
270 		 */
271 		printk(KERN_ERR "Cannot add cpu %pOF; this system configuration"
272 		       " supports %d logical cpus.\n", np,
273 		       num_possible_cpus());
274 		goto out_unlock;
275 	}
276 
277 	while (!cpumask_empty(tmp))
278 		if (cpumask_subset(tmp, candidate_mask))
279 			/* Found a range where we can insert the new cpu(s) */
280 			break;
281 		else
282 			cpumask_shift_left(tmp, tmp, nthreads);
283 
284 	if (cpumask_empty(tmp)) {
285 		printk(KERN_ERR "Unable to find space in cpu_present_mask for"
286 		       " processor %pOFn with %d thread(s)\n", np,
287 		       nthreads);
288 		goto out_unlock;
289 	}
290 
291 	for_each_cpu(cpu, tmp) {
292 		BUG_ON(cpu_present(cpu));
293 		set_cpu_present(cpu, true);
294 		set_hard_smp_processor_id(cpu, be32_to_cpu(*intserv++));
295 	}
296 	err = 0;
297 out_unlock:
298 	cpu_maps_update_done();
299 	free_cpumask_var(candidate_mask);
300 	free_cpumask_var(tmp);
301 	return err;
302 }
303 
304 /*
305  * Update the present map for a cpu node which is going away, and set
306  * the hard id in the paca(s) to -1 to be consistent with boot time
307  * convention for non-present cpus.
308  */
309 static void pseries_remove_processor(struct device_node *np)
310 {
311 	unsigned int cpu;
312 	int len, nthreads, i;
313 	const __be32 *intserv;
314 	u32 thread;
315 
316 	intserv = of_get_property(np, "ibm,ppc-interrupt-server#s", &len);
317 	if (!intserv)
318 		return;
319 
320 	nthreads = len / sizeof(u32);
321 
322 	cpu_maps_update_begin();
323 	for (i = 0; i < nthreads; i++) {
324 		thread = be32_to_cpu(intserv[i]);
325 		for_each_present_cpu(cpu) {
326 			if (get_hard_smp_processor_id(cpu) != thread)
327 				continue;
328 			BUG_ON(cpu_online(cpu));
329 			set_cpu_present(cpu, false);
330 			set_hard_smp_processor_id(cpu, -1);
331 			update_numa_cpu_lookup_table(cpu, -1);
332 			break;
333 		}
334 		if (cpu >= nr_cpu_ids)
335 			printk(KERN_WARNING "Could not find cpu to remove "
336 			       "with physical id 0x%x\n", thread);
337 	}
338 	cpu_maps_update_done();
339 }
340 
341 static int dlpar_offline_cpu(struct device_node *dn)
342 {
343 	int rc = 0;
344 	unsigned int cpu;
345 	int len, nthreads, i;
346 	const __be32 *intserv;
347 	u32 thread;
348 
349 	intserv = of_get_property(dn, "ibm,ppc-interrupt-server#s", &len);
350 	if (!intserv)
351 		return -EINVAL;
352 
353 	nthreads = len / sizeof(u32);
354 
355 	cpu_maps_update_begin();
356 	for (i = 0; i < nthreads; i++) {
357 		thread = be32_to_cpu(intserv[i]);
358 		for_each_present_cpu(cpu) {
359 			if (get_hard_smp_processor_id(cpu) != thread)
360 				continue;
361 
362 			if (get_cpu_current_state(cpu) == CPU_STATE_OFFLINE)
363 				break;
364 
365 			if (get_cpu_current_state(cpu) == CPU_STATE_ONLINE) {
366 				set_preferred_offline_state(cpu,
367 							    CPU_STATE_OFFLINE);
368 				cpu_maps_update_done();
369 				timed_topology_update(1);
370 				rc = device_offline(get_cpu_device(cpu));
371 				if (rc)
372 					goto out;
373 				cpu_maps_update_begin();
374 				break;
375 			}
376 
377 			/*
378 			 * The cpu is in CPU_STATE_INACTIVE.
379 			 * Upgrade it's state to CPU_STATE_OFFLINE.
380 			 */
381 			set_preferred_offline_state(cpu, CPU_STATE_OFFLINE);
382 			WARN_ON(plpar_hcall_norets(H_PROD, thread) != H_SUCCESS);
383 			__cpu_die(cpu);
384 			break;
385 		}
386 		if (cpu == num_possible_cpus()) {
387 			pr_warn("Could not find cpu to offline with physical id 0x%x\n",
388 				thread);
389 		}
390 	}
391 	cpu_maps_update_done();
392 
393 out:
394 	return rc;
395 }
396 
397 static int dlpar_online_cpu(struct device_node *dn)
398 {
399 	int rc = 0;
400 	unsigned int cpu;
401 	int len, nthreads, i;
402 	const __be32 *intserv;
403 	u32 thread;
404 
405 	intserv = of_get_property(dn, "ibm,ppc-interrupt-server#s", &len);
406 	if (!intserv)
407 		return -EINVAL;
408 
409 	nthreads = len / sizeof(u32);
410 
411 	cpu_maps_update_begin();
412 	for (i = 0; i < nthreads; i++) {
413 		thread = be32_to_cpu(intserv[i]);
414 		for_each_present_cpu(cpu) {
415 			if (get_hard_smp_processor_id(cpu) != thread)
416 				continue;
417 			BUG_ON(get_cpu_current_state(cpu)
418 					!= CPU_STATE_OFFLINE);
419 			cpu_maps_update_done();
420 			timed_topology_update(1);
421 			find_and_online_cpu_nid(cpu);
422 			rc = device_online(get_cpu_device(cpu));
423 			if (rc) {
424 				dlpar_offline_cpu(dn);
425 				goto out;
426 			}
427 			cpu_maps_update_begin();
428 
429 			break;
430 		}
431 		if (cpu == num_possible_cpus())
432 			printk(KERN_WARNING "Could not find cpu to online "
433 			       "with physical id 0x%x\n", thread);
434 	}
435 	cpu_maps_update_done();
436 
437 out:
438 	return rc;
439 
440 }
441 
442 static bool dlpar_cpu_exists(struct device_node *parent, u32 drc_index)
443 {
444 	struct device_node *child = NULL;
445 	u32 my_drc_index;
446 	bool found;
447 	int rc;
448 
449 	/* Assume cpu doesn't exist */
450 	found = false;
451 
452 	for_each_child_of_node(parent, child) {
453 		rc = of_property_read_u32(child, "ibm,my-drc-index",
454 					  &my_drc_index);
455 		if (rc)
456 			continue;
457 
458 		if (my_drc_index == drc_index) {
459 			of_node_put(child);
460 			found = true;
461 			break;
462 		}
463 	}
464 
465 	return found;
466 }
467 
468 static bool drc_info_valid_index(struct device_node *parent, u32 drc_index)
469 {
470 	struct property *info;
471 	struct of_drc_info drc;
472 	const __be32 *value;
473 	u32 index;
474 	int count, i, j;
475 
476 	info = of_find_property(parent, "ibm,drc-info", NULL);
477 	if (!info)
478 		return false;
479 
480 	value = of_prop_next_u32(info, NULL, &count);
481 
482 	/* First value of ibm,drc-info is number of drc-info records */
483 	if (value)
484 		value++;
485 	else
486 		return false;
487 
488 	for (i = 0; i < count; i++) {
489 		if (of_read_drc_info_cell(&info, &value, &drc))
490 			return false;
491 
492 		if (strncmp(drc.drc_type, "CPU", 3))
493 			break;
494 
495 		if (drc_index > drc.last_drc_index)
496 			continue;
497 
498 		index = drc.drc_index_start;
499 		for (j = 0; j < drc.num_sequential_elems; j++) {
500 			if (drc_index == index)
501 				return true;
502 
503 			index += drc.sequential_inc;
504 		}
505 	}
506 
507 	return false;
508 }
509 
510 static bool valid_cpu_drc_index(struct device_node *parent, u32 drc_index)
511 {
512 	bool found = false;
513 	int rc, index;
514 
515 	if (of_find_property(parent, "ibm,drc-info", NULL))
516 		return drc_info_valid_index(parent, drc_index);
517 
518 	/* Note that the format of the ibm,drc-indexes array is
519 	 * the number of entries in the array followed by the array
520 	 * of drc values so we start looking at index = 1.
521 	 */
522 	index = 1;
523 	while (!found) {
524 		u32 drc;
525 
526 		rc = of_property_read_u32_index(parent, "ibm,drc-indexes",
527 						index++, &drc);
528 
529 		if (rc)
530 			break;
531 
532 		if (drc == drc_index)
533 			found = true;
534 	}
535 
536 	return found;
537 }
538 
539 static ssize_t dlpar_cpu_add(u32 drc_index)
540 {
541 	struct device_node *dn, *parent;
542 	int rc, saved_rc;
543 
544 	pr_debug("Attempting to add CPU, drc index: %x\n", drc_index);
545 
546 	parent = of_find_node_by_path("/cpus");
547 	if (!parent) {
548 		pr_warn("Failed to find CPU root node \"/cpus\"\n");
549 		return -ENODEV;
550 	}
551 
552 	if (dlpar_cpu_exists(parent, drc_index)) {
553 		of_node_put(parent);
554 		pr_warn("CPU with drc index %x already exists\n", drc_index);
555 		return -EINVAL;
556 	}
557 
558 	if (!valid_cpu_drc_index(parent, drc_index)) {
559 		of_node_put(parent);
560 		pr_warn("Cannot find CPU (drc index %x) to add.\n", drc_index);
561 		return -EINVAL;
562 	}
563 
564 	rc = dlpar_acquire_drc(drc_index);
565 	if (rc) {
566 		pr_warn("Failed to acquire DRC, rc: %d, drc index: %x\n",
567 			rc, drc_index);
568 		of_node_put(parent);
569 		return -EINVAL;
570 	}
571 
572 	dn = dlpar_configure_connector(cpu_to_be32(drc_index), parent);
573 	if (!dn) {
574 		pr_warn("Failed call to configure-connector, drc index: %x\n",
575 			drc_index);
576 		dlpar_release_drc(drc_index);
577 		of_node_put(parent);
578 		return -EINVAL;
579 	}
580 
581 	rc = dlpar_attach_node(dn, parent);
582 
583 	/* Regardless we are done with parent now */
584 	of_node_put(parent);
585 
586 	if (rc) {
587 		saved_rc = rc;
588 		pr_warn("Failed to attach node %pOFn, rc: %d, drc index: %x\n",
589 			dn, rc, drc_index);
590 
591 		rc = dlpar_release_drc(drc_index);
592 		if (!rc)
593 			dlpar_free_cc_nodes(dn);
594 
595 		return saved_rc;
596 	}
597 
598 	rc = dlpar_online_cpu(dn);
599 	if (rc) {
600 		saved_rc = rc;
601 		pr_warn("Failed to online cpu %pOFn, rc: %d, drc index: %x\n",
602 			dn, rc, drc_index);
603 
604 		rc = dlpar_detach_node(dn);
605 		if (!rc)
606 			dlpar_release_drc(drc_index);
607 
608 		return saved_rc;
609 	}
610 
611 	pr_debug("Successfully added CPU %pOFn, drc index: %x\n", dn,
612 		 drc_index);
613 	return rc;
614 }
615 
616 static ssize_t dlpar_cpu_remove(struct device_node *dn, u32 drc_index)
617 {
618 	int rc;
619 
620 	pr_debug("Attempting to remove CPU %pOFn, drc index: %x\n",
621 		 dn, drc_index);
622 
623 	rc = dlpar_offline_cpu(dn);
624 	if (rc) {
625 		pr_warn("Failed to offline CPU %pOFn, rc: %d\n", dn, rc);
626 		return -EINVAL;
627 	}
628 
629 	rc = dlpar_release_drc(drc_index);
630 	if (rc) {
631 		pr_warn("Failed to release drc (%x) for CPU %pOFn, rc: %d\n",
632 			drc_index, dn, rc);
633 		dlpar_online_cpu(dn);
634 		return rc;
635 	}
636 
637 	rc = dlpar_detach_node(dn);
638 	if (rc) {
639 		int saved_rc = rc;
640 
641 		pr_warn("Failed to detach CPU %pOFn, rc: %d", dn, rc);
642 
643 		rc = dlpar_acquire_drc(drc_index);
644 		if (!rc)
645 			dlpar_online_cpu(dn);
646 
647 		return saved_rc;
648 	}
649 
650 	pr_debug("Successfully removed CPU, drc index: %x\n", drc_index);
651 	return 0;
652 }
653 
654 static struct device_node *cpu_drc_index_to_dn(u32 drc_index)
655 {
656 	struct device_node *dn;
657 	u32 my_index;
658 	int rc;
659 
660 	for_each_node_by_type(dn, "cpu") {
661 		rc = of_property_read_u32(dn, "ibm,my-drc-index", &my_index);
662 		if (rc)
663 			continue;
664 
665 		if (my_index == drc_index)
666 			break;
667 	}
668 
669 	return dn;
670 }
671 
672 static int dlpar_cpu_remove_by_index(u32 drc_index)
673 {
674 	struct device_node *dn;
675 	int rc;
676 
677 	dn = cpu_drc_index_to_dn(drc_index);
678 	if (!dn) {
679 		pr_warn("Cannot find CPU (drc index %x) to remove\n",
680 			drc_index);
681 		return -ENODEV;
682 	}
683 
684 	rc = dlpar_cpu_remove(dn, drc_index);
685 	of_node_put(dn);
686 	return rc;
687 }
688 
689 static int find_dlpar_cpus_to_remove(u32 *cpu_drcs, int cpus_to_remove)
690 {
691 	struct device_node *dn;
692 	int cpus_found = 0;
693 	int rc;
694 
695 	/* We want to find cpus_to_remove + 1 CPUs to ensure we do not
696 	 * remove the last CPU.
697 	 */
698 	for_each_node_by_type(dn, "cpu") {
699 		cpus_found++;
700 
701 		if (cpus_found > cpus_to_remove) {
702 			of_node_put(dn);
703 			break;
704 		}
705 
706 		/* Note that cpus_found is always 1 ahead of the index
707 		 * into the cpu_drcs array, so we use cpus_found - 1
708 		 */
709 		rc = of_property_read_u32(dn, "ibm,my-drc-index",
710 					  &cpu_drcs[cpus_found - 1]);
711 		if (rc) {
712 			pr_warn("Error occurred getting drc-index for %pOFn\n",
713 				dn);
714 			of_node_put(dn);
715 			return -1;
716 		}
717 	}
718 
719 	if (cpus_found < cpus_to_remove) {
720 		pr_warn("Failed to find enough CPUs (%d of %d) to remove\n",
721 			cpus_found, cpus_to_remove);
722 	} else if (cpus_found == cpus_to_remove) {
723 		pr_warn("Cannot remove all CPUs\n");
724 	}
725 
726 	return cpus_found;
727 }
728 
729 static int dlpar_cpu_remove_by_count(u32 cpus_to_remove)
730 {
731 	u32 *cpu_drcs;
732 	int cpus_found;
733 	int cpus_removed = 0;
734 	int i, rc;
735 
736 	pr_debug("Attempting to hot-remove %d CPUs\n", cpus_to_remove);
737 
738 	cpu_drcs = kcalloc(cpus_to_remove, sizeof(*cpu_drcs), GFP_KERNEL);
739 	if (!cpu_drcs)
740 		return -EINVAL;
741 
742 	cpus_found = find_dlpar_cpus_to_remove(cpu_drcs, cpus_to_remove);
743 	if (cpus_found <= cpus_to_remove) {
744 		kfree(cpu_drcs);
745 		return -EINVAL;
746 	}
747 
748 	for (i = 0; i < cpus_to_remove; i++) {
749 		rc = dlpar_cpu_remove_by_index(cpu_drcs[i]);
750 		if (rc)
751 			break;
752 
753 		cpus_removed++;
754 	}
755 
756 	if (cpus_removed != cpus_to_remove) {
757 		pr_warn("CPU hot-remove failed, adding back removed CPUs\n");
758 
759 		for (i = 0; i < cpus_removed; i++)
760 			dlpar_cpu_add(cpu_drcs[i]);
761 
762 		rc = -EINVAL;
763 	} else {
764 		rc = 0;
765 	}
766 
767 	kfree(cpu_drcs);
768 	return rc;
769 }
770 
771 static int find_drc_info_cpus_to_add(struct device_node *cpus,
772 				     struct property *info,
773 				     u32 *cpu_drcs, u32 cpus_to_add)
774 {
775 	struct of_drc_info drc;
776 	const __be32 *value;
777 	u32 count, drc_index;
778 	int cpus_found = 0;
779 	int i, j;
780 
781 	if (!info)
782 		return -1;
783 
784 	value = of_prop_next_u32(info, NULL, &count);
785 	if (value)
786 		value++;
787 
788 	for (i = 0; i < count; i++) {
789 		of_read_drc_info_cell(&info, &value, &drc);
790 		if (strncmp(drc.drc_type, "CPU", 3))
791 			break;
792 
793 		drc_index = drc.drc_index_start;
794 		for (j = 0; j < drc.num_sequential_elems; j++) {
795 			if (dlpar_cpu_exists(cpus, drc_index))
796 				continue;
797 
798 			cpu_drcs[cpus_found++] = drc_index;
799 
800 			if (cpus_found == cpus_to_add)
801 				return cpus_found;
802 
803 			drc_index += drc.sequential_inc;
804 		}
805 	}
806 
807 	return cpus_found;
808 }
809 
810 static int find_drc_index_cpus_to_add(struct device_node *cpus,
811 				      u32 *cpu_drcs, u32 cpus_to_add)
812 {
813 	int cpus_found = 0;
814 	int index, rc;
815 	u32 drc_index;
816 
817 	/* Search the ibm,drc-indexes array for possible CPU drcs to
818 	 * add. Note that the format of the ibm,drc-indexes array is
819 	 * the number of entries in the array followed by the array
820 	 * of drc values so we start looking at index = 1.
821 	 */
822 	index = 1;
823 	while (cpus_found < cpus_to_add) {
824 		rc = of_property_read_u32_index(cpus, "ibm,drc-indexes",
825 						index++, &drc_index);
826 
827 		if (rc)
828 			break;
829 
830 		if (dlpar_cpu_exists(cpus, drc_index))
831 			continue;
832 
833 		cpu_drcs[cpus_found++] = drc_index;
834 	}
835 
836 	return cpus_found;
837 }
838 
839 static int dlpar_cpu_add_by_count(u32 cpus_to_add)
840 {
841 	struct device_node *parent;
842 	struct property *info;
843 	u32 *cpu_drcs;
844 	int cpus_added = 0;
845 	int cpus_found;
846 	int i, rc;
847 
848 	pr_debug("Attempting to hot-add %d CPUs\n", cpus_to_add);
849 
850 	cpu_drcs = kcalloc(cpus_to_add, sizeof(*cpu_drcs), GFP_KERNEL);
851 	if (!cpu_drcs)
852 		return -EINVAL;
853 
854 	parent = of_find_node_by_path("/cpus");
855 	if (!parent) {
856 		pr_warn("Could not find CPU root node in device tree\n");
857 		kfree(cpu_drcs);
858 		return -1;
859 	}
860 
861 	info = of_find_property(parent, "ibm,drc-info", NULL);
862 	if (info)
863 		cpus_found = find_drc_info_cpus_to_add(parent, info, cpu_drcs, cpus_to_add);
864 	else
865 		cpus_found = find_drc_index_cpus_to_add(parent, cpu_drcs, cpus_to_add);
866 
867 	of_node_put(parent);
868 
869 	if (cpus_found < cpus_to_add) {
870 		pr_warn("Failed to find enough CPUs (%d of %d) to add\n",
871 			cpus_found, cpus_to_add);
872 		kfree(cpu_drcs);
873 		return -EINVAL;
874 	}
875 
876 	for (i = 0; i < cpus_to_add; i++) {
877 		rc = dlpar_cpu_add(cpu_drcs[i]);
878 		if (rc)
879 			break;
880 
881 		cpus_added++;
882 	}
883 
884 	if (cpus_added < cpus_to_add) {
885 		pr_warn("CPU hot-add failed, removing any added CPUs\n");
886 
887 		for (i = 0; i < cpus_added; i++)
888 			dlpar_cpu_remove_by_index(cpu_drcs[i]);
889 
890 		rc = -EINVAL;
891 	} else {
892 		rc = 0;
893 	}
894 
895 	kfree(cpu_drcs);
896 	return rc;
897 }
898 
899 int dlpar_cpu_readd(int cpu)
900 {
901 	struct device_node *dn;
902 	struct device *dev;
903 	u32 drc_index;
904 	int rc;
905 
906 	dev = get_cpu_device(cpu);
907 	dn = dev->of_node;
908 
909 	rc = of_property_read_u32(dn, "ibm,my-drc-index", &drc_index);
910 
911 	rc = dlpar_cpu_remove_by_index(drc_index);
912 	if (!rc)
913 		rc = dlpar_cpu_add(drc_index);
914 
915 	return rc;
916 }
917 
918 int dlpar_cpu(struct pseries_hp_errorlog *hp_elog)
919 {
920 	u32 count, drc_index;
921 	int rc;
922 
923 	count = hp_elog->_drc_u.drc_count;
924 	drc_index = hp_elog->_drc_u.drc_index;
925 
926 	lock_device_hotplug();
927 
928 	switch (hp_elog->action) {
929 	case PSERIES_HP_ELOG_ACTION_REMOVE:
930 		if (hp_elog->id_type == PSERIES_HP_ELOG_ID_DRC_COUNT)
931 			rc = dlpar_cpu_remove_by_count(count);
932 		else if (hp_elog->id_type == PSERIES_HP_ELOG_ID_DRC_INDEX)
933 			rc = dlpar_cpu_remove_by_index(drc_index);
934 		else
935 			rc = -EINVAL;
936 		break;
937 	case PSERIES_HP_ELOG_ACTION_ADD:
938 		if (hp_elog->id_type == PSERIES_HP_ELOG_ID_DRC_COUNT)
939 			rc = dlpar_cpu_add_by_count(count);
940 		else if (hp_elog->id_type == PSERIES_HP_ELOG_ID_DRC_INDEX)
941 			rc = dlpar_cpu_add(drc_index);
942 		else
943 			rc = -EINVAL;
944 		break;
945 	default:
946 		pr_err("Invalid action (%d) specified\n", hp_elog->action);
947 		rc = -EINVAL;
948 		break;
949 	}
950 
951 	unlock_device_hotplug();
952 	return rc;
953 }
954 
955 #ifdef CONFIG_ARCH_CPU_PROBE_RELEASE
956 
957 static ssize_t dlpar_cpu_probe(const char *buf, size_t count)
958 {
959 	u32 drc_index;
960 	int rc;
961 
962 	rc = kstrtou32(buf, 0, &drc_index);
963 	if (rc)
964 		return -EINVAL;
965 
966 	rc = dlpar_cpu_add(drc_index);
967 
968 	return rc ? rc : count;
969 }
970 
971 static ssize_t dlpar_cpu_release(const char *buf, size_t count)
972 {
973 	struct device_node *dn;
974 	u32 drc_index;
975 	int rc;
976 
977 	dn = of_find_node_by_path(buf);
978 	if (!dn)
979 		return -EINVAL;
980 
981 	rc = of_property_read_u32(dn, "ibm,my-drc-index", &drc_index);
982 	if (rc) {
983 		of_node_put(dn);
984 		return -EINVAL;
985 	}
986 
987 	rc = dlpar_cpu_remove(dn, drc_index);
988 	of_node_put(dn);
989 
990 	return rc ? rc : count;
991 }
992 
993 #endif /* CONFIG_ARCH_CPU_PROBE_RELEASE */
994 
995 static int pseries_smp_notifier(struct notifier_block *nb,
996 				unsigned long action, void *data)
997 {
998 	struct of_reconfig_data *rd = data;
999 	int err = 0;
1000 
1001 	switch (action) {
1002 	case OF_RECONFIG_ATTACH_NODE:
1003 		err = pseries_add_processor(rd->dn);
1004 		break;
1005 	case OF_RECONFIG_DETACH_NODE:
1006 		pseries_remove_processor(rd->dn);
1007 		break;
1008 	}
1009 	return notifier_from_errno(err);
1010 }
1011 
1012 static struct notifier_block pseries_smp_nb = {
1013 	.notifier_call = pseries_smp_notifier,
1014 };
1015 
1016 #define MAX_CEDE_LATENCY_LEVELS		4
1017 #define	CEDE_LATENCY_PARAM_LENGTH	10
1018 #define CEDE_LATENCY_PARAM_MAX_LENGTH	\
1019 	(MAX_CEDE_LATENCY_LEVELS * CEDE_LATENCY_PARAM_LENGTH * sizeof(char))
1020 #define CEDE_LATENCY_TOKEN		45
1021 
1022 static char cede_parameters[CEDE_LATENCY_PARAM_MAX_LENGTH];
1023 
1024 static int parse_cede_parameters(void)
1025 {
1026 	memset(cede_parameters, 0, CEDE_LATENCY_PARAM_MAX_LENGTH);
1027 	return rtas_call(rtas_token("ibm,get-system-parameter"), 3, 1,
1028 			 NULL,
1029 			 CEDE_LATENCY_TOKEN,
1030 			 __pa(cede_parameters),
1031 			 CEDE_LATENCY_PARAM_MAX_LENGTH);
1032 }
1033 
1034 static int __init pseries_cpu_hotplug_init(void)
1035 {
1036 	int cpu;
1037 	int qcss_tok;
1038 
1039 #ifdef CONFIG_ARCH_CPU_PROBE_RELEASE
1040 	ppc_md.cpu_probe = dlpar_cpu_probe;
1041 	ppc_md.cpu_release = dlpar_cpu_release;
1042 #endif /* CONFIG_ARCH_CPU_PROBE_RELEASE */
1043 
1044 	rtas_stop_self_token = rtas_token("stop-self");
1045 	qcss_tok = rtas_token("query-cpu-stopped-state");
1046 
1047 	if (rtas_stop_self_token == RTAS_UNKNOWN_SERVICE ||
1048 			qcss_tok == RTAS_UNKNOWN_SERVICE) {
1049 		printk(KERN_INFO "CPU Hotplug not supported by firmware "
1050 				"- disabling.\n");
1051 		return 0;
1052 	}
1053 
1054 	ppc_md.cpu_die = pseries_mach_cpu_die;
1055 	smp_ops->cpu_disable = pseries_cpu_disable;
1056 	smp_ops->cpu_die = pseries_cpu_die;
1057 
1058 	/* Processors can be added/removed only on LPAR */
1059 	if (firmware_has_feature(FW_FEATURE_LPAR)) {
1060 		of_reconfig_notifier_register(&pseries_smp_nb);
1061 		cpu_maps_update_begin();
1062 		if (cede_offline_enabled && parse_cede_parameters() == 0) {
1063 			default_offline_state = CPU_STATE_INACTIVE;
1064 			for_each_online_cpu(cpu)
1065 				set_default_offline_state(cpu);
1066 		}
1067 		cpu_maps_update_done();
1068 	}
1069 
1070 	return 0;
1071 }
1072 machine_arch_initcall(pseries, pseries_cpu_hotplug_init);
1073