xref: /openbmc/linux/arch/powerpc/kernel/smp.c (revision bdeeed09)
1 /*
2  * SMP support for ppc.
3  *
4  * Written by Cort Dougan (cort@cs.nmt.edu) borrowing a great
5  * deal of code from the sparc and intel versions.
6  *
7  * Copyright (C) 1999 Cort Dougan <cort@cs.nmt.edu>
8  *
9  * PowerPC-64 Support added by Dave Engebretsen, Peter Bergner, and
10  * Mike Corrigan {engebret|bergner|mikec}@us.ibm.com
11  *
12  *      This program is free software; you can redistribute it and/or
13  *      modify it under the terms of the GNU General Public License
14  *      as published by the Free Software Foundation; either version
15  *      2 of the License, or (at your option) any later version.
16  */
17 
18 #undef DEBUG
19 
20 #include <linux/kernel.h>
21 #include <linux/export.h>
22 #include <linux/sched/mm.h>
23 #include <linux/sched/topology.h>
24 #include <linux/smp.h>
25 #include <linux/interrupt.h>
26 #include <linux/delay.h>
27 #include <linux/init.h>
28 #include <linux/spinlock.h>
29 #include <linux/cache.h>
30 #include <linux/err.h>
31 #include <linux/device.h>
32 #include <linux/cpu.h>
33 #include <linux/notifier.h>
34 #include <linux/topology.h>
35 #include <linux/profile.h>
36 #include <linux/processor.h>
37 
38 #include <asm/ptrace.h>
39 #include <linux/atomic.h>
40 #include <asm/irq.h>
41 #include <asm/hw_irq.h>
42 #include <asm/kvm_ppc.h>
43 #include <asm/dbell.h>
44 #include <asm/page.h>
45 #include <asm/pgtable.h>
46 #include <asm/prom.h>
47 #include <asm/smp.h>
48 #include <asm/time.h>
49 #include <asm/machdep.h>
50 #include <asm/cputhreads.h>
51 #include <asm/cputable.h>
52 #include <asm/mpic.h>
53 #include <asm/vdso_datapage.h>
54 #ifdef CONFIG_PPC64
55 #include <asm/paca.h>
56 #endif
57 #include <asm/vdso.h>
58 #include <asm/debug.h>
59 #include <asm/kexec.h>
60 #include <asm/asm-prototypes.h>
61 #include <asm/cpu_has_feature.h>
62 
63 #ifdef DEBUG
64 #include <asm/udbg.h>
65 #define DBG(fmt...) udbg_printf(fmt)
66 #else
67 #define DBG(fmt...)
68 #endif
69 
70 #ifdef CONFIG_HOTPLUG_CPU
71 /* State of each CPU during hotplug phases */
72 static DEFINE_PER_CPU(int, cpu_state) = { 0 };
73 #endif
74 
75 struct thread_info *secondary_ti;
76 
77 DEFINE_PER_CPU(cpumask_var_t, cpu_sibling_map);
78 DEFINE_PER_CPU(cpumask_var_t, cpu_l2_cache_map);
79 DEFINE_PER_CPU(cpumask_var_t, cpu_core_map);
80 
81 EXPORT_PER_CPU_SYMBOL(cpu_sibling_map);
82 EXPORT_PER_CPU_SYMBOL(cpu_l2_cache_map);
83 EXPORT_PER_CPU_SYMBOL(cpu_core_map);
84 
85 /* SMP operations for this machine */
86 struct smp_ops_t *smp_ops;
87 
88 /* Can't be static due to PowerMac hackery */
89 volatile unsigned int cpu_callin_map[NR_CPUS];
90 
91 int smt_enabled_at_boot = 1;
92 
93 /*
94  * Returns 1 if the specified cpu should be brought up during boot.
95  * Used to inhibit booting threads if they've been disabled or
96  * limited on the command line
97  */
98 int smp_generic_cpu_bootable(unsigned int nr)
99 {
100 	/* Special case - we inhibit secondary thread startup
101 	 * during boot if the user requests it.
102 	 */
103 	if (system_state < SYSTEM_RUNNING && cpu_has_feature(CPU_FTR_SMT)) {
104 		if (!smt_enabled_at_boot && cpu_thread_in_core(nr) != 0)
105 			return 0;
106 		if (smt_enabled_at_boot
107 		    && cpu_thread_in_core(nr) >= smt_enabled_at_boot)
108 			return 0;
109 	}
110 
111 	return 1;
112 }
113 
114 
115 #ifdef CONFIG_PPC64
116 int smp_generic_kick_cpu(int nr)
117 {
118 	if (nr < 0 || nr >= nr_cpu_ids)
119 		return -EINVAL;
120 
121 	/*
122 	 * The processor is currently spinning, waiting for the
123 	 * cpu_start field to become non-zero After we set cpu_start,
124 	 * the processor will continue on to secondary_start
125 	 */
126 	if (!paca_ptrs[nr]->cpu_start) {
127 		paca_ptrs[nr]->cpu_start = 1;
128 		smp_mb();
129 		return 0;
130 	}
131 
132 #ifdef CONFIG_HOTPLUG_CPU
133 	/*
134 	 * Ok it's not there, so it might be soft-unplugged, let's
135 	 * try to bring it back
136 	 */
137 	generic_set_cpu_up(nr);
138 	smp_wmb();
139 	smp_send_reschedule(nr);
140 #endif /* CONFIG_HOTPLUG_CPU */
141 
142 	return 0;
143 }
144 #endif /* CONFIG_PPC64 */
145 
146 static irqreturn_t call_function_action(int irq, void *data)
147 {
148 	generic_smp_call_function_interrupt();
149 	return IRQ_HANDLED;
150 }
151 
152 static irqreturn_t reschedule_action(int irq, void *data)
153 {
154 	scheduler_ipi();
155 	return IRQ_HANDLED;
156 }
157 
158 static irqreturn_t tick_broadcast_ipi_action(int irq, void *data)
159 {
160 	tick_broadcast_ipi_handler();
161 	return IRQ_HANDLED;
162 }
163 
164 #ifdef CONFIG_NMI_IPI
165 static irqreturn_t nmi_ipi_action(int irq, void *data)
166 {
167 	smp_handle_nmi_ipi(get_irq_regs());
168 	return IRQ_HANDLED;
169 }
170 #endif
171 
172 static irq_handler_t smp_ipi_action[] = {
173 	[PPC_MSG_CALL_FUNCTION] =  call_function_action,
174 	[PPC_MSG_RESCHEDULE] = reschedule_action,
175 	[PPC_MSG_TICK_BROADCAST] = tick_broadcast_ipi_action,
176 #ifdef CONFIG_NMI_IPI
177 	[PPC_MSG_NMI_IPI] = nmi_ipi_action,
178 #endif
179 };
180 
181 /*
182  * The NMI IPI is a fallback and not truly non-maskable. It is simpler
183  * than going through the call function infrastructure, and strongly
184  * serialized, so it is more appropriate for debugging.
185  */
186 const char *smp_ipi_name[] = {
187 	[PPC_MSG_CALL_FUNCTION] =  "ipi call function",
188 	[PPC_MSG_RESCHEDULE] = "ipi reschedule",
189 	[PPC_MSG_TICK_BROADCAST] = "ipi tick-broadcast",
190 	[PPC_MSG_NMI_IPI] = "nmi ipi",
191 };
192 
193 /* optional function to request ipi, for controllers with >= 4 ipis */
194 int smp_request_message_ipi(int virq, int msg)
195 {
196 	int err;
197 
198 	if (msg < 0 || msg > PPC_MSG_NMI_IPI)
199 		return -EINVAL;
200 #ifndef CONFIG_NMI_IPI
201 	if (msg == PPC_MSG_NMI_IPI)
202 		return 1;
203 #endif
204 
205 	err = request_irq(virq, smp_ipi_action[msg],
206 			  IRQF_PERCPU | IRQF_NO_THREAD | IRQF_NO_SUSPEND,
207 			  smp_ipi_name[msg], NULL);
208 	WARN(err < 0, "unable to request_irq %d for %s (rc %d)\n",
209 		virq, smp_ipi_name[msg], err);
210 
211 	return err;
212 }
213 
214 #ifdef CONFIG_PPC_SMP_MUXED_IPI
215 struct cpu_messages {
216 	long messages;			/* current messages */
217 };
218 static DEFINE_PER_CPU_SHARED_ALIGNED(struct cpu_messages, ipi_message);
219 
220 void smp_muxed_ipi_set_message(int cpu, int msg)
221 {
222 	struct cpu_messages *info = &per_cpu(ipi_message, cpu);
223 	char *message = (char *)&info->messages;
224 
225 	/*
226 	 * Order previous accesses before accesses in the IPI handler.
227 	 */
228 	smp_mb();
229 	message[msg] = 1;
230 }
231 
232 void smp_muxed_ipi_message_pass(int cpu, int msg)
233 {
234 	smp_muxed_ipi_set_message(cpu, msg);
235 
236 	/*
237 	 * cause_ipi functions are required to include a full barrier
238 	 * before doing whatever causes the IPI.
239 	 */
240 	smp_ops->cause_ipi(cpu);
241 }
242 
243 #ifdef __BIG_ENDIAN__
244 #define IPI_MESSAGE(A) (1uL << ((BITS_PER_LONG - 8) - 8 * (A)))
245 #else
246 #define IPI_MESSAGE(A) (1uL << (8 * (A)))
247 #endif
248 
249 irqreturn_t smp_ipi_demux(void)
250 {
251 	mb();	/* order any irq clear */
252 
253 	return smp_ipi_demux_relaxed();
254 }
255 
256 /* sync-free variant. Callers should ensure synchronization */
257 irqreturn_t smp_ipi_demux_relaxed(void)
258 {
259 	struct cpu_messages *info;
260 	unsigned long all;
261 
262 	info = this_cpu_ptr(&ipi_message);
263 	do {
264 		all = xchg(&info->messages, 0);
265 #if defined(CONFIG_KVM_XICS) && defined(CONFIG_KVM_BOOK3S_HV_POSSIBLE)
266 		/*
267 		 * Must check for PPC_MSG_RM_HOST_ACTION messages
268 		 * before PPC_MSG_CALL_FUNCTION messages because when
269 		 * a VM is destroyed, we call kick_all_cpus_sync()
270 		 * to ensure that any pending PPC_MSG_RM_HOST_ACTION
271 		 * messages have completed before we free any VCPUs.
272 		 */
273 		if (all & IPI_MESSAGE(PPC_MSG_RM_HOST_ACTION))
274 			kvmppc_xics_ipi_action();
275 #endif
276 		if (all & IPI_MESSAGE(PPC_MSG_CALL_FUNCTION))
277 			generic_smp_call_function_interrupt();
278 		if (all & IPI_MESSAGE(PPC_MSG_RESCHEDULE))
279 			scheduler_ipi();
280 		if (all & IPI_MESSAGE(PPC_MSG_TICK_BROADCAST))
281 			tick_broadcast_ipi_handler();
282 #ifdef CONFIG_NMI_IPI
283 		if (all & IPI_MESSAGE(PPC_MSG_NMI_IPI))
284 			nmi_ipi_action(0, NULL);
285 #endif
286 	} while (info->messages);
287 
288 	return IRQ_HANDLED;
289 }
290 #endif /* CONFIG_PPC_SMP_MUXED_IPI */
291 
292 static inline void do_message_pass(int cpu, int msg)
293 {
294 	if (smp_ops->message_pass)
295 		smp_ops->message_pass(cpu, msg);
296 #ifdef CONFIG_PPC_SMP_MUXED_IPI
297 	else
298 		smp_muxed_ipi_message_pass(cpu, msg);
299 #endif
300 }
301 
302 void smp_send_reschedule(int cpu)
303 {
304 	if (likely(smp_ops))
305 		do_message_pass(cpu, PPC_MSG_RESCHEDULE);
306 }
307 EXPORT_SYMBOL_GPL(smp_send_reschedule);
308 
309 void arch_send_call_function_single_ipi(int cpu)
310 {
311 	do_message_pass(cpu, PPC_MSG_CALL_FUNCTION);
312 }
313 
314 void arch_send_call_function_ipi_mask(const struct cpumask *mask)
315 {
316 	unsigned int cpu;
317 
318 	for_each_cpu(cpu, mask)
319 		do_message_pass(cpu, PPC_MSG_CALL_FUNCTION);
320 }
321 
322 #ifdef CONFIG_NMI_IPI
323 
324 /*
325  * "NMI IPI" system.
326  *
327  * NMI IPIs may not be recoverable, so should not be used as ongoing part of
328  * a running system. They can be used for crash, debug, halt/reboot, etc.
329  *
330  * NMI IPIs are globally single threaded. No more than one in progress at
331  * any time.
332  *
333  * The IPI call waits with interrupts disabled until all targets enter the
334  * NMI handler, then the call returns.
335  *
336  * No new NMI can be initiated until targets exit the handler.
337  *
338  * The IPI call may time out without all targets entering the NMI handler.
339  * In that case, there is some logic to recover (and ignore subsequent
340  * NMI interrupts that may eventually be raised), but the platform interrupt
341  * handler may not be able to distinguish this from other exception causes,
342  * which may cause a crash.
343  */
344 
345 static atomic_t __nmi_ipi_lock = ATOMIC_INIT(0);
346 static struct cpumask nmi_ipi_pending_mask;
347 static int nmi_ipi_busy_count = 0;
348 static void (*nmi_ipi_function)(struct pt_regs *) = NULL;
349 
350 static void nmi_ipi_lock_start(unsigned long *flags)
351 {
352 	raw_local_irq_save(*flags);
353 	hard_irq_disable();
354 	while (atomic_cmpxchg(&__nmi_ipi_lock, 0, 1) == 1) {
355 		raw_local_irq_restore(*flags);
356 		spin_until_cond(atomic_read(&__nmi_ipi_lock) == 0);
357 		raw_local_irq_save(*flags);
358 		hard_irq_disable();
359 	}
360 }
361 
362 static void nmi_ipi_lock(void)
363 {
364 	while (atomic_cmpxchg(&__nmi_ipi_lock, 0, 1) == 1)
365 		spin_until_cond(atomic_read(&__nmi_ipi_lock) == 0);
366 }
367 
368 static void nmi_ipi_unlock(void)
369 {
370 	smp_mb();
371 	WARN_ON(atomic_read(&__nmi_ipi_lock) != 1);
372 	atomic_set(&__nmi_ipi_lock, 0);
373 }
374 
375 static void nmi_ipi_unlock_end(unsigned long *flags)
376 {
377 	nmi_ipi_unlock();
378 	raw_local_irq_restore(*flags);
379 }
380 
381 /*
382  * Platform NMI handler calls this to ack
383  */
384 int smp_handle_nmi_ipi(struct pt_regs *regs)
385 {
386 	void (*fn)(struct pt_regs *);
387 	unsigned long flags;
388 	int me = raw_smp_processor_id();
389 	int ret = 0;
390 
391 	/*
392 	 * Unexpected NMIs are possible here because the interrupt may not
393 	 * be able to distinguish NMI IPIs from other types of NMIs, or
394 	 * because the caller may have timed out.
395 	 */
396 	nmi_ipi_lock_start(&flags);
397 	if (!nmi_ipi_busy_count)
398 		goto out;
399 	if (!cpumask_test_cpu(me, &nmi_ipi_pending_mask))
400 		goto out;
401 
402 	fn = nmi_ipi_function;
403 	if (!fn)
404 		goto out;
405 
406 	cpumask_clear_cpu(me, &nmi_ipi_pending_mask);
407 	nmi_ipi_busy_count++;
408 	nmi_ipi_unlock();
409 
410 	ret = 1;
411 
412 	fn(regs);
413 
414 	nmi_ipi_lock();
415 	nmi_ipi_busy_count--;
416 out:
417 	nmi_ipi_unlock_end(&flags);
418 
419 	return ret;
420 }
421 
422 static void do_smp_send_nmi_ipi(int cpu)
423 {
424 	if (smp_ops->cause_nmi_ipi && smp_ops->cause_nmi_ipi(cpu))
425 		return;
426 
427 	if (cpu >= 0) {
428 		do_message_pass(cpu, PPC_MSG_NMI_IPI);
429 	} else {
430 		int c;
431 
432 		for_each_online_cpu(c) {
433 			if (c == raw_smp_processor_id())
434 				continue;
435 			do_message_pass(c, PPC_MSG_NMI_IPI);
436 		}
437 	}
438 }
439 
440 void smp_flush_nmi_ipi(u64 delay_us)
441 {
442 	unsigned long flags;
443 
444 	nmi_ipi_lock_start(&flags);
445 	while (nmi_ipi_busy_count) {
446 		nmi_ipi_unlock_end(&flags);
447 		udelay(1);
448 		if (delay_us) {
449 			delay_us--;
450 			if (!delay_us)
451 				return;
452 		}
453 		nmi_ipi_lock_start(&flags);
454 	}
455 	nmi_ipi_unlock_end(&flags);
456 }
457 
458 /*
459  * - cpu is the target CPU (must not be this CPU), or NMI_IPI_ALL_OTHERS.
460  * - fn is the target callback function.
461  * - delay_us > 0 is the delay before giving up waiting for targets to
462  *   enter the handler, == 0 specifies indefinite delay.
463  */
464 int smp_send_nmi_ipi(int cpu, void (*fn)(struct pt_regs *), u64 delay_us)
465 {
466 	unsigned long flags;
467 	int me = raw_smp_processor_id();
468 	int ret = 1;
469 
470 	BUG_ON(cpu == me);
471 	BUG_ON(cpu < 0 && cpu != NMI_IPI_ALL_OTHERS);
472 
473 	if (unlikely(!smp_ops))
474 		return 0;
475 
476 	/* Take the nmi_ipi_busy count/lock with interrupts hard disabled */
477 	nmi_ipi_lock_start(&flags);
478 	while (nmi_ipi_busy_count) {
479 		nmi_ipi_unlock_end(&flags);
480 		spin_until_cond(nmi_ipi_busy_count == 0);
481 		nmi_ipi_lock_start(&flags);
482 	}
483 
484 	nmi_ipi_function = fn;
485 
486 	if (cpu < 0) {
487 		/* ALL_OTHERS */
488 		cpumask_copy(&nmi_ipi_pending_mask, cpu_online_mask);
489 		cpumask_clear_cpu(me, &nmi_ipi_pending_mask);
490 	} else {
491 		/* cpumask starts clear */
492 		cpumask_set_cpu(cpu, &nmi_ipi_pending_mask);
493 	}
494 	nmi_ipi_busy_count++;
495 	nmi_ipi_unlock();
496 
497 	do_smp_send_nmi_ipi(cpu);
498 
499 	while (!cpumask_empty(&nmi_ipi_pending_mask)) {
500 		udelay(1);
501 		if (delay_us) {
502 			delay_us--;
503 			if (!delay_us)
504 				break;
505 		}
506 	}
507 
508 	nmi_ipi_lock();
509 	if (!cpumask_empty(&nmi_ipi_pending_mask)) {
510 		/* Could not gather all CPUs */
511 		ret = 0;
512 		cpumask_clear(&nmi_ipi_pending_mask);
513 	}
514 	nmi_ipi_busy_count--;
515 	nmi_ipi_unlock_end(&flags);
516 
517 	return ret;
518 }
519 #endif /* CONFIG_NMI_IPI */
520 
521 #ifdef CONFIG_GENERIC_CLOCKEVENTS_BROADCAST
522 void tick_broadcast(const struct cpumask *mask)
523 {
524 	unsigned int cpu;
525 
526 	for_each_cpu(cpu, mask)
527 		do_message_pass(cpu, PPC_MSG_TICK_BROADCAST);
528 }
529 #endif
530 
531 #ifdef CONFIG_DEBUGGER
532 void debugger_ipi_callback(struct pt_regs *regs)
533 {
534 	debugger_ipi(regs);
535 }
536 
537 void smp_send_debugger_break(void)
538 {
539 	smp_send_nmi_ipi(NMI_IPI_ALL_OTHERS, debugger_ipi_callback, 1000000);
540 }
541 #endif
542 
543 #ifdef CONFIG_KEXEC_CORE
544 void crash_send_ipi(void (*crash_ipi_callback)(struct pt_regs *))
545 {
546 	int cpu;
547 
548 	smp_send_nmi_ipi(NMI_IPI_ALL_OTHERS, crash_ipi_callback, 1000000);
549 	if (kdump_in_progress() && crash_wake_offline) {
550 		for_each_present_cpu(cpu) {
551 			if (cpu_online(cpu))
552 				continue;
553 			/*
554 			 * crash_ipi_callback will wait for
555 			 * all cpus, including offline CPUs.
556 			 * We don't care about nmi_ipi_function.
557 			 * Offline cpus will jump straight into
558 			 * crash_ipi_callback, we can skip the
559 			 * entire NMI dance and waiting for
560 			 * cpus to clear pending mask, etc.
561 			 */
562 			do_smp_send_nmi_ipi(cpu);
563 		}
564 	}
565 }
566 #endif
567 
568 #ifdef CONFIG_NMI_IPI
569 static void nmi_stop_this_cpu(struct pt_regs *regs)
570 {
571 	/*
572 	 * This is a special case because it never returns, so the NMI IPI
573 	 * handling would never mark it as done, which makes any later
574 	 * smp_send_nmi_ipi() call spin forever. Mark it done now.
575 	 *
576 	 * IRQs are already hard disabled by the smp_handle_nmi_ipi.
577 	 */
578 	nmi_ipi_lock();
579 	nmi_ipi_busy_count--;
580 	nmi_ipi_unlock();
581 
582 	/* Remove this CPU */
583 	set_cpu_online(smp_processor_id(), false);
584 
585 	spin_begin();
586 	while (1)
587 		spin_cpu_relax();
588 }
589 
590 void smp_send_stop(void)
591 {
592 	smp_send_nmi_ipi(NMI_IPI_ALL_OTHERS, nmi_stop_this_cpu, 1000000);
593 }
594 
595 #else /* CONFIG_NMI_IPI */
596 
597 static void stop_this_cpu(void *dummy)
598 {
599 	/* Remove this CPU */
600 	set_cpu_online(smp_processor_id(), false);
601 
602 	hard_irq_disable();
603 	spin_begin();
604 	while (1)
605 		spin_cpu_relax();
606 }
607 
608 void smp_send_stop(void)
609 {
610 	static bool stopped = false;
611 
612 	/*
613 	 * Prevent waiting on csd lock from a previous smp_send_stop.
614 	 * This is racy, but in general callers try to do the right
615 	 * thing and only fire off one smp_send_stop (e.g., see
616 	 * kernel/panic.c)
617 	 */
618 	if (stopped)
619 		return;
620 
621 	stopped = true;
622 
623 	smp_call_function(stop_this_cpu, NULL, 0);
624 }
625 #endif /* CONFIG_NMI_IPI */
626 
627 struct thread_info *current_set[NR_CPUS];
628 
629 static void smp_store_cpu_info(int id)
630 {
631 	per_cpu(cpu_pvr, id) = mfspr(SPRN_PVR);
632 #ifdef CONFIG_PPC_FSL_BOOK3E
633 	per_cpu(next_tlbcam_idx, id)
634 		= (mfspr(SPRN_TLB1CFG) & TLBnCFG_N_ENTRY) - 1;
635 #endif
636 }
637 
638 /*
639  * Relationships between CPUs are maintained in a set of per-cpu cpumasks so
640  * rather than just passing around the cpumask we pass around a function that
641  * returns the that cpumask for the given CPU.
642  */
643 static void set_cpus_related(int i, int j, struct cpumask *(*get_cpumask)(int))
644 {
645 	cpumask_set_cpu(i, get_cpumask(j));
646 	cpumask_set_cpu(j, get_cpumask(i));
647 }
648 
649 #ifdef CONFIG_HOTPLUG_CPU
650 static void set_cpus_unrelated(int i, int j,
651 		struct cpumask *(*get_cpumask)(int))
652 {
653 	cpumask_clear_cpu(i, get_cpumask(j));
654 	cpumask_clear_cpu(j, get_cpumask(i));
655 }
656 #endif
657 
658 void __init smp_prepare_cpus(unsigned int max_cpus)
659 {
660 	unsigned int cpu;
661 
662 	DBG("smp_prepare_cpus\n");
663 
664 	/*
665 	 * setup_cpu may need to be called on the boot cpu. We havent
666 	 * spun any cpus up but lets be paranoid.
667 	 */
668 	BUG_ON(boot_cpuid != smp_processor_id());
669 
670 	/* Fixup boot cpu */
671 	smp_store_cpu_info(boot_cpuid);
672 	cpu_callin_map[boot_cpuid] = 1;
673 
674 	for_each_possible_cpu(cpu) {
675 		zalloc_cpumask_var_node(&per_cpu(cpu_sibling_map, cpu),
676 					GFP_KERNEL, cpu_to_node(cpu));
677 		zalloc_cpumask_var_node(&per_cpu(cpu_l2_cache_map, cpu),
678 					GFP_KERNEL, cpu_to_node(cpu));
679 		zalloc_cpumask_var_node(&per_cpu(cpu_core_map, cpu),
680 					GFP_KERNEL, cpu_to_node(cpu));
681 		/*
682 		 * numa_node_id() works after this.
683 		 */
684 		if (cpu_present(cpu)) {
685 			set_cpu_numa_node(cpu, numa_cpu_lookup_table[cpu]);
686 			set_cpu_numa_mem(cpu,
687 				local_memory_node(numa_cpu_lookup_table[cpu]));
688 		}
689 	}
690 
691 	/* Init the cpumasks so the boot CPU is related to itself */
692 	cpumask_set_cpu(boot_cpuid, cpu_sibling_mask(boot_cpuid));
693 	cpumask_set_cpu(boot_cpuid, cpu_l2_cache_mask(boot_cpuid));
694 	cpumask_set_cpu(boot_cpuid, cpu_core_mask(boot_cpuid));
695 
696 	if (smp_ops && smp_ops->probe)
697 		smp_ops->probe();
698 }
699 
700 void smp_prepare_boot_cpu(void)
701 {
702 	BUG_ON(smp_processor_id() != boot_cpuid);
703 #ifdef CONFIG_PPC64
704 	paca_ptrs[boot_cpuid]->__current = current;
705 #endif
706 	set_numa_node(numa_cpu_lookup_table[boot_cpuid]);
707 	current_set[boot_cpuid] = task_thread_info(current);
708 }
709 
710 #ifdef CONFIG_HOTPLUG_CPU
711 
712 int generic_cpu_disable(void)
713 {
714 	unsigned int cpu = smp_processor_id();
715 
716 	if (cpu == boot_cpuid)
717 		return -EBUSY;
718 
719 	set_cpu_online(cpu, false);
720 #ifdef CONFIG_PPC64
721 	vdso_data->processorCount--;
722 #endif
723 	/* Update affinity of all IRQs previously aimed at this CPU */
724 	irq_migrate_all_off_this_cpu();
725 
726 	/*
727 	 * Depending on the details of the interrupt controller, it's possible
728 	 * that one of the interrupts we just migrated away from this CPU is
729 	 * actually already pending on this CPU. If we leave it in that state
730 	 * the interrupt will never be EOI'ed, and will never fire again. So
731 	 * temporarily enable interrupts here, to allow any pending interrupt to
732 	 * be received (and EOI'ed), before we take this CPU offline.
733 	 */
734 	local_irq_enable();
735 	mdelay(1);
736 	local_irq_disable();
737 
738 	return 0;
739 }
740 
741 void generic_cpu_die(unsigned int cpu)
742 {
743 	int i;
744 
745 	for (i = 0; i < 100; i++) {
746 		smp_rmb();
747 		if (is_cpu_dead(cpu))
748 			return;
749 		msleep(100);
750 	}
751 	printk(KERN_ERR "CPU%d didn't die...\n", cpu);
752 }
753 
754 void generic_set_cpu_dead(unsigned int cpu)
755 {
756 	per_cpu(cpu_state, cpu) = CPU_DEAD;
757 }
758 
759 /*
760  * The cpu_state should be set to CPU_UP_PREPARE in kick_cpu(), otherwise
761  * the cpu_state is always CPU_DEAD after calling generic_set_cpu_dead(),
762  * which makes the delay in generic_cpu_die() not happen.
763  */
764 void generic_set_cpu_up(unsigned int cpu)
765 {
766 	per_cpu(cpu_state, cpu) = CPU_UP_PREPARE;
767 }
768 
769 int generic_check_cpu_restart(unsigned int cpu)
770 {
771 	return per_cpu(cpu_state, cpu) == CPU_UP_PREPARE;
772 }
773 
774 int is_cpu_dead(unsigned int cpu)
775 {
776 	return per_cpu(cpu_state, cpu) == CPU_DEAD;
777 }
778 
779 static bool secondaries_inhibited(void)
780 {
781 	return kvm_hv_mode_active();
782 }
783 
784 #else /* HOTPLUG_CPU */
785 
786 #define secondaries_inhibited()		0
787 
788 #endif
789 
790 static void cpu_idle_thread_init(unsigned int cpu, struct task_struct *idle)
791 {
792 	struct thread_info *ti = task_thread_info(idle);
793 
794 #ifdef CONFIG_PPC64
795 	paca_ptrs[cpu]->__current = idle;
796 	paca_ptrs[cpu]->kstack = (unsigned long)ti + THREAD_SIZE - STACK_FRAME_OVERHEAD;
797 #endif
798 	ti->cpu = cpu;
799 	secondary_ti = current_set[cpu] = ti;
800 }
801 
802 int __cpu_up(unsigned int cpu, struct task_struct *tidle)
803 {
804 	int rc, c;
805 
806 	/*
807 	 * Don't allow secondary threads to come online if inhibited
808 	 */
809 	if (threads_per_core > 1 && secondaries_inhibited() &&
810 	    cpu_thread_in_subcore(cpu))
811 		return -EBUSY;
812 
813 	if (smp_ops == NULL ||
814 	    (smp_ops->cpu_bootable && !smp_ops->cpu_bootable(cpu)))
815 		return -EINVAL;
816 
817 	cpu_idle_thread_init(cpu, tidle);
818 
819 	/*
820 	 * The platform might need to allocate resources prior to bringing
821 	 * up the CPU
822 	 */
823 	if (smp_ops->prepare_cpu) {
824 		rc = smp_ops->prepare_cpu(cpu);
825 		if (rc)
826 			return rc;
827 	}
828 
829 	/* Make sure callin-map entry is 0 (can be leftover a CPU
830 	 * hotplug
831 	 */
832 	cpu_callin_map[cpu] = 0;
833 
834 	/* The information for processor bringup must
835 	 * be written out to main store before we release
836 	 * the processor.
837 	 */
838 	smp_mb();
839 
840 	/* wake up cpus */
841 	DBG("smp: kicking cpu %d\n", cpu);
842 	rc = smp_ops->kick_cpu(cpu);
843 	if (rc) {
844 		pr_err("smp: failed starting cpu %d (rc %d)\n", cpu, rc);
845 		return rc;
846 	}
847 
848 	/*
849 	 * wait to see if the cpu made a callin (is actually up).
850 	 * use this value that I found through experimentation.
851 	 * -- Cort
852 	 */
853 	if (system_state < SYSTEM_RUNNING)
854 		for (c = 50000; c && !cpu_callin_map[cpu]; c--)
855 			udelay(100);
856 #ifdef CONFIG_HOTPLUG_CPU
857 	else
858 		/*
859 		 * CPUs can take much longer to come up in the
860 		 * hotplug case.  Wait five seconds.
861 		 */
862 		for (c = 5000; c && !cpu_callin_map[cpu]; c--)
863 			msleep(1);
864 #endif
865 
866 	if (!cpu_callin_map[cpu]) {
867 		printk(KERN_ERR "Processor %u is stuck.\n", cpu);
868 		return -ENOENT;
869 	}
870 
871 	DBG("Processor %u found.\n", cpu);
872 
873 	if (smp_ops->give_timebase)
874 		smp_ops->give_timebase();
875 
876 	/* Wait until cpu puts itself in the online & active maps */
877 	spin_until_cond(cpu_online(cpu));
878 
879 	return 0;
880 }
881 
882 /* Return the value of the reg property corresponding to the given
883  * logical cpu.
884  */
885 int cpu_to_core_id(int cpu)
886 {
887 	struct device_node *np;
888 	const __be32 *reg;
889 	int id = -1;
890 
891 	np = of_get_cpu_node(cpu, NULL);
892 	if (!np)
893 		goto out;
894 
895 	reg = of_get_property(np, "reg", NULL);
896 	if (!reg)
897 		goto out;
898 
899 	id = be32_to_cpup(reg);
900 out:
901 	of_node_put(np);
902 	return id;
903 }
904 EXPORT_SYMBOL_GPL(cpu_to_core_id);
905 
906 /* Helper routines for cpu to core mapping */
907 int cpu_core_index_of_thread(int cpu)
908 {
909 	return cpu >> threads_shift;
910 }
911 EXPORT_SYMBOL_GPL(cpu_core_index_of_thread);
912 
913 int cpu_first_thread_of_core(int core)
914 {
915 	return core << threads_shift;
916 }
917 EXPORT_SYMBOL_GPL(cpu_first_thread_of_core);
918 
919 /* Must be called when no change can occur to cpu_present_mask,
920  * i.e. during cpu online or offline.
921  */
922 static struct device_node *cpu_to_l2cache(int cpu)
923 {
924 	struct device_node *np;
925 	struct device_node *cache;
926 
927 	if (!cpu_present(cpu))
928 		return NULL;
929 
930 	np = of_get_cpu_node(cpu, NULL);
931 	if (np == NULL)
932 		return NULL;
933 
934 	cache = of_find_next_cache_node(np);
935 
936 	of_node_put(np);
937 
938 	return cache;
939 }
940 
941 static bool update_mask_by_l2(int cpu, struct cpumask *(*mask_fn)(int))
942 {
943 	struct device_node *l2_cache, *np;
944 	int i;
945 
946 	l2_cache = cpu_to_l2cache(cpu);
947 	if (!l2_cache)
948 		return false;
949 
950 	for_each_cpu(i, cpu_online_mask) {
951 		/*
952 		 * when updating the marks the current CPU has not been marked
953 		 * online, but we need to update the cache masks
954 		 */
955 		np = cpu_to_l2cache(i);
956 		if (!np)
957 			continue;
958 
959 		if (np == l2_cache)
960 			set_cpus_related(cpu, i, mask_fn);
961 
962 		of_node_put(np);
963 	}
964 	of_node_put(l2_cache);
965 
966 	return true;
967 }
968 
969 #ifdef CONFIG_HOTPLUG_CPU
970 static void remove_cpu_from_masks(int cpu)
971 {
972 	int i;
973 
974 	/* NB: cpu_core_mask is a superset of the others */
975 	for_each_cpu(i, cpu_core_mask(cpu)) {
976 		set_cpus_unrelated(cpu, i, cpu_core_mask);
977 		set_cpus_unrelated(cpu, i, cpu_l2_cache_mask);
978 		set_cpus_unrelated(cpu, i, cpu_sibling_mask);
979 	}
980 }
981 #endif
982 
983 static void add_cpu_to_masks(int cpu)
984 {
985 	int first_thread = cpu_first_thread_sibling(cpu);
986 	int chipid = cpu_to_chip_id(cpu);
987 	int i;
988 
989 	/*
990 	 * This CPU will not be in the online mask yet so we need to manually
991 	 * add it to it's own thread sibling mask.
992 	 */
993 	cpumask_set_cpu(cpu, cpu_sibling_mask(cpu));
994 
995 	for (i = first_thread; i < first_thread + threads_per_core; i++)
996 		if (cpu_online(i))
997 			set_cpus_related(i, cpu, cpu_sibling_mask);
998 
999 	/*
1000 	 * Copy the thread sibling mask into the cache sibling mask
1001 	 * and mark any CPUs that share an L2 with this CPU.
1002 	 */
1003 	for_each_cpu(i, cpu_sibling_mask(cpu))
1004 		set_cpus_related(cpu, i, cpu_l2_cache_mask);
1005 	update_mask_by_l2(cpu, cpu_l2_cache_mask);
1006 
1007 	/*
1008 	 * Copy the cache sibling mask into core sibling mask and mark
1009 	 * any CPUs on the same chip as this CPU.
1010 	 */
1011 	for_each_cpu(i, cpu_l2_cache_mask(cpu))
1012 		set_cpus_related(cpu, i, cpu_core_mask);
1013 
1014 	if (chipid == -1)
1015 		return;
1016 
1017 	for_each_cpu(i, cpu_online_mask)
1018 		if (cpu_to_chip_id(i) == chipid)
1019 			set_cpus_related(cpu, i, cpu_core_mask);
1020 }
1021 
1022 static bool shared_caches;
1023 
1024 /* Activate a secondary processor. */
1025 void start_secondary(void *unused)
1026 {
1027 	unsigned int cpu = smp_processor_id();
1028 
1029 	mmgrab(&init_mm);
1030 	current->active_mm = &init_mm;
1031 
1032 	smp_store_cpu_info(cpu);
1033 	set_dec(tb_ticks_per_jiffy);
1034 	preempt_disable();
1035 	cpu_callin_map[cpu] = 1;
1036 
1037 	if (smp_ops->setup_cpu)
1038 		smp_ops->setup_cpu(cpu);
1039 	if (smp_ops->take_timebase)
1040 		smp_ops->take_timebase();
1041 
1042 	secondary_cpu_time_init();
1043 
1044 #ifdef CONFIG_PPC64
1045 	if (system_state == SYSTEM_RUNNING)
1046 		vdso_data->processorCount++;
1047 
1048 	vdso_getcpu_init();
1049 #endif
1050 	/* Update topology CPU masks */
1051 	add_cpu_to_masks(cpu);
1052 
1053 	/*
1054 	 * Check for any shared caches. Note that this must be done on a
1055 	 * per-core basis because one core in the pair might be disabled.
1056 	 */
1057 	if (!cpumask_equal(cpu_l2_cache_mask(cpu), cpu_sibling_mask(cpu)))
1058 		shared_caches = true;
1059 
1060 	set_numa_node(numa_cpu_lookup_table[cpu]);
1061 	set_numa_mem(local_memory_node(numa_cpu_lookup_table[cpu]));
1062 
1063 	smp_wmb();
1064 	notify_cpu_starting(cpu);
1065 	set_cpu_online(cpu, true);
1066 
1067 	local_irq_enable();
1068 
1069 	cpu_startup_entry(CPUHP_AP_ONLINE_IDLE);
1070 
1071 	BUG();
1072 }
1073 
1074 int setup_profiling_timer(unsigned int multiplier)
1075 {
1076 	return 0;
1077 }
1078 
1079 #ifdef CONFIG_SCHED_SMT
1080 /* cpumask of CPUs with asymetric SMT dependancy */
1081 static int powerpc_smt_flags(void)
1082 {
1083 	int flags = SD_SHARE_CPUCAPACITY | SD_SHARE_PKG_RESOURCES;
1084 
1085 	if (cpu_has_feature(CPU_FTR_ASYM_SMT)) {
1086 		printk_once(KERN_INFO "Enabling Asymmetric SMT scheduling\n");
1087 		flags |= SD_ASYM_PACKING;
1088 	}
1089 	return flags;
1090 }
1091 #endif
1092 
1093 static struct sched_domain_topology_level powerpc_topology[] = {
1094 #ifdef CONFIG_SCHED_SMT
1095 	{ cpu_smt_mask, powerpc_smt_flags, SD_INIT_NAME(SMT) },
1096 #endif
1097 	{ cpu_cpu_mask, SD_INIT_NAME(DIE) },
1098 	{ NULL, },
1099 };
1100 
1101 /*
1102  * P9 has a slightly odd architecture where pairs of cores share an L2 cache.
1103  * This topology makes it *much* cheaper to migrate tasks between adjacent cores
1104  * since the migrated task remains cache hot. We want to take advantage of this
1105  * at the scheduler level so an extra topology level is required.
1106  */
1107 static int powerpc_shared_cache_flags(void)
1108 {
1109 	return SD_SHARE_PKG_RESOURCES;
1110 }
1111 
1112 /*
1113  * We can't just pass cpu_l2_cache_mask() directly because
1114  * returns a non-const pointer and the compiler barfs on that.
1115  */
1116 static const struct cpumask *shared_cache_mask(int cpu)
1117 {
1118 	return cpu_l2_cache_mask(cpu);
1119 }
1120 
1121 static struct sched_domain_topology_level power9_topology[] = {
1122 #ifdef CONFIG_SCHED_SMT
1123 	{ cpu_smt_mask, powerpc_smt_flags, SD_INIT_NAME(SMT) },
1124 #endif
1125 	{ shared_cache_mask, powerpc_shared_cache_flags, SD_INIT_NAME(CACHE) },
1126 	{ cpu_cpu_mask, SD_INIT_NAME(DIE) },
1127 	{ NULL, },
1128 };
1129 
1130 void __init smp_cpus_done(unsigned int max_cpus)
1131 {
1132 	/*
1133 	 * We are running pinned to the boot CPU, see rest_init().
1134 	 */
1135 	if (smp_ops && smp_ops->setup_cpu)
1136 		smp_ops->setup_cpu(boot_cpuid);
1137 
1138 	if (smp_ops && smp_ops->bringup_done)
1139 		smp_ops->bringup_done();
1140 
1141 	dump_numa_cpu_topology();
1142 
1143 	/*
1144 	 * If any CPU detects that it's sharing a cache with another CPU then
1145 	 * use the deeper topology that is aware of this sharing.
1146 	 */
1147 	if (shared_caches) {
1148 		pr_info("Using shared cache scheduler topology\n");
1149 		set_sched_topology(power9_topology);
1150 	} else {
1151 		pr_info("Using standard scheduler topology\n");
1152 		set_sched_topology(powerpc_topology);
1153 	}
1154 }
1155 
1156 #ifdef CONFIG_HOTPLUG_CPU
1157 int __cpu_disable(void)
1158 {
1159 	int cpu = smp_processor_id();
1160 	int err;
1161 
1162 	if (!smp_ops->cpu_disable)
1163 		return -ENOSYS;
1164 
1165 	err = smp_ops->cpu_disable();
1166 	if (err)
1167 		return err;
1168 
1169 	/* Update sibling maps */
1170 	remove_cpu_from_masks(cpu);
1171 
1172 	return 0;
1173 }
1174 
1175 void __cpu_die(unsigned int cpu)
1176 {
1177 	if (smp_ops->cpu_die)
1178 		smp_ops->cpu_die(cpu);
1179 }
1180 
1181 void cpu_die(void)
1182 {
1183 	if (ppc_md.cpu_die)
1184 		ppc_md.cpu_die();
1185 
1186 	/* If we return, we re-enter start_secondary */
1187 	start_secondary_resume();
1188 }
1189 
1190 #endif
1191