1 /*
2  * SMP support for PowerNV machines.
3  *
4  * Copyright 2011 IBM Corp.
5  *
6  * This program is free software; you can redistribute it and/or
7  * modify it under the terms of the GNU General Public License
8  * as published by the Free Software Foundation; either version
9  * 2 of the License, or (at your option) any later version.
10  */
11 
12 #include <linux/kernel.h>
13 #include <linux/module.h>
14 #include <linux/sched.h>
15 #include <linux/sched/hotplug.h>
16 #include <linux/smp.h>
17 #include <linux/interrupt.h>
18 #include <linux/delay.h>
19 #include <linux/init.h>
20 #include <linux/spinlock.h>
21 #include <linux/cpu.h>
22 
23 #include <asm/irq.h>
24 #include <asm/smp.h>
25 #include <asm/paca.h>
26 #include <asm/machdep.h>
27 #include <asm/cputable.h>
28 #include <asm/firmware.h>
29 #include <asm/vdso_datapage.h>
30 #include <asm/cputhreads.h>
31 #include <asm/xics.h>
32 #include <asm/xive.h>
33 #include <asm/opal.h>
34 #include <asm/runlatch.h>
35 #include <asm/code-patching.h>
36 #include <asm/dbell.h>
37 #include <asm/kvm_ppc.h>
38 #include <asm/ppc-opcode.h>
39 #include <asm/cpuidle.h>
40 #include <asm/kexec.h>
41 #include <asm/reg.h>
42 
43 #include "powernv.h"
44 
45 #ifdef DEBUG
46 #include <asm/udbg.h>
47 #define DBG(fmt...) udbg_printf(fmt)
48 #else
49 #define DBG(fmt...)
50 #endif
51 
52 static void pnv_smp_setup_cpu(int cpu)
53 {
54 	/*
55 	 * P9 workaround for CI vector load (see traps.c),
56 	 * enable the corresponding HMI interrupt
57 	 */
58 	if (pvr_version_is(PVR_POWER9))
59 		mtspr(SPRN_HMEER, mfspr(SPRN_HMEER) | PPC_BIT(17));
60 
61 	if (xive_enabled())
62 		xive_smp_setup_cpu();
63 	else if (cpu != boot_cpuid)
64 		xics_setup_cpu();
65 }
66 
67 static int pnv_smp_kick_cpu(int nr)
68 {
69 	unsigned int pcpu;
70 	unsigned long start_here =
71 			__pa(ppc_function_entry(generic_secondary_smp_init));
72 	long rc;
73 	uint8_t status;
74 
75 	if (nr < 0 || nr >= nr_cpu_ids)
76 		return -EINVAL;
77 
78 	pcpu = get_hard_smp_processor_id(nr);
79 	/*
80 	 * If we already started or OPAL is not supported, we just
81 	 * kick the CPU via the PACA
82 	 */
83 	if (paca[nr].cpu_start || !firmware_has_feature(FW_FEATURE_OPAL))
84 		goto kick;
85 
86 	/*
87 	 * At this point, the CPU can either be spinning on the way in
88 	 * from kexec or be inside OPAL waiting to be started for the
89 	 * first time. OPAL v3 allows us to query OPAL to know if it
90 	 * has the CPUs, so we do that
91 	 */
92 	rc = opal_query_cpu_status(pcpu, &status);
93 	if (rc != OPAL_SUCCESS) {
94 		pr_warn("OPAL Error %ld querying CPU %d state\n", rc, nr);
95 		return -ENODEV;
96 	}
97 
98 	/*
99 	 * Already started, just kick it, probably coming from
100 	 * kexec and spinning
101 	 */
102 	if (status == OPAL_THREAD_STARTED)
103 		goto kick;
104 
105 	/*
106 	 * Available/inactive, let's kick it
107 	 */
108 	if (status == OPAL_THREAD_INACTIVE) {
109 		pr_devel("OPAL: Starting CPU %d (HW 0x%x)...\n", nr, pcpu);
110 		rc = opal_start_cpu(pcpu, start_here);
111 		if (rc != OPAL_SUCCESS) {
112 			pr_warn("OPAL Error %ld starting CPU %d\n", rc, nr);
113 			return -ENODEV;
114 		}
115 	} else {
116 		/*
117 		 * An unavailable CPU (or any other unknown status)
118 		 * shouldn't be started. It should also
119 		 * not be in the possible map but currently it can
120 		 * happen
121 		 */
122 		pr_devel("OPAL: CPU %d (HW 0x%x) is unavailable"
123 			 " (status %d)...\n", nr, pcpu, status);
124 		return -ENODEV;
125 	}
126 
127 kick:
128 	return smp_generic_kick_cpu(nr);
129 }
130 
131 #ifdef CONFIG_HOTPLUG_CPU
132 
133 static int pnv_smp_cpu_disable(void)
134 {
135 	int cpu = smp_processor_id();
136 
137 	/* This is identical to pSeries... might consolidate by
138 	 * moving migrate_irqs_away to a ppc_md with default to
139 	 * the generic fixup_irqs. --BenH.
140 	 */
141 	set_cpu_online(cpu, false);
142 	vdso_data->processorCount--;
143 	if (cpu == boot_cpuid)
144 		boot_cpuid = cpumask_any(cpu_online_mask);
145 	if (xive_enabled())
146 		xive_smp_disable_cpu();
147 	else
148 		xics_migrate_irqs_away();
149 	return 0;
150 }
151 
152 static void pnv_smp_cpu_kill_self(void)
153 {
154 	unsigned int cpu;
155 	unsigned long srr1, wmask;
156 
157 	/* Standard hot unplug procedure */
158 	/*
159 	 * This hard disables local interurpts, ensuring we have no lazy
160 	 * irqs pending.
161 	 */
162 	WARN_ON(irqs_disabled());
163 	hard_irq_disable();
164 	WARN_ON(lazy_irq_pending());
165 
166 	idle_task_exit();
167 	current->active_mm = NULL; /* for sanity */
168 	cpu = smp_processor_id();
169 	DBG("CPU%d offline\n", cpu);
170 	generic_set_cpu_dead(cpu);
171 	smp_wmb();
172 
173 	wmask = SRR1_WAKEMASK;
174 	if (cpu_has_feature(CPU_FTR_ARCH_207S))
175 		wmask = SRR1_WAKEMASK_P8;
176 
177 	while (!generic_check_cpu_restart(cpu)) {
178 		/*
179 		 * Clear IPI flag, since we don't handle IPIs while
180 		 * offline, except for those when changing micro-threading
181 		 * mode, which are handled explicitly below, and those
182 		 * for coming online, which are handled via
183 		 * generic_check_cpu_restart() calls.
184 		 */
185 		kvmppc_set_host_ipi(cpu, 0);
186 
187 		srr1 = pnv_cpu_offline(cpu);
188 
189 		WARN_ON(lazy_irq_pending());
190 
191 		/*
192 		 * If the SRR1 value indicates that we woke up due to
193 		 * an external interrupt, then clear the interrupt.
194 		 * We clear the interrupt before checking for the
195 		 * reason, so as to avoid a race where we wake up for
196 		 * some other reason, find nothing and clear the interrupt
197 		 * just as some other cpu is sending us an interrupt.
198 		 * If we returned from power7_nap as a result of
199 		 * having finished executing in a KVM guest, then srr1
200 		 * contains 0.
201 		 */
202 		if (((srr1 & wmask) == SRR1_WAKEEE) ||
203 		    ((srr1 & wmask) == SRR1_WAKEHVI)) {
204 			if (cpu_has_feature(CPU_FTR_ARCH_300)) {
205 				if (xive_enabled())
206 					xive_flush_interrupt();
207 				else
208 					icp_opal_flush_interrupt();
209 			} else
210 				icp_native_flush_interrupt();
211 		} else if ((srr1 & wmask) == SRR1_WAKEHDBELL) {
212 			unsigned long msg = PPC_DBELL_TYPE(PPC_DBELL_SERVER);
213 			asm volatile(PPC_MSGCLR(%0) : : "r" (msg));
214 		} else if ((srr1 & wmask) == SRR1_WAKERESET) {
215 			irq_set_pending_from_srr1(srr1);
216 			/* Does not return */
217 		}
218 
219 		smp_mb();
220 
221 		/*
222 		 * For kdump kernels, we process the ipi and jump to
223 		 * crash_ipi_callback
224 		 */
225 		if (kdump_in_progress()) {
226 			/*
227 			 * If we got to this point, we've not used
228 			 * NMI's, otherwise we would have gone
229 			 * via the SRR1_WAKERESET path. We are
230 			 * using regular IPI's for waking up offline
231 			 * threads.
232 			 */
233 			struct pt_regs regs;
234 
235 			ppc_save_regs(&regs);
236 			crash_ipi_callback(&regs);
237 			/* Does not return */
238 		}
239 
240 		if (cpu_core_split_required())
241 			continue;
242 
243 		if (srr1 && !generic_check_cpu_restart(cpu))
244 			DBG("CPU%d Unexpected exit while offline srr1=%lx!\n",
245 					cpu, srr1);
246 
247 	}
248 
249 	DBG("CPU%d coming online...\n", cpu);
250 }
251 
252 #endif /* CONFIG_HOTPLUG_CPU */
253 
254 static int pnv_cpu_bootable(unsigned int nr)
255 {
256 	/*
257 	 * Starting with POWER8, the subcore logic relies on all threads of a
258 	 * core being booted so that they can participate in split mode
259 	 * switches. So on those machines we ignore the smt_enabled_at_boot
260 	 * setting (smt-enabled on the kernel command line).
261 	 */
262 	if (cpu_has_feature(CPU_FTR_ARCH_207S))
263 		return 1;
264 
265 	return smp_generic_cpu_bootable(nr);
266 }
267 
268 static int pnv_smp_prepare_cpu(int cpu)
269 {
270 	if (xive_enabled())
271 		return xive_smp_prepare_cpu(cpu);
272 	return 0;
273 }
274 
275 /* Cause IPI as setup by the interrupt controller (xics or xive) */
276 static void (*ic_cause_ipi)(int cpu);
277 
278 static void pnv_cause_ipi(int cpu)
279 {
280 	if (doorbell_try_core_ipi(cpu))
281 		return;
282 
283 	ic_cause_ipi(cpu);
284 }
285 
286 static void pnv_p9_dd1_cause_ipi(int cpu)
287 {
288 	int this_cpu = get_cpu();
289 
290 	/*
291 	 * POWER9 DD1 has a global addressed msgsnd, but for now we restrict
292 	 * IPIs to same core, because it requires additional synchronization
293 	 * for inter-core doorbells which we do not implement.
294 	 */
295 	if (cpumask_test_cpu(cpu, cpu_sibling_mask(this_cpu)))
296 		doorbell_global_ipi(cpu);
297 	else
298 		ic_cause_ipi(cpu);
299 
300 	put_cpu();
301 }
302 
303 static void __init pnv_smp_probe(void)
304 {
305 	if (xive_enabled())
306 		xive_smp_probe();
307 	else
308 		xics_smp_probe();
309 
310 	if (cpu_has_feature(CPU_FTR_DBELL)) {
311 		ic_cause_ipi = smp_ops->cause_ipi;
312 		WARN_ON(!ic_cause_ipi);
313 
314 		if (cpu_has_feature(CPU_FTR_ARCH_300)) {
315 			if (cpu_has_feature(CPU_FTR_POWER9_DD1))
316 				smp_ops->cause_ipi = pnv_p9_dd1_cause_ipi;
317 			else
318 				smp_ops->cause_ipi = doorbell_global_ipi;
319 		} else {
320 			smp_ops->cause_ipi = pnv_cause_ipi;
321 		}
322 	}
323 }
324 
325 static int pnv_system_reset_exception(struct pt_regs *regs)
326 {
327 	if (smp_handle_nmi_ipi(regs))
328 		return 1;
329 	return 0;
330 }
331 
332 static int pnv_cause_nmi_ipi(int cpu)
333 {
334 	int64_t rc;
335 
336 	if (cpu >= 0) {
337 		rc = opal_signal_system_reset(get_hard_smp_processor_id(cpu));
338 		if (rc != OPAL_SUCCESS)
339 			return 0;
340 		return 1;
341 
342 	} else if (cpu == NMI_IPI_ALL_OTHERS) {
343 		bool success = true;
344 		int c;
345 
346 
347 		/*
348 		 * We do not use broadcasts (yet), because it's not clear
349 		 * exactly what semantics Linux wants or the firmware should
350 		 * provide.
351 		 */
352 		for_each_online_cpu(c) {
353 			if (c == smp_processor_id())
354 				continue;
355 
356 			rc = opal_signal_system_reset(
357 						get_hard_smp_processor_id(c));
358 			if (rc != OPAL_SUCCESS)
359 				success = false;
360 		}
361 		if (success)
362 			return 1;
363 
364 		/*
365 		 * Caller will fall back to doorbells, which may pick
366 		 * up the remainders.
367 		 */
368 	}
369 
370 	return 0;
371 }
372 
373 static struct smp_ops_t pnv_smp_ops = {
374 	.message_pass	= NULL, /* Use smp_muxed_ipi_message_pass */
375 	.cause_ipi	= NULL,	/* Filled at runtime by pnv_smp_probe() */
376 	.cause_nmi_ipi	= NULL,
377 	.probe		= pnv_smp_probe,
378 	.prepare_cpu	= pnv_smp_prepare_cpu,
379 	.kick_cpu	= pnv_smp_kick_cpu,
380 	.setup_cpu	= pnv_smp_setup_cpu,
381 	.cpu_bootable	= pnv_cpu_bootable,
382 #ifdef CONFIG_HOTPLUG_CPU
383 	.cpu_disable	= pnv_smp_cpu_disable,
384 	.cpu_die	= generic_cpu_die,
385 #endif /* CONFIG_HOTPLUG_CPU */
386 };
387 
388 /* This is called very early during platform setup_arch */
389 void __init pnv_smp_init(void)
390 {
391 	if (opal_check_token(OPAL_SIGNAL_SYSTEM_RESET)) {
392 		ppc_md.system_reset_exception = pnv_system_reset_exception;
393 		pnv_smp_ops.cause_nmi_ipi = pnv_cause_nmi_ipi;
394 	}
395 	smp_ops = &pnv_smp_ops;
396 
397 #ifdef CONFIG_HOTPLUG_CPU
398 	ppc_md.cpu_die	= pnv_smp_cpu_kill_self;
399 #ifdef CONFIG_KEXEC_CORE
400 	crash_wake_offline = 1;
401 #endif
402 #endif
403 }
404