1 /*
2  * SMP support for PowerNV machines.
3  *
4  * Copyright 2011 IBM Corp.
5  *
6  * This program is free software; you can redistribute it and/or
7  * modify it under the terms of the GNU General Public License
8  * as published by the Free Software Foundation; either version
9  * 2 of the License, or (at your option) any later version.
10  */
11 
12 #include <linux/kernel.h>
13 #include <linux/module.h>
14 #include <linux/sched.h>
15 #include <linux/sched/hotplug.h>
16 #include <linux/smp.h>
17 #include <linux/interrupt.h>
18 #include <linux/delay.h>
19 #include <linux/init.h>
20 #include <linux/spinlock.h>
21 #include <linux/cpu.h>
22 
23 #include <asm/irq.h>
24 #include <asm/smp.h>
25 #include <asm/paca.h>
26 #include <asm/machdep.h>
27 #include <asm/cputable.h>
28 #include <asm/firmware.h>
29 #include <asm/vdso_datapage.h>
30 #include <asm/cputhreads.h>
31 #include <asm/xics.h>
32 #include <asm/xive.h>
33 #include <asm/opal.h>
34 #include <asm/runlatch.h>
35 #include <asm/code-patching.h>
36 #include <asm/dbell.h>
37 #include <asm/kvm_ppc.h>
38 #include <asm/ppc-opcode.h>
39 #include <asm/cpuidle.h>
40 
41 #include "powernv.h"
42 
43 #ifdef DEBUG
44 #include <asm/udbg.h>
45 #define DBG(fmt...) udbg_printf(fmt)
46 #else
47 #define DBG(fmt...)
48 #endif
49 
50 static void pnv_smp_setup_cpu(int cpu)
51 {
52 	if (xive_enabled())
53 		xive_smp_setup_cpu();
54 	else if (cpu != boot_cpuid)
55 		xics_setup_cpu();
56 }
57 
58 static int pnv_smp_kick_cpu(int nr)
59 {
60 	unsigned int pcpu = get_hard_smp_processor_id(nr);
61 	unsigned long start_here =
62 			__pa(ppc_function_entry(generic_secondary_smp_init));
63 	long rc;
64 	uint8_t status;
65 
66 	BUG_ON(nr < 0 || nr >= NR_CPUS);
67 
68 	/*
69 	 * If we already started or OPAL is not supported, we just
70 	 * kick the CPU via the PACA
71 	 */
72 	if (paca[nr].cpu_start || !firmware_has_feature(FW_FEATURE_OPAL))
73 		goto kick;
74 
75 	/*
76 	 * At this point, the CPU can either be spinning on the way in
77 	 * from kexec or be inside OPAL waiting to be started for the
78 	 * first time. OPAL v3 allows us to query OPAL to know if it
79 	 * has the CPUs, so we do that
80 	 */
81 	rc = opal_query_cpu_status(pcpu, &status);
82 	if (rc != OPAL_SUCCESS) {
83 		pr_warn("OPAL Error %ld querying CPU %d state\n", rc, nr);
84 		return -ENODEV;
85 	}
86 
87 	/*
88 	 * Already started, just kick it, probably coming from
89 	 * kexec and spinning
90 	 */
91 	if (status == OPAL_THREAD_STARTED)
92 		goto kick;
93 
94 	/*
95 	 * Available/inactive, let's kick it
96 	 */
97 	if (status == OPAL_THREAD_INACTIVE) {
98 		pr_devel("OPAL: Starting CPU %d (HW 0x%x)...\n", nr, pcpu);
99 		rc = opal_start_cpu(pcpu, start_here);
100 		if (rc != OPAL_SUCCESS) {
101 			pr_warn("OPAL Error %ld starting CPU %d\n", rc, nr);
102 			return -ENODEV;
103 		}
104 	} else {
105 		/*
106 		 * An unavailable CPU (or any other unknown status)
107 		 * shouldn't be started. It should also
108 		 * not be in the possible map but currently it can
109 		 * happen
110 		 */
111 		pr_devel("OPAL: CPU %d (HW 0x%x) is unavailable"
112 			 " (status %d)...\n", nr, pcpu, status);
113 		return -ENODEV;
114 	}
115 
116 kick:
117 	return smp_generic_kick_cpu(nr);
118 }
119 
120 #ifdef CONFIG_HOTPLUG_CPU
121 
122 static int pnv_smp_cpu_disable(void)
123 {
124 	int cpu = smp_processor_id();
125 
126 	/* This is identical to pSeries... might consolidate by
127 	 * moving migrate_irqs_away to a ppc_md with default to
128 	 * the generic fixup_irqs. --BenH.
129 	 */
130 	set_cpu_online(cpu, false);
131 	vdso_data->processorCount--;
132 	if (cpu == boot_cpuid)
133 		boot_cpuid = cpumask_any(cpu_online_mask);
134 	if (xive_enabled())
135 		xive_smp_disable_cpu();
136 	else
137 		xics_migrate_irqs_away();
138 	return 0;
139 }
140 
141 static void pnv_smp_cpu_kill_self(void)
142 {
143 	unsigned int cpu;
144 	unsigned long srr1, wmask;
145 
146 	/* Standard hot unplug procedure */
147 	local_irq_disable();
148 	idle_task_exit();
149 	current->active_mm = NULL; /* for sanity */
150 	cpu = smp_processor_id();
151 	DBG("CPU%d offline\n", cpu);
152 	generic_set_cpu_dead(cpu);
153 	smp_wmb();
154 
155 	wmask = SRR1_WAKEMASK;
156 	if (cpu_has_feature(CPU_FTR_ARCH_207S))
157 		wmask = SRR1_WAKEMASK_P8;
158 
159 	/* We don't want to take decrementer interrupts while we are offline,
160 	 * so clear LPCR:PECE1. We keep PECE2 (and LPCR_PECE_HVEE on P9)
161 	 * enabled as to let IPIs in.
162 	 */
163 	mtspr(SPRN_LPCR, mfspr(SPRN_LPCR) & ~(u64)LPCR_PECE1);
164 
165 	/*
166 	 * Hard-disable interrupts, and then clear irq_happened flags
167 	 * that we can safely ignore while off-line, since they
168 	 * are for things for which we do no processing when off-line
169 	 * (or in the case of HMI, all the processing we need to do
170 	 * is done in lower-level real-mode code).
171 	 */
172 	hard_irq_disable();
173 	local_paca->irq_happened &= ~(PACA_IRQ_DEC | PACA_IRQ_HMI);
174 
175 	while (!generic_check_cpu_restart(cpu)) {
176 		/*
177 		 * Clear IPI flag, since we don't handle IPIs while
178 		 * offline, except for those when changing micro-threading
179 		 * mode, which are handled explicitly below, and those
180 		 * for coming online, which are handled via
181 		 * generic_check_cpu_restart() calls.
182 		 */
183 		kvmppc_set_host_ipi(cpu, 0);
184 
185 		ppc64_runlatch_off();
186 		srr1 = pnv_cpu_offline(cpu);
187 		ppc64_runlatch_on();
188 
189 		/*
190 		 * If the SRR1 value indicates that we woke up due to
191 		 * an external interrupt, then clear the interrupt.
192 		 * We clear the interrupt before checking for the
193 		 * reason, so as to avoid a race where we wake up for
194 		 * some other reason, find nothing and clear the interrupt
195 		 * just as some other cpu is sending us an interrupt.
196 		 * If we returned from power7_nap as a result of
197 		 * having finished executing in a KVM guest, then srr1
198 		 * contains 0.
199 		 */
200 		if (((srr1 & wmask) == SRR1_WAKEEE) ||
201 		    ((srr1 & wmask) == SRR1_WAKEHVI) ||
202 		    (local_paca->irq_happened & PACA_IRQ_EE)) {
203 			if (cpu_has_feature(CPU_FTR_ARCH_300)) {
204 				if (xive_enabled())
205 					xive_flush_interrupt();
206 				else
207 					icp_opal_flush_interrupt();
208 			} else
209 				icp_native_flush_interrupt();
210 		} else if ((srr1 & wmask) == SRR1_WAKEHDBELL) {
211 			unsigned long msg = PPC_DBELL_TYPE(PPC_DBELL_SERVER);
212 			asm volatile(PPC_MSGCLR(%0) : : "r" (msg));
213 		}
214 		local_paca->irq_happened &= ~(PACA_IRQ_EE | PACA_IRQ_DBELL);
215 		smp_mb();
216 
217 		if (cpu_core_split_required())
218 			continue;
219 
220 		if (srr1 && !generic_check_cpu_restart(cpu))
221 			DBG("CPU%d Unexpected exit while offline !\n", cpu);
222 	}
223 
224 	/* Re-enable decrementer interrupts */
225 	mtspr(SPRN_LPCR, mfspr(SPRN_LPCR) | LPCR_PECE1);
226 	DBG("CPU%d coming online...\n", cpu);
227 }
228 
229 #endif /* CONFIG_HOTPLUG_CPU */
230 
231 static int pnv_cpu_bootable(unsigned int nr)
232 {
233 	/*
234 	 * Starting with POWER8, the subcore logic relies on all threads of a
235 	 * core being booted so that they can participate in split mode
236 	 * switches. So on those machines we ignore the smt_enabled_at_boot
237 	 * setting (smt-enabled on the kernel command line).
238 	 */
239 	if (cpu_has_feature(CPU_FTR_ARCH_207S))
240 		return 1;
241 
242 	return smp_generic_cpu_bootable(nr);
243 }
244 
245 static int pnv_smp_prepare_cpu(int cpu)
246 {
247 	if (xive_enabled())
248 		return xive_smp_prepare_cpu(cpu);
249 	return 0;
250 }
251 
252 static void pnv_cause_ipi(int cpu)
253 {
254 	if (doorbell_try_core_ipi(cpu))
255 		return;
256 
257 	icp_ops->cause_ipi(cpu);
258 }
259 
260 static void pnv_p9_dd1_cause_ipi(int cpu)
261 {
262 	int this_cpu = get_cpu();
263 
264 	/*
265 	 * POWER9 DD1 has a global addressed msgsnd, but for now we restrict
266 	 * IPIs to same core, because it requires additional synchronization
267 	 * for inter-core doorbells which we do not implement.
268 	 */
269 	if (cpumask_test_cpu(cpu, cpu_sibling_mask(this_cpu)))
270 		doorbell_global_ipi(cpu);
271 	else
272 		icp_ops->cause_ipi(cpu);
273 
274 	put_cpu();
275 }
276 
277 static void __init pnv_smp_probe(void)
278 {
279 	if (xive_enabled())
280 		xive_smp_probe();
281 	else
282 		xics_smp_probe();
283 
284 	if (cpu_has_feature(CPU_FTR_DBELL)) {
285 		if (cpu_has_feature(CPU_FTR_ARCH_300)) {
286 			if (cpu_has_feature(CPU_FTR_POWER9_DD1))
287 				smp_ops->cause_ipi = pnv_p9_dd1_cause_ipi;
288 			else
289 				smp_ops->cause_ipi = doorbell_global_ipi;
290 		} else {
291 			smp_ops->cause_ipi = pnv_cause_ipi;
292 		}
293 	} else {
294 		smp_ops->cause_ipi = icp_ops->cause_ipi;
295 	}
296 }
297 
298 static struct smp_ops_t pnv_smp_ops = {
299 	.message_pass	= NULL, /* Use smp_muxed_ipi_message_pass */
300 	.cause_ipi	= NULL,	/* Filled at runtime by pnv_smp_probe() */
301 	.probe		= pnv_smp_probe,
302 	.prepare_cpu	= pnv_smp_prepare_cpu,
303 	.kick_cpu	= pnv_smp_kick_cpu,
304 	.setup_cpu	= pnv_smp_setup_cpu,
305 	.cpu_bootable	= pnv_cpu_bootable,
306 #ifdef CONFIG_HOTPLUG_CPU
307 	.cpu_disable	= pnv_smp_cpu_disable,
308 	.cpu_die	= generic_cpu_die,
309 #endif /* CONFIG_HOTPLUG_CPU */
310 };
311 
312 /* This is called very early during platform setup_arch */
313 void __init pnv_smp_init(void)
314 {
315 	smp_ops = &pnv_smp_ops;
316 
317 #ifdef CONFIG_HOTPLUG_CPU
318 	ppc_md.cpu_die	= pnv_smp_cpu_kill_self;
319 #endif
320 }
321