12874c5fdSThomas Gleixner // SPDX-License-Identifier: GPL-2.0-or-later
269a80d3fSPaul Mackerras /*
369a80d3fSPaul Mackerras * SMP support for pSeries machines.
469a80d3fSPaul Mackerras *
569a80d3fSPaul Mackerras * Dave Engebretsen, Peter Bergner, and
669a80d3fSPaul Mackerras * Mike Corrigan {engebret|bergner|mikec}@us.ibm.com
769a80d3fSPaul Mackerras *
869a80d3fSPaul Mackerras * Plus various changes from other IBM teams...
969a80d3fSPaul Mackerras */
1069a80d3fSPaul Mackerras
1169a80d3fSPaul Mackerras
1269a80d3fSPaul Mackerras #include <linux/kernel.h>
1369a80d3fSPaul Mackerras #include <linux/sched.h>
1469a80d3fSPaul Mackerras #include <linux/smp.h>
1569a80d3fSPaul Mackerras #include <linux/interrupt.h>
1669a80d3fSPaul Mackerras #include <linux/delay.h>
1769a80d3fSPaul Mackerras #include <linux/init.h>
1869a80d3fSPaul Mackerras #include <linux/spinlock.h>
1969a80d3fSPaul Mackerras #include <linux/cache.h>
2069a80d3fSPaul Mackerras #include <linux/err.h>
21edbaa603SKay Sievers #include <linux/device.h>
2269a80d3fSPaul Mackerras #include <linux/cpu.h>
2365fddcfcSMike Rapoport #include <linux/pgtable.h>
2469a80d3fSPaul Mackerras
2569a80d3fSPaul Mackerras #include <asm/ptrace.h>
2660063497SArun Sharma #include <linux/atomic.h>
2769a80d3fSPaul Mackerras #include <asm/irq.h>
2869a80d3fSPaul Mackerras #include <asm/page.h>
2969a80d3fSPaul Mackerras #include <asm/io.h>
3069a80d3fSPaul Mackerras #include <asm/smp.h>
3169a80d3fSPaul Mackerras #include <asm/paca.h>
3269a80d3fSPaul Mackerras #include <asm/machdep.h>
3369a80d3fSPaul Mackerras #include <asm/cputable.h>
3469a80d3fSPaul Mackerras #include <asm/firmware.h>
3569a80d3fSPaul Mackerras #include <asm/rtas.h>
36271c3f35SPaul Mackerras #include <asm/vdso_datapage.h>
378d089085SBenjamin Herrenschmidt #include <asm/cputhreads.h>
380b05ac6eSBenjamin Herrenschmidt #include <asm/xics.h>
39eac1e731SCédric Le Goater #include <asm/xive.h>
40e5e84f0aSIan Munsie #include <asm/dbell.h>
41212bebb4SDeepthi Dharwar #include <asm/plpar_wrappers.h>
422751b628SAnton Blanchard #include <asm/code-patching.h>
434edaac51SSukadev Bhattiprolu #include <asm/svm.h>
4492cc6bf0SSrikar Dronamraju #include <asm/kvm_guest.h>
4569a80d3fSPaul Mackerras
468feaeca2SMichael Ellerman #include "pseries.h"
4769a80d3fSPaul Mackerras
4869a80d3fSPaul Mackerras /*
496a75a6b8SMilton Miller * The Primary thread of each non-boot processor was started from the OF client
506a75a6b8SMilton Miller * interface by prom_hold_cpus and is spinning on secondary_hold_spinloop.
5169a80d3fSPaul Mackerras */
52af831e1eSAnton Blanchard static cpumask_var_t of_spin_mask;
5369a80d3fSPaul Mackerras
54f8b67691SMichael Neuling /* Query where a cpu is now. Return codes #defined in plpar_wrappers.h */
smp_query_cpu_stopped(unsigned int pcpu)55f8b67691SMichael Neuling int smp_query_cpu_stopped(unsigned int pcpu)
56f8b67691SMichael Neuling {
57f8b67691SMichael Neuling int cpu_status, status;
58*08273c9fSNathan Lynch int qcss_tok = rtas_function_token(RTAS_FN_QUERY_CPU_STOPPED_STATE);
59f8b67691SMichael Neuling
60f8b67691SMichael Neuling if (qcss_tok == RTAS_UNKNOWN_SERVICE) {
612d86938aSMilton Miller printk_once(KERN_INFO
622d86938aSMilton Miller "Firmware doesn't support query-cpu-stopped-state\n");
63f8b67691SMichael Neuling return QCSS_HARDWARE_ERROR;
64f8b67691SMichael Neuling }
65f8b67691SMichael Neuling
66f8b67691SMichael Neuling status = rtas_call(qcss_tok, 1, 2, &cpu_status, pcpu);
67f8b67691SMichael Neuling if (status != 0) {
68f8b67691SMichael Neuling printk(KERN_ERR
69f8b67691SMichael Neuling "RTAS query-cpu-stopped-state failed: %i\n", status);
70f8b67691SMichael Neuling return status;
71f8b67691SMichael Neuling }
72f8b67691SMichael Neuling
73f8b67691SMichael Neuling return cpu_status;
74f8b67691SMichael Neuling }
75f8b67691SMichael Neuling
7669a80d3fSPaul Mackerras /**
7769a80d3fSPaul Mackerras * smp_startup_cpu() - start the given cpu
7869a80d3fSPaul Mackerras *
7969a80d3fSPaul Mackerras * At boot time, there is nothing to do for primary threads which were
8069a80d3fSPaul Mackerras * started from Open Firmware. For anything else, call RTAS with the
8169a80d3fSPaul Mackerras * appropriate start location.
8269a80d3fSPaul Mackerras *
8369a80d3fSPaul Mackerras * Returns:
8469a80d3fSPaul Mackerras * 0 - failure
8569a80d3fSPaul Mackerras * 1 - success
8669a80d3fSPaul Mackerras */
smp_startup_cpu(unsigned int lcpu)87cad5cef6SGreg Kroah-Hartman static inline int smp_startup_cpu(unsigned int lcpu)
8869a80d3fSPaul Mackerras {
8969a80d3fSPaul Mackerras int status;
902751b628SAnton Blanchard unsigned long start_here =
912751b628SAnton Blanchard __pa(ppc_function_entry(generic_secondary_smp_init));
9269a80d3fSPaul Mackerras unsigned int pcpu;
9369a80d3fSPaul Mackerras int start_cpu;
9469a80d3fSPaul Mackerras
95af831e1eSAnton Blanchard if (cpumask_test_cpu(lcpu, of_spin_mask))
9669a80d3fSPaul Mackerras /* Already started by OF and sitting in spin loop */
9769a80d3fSPaul Mackerras return 1;
9869a80d3fSPaul Mackerras
9969a80d3fSPaul Mackerras pcpu = get_hard_smp_processor_id(lcpu);
10069a80d3fSPaul Mackerras
101aef40e87SMichael Neuling /* Check to see if the CPU out of FW already for kexec */
102aef40e87SMichael Neuling if (smp_query_cpu_stopped(pcpu) == QCSS_NOT_STOPPED){
103af831e1eSAnton Blanchard cpumask_set_cpu(lcpu, of_spin_mask);
104aef40e87SMichael Neuling return 1;
105aef40e87SMichael Neuling }
106aef40e87SMichael Neuling
10769a80d3fSPaul Mackerras /*
10869a80d3fSPaul Mackerras * If the RTAS start-cpu token does not exist then presume the
10969a80d3fSPaul Mackerras * cpu is already spinning.
11069a80d3fSPaul Mackerras */
111*08273c9fSNathan Lynch start_cpu = rtas_function_token(RTAS_FN_START_CPU);
11269a80d3fSPaul Mackerras if (start_cpu == RTAS_UNKNOWN_SERVICE)
11369a80d3fSPaul Mackerras return 1;
11469a80d3fSPaul Mackerras
115496b7a51SMichael Ellerman status = rtas_call(start_cpu, 3, 1, NULL, pcpu, start_here, pcpu);
11669a80d3fSPaul Mackerras if (status != 0) {
11769a80d3fSPaul Mackerras printk(KERN_ERR "start-cpu failed: %i\n", status);
11869a80d3fSPaul Mackerras return 0;
11969a80d3fSPaul Mackerras }
12069a80d3fSPaul Mackerras
12169a80d3fSPaul Mackerras return 1;
12269a80d3fSPaul Mackerras }
12369a80d3fSPaul Mackerras
smp_setup_cpu(int cpu)12486425bedSRashmica Gupta static void smp_setup_cpu(int cpu)
12569a80d3fSPaul Mackerras {
126eac1e731SCédric Le Goater if (xive_enabled())
127eac1e731SCédric Le Goater xive_smp_setup_cpu();
128eac1e731SCédric Le Goater else if (cpu != boot_cpuid)
12969a80d3fSPaul Mackerras xics_setup_cpu();
13069a80d3fSPaul Mackerras
13169a80d3fSPaul Mackerras if (firmware_has_feature(FW_FEATURE_SPLPAR))
13269a80d3fSPaul Mackerras vpa_init(cpu);
13369a80d3fSPaul Mackerras
134af831e1eSAnton Blanchard cpumask_clear_cpu(cpu, of_spin_mask);
13569a80d3fSPaul Mackerras }
13669a80d3fSPaul Mackerras
smp_pSeries_kick_cpu(int nr)137cad5cef6SGreg Kroah-Hartman static int smp_pSeries_kick_cpu(int nr)
13869a80d3fSPaul Mackerras {
139c642af9cSSantosh Sivaraj if (nr < 0 || nr >= nr_cpu_ids)
140f8d0d5dcSSantosh Sivaraj return -EINVAL;
14169a80d3fSPaul Mackerras
14269a80d3fSPaul Mackerras if (!smp_startup_cpu(nr))
143de300974SMichael Ellerman return -ENOENT;
14469a80d3fSPaul Mackerras
14569a80d3fSPaul Mackerras /*
14669a80d3fSPaul Mackerras * The processor is currently spinning, waiting for the
14769a80d3fSPaul Mackerras * cpu_start field to become non-zero After we set cpu_start,
14869a80d3fSPaul Mackerras * the processor will continue on to secondary_start
14969a80d3fSPaul Mackerras */
150d2e60075SNicholas Piggin paca_ptrs[nr]->cpu_start = 1;
151de300974SMichael Ellerman
152de300974SMichael Ellerman return 0;
15369a80d3fSPaul Mackerras }
15469a80d3fSPaul Mackerras
pseries_smp_prepare_cpu(int cpu)155eac1e731SCédric Le Goater static int pseries_smp_prepare_cpu(int cpu)
156eac1e731SCédric Le Goater {
157eac1e731SCédric Le Goater if (xive_enabled())
158eac1e731SCédric Le Goater return xive_smp_prepare_cpu(cpu);
159eac1e731SCédric Le Goater return 0;
160eac1e731SCédric Le Goater }
161eac1e731SCédric Le Goater
1625b06d167SNicholas Piggin /* Cause IPI as setup by the interrupt controller (xics or xive) */
1635b06d167SNicholas Piggin static void (*ic_cause_ipi)(int cpu) __ro_after_init;
1645b06d167SNicholas Piggin
1655b06d167SNicholas Piggin /* Use msgsndp doorbells target is a sibling, else use interrupt controller */
dbell_or_ic_cause_ipi(int cpu)1665b06d167SNicholas Piggin static void dbell_or_ic_cause_ipi(int cpu)
167e5e84f0aSIan Munsie {
168b866cc21SNicholas Piggin if (doorbell_try_core_ipi(cpu))
169b866cc21SNicholas Piggin return;
170b866cc21SNicholas Piggin
1715b06d167SNicholas Piggin ic_cause_ipi(cpu);
172e5e84f0aSIan Munsie }
173e5e84f0aSIan Munsie
pseries_cause_nmi_ipi(int cpu)174102c05e8SNicholas Piggin static int pseries_cause_nmi_ipi(int cpu)
175102c05e8SNicholas Piggin {
176102c05e8SNicholas Piggin int hwcpu;
177102c05e8SNicholas Piggin
178102c05e8SNicholas Piggin if (cpu == NMI_IPI_ALL_OTHERS) {
179102c05e8SNicholas Piggin hwcpu = H_SIGNAL_SYS_RESET_ALL_OTHERS;
180102c05e8SNicholas Piggin } else {
181102c05e8SNicholas Piggin if (cpu < 0) {
182102c05e8SNicholas Piggin WARN_ONCE(true, "incorrect cpu parameter %d", cpu);
183102c05e8SNicholas Piggin return 0;
184102c05e8SNicholas Piggin }
185102c05e8SNicholas Piggin
186102c05e8SNicholas Piggin hwcpu = get_hard_smp_processor_id(cpu);
187102c05e8SNicholas Piggin }
188102c05e8SNicholas Piggin
1897c09c186SMichael Ellerman if (plpar_signal_sys_reset(hwcpu) == H_SUCCESS)
190102c05e8SNicholas Piggin return 1;
191102c05e8SNicholas Piggin
192102c05e8SNicholas Piggin return 0;
193102c05e8SNicholas Piggin }
194102c05e8SNicholas Piggin
pSeries_smp_probe(void)195eac1e731SCédric Le Goater static __init void pSeries_smp_probe(void)
196eac1e731SCédric Le Goater {
197eac1e731SCédric Le Goater if (xive_enabled())
198eac1e731SCédric Le Goater xive_smp_probe();
199eac1e731SCédric Le Goater else
2005b06d167SNicholas Piggin xics_smp_probe();
2015b06d167SNicholas Piggin
2025b06d167SNicholas Piggin /* No doorbell facility, must use the interrupt controller for IPIs */
2035b06d167SNicholas Piggin if (!cpu_has_feature(CPU_FTR_DBELL))
2045b06d167SNicholas Piggin return;
2055b06d167SNicholas Piggin
2065b06d167SNicholas Piggin /* Doorbells can only be used for IPIs between SMT siblings */
2075b06d167SNicholas Piggin if (!cpu_has_feature(CPU_FTR_SMT))
2085b06d167SNicholas Piggin return;
2095b06d167SNicholas Piggin
21095839225SMichael Ellerman check_kvm_guest();
21195839225SMichael Ellerman
21295839225SMichael Ellerman if (is_kvm_guest()) {
2135b06d167SNicholas Piggin /*
214107c5500SNicholas Piggin * KVM emulates doorbells by disabling FSCR[MSGP] so msgsndp
215107c5500SNicholas Piggin * faults to the hypervisor which then reads the instruction
216107c5500SNicholas Piggin * from guest memory, which tends to be slower than using XIVE.
2175b06d167SNicholas Piggin */
218107c5500SNicholas Piggin if (xive_enabled())
2195b06d167SNicholas Piggin return;
2205b06d167SNicholas Piggin
2215b06d167SNicholas Piggin /*
222107c5500SNicholas Piggin * XICS hcalls aren't as fast, so we can use msgsndp (which
223107c5500SNicholas Piggin * also helps exercise KVM emulation), however KVM can't
224107c5500SNicholas Piggin * emulate secure guests because it can't read the instruction
225107c5500SNicholas Piggin * out of their memory.
2265b06d167SNicholas Piggin */
227107c5500SNicholas Piggin if (is_secure_guest())
228107c5500SNicholas Piggin return;
229107c5500SNicholas Piggin }
230107c5500SNicholas Piggin
231107c5500SNicholas Piggin /*
232107c5500SNicholas Piggin * Under PowerVM, FSCR[MSGP] is enabled as guest vCPU siblings are
233107c5500SNicholas Piggin * gang scheduled on the same physical core, so doorbells are always
234107c5500SNicholas Piggin * faster than the interrupt controller, and they can be used by
235107c5500SNicholas Piggin * secure guests.
236107c5500SNicholas Piggin */
237107c5500SNicholas Piggin
2385b06d167SNicholas Piggin ic_cause_ipi = smp_ops->cause_ipi;
2395b06d167SNicholas Piggin smp_ops->cause_ipi = dbell_or_ic_cause_ipi;
240eac1e731SCédric Le Goater }
241eac1e731SCédric Le Goater
24286425bedSRashmica Gupta static struct smp_ops_t pseries_smp_ops = {
2439ca980dcSPaul Mackerras .message_pass = NULL, /* Use smp_muxed_ipi_message_pass */
244e5e84f0aSIan Munsie .cause_ipi = NULL, /* Filled at runtime by pSeries_smp_probe() */
245102c05e8SNicholas Piggin .cause_nmi_ipi = pseries_cause_nmi_ipi,
246e5e84f0aSIan Munsie .probe = pSeries_smp_probe,
247eac1e731SCédric Le Goater .prepare_cpu = pseries_smp_prepare_cpu,
24869a80d3fSPaul Mackerras .kick_cpu = smp_pSeries_kick_cpu,
24986425bedSRashmica Gupta .setup_cpu = smp_setup_cpu,
25039fd4027SAndy Fleming .cpu_bootable = smp_generic_cpu_bootable,
25169a80d3fSPaul Mackerras };
25269a80d3fSPaul Mackerras
25369a80d3fSPaul Mackerras /* This is called very early */
smp_init_pseries(void)25486425bedSRashmica Gupta void __init smp_init_pseries(void)
25569a80d3fSPaul Mackerras {
25669a80d3fSPaul Mackerras int i;
25769a80d3fSPaul Mackerras
258f7ebf352SMichael Ellerman pr_debug(" -> smp_init_pSeries()\n");
25986425bedSRashmica Gupta smp_ops = &pseries_smp_ops;
26069a80d3fSPaul Mackerras
261af831e1eSAnton Blanchard alloc_bootmem_cpumask_var(&of_spin_mask);
262af831e1eSAnton Blanchard
263dbe78b40SBenjamin Herrenschmidt /*
264dbe78b40SBenjamin Herrenschmidt * Mark threads which are still spinning in hold loops
265dbe78b40SBenjamin Herrenschmidt *
266dbe78b40SBenjamin Herrenschmidt * We know prom_init will not have started them if RTAS supports
267dbe78b40SBenjamin Herrenschmidt * query-cpu-stopped-state.
268dbe78b40SBenjamin Herrenschmidt */
269*08273c9fSNathan Lynch if (rtas_function_token(RTAS_FN_QUERY_CPU_STOPPED_STATE) == RTAS_UNKNOWN_SERVICE) {
27069a80d3fSPaul Mackerras if (cpu_has_feature(CPU_FTR_SMT)) {
27169a80d3fSPaul Mackerras for_each_present_cpu(i) {
2726a75a6b8SMilton Miller if (cpu_thread_in_core(i) == 0)
273af831e1eSAnton Blanchard cpumask_set_cpu(i, of_spin_mask);
27469a80d3fSPaul Mackerras }
275dbe78b40SBenjamin Herrenschmidt } else
276af831e1eSAnton Blanchard cpumask_copy(of_spin_mask, cpu_present_mask);
27769a80d3fSPaul Mackerras
278af831e1eSAnton Blanchard cpumask_clear_cpu(boot_cpuid, of_spin_mask);
279dbe78b40SBenjamin Herrenschmidt }
28069a80d3fSPaul Mackerras
281f7ebf352SMichael Ellerman pr_debug(" <- smp_init_pSeries()\n");
28269a80d3fSPaul Mackerras }
283