155190f88SBenjamin Herrenschmidt /* 255190f88SBenjamin Herrenschmidt * SMP support for PowerNV machines. 355190f88SBenjamin Herrenschmidt * 455190f88SBenjamin Herrenschmidt * Copyright 2011 IBM Corp. 555190f88SBenjamin Herrenschmidt * 655190f88SBenjamin Herrenschmidt * This program is free software; you can redistribute it and/or 755190f88SBenjamin Herrenschmidt * modify it under the terms of the GNU General Public License 855190f88SBenjamin Herrenschmidt * as published by the Free Software Foundation; either version 955190f88SBenjamin Herrenschmidt * 2 of the License, or (at your option) any later version. 1055190f88SBenjamin Herrenschmidt */ 1155190f88SBenjamin Herrenschmidt 1255190f88SBenjamin Herrenschmidt #include <linux/kernel.h> 1355190f88SBenjamin Herrenschmidt #include <linux/module.h> 1455190f88SBenjamin Herrenschmidt #include <linux/sched.h> 15ef8bd77fSIngo Molnar #include <linux/sched/hotplug.h> 1655190f88SBenjamin Herrenschmidt #include <linux/smp.h> 1755190f88SBenjamin Herrenschmidt #include <linux/interrupt.h> 1855190f88SBenjamin Herrenschmidt #include <linux/delay.h> 1955190f88SBenjamin Herrenschmidt #include <linux/init.h> 2055190f88SBenjamin Herrenschmidt #include <linux/spinlock.h> 2155190f88SBenjamin Herrenschmidt #include <linux/cpu.h> 2255190f88SBenjamin Herrenschmidt 2355190f88SBenjamin Herrenschmidt #include <asm/irq.h> 2455190f88SBenjamin Herrenschmidt #include <asm/smp.h> 2555190f88SBenjamin Herrenschmidt #include <asm/paca.h> 2655190f88SBenjamin Herrenschmidt #include <asm/machdep.h> 2755190f88SBenjamin Herrenschmidt #include <asm/cputable.h> 2855190f88SBenjamin Herrenschmidt #include <asm/firmware.h> 2955190f88SBenjamin Herrenschmidt #include <asm/vdso_datapage.h> 3055190f88SBenjamin Herrenschmidt #include <asm/cputhreads.h> 3155190f88SBenjamin Herrenschmidt #include <asm/xics.h> 32243e2511SBenjamin Herrenschmidt #include <asm/xive.h> 3314a43e69SBenjamin Herrenschmidt #include <asm/opal.h> 34f2038911SPreeti U Murthy #include <asm/runlatch.h> 352751b628SAnton Blanchard #include <asm/code-patching.h> 36d4e58e59SMichael Neuling #include <asm/dbell.h> 37755563bcSPaul Mackerras #include <asm/kvm_ppc.h> 38755563bcSPaul Mackerras #include <asm/ppc-opcode.h> 39a7cd88daSGautham R. Shenoy #include <asm/cpuidle.h> 4055190f88SBenjamin Herrenschmidt 4155190f88SBenjamin Herrenschmidt #include "powernv.h" 4255190f88SBenjamin Herrenschmidt 43344eb010SBenjamin Herrenschmidt #ifdef DEBUG 44344eb010SBenjamin Herrenschmidt #include <asm/udbg.h> 45344eb010SBenjamin Herrenschmidt #define DBG(fmt...) udbg_printf(fmt) 46344eb010SBenjamin Herrenschmidt #else 47344eb010SBenjamin Herrenschmidt #define DBG(fmt...) 48344eb010SBenjamin Herrenschmidt #endif 49344eb010SBenjamin Herrenschmidt 50061d19f2SPaul Gortmaker static void pnv_smp_setup_cpu(int cpu) 5155190f88SBenjamin Herrenschmidt { 525080332cSMichael Neuling /* 535080332cSMichael Neuling * P9 workaround for CI vector load (see traps.c), 545080332cSMichael Neuling * enable the corresponding HMI interrupt 555080332cSMichael Neuling */ 565080332cSMichael Neuling if (pvr_version_is(PVR_POWER9)) 575080332cSMichael Neuling mtspr(SPRN_HMEER, mfspr(SPRN_HMEER) | PPC_BIT(17)); 585080332cSMichael Neuling 59243e2511SBenjamin Herrenschmidt if (xive_enabled()) 60243e2511SBenjamin Herrenschmidt xive_smp_setup_cpu(); 61243e2511SBenjamin Herrenschmidt else if (cpu != boot_cpuid) 6255190f88SBenjamin Herrenschmidt xics_setup_cpu(); 6355190f88SBenjamin Herrenschmidt } 6455190f88SBenjamin Herrenschmidt 65e51df2c1SAnton Blanchard static int pnv_smp_kick_cpu(int nr) 6614a43e69SBenjamin Herrenschmidt { 6776d98ab4SSantosh Sivaraj unsigned int pcpu; 682751b628SAnton Blanchard unsigned long start_here = 692751b628SAnton Blanchard __pa(ppc_function_entry(generic_secondary_smp_init)); 7014a43e69SBenjamin Herrenschmidt long rc; 71e4d54f71SStewart Smith uint8_t status; 7214a43e69SBenjamin Herrenschmidt 73c642af9cSSantosh Sivaraj if (nr < 0 || nr >= nr_cpu_ids) 74f8d0d5dcSSantosh Sivaraj return -EINVAL; 7514a43e69SBenjamin Herrenschmidt 7676d98ab4SSantosh Sivaraj pcpu = get_hard_smp_processor_id(nr); 77b2b48584SBenjamin Herrenschmidt /* 78e4d54f71SStewart Smith * If we already started or OPAL is not supported, we just 79b2b48584SBenjamin Herrenschmidt * kick the CPU via the PACA 8014a43e69SBenjamin Herrenschmidt */ 81e4d54f71SStewart Smith if (paca[nr].cpu_start || !firmware_has_feature(FW_FEATURE_OPAL)) 82b2b48584SBenjamin Herrenschmidt goto kick; 83b2b48584SBenjamin Herrenschmidt 84b2b48584SBenjamin Herrenschmidt /* 85b2b48584SBenjamin Herrenschmidt * At this point, the CPU can either be spinning on the way in 86b2b48584SBenjamin Herrenschmidt * from kexec or be inside OPAL waiting to be started for the 87b2b48584SBenjamin Herrenschmidt * first time. OPAL v3 allows us to query OPAL to know if it 88b2b48584SBenjamin Herrenschmidt * has the CPUs, so we do that 89b2b48584SBenjamin Herrenschmidt */ 90b2b48584SBenjamin Herrenschmidt rc = opal_query_cpu_status(pcpu, &status); 91b2b48584SBenjamin Herrenschmidt if (rc != OPAL_SUCCESS) { 92e4d54f71SStewart Smith pr_warn("OPAL Error %ld querying CPU %d state\n", rc, nr); 93b2b48584SBenjamin Herrenschmidt return -ENODEV; 94b2b48584SBenjamin Herrenschmidt } 95b2b48584SBenjamin Herrenschmidt 96b2b48584SBenjamin Herrenschmidt /* 97b2b48584SBenjamin Herrenschmidt * Already started, just kick it, probably coming from 98b2b48584SBenjamin Herrenschmidt * kexec and spinning 99b2b48584SBenjamin Herrenschmidt */ 100b2b48584SBenjamin Herrenschmidt if (status == OPAL_THREAD_STARTED) 101b2b48584SBenjamin Herrenschmidt goto kick; 102b2b48584SBenjamin Herrenschmidt 103b2b48584SBenjamin Herrenschmidt /* 104b2b48584SBenjamin Herrenschmidt * Available/inactive, let's kick it 105b2b48584SBenjamin Herrenschmidt */ 106b2b48584SBenjamin Herrenschmidt if (status == OPAL_THREAD_INACTIVE) { 107e4d54f71SStewart Smith pr_devel("OPAL: Starting CPU %d (HW 0x%x)...\n", nr, pcpu); 10814a43e69SBenjamin Herrenschmidt rc = opal_start_cpu(pcpu, start_here); 1094ea9008bSBenjamin Herrenschmidt if (rc != OPAL_SUCCESS) { 110e4d54f71SStewart Smith pr_warn("OPAL Error %ld starting CPU %d\n", rc, nr); 1114ea9008bSBenjamin Herrenschmidt return -ENODEV; 1124ea9008bSBenjamin Herrenschmidt } 113b2b48584SBenjamin Herrenschmidt } else { 114b2b48584SBenjamin Herrenschmidt /* 115b2b48584SBenjamin Herrenschmidt * An unavailable CPU (or any other unknown status) 116b2b48584SBenjamin Herrenschmidt * shouldn't be started. It should also 117b2b48584SBenjamin Herrenschmidt * not be in the possible map but currently it can 118b2b48584SBenjamin Herrenschmidt * happen 119b2b48584SBenjamin Herrenschmidt */ 120b2b48584SBenjamin Herrenschmidt pr_devel("OPAL: CPU %d (HW 0x%x) is unavailable" 121b2b48584SBenjamin Herrenschmidt " (status %d)...\n", nr, pcpu, status); 122b2b48584SBenjamin Herrenschmidt return -ENODEV; 12314a43e69SBenjamin Herrenschmidt } 124e4d54f71SStewart Smith 125b2b48584SBenjamin Herrenschmidt kick: 12614a43e69SBenjamin Herrenschmidt return smp_generic_kick_cpu(nr); 12714a43e69SBenjamin Herrenschmidt } 12814a43e69SBenjamin Herrenschmidt 129344eb010SBenjamin Herrenschmidt #ifdef CONFIG_HOTPLUG_CPU 130344eb010SBenjamin Herrenschmidt 131344eb010SBenjamin Herrenschmidt static int pnv_smp_cpu_disable(void) 132344eb010SBenjamin Herrenschmidt { 133344eb010SBenjamin Herrenschmidt int cpu = smp_processor_id(); 134344eb010SBenjamin Herrenschmidt 135344eb010SBenjamin Herrenschmidt /* This is identical to pSeries... might consolidate by 136344eb010SBenjamin Herrenschmidt * moving migrate_irqs_away to a ppc_md with default to 137344eb010SBenjamin Herrenschmidt * the generic fixup_irqs. --BenH. 138344eb010SBenjamin Herrenschmidt */ 139344eb010SBenjamin Herrenschmidt set_cpu_online(cpu, false); 140344eb010SBenjamin Herrenschmidt vdso_data->processorCount--; 141344eb010SBenjamin Herrenschmidt if (cpu == boot_cpuid) 142344eb010SBenjamin Herrenschmidt boot_cpuid = cpumask_any(cpu_online_mask); 143243e2511SBenjamin Herrenschmidt if (xive_enabled()) 144243e2511SBenjamin Herrenschmidt xive_smp_disable_cpu(); 145243e2511SBenjamin Herrenschmidt else 146344eb010SBenjamin Herrenschmidt xics_migrate_irqs_away(); 147344eb010SBenjamin Herrenschmidt return 0; 148344eb010SBenjamin Herrenschmidt } 149344eb010SBenjamin Herrenschmidt 150344eb010SBenjamin Herrenschmidt static void pnv_smp_cpu_kill_self(void) 151344eb010SBenjamin Herrenschmidt { 152344eb010SBenjamin Herrenschmidt unsigned int cpu; 153755563bcSPaul Mackerras unsigned long srr1, wmask; 154344eb010SBenjamin Herrenschmidt 155344eb010SBenjamin Herrenschmidt /* Standard hot unplug procedure */ 1562525db04SNicholas Piggin /* 1572525db04SNicholas Piggin * This hard disables local interurpts, ensuring we have no lazy 1582525db04SNicholas Piggin * irqs pending. 1592525db04SNicholas Piggin */ 1602525db04SNicholas Piggin WARN_ON(irqs_disabled()); 1612525db04SNicholas Piggin hard_irq_disable(); 1622525db04SNicholas Piggin WARN_ON(lazy_irq_pending()); 1632525db04SNicholas Piggin 164344eb010SBenjamin Herrenschmidt idle_task_exit(); 165344eb010SBenjamin Herrenschmidt current->active_mm = NULL; /* for sanity */ 166344eb010SBenjamin Herrenschmidt cpu = smp_processor_id(); 167344eb010SBenjamin Herrenschmidt DBG("CPU%d offline\n", cpu); 168344eb010SBenjamin Herrenschmidt generic_set_cpu_dead(cpu); 169344eb010SBenjamin Herrenschmidt smp_wmb(); 170344eb010SBenjamin Herrenschmidt 171755563bcSPaul Mackerras wmask = SRR1_WAKEMASK; 172755563bcSPaul Mackerras if (cpu_has_feature(CPU_FTR_ARCH_207S)) 173755563bcSPaul Mackerras wmask = SRR1_WAKEMASK_P8; 174755563bcSPaul Mackerras 175344eb010SBenjamin Herrenschmidt while (!generic_check_cpu_restart(cpu)) { 17653c656c4SPaul Mackerras /* 17753c656c4SPaul Mackerras * Clear IPI flag, since we don't handle IPIs while 17853c656c4SPaul Mackerras * offline, except for those when changing micro-threading 17953c656c4SPaul Mackerras * mode, which are handled explicitly below, and those 18053c656c4SPaul Mackerras * for coming online, which are handled via 18153c656c4SPaul Mackerras * generic_check_cpu_restart() calls. 18253c656c4SPaul Mackerras */ 18353c656c4SPaul Mackerras kvmppc_set_host_ipi(cpu, 0); 18477b54e9fSShreyas B. Prabhu 185a7cd88daSGautham R. Shenoy srr1 = pnv_cpu_offline(cpu); 186e2186023SMichael Ellerman 1872525db04SNicholas Piggin WARN_ON(lazy_irq_pending()); 1882525db04SNicholas Piggin 18956548fc0SPaul Mackerras /* 19056548fc0SPaul Mackerras * If the SRR1 value indicates that we woke up due to 19156548fc0SPaul Mackerras * an external interrupt, then clear the interrupt. 19256548fc0SPaul Mackerras * We clear the interrupt before checking for the 19356548fc0SPaul Mackerras * reason, so as to avoid a race where we wake up for 19456548fc0SPaul Mackerras * some other reason, find nothing and clear the interrupt 19556548fc0SPaul Mackerras * just as some other cpu is sending us an interrupt. 19656548fc0SPaul Mackerras * If we returned from power7_nap as a result of 19756548fc0SPaul Mackerras * having finished executing in a KVM guest, then srr1 19856548fc0SPaul Mackerras * contains 0. 19956548fc0SPaul Mackerras */ 20053c656c4SPaul Mackerras if (((srr1 & wmask) == SRR1_WAKEEE) || 2012525db04SNicholas Piggin ((srr1 & wmask) == SRR1_WAKEHVI)) { 202243e2511SBenjamin Herrenschmidt if (cpu_has_feature(CPU_FTR_ARCH_300)) { 203243e2511SBenjamin Herrenschmidt if (xive_enabled()) 204243e2511SBenjamin Herrenschmidt xive_flush_interrupt(); 2059b256714SBenjamin Herrenschmidt else 206243e2511SBenjamin Herrenschmidt icp_opal_flush_interrupt(); 207243e2511SBenjamin Herrenschmidt } else 208d6a4f709SPaul Mackerras icp_native_flush_interrupt(); 209755563bcSPaul Mackerras } else if ((srr1 & wmask) == SRR1_WAKEHDBELL) { 210755563bcSPaul Mackerras unsigned long msg = PPC_DBELL_TYPE(PPC_DBELL_SERVER); 211755563bcSPaul Mackerras asm volatile(PPC_MSGCLR(%0) : : "r" (msg)); 21256548fc0SPaul Mackerras } 21353c656c4SPaul Mackerras smp_mb(); 214e2186023SMichael Ellerman 215e2186023SMichael Ellerman if (cpu_core_split_required()) 216e2186023SMichael Ellerman continue; 217e2186023SMichael Ellerman 21853c656c4SPaul Mackerras if (srr1 && !generic_check_cpu_restart(cpu)) 2192525db04SNicholas Piggin DBG("CPU%d Unexpected exit while offline srr1=%lx!\n", 2202525db04SNicholas Piggin cpu, srr1); 2212525db04SNicholas Piggin 222344eb010SBenjamin Herrenschmidt } 2239b256714SBenjamin Herrenschmidt 224344eb010SBenjamin Herrenschmidt DBG("CPU%d coming online...\n", cpu); 225344eb010SBenjamin Herrenschmidt } 226344eb010SBenjamin Herrenschmidt 227344eb010SBenjamin Herrenschmidt #endif /* CONFIG_HOTPLUG_CPU */ 228344eb010SBenjamin Herrenschmidt 229d70a54e2SGreg Kurz static int pnv_cpu_bootable(unsigned int nr) 230d70a54e2SGreg Kurz { 231d70a54e2SGreg Kurz /* 232d70a54e2SGreg Kurz * Starting with POWER8, the subcore logic relies on all threads of a 233d70a54e2SGreg Kurz * core being booted so that they can participate in split mode 234d70a54e2SGreg Kurz * switches. So on those machines we ignore the smt_enabled_at_boot 235d70a54e2SGreg Kurz * setting (smt-enabled on the kernel command line). 236d70a54e2SGreg Kurz */ 237d70a54e2SGreg Kurz if (cpu_has_feature(CPU_FTR_ARCH_207S)) 238d70a54e2SGreg Kurz return 1; 239d70a54e2SGreg Kurz 240d70a54e2SGreg Kurz return smp_generic_cpu_bootable(nr); 241d70a54e2SGreg Kurz } 242d70a54e2SGreg Kurz 243243e2511SBenjamin Herrenschmidt static int pnv_smp_prepare_cpu(int cpu) 244243e2511SBenjamin Herrenschmidt { 245243e2511SBenjamin Herrenschmidt if (xive_enabled()) 246243e2511SBenjamin Herrenschmidt return xive_smp_prepare_cpu(cpu); 247243e2511SBenjamin Herrenschmidt return 0; 248243e2511SBenjamin Herrenschmidt } 249243e2511SBenjamin Herrenschmidt 25045b21cfeSMichael Ellerman /* Cause IPI as setup by the interrupt controller (xics or xive) */ 25145b21cfeSMichael Ellerman static void (*ic_cause_ipi)(int cpu); 25245b21cfeSMichael Ellerman 253b866cc21SNicholas Piggin static void pnv_cause_ipi(int cpu) 254b866cc21SNicholas Piggin { 255b866cc21SNicholas Piggin if (doorbell_try_core_ipi(cpu)) 256b866cc21SNicholas Piggin return; 257b866cc21SNicholas Piggin 25845b21cfeSMichael Ellerman ic_cause_ipi(cpu); 259b866cc21SNicholas Piggin } 260b866cc21SNicholas Piggin 2616b3edefeSNicholas Piggin static void pnv_p9_dd1_cause_ipi(int cpu) 2626b3edefeSNicholas Piggin { 2636b3edefeSNicholas Piggin int this_cpu = get_cpu(); 2646b3edefeSNicholas Piggin 2656b3edefeSNicholas Piggin /* 2666b3edefeSNicholas Piggin * POWER9 DD1 has a global addressed msgsnd, but for now we restrict 2676b3edefeSNicholas Piggin * IPIs to same core, because it requires additional synchronization 2686b3edefeSNicholas Piggin * for inter-core doorbells which we do not implement. 2696b3edefeSNicholas Piggin */ 2706b3edefeSNicholas Piggin if (cpumask_test_cpu(cpu, cpu_sibling_mask(this_cpu))) 2716b3edefeSNicholas Piggin doorbell_global_ipi(cpu); 2726b3edefeSNicholas Piggin else 27345b21cfeSMichael Ellerman ic_cause_ipi(cpu); 2746b3edefeSNicholas Piggin 2756b3edefeSNicholas Piggin put_cpu(); 2766b3edefeSNicholas Piggin } 2776b3edefeSNicholas Piggin 278243e2511SBenjamin Herrenschmidt static void __init pnv_smp_probe(void) 279243e2511SBenjamin Herrenschmidt { 280243e2511SBenjamin Herrenschmidt if (xive_enabled()) 281243e2511SBenjamin Herrenschmidt xive_smp_probe(); 282243e2511SBenjamin Herrenschmidt else 283243e2511SBenjamin Herrenschmidt xics_smp_probe(); 284b866cc21SNicholas Piggin 2856b3edefeSNicholas Piggin if (cpu_has_feature(CPU_FTR_DBELL)) { 28645b21cfeSMichael Ellerman ic_cause_ipi = smp_ops->cause_ipi; 28745b21cfeSMichael Ellerman WARN_ON(!ic_cause_ipi); 28845b21cfeSMichael Ellerman 2896b3edefeSNicholas Piggin if (cpu_has_feature(CPU_FTR_ARCH_300)) { 2906b3edefeSNicholas Piggin if (cpu_has_feature(CPU_FTR_POWER9_DD1)) 2916b3edefeSNicholas Piggin smp_ops->cause_ipi = pnv_p9_dd1_cause_ipi; 2926b3edefeSNicholas Piggin else 2936b3edefeSNicholas Piggin smp_ops->cause_ipi = doorbell_global_ipi; 2946b3edefeSNicholas Piggin } else { 295b866cc21SNicholas Piggin smp_ops->cause_ipi = pnv_cause_ipi; 2966b3edefeSNicholas Piggin } 297b866cc21SNicholas Piggin } 298243e2511SBenjamin Herrenschmidt } 299243e2511SBenjamin Herrenschmidt 30055190f88SBenjamin Herrenschmidt static struct smp_ops_t pnv_smp_ops = { 301b866cc21SNicholas Piggin .message_pass = NULL, /* Use smp_muxed_ipi_message_pass */ 302b866cc21SNicholas Piggin .cause_ipi = NULL, /* Filled at runtime by pnv_smp_probe() */ 303c64af645SNicholas Piggin .cause_nmi_ipi = NULL, 304243e2511SBenjamin Herrenschmidt .probe = pnv_smp_probe, 305243e2511SBenjamin Herrenschmidt .prepare_cpu = pnv_smp_prepare_cpu, 30614a43e69SBenjamin Herrenschmidt .kick_cpu = pnv_smp_kick_cpu, 30755190f88SBenjamin Herrenschmidt .setup_cpu = pnv_smp_setup_cpu, 308d70a54e2SGreg Kurz .cpu_bootable = pnv_cpu_bootable, 309344eb010SBenjamin Herrenschmidt #ifdef CONFIG_HOTPLUG_CPU 310344eb010SBenjamin Herrenschmidt .cpu_disable = pnv_smp_cpu_disable, 311344eb010SBenjamin Herrenschmidt .cpu_die = generic_cpu_die, 312344eb010SBenjamin Herrenschmidt #endif /* CONFIG_HOTPLUG_CPU */ 31355190f88SBenjamin Herrenschmidt }; 31455190f88SBenjamin Herrenschmidt 31555190f88SBenjamin Herrenschmidt /* This is called very early during platform setup_arch */ 31655190f88SBenjamin Herrenschmidt void __init pnv_smp_init(void) 31755190f88SBenjamin Herrenschmidt { 31855190f88SBenjamin Herrenschmidt smp_ops = &pnv_smp_ops; 31955190f88SBenjamin Herrenschmidt 320344eb010SBenjamin Herrenschmidt #ifdef CONFIG_HOTPLUG_CPU 321344eb010SBenjamin Herrenschmidt ppc_md.cpu_die = pnv_smp_cpu_kill_self; 322344eb010SBenjamin Herrenschmidt #endif 32355190f88SBenjamin Herrenschmidt } 324