155190f88SBenjamin Herrenschmidt /* 255190f88SBenjamin Herrenschmidt * SMP support for PowerNV machines. 355190f88SBenjamin Herrenschmidt * 455190f88SBenjamin Herrenschmidt * Copyright 2011 IBM Corp. 555190f88SBenjamin Herrenschmidt * 655190f88SBenjamin Herrenschmidt * This program is free software; you can redistribute it and/or 755190f88SBenjamin Herrenschmidt * modify it under the terms of the GNU General Public License 855190f88SBenjamin Herrenschmidt * as published by the Free Software Foundation; either version 955190f88SBenjamin Herrenschmidt * 2 of the License, or (at your option) any later version. 1055190f88SBenjamin Herrenschmidt */ 1155190f88SBenjamin Herrenschmidt 1255190f88SBenjamin Herrenschmidt #include <linux/kernel.h> 1355190f88SBenjamin Herrenschmidt #include <linux/module.h> 1455190f88SBenjamin Herrenschmidt #include <linux/sched.h> 15ef8bd77fSIngo Molnar #include <linux/sched/hotplug.h> 1655190f88SBenjamin Herrenschmidt #include <linux/smp.h> 1755190f88SBenjamin Herrenschmidt #include <linux/interrupt.h> 1855190f88SBenjamin Herrenschmidt #include <linux/delay.h> 1955190f88SBenjamin Herrenschmidt #include <linux/init.h> 2055190f88SBenjamin Herrenschmidt #include <linux/spinlock.h> 2155190f88SBenjamin Herrenschmidt #include <linux/cpu.h> 2255190f88SBenjamin Herrenschmidt 2355190f88SBenjamin Herrenschmidt #include <asm/irq.h> 2455190f88SBenjamin Herrenschmidt #include <asm/smp.h> 2555190f88SBenjamin Herrenschmidt #include <asm/paca.h> 2655190f88SBenjamin Herrenschmidt #include <asm/machdep.h> 2755190f88SBenjamin Herrenschmidt #include <asm/cputable.h> 2855190f88SBenjamin Herrenschmidt #include <asm/firmware.h> 2955190f88SBenjamin Herrenschmidt #include <asm/vdso_datapage.h> 3055190f88SBenjamin Herrenschmidt #include <asm/cputhreads.h> 3155190f88SBenjamin Herrenschmidt #include <asm/xics.h> 32243e2511SBenjamin Herrenschmidt #include <asm/xive.h> 3314a43e69SBenjamin Herrenschmidt #include <asm/opal.h> 34f2038911SPreeti U Murthy #include <asm/runlatch.h> 352751b628SAnton Blanchard #include <asm/code-patching.h> 36d4e58e59SMichael Neuling #include <asm/dbell.h> 37755563bcSPaul Mackerras #include <asm/kvm_ppc.h> 38755563bcSPaul Mackerras #include <asm/ppc-opcode.h> 3955190f88SBenjamin Herrenschmidt 4055190f88SBenjamin Herrenschmidt #include "powernv.h" 4155190f88SBenjamin Herrenschmidt 42344eb010SBenjamin Herrenschmidt #ifdef DEBUG 43344eb010SBenjamin Herrenschmidt #include <asm/udbg.h> 44344eb010SBenjamin Herrenschmidt #define DBG(fmt...) udbg_printf(fmt) 45344eb010SBenjamin Herrenschmidt #else 46344eb010SBenjamin Herrenschmidt #define DBG(fmt...) 47344eb010SBenjamin Herrenschmidt #endif 48344eb010SBenjamin Herrenschmidt 49061d19f2SPaul Gortmaker static void pnv_smp_setup_cpu(int cpu) 5055190f88SBenjamin Herrenschmidt { 51243e2511SBenjamin Herrenschmidt if (xive_enabled()) 52243e2511SBenjamin Herrenschmidt xive_smp_setup_cpu(); 53243e2511SBenjamin Herrenschmidt else if (cpu != boot_cpuid) 5455190f88SBenjamin Herrenschmidt xics_setup_cpu(); 55d4e58e59SMichael Neuling 56d4e58e59SMichael Neuling #ifdef CONFIG_PPC_DOORBELL 57d4e58e59SMichael Neuling if (cpu_has_feature(CPU_FTR_DBELL)) 58d4e58e59SMichael Neuling doorbell_setup_this_cpu(); 59d4e58e59SMichael Neuling #endif 6055190f88SBenjamin Herrenschmidt } 6155190f88SBenjamin Herrenschmidt 62e51df2c1SAnton Blanchard static int pnv_smp_kick_cpu(int nr) 6314a43e69SBenjamin Herrenschmidt { 6414a43e69SBenjamin Herrenschmidt unsigned int pcpu = get_hard_smp_processor_id(nr); 652751b628SAnton Blanchard unsigned long start_here = 662751b628SAnton Blanchard __pa(ppc_function_entry(generic_secondary_smp_init)); 6714a43e69SBenjamin Herrenschmidt long rc; 68e4d54f71SStewart Smith uint8_t status; 6914a43e69SBenjamin Herrenschmidt 7014a43e69SBenjamin Herrenschmidt BUG_ON(nr < 0 || nr >= NR_CPUS); 7114a43e69SBenjamin Herrenschmidt 72b2b48584SBenjamin Herrenschmidt /* 73e4d54f71SStewart Smith * If we already started or OPAL is not supported, we just 74b2b48584SBenjamin Herrenschmidt * kick the CPU via the PACA 7514a43e69SBenjamin Herrenschmidt */ 76e4d54f71SStewart Smith if (paca[nr].cpu_start || !firmware_has_feature(FW_FEATURE_OPAL)) 77b2b48584SBenjamin Herrenschmidt goto kick; 78b2b48584SBenjamin Herrenschmidt 79b2b48584SBenjamin Herrenschmidt /* 80b2b48584SBenjamin Herrenschmidt * At this point, the CPU can either be spinning on the way in 81b2b48584SBenjamin Herrenschmidt * from kexec or be inside OPAL waiting to be started for the 82b2b48584SBenjamin Herrenschmidt * first time. OPAL v3 allows us to query OPAL to know if it 83b2b48584SBenjamin Herrenschmidt * has the CPUs, so we do that 84b2b48584SBenjamin Herrenschmidt */ 85b2b48584SBenjamin Herrenschmidt rc = opal_query_cpu_status(pcpu, &status); 86b2b48584SBenjamin Herrenschmidt if (rc != OPAL_SUCCESS) { 87e4d54f71SStewart Smith pr_warn("OPAL Error %ld querying CPU %d state\n", rc, nr); 88b2b48584SBenjamin Herrenschmidt return -ENODEV; 89b2b48584SBenjamin Herrenschmidt } 90b2b48584SBenjamin Herrenschmidt 91b2b48584SBenjamin Herrenschmidt /* 92b2b48584SBenjamin Herrenschmidt * Already started, just kick it, probably coming from 93b2b48584SBenjamin Herrenschmidt * kexec and spinning 94b2b48584SBenjamin Herrenschmidt */ 95b2b48584SBenjamin Herrenschmidt if (status == OPAL_THREAD_STARTED) 96b2b48584SBenjamin Herrenschmidt goto kick; 97b2b48584SBenjamin Herrenschmidt 98b2b48584SBenjamin Herrenschmidt /* 99b2b48584SBenjamin Herrenschmidt * Available/inactive, let's kick it 100b2b48584SBenjamin Herrenschmidt */ 101b2b48584SBenjamin Herrenschmidt if (status == OPAL_THREAD_INACTIVE) { 102e4d54f71SStewart Smith pr_devel("OPAL: Starting CPU %d (HW 0x%x)...\n", nr, pcpu); 10314a43e69SBenjamin Herrenschmidt rc = opal_start_cpu(pcpu, start_here); 1044ea9008bSBenjamin Herrenschmidt if (rc != OPAL_SUCCESS) { 105e4d54f71SStewart Smith pr_warn("OPAL Error %ld starting CPU %d\n", rc, nr); 1064ea9008bSBenjamin Herrenschmidt return -ENODEV; 1074ea9008bSBenjamin Herrenschmidt } 108b2b48584SBenjamin Herrenschmidt } else { 109b2b48584SBenjamin Herrenschmidt /* 110b2b48584SBenjamin Herrenschmidt * An unavailable CPU (or any other unknown status) 111b2b48584SBenjamin Herrenschmidt * shouldn't be started. It should also 112b2b48584SBenjamin Herrenschmidt * not be in the possible map but currently it can 113b2b48584SBenjamin Herrenschmidt * happen 114b2b48584SBenjamin Herrenschmidt */ 115b2b48584SBenjamin Herrenschmidt pr_devel("OPAL: CPU %d (HW 0x%x) is unavailable" 116b2b48584SBenjamin Herrenschmidt " (status %d)...\n", nr, pcpu, status); 117b2b48584SBenjamin Herrenschmidt return -ENODEV; 11814a43e69SBenjamin Herrenschmidt } 119e4d54f71SStewart Smith 120b2b48584SBenjamin Herrenschmidt kick: 12114a43e69SBenjamin Herrenschmidt return smp_generic_kick_cpu(nr); 12214a43e69SBenjamin Herrenschmidt } 12314a43e69SBenjamin Herrenschmidt 124344eb010SBenjamin Herrenschmidt #ifdef CONFIG_HOTPLUG_CPU 125344eb010SBenjamin Herrenschmidt 126344eb010SBenjamin Herrenschmidt static int pnv_smp_cpu_disable(void) 127344eb010SBenjamin Herrenschmidt { 128344eb010SBenjamin Herrenschmidt int cpu = smp_processor_id(); 129344eb010SBenjamin Herrenschmidt 130344eb010SBenjamin Herrenschmidt /* This is identical to pSeries... might consolidate by 131344eb010SBenjamin Herrenschmidt * moving migrate_irqs_away to a ppc_md with default to 132344eb010SBenjamin Herrenschmidt * the generic fixup_irqs. --BenH. 133344eb010SBenjamin Herrenschmidt */ 134344eb010SBenjamin Herrenschmidt set_cpu_online(cpu, false); 135344eb010SBenjamin Herrenschmidt vdso_data->processorCount--; 136344eb010SBenjamin Herrenschmidt if (cpu == boot_cpuid) 137344eb010SBenjamin Herrenschmidt boot_cpuid = cpumask_any(cpu_online_mask); 138243e2511SBenjamin Herrenschmidt if (xive_enabled()) 139243e2511SBenjamin Herrenschmidt xive_smp_disable_cpu(); 140243e2511SBenjamin Herrenschmidt else 141344eb010SBenjamin Herrenschmidt xics_migrate_irqs_away(); 142344eb010SBenjamin Herrenschmidt return 0; 143344eb010SBenjamin Herrenschmidt } 144344eb010SBenjamin Herrenschmidt 145344eb010SBenjamin Herrenschmidt static void pnv_smp_cpu_kill_self(void) 146344eb010SBenjamin Herrenschmidt { 147344eb010SBenjamin Herrenschmidt unsigned int cpu; 148755563bcSPaul Mackerras unsigned long srr1, wmask; 1498eb8ac89SShreyas B. Prabhu u32 idle_states; 150344eb010SBenjamin Herrenschmidt 151344eb010SBenjamin Herrenschmidt /* Standard hot unplug procedure */ 152344eb010SBenjamin Herrenschmidt local_irq_disable(); 153344eb010SBenjamin Herrenschmidt idle_task_exit(); 154344eb010SBenjamin Herrenschmidt current->active_mm = NULL; /* for sanity */ 155344eb010SBenjamin Herrenschmidt cpu = smp_processor_id(); 156344eb010SBenjamin Herrenschmidt DBG("CPU%d offline\n", cpu); 157344eb010SBenjamin Herrenschmidt generic_set_cpu_dead(cpu); 158344eb010SBenjamin Herrenschmidt smp_wmb(); 159344eb010SBenjamin Herrenschmidt 160755563bcSPaul Mackerras wmask = SRR1_WAKEMASK; 161755563bcSPaul Mackerras if (cpu_has_feature(CPU_FTR_ARCH_207S)) 162755563bcSPaul Mackerras wmask = SRR1_WAKEMASK_P8; 163755563bcSPaul Mackerras 1648eb8ac89SShreyas B. Prabhu idle_states = pnv_get_supported_cpuidle_states(); 1659b256714SBenjamin Herrenschmidt 166344eb010SBenjamin Herrenschmidt /* We don't want to take decrementer interrupts while we are offline, 1679b256714SBenjamin Herrenschmidt * so clear LPCR:PECE1. We keep PECE2 (and LPCR_PECE_HVEE on P9) 1689b256714SBenjamin Herrenschmidt * enabled as to let IPIs in. 169344eb010SBenjamin Herrenschmidt */ 170344eb010SBenjamin Herrenschmidt mtspr(SPRN_LPCR, mfspr(SPRN_LPCR) & ~(u64)LPCR_PECE1); 17153c656c4SPaul Mackerras 17253c656c4SPaul Mackerras /* 17353c656c4SPaul Mackerras * Hard-disable interrupts, and then clear irq_happened flags 17453c656c4SPaul Mackerras * that we can safely ignore while off-line, since they 17553c656c4SPaul Mackerras * are for things for which we do no processing when off-line 17653c656c4SPaul Mackerras * (or in the case of HMI, all the processing we need to do 17753c656c4SPaul Mackerras * is done in lower-level real-mode code). 17853c656c4SPaul Mackerras */ 17953c656c4SPaul Mackerras hard_irq_disable(); 18053c656c4SPaul Mackerras local_paca->irq_happened &= ~(PACA_IRQ_DEC | PACA_IRQ_HMI); 18153c656c4SPaul Mackerras 182344eb010SBenjamin Herrenschmidt while (!generic_check_cpu_restart(cpu)) { 18353c656c4SPaul Mackerras /* 18453c656c4SPaul Mackerras * Clear IPI flag, since we don't handle IPIs while 18553c656c4SPaul Mackerras * offline, except for those when changing micro-threading 18653c656c4SPaul Mackerras * mode, which are handled explicitly below, and those 18753c656c4SPaul Mackerras * for coming online, which are handled via 18853c656c4SPaul Mackerras * generic_check_cpu_restart() calls. 18953c656c4SPaul Mackerras */ 19053c656c4SPaul Mackerras kvmppc_set_host_ipi(cpu, 0); 19177b54e9fSShreyas B. Prabhu 192f2038911SPreeti U Murthy ppc64_runlatch_off(); 19377b54e9fSShreyas B. Prabhu 19409206b60SGautham R. Shenoy if (cpu_has_feature(CPU_FTR_ARCH_300)) { 19509206b60SGautham R. Shenoy srr1 = power9_idle_stop(pnv_deepest_stop_psscr_val, 19609206b60SGautham R. Shenoy pnv_deepest_stop_psscr_mask); 19709206b60SGautham R. Shenoy } else if (idle_states & OPAL_PM_WINKLE_ENABLED) { 19877b54e9fSShreyas B. Prabhu srr1 = power7_winkle(); 19909206b60SGautham R. Shenoy } else if ((idle_states & OPAL_PM_SLEEP_ENABLED) || 20009206b60SGautham R. Shenoy (idle_states & OPAL_PM_SLEEP_ENABLED_ER1)) { 2018eb8ac89SShreyas B. Prabhu srr1 = power7_sleep(); 20209206b60SGautham R. Shenoy } else { 20356548fc0SPaul Mackerras srr1 = power7_nap(1); 20409206b60SGautham R. Shenoy } 20577b54e9fSShreyas B. Prabhu 206f2038911SPreeti U Murthy ppc64_runlatch_on(); 207e2186023SMichael Ellerman 20856548fc0SPaul Mackerras /* 20956548fc0SPaul Mackerras * If the SRR1 value indicates that we woke up due to 21056548fc0SPaul Mackerras * an external interrupt, then clear the interrupt. 21156548fc0SPaul Mackerras * We clear the interrupt before checking for the 21256548fc0SPaul Mackerras * reason, so as to avoid a race where we wake up for 21356548fc0SPaul Mackerras * some other reason, find nothing and clear the interrupt 21456548fc0SPaul Mackerras * just as some other cpu is sending us an interrupt. 21556548fc0SPaul Mackerras * If we returned from power7_nap as a result of 21656548fc0SPaul Mackerras * having finished executing in a KVM guest, then srr1 21756548fc0SPaul Mackerras * contains 0. 21856548fc0SPaul Mackerras */ 21953c656c4SPaul Mackerras if (((srr1 & wmask) == SRR1_WAKEEE) || 2209b256714SBenjamin Herrenschmidt ((srr1 & wmask) == SRR1_WAKEHVI) || 22153c656c4SPaul Mackerras (local_paca->irq_happened & PACA_IRQ_EE)) { 222243e2511SBenjamin Herrenschmidt if (cpu_has_feature(CPU_FTR_ARCH_300)) { 223243e2511SBenjamin Herrenschmidt if (xive_enabled()) 224243e2511SBenjamin Herrenschmidt xive_flush_interrupt(); 2259b256714SBenjamin Herrenschmidt else 226243e2511SBenjamin Herrenschmidt icp_opal_flush_interrupt(); 227243e2511SBenjamin Herrenschmidt } else 228d6a4f709SPaul Mackerras icp_native_flush_interrupt(); 229755563bcSPaul Mackerras } else if ((srr1 & wmask) == SRR1_WAKEHDBELL) { 230755563bcSPaul Mackerras unsigned long msg = PPC_DBELL_TYPE(PPC_DBELL_SERVER); 231755563bcSPaul Mackerras asm volatile(PPC_MSGCLR(%0) : : "r" (msg)); 23256548fc0SPaul Mackerras } 23353c656c4SPaul Mackerras local_paca->irq_happened &= ~(PACA_IRQ_EE | PACA_IRQ_DBELL); 23453c656c4SPaul Mackerras smp_mb(); 235e2186023SMichael Ellerman 236e2186023SMichael Ellerman if (cpu_core_split_required()) 237e2186023SMichael Ellerman continue; 238e2186023SMichael Ellerman 23953c656c4SPaul Mackerras if (srr1 && !generic_check_cpu_restart(cpu)) 240e2186023SMichael Ellerman DBG("CPU%d Unexpected exit while offline !\n", cpu); 241344eb010SBenjamin Herrenschmidt } 2429b256714SBenjamin Herrenschmidt 2439b256714SBenjamin Herrenschmidt /* Re-enable decrementer interrupts */ 244344eb010SBenjamin Herrenschmidt mtspr(SPRN_LPCR, mfspr(SPRN_LPCR) | LPCR_PECE1); 245344eb010SBenjamin Herrenschmidt DBG("CPU%d coming online...\n", cpu); 246344eb010SBenjamin Herrenschmidt } 247344eb010SBenjamin Herrenschmidt 248344eb010SBenjamin Herrenschmidt #endif /* CONFIG_HOTPLUG_CPU */ 249344eb010SBenjamin Herrenschmidt 250d70a54e2SGreg Kurz static int pnv_cpu_bootable(unsigned int nr) 251d70a54e2SGreg Kurz { 252d70a54e2SGreg Kurz /* 253d70a54e2SGreg Kurz * Starting with POWER8, the subcore logic relies on all threads of a 254d70a54e2SGreg Kurz * core being booted so that they can participate in split mode 255d70a54e2SGreg Kurz * switches. So on those machines we ignore the smt_enabled_at_boot 256d70a54e2SGreg Kurz * setting (smt-enabled on the kernel command line). 257d70a54e2SGreg Kurz */ 258d70a54e2SGreg Kurz if (cpu_has_feature(CPU_FTR_ARCH_207S)) 259d70a54e2SGreg Kurz return 1; 260d70a54e2SGreg Kurz 261d70a54e2SGreg Kurz return smp_generic_cpu_bootable(nr); 262d70a54e2SGreg Kurz } 263d70a54e2SGreg Kurz 264243e2511SBenjamin Herrenschmidt static int pnv_smp_prepare_cpu(int cpu) 265243e2511SBenjamin Herrenschmidt { 266243e2511SBenjamin Herrenschmidt if (xive_enabled()) 267243e2511SBenjamin Herrenschmidt return xive_smp_prepare_cpu(cpu); 268243e2511SBenjamin Herrenschmidt return 0; 269243e2511SBenjamin Herrenschmidt } 270243e2511SBenjamin Herrenschmidt 271243e2511SBenjamin Herrenschmidt static void __init pnv_smp_probe(void) 272243e2511SBenjamin Herrenschmidt { 273243e2511SBenjamin Herrenschmidt if (xive_enabled()) 274243e2511SBenjamin Herrenschmidt xive_smp_probe(); 275243e2511SBenjamin Herrenschmidt else 276243e2511SBenjamin Herrenschmidt xics_smp_probe(); 277243e2511SBenjamin Herrenschmidt } 278243e2511SBenjamin Herrenschmidt 27955190f88SBenjamin Herrenschmidt static struct smp_ops_t pnv_smp_ops = { 28055190f88SBenjamin Herrenschmidt .message_pass = smp_muxed_ipi_message_pass, 281243e2511SBenjamin Herrenschmidt .cause_ipi = NULL, /* Filled at runtime by xi{cs,ve}_smp_probe() */ 282243e2511SBenjamin Herrenschmidt .probe = pnv_smp_probe, 283243e2511SBenjamin Herrenschmidt .prepare_cpu = pnv_smp_prepare_cpu, 28414a43e69SBenjamin Herrenschmidt .kick_cpu = pnv_smp_kick_cpu, 28555190f88SBenjamin Herrenschmidt .setup_cpu = pnv_smp_setup_cpu, 286d70a54e2SGreg Kurz .cpu_bootable = pnv_cpu_bootable, 287344eb010SBenjamin Herrenschmidt #ifdef CONFIG_HOTPLUG_CPU 288344eb010SBenjamin Herrenschmidt .cpu_disable = pnv_smp_cpu_disable, 289344eb010SBenjamin Herrenschmidt .cpu_die = generic_cpu_die, 290344eb010SBenjamin Herrenschmidt #endif /* CONFIG_HOTPLUG_CPU */ 29155190f88SBenjamin Herrenschmidt }; 29255190f88SBenjamin Herrenschmidt 29355190f88SBenjamin Herrenschmidt /* This is called very early during platform setup_arch */ 29455190f88SBenjamin Herrenschmidt void __init pnv_smp_init(void) 29555190f88SBenjamin Herrenschmidt { 29655190f88SBenjamin Herrenschmidt smp_ops = &pnv_smp_ops; 29755190f88SBenjamin Herrenschmidt 298344eb010SBenjamin Herrenschmidt #ifdef CONFIG_HOTPLUG_CPU 299344eb010SBenjamin Herrenschmidt ppc_md.cpu_die = pnv_smp_cpu_kill_self; 300344eb010SBenjamin Herrenschmidt #endif 30155190f88SBenjamin Herrenschmidt } 302