1d405a98cSShreyas B. Prabhu /* 2d405a98cSShreyas B. Prabhu * PowerNV cpuidle code 3d405a98cSShreyas B. Prabhu * 4d405a98cSShreyas B. Prabhu * Copyright 2015 IBM Corp. 5d405a98cSShreyas B. Prabhu * 6d405a98cSShreyas B. Prabhu * This program is free software; you can redistribute it and/or 7d405a98cSShreyas B. Prabhu * modify it under the terms of the GNU General Public License 8d405a98cSShreyas B. Prabhu * as published by the Free Software Foundation; either version 9d405a98cSShreyas B. Prabhu * 2 of the License, or (at your option) any later version. 10d405a98cSShreyas B. Prabhu */ 11d405a98cSShreyas B. Prabhu 12d405a98cSShreyas B. Prabhu #include <linux/types.h> 13d405a98cSShreyas B. Prabhu #include <linux/mm.h> 14d405a98cSShreyas B. Prabhu #include <linux/slab.h> 15d405a98cSShreyas B. Prabhu #include <linux/of.h> 165703d2f4SShreyas B. Prabhu #include <linux/device.h> 175703d2f4SShreyas B. Prabhu #include <linux/cpu.h> 18d405a98cSShreyas B. Prabhu 19d405a98cSShreyas B. Prabhu #include <asm/firmware.h> 204bece972SMichael Ellerman #include <asm/machdep.h> 21d405a98cSShreyas B. Prabhu #include <asm/opal.h> 22d405a98cSShreyas B. Prabhu #include <asm/cputhreads.h> 23d405a98cSShreyas B. Prabhu #include <asm/cpuidle.h> 24d405a98cSShreyas B. Prabhu #include <asm/code-patching.h> 25d405a98cSShreyas B. Prabhu #include <asm/smp.h> 262201f994SNicholas Piggin #include <asm/runlatch.h> 277672691aSPaul Mackerras #include <asm/dbell.h> 28d405a98cSShreyas B. Prabhu 29d405a98cSShreyas B. Prabhu #include "powernv.h" 30d405a98cSShreyas B. Prabhu #include "subcore.h" 31d405a98cSShreyas B. Prabhu 32bcef83a0SShreyas B. Prabhu /* Power ISA 3.0 allows for stop states 0x0 - 0xF */ 33bcef83a0SShreyas B. Prabhu #define MAX_STOP_STATE 0xF 34bcef83a0SShreyas B. Prabhu 351e1601b3SAkshay Adiga #define P9_STOP_SPR_MSR 2000 361e1601b3SAkshay Adiga #define P9_STOP_SPR_PSSCR 855 371e1601b3SAkshay Adiga 38d405a98cSShreyas B. Prabhu static u32 supported_cpuidle_states; 399c7b185aSAkshay Adiga struct pnv_idle_states_t *pnv_idle_states; 409c7b185aSAkshay Adiga int nr_pnv_idle_states; 41d405a98cSShreyas B. Prabhu 421e1601b3SAkshay Adiga /* 431e1601b3SAkshay Adiga * The default stop state that will be used by ppc_md.power_save 441e1601b3SAkshay Adiga * function on platforms that support stop instruction. 451e1601b3SAkshay Adiga */ 461e1601b3SAkshay Adiga static u64 pnv_default_stop_val; 471e1601b3SAkshay Adiga static u64 pnv_default_stop_mask; 481e1601b3SAkshay Adiga static bool default_stop_found; 491e1601b3SAkshay Adiga 501e1601b3SAkshay Adiga /* 511e1601b3SAkshay Adiga * First deep stop state. Used to figure out when to save/restore 521e1601b3SAkshay Adiga * hypervisor context. 531e1601b3SAkshay Adiga */ 541e1601b3SAkshay Adiga u64 pnv_first_deep_stop_state = MAX_STOP_STATE; 551e1601b3SAkshay Adiga 561e1601b3SAkshay Adiga /* 571e1601b3SAkshay Adiga * psscr value and mask of the deepest stop idle state. 581e1601b3SAkshay Adiga * Used when a cpu is offlined. 591e1601b3SAkshay Adiga */ 601e1601b3SAkshay Adiga static u64 pnv_deepest_stop_psscr_val; 611e1601b3SAkshay Adiga static u64 pnv_deepest_stop_psscr_mask; 62785a12afSGautham R. Shenoy static u64 pnv_deepest_stop_flag; 631e1601b3SAkshay Adiga static bool deepest_stop_found; 641e1601b3SAkshay Adiga 65bcef83a0SShreyas B. Prabhu static int pnv_save_sprs_for_deep_states(void) 66d405a98cSShreyas B. Prabhu { 67d405a98cSShreyas B. Prabhu int cpu; 68d405a98cSShreyas B. Prabhu int rc; 69d405a98cSShreyas B. Prabhu 70d405a98cSShreyas B. Prabhu /* 71446957baSAdam Buchbinder * hid0, hid1, hid4, hid5, hmeer and lpcr values are symmetric across 72d405a98cSShreyas B. Prabhu * all cpus at boot. Get these reg values of current cpu and use the 73446957baSAdam Buchbinder * same across all cpus. 74d405a98cSShreyas B. Prabhu */ 7524be85a2SGautham R. Shenoy uint64_t lpcr_val = mfspr(SPRN_LPCR); 76d405a98cSShreyas B. Prabhu uint64_t hid0_val = mfspr(SPRN_HID0); 77d405a98cSShreyas B. Prabhu uint64_t hid1_val = mfspr(SPRN_HID1); 78d405a98cSShreyas B. Prabhu uint64_t hid4_val = mfspr(SPRN_HID4); 79d405a98cSShreyas B. Prabhu uint64_t hid5_val = mfspr(SPRN_HID5); 80d405a98cSShreyas B. Prabhu uint64_t hmeer_val = mfspr(SPRN_HMEER); 811e1601b3SAkshay Adiga uint64_t msr_val = MSR_IDLE; 821e1601b3SAkshay Adiga uint64_t psscr_val = pnv_deepest_stop_psscr_val; 83d405a98cSShreyas B. Prabhu 84ac9816dcSAkshay Adiga for_each_present_cpu(cpu) { 85d405a98cSShreyas B. Prabhu uint64_t pir = get_hard_smp_processor_id(cpu); 86d2e60075SNicholas Piggin uint64_t hsprg0_val = (uint64_t)paca_ptrs[cpu]; 87d405a98cSShreyas B. Prabhu 88d405a98cSShreyas B. Prabhu rc = opal_slw_set_reg(pir, SPRN_HSPRG0, hsprg0_val); 89d405a98cSShreyas B. Prabhu if (rc != 0) 90d405a98cSShreyas B. Prabhu return rc; 91d405a98cSShreyas B. Prabhu 92d405a98cSShreyas B. Prabhu rc = opal_slw_set_reg(pir, SPRN_LPCR, lpcr_val); 93d405a98cSShreyas B. Prabhu if (rc != 0) 94d405a98cSShreyas B. Prabhu return rc; 95d405a98cSShreyas B. Prabhu 961e1601b3SAkshay Adiga if (cpu_has_feature(CPU_FTR_ARCH_300)) { 971e1601b3SAkshay Adiga rc = opal_slw_set_reg(pir, P9_STOP_SPR_MSR, msr_val); 981e1601b3SAkshay Adiga if (rc) 991e1601b3SAkshay Adiga return rc; 1001e1601b3SAkshay Adiga 1011e1601b3SAkshay Adiga rc = opal_slw_set_reg(pir, 1021e1601b3SAkshay Adiga P9_STOP_SPR_PSSCR, psscr_val); 1031e1601b3SAkshay Adiga 1041e1601b3SAkshay Adiga if (rc) 1051e1601b3SAkshay Adiga return rc; 1061e1601b3SAkshay Adiga } 1071e1601b3SAkshay Adiga 108d405a98cSShreyas B. Prabhu /* HIDs are per core registers */ 109d405a98cSShreyas B. Prabhu if (cpu_thread_in_core(cpu) == 0) { 110d405a98cSShreyas B. Prabhu 111d405a98cSShreyas B. Prabhu rc = opal_slw_set_reg(pir, SPRN_HMEER, hmeer_val); 112d405a98cSShreyas B. Prabhu if (rc != 0) 113d405a98cSShreyas B. Prabhu return rc; 114d405a98cSShreyas B. Prabhu 115d405a98cSShreyas B. Prabhu rc = opal_slw_set_reg(pir, SPRN_HID0, hid0_val); 116d405a98cSShreyas B. Prabhu if (rc != 0) 117d405a98cSShreyas B. Prabhu return rc; 118d405a98cSShreyas B. Prabhu 1191e1601b3SAkshay Adiga /* Only p8 needs to set extra HID regiters */ 1201e1601b3SAkshay Adiga if (!cpu_has_feature(CPU_FTR_ARCH_300)) { 1211e1601b3SAkshay Adiga 122d405a98cSShreyas B. Prabhu rc = opal_slw_set_reg(pir, SPRN_HID1, hid1_val); 123d405a98cSShreyas B. Prabhu if (rc != 0) 124d405a98cSShreyas B. Prabhu return rc; 125d405a98cSShreyas B. Prabhu 126d405a98cSShreyas B. Prabhu rc = opal_slw_set_reg(pir, SPRN_HID4, hid4_val); 127d405a98cSShreyas B. Prabhu if (rc != 0) 128d405a98cSShreyas B. Prabhu return rc; 129d405a98cSShreyas B. Prabhu 130d405a98cSShreyas B. Prabhu rc = opal_slw_set_reg(pir, SPRN_HID5, hid5_val); 131d405a98cSShreyas B. Prabhu if (rc != 0) 132d405a98cSShreyas B. Prabhu return rc; 133d405a98cSShreyas B. Prabhu } 134d405a98cSShreyas B. Prabhu } 1351e1601b3SAkshay Adiga } 136d405a98cSShreyas B. Prabhu 137d405a98cSShreyas B. Prabhu return 0; 138d405a98cSShreyas B. Prabhu } 139d405a98cSShreyas B. Prabhu 140d405a98cSShreyas B. Prabhu static void pnv_alloc_idle_core_states(void) 141d405a98cSShreyas B. Prabhu { 142d405a98cSShreyas B. Prabhu int i, j; 143d405a98cSShreyas B. Prabhu int nr_cores = cpu_nr_cores(); 144d405a98cSShreyas B. Prabhu u32 *core_idle_state; 145d405a98cSShreyas B. Prabhu 146d405a98cSShreyas B. Prabhu /* 1475f221c3cSGautham R. Shenoy * core_idle_state - The lower 8 bits track the idle state of 1485f221c3cSGautham R. Shenoy * each thread of the core. 1495f221c3cSGautham R. Shenoy * 1505f221c3cSGautham R. Shenoy * The most significant bit is the lock bit. 1515f221c3cSGautham R. Shenoy * 1525f221c3cSGautham R. Shenoy * Initially all the bits corresponding to threads_per_core 1535f221c3cSGautham R. Shenoy * are set. They are cleared when the thread enters deep idle 1545f221c3cSGautham R. Shenoy * state like sleep and winkle/stop. 1555f221c3cSGautham R. Shenoy * 1565f221c3cSGautham R. Shenoy * Initially the lock bit is cleared. The lock bit has 2 1575f221c3cSGautham R. Shenoy * purposes: 1585f221c3cSGautham R. Shenoy * a. While the first thread in the core waking up from 1595f221c3cSGautham R. Shenoy * idle is restoring core state, it prevents other 1605f221c3cSGautham R. Shenoy * threads in the core from switching to process 1615f221c3cSGautham R. Shenoy * context. 1625f221c3cSGautham R. Shenoy * b. While the last thread in the core is saving the 1635f221c3cSGautham R. Shenoy * core state, it prevents a different thread from 1645f221c3cSGautham R. Shenoy * waking up. 165d405a98cSShreyas B. Prabhu */ 166d405a98cSShreyas B. Prabhu for (i = 0; i < nr_cores; i++) { 167d405a98cSShreyas B. Prabhu int first_cpu = i * threads_per_core; 168d405a98cSShreyas B. Prabhu int node = cpu_to_node(first_cpu); 16917ed4c8fSGautham R. Shenoy size_t paca_ptr_array_size; 170d405a98cSShreyas B. Prabhu 171d405a98cSShreyas B. Prabhu core_idle_state = kmalloc_node(sizeof(u32), GFP_KERNEL, node); 1725f221c3cSGautham R. Shenoy *core_idle_state = (1 << threads_per_core) - 1; 17317ed4c8fSGautham R. Shenoy paca_ptr_array_size = (threads_per_core * 17417ed4c8fSGautham R. Shenoy sizeof(struct paca_struct *)); 175d405a98cSShreyas B. Prabhu 176d405a98cSShreyas B. Prabhu for (j = 0; j < threads_per_core; j++) { 177d405a98cSShreyas B. Prabhu int cpu = first_cpu + j; 178d405a98cSShreyas B. Prabhu 179d2e60075SNicholas Piggin paca_ptrs[cpu]->core_idle_state_ptr = core_idle_state; 180d2e60075SNicholas Piggin paca_ptrs[cpu]->thread_idle_state = PNV_THREAD_RUNNING; 181d2e60075SNicholas Piggin paca_ptrs[cpu]->thread_mask = 1 << j; 182d405a98cSShreyas B. Prabhu } 183d405a98cSShreyas B. Prabhu } 184d405a98cSShreyas B. Prabhu 185d405a98cSShreyas B. Prabhu update_subcore_sibling_mask(); 186d405a98cSShreyas B. Prabhu 187785a12afSGautham R. Shenoy if (supported_cpuidle_states & OPAL_PM_LOSE_FULL_CONTEXT) { 188785a12afSGautham R. Shenoy int rc = pnv_save_sprs_for_deep_states(); 189785a12afSGautham R. Shenoy 190785a12afSGautham R. Shenoy if (likely(!rc)) 191785a12afSGautham R. Shenoy return; 192785a12afSGautham R. Shenoy 193785a12afSGautham R. Shenoy /* 194785a12afSGautham R. Shenoy * The stop-api is unable to restore hypervisor 195785a12afSGautham R. Shenoy * resources on wakeup from platform idle states which 196785a12afSGautham R. Shenoy * lose full context. So disable such states. 197785a12afSGautham R. Shenoy */ 198785a12afSGautham R. Shenoy supported_cpuidle_states &= ~OPAL_PM_LOSE_FULL_CONTEXT; 199785a12afSGautham R. Shenoy pr_warn("cpuidle-powernv: Disabling idle states that lose full context\n"); 200785a12afSGautham R. Shenoy pr_warn("cpuidle-powernv: Idle power-savings, CPU-Hotplug affected\n"); 201785a12afSGautham R. Shenoy 202785a12afSGautham R. Shenoy if (cpu_has_feature(CPU_FTR_ARCH_300) && 203785a12afSGautham R. Shenoy (pnv_deepest_stop_flag & OPAL_PM_LOSE_FULL_CONTEXT)) { 204785a12afSGautham R. Shenoy /* 205785a12afSGautham R. Shenoy * Use the default stop state for CPU-Hotplug 206785a12afSGautham R. Shenoy * if available. 207785a12afSGautham R. Shenoy */ 208785a12afSGautham R. Shenoy if (default_stop_found) { 209785a12afSGautham R. Shenoy pnv_deepest_stop_psscr_val = 210785a12afSGautham R. Shenoy pnv_default_stop_val; 211785a12afSGautham R. Shenoy pnv_deepest_stop_psscr_mask = 212785a12afSGautham R. Shenoy pnv_default_stop_mask; 213785a12afSGautham R. Shenoy pr_warn("cpuidle-powernv: Offlined CPUs will stop with psscr = 0x%016llx\n", 214785a12afSGautham R. Shenoy pnv_deepest_stop_psscr_val); 215785a12afSGautham R. Shenoy } else { /* Fallback to snooze loop for CPU-Hotplug */ 216785a12afSGautham R. Shenoy deepest_stop_found = false; 217785a12afSGautham R. Shenoy pr_warn("cpuidle-powernv: Offlined CPUs will busy wait\n"); 218785a12afSGautham R. Shenoy } 219785a12afSGautham R. Shenoy } 220785a12afSGautham R. Shenoy } 221d405a98cSShreyas B. Prabhu } 222d405a98cSShreyas B. Prabhu 223d405a98cSShreyas B. Prabhu u32 pnv_get_supported_cpuidle_states(void) 224d405a98cSShreyas B. Prabhu { 225d405a98cSShreyas B. Prabhu return supported_cpuidle_states; 226d405a98cSShreyas B. Prabhu } 227d405a98cSShreyas B. Prabhu EXPORT_SYMBOL_GPL(pnv_get_supported_cpuidle_states); 228d405a98cSShreyas B. Prabhu 2295703d2f4SShreyas B. Prabhu static void pnv_fastsleep_workaround_apply(void *info) 2305703d2f4SShreyas B. Prabhu 2315703d2f4SShreyas B. Prabhu { 2325703d2f4SShreyas B. Prabhu int rc; 2335703d2f4SShreyas B. Prabhu int *err = info; 2345703d2f4SShreyas B. Prabhu 2355703d2f4SShreyas B. Prabhu rc = opal_config_cpu_idle_state(OPAL_CONFIG_IDLE_FASTSLEEP, 2365703d2f4SShreyas B. Prabhu OPAL_CONFIG_IDLE_APPLY); 2375703d2f4SShreyas B. Prabhu if (rc) 2385703d2f4SShreyas B. Prabhu *err = 1; 2395703d2f4SShreyas B. Prabhu } 2405703d2f4SShreyas B. Prabhu 2415703d2f4SShreyas B. Prabhu /* 2425703d2f4SShreyas B. Prabhu * Used to store fastsleep workaround state 2435703d2f4SShreyas B. Prabhu * 0 - Workaround applied/undone at fastsleep entry/exit path (Default) 2445703d2f4SShreyas B. Prabhu * 1 - Workaround applied once, never undone. 2455703d2f4SShreyas B. Prabhu */ 2465703d2f4SShreyas B. Prabhu static u8 fastsleep_workaround_applyonce; 2475703d2f4SShreyas B. Prabhu 2485703d2f4SShreyas B. Prabhu static ssize_t show_fastsleep_workaround_applyonce(struct device *dev, 2495703d2f4SShreyas B. Prabhu struct device_attribute *attr, char *buf) 2505703d2f4SShreyas B. Prabhu { 2515703d2f4SShreyas B. Prabhu return sprintf(buf, "%u\n", fastsleep_workaround_applyonce); 2525703d2f4SShreyas B. Prabhu } 2535703d2f4SShreyas B. Prabhu 2545703d2f4SShreyas B. Prabhu static ssize_t store_fastsleep_workaround_applyonce(struct device *dev, 2555703d2f4SShreyas B. Prabhu struct device_attribute *attr, const char *buf, 2565703d2f4SShreyas B. Prabhu size_t count) 2575703d2f4SShreyas B. Prabhu { 2585703d2f4SShreyas B. Prabhu cpumask_t primary_thread_mask; 2595703d2f4SShreyas B. Prabhu int err; 2605703d2f4SShreyas B. Prabhu u8 val; 2615703d2f4SShreyas B. Prabhu 2625703d2f4SShreyas B. Prabhu if (kstrtou8(buf, 0, &val) || val != 1) 2635703d2f4SShreyas B. Prabhu return -EINVAL; 2645703d2f4SShreyas B. Prabhu 2655703d2f4SShreyas B. Prabhu if (fastsleep_workaround_applyonce == 1) 2665703d2f4SShreyas B. Prabhu return count; 2675703d2f4SShreyas B. Prabhu 2685703d2f4SShreyas B. Prabhu /* 2695703d2f4SShreyas B. Prabhu * fastsleep_workaround_applyonce = 1 implies 2705703d2f4SShreyas B. Prabhu * fastsleep workaround needs to be left in 'applied' state on all 2715703d2f4SShreyas B. Prabhu * the cores. Do this by- 2725703d2f4SShreyas B. Prabhu * 1. Patching out the call to 'undo' workaround in fastsleep exit path 2735703d2f4SShreyas B. Prabhu * 2. Sending ipi to all the cores which have at least one online thread 2745703d2f4SShreyas B. Prabhu * 3. Patching out the call to 'apply' workaround in fastsleep entry 2755703d2f4SShreyas B. Prabhu * path 2765703d2f4SShreyas B. Prabhu * There is no need to send ipi to cores which have all threads 2775703d2f4SShreyas B. Prabhu * offlined, as last thread of the core entering fastsleep or deeper 2785703d2f4SShreyas B. Prabhu * state would have applied workaround. 2795703d2f4SShreyas B. Prabhu */ 2805703d2f4SShreyas B. Prabhu err = patch_instruction( 2815703d2f4SShreyas B. Prabhu (unsigned int *)pnv_fastsleep_workaround_at_exit, 2825703d2f4SShreyas B. Prabhu PPC_INST_NOP); 2835703d2f4SShreyas B. Prabhu if (err) { 2845703d2f4SShreyas B. Prabhu pr_err("fastsleep_workaround_applyonce change failed while patching pnv_fastsleep_workaround_at_exit"); 2855703d2f4SShreyas B. Prabhu goto fail; 2865703d2f4SShreyas B. Prabhu } 2875703d2f4SShreyas B. Prabhu 2885703d2f4SShreyas B. Prabhu get_online_cpus(); 2895703d2f4SShreyas B. Prabhu primary_thread_mask = cpu_online_cores_map(); 2905703d2f4SShreyas B. Prabhu on_each_cpu_mask(&primary_thread_mask, 2915703d2f4SShreyas B. Prabhu pnv_fastsleep_workaround_apply, 2925703d2f4SShreyas B. Prabhu &err, 1); 2935703d2f4SShreyas B. Prabhu put_online_cpus(); 2945703d2f4SShreyas B. Prabhu if (err) { 2955703d2f4SShreyas B. Prabhu pr_err("fastsleep_workaround_applyonce change failed while running pnv_fastsleep_workaround_apply"); 2965703d2f4SShreyas B. Prabhu goto fail; 2975703d2f4SShreyas B. Prabhu } 2985703d2f4SShreyas B. Prabhu 2995703d2f4SShreyas B. Prabhu err = patch_instruction( 3005703d2f4SShreyas B. Prabhu (unsigned int *)pnv_fastsleep_workaround_at_entry, 3015703d2f4SShreyas B. Prabhu PPC_INST_NOP); 3025703d2f4SShreyas B. Prabhu if (err) { 3035703d2f4SShreyas B. Prabhu pr_err("fastsleep_workaround_applyonce change failed while patching pnv_fastsleep_workaround_at_entry"); 3045703d2f4SShreyas B. Prabhu goto fail; 3055703d2f4SShreyas B. Prabhu } 3065703d2f4SShreyas B. Prabhu 3075703d2f4SShreyas B. Prabhu fastsleep_workaround_applyonce = 1; 3085703d2f4SShreyas B. Prabhu 3095703d2f4SShreyas B. Prabhu return count; 3105703d2f4SShreyas B. Prabhu fail: 3115703d2f4SShreyas B. Prabhu return -EIO; 3125703d2f4SShreyas B. Prabhu } 3135703d2f4SShreyas B. Prabhu 3145703d2f4SShreyas B. Prabhu static DEVICE_ATTR(fastsleep_workaround_applyonce, 0600, 3155703d2f4SShreyas B. Prabhu show_fastsleep_workaround_applyonce, 3165703d2f4SShreyas B. Prabhu store_fastsleep_workaround_applyonce); 3175703d2f4SShreyas B. Prabhu 3182201f994SNicholas Piggin static unsigned long __power7_idle_type(unsigned long type) 3192201f994SNicholas Piggin { 3202201f994SNicholas Piggin unsigned long srr1; 3212201f994SNicholas Piggin 3222201f994SNicholas Piggin if (!prep_irq_for_idle_irqsoff()) 3232201f994SNicholas Piggin return 0; 3242201f994SNicholas Piggin 32540d24343SNicholas Piggin __ppc64_runlatch_off(); 3262201f994SNicholas Piggin srr1 = power7_idle_insn(type); 32740d24343SNicholas Piggin __ppc64_runlatch_on(); 3282201f994SNicholas Piggin 3292201f994SNicholas Piggin fini_irq_for_idle_irqsoff(); 3302201f994SNicholas Piggin 3312201f994SNicholas Piggin return srr1; 3322201f994SNicholas Piggin } 3332201f994SNicholas Piggin 3342201f994SNicholas Piggin void power7_idle_type(unsigned long type) 3352201f994SNicholas Piggin { 336771d4304SNicholas Piggin unsigned long srr1; 337771d4304SNicholas Piggin 338771d4304SNicholas Piggin srr1 = __power7_idle_type(type); 339771d4304SNicholas Piggin irq_set_pending_from_srr1(srr1); 3402201f994SNicholas Piggin } 3412201f994SNicholas Piggin 3422201f994SNicholas Piggin void power7_idle(void) 3432201f994SNicholas Piggin { 3442201f994SNicholas Piggin if (!powersave_nap) 3452201f994SNicholas Piggin return; 3462201f994SNicholas Piggin 3472201f994SNicholas Piggin power7_idle_type(PNV_THREAD_NAP); 3482201f994SNicholas Piggin } 3492201f994SNicholas Piggin 3502201f994SNicholas Piggin static unsigned long __power9_idle_type(unsigned long stop_psscr_val, 3512201f994SNicholas Piggin unsigned long stop_psscr_mask) 3522201f994SNicholas Piggin { 3532201f994SNicholas Piggin unsigned long psscr; 3542201f994SNicholas Piggin unsigned long srr1; 3552201f994SNicholas Piggin 3562201f994SNicholas Piggin if (!prep_irq_for_idle_irqsoff()) 3572201f994SNicholas Piggin return 0; 3582201f994SNicholas Piggin 3592201f994SNicholas Piggin psscr = mfspr(SPRN_PSSCR); 3602201f994SNicholas Piggin psscr = (psscr & ~stop_psscr_mask) | stop_psscr_val; 3612201f994SNicholas Piggin 36240d24343SNicholas Piggin __ppc64_runlatch_off(); 3632201f994SNicholas Piggin srr1 = power9_idle_stop(psscr); 36440d24343SNicholas Piggin __ppc64_runlatch_on(); 3652201f994SNicholas Piggin 3662201f994SNicholas Piggin fini_irq_for_idle_irqsoff(); 3672201f994SNicholas Piggin 3682201f994SNicholas Piggin return srr1; 3692201f994SNicholas Piggin } 3702201f994SNicholas Piggin 3712201f994SNicholas Piggin void power9_idle_type(unsigned long stop_psscr_val, 3722201f994SNicholas Piggin unsigned long stop_psscr_mask) 3732201f994SNicholas Piggin { 374771d4304SNicholas Piggin unsigned long srr1; 375771d4304SNicholas Piggin 376771d4304SNicholas Piggin srr1 = __power9_idle_type(stop_psscr_val, stop_psscr_mask); 377771d4304SNicholas Piggin irq_set_pending_from_srr1(srr1); 3782201f994SNicholas Piggin } 3792201f994SNicholas Piggin 38009206b60SGautham R. Shenoy /* 381bcef83a0SShreyas B. Prabhu * Used for ppc_md.power_save which needs a function with no parameters 382bcef83a0SShreyas B. Prabhu */ 3832201f994SNicholas Piggin void power9_idle(void) 384d405a98cSShreyas B. Prabhu { 3852201f994SNicholas Piggin power9_idle_type(pnv_default_stop_val, pnv_default_stop_mask); 386bcef83a0SShreyas B. Prabhu } 38709206b60SGautham R. Shenoy 3887672691aSPaul Mackerras #ifdef CONFIG_KVM_BOOK3S_HV_POSSIBLE 3897672691aSPaul Mackerras /* 3907672691aSPaul Mackerras * This is used in working around bugs in thread reconfiguration 3917672691aSPaul Mackerras * on POWER9 (at least up to Nimbus DD2.2) relating to transactional 3927672691aSPaul Mackerras * memory and the way that XER[SO] is checkpointed. 3937672691aSPaul Mackerras * This function forces the core into SMT4 in order by asking 3947672691aSPaul Mackerras * all other threads not to stop, and sending a message to any 3957672691aSPaul Mackerras * that are in a stop state. 3967672691aSPaul Mackerras * Must be called with preemption disabled. 3977672691aSPaul Mackerras */ 3987672691aSPaul Mackerras void pnv_power9_force_smt4_catch(void) 3997672691aSPaul Mackerras { 4007672691aSPaul Mackerras int cpu, cpu0, thr; 4017672691aSPaul Mackerras int awake_threads = 1; /* this thread is awake */ 4027672691aSPaul Mackerras int poke_threads = 0; 4037672691aSPaul Mackerras int need_awake = threads_per_core; 4047672691aSPaul Mackerras 4057672691aSPaul Mackerras cpu = smp_processor_id(); 4067672691aSPaul Mackerras cpu0 = cpu & ~(threads_per_core - 1); 4077672691aSPaul Mackerras for (thr = 0; thr < threads_per_core; ++thr) { 4087672691aSPaul Mackerras if (cpu != cpu0 + thr) 409f437c517SMichael Ellerman atomic_inc(&paca_ptrs[cpu0+thr]->dont_stop); 4107672691aSPaul Mackerras } 4117672691aSPaul Mackerras /* order setting dont_stop vs testing requested_psscr */ 4127672691aSPaul Mackerras mb(); 4137672691aSPaul Mackerras for (thr = 0; thr < threads_per_core; ++thr) { 414f437c517SMichael Ellerman if (!paca_ptrs[cpu0+thr]->requested_psscr) 4157672691aSPaul Mackerras ++awake_threads; 4167672691aSPaul Mackerras else 4177672691aSPaul Mackerras poke_threads |= (1 << thr); 4187672691aSPaul Mackerras } 4197672691aSPaul Mackerras 4207672691aSPaul Mackerras /* If at least 3 threads are awake, the core is in SMT4 already */ 4217672691aSPaul Mackerras if (awake_threads < need_awake) { 4227672691aSPaul Mackerras /* We have to wake some threads; we'll use msgsnd */ 4237672691aSPaul Mackerras for (thr = 0; thr < threads_per_core; ++thr) { 4247672691aSPaul Mackerras if (poke_threads & (1 << thr)) { 4257672691aSPaul Mackerras ppc_msgsnd_sync(); 4267672691aSPaul Mackerras ppc_msgsnd(PPC_DBELL_MSGTYPE, 0, 427f437c517SMichael Ellerman paca_ptrs[cpu0+thr]->hw_cpu_id); 4287672691aSPaul Mackerras } 4297672691aSPaul Mackerras } 4307672691aSPaul Mackerras /* now spin until at least 3 threads are awake */ 4317672691aSPaul Mackerras do { 4327672691aSPaul Mackerras for (thr = 0; thr < threads_per_core; ++thr) { 4337672691aSPaul Mackerras if ((poke_threads & (1 << thr)) && 434f437c517SMichael Ellerman !paca_ptrs[cpu0+thr]->requested_psscr) { 4357672691aSPaul Mackerras ++awake_threads; 4367672691aSPaul Mackerras poke_threads &= ~(1 << thr); 4377672691aSPaul Mackerras } 4387672691aSPaul Mackerras } 4397672691aSPaul Mackerras } while (awake_threads < need_awake); 4407672691aSPaul Mackerras } 4417672691aSPaul Mackerras } 4427672691aSPaul Mackerras EXPORT_SYMBOL_GPL(pnv_power9_force_smt4_catch); 4437672691aSPaul Mackerras 4447672691aSPaul Mackerras void pnv_power9_force_smt4_release(void) 4457672691aSPaul Mackerras { 4467672691aSPaul Mackerras int cpu, cpu0, thr; 4477672691aSPaul Mackerras 4487672691aSPaul Mackerras cpu = smp_processor_id(); 4497672691aSPaul Mackerras cpu0 = cpu & ~(threads_per_core - 1); 4507672691aSPaul Mackerras 4517672691aSPaul Mackerras /* clear all the dont_stop flags */ 4527672691aSPaul Mackerras for (thr = 0; thr < threads_per_core; ++thr) { 4537672691aSPaul Mackerras if (cpu != cpu0 + thr) 454f437c517SMichael Ellerman atomic_dec(&paca_ptrs[cpu0+thr]->dont_stop); 4557672691aSPaul Mackerras } 4567672691aSPaul Mackerras } 4577672691aSPaul Mackerras EXPORT_SYMBOL_GPL(pnv_power9_force_smt4_release); 4587672691aSPaul Mackerras #endif /* CONFIG_KVM_BOOK3S_HV_POSSIBLE */ 4597672691aSPaul Mackerras 46067d20418SNicholas Piggin #ifdef CONFIG_HOTPLUG_CPU 46124be85a2SGautham R. Shenoy static void pnv_program_cpu_hotplug_lpcr(unsigned int cpu, u64 lpcr_val) 46224be85a2SGautham R. Shenoy { 46324be85a2SGautham R. Shenoy u64 pir = get_hard_smp_processor_id(cpu); 46424be85a2SGautham R. Shenoy 46524be85a2SGautham R. Shenoy mtspr(SPRN_LPCR, lpcr_val); 4665d298baaSGautham R. Shenoy 4675d298baaSGautham R. Shenoy /* 4685d298baaSGautham R. Shenoy * Program the LPCR via stop-api only if the deepest stop state 4695d298baaSGautham R. Shenoy * can lose hypervisor context. 4705d298baaSGautham R. Shenoy */ 4715d298baaSGautham R. Shenoy if (supported_cpuidle_states & OPAL_PM_LOSE_FULL_CONTEXT) 47224be85a2SGautham R. Shenoy opal_slw_set_reg(pir, SPRN_LPCR, lpcr_val); 47324be85a2SGautham R. Shenoy } 47424be85a2SGautham R. Shenoy 475c0691f9dSShreyas B. Prabhu /* 476a7cd88daSGautham R. Shenoy * pnv_cpu_offline: A function that puts the CPU into the deepest 477a7cd88daSGautham R. Shenoy * available platform idle state on a CPU-Offline. 4782525db04SNicholas Piggin * interrupts hard disabled and no lazy irq pending. 479a7cd88daSGautham R. Shenoy */ 480a7cd88daSGautham R. Shenoy unsigned long pnv_cpu_offline(unsigned int cpu) 481a7cd88daSGautham R. Shenoy { 482a7cd88daSGautham R. Shenoy unsigned long srr1; 483a7cd88daSGautham R. Shenoy u32 idle_states = pnv_get_supported_cpuidle_states(); 48424be85a2SGautham R. Shenoy u64 lpcr_val; 48524be85a2SGautham R. Shenoy 48624be85a2SGautham R. Shenoy /* 48724be85a2SGautham R. Shenoy * We don't want to take decrementer interrupts while we are 48824be85a2SGautham R. Shenoy * offline, so clear LPCR:PECE1. We keep PECE2 (and 48924be85a2SGautham R. Shenoy * LPCR_PECE_HVEE on P9) enabled as to let IPIs in. 49024be85a2SGautham R. Shenoy * 49124be85a2SGautham R. Shenoy * If the CPU gets woken up by a special wakeup, ensure that 49224be85a2SGautham R. Shenoy * the SLW engine sets LPCR with decrementer bit cleared, else 49324be85a2SGautham R. Shenoy * the CPU will come back to the kernel due to a spurious 49424be85a2SGautham R. Shenoy * wakeup. 49524be85a2SGautham R. Shenoy */ 49624be85a2SGautham R. Shenoy lpcr_val = mfspr(SPRN_LPCR) & ~(u64)LPCR_PECE1; 49724be85a2SGautham R. Shenoy pnv_program_cpu_hotplug_lpcr(cpu, lpcr_val); 498a7cd88daSGautham R. Shenoy 49940d24343SNicholas Piggin __ppc64_runlatch_off(); 5002525db04SNicholas Piggin 501f3b3f284SGautham R. Shenoy if (cpu_has_feature(CPU_FTR_ARCH_300) && deepest_stop_found) { 5022525db04SNicholas Piggin unsigned long psscr; 5032525db04SNicholas Piggin 5042525db04SNicholas Piggin psscr = mfspr(SPRN_PSSCR); 5052525db04SNicholas Piggin psscr = (psscr & ~pnv_deepest_stop_psscr_mask) | 5062525db04SNicholas Piggin pnv_deepest_stop_psscr_val; 5073d4fbffdSNicholas Piggin srr1 = power9_offline_stop(psscr); 5082525db04SNicholas Piggin 509785a12afSGautham R. Shenoy } else if ((idle_states & OPAL_PM_WINKLE_ENABLED) && 510785a12afSGautham R. Shenoy (idle_states & OPAL_PM_LOSE_FULL_CONTEXT)) { 5112525db04SNicholas Piggin srr1 = power7_idle_insn(PNV_THREAD_WINKLE); 512a7cd88daSGautham R. Shenoy } else if ((idle_states & OPAL_PM_SLEEP_ENABLED) || 513a7cd88daSGautham R. Shenoy (idle_states & OPAL_PM_SLEEP_ENABLED_ER1)) { 5142525db04SNicholas Piggin srr1 = power7_idle_insn(PNV_THREAD_SLEEP); 51590061231SGautham R. Shenoy } else if (idle_states & OPAL_PM_NAP_ENABLED) { 5162525db04SNicholas Piggin srr1 = power7_idle_insn(PNV_THREAD_NAP); 51790061231SGautham R. Shenoy } else { 51890061231SGautham R. Shenoy /* This is the fallback method. We emulate snooze */ 51990061231SGautham R. Shenoy while (!generic_check_cpu_restart(cpu)) { 52090061231SGautham R. Shenoy HMT_low(); 52190061231SGautham R. Shenoy HMT_very_low(); 52290061231SGautham R. Shenoy } 52390061231SGautham R. Shenoy srr1 = 0; 52490061231SGautham R. Shenoy HMT_medium(); 525a7cd88daSGautham R. Shenoy } 526a7cd88daSGautham R. Shenoy 52740d24343SNicholas Piggin __ppc64_runlatch_on(); 5282525db04SNicholas Piggin 52924be85a2SGautham R. Shenoy /* 53024be85a2SGautham R. Shenoy * Re-enable decrementer interrupts in LPCR. 53124be85a2SGautham R. Shenoy * 53224be85a2SGautham R. Shenoy * Further, we want stop states to be woken up by decrementer 53324be85a2SGautham R. Shenoy * for non-hotplug cases. So program the LPCR via stop api as 53424be85a2SGautham R. Shenoy * well. 53524be85a2SGautham R. Shenoy */ 53624be85a2SGautham R. Shenoy lpcr_val = mfspr(SPRN_LPCR) | (u64)LPCR_PECE1; 53724be85a2SGautham R. Shenoy pnv_program_cpu_hotplug_lpcr(cpu, lpcr_val); 53824be85a2SGautham R. Shenoy 539a7cd88daSGautham R. Shenoy return srr1; 540a7cd88daSGautham R. Shenoy } 54167d20418SNicholas Piggin #endif 542a7cd88daSGautham R. Shenoy 543a7cd88daSGautham R. Shenoy /* 544bcef83a0SShreyas B. Prabhu * Power ISA 3.0 idle initialization. 545bcef83a0SShreyas B. Prabhu * 546bcef83a0SShreyas B. Prabhu * POWER ISA 3.0 defines a new SPR Processor stop Status and Control 547bcef83a0SShreyas B. Prabhu * Register (PSSCR) to control idle behavior. 548bcef83a0SShreyas B. Prabhu * 549bcef83a0SShreyas B. Prabhu * PSSCR layout: 550bcef83a0SShreyas B. Prabhu * ---------------------------------------------------------- 551bcef83a0SShreyas B. Prabhu * | PLS | /// | SD | ESL | EC | PSLL | /// | TR | MTL | RL | 552bcef83a0SShreyas B. Prabhu * ---------------------------------------------------------- 553bcef83a0SShreyas B. Prabhu * 0 4 41 42 43 44 48 54 56 60 554bcef83a0SShreyas B. Prabhu * 555bcef83a0SShreyas B. Prabhu * PSSCR key fields: 556bcef83a0SShreyas B. Prabhu * Bits 0:3 - Power-Saving Level Status (PLS). This field indicates the 557bcef83a0SShreyas B. Prabhu * lowest power-saving state the thread entered since stop instruction was 558bcef83a0SShreyas B. Prabhu * last executed. 559bcef83a0SShreyas B. Prabhu * 560bcef83a0SShreyas B. Prabhu * Bit 41 - Status Disable(SD) 561bcef83a0SShreyas B. Prabhu * 0 - Shows PLS entries 562bcef83a0SShreyas B. Prabhu * 1 - PLS entries are all 0 563bcef83a0SShreyas B. Prabhu * 564bcef83a0SShreyas B. Prabhu * Bit 42 - Enable State Loss 565bcef83a0SShreyas B. Prabhu * 0 - No state is lost irrespective of other fields 566bcef83a0SShreyas B. Prabhu * 1 - Allows state loss 567bcef83a0SShreyas B. Prabhu * 568bcef83a0SShreyas B. Prabhu * Bit 43 - Exit Criterion 569bcef83a0SShreyas B. Prabhu * 0 - Exit from power-save mode on any interrupt 570bcef83a0SShreyas B. Prabhu * 1 - Exit from power-save mode controlled by LPCR's PECE bits 571bcef83a0SShreyas B. Prabhu * 572bcef83a0SShreyas B. Prabhu * Bits 44:47 - Power-Saving Level Limit 573bcef83a0SShreyas B. Prabhu * This limits the power-saving level that can be entered into. 574bcef83a0SShreyas B. Prabhu * 575bcef83a0SShreyas B. Prabhu * Bits 60:63 - Requested Level 576bcef83a0SShreyas B. Prabhu * Used to specify which power-saving level must be entered on executing 577bcef83a0SShreyas B. Prabhu * stop instruction 57809206b60SGautham R. Shenoy */ 57909206b60SGautham R. Shenoy 58009206b60SGautham R. Shenoy int validate_psscr_val_mask(u64 *psscr_val, u64 *psscr_mask, u32 flags) 58109206b60SGautham R. Shenoy { 58209206b60SGautham R. Shenoy int err = 0; 58309206b60SGautham R. Shenoy 58409206b60SGautham R. Shenoy /* 58509206b60SGautham R. Shenoy * psscr_mask == 0xf indicates an older firmware. 58609206b60SGautham R. Shenoy * Set remaining fields of psscr to the default values. 58709206b60SGautham R. Shenoy * See NOTE above definition of PSSCR_HV_DEFAULT_VAL 58809206b60SGautham R. Shenoy */ 58909206b60SGautham R. Shenoy if (*psscr_mask == 0xf) { 59009206b60SGautham R. Shenoy *psscr_val = *psscr_val | PSSCR_HV_DEFAULT_VAL; 59109206b60SGautham R. Shenoy *psscr_mask = PSSCR_HV_DEFAULT_MASK; 59209206b60SGautham R. Shenoy return err; 59309206b60SGautham R. Shenoy } 59409206b60SGautham R. Shenoy 59509206b60SGautham R. Shenoy /* 59609206b60SGautham R. Shenoy * New firmware is expected to set the psscr_val bits correctly. 59709206b60SGautham R. Shenoy * Validate that the following invariants are correctly maintained by 59809206b60SGautham R. Shenoy * the new firmware. 59909206b60SGautham R. Shenoy * - ESL bit value matches the EC bit value. 60009206b60SGautham R. Shenoy * - ESL bit is set for all the deep stop states. 60109206b60SGautham R. Shenoy */ 60209206b60SGautham R. Shenoy if (GET_PSSCR_ESL(*psscr_val) != GET_PSSCR_EC(*psscr_val)) { 60309206b60SGautham R. Shenoy err = ERR_EC_ESL_MISMATCH; 60409206b60SGautham R. Shenoy } else if ((flags & OPAL_PM_LOSE_FULL_CONTEXT) && 60509206b60SGautham R. Shenoy GET_PSSCR_ESL(*psscr_val) == 0) { 60609206b60SGautham R. Shenoy err = ERR_DEEP_STATE_ESL_MISMATCH; 60709206b60SGautham R. Shenoy } 60809206b60SGautham R. Shenoy 60909206b60SGautham R. Shenoy return err; 61009206b60SGautham R. Shenoy } 61109206b60SGautham R. Shenoy 61209206b60SGautham R. Shenoy /* 61309206b60SGautham R. Shenoy * pnv_arch300_idle_init: Initializes the default idle state, first 61409206b60SGautham R. Shenoy * deep idle state and deepest idle state on 61509206b60SGautham R. Shenoy * ISA 3.0 CPUs. 616bcef83a0SShreyas B. Prabhu * 617bcef83a0SShreyas B. Prabhu * @np: /ibm,opal/power-mgt device node 618bcef83a0SShreyas B. Prabhu * @flags: cpu-idle-state-flags array 619bcef83a0SShreyas B. Prabhu * @dt_idle_states: Number of idle state entries 620bcef83a0SShreyas B. Prabhu * Returns 0 on success 621bcef83a0SShreyas B. Prabhu */ 6229c7b185aSAkshay Adiga static int __init pnv_power9_idle_init(void) 623bcef83a0SShreyas B. Prabhu { 62409206b60SGautham R. Shenoy u64 max_residency_ns = 0; 6259c7b185aSAkshay Adiga int i; 626bcef83a0SShreyas B. Prabhu 627bcef83a0SShreyas B. Prabhu /* 62809206b60SGautham R. Shenoy * Set pnv_first_deep_stop_state, pnv_deepest_stop_psscr_{val,mask}, 62909206b60SGautham R. Shenoy * and the pnv_default_stop_{val,mask}. 63009206b60SGautham R. Shenoy * 631c0691f9dSShreyas B. Prabhu * pnv_first_deep_stop_state should be set to the first stop 632c0691f9dSShreyas B. Prabhu * level to cause hypervisor state loss. 63309206b60SGautham R. Shenoy * 63409206b60SGautham R. Shenoy * pnv_deepest_stop_{val,mask} should be set to values corresponding to 63509206b60SGautham R. Shenoy * the deepest stop state. 63609206b60SGautham R. Shenoy * 63709206b60SGautham R. Shenoy * pnv_default_stop_{val,mask} should be set to values corresponding to 63809206b60SGautham R. Shenoy * the shallowest (OPAL_PM_STOP_INST_FAST) loss-less stop state. 639bcef83a0SShreyas B. Prabhu */ 640bcef83a0SShreyas B. Prabhu pnv_first_deep_stop_state = MAX_STOP_STATE; 6419c7b185aSAkshay Adiga for (i = 0; i < nr_pnv_idle_states; i++) { 64209206b60SGautham R. Shenoy int err; 6439c7b185aSAkshay Adiga struct pnv_idle_states_t *state = &pnv_idle_states[i]; 6449c7b185aSAkshay Adiga u64 psscr_rl = state->psscr_val & PSSCR_RL_MASK; 645bcef83a0SShreyas B. Prabhu 6469c7b185aSAkshay Adiga if ((state->flags & OPAL_PM_LOSE_FULL_CONTEXT) && 6479c7b185aSAkshay Adiga pnv_first_deep_stop_state > psscr_rl) 648bcef83a0SShreyas B. Prabhu pnv_first_deep_stop_state = psscr_rl; 649c0691f9dSShreyas B. Prabhu 6509c7b185aSAkshay Adiga err = validate_psscr_val_mask(&state->psscr_val, 6519c7b185aSAkshay Adiga &state->psscr_mask, 6529c7b185aSAkshay Adiga state->flags); 65309206b60SGautham R. Shenoy if (err) { 6549c7b185aSAkshay Adiga state->valid = false; 6559c7b185aSAkshay Adiga report_invalid_psscr_val(state->psscr_val, err); 65609206b60SGautham R. Shenoy continue; 65709206b60SGautham R. Shenoy } 65809206b60SGautham R. Shenoy 6599c7b185aSAkshay Adiga if (max_residency_ns < state->residency_ns) { 6609c7b185aSAkshay Adiga max_residency_ns = state->residency_ns; 6619c7b185aSAkshay Adiga pnv_deepest_stop_psscr_val = state->psscr_val; 6629c7b185aSAkshay Adiga pnv_deepest_stop_psscr_mask = state->psscr_mask; 6639c7b185aSAkshay Adiga pnv_deepest_stop_flag = state->flags; 66409206b60SGautham R. Shenoy deepest_stop_found = true; 66509206b60SGautham R. Shenoy } 66609206b60SGautham R. Shenoy 66709206b60SGautham R. Shenoy if (!default_stop_found && 6689c7b185aSAkshay Adiga (state->flags & OPAL_PM_STOP_INST_FAST)) { 6699c7b185aSAkshay Adiga pnv_default_stop_val = state->psscr_val; 6709c7b185aSAkshay Adiga pnv_default_stop_mask = state->psscr_mask; 67109206b60SGautham R. Shenoy default_stop_found = true; 67209206b60SGautham R. Shenoy } 67309206b60SGautham R. Shenoy } 67409206b60SGautham R. Shenoy 675f3b3f284SGautham R. Shenoy if (unlikely(!default_stop_found)) { 676f3b3f284SGautham R. Shenoy pr_warn("cpuidle-powernv: No suitable default stop state found. Disabling platform idle.\n"); 677f3b3f284SGautham R. Shenoy } else { 678f3b3f284SGautham R. Shenoy ppc_md.power_save = power9_idle; 679f3b3f284SGautham R. Shenoy pr_info("cpuidle-powernv: Default stop: psscr = 0x%016llx,mask=0x%016llx\n", 68009206b60SGautham R. Shenoy pnv_default_stop_val, pnv_default_stop_mask); 68109206b60SGautham R. Shenoy } 68209206b60SGautham R. Shenoy 683f3b3f284SGautham R. Shenoy if (unlikely(!deepest_stop_found)) { 684f3b3f284SGautham R. Shenoy pr_warn("cpuidle-powernv: No suitable stop state for CPU-Hotplug. Offlined CPUs will busy wait"); 685f3b3f284SGautham R. Shenoy } else { 686f3b3f284SGautham R. Shenoy pr_info("cpuidle-powernv: Deepest stop: psscr = 0x%016llx,mask=0x%016llx\n", 68709206b60SGautham R. Shenoy pnv_deepest_stop_psscr_val, 68809206b60SGautham R. Shenoy pnv_deepest_stop_psscr_mask); 689bcef83a0SShreyas B. Prabhu } 690bcef83a0SShreyas B. Prabhu 691f3b3f284SGautham R. Shenoy pr_info("cpuidle-powernv: Requested Level (RL) value of first deep stop = 0x%llx\n", 692f3b3f284SGautham R. Shenoy pnv_first_deep_stop_state); 6939c7b185aSAkshay Adiga 6949c7b185aSAkshay Adiga return 0; 695bcef83a0SShreyas B. Prabhu } 696bcef83a0SShreyas B. Prabhu 697bcef83a0SShreyas B. Prabhu /* 698bcef83a0SShreyas B. Prabhu * Probe device tree for supported idle states 699bcef83a0SShreyas B. Prabhu */ 700bcef83a0SShreyas B. Prabhu static void __init pnv_probe_idle_states(void) 701bcef83a0SShreyas B. Prabhu { 702d405a98cSShreyas B. Prabhu int i; 703d405a98cSShreyas B. Prabhu 7049c7b185aSAkshay Adiga if (nr_pnv_idle_states < 0) { 7059c7b185aSAkshay Adiga pr_warn("cpuidle-powernv: no idle states found in the DT\n"); 7069c7b185aSAkshay Adiga return; 7079c7b185aSAkshay Adiga } 7089c7b185aSAkshay Adiga 7099c7b185aSAkshay Adiga if (cpu_has_feature(CPU_FTR_ARCH_300)) { 7109c7b185aSAkshay Adiga if (pnv_power9_idle_init()) 7119c7b185aSAkshay Adiga return; 7129c7b185aSAkshay Adiga } 7139c7b185aSAkshay Adiga 7149c7b185aSAkshay Adiga for (i = 0; i < nr_pnv_idle_states; i++) 7159c7b185aSAkshay Adiga supported_cpuidle_states |= pnv_idle_states[i].flags; 7169c7b185aSAkshay Adiga } 7179c7b185aSAkshay Adiga 7189c7b185aSAkshay Adiga /* 7199c7b185aSAkshay Adiga * This function parses device-tree and populates all the information 7209c7b185aSAkshay Adiga * into pnv_idle_states structure. It also sets up nr_pnv_idle_states 7219c7b185aSAkshay Adiga * which is the number of cpuidle states discovered through device-tree. 7229c7b185aSAkshay Adiga */ 7239c7b185aSAkshay Adiga 7249c7b185aSAkshay Adiga static int pnv_parse_cpuidle_dt(void) 7259c7b185aSAkshay Adiga { 7269c7b185aSAkshay Adiga struct device_node *np; 7279c7b185aSAkshay Adiga int nr_idle_states, i; 7289c7b185aSAkshay Adiga int rc = 0; 7299c7b185aSAkshay Adiga u32 *temp_u32; 7309c7b185aSAkshay Adiga u64 *temp_u64; 7319c7b185aSAkshay Adiga const char **temp_string; 7329c7b185aSAkshay Adiga 733bcef83a0SShreyas B. Prabhu np = of_find_node_by_path("/ibm,opal/power-mgt"); 734bcef83a0SShreyas B. Prabhu if (!np) { 735d405a98cSShreyas B. Prabhu pr_warn("opal: PowerMgmt Node not found\n"); 7369c7b185aSAkshay Adiga return -ENODEV; 737d405a98cSShreyas B. Prabhu } 7389c7b185aSAkshay Adiga nr_idle_states = of_property_count_u32_elems(np, 739d405a98cSShreyas B. Prabhu "ibm,cpu-idle-state-flags"); 7409c7b185aSAkshay Adiga 7419c7b185aSAkshay Adiga pnv_idle_states = kcalloc(nr_idle_states, sizeof(*pnv_idle_states), 7429c7b185aSAkshay Adiga GFP_KERNEL); 7439c7b185aSAkshay Adiga temp_u32 = kcalloc(nr_idle_states, sizeof(u32), GFP_KERNEL); 7449c7b185aSAkshay Adiga temp_u64 = kcalloc(nr_idle_states, sizeof(u64), GFP_KERNEL); 7459c7b185aSAkshay Adiga temp_string = kcalloc(nr_idle_states, sizeof(char *), GFP_KERNEL); 7469c7b185aSAkshay Adiga 7479c7b185aSAkshay Adiga if (!(pnv_idle_states && temp_u32 && temp_u64 && temp_string)) { 7489c7b185aSAkshay Adiga pr_err("Could not allocate memory for dt parsing\n"); 7499c7b185aSAkshay Adiga rc = -ENOMEM; 750d405a98cSShreyas B. Prabhu goto out; 751d405a98cSShreyas B. Prabhu } 752d405a98cSShreyas B. Prabhu 7539c7b185aSAkshay Adiga /* Read flags */ 7549c7b185aSAkshay Adiga if (of_property_read_u32_array(np, "ibm,cpu-idle-state-flags", 7559c7b185aSAkshay Adiga temp_u32, nr_idle_states)) { 756d405a98cSShreyas B. Prabhu pr_warn("cpuidle-powernv: missing ibm,cpu-idle-state-flags in DT\n"); 7579c7b185aSAkshay Adiga rc = -EINVAL; 758bcef83a0SShreyas B. Prabhu goto out; 759bcef83a0SShreyas B. Prabhu } 7609c7b185aSAkshay Adiga for (i = 0; i < nr_idle_states; i++) 7619c7b185aSAkshay Adiga pnv_idle_states[i].flags = temp_u32[i]; 762bcef83a0SShreyas B. Prabhu 7639c7b185aSAkshay Adiga /* Read latencies */ 7649c7b185aSAkshay Adiga if (of_property_read_u32_array(np, "ibm,cpu-idle-state-latencies-ns", 7659c7b185aSAkshay Adiga temp_u32, nr_idle_states)) { 7669c7b185aSAkshay Adiga pr_warn("cpuidle-powernv: missing ibm,cpu-idle-state-latencies-ns in DT\n"); 7679c7b185aSAkshay Adiga rc = -EINVAL; 7689c7b185aSAkshay Adiga goto out; 7699c7b185aSAkshay Adiga } 7709c7b185aSAkshay Adiga for (i = 0; i < nr_idle_states; i++) 7719c7b185aSAkshay Adiga pnv_idle_states[i].latency_ns = temp_u32[i]; 7729c7b185aSAkshay Adiga 7739c7b185aSAkshay Adiga /* Read residencies */ 7749c7b185aSAkshay Adiga if (of_property_read_u32_array(np, "ibm,cpu-idle-state-residency-ns", 7759c7b185aSAkshay Adiga temp_u32, nr_idle_states)) { 7769c7b185aSAkshay Adiga pr_warn("cpuidle-powernv: missing ibm,cpu-idle-state-latencies-ns in DT\n"); 7779c7b185aSAkshay Adiga rc = -EINVAL; 7789c7b185aSAkshay Adiga goto out; 7799c7b185aSAkshay Adiga } 7809c7b185aSAkshay Adiga for (i = 0; i < nr_idle_states; i++) 7819c7b185aSAkshay Adiga pnv_idle_states[i].residency_ns = temp_u32[i]; 7829c7b185aSAkshay Adiga 7839c7b185aSAkshay Adiga /* For power9 */ 784bcef83a0SShreyas B. Prabhu if (cpu_has_feature(CPU_FTR_ARCH_300)) { 7859c7b185aSAkshay Adiga /* Read pm_crtl_val */ 7869c7b185aSAkshay Adiga if (of_property_read_u64_array(np, "ibm,cpu-idle-state-psscr", 7879c7b185aSAkshay Adiga temp_u64, nr_idle_states)) { 7889c7b185aSAkshay Adiga pr_warn("cpuidle-powernv: missing ibm,cpu-idle-state-psscr in DT\n"); 7899c7b185aSAkshay Adiga rc = -EINVAL; 790bcef83a0SShreyas B. Prabhu goto out; 791d405a98cSShreyas B. Prabhu } 7929c7b185aSAkshay Adiga for (i = 0; i < nr_idle_states; i++) 7939c7b185aSAkshay Adiga pnv_idle_states[i].psscr_val = temp_u64[i]; 794d405a98cSShreyas B. Prabhu 7959c7b185aSAkshay Adiga /* Read pm_crtl_mask */ 7969c7b185aSAkshay Adiga if (of_property_read_u64_array(np, "ibm,cpu-idle-state-psscr-mask", 7979c7b185aSAkshay Adiga temp_u64, nr_idle_states)) { 7989c7b185aSAkshay Adiga pr_warn("cpuidle-powernv: missing ibm,cpu-idle-state-psscr-mask in DT\n"); 7999c7b185aSAkshay Adiga rc = -EINVAL; 8009c7b185aSAkshay Adiga goto out; 801bcef83a0SShreyas B. Prabhu } 8029c7b185aSAkshay Adiga for (i = 0; i < nr_idle_states; i++) 8039c7b185aSAkshay Adiga pnv_idle_states[i].psscr_mask = temp_u64[i]; 8049c7b185aSAkshay Adiga } 8059c7b185aSAkshay Adiga 8069c7b185aSAkshay Adiga /* 8079c7b185aSAkshay Adiga * power8 specific properties ibm,cpu-idle-state-pmicr-mask and 8089c7b185aSAkshay Adiga * ibm,cpu-idle-state-pmicr-val were never used and there is no 8099c7b185aSAkshay Adiga * plan to use it in near future. Hence, not parsing these properties 8109c7b185aSAkshay Adiga */ 8119c7b185aSAkshay Adiga 8129c7b185aSAkshay Adiga if (of_property_read_string_array(np, "ibm,cpu-idle-state-names", 8139c7b185aSAkshay Adiga temp_string, nr_idle_states) < 0) { 8149c7b185aSAkshay Adiga pr_warn("cpuidle-powernv: missing ibm,cpu-idle-state-names in DT\n"); 8159c7b185aSAkshay Adiga rc = -EINVAL; 8169c7b185aSAkshay Adiga goto out; 8179c7b185aSAkshay Adiga } 8189c7b185aSAkshay Adiga for (i = 0; i < nr_idle_states; i++) 8199c7b185aSAkshay Adiga strncpy(pnv_idle_states[i].name, temp_string[i], 8209c7b185aSAkshay Adiga PNV_IDLE_NAME_LEN); 8219c7b185aSAkshay Adiga nr_pnv_idle_states = nr_idle_states; 8229c7b185aSAkshay Adiga rc = 0; 8239c7b185aSAkshay Adiga out: 8249c7b185aSAkshay Adiga kfree(temp_u32); 8259c7b185aSAkshay Adiga kfree(temp_u64); 8269c7b185aSAkshay Adiga kfree(temp_string); 8279c7b185aSAkshay Adiga return rc; 8289c7b185aSAkshay Adiga } 8299c7b185aSAkshay Adiga 830bcef83a0SShreyas B. Prabhu static int __init pnv_init_idle_states(void) 831bcef83a0SShreyas B. Prabhu { 8329c7b185aSAkshay Adiga int rc = 0; 833bcef83a0SShreyas B. Prabhu supported_cpuidle_states = 0; 834bcef83a0SShreyas B. Prabhu 8359c7b185aSAkshay Adiga /* In case we error out nr_pnv_idle_states will be zero */ 8369c7b185aSAkshay Adiga nr_pnv_idle_states = 0; 837bcef83a0SShreyas B. Prabhu if (cpuidle_disable != IDLE_NO_OVERRIDE) 838bcef83a0SShreyas B. Prabhu goto out; 8399c7b185aSAkshay Adiga rc = pnv_parse_cpuidle_dt(); 8409c7b185aSAkshay Adiga if (rc) 8419c7b185aSAkshay Adiga return rc; 842bcef83a0SShreyas B. Prabhu pnv_probe_idle_states(); 843bcef83a0SShreyas B. Prabhu 844d405a98cSShreyas B. Prabhu if (!(supported_cpuidle_states & OPAL_PM_SLEEP_ENABLED_ER1)) { 845d405a98cSShreyas B. Prabhu patch_instruction( 846d405a98cSShreyas B. Prabhu (unsigned int *)pnv_fastsleep_workaround_at_entry, 847d405a98cSShreyas B. Prabhu PPC_INST_NOP); 848d405a98cSShreyas B. Prabhu patch_instruction( 849d405a98cSShreyas B. Prabhu (unsigned int *)pnv_fastsleep_workaround_at_exit, 850d405a98cSShreyas B. Prabhu PPC_INST_NOP); 8515703d2f4SShreyas B. Prabhu } else { 8525703d2f4SShreyas B. Prabhu /* 8535703d2f4SShreyas B. Prabhu * OPAL_PM_SLEEP_ENABLED_ER1 is set. It indicates that 8545703d2f4SShreyas B. Prabhu * workaround is needed to use fastsleep. Provide sysfs 8555703d2f4SShreyas B. Prabhu * control to choose how this workaround has to be applied. 8565703d2f4SShreyas B. Prabhu */ 8575703d2f4SShreyas B. Prabhu device_create_file(cpu_subsys.dev_root, 8585703d2f4SShreyas B. Prabhu &dev_attr_fastsleep_workaround_applyonce); 859d405a98cSShreyas B. Prabhu } 8605703d2f4SShreyas B. Prabhu 861d405a98cSShreyas B. Prabhu pnv_alloc_idle_core_states(); 8625593e303SShreyas B. Prabhu 8635593e303SShreyas B. Prabhu if (supported_cpuidle_states & OPAL_PM_NAP_ENABLED) 8645593e303SShreyas B. Prabhu ppc_md.power_save = power7_idle; 865bcef83a0SShreyas B. Prabhu 866d405a98cSShreyas B. Prabhu out: 867d405a98cSShreyas B. Prabhu return 0; 868d405a98cSShreyas B. Prabhu } 8694bece972SMichael Ellerman machine_subsys_initcall(powernv, pnv_init_idle_states); 870