1d405a98cSShreyas B. Prabhu /* 2d405a98cSShreyas B. Prabhu * PowerNV cpuidle code 3d405a98cSShreyas B. Prabhu * 4d405a98cSShreyas B. Prabhu * Copyright 2015 IBM Corp. 5d405a98cSShreyas B. Prabhu * 6d405a98cSShreyas B. Prabhu * This program is free software; you can redistribute it and/or 7d405a98cSShreyas B. Prabhu * modify it under the terms of the GNU General Public License 8d405a98cSShreyas B. Prabhu * as published by the Free Software Foundation; either version 9d405a98cSShreyas B. Prabhu * 2 of the License, or (at your option) any later version. 10d405a98cSShreyas B. Prabhu */ 11d405a98cSShreyas B. Prabhu 12d405a98cSShreyas B. Prabhu #include <linux/types.h> 13d405a98cSShreyas B. Prabhu #include <linux/mm.h> 14d405a98cSShreyas B. Prabhu #include <linux/slab.h> 15d405a98cSShreyas B. Prabhu #include <linux/of.h> 165703d2f4SShreyas B. Prabhu #include <linux/device.h> 175703d2f4SShreyas B. Prabhu #include <linux/cpu.h> 18d405a98cSShreyas B. Prabhu 19d405a98cSShreyas B. Prabhu #include <asm/firmware.h> 204bece972SMichael Ellerman #include <asm/machdep.h> 21d405a98cSShreyas B. Prabhu #include <asm/opal.h> 22d405a98cSShreyas B. Prabhu #include <asm/cputhreads.h> 23d405a98cSShreyas B. Prabhu #include <asm/cpuidle.h> 24d405a98cSShreyas B. Prabhu #include <asm/code-patching.h> 25d405a98cSShreyas B. Prabhu #include <asm/smp.h> 26d405a98cSShreyas B. Prabhu 27d405a98cSShreyas B. Prabhu #include "powernv.h" 28d405a98cSShreyas B. Prabhu #include "subcore.h" 29d405a98cSShreyas B. Prabhu 30bcef83a0SShreyas B. Prabhu /* Power ISA 3.0 allows for stop states 0x0 - 0xF */ 31bcef83a0SShreyas B. Prabhu #define MAX_STOP_STATE 0xF 32bcef83a0SShreyas B. Prabhu 331e1601b3SAkshay Adiga #define P9_STOP_SPR_MSR 2000 341e1601b3SAkshay Adiga #define P9_STOP_SPR_PSSCR 855 351e1601b3SAkshay Adiga 36d405a98cSShreyas B. Prabhu static u32 supported_cpuidle_states; 37d405a98cSShreyas B. Prabhu 381e1601b3SAkshay Adiga /* 391e1601b3SAkshay Adiga * The default stop state that will be used by ppc_md.power_save 401e1601b3SAkshay Adiga * function on platforms that support stop instruction. 411e1601b3SAkshay Adiga */ 421e1601b3SAkshay Adiga static u64 pnv_default_stop_val; 431e1601b3SAkshay Adiga static u64 pnv_default_stop_mask; 441e1601b3SAkshay Adiga static bool default_stop_found; 451e1601b3SAkshay Adiga 461e1601b3SAkshay Adiga /* 471e1601b3SAkshay Adiga * First deep stop state. Used to figure out when to save/restore 481e1601b3SAkshay Adiga * hypervisor context. 491e1601b3SAkshay Adiga */ 501e1601b3SAkshay Adiga u64 pnv_first_deep_stop_state = MAX_STOP_STATE; 511e1601b3SAkshay Adiga 521e1601b3SAkshay Adiga /* 531e1601b3SAkshay Adiga * psscr value and mask of the deepest stop idle state. 541e1601b3SAkshay Adiga * Used when a cpu is offlined. 551e1601b3SAkshay Adiga */ 561e1601b3SAkshay Adiga static u64 pnv_deepest_stop_psscr_val; 571e1601b3SAkshay Adiga static u64 pnv_deepest_stop_psscr_mask; 581e1601b3SAkshay Adiga static bool deepest_stop_found; 591e1601b3SAkshay Adiga 60bcef83a0SShreyas B. Prabhu static int pnv_save_sprs_for_deep_states(void) 61d405a98cSShreyas B. Prabhu { 62d405a98cSShreyas B. Prabhu int cpu; 63d405a98cSShreyas B. Prabhu int rc; 64d405a98cSShreyas B. Prabhu 65d405a98cSShreyas B. Prabhu /* 66446957baSAdam Buchbinder * hid0, hid1, hid4, hid5, hmeer and lpcr values are symmetric across 67d405a98cSShreyas B. Prabhu * all cpus at boot. Get these reg values of current cpu and use the 68446957baSAdam Buchbinder * same across all cpus. 69d405a98cSShreyas B. Prabhu */ 70d405a98cSShreyas B. Prabhu uint64_t lpcr_val = mfspr(SPRN_LPCR) & ~(u64)LPCR_PECE1; 71d405a98cSShreyas B. Prabhu uint64_t hid0_val = mfspr(SPRN_HID0); 72d405a98cSShreyas B. Prabhu uint64_t hid1_val = mfspr(SPRN_HID1); 73d405a98cSShreyas B. Prabhu uint64_t hid4_val = mfspr(SPRN_HID4); 74d405a98cSShreyas B. Prabhu uint64_t hid5_val = mfspr(SPRN_HID5); 75d405a98cSShreyas B. Prabhu uint64_t hmeer_val = mfspr(SPRN_HMEER); 761e1601b3SAkshay Adiga uint64_t msr_val = MSR_IDLE; 771e1601b3SAkshay Adiga uint64_t psscr_val = pnv_deepest_stop_psscr_val; 78d405a98cSShreyas B. Prabhu 79d405a98cSShreyas B. Prabhu for_each_possible_cpu(cpu) { 80d405a98cSShreyas B. Prabhu uint64_t pir = get_hard_smp_processor_id(cpu); 81d405a98cSShreyas B. Prabhu uint64_t hsprg0_val = (uint64_t)&paca[cpu]; 82d405a98cSShreyas B. Prabhu 83d405a98cSShreyas B. Prabhu rc = opal_slw_set_reg(pir, SPRN_HSPRG0, hsprg0_val); 84d405a98cSShreyas B. Prabhu if (rc != 0) 85d405a98cSShreyas B. Prabhu return rc; 86d405a98cSShreyas B. Prabhu 87d405a98cSShreyas B. Prabhu rc = opal_slw_set_reg(pir, SPRN_LPCR, lpcr_val); 88d405a98cSShreyas B. Prabhu if (rc != 0) 89d405a98cSShreyas B. Prabhu return rc; 90d405a98cSShreyas B. Prabhu 911e1601b3SAkshay Adiga if (cpu_has_feature(CPU_FTR_ARCH_300)) { 921e1601b3SAkshay Adiga rc = opal_slw_set_reg(pir, P9_STOP_SPR_MSR, msr_val); 931e1601b3SAkshay Adiga if (rc) 941e1601b3SAkshay Adiga return rc; 951e1601b3SAkshay Adiga 961e1601b3SAkshay Adiga rc = opal_slw_set_reg(pir, 971e1601b3SAkshay Adiga P9_STOP_SPR_PSSCR, psscr_val); 981e1601b3SAkshay Adiga 991e1601b3SAkshay Adiga if (rc) 1001e1601b3SAkshay Adiga return rc; 1011e1601b3SAkshay Adiga } 1021e1601b3SAkshay Adiga 103d405a98cSShreyas B. Prabhu /* HIDs are per core registers */ 104d405a98cSShreyas B. Prabhu if (cpu_thread_in_core(cpu) == 0) { 105d405a98cSShreyas B. Prabhu 106d405a98cSShreyas B. Prabhu rc = opal_slw_set_reg(pir, SPRN_HMEER, hmeer_val); 107d405a98cSShreyas B. Prabhu if (rc != 0) 108d405a98cSShreyas B. Prabhu return rc; 109d405a98cSShreyas B. Prabhu 110d405a98cSShreyas B. Prabhu rc = opal_slw_set_reg(pir, SPRN_HID0, hid0_val); 111d405a98cSShreyas B. Prabhu if (rc != 0) 112d405a98cSShreyas B. Prabhu return rc; 113d405a98cSShreyas B. Prabhu 1141e1601b3SAkshay Adiga /* Only p8 needs to set extra HID regiters */ 1151e1601b3SAkshay Adiga if (!cpu_has_feature(CPU_FTR_ARCH_300)) { 1161e1601b3SAkshay Adiga 117d405a98cSShreyas B. Prabhu rc = opal_slw_set_reg(pir, SPRN_HID1, hid1_val); 118d405a98cSShreyas B. Prabhu if (rc != 0) 119d405a98cSShreyas B. Prabhu return rc; 120d405a98cSShreyas B. Prabhu 121d405a98cSShreyas B. Prabhu rc = opal_slw_set_reg(pir, SPRN_HID4, hid4_val); 122d405a98cSShreyas B. Prabhu if (rc != 0) 123d405a98cSShreyas B. Prabhu return rc; 124d405a98cSShreyas B. Prabhu 125d405a98cSShreyas B. Prabhu rc = opal_slw_set_reg(pir, SPRN_HID5, hid5_val); 126d405a98cSShreyas B. Prabhu if (rc != 0) 127d405a98cSShreyas B. Prabhu return rc; 128d405a98cSShreyas B. Prabhu } 129d405a98cSShreyas B. Prabhu } 1301e1601b3SAkshay Adiga } 131d405a98cSShreyas B. Prabhu 132d405a98cSShreyas B. Prabhu return 0; 133d405a98cSShreyas B. Prabhu } 134d405a98cSShreyas B. Prabhu 135d405a98cSShreyas B. Prabhu static void pnv_alloc_idle_core_states(void) 136d405a98cSShreyas B. Prabhu { 137d405a98cSShreyas B. Prabhu int i, j; 138d405a98cSShreyas B. Prabhu int nr_cores = cpu_nr_cores(); 139d405a98cSShreyas B. Prabhu u32 *core_idle_state; 140d405a98cSShreyas B. Prabhu 141d405a98cSShreyas B. Prabhu /* 1425f221c3cSGautham R. Shenoy * core_idle_state - The lower 8 bits track the idle state of 1435f221c3cSGautham R. Shenoy * each thread of the core. 1445f221c3cSGautham R. Shenoy * 1455f221c3cSGautham R. Shenoy * The most significant bit is the lock bit. 1465f221c3cSGautham R. Shenoy * 1475f221c3cSGautham R. Shenoy * Initially all the bits corresponding to threads_per_core 1485f221c3cSGautham R. Shenoy * are set. They are cleared when the thread enters deep idle 1495f221c3cSGautham R. Shenoy * state like sleep and winkle/stop. 1505f221c3cSGautham R. Shenoy * 1515f221c3cSGautham R. Shenoy * Initially the lock bit is cleared. The lock bit has 2 1525f221c3cSGautham R. Shenoy * purposes: 1535f221c3cSGautham R. Shenoy * a. While the first thread in the core waking up from 1545f221c3cSGautham R. Shenoy * idle is restoring core state, it prevents other 1555f221c3cSGautham R. Shenoy * threads in the core from switching to process 1565f221c3cSGautham R. Shenoy * context. 1575f221c3cSGautham R. Shenoy * b. While the last thread in the core is saving the 1585f221c3cSGautham R. Shenoy * core state, it prevents a different thread from 1595f221c3cSGautham R. Shenoy * waking up. 160d405a98cSShreyas B. Prabhu */ 161d405a98cSShreyas B. Prabhu for (i = 0; i < nr_cores; i++) { 162d405a98cSShreyas B. Prabhu int first_cpu = i * threads_per_core; 163d405a98cSShreyas B. Prabhu int node = cpu_to_node(first_cpu); 16417ed4c8fSGautham R. Shenoy size_t paca_ptr_array_size; 165d405a98cSShreyas B. Prabhu 166d405a98cSShreyas B. Prabhu core_idle_state = kmalloc_node(sizeof(u32), GFP_KERNEL, node); 1675f221c3cSGautham R. Shenoy *core_idle_state = (1 << threads_per_core) - 1; 16817ed4c8fSGautham R. Shenoy paca_ptr_array_size = (threads_per_core * 16917ed4c8fSGautham R. Shenoy sizeof(struct paca_struct *)); 170d405a98cSShreyas B. Prabhu 171d405a98cSShreyas B. Prabhu for (j = 0; j < threads_per_core; j++) { 172d405a98cSShreyas B. Prabhu int cpu = first_cpu + j; 173d405a98cSShreyas B. Prabhu 174d405a98cSShreyas B. Prabhu paca[cpu].core_idle_state_ptr = core_idle_state; 175d405a98cSShreyas B. Prabhu paca[cpu].thread_idle_state = PNV_THREAD_RUNNING; 176d405a98cSShreyas B. Prabhu paca[cpu].thread_mask = 1 << j; 17717ed4c8fSGautham R. Shenoy if (!cpu_has_feature(CPU_FTR_POWER9_DD1)) 17817ed4c8fSGautham R. Shenoy continue; 17917ed4c8fSGautham R. Shenoy paca[cpu].thread_sibling_pacas = 18017ed4c8fSGautham R. Shenoy kmalloc_node(paca_ptr_array_size, 18117ed4c8fSGautham R. Shenoy GFP_KERNEL, node); 182d405a98cSShreyas B. Prabhu } 183d405a98cSShreyas B. Prabhu } 184d405a98cSShreyas B. Prabhu 185d405a98cSShreyas B. Prabhu update_subcore_sibling_mask(); 186d405a98cSShreyas B. Prabhu 187bcef83a0SShreyas B. Prabhu if (supported_cpuidle_states & OPAL_PM_LOSE_FULL_CONTEXT) 188bcef83a0SShreyas B. Prabhu pnv_save_sprs_for_deep_states(); 189d405a98cSShreyas B. Prabhu } 190d405a98cSShreyas B. Prabhu 191d405a98cSShreyas B. Prabhu u32 pnv_get_supported_cpuidle_states(void) 192d405a98cSShreyas B. Prabhu { 193d405a98cSShreyas B. Prabhu return supported_cpuidle_states; 194d405a98cSShreyas B. Prabhu } 195d405a98cSShreyas B. Prabhu EXPORT_SYMBOL_GPL(pnv_get_supported_cpuidle_states); 196d405a98cSShreyas B. Prabhu 1975703d2f4SShreyas B. Prabhu static void pnv_fastsleep_workaround_apply(void *info) 1985703d2f4SShreyas B. Prabhu 1995703d2f4SShreyas B. Prabhu { 2005703d2f4SShreyas B. Prabhu int rc; 2015703d2f4SShreyas B. Prabhu int *err = info; 2025703d2f4SShreyas B. Prabhu 2035703d2f4SShreyas B. Prabhu rc = opal_config_cpu_idle_state(OPAL_CONFIG_IDLE_FASTSLEEP, 2045703d2f4SShreyas B. Prabhu OPAL_CONFIG_IDLE_APPLY); 2055703d2f4SShreyas B. Prabhu if (rc) 2065703d2f4SShreyas B. Prabhu *err = 1; 2075703d2f4SShreyas B. Prabhu } 2085703d2f4SShreyas B. Prabhu 2095703d2f4SShreyas B. Prabhu /* 2105703d2f4SShreyas B. Prabhu * Used to store fastsleep workaround state 2115703d2f4SShreyas B. Prabhu * 0 - Workaround applied/undone at fastsleep entry/exit path (Default) 2125703d2f4SShreyas B. Prabhu * 1 - Workaround applied once, never undone. 2135703d2f4SShreyas B. Prabhu */ 2145703d2f4SShreyas B. Prabhu static u8 fastsleep_workaround_applyonce; 2155703d2f4SShreyas B. Prabhu 2165703d2f4SShreyas B. Prabhu static ssize_t show_fastsleep_workaround_applyonce(struct device *dev, 2175703d2f4SShreyas B. Prabhu struct device_attribute *attr, char *buf) 2185703d2f4SShreyas B. Prabhu { 2195703d2f4SShreyas B. Prabhu return sprintf(buf, "%u\n", fastsleep_workaround_applyonce); 2205703d2f4SShreyas B. Prabhu } 2215703d2f4SShreyas B. Prabhu 2225703d2f4SShreyas B. Prabhu static ssize_t store_fastsleep_workaround_applyonce(struct device *dev, 2235703d2f4SShreyas B. Prabhu struct device_attribute *attr, const char *buf, 2245703d2f4SShreyas B. Prabhu size_t count) 2255703d2f4SShreyas B. Prabhu { 2265703d2f4SShreyas B. Prabhu cpumask_t primary_thread_mask; 2275703d2f4SShreyas B. Prabhu int err; 2285703d2f4SShreyas B. Prabhu u8 val; 2295703d2f4SShreyas B. Prabhu 2305703d2f4SShreyas B. Prabhu if (kstrtou8(buf, 0, &val) || val != 1) 2315703d2f4SShreyas B. Prabhu return -EINVAL; 2325703d2f4SShreyas B. Prabhu 2335703d2f4SShreyas B. Prabhu if (fastsleep_workaround_applyonce == 1) 2345703d2f4SShreyas B. Prabhu return count; 2355703d2f4SShreyas B. Prabhu 2365703d2f4SShreyas B. Prabhu /* 2375703d2f4SShreyas B. Prabhu * fastsleep_workaround_applyonce = 1 implies 2385703d2f4SShreyas B. Prabhu * fastsleep workaround needs to be left in 'applied' state on all 2395703d2f4SShreyas B. Prabhu * the cores. Do this by- 2405703d2f4SShreyas B. Prabhu * 1. Patching out the call to 'undo' workaround in fastsleep exit path 2415703d2f4SShreyas B. Prabhu * 2. Sending ipi to all the cores which have at least one online thread 2425703d2f4SShreyas B. Prabhu * 3. Patching out the call to 'apply' workaround in fastsleep entry 2435703d2f4SShreyas B. Prabhu * path 2445703d2f4SShreyas B. Prabhu * There is no need to send ipi to cores which have all threads 2455703d2f4SShreyas B. Prabhu * offlined, as last thread of the core entering fastsleep or deeper 2465703d2f4SShreyas B. Prabhu * state would have applied workaround. 2475703d2f4SShreyas B. Prabhu */ 2485703d2f4SShreyas B. Prabhu err = patch_instruction( 2495703d2f4SShreyas B. Prabhu (unsigned int *)pnv_fastsleep_workaround_at_exit, 2505703d2f4SShreyas B. Prabhu PPC_INST_NOP); 2515703d2f4SShreyas B. Prabhu if (err) { 2525703d2f4SShreyas B. Prabhu pr_err("fastsleep_workaround_applyonce change failed while patching pnv_fastsleep_workaround_at_exit"); 2535703d2f4SShreyas B. Prabhu goto fail; 2545703d2f4SShreyas B. Prabhu } 2555703d2f4SShreyas B. Prabhu 2565703d2f4SShreyas B. Prabhu get_online_cpus(); 2575703d2f4SShreyas B. Prabhu primary_thread_mask = cpu_online_cores_map(); 2585703d2f4SShreyas B. Prabhu on_each_cpu_mask(&primary_thread_mask, 2595703d2f4SShreyas B. Prabhu pnv_fastsleep_workaround_apply, 2605703d2f4SShreyas B. Prabhu &err, 1); 2615703d2f4SShreyas B. Prabhu put_online_cpus(); 2625703d2f4SShreyas B. Prabhu if (err) { 2635703d2f4SShreyas B. Prabhu pr_err("fastsleep_workaround_applyonce change failed while running pnv_fastsleep_workaround_apply"); 2645703d2f4SShreyas B. Prabhu goto fail; 2655703d2f4SShreyas B. Prabhu } 2665703d2f4SShreyas B. Prabhu 2675703d2f4SShreyas B. Prabhu err = patch_instruction( 2685703d2f4SShreyas B. Prabhu (unsigned int *)pnv_fastsleep_workaround_at_entry, 2695703d2f4SShreyas B. Prabhu PPC_INST_NOP); 2705703d2f4SShreyas B. Prabhu if (err) { 2715703d2f4SShreyas B. Prabhu pr_err("fastsleep_workaround_applyonce change failed while patching pnv_fastsleep_workaround_at_entry"); 2725703d2f4SShreyas B. Prabhu goto fail; 2735703d2f4SShreyas B. Prabhu } 2745703d2f4SShreyas B. Prabhu 2755703d2f4SShreyas B. Prabhu fastsleep_workaround_applyonce = 1; 2765703d2f4SShreyas B. Prabhu 2775703d2f4SShreyas B. Prabhu return count; 2785703d2f4SShreyas B. Prabhu fail: 2795703d2f4SShreyas B. Prabhu return -EIO; 2805703d2f4SShreyas B. Prabhu } 2815703d2f4SShreyas B. Prabhu 2825703d2f4SShreyas B. Prabhu static DEVICE_ATTR(fastsleep_workaround_applyonce, 0600, 2835703d2f4SShreyas B. Prabhu show_fastsleep_workaround_applyonce, 2845703d2f4SShreyas B. Prabhu store_fastsleep_workaround_applyonce); 2855703d2f4SShreyas B. Prabhu 28609206b60SGautham R. Shenoy /* 287bcef83a0SShreyas B. Prabhu * Used for ppc_md.power_save which needs a function with no parameters 288bcef83a0SShreyas B. Prabhu */ 289bcef83a0SShreyas B. Prabhu static void power9_idle(void) 290d405a98cSShreyas B. Prabhu { 29109206b60SGautham R. Shenoy power9_idle_stop(pnv_default_stop_val, pnv_default_stop_mask); 292bcef83a0SShreyas B. Prabhu } 29309206b60SGautham R. Shenoy 29467d20418SNicholas Piggin #ifdef CONFIG_HOTPLUG_CPU 295c0691f9dSShreyas B. Prabhu /* 296a7cd88daSGautham R. Shenoy * pnv_cpu_offline: A function that puts the CPU into the deepest 297a7cd88daSGautham R. Shenoy * available platform idle state on a CPU-Offline. 298a7cd88daSGautham R. Shenoy */ 299a7cd88daSGautham R. Shenoy unsigned long pnv_cpu_offline(unsigned int cpu) 300a7cd88daSGautham R. Shenoy { 301a7cd88daSGautham R. Shenoy unsigned long srr1; 302a7cd88daSGautham R. Shenoy 303a7cd88daSGautham R. Shenoy u32 idle_states = pnv_get_supported_cpuidle_states(); 304a7cd88daSGautham R. Shenoy 305f3b3f284SGautham R. Shenoy if (cpu_has_feature(CPU_FTR_ARCH_300) && deepest_stop_found) { 306a7cd88daSGautham R. Shenoy srr1 = power9_idle_stop(pnv_deepest_stop_psscr_val, 307a7cd88daSGautham R. Shenoy pnv_deepest_stop_psscr_mask); 308a7cd88daSGautham R. Shenoy } else if (idle_states & OPAL_PM_WINKLE_ENABLED) { 309a7cd88daSGautham R. Shenoy srr1 = power7_winkle(); 310a7cd88daSGautham R. Shenoy } else if ((idle_states & OPAL_PM_SLEEP_ENABLED) || 311a7cd88daSGautham R. Shenoy (idle_states & OPAL_PM_SLEEP_ENABLED_ER1)) { 312a7cd88daSGautham R. Shenoy srr1 = power7_sleep(); 31390061231SGautham R. Shenoy } else if (idle_states & OPAL_PM_NAP_ENABLED) { 314a7cd88daSGautham R. Shenoy srr1 = power7_nap(1); 31590061231SGautham R. Shenoy } else { 31690061231SGautham R. Shenoy /* This is the fallback method. We emulate snooze */ 31790061231SGautham R. Shenoy while (!generic_check_cpu_restart(cpu)) { 31890061231SGautham R. Shenoy HMT_low(); 31990061231SGautham R. Shenoy HMT_very_low(); 32090061231SGautham R. Shenoy } 32190061231SGautham R. Shenoy srr1 = 0; 32290061231SGautham R. Shenoy HMT_medium(); 323a7cd88daSGautham R. Shenoy } 324a7cd88daSGautham R. Shenoy 325a7cd88daSGautham R. Shenoy return srr1; 326a7cd88daSGautham R. Shenoy } 32767d20418SNicholas Piggin #endif 328a7cd88daSGautham R. Shenoy 329a7cd88daSGautham R. Shenoy /* 330bcef83a0SShreyas B. Prabhu * Power ISA 3.0 idle initialization. 331bcef83a0SShreyas B. Prabhu * 332bcef83a0SShreyas B. Prabhu * POWER ISA 3.0 defines a new SPR Processor stop Status and Control 333bcef83a0SShreyas B. Prabhu * Register (PSSCR) to control idle behavior. 334bcef83a0SShreyas B. Prabhu * 335bcef83a0SShreyas B. Prabhu * PSSCR layout: 336bcef83a0SShreyas B. Prabhu * ---------------------------------------------------------- 337bcef83a0SShreyas B. Prabhu * | PLS | /// | SD | ESL | EC | PSLL | /// | TR | MTL | RL | 338bcef83a0SShreyas B. Prabhu * ---------------------------------------------------------- 339bcef83a0SShreyas B. Prabhu * 0 4 41 42 43 44 48 54 56 60 340bcef83a0SShreyas B. Prabhu * 341bcef83a0SShreyas B. Prabhu * PSSCR key fields: 342bcef83a0SShreyas B. Prabhu * Bits 0:3 - Power-Saving Level Status (PLS). This field indicates the 343bcef83a0SShreyas B. Prabhu * lowest power-saving state the thread entered since stop instruction was 344bcef83a0SShreyas B. Prabhu * last executed. 345bcef83a0SShreyas B. Prabhu * 346bcef83a0SShreyas B. Prabhu * Bit 41 - Status Disable(SD) 347bcef83a0SShreyas B. Prabhu * 0 - Shows PLS entries 348bcef83a0SShreyas B. Prabhu * 1 - PLS entries are all 0 349bcef83a0SShreyas B. Prabhu * 350bcef83a0SShreyas B. Prabhu * Bit 42 - Enable State Loss 351bcef83a0SShreyas B. Prabhu * 0 - No state is lost irrespective of other fields 352bcef83a0SShreyas B. Prabhu * 1 - Allows state loss 353bcef83a0SShreyas B. Prabhu * 354bcef83a0SShreyas B. Prabhu * Bit 43 - Exit Criterion 355bcef83a0SShreyas B. Prabhu * 0 - Exit from power-save mode on any interrupt 356bcef83a0SShreyas B. Prabhu * 1 - Exit from power-save mode controlled by LPCR's PECE bits 357bcef83a0SShreyas B. Prabhu * 358bcef83a0SShreyas B. Prabhu * Bits 44:47 - Power-Saving Level Limit 359bcef83a0SShreyas B. Prabhu * This limits the power-saving level that can be entered into. 360bcef83a0SShreyas B. Prabhu * 361bcef83a0SShreyas B. Prabhu * Bits 60:63 - Requested Level 362bcef83a0SShreyas B. Prabhu * Used to specify which power-saving level must be entered on executing 363bcef83a0SShreyas B. Prabhu * stop instruction 36409206b60SGautham R. Shenoy */ 36509206b60SGautham R. Shenoy 36609206b60SGautham R. Shenoy int validate_psscr_val_mask(u64 *psscr_val, u64 *psscr_mask, u32 flags) 36709206b60SGautham R. Shenoy { 36809206b60SGautham R. Shenoy int err = 0; 36909206b60SGautham R. Shenoy 37009206b60SGautham R. Shenoy /* 37109206b60SGautham R. Shenoy * psscr_mask == 0xf indicates an older firmware. 37209206b60SGautham R. Shenoy * Set remaining fields of psscr to the default values. 37309206b60SGautham R. Shenoy * See NOTE above definition of PSSCR_HV_DEFAULT_VAL 37409206b60SGautham R. Shenoy */ 37509206b60SGautham R. Shenoy if (*psscr_mask == 0xf) { 37609206b60SGautham R. Shenoy *psscr_val = *psscr_val | PSSCR_HV_DEFAULT_VAL; 37709206b60SGautham R. Shenoy *psscr_mask = PSSCR_HV_DEFAULT_MASK; 37809206b60SGautham R. Shenoy return err; 37909206b60SGautham R. Shenoy } 38009206b60SGautham R. Shenoy 38109206b60SGautham R. Shenoy /* 38209206b60SGautham R. Shenoy * New firmware is expected to set the psscr_val bits correctly. 38309206b60SGautham R. Shenoy * Validate that the following invariants are correctly maintained by 38409206b60SGautham R. Shenoy * the new firmware. 38509206b60SGautham R. Shenoy * - ESL bit value matches the EC bit value. 38609206b60SGautham R. Shenoy * - ESL bit is set for all the deep stop states. 38709206b60SGautham R. Shenoy */ 38809206b60SGautham R. Shenoy if (GET_PSSCR_ESL(*psscr_val) != GET_PSSCR_EC(*psscr_val)) { 38909206b60SGautham R. Shenoy err = ERR_EC_ESL_MISMATCH; 39009206b60SGautham R. Shenoy } else if ((flags & OPAL_PM_LOSE_FULL_CONTEXT) && 39109206b60SGautham R. Shenoy GET_PSSCR_ESL(*psscr_val) == 0) { 39209206b60SGautham R. Shenoy err = ERR_DEEP_STATE_ESL_MISMATCH; 39309206b60SGautham R. Shenoy } 39409206b60SGautham R. Shenoy 39509206b60SGautham R. Shenoy return err; 39609206b60SGautham R. Shenoy } 39709206b60SGautham R. Shenoy 39809206b60SGautham R. Shenoy /* 39909206b60SGautham R. Shenoy * pnv_arch300_idle_init: Initializes the default idle state, first 40009206b60SGautham R. Shenoy * deep idle state and deepest idle state on 40109206b60SGautham R. Shenoy * ISA 3.0 CPUs. 402bcef83a0SShreyas B. Prabhu * 403bcef83a0SShreyas B. Prabhu * @np: /ibm,opal/power-mgt device node 404bcef83a0SShreyas B. Prabhu * @flags: cpu-idle-state-flags array 405bcef83a0SShreyas B. Prabhu * @dt_idle_states: Number of idle state entries 406bcef83a0SShreyas B. Prabhu * Returns 0 on success 407bcef83a0SShreyas B. Prabhu */ 408dd34c74cSGautham R. Shenoy static int __init pnv_power9_idle_init(struct device_node *np, u32 *flags, 409bcef83a0SShreyas B. Prabhu int dt_idle_states) 410bcef83a0SShreyas B. Prabhu { 411bcef83a0SShreyas B. Prabhu u64 *psscr_val = NULL; 41209206b60SGautham R. Shenoy u64 *psscr_mask = NULL; 41309206b60SGautham R. Shenoy u32 *residency_ns = NULL; 41409206b60SGautham R. Shenoy u64 max_residency_ns = 0; 415bcef83a0SShreyas B. Prabhu int rc = 0, i; 416bcef83a0SShreyas B. Prabhu 41709206b60SGautham R. Shenoy psscr_val = kcalloc(dt_idle_states, sizeof(*psscr_val), GFP_KERNEL); 41809206b60SGautham R. Shenoy psscr_mask = kcalloc(dt_idle_states, sizeof(*psscr_mask), GFP_KERNEL); 41909206b60SGautham R. Shenoy residency_ns = kcalloc(dt_idle_states, sizeof(*residency_ns), 420bcef83a0SShreyas B. Prabhu GFP_KERNEL); 42109206b60SGautham R. Shenoy 42209206b60SGautham R. Shenoy if (!psscr_val || !psscr_mask || !residency_ns) { 423bcef83a0SShreyas B. Prabhu rc = -1; 424bcef83a0SShreyas B. Prabhu goto out; 425bcef83a0SShreyas B. Prabhu } 42609206b60SGautham R. Shenoy 427bcef83a0SShreyas B. Prabhu if (of_property_read_u64_array(np, 428bcef83a0SShreyas B. Prabhu "ibm,cpu-idle-state-psscr", 429bcef83a0SShreyas B. Prabhu psscr_val, dt_idle_states)) { 43009206b60SGautham R. Shenoy pr_warn("cpuidle-powernv: missing ibm,cpu-idle-state-psscr in DT\n"); 43109206b60SGautham R. Shenoy rc = -1; 43209206b60SGautham R. Shenoy goto out; 43309206b60SGautham R. Shenoy } 43409206b60SGautham R. Shenoy 43509206b60SGautham R. Shenoy if (of_property_read_u64_array(np, 43609206b60SGautham R. Shenoy "ibm,cpu-idle-state-psscr-mask", 43709206b60SGautham R. Shenoy psscr_mask, dt_idle_states)) { 43809206b60SGautham R. Shenoy pr_warn("cpuidle-powernv: missing ibm,cpu-idle-state-psscr-mask in DT\n"); 43909206b60SGautham R. Shenoy rc = -1; 44009206b60SGautham R. Shenoy goto out; 44109206b60SGautham R. Shenoy } 44209206b60SGautham R. Shenoy 44309206b60SGautham R. Shenoy if (of_property_read_u32_array(np, 44409206b60SGautham R. Shenoy "ibm,cpu-idle-state-residency-ns", 44509206b60SGautham R. Shenoy residency_ns, dt_idle_states)) { 44609206b60SGautham R. Shenoy pr_warn("cpuidle-powernv: missing ibm,cpu-idle-state-residency-ns in DT\n"); 447bcef83a0SShreyas B. Prabhu rc = -1; 448bcef83a0SShreyas B. Prabhu goto out; 449bcef83a0SShreyas B. Prabhu } 450bcef83a0SShreyas B. Prabhu 451bcef83a0SShreyas B. Prabhu /* 45209206b60SGautham R. Shenoy * Set pnv_first_deep_stop_state, pnv_deepest_stop_psscr_{val,mask}, 45309206b60SGautham R. Shenoy * and the pnv_default_stop_{val,mask}. 45409206b60SGautham R. Shenoy * 455c0691f9dSShreyas B. Prabhu * pnv_first_deep_stop_state should be set to the first stop 456c0691f9dSShreyas B. Prabhu * level to cause hypervisor state loss. 45709206b60SGautham R. Shenoy * 45809206b60SGautham R. Shenoy * pnv_deepest_stop_{val,mask} should be set to values corresponding to 45909206b60SGautham R. Shenoy * the deepest stop state. 46009206b60SGautham R. Shenoy * 46109206b60SGautham R. Shenoy * pnv_default_stop_{val,mask} should be set to values corresponding to 46209206b60SGautham R. Shenoy * the shallowest (OPAL_PM_STOP_INST_FAST) loss-less stop state. 463bcef83a0SShreyas B. Prabhu */ 464bcef83a0SShreyas B. Prabhu pnv_first_deep_stop_state = MAX_STOP_STATE; 465bcef83a0SShreyas B. Prabhu for (i = 0; i < dt_idle_states; i++) { 46609206b60SGautham R. Shenoy int err; 467bcef83a0SShreyas B. Prabhu u64 psscr_rl = psscr_val[i] & PSSCR_RL_MASK; 468bcef83a0SShreyas B. Prabhu 469bcef83a0SShreyas B. Prabhu if ((flags[i] & OPAL_PM_LOSE_FULL_CONTEXT) && 470bcef83a0SShreyas B. Prabhu (pnv_first_deep_stop_state > psscr_rl)) 471bcef83a0SShreyas B. Prabhu pnv_first_deep_stop_state = psscr_rl; 472c0691f9dSShreyas B. Prabhu 47309206b60SGautham R. Shenoy err = validate_psscr_val_mask(&psscr_val[i], &psscr_mask[i], 47409206b60SGautham R. Shenoy flags[i]); 47509206b60SGautham R. Shenoy if (err) { 47609206b60SGautham R. Shenoy report_invalid_psscr_val(psscr_val[i], err); 47709206b60SGautham R. Shenoy continue; 47809206b60SGautham R. Shenoy } 47909206b60SGautham R. Shenoy 48009206b60SGautham R. Shenoy if (max_residency_ns < residency_ns[i]) { 48109206b60SGautham R. Shenoy max_residency_ns = residency_ns[i]; 48209206b60SGautham R. Shenoy pnv_deepest_stop_psscr_val = psscr_val[i]; 48309206b60SGautham R. Shenoy pnv_deepest_stop_psscr_mask = psscr_mask[i]; 48409206b60SGautham R. Shenoy deepest_stop_found = true; 48509206b60SGautham R. Shenoy } 48609206b60SGautham R. Shenoy 48709206b60SGautham R. Shenoy if (!default_stop_found && 48809206b60SGautham R. Shenoy (flags[i] & OPAL_PM_STOP_INST_FAST)) { 48909206b60SGautham R. Shenoy pnv_default_stop_val = psscr_val[i]; 49009206b60SGautham R. Shenoy pnv_default_stop_mask = psscr_mask[i]; 49109206b60SGautham R. Shenoy default_stop_found = true; 49209206b60SGautham R. Shenoy } 49309206b60SGautham R. Shenoy } 49409206b60SGautham R. Shenoy 495f3b3f284SGautham R. Shenoy if (unlikely(!default_stop_found)) { 496f3b3f284SGautham R. Shenoy pr_warn("cpuidle-powernv: No suitable default stop state found. Disabling platform idle.\n"); 497f3b3f284SGautham R. Shenoy } else { 498f3b3f284SGautham R. Shenoy ppc_md.power_save = power9_idle; 499f3b3f284SGautham R. Shenoy pr_info("cpuidle-powernv: Default stop: psscr = 0x%016llx,mask=0x%016llx\n", 50009206b60SGautham R. Shenoy pnv_default_stop_val, pnv_default_stop_mask); 50109206b60SGautham R. Shenoy } 50209206b60SGautham R. Shenoy 503f3b3f284SGautham R. Shenoy if (unlikely(!deepest_stop_found)) { 504f3b3f284SGautham R. Shenoy pr_warn("cpuidle-powernv: No suitable stop state for CPU-Hotplug. Offlined CPUs will busy wait"); 505f3b3f284SGautham R. Shenoy } else { 506f3b3f284SGautham R. Shenoy pr_info("cpuidle-powernv: Deepest stop: psscr = 0x%016llx,mask=0x%016llx\n", 50709206b60SGautham R. Shenoy pnv_deepest_stop_psscr_val, 50809206b60SGautham R. Shenoy pnv_deepest_stop_psscr_mask); 509bcef83a0SShreyas B. Prabhu } 510bcef83a0SShreyas B. Prabhu 511f3b3f284SGautham R. Shenoy pr_info("cpuidle-powernv: Requested Level (RL) value of first deep stop = 0x%llx\n", 512f3b3f284SGautham R. Shenoy pnv_first_deep_stop_state); 513bcef83a0SShreyas B. Prabhu out: 514bcef83a0SShreyas B. Prabhu kfree(psscr_val); 51509206b60SGautham R. Shenoy kfree(psscr_mask); 51609206b60SGautham R. Shenoy kfree(residency_ns); 517bcef83a0SShreyas B. Prabhu return rc; 518bcef83a0SShreyas B. Prabhu } 519bcef83a0SShreyas B. Prabhu 520bcef83a0SShreyas B. Prabhu /* 521bcef83a0SShreyas B. Prabhu * Probe device tree for supported idle states 522bcef83a0SShreyas B. Prabhu */ 523bcef83a0SShreyas B. Prabhu static void __init pnv_probe_idle_states(void) 524bcef83a0SShreyas B. Prabhu { 525bcef83a0SShreyas B. Prabhu struct device_node *np; 526d405a98cSShreyas B. Prabhu int dt_idle_states; 527bcef83a0SShreyas B. Prabhu u32 *flags = NULL; 528d405a98cSShreyas B. Prabhu int i; 529d405a98cSShreyas B. Prabhu 530bcef83a0SShreyas B. Prabhu np = of_find_node_by_path("/ibm,opal/power-mgt"); 531bcef83a0SShreyas B. Prabhu if (!np) { 532d405a98cSShreyas B. Prabhu pr_warn("opal: PowerMgmt Node not found\n"); 533d405a98cSShreyas B. Prabhu goto out; 534d405a98cSShreyas B. Prabhu } 535bcef83a0SShreyas B. Prabhu dt_idle_states = of_property_count_u32_elems(np, 536d405a98cSShreyas B. Prabhu "ibm,cpu-idle-state-flags"); 537d405a98cSShreyas B. Prabhu if (dt_idle_states < 0) { 538d405a98cSShreyas B. Prabhu pr_warn("cpuidle-powernv: no idle states found in the DT\n"); 539d405a98cSShreyas B. Prabhu goto out; 540d405a98cSShreyas B. Prabhu } 541d405a98cSShreyas B. Prabhu 542bcef83a0SShreyas B. Prabhu flags = kcalloc(dt_idle_states, sizeof(*flags), GFP_KERNEL); 543bcef83a0SShreyas B. Prabhu 544bcef83a0SShreyas B. Prabhu if (of_property_read_u32_array(np, 545d405a98cSShreyas B. Prabhu "ibm,cpu-idle-state-flags", flags, dt_idle_states)) { 546d405a98cSShreyas B. Prabhu pr_warn("cpuidle-powernv: missing ibm,cpu-idle-state-flags in DT\n"); 547bcef83a0SShreyas B. Prabhu goto out; 548bcef83a0SShreyas B. Prabhu } 549bcef83a0SShreyas B. Prabhu 550bcef83a0SShreyas B. Prabhu if (cpu_has_feature(CPU_FTR_ARCH_300)) { 551dd34c74cSGautham R. Shenoy if (pnv_power9_idle_init(np, flags, dt_idle_states)) 552bcef83a0SShreyas B. Prabhu goto out; 553d405a98cSShreyas B. Prabhu } 554d405a98cSShreyas B. Prabhu 555d405a98cSShreyas B. Prabhu for (i = 0; i < dt_idle_states; i++) 556d405a98cSShreyas B. Prabhu supported_cpuidle_states |= flags[i]; 557d405a98cSShreyas B. Prabhu 558bcef83a0SShreyas B. Prabhu out: 559bcef83a0SShreyas B. Prabhu kfree(flags); 560bcef83a0SShreyas B. Prabhu } 561bcef83a0SShreyas B. Prabhu static int __init pnv_init_idle_states(void) 562bcef83a0SShreyas B. Prabhu { 563bcef83a0SShreyas B. Prabhu 564bcef83a0SShreyas B. Prabhu supported_cpuidle_states = 0; 565bcef83a0SShreyas B. Prabhu 566bcef83a0SShreyas B. Prabhu if (cpuidle_disable != IDLE_NO_OVERRIDE) 567bcef83a0SShreyas B. Prabhu goto out; 568bcef83a0SShreyas B. Prabhu 569bcef83a0SShreyas B. Prabhu pnv_probe_idle_states(); 570bcef83a0SShreyas B. Prabhu 571d405a98cSShreyas B. Prabhu if (!(supported_cpuidle_states & OPAL_PM_SLEEP_ENABLED_ER1)) { 572d405a98cSShreyas B. Prabhu patch_instruction( 573d405a98cSShreyas B. Prabhu (unsigned int *)pnv_fastsleep_workaround_at_entry, 574d405a98cSShreyas B. Prabhu PPC_INST_NOP); 575d405a98cSShreyas B. Prabhu patch_instruction( 576d405a98cSShreyas B. Prabhu (unsigned int *)pnv_fastsleep_workaround_at_exit, 577d405a98cSShreyas B. Prabhu PPC_INST_NOP); 5785703d2f4SShreyas B. Prabhu } else { 5795703d2f4SShreyas B. Prabhu /* 5805703d2f4SShreyas B. Prabhu * OPAL_PM_SLEEP_ENABLED_ER1 is set. It indicates that 5815703d2f4SShreyas B. Prabhu * workaround is needed to use fastsleep. Provide sysfs 5825703d2f4SShreyas B. Prabhu * control to choose how this workaround has to be applied. 5835703d2f4SShreyas B. Prabhu */ 5845703d2f4SShreyas B. Prabhu device_create_file(cpu_subsys.dev_root, 5855703d2f4SShreyas B. Prabhu &dev_attr_fastsleep_workaround_applyonce); 586d405a98cSShreyas B. Prabhu } 5875703d2f4SShreyas B. Prabhu 588d405a98cSShreyas B. Prabhu pnv_alloc_idle_core_states(); 5895593e303SShreyas B. Prabhu 59017ed4c8fSGautham R. Shenoy /* 59117ed4c8fSGautham R. Shenoy * For each CPU, record its PACA address in each of it's 59217ed4c8fSGautham R. Shenoy * sibling thread's PACA at the slot corresponding to this 59317ed4c8fSGautham R. Shenoy * CPU's index in the core. 59417ed4c8fSGautham R. Shenoy */ 59517ed4c8fSGautham R. Shenoy if (cpu_has_feature(CPU_FTR_POWER9_DD1)) { 59617ed4c8fSGautham R. Shenoy int cpu; 59717ed4c8fSGautham R. Shenoy 59817ed4c8fSGautham R. Shenoy pr_info("powernv: idle: Saving PACA pointers of all CPUs in their thread sibling PACA\n"); 59917ed4c8fSGautham R. Shenoy for_each_possible_cpu(cpu) { 60017ed4c8fSGautham R. Shenoy int base_cpu = cpu_first_thread_sibling(cpu); 60117ed4c8fSGautham R. Shenoy int idx = cpu_thread_in_core(cpu); 60217ed4c8fSGautham R. Shenoy int i; 60317ed4c8fSGautham R. Shenoy 60417ed4c8fSGautham R. Shenoy for (i = 0; i < threads_per_core; i++) { 60517ed4c8fSGautham R. Shenoy int j = base_cpu + i; 60617ed4c8fSGautham R. Shenoy 60717ed4c8fSGautham R. Shenoy paca[j].thread_sibling_pacas[idx] = &paca[cpu]; 60817ed4c8fSGautham R. Shenoy } 60917ed4c8fSGautham R. Shenoy } 61017ed4c8fSGautham R. Shenoy } 61117ed4c8fSGautham R. Shenoy 6125593e303SShreyas B. Prabhu if (supported_cpuidle_states & OPAL_PM_NAP_ENABLED) 6135593e303SShreyas B. Prabhu ppc_md.power_save = power7_idle; 614bcef83a0SShreyas B. Prabhu 615d405a98cSShreyas B. Prabhu out: 616d405a98cSShreyas B. Prabhu return 0; 617d405a98cSShreyas B. Prabhu } 6184bece972SMichael Ellerman machine_subsys_initcall(powernv, pnv_init_idle_states); 619