1d405a98cSShreyas B. Prabhu /* 2d405a98cSShreyas B. Prabhu * PowerNV cpuidle code 3d405a98cSShreyas B. Prabhu * 4d405a98cSShreyas B. Prabhu * Copyright 2015 IBM Corp. 5d405a98cSShreyas B. Prabhu * 6d405a98cSShreyas B. Prabhu * This program is free software; you can redistribute it and/or 7d405a98cSShreyas B. Prabhu * modify it under the terms of the GNU General Public License 8d405a98cSShreyas B. Prabhu * as published by the Free Software Foundation; either version 9d405a98cSShreyas B. Prabhu * 2 of the License, or (at your option) any later version. 10d405a98cSShreyas B. Prabhu */ 11d405a98cSShreyas B. Prabhu 12d405a98cSShreyas B. Prabhu #include <linux/types.h> 13d405a98cSShreyas B. Prabhu #include <linux/mm.h> 14d405a98cSShreyas B. Prabhu #include <linux/slab.h> 15d405a98cSShreyas B. Prabhu #include <linux/of.h> 165703d2f4SShreyas B. Prabhu #include <linux/device.h> 175703d2f4SShreyas B. Prabhu #include <linux/cpu.h> 18d405a98cSShreyas B. Prabhu 19d405a98cSShreyas B. Prabhu #include <asm/firmware.h> 204bece972SMichael Ellerman #include <asm/machdep.h> 21d405a98cSShreyas B. Prabhu #include <asm/opal.h> 22d405a98cSShreyas B. Prabhu #include <asm/cputhreads.h> 23d405a98cSShreyas B. Prabhu #include <asm/cpuidle.h> 24d405a98cSShreyas B. Prabhu #include <asm/code-patching.h> 25d405a98cSShreyas B. Prabhu #include <asm/smp.h> 26d405a98cSShreyas B. Prabhu 27d405a98cSShreyas B. Prabhu #include "powernv.h" 28d405a98cSShreyas B. Prabhu #include "subcore.h" 29d405a98cSShreyas B. Prabhu 30bcef83a0SShreyas B. Prabhu /* Power ISA 3.0 allows for stop states 0x0 - 0xF */ 31bcef83a0SShreyas B. Prabhu #define MAX_STOP_STATE 0xF 32bcef83a0SShreyas B. Prabhu 33d405a98cSShreyas B. Prabhu static u32 supported_cpuidle_states; 34d405a98cSShreyas B. Prabhu 35bcef83a0SShreyas B. Prabhu static int pnv_save_sprs_for_deep_states(void) 36d405a98cSShreyas B. Prabhu { 37d405a98cSShreyas B. Prabhu int cpu; 38d405a98cSShreyas B. Prabhu int rc; 39d405a98cSShreyas B. Prabhu 40d405a98cSShreyas B. Prabhu /* 41446957baSAdam Buchbinder * hid0, hid1, hid4, hid5, hmeer and lpcr values are symmetric across 42d405a98cSShreyas B. Prabhu * all cpus at boot. Get these reg values of current cpu and use the 43446957baSAdam Buchbinder * same across all cpus. 44d405a98cSShreyas B. Prabhu */ 45d405a98cSShreyas B. Prabhu uint64_t lpcr_val = mfspr(SPRN_LPCR) & ~(u64)LPCR_PECE1; 46d405a98cSShreyas B. Prabhu uint64_t hid0_val = mfspr(SPRN_HID0); 47d405a98cSShreyas B. Prabhu uint64_t hid1_val = mfspr(SPRN_HID1); 48d405a98cSShreyas B. Prabhu uint64_t hid4_val = mfspr(SPRN_HID4); 49d405a98cSShreyas B. Prabhu uint64_t hid5_val = mfspr(SPRN_HID5); 50d405a98cSShreyas B. Prabhu uint64_t hmeer_val = mfspr(SPRN_HMEER); 51d405a98cSShreyas B. Prabhu 52d405a98cSShreyas B. Prabhu for_each_possible_cpu(cpu) { 53d405a98cSShreyas B. Prabhu uint64_t pir = get_hard_smp_processor_id(cpu); 54d405a98cSShreyas B. Prabhu uint64_t hsprg0_val = (uint64_t)&paca[cpu]; 55d405a98cSShreyas B. Prabhu 56bcef83a0SShreyas B. Prabhu if (!cpu_has_feature(CPU_FTR_ARCH_300)) { 57d405a98cSShreyas B. Prabhu /* 58bcef83a0SShreyas B. Prabhu * HSPRG0 is used to store the cpu's pointer to paca. 59bcef83a0SShreyas B. Prabhu * Hence last 3 bits are guaranteed to be 0. Program 60bcef83a0SShreyas B. Prabhu * slw to restore HSPRG0 with 63rd bit set, so that 61bcef83a0SShreyas B. Prabhu * when a thread wakes up at 0x100 we can use this bit 62bcef83a0SShreyas B. Prabhu * to distinguish between fastsleep and deep winkle. 63bcef83a0SShreyas B. Prabhu * This is not necessary with stop/psscr since PLS 64bcef83a0SShreyas B. Prabhu * field of psscr indicates which state we are waking 65bcef83a0SShreyas B. Prabhu * up from. 66d405a98cSShreyas B. Prabhu */ 67d405a98cSShreyas B. Prabhu hsprg0_val |= 1; 68bcef83a0SShreyas B. Prabhu } 69d405a98cSShreyas B. Prabhu rc = opal_slw_set_reg(pir, SPRN_HSPRG0, hsprg0_val); 70d405a98cSShreyas B. Prabhu if (rc != 0) 71d405a98cSShreyas B. Prabhu return rc; 72d405a98cSShreyas B. Prabhu 73d405a98cSShreyas B. Prabhu rc = opal_slw_set_reg(pir, SPRN_LPCR, lpcr_val); 74d405a98cSShreyas B. Prabhu if (rc != 0) 75d405a98cSShreyas B. Prabhu return rc; 76d405a98cSShreyas B. Prabhu 77d405a98cSShreyas B. Prabhu /* HIDs are per core registers */ 78d405a98cSShreyas B. Prabhu if (cpu_thread_in_core(cpu) == 0) { 79d405a98cSShreyas B. Prabhu 80d405a98cSShreyas B. Prabhu rc = opal_slw_set_reg(pir, SPRN_HMEER, hmeer_val); 81d405a98cSShreyas B. Prabhu if (rc != 0) 82d405a98cSShreyas B. Prabhu return rc; 83d405a98cSShreyas B. Prabhu 84d405a98cSShreyas B. Prabhu rc = opal_slw_set_reg(pir, SPRN_HID0, hid0_val); 85d405a98cSShreyas B. Prabhu if (rc != 0) 86d405a98cSShreyas B. Prabhu return rc; 87d405a98cSShreyas B. Prabhu 88d405a98cSShreyas B. Prabhu rc = opal_slw_set_reg(pir, SPRN_HID1, hid1_val); 89d405a98cSShreyas B. Prabhu if (rc != 0) 90d405a98cSShreyas B. Prabhu return rc; 91d405a98cSShreyas B. Prabhu 92d405a98cSShreyas B. Prabhu rc = opal_slw_set_reg(pir, SPRN_HID4, hid4_val); 93d405a98cSShreyas B. Prabhu if (rc != 0) 94d405a98cSShreyas B. Prabhu return rc; 95d405a98cSShreyas B. Prabhu 96d405a98cSShreyas B. Prabhu rc = opal_slw_set_reg(pir, SPRN_HID5, hid5_val); 97d405a98cSShreyas B. Prabhu if (rc != 0) 98d405a98cSShreyas B. Prabhu return rc; 99d405a98cSShreyas B. Prabhu } 100d405a98cSShreyas B. Prabhu } 101d405a98cSShreyas B. Prabhu 102d405a98cSShreyas B. Prabhu return 0; 103d405a98cSShreyas B. Prabhu } 104d405a98cSShreyas B. Prabhu 105d405a98cSShreyas B. Prabhu static void pnv_alloc_idle_core_states(void) 106d405a98cSShreyas B. Prabhu { 107d405a98cSShreyas B. Prabhu int i, j; 108d405a98cSShreyas B. Prabhu int nr_cores = cpu_nr_cores(); 109d405a98cSShreyas B. Prabhu u32 *core_idle_state; 110d405a98cSShreyas B. Prabhu 111d405a98cSShreyas B. Prabhu /* 112d405a98cSShreyas B. Prabhu * core_idle_state - First 8 bits track the idle state of each thread 113d405a98cSShreyas B. Prabhu * of the core. The 8th bit is the lock bit. Initially all thread bits 114d405a98cSShreyas B. Prabhu * are set. They are cleared when the thread enters deep idle state 115d405a98cSShreyas B. Prabhu * like sleep and winkle. Initially the lock bit is cleared. 116d405a98cSShreyas B. Prabhu * The lock bit has 2 purposes 117d405a98cSShreyas B. Prabhu * a. While the first thread is restoring core state, it prevents 118d405a98cSShreyas B. Prabhu * other threads in the core from switching to process context. 119d405a98cSShreyas B. Prabhu * b. While the last thread in the core is saving the core state, it 120d405a98cSShreyas B. Prabhu * prevents a different thread from waking up. 121d405a98cSShreyas B. Prabhu */ 122d405a98cSShreyas B. Prabhu for (i = 0; i < nr_cores; i++) { 123d405a98cSShreyas B. Prabhu int first_cpu = i * threads_per_core; 124d405a98cSShreyas B. Prabhu int node = cpu_to_node(first_cpu); 12517ed4c8fSGautham R. Shenoy size_t paca_ptr_array_size; 126d405a98cSShreyas B. Prabhu 127d405a98cSShreyas B. Prabhu core_idle_state = kmalloc_node(sizeof(u32), GFP_KERNEL, node); 128d405a98cSShreyas B. Prabhu *core_idle_state = PNV_CORE_IDLE_THREAD_BITS; 12917ed4c8fSGautham R. Shenoy paca_ptr_array_size = (threads_per_core * 13017ed4c8fSGautham R. Shenoy sizeof(struct paca_struct *)); 131d405a98cSShreyas B. Prabhu 132d405a98cSShreyas B. Prabhu for (j = 0; j < threads_per_core; j++) { 133d405a98cSShreyas B. Prabhu int cpu = first_cpu + j; 134d405a98cSShreyas B. Prabhu 135d405a98cSShreyas B. Prabhu paca[cpu].core_idle_state_ptr = core_idle_state; 136d405a98cSShreyas B. Prabhu paca[cpu].thread_idle_state = PNV_THREAD_RUNNING; 137d405a98cSShreyas B. Prabhu paca[cpu].thread_mask = 1 << j; 13817ed4c8fSGautham R. Shenoy if (!cpu_has_feature(CPU_FTR_POWER9_DD1)) 13917ed4c8fSGautham R. Shenoy continue; 14017ed4c8fSGautham R. Shenoy paca[cpu].thread_sibling_pacas = 14117ed4c8fSGautham R. Shenoy kmalloc_node(paca_ptr_array_size, 14217ed4c8fSGautham R. Shenoy GFP_KERNEL, node); 143d405a98cSShreyas B. Prabhu } 144d405a98cSShreyas B. Prabhu } 145d405a98cSShreyas B. Prabhu 146d405a98cSShreyas B. Prabhu update_subcore_sibling_mask(); 147d405a98cSShreyas B. Prabhu 148bcef83a0SShreyas B. Prabhu if (supported_cpuidle_states & OPAL_PM_LOSE_FULL_CONTEXT) 149bcef83a0SShreyas B. Prabhu pnv_save_sprs_for_deep_states(); 150d405a98cSShreyas B. Prabhu } 151d405a98cSShreyas B. Prabhu 152d405a98cSShreyas B. Prabhu u32 pnv_get_supported_cpuidle_states(void) 153d405a98cSShreyas B. Prabhu { 154d405a98cSShreyas B. Prabhu return supported_cpuidle_states; 155d405a98cSShreyas B. Prabhu } 156d405a98cSShreyas B. Prabhu EXPORT_SYMBOL_GPL(pnv_get_supported_cpuidle_states); 157d405a98cSShreyas B. Prabhu 1585703d2f4SShreyas B. Prabhu static void pnv_fastsleep_workaround_apply(void *info) 1595703d2f4SShreyas B. Prabhu 1605703d2f4SShreyas B. Prabhu { 1615703d2f4SShreyas B. Prabhu int rc; 1625703d2f4SShreyas B. Prabhu int *err = info; 1635703d2f4SShreyas B. Prabhu 1645703d2f4SShreyas B. Prabhu rc = opal_config_cpu_idle_state(OPAL_CONFIG_IDLE_FASTSLEEP, 1655703d2f4SShreyas B. Prabhu OPAL_CONFIG_IDLE_APPLY); 1665703d2f4SShreyas B. Prabhu if (rc) 1675703d2f4SShreyas B. Prabhu *err = 1; 1685703d2f4SShreyas B. Prabhu } 1695703d2f4SShreyas B. Prabhu 1705703d2f4SShreyas B. Prabhu /* 1715703d2f4SShreyas B. Prabhu * Used to store fastsleep workaround state 1725703d2f4SShreyas B. Prabhu * 0 - Workaround applied/undone at fastsleep entry/exit path (Default) 1735703d2f4SShreyas B. Prabhu * 1 - Workaround applied once, never undone. 1745703d2f4SShreyas B. Prabhu */ 1755703d2f4SShreyas B. Prabhu static u8 fastsleep_workaround_applyonce; 1765703d2f4SShreyas B. Prabhu 1775703d2f4SShreyas B. Prabhu static ssize_t show_fastsleep_workaround_applyonce(struct device *dev, 1785703d2f4SShreyas B. Prabhu struct device_attribute *attr, char *buf) 1795703d2f4SShreyas B. Prabhu { 1805703d2f4SShreyas B. Prabhu return sprintf(buf, "%u\n", fastsleep_workaround_applyonce); 1815703d2f4SShreyas B. Prabhu } 1825703d2f4SShreyas B. Prabhu 1835703d2f4SShreyas B. Prabhu static ssize_t store_fastsleep_workaround_applyonce(struct device *dev, 1845703d2f4SShreyas B. Prabhu struct device_attribute *attr, const char *buf, 1855703d2f4SShreyas B. Prabhu size_t count) 1865703d2f4SShreyas B. Prabhu { 1875703d2f4SShreyas B. Prabhu cpumask_t primary_thread_mask; 1885703d2f4SShreyas B. Prabhu int err; 1895703d2f4SShreyas B. Prabhu u8 val; 1905703d2f4SShreyas B. Prabhu 1915703d2f4SShreyas B. Prabhu if (kstrtou8(buf, 0, &val) || val != 1) 1925703d2f4SShreyas B. Prabhu return -EINVAL; 1935703d2f4SShreyas B. Prabhu 1945703d2f4SShreyas B. Prabhu if (fastsleep_workaround_applyonce == 1) 1955703d2f4SShreyas B. Prabhu return count; 1965703d2f4SShreyas B. Prabhu 1975703d2f4SShreyas B. Prabhu /* 1985703d2f4SShreyas B. Prabhu * fastsleep_workaround_applyonce = 1 implies 1995703d2f4SShreyas B. Prabhu * fastsleep workaround needs to be left in 'applied' state on all 2005703d2f4SShreyas B. Prabhu * the cores. Do this by- 2015703d2f4SShreyas B. Prabhu * 1. Patching out the call to 'undo' workaround in fastsleep exit path 2025703d2f4SShreyas B. Prabhu * 2. Sending ipi to all the cores which have at least one online thread 2035703d2f4SShreyas B. Prabhu * 3. Patching out the call to 'apply' workaround in fastsleep entry 2045703d2f4SShreyas B. Prabhu * path 2055703d2f4SShreyas B. Prabhu * There is no need to send ipi to cores which have all threads 2065703d2f4SShreyas B. Prabhu * offlined, as last thread of the core entering fastsleep or deeper 2075703d2f4SShreyas B. Prabhu * state would have applied workaround. 2085703d2f4SShreyas B. Prabhu */ 2095703d2f4SShreyas B. Prabhu err = patch_instruction( 2105703d2f4SShreyas B. Prabhu (unsigned int *)pnv_fastsleep_workaround_at_exit, 2115703d2f4SShreyas B. Prabhu PPC_INST_NOP); 2125703d2f4SShreyas B. Prabhu if (err) { 2135703d2f4SShreyas B. Prabhu pr_err("fastsleep_workaround_applyonce change failed while patching pnv_fastsleep_workaround_at_exit"); 2145703d2f4SShreyas B. Prabhu goto fail; 2155703d2f4SShreyas B. Prabhu } 2165703d2f4SShreyas B. Prabhu 2175703d2f4SShreyas B. Prabhu get_online_cpus(); 2185703d2f4SShreyas B. Prabhu primary_thread_mask = cpu_online_cores_map(); 2195703d2f4SShreyas B. Prabhu on_each_cpu_mask(&primary_thread_mask, 2205703d2f4SShreyas B. Prabhu pnv_fastsleep_workaround_apply, 2215703d2f4SShreyas B. Prabhu &err, 1); 2225703d2f4SShreyas B. Prabhu put_online_cpus(); 2235703d2f4SShreyas B. Prabhu if (err) { 2245703d2f4SShreyas B. Prabhu pr_err("fastsleep_workaround_applyonce change failed while running pnv_fastsleep_workaround_apply"); 2255703d2f4SShreyas B. Prabhu goto fail; 2265703d2f4SShreyas B. Prabhu } 2275703d2f4SShreyas B. Prabhu 2285703d2f4SShreyas B. Prabhu err = patch_instruction( 2295703d2f4SShreyas B. Prabhu (unsigned int *)pnv_fastsleep_workaround_at_entry, 2305703d2f4SShreyas B. Prabhu PPC_INST_NOP); 2315703d2f4SShreyas B. Prabhu if (err) { 2325703d2f4SShreyas B. Prabhu pr_err("fastsleep_workaround_applyonce change failed while patching pnv_fastsleep_workaround_at_entry"); 2335703d2f4SShreyas B. Prabhu goto fail; 2345703d2f4SShreyas B. Prabhu } 2355703d2f4SShreyas B. Prabhu 2365703d2f4SShreyas B. Prabhu fastsleep_workaround_applyonce = 1; 2375703d2f4SShreyas B. Prabhu 2385703d2f4SShreyas B. Prabhu return count; 2395703d2f4SShreyas B. Prabhu fail: 2405703d2f4SShreyas B. Prabhu return -EIO; 2415703d2f4SShreyas B. Prabhu } 2425703d2f4SShreyas B. Prabhu 2435703d2f4SShreyas B. Prabhu static DEVICE_ATTR(fastsleep_workaround_applyonce, 0600, 2445703d2f4SShreyas B. Prabhu show_fastsleep_workaround_applyonce, 2455703d2f4SShreyas B. Prabhu store_fastsleep_workaround_applyonce); 2465703d2f4SShreyas B. Prabhu 24709206b60SGautham R. Shenoy /* 24809206b60SGautham R. Shenoy * The default stop state that will be used by ppc_md.power_save 24909206b60SGautham R. Shenoy * function on platforms that support stop instruction. 25009206b60SGautham R. Shenoy */ 251f3b3f284SGautham R. Shenoy static u64 pnv_default_stop_val; 252f3b3f284SGautham R. Shenoy static u64 pnv_default_stop_mask; 253f3b3f284SGautham R. Shenoy static bool default_stop_found; 254bcef83a0SShreyas B. Prabhu 255bcef83a0SShreyas B. Prabhu /* 256bcef83a0SShreyas B. Prabhu * Used for ppc_md.power_save which needs a function with no parameters 257bcef83a0SShreyas B. Prabhu */ 258bcef83a0SShreyas B. Prabhu static void power9_idle(void) 259d405a98cSShreyas B. Prabhu { 26009206b60SGautham R. Shenoy power9_idle_stop(pnv_default_stop_val, pnv_default_stop_mask); 261bcef83a0SShreyas B. Prabhu } 26209206b60SGautham R. Shenoy 263bcef83a0SShreyas B. Prabhu /* 264bcef83a0SShreyas B. Prabhu * First deep stop state. Used to figure out when to save/restore 265bcef83a0SShreyas B. Prabhu * hypervisor context. 266bcef83a0SShreyas B. Prabhu */ 267bcef83a0SShreyas B. Prabhu u64 pnv_first_deep_stop_state = MAX_STOP_STATE; 268bcef83a0SShreyas B. Prabhu 269bcef83a0SShreyas B. Prabhu /* 27009206b60SGautham R. Shenoy * psscr value and mask of the deepest stop idle state. 27109206b60SGautham R. Shenoy * Used when a cpu is offlined. 272c0691f9dSShreyas B. Prabhu */ 273f3b3f284SGautham R. Shenoy static u64 pnv_deepest_stop_psscr_val; 274f3b3f284SGautham R. Shenoy static u64 pnv_deepest_stop_psscr_mask; 275f3b3f284SGautham R. Shenoy static bool deepest_stop_found; 276c0691f9dSShreyas B. Prabhu 277c0691f9dSShreyas B. Prabhu /* 278a7cd88daSGautham R. Shenoy * pnv_cpu_offline: A function that puts the CPU into the deepest 279a7cd88daSGautham R. Shenoy * available platform idle state on a CPU-Offline. 280a7cd88daSGautham R. Shenoy */ 281a7cd88daSGautham R. Shenoy unsigned long pnv_cpu_offline(unsigned int cpu) 282a7cd88daSGautham R. Shenoy { 283a7cd88daSGautham R. Shenoy unsigned long srr1; 284a7cd88daSGautham R. Shenoy 285a7cd88daSGautham R. Shenoy u32 idle_states = pnv_get_supported_cpuidle_states(); 286a7cd88daSGautham R. Shenoy 287f3b3f284SGautham R. Shenoy if (cpu_has_feature(CPU_FTR_ARCH_300) && deepest_stop_found) { 288a7cd88daSGautham R. Shenoy srr1 = power9_idle_stop(pnv_deepest_stop_psscr_val, 289a7cd88daSGautham R. Shenoy pnv_deepest_stop_psscr_mask); 290a7cd88daSGautham R. Shenoy } else if (idle_states & OPAL_PM_WINKLE_ENABLED) { 291a7cd88daSGautham R. Shenoy srr1 = power7_winkle(); 292a7cd88daSGautham R. Shenoy } else if ((idle_states & OPAL_PM_SLEEP_ENABLED) || 293a7cd88daSGautham R. Shenoy (idle_states & OPAL_PM_SLEEP_ENABLED_ER1)) { 294a7cd88daSGautham R. Shenoy srr1 = power7_sleep(); 29590061231SGautham R. Shenoy } else if (idle_states & OPAL_PM_NAP_ENABLED) { 296a7cd88daSGautham R. Shenoy srr1 = power7_nap(1); 29790061231SGautham R. Shenoy } else { 29890061231SGautham R. Shenoy /* This is the fallback method. We emulate snooze */ 29990061231SGautham R. Shenoy while (!generic_check_cpu_restart(cpu)) { 30090061231SGautham R. Shenoy HMT_low(); 30190061231SGautham R. Shenoy HMT_very_low(); 30290061231SGautham R. Shenoy } 30390061231SGautham R. Shenoy srr1 = 0; 30490061231SGautham R. Shenoy HMT_medium(); 305a7cd88daSGautham R. Shenoy } 306a7cd88daSGautham R. Shenoy 307a7cd88daSGautham R. Shenoy return srr1; 308a7cd88daSGautham R. Shenoy } 309a7cd88daSGautham R. Shenoy 310a7cd88daSGautham R. Shenoy /* 311bcef83a0SShreyas B. Prabhu * Power ISA 3.0 idle initialization. 312bcef83a0SShreyas B. Prabhu * 313bcef83a0SShreyas B. Prabhu * POWER ISA 3.0 defines a new SPR Processor stop Status and Control 314bcef83a0SShreyas B. Prabhu * Register (PSSCR) to control idle behavior. 315bcef83a0SShreyas B. Prabhu * 316bcef83a0SShreyas B. Prabhu * PSSCR layout: 317bcef83a0SShreyas B. Prabhu * ---------------------------------------------------------- 318bcef83a0SShreyas B. Prabhu * | PLS | /// | SD | ESL | EC | PSLL | /// | TR | MTL | RL | 319bcef83a0SShreyas B. Prabhu * ---------------------------------------------------------- 320bcef83a0SShreyas B. Prabhu * 0 4 41 42 43 44 48 54 56 60 321bcef83a0SShreyas B. Prabhu * 322bcef83a0SShreyas B. Prabhu * PSSCR key fields: 323bcef83a0SShreyas B. Prabhu * Bits 0:3 - Power-Saving Level Status (PLS). This field indicates the 324bcef83a0SShreyas B. Prabhu * lowest power-saving state the thread entered since stop instruction was 325bcef83a0SShreyas B. Prabhu * last executed. 326bcef83a0SShreyas B. Prabhu * 327bcef83a0SShreyas B. Prabhu * Bit 41 - Status Disable(SD) 328bcef83a0SShreyas B. Prabhu * 0 - Shows PLS entries 329bcef83a0SShreyas B. Prabhu * 1 - PLS entries are all 0 330bcef83a0SShreyas B. Prabhu * 331bcef83a0SShreyas B. Prabhu * Bit 42 - Enable State Loss 332bcef83a0SShreyas B. Prabhu * 0 - No state is lost irrespective of other fields 333bcef83a0SShreyas B. Prabhu * 1 - Allows state loss 334bcef83a0SShreyas B. Prabhu * 335bcef83a0SShreyas B. Prabhu * Bit 43 - Exit Criterion 336bcef83a0SShreyas B. Prabhu * 0 - Exit from power-save mode on any interrupt 337bcef83a0SShreyas B. Prabhu * 1 - Exit from power-save mode controlled by LPCR's PECE bits 338bcef83a0SShreyas B. Prabhu * 339bcef83a0SShreyas B. Prabhu * Bits 44:47 - Power-Saving Level Limit 340bcef83a0SShreyas B. Prabhu * This limits the power-saving level that can be entered into. 341bcef83a0SShreyas B. Prabhu * 342bcef83a0SShreyas B. Prabhu * Bits 60:63 - Requested Level 343bcef83a0SShreyas B. Prabhu * Used to specify which power-saving level must be entered on executing 344bcef83a0SShreyas B. Prabhu * stop instruction 34509206b60SGautham R. Shenoy */ 34609206b60SGautham R. Shenoy 34709206b60SGautham R. Shenoy int validate_psscr_val_mask(u64 *psscr_val, u64 *psscr_mask, u32 flags) 34809206b60SGautham R. Shenoy { 34909206b60SGautham R. Shenoy int err = 0; 35009206b60SGautham R. Shenoy 35109206b60SGautham R. Shenoy /* 35209206b60SGautham R. Shenoy * psscr_mask == 0xf indicates an older firmware. 35309206b60SGautham R. Shenoy * Set remaining fields of psscr to the default values. 35409206b60SGautham R. Shenoy * See NOTE above definition of PSSCR_HV_DEFAULT_VAL 35509206b60SGautham R. Shenoy */ 35609206b60SGautham R. Shenoy if (*psscr_mask == 0xf) { 35709206b60SGautham R. Shenoy *psscr_val = *psscr_val | PSSCR_HV_DEFAULT_VAL; 35809206b60SGautham R. Shenoy *psscr_mask = PSSCR_HV_DEFAULT_MASK; 35909206b60SGautham R. Shenoy return err; 36009206b60SGautham R. Shenoy } 36109206b60SGautham R. Shenoy 36209206b60SGautham R. Shenoy /* 36309206b60SGautham R. Shenoy * New firmware is expected to set the psscr_val bits correctly. 36409206b60SGautham R. Shenoy * Validate that the following invariants are correctly maintained by 36509206b60SGautham R. Shenoy * the new firmware. 36609206b60SGautham R. Shenoy * - ESL bit value matches the EC bit value. 36709206b60SGautham R. Shenoy * - ESL bit is set for all the deep stop states. 36809206b60SGautham R. Shenoy */ 36909206b60SGautham R. Shenoy if (GET_PSSCR_ESL(*psscr_val) != GET_PSSCR_EC(*psscr_val)) { 37009206b60SGautham R. Shenoy err = ERR_EC_ESL_MISMATCH; 37109206b60SGautham R. Shenoy } else if ((flags & OPAL_PM_LOSE_FULL_CONTEXT) && 37209206b60SGautham R. Shenoy GET_PSSCR_ESL(*psscr_val) == 0) { 37309206b60SGautham R. Shenoy err = ERR_DEEP_STATE_ESL_MISMATCH; 37409206b60SGautham R. Shenoy } 37509206b60SGautham R. Shenoy 37609206b60SGautham R. Shenoy return err; 37709206b60SGautham R. Shenoy } 37809206b60SGautham R. Shenoy 37909206b60SGautham R. Shenoy /* 38009206b60SGautham R. Shenoy * pnv_arch300_idle_init: Initializes the default idle state, first 38109206b60SGautham R. Shenoy * deep idle state and deepest idle state on 38209206b60SGautham R. Shenoy * ISA 3.0 CPUs. 383bcef83a0SShreyas B. Prabhu * 384bcef83a0SShreyas B. Prabhu * @np: /ibm,opal/power-mgt device node 385bcef83a0SShreyas B. Prabhu * @flags: cpu-idle-state-flags array 386bcef83a0SShreyas B. Prabhu * @dt_idle_states: Number of idle state entries 387bcef83a0SShreyas B. Prabhu * Returns 0 on success 388bcef83a0SShreyas B. Prabhu */ 389dd34c74cSGautham R. Shenoy static int __init pnv_power9_idle_init(struct device_node *np, u32 *flags, 390bcef83a0SShreyas B. Prabhu int dt_idle_states) 391bcef83a0SShreyas B. Prabhu { 392bcef83a0SShreyas B. Prabhu u64 *psscr_val = NULL; 39309206b60SGautham R. Shenoy u64 *psscr_mask = NULL; 39409206b60SGautham R. Shenoy u32 *residency_ns = NULL; 39509206b60SGautham R. Shenoy u64 max_residency_ns = 0; 396bcef83a0SShreyas B. Prabhu int rc = 0, i; 397bcef83a0SShreyas B. Prabhu 39809206b60SGautham R. Shenoy psscr_val = kcalloc(dt_idle_states, sizeof(*psscr_val), GFP_KERNEL); 39909206b60SGautham R. Shenoy psscr_mask = kcalloc(dt_idle_states, sizeof(*psscr_mask), GFP_KERNEL); 40009206b60SGautham R. Shenoy residency_ns = kcalloc(dt_idle_states, sizeof(*residency_ns), 401bcef83a0SShreyas B. Prabhu GFP_KERNEL); 40209206b60SGautham R. Shenoy 40309206b60SGautham R. Shenoy if (!psscr_val || !psscr_mask || !residency_ns) { 404bcef83a0SShreyas B. Prabhu rc = -1; 405bcef83a0SShreyas B. Prabhu goto out; 406bcef83a0SShreyas B. Prabhu } 40709206b60SGautham R. Shenoy 408bcef83a0SShreyas B. Prabhu if (of_property_read_u64_array(np, 409bcef83a0SShreyas B. Prabhu "ibm,cpu-idle-state-psscr", 410bcef83a0SShreyas B. Prabhu psscr_val, dt_idle_states)) { 41109206b60SGautham R. Shenoy pr_warn("cpuidle-powernv: missing ibm,cpu-idle-state-psscr in DT\n"); 41209206b60SGautham R. Shenoy rc = -1; 41309206b60SGautham R. Shenoy goto out; 41409206b60SGautham R. Shenoy } 41509206b60SGautham R. Shenoy 41609206b60SGautham R. Shenoy if (of_property_read_u64_array(np, 41709206b60SGautham R. Shenoy "ibm,cpu-idle-state-psscr-mask", 41809206b60SGautham R. Shenoy psscr_mask, dt_idle_states)) { 41909206b60SGautham R. Shenoy pr_warn("cpuidle-powernv: missing ibm,cpu-idle-state-psscr-mask in DT\n"); 42009206b60SGautham R. Shenoy rc = -1; 42109206b60SGautham R. Shenoy goto out; 42209206b60SGautham R. Shenoy } 42309206b60SGautham R. Shenoy 42409206b60SGautham R. Shenoy if (of_property_read_u32_array(np, 42509206b60SGautham R. Shenoy "ibm,cpu-idle-state-residency-ns", 42609206b60SGautham R. Shenoy residency_ns, dt_idle_states)) { 42709206b60SGautham R. Shenoy pr_warn("cpuidle-powernv: missing ibm,cpu-idle-state-residency-ns in DT\n"); 428bcef83a0SShreyas B. Prabhu rc = -1; 429bcef83a0SShreyas B. Prabhu goto out; 430bcef83a0SShreyas B. Prabhu } 431bcef83a0SShreyas B. Prabhu 432bcef83a0SShreyas B. Prabhu /* 43309206b60SGautham R. Shenoy * Set pnv_first_deep_stop_state, pnv_deepest_stop_psscr_{val,mask}, 43409206b60SGautham R. Shenoy * and the pnv_default_stop_{val,mask}. 43509206b60SGautham R. Shenoy * 436c0691f9dSShreyas B. Prabhu * pnv_first_deep_stop_state should be set to the first stop 437c0691f9dSShreyas B. Prabhu * level to cause hypervisor state loss. 43809206b60SGautham R. Shenoy * 43909206b60SGautham R. Shenoy * pnv_deepest_stop_{val,mask} should be set to values corresponding to 44009206b60SGautham R. Shenoy * the deepest stop state. 44109206b60SGautham R. Shenoy * 44209206b60SGautham R. Shenoy * pnv_default_stop_{val,mask} should be set to values corresponding to 44309206b60SGautham R. Shenoy * the shallowest (OPAL_PM_STOP_INST_FAST) loss-less stop state. 444bcef83a0SShreyas B. Prabhu */ 445bcef83a0SShreyas B. Prabhu pnv_first_deep_stop_state = MAX_STOP_STATE; 446bcef83a0SShreyas B. Prabhu for (i = 0; i < dt_idle_states; i++) { 44709206b60SGautham R. Shenoy int err; 448bcef83a0SShreyas B. Prabhu u64 psscr_rl = psscr_val[i] & PSSCR_RL_MASK; 449bcef83a0SShreyas B. Prabhu 450bcef83a0SShreyas B. Prabhu if ((flags[i] & OPAL_PM_LOSE_FULL_CONTEXT) && 451bcef83a0SShreyas B. Prabhu (pnv_first_deep_stop_state > psscr_rl)) 452bcef83a0SShreyas B. Prabhu pnv_first_deep_stop_state = psscr_rl; 453c0691f9dSShreyas B. Prabhu 45409206b60SGautham R. Shenoy err = validate_psscr_val_mask(&psscr_val[i], &psscr_mask[i], 45509206b60SGautham R. Shenoy flags[i]); 45609206b60SGautham R. Shenoy if (err) { 45709206b60SGautham R. Shenoy report_invalid_psscr_val(psscr_val[i], err); 45809206b60SGautham R. Shenoy continue; 45909206b60SGautham R. Shenoy } 46009206b60SGautham R. Shenoy 46109206b60SGautham R. Shenoy if (max_residency_ns < residency_ns[i]) { 46209206b60SGautham R. Shenoy max_residency_ns = residency_ns[i]; 46309206b60SGautham R. Shenoy pnv_deepest_stop_psscr_val = psscr_val[i]; 46409206b60SGautham R. Shenoy pnv_deepest_stop_psscr_mask = psscr_mask[i]; 46509206b60SGautham R. Shenoy deepest_stop_found = true; 46609206b60SGautham R. Shenoy } 46709206b60SGautham R. Shenoy 46809206b60SGautham R. Shenoy if (!default_stop_found && 46909206b60SGautham R. Shenoy (flags[i] & OPAL_PM_STOP_INST_FAST)) { 47009206b60SGautham R. Shenoy pnv_default_stop_val = psscr_val[i]; 47109206b60SGautham R. Shenoy pnv_default_stop_mask = psscr_mask[i]; 47209206b60SGautham R. Shenoy default_stop_found = true; 47309206b60SGautham R. Shenoy } 47409206b60SGautham R. Shenoy } 47509206b60SGautham R. Shenoy 476f3b3f284SGautham R. Shenoy if (unlikely(!default_stop_found)) { 477f3b3f284SGautham R. Shenoy pr_warn("cpuidle-powernv: No suitable default stop state found. Disabling platform idle.\n"); 478f3b3f284SGautham R. Shenoy } else { 479f3b3f284SGautham R. Shenoy ppc_md.power_save = power9_idle; 480f3b3f284SGautham R. Shenoy pr_info("cpuidle-powernv: Default stop: psscr = 0x%016llx,mask=0x%016llx\n", 48109206b60SGautham R. Shenoy pnv_default_stop_val, pnv_default_stop_mask); 48209206b60SGautham R. Shenoy } 48309206b60SGautham R. Shenoy 484f3b3f284SGautham R. Shenoy if (unlikely(!deepest_stop_found)) { 485f3b3f284SGautham R. Shenoy pr_warn("cpuidle-powernv: No suitable stop state for CPU-Hotplug. Offlined CPUs will busy wait"); 486f3b3f284SGautham R. Shenoy } else { 487f3b3f284SGautham R. Shenoy pr_info("cpuidle-powernv: Deepest stop: psscr = 0x%016llx,mask=0x%016llx\n", 48809206b60SGautham R. Shenoy pnv_deepest_stop_psscr_val, 48909206b60SGautham R. Shenoy pnv_deepest_stop_psscr_mask); 490bcef83a0SShreyas B. Prabhu } 491bcef83a0SShreyas B. Prabhu 492f3b3f284SGautham R. Shenoy pr_info("cpuidle-powernv: Requested Level (RL) value of first deep stop = 0x%llx\n", 493f3b3f284SGautham R. Shenoy pnv_first_deep_stop_state); 494bcef83a0SShreyas B. Prabhu out: 495bcef83a0SShreyas B. Prabhu kfree(psscr_val); 49609206b60SGautham R. Shenoy kfree(psscr_mask); 49709206b60SGautham R. Shenoy kfree(residency_ns); 498bcef83a0SShreyas B. Prabhu return rc; 499bcef83a0SShreyas B. Prabhu } 500bcef83a0SShreyas B. Prabhu 501bcef83a0SShreyas B. Prabhu /* 502bcef83a0SShreyas B. Prabhu * Probe device tree for supported idle states 503bcef83a0SShreyas B. Prabhu */ 504bcef83a0SShreyas B. Prabhu static void __init pnv_probe_idle_states(void) 505bcef83a0SShreyas B. Prabhu { 506bcef83a0SShreyas B. Prabhu struct device_node *np; 507d405a98cSShreyas B. Prabhu int dt_idle_states; 508bcef83a0SShreyas B. Prabhu u32 *flags = NULL; 509d405a98cSShreyas B. Prabhu int i; 510d405a98cSShreyas B. Prabhu 511bcef83a0SShreyas B. Prabhu np = of_find_node_by_path("/ibm,opal/power-mgt"); 512bcef83a0SShreyas B. Prabhu if (!np) { 513d405a98cSShreyas B. Prabhu pr_warn("opal: PowerMgmt Node not found\n"); 514d405a98cSShreyas B. Prabhu goto out; 515d405a98cSShreyas B. Prabhu } 516bcef83a0SShreyas B. Prabhu dt_idle_states = of_property_count_u32_elems(np, 517d405a98cSShreyas B. Prabhu "ibm,cpu-idle-state-flags"); 518d405a98cSShreyas B. Prabhu if (dt_idle_states < 0) { 519d405a98cSShreyas B. Prabhu pr_warn("cpuidle-powernv: no idle states found in the DT\n"); 520d405a98cSShreyas B. Prabhu goto out; 521d405a98cSShreyas B. Prabhu } 522d405a98cSShreyas B. Prabhu 523bcef83a0SShreyas B. Prabhu flags = kcalloc(dt_idle_states, sizeof(*flags), GFP_KERNEL); 524bcef83a0SShreyas B. Prabhu 525bcef83a0SShreyas B. Prabhu if (of_property_read_u32_array(np, 526d405a98cSShreyas B. Prabhu "ibm,cpu-idle-state-flags", flags, dt_idle_states)) { 527d405a98cSShreyas B. Prabhu pr_warn("cpuidle-powernv: missing ibm,cpu-idle-state-flags in DT\n"); 528bcef83a0SShreyas B. Prabhu goto out; 529bcef83a0SShreyas B. Prabhu } 530bcef83a0SShreyas B. Prabhu 531bcef83a0SShreyas B. Prabhu if (cpu_has_feature(CPU_FTR_ARCH_300)) { 532dd34c74cSGautham R. Shenoy if (pnv_power9_idle_init(np, flags, dt_idle_states)) 533bcef83a0SShreyas B. Prabhu goto out; 534d405a98cSShreyas B. Prabhu } 535d405a98cSShreyas B. Prabhu 536d405a98cSShreyas B. Prabhu for (i = 0; i < dt_idle_states; i++) 537d405a98cSShreyas B. Prabhu supported_cpuidle_states |= flags[i]; 538d405a98cSShreyas B. Prabhu 539bcef83a0SShreyas B. Prabhu out: 540bcef83a0SShreyas B. Prabhu kfree(flags); 541bcef83a0SShreyas B. Prabhu } 542bcef83a0SShreyas B. Prabhu static int __init pnv_init_idle_states(void) 543bcef83a0SShreyas B. Prabhu { 544bcef83a0SShreyas B. Prabhu 545bcef83a0SShreyas B. Prabhu supported_cpuidle_states = 0; 546bcef83a0SShreyas B. Prabhu 547bcef83a0SShreyas B. Prabhu if (cpuidle_disable != IDLE_NO_OVERRIDE) 548bcef83a0SShreyas B. Prabhu goto out; 549bcef83a0SShreyas B. Prabhu 550bcef83a0SShreyas B. Prabhu pnv_probe_idle_states(); 551bcef83a0SShreyas B. Prabhu 552d405a98cSShreyas B. Prabhu if (!(supported_cpuidle_states & OPAL_PM_SLEEP_ENABLED_ER1)) { 553d405a98cSShreyas B. Prabhu patch_instruction( 554d405a98cSShreyas B. Prabhu (unsigned int *)pnv_fastsleep_workaround_at_entry, 555d405a98cSShreyas B. Prabhu PPC_INST_NOP); 556d405a98cSShreyas B. Prabhu patch_instruction( 557d405a98cSShreyas B. Prabhu (unsigned int *)pnv_fastsleep_workaround_at_exit, 558d405a98cSShreyas B. Prabhu PPC_INST_NOP); 5595703d2f4SShreyas B. Prabhu } else { 5605703d2f4SShreyas B. Prabhu /* 5615703d2f4SShreyas B. Prabhu * OPAL_PM_SLEEP_ENABLED_ER1 is set. It indicates that 5625703d2f4SShreyas B. Prabhu * workaround is needed to use fastsleep. Provide sysfs 5635703d2f4SShreyas B. Prabhu * control to choose how this workaround has to be applied. 5645703d2f4SShreyas B. Prabhu */ 5655703d2f4SShreyas B. Prabhu device_create_file(cpu_subsys.dev_root, 5665703d2f4SShreyas B. Prabhu &dev_attr_fastsleep_workaround_applyonce); 567d405a98cSShreyas B. Prabhu } 5685703d2f4SShreyas B. Prabhu 569d405a98cSShreyas B. Prabhu pnv_alloc_idle_core_states(); 5705593e303SShreyas B. Prabhu 57117ed4c8fSGautham R. Shenoy /* 57217ed4c8fSGautham R. Shenoy * For each CPU, record its PACA address in each of it's 57317ed4c8fSGautham R. Shenoy * sibling thread's PACA at the slot corresponding to this 57417ed4c8fSGautham R. Shenoy * CPU's index in the core. 57517ed4c8fSGautham R. Shenoy */ 57617ed4c8fSGautham R. Shenoy if (cpu_has_feature(CPU_FTR_POWER9_DD1)) { 57717ed4c8fSGautham R. Shenoy int cpu; 57817ed4c8fSGautham R. Shenoy 57917ed4c8fSGautham R. Shenoy pr_info("powernv: idle: Saving PACA pointers of all CPUs in their thread sibling PACA\n"); 58017ed4c8fSGautham R. Shenoy for_each_possible_cpu(cpu) { 58117ed4c8fSGautham R. Shenoy int base_cpu = cpu_first_thread_sibling(cpu); 58217ed4c8fSGautham R. Shenoy int idx = cpu_thread_in_core(cpu); 58317ed4c8fSGautham R. Shenoy int i; 58417ed4c8fSGautham R. Shenoy 58517ed4c8fSGautham R. Shenoy for (i = 0; i < threads_per_core; i++) { 58617ed4c8fSGautham R. Shenoy int j = base_cpu + i; 58717ed4c8fSGautham R. Shenoy 58817ed4c8fSGautham R. Shenoy paca[j].thread_sibling_pacas[idx] = &paca[cpu]; 58917ed4c8fSGautham R. Shenoy } 59017ed4c8fSGautham R. Shenoy } 59117ed4c8fSGautham R. Shenoy } 59217ed4c8fSGautham R. Shenoy 5935593e303SShreyas B. Prabhu if (supported_cpuidle_states & OPAL_PM_NAP_ENABLED) 5945593e303SShreyas B. Prabhu ppc_md.power_save = power7_idle; 595bcef83a0SShreyas B. Prabhu 596d405a98cSShreyas B. Prabhu out: 597d405a98cSShreyas B. Prabhu return 0; 598d405a98cSShreyas B. Prabhu } 5994bece972SMichael Ellerman machine_subsys_initcall(powernv, pnv_init_idle_states); 600