1d405a98cSShreyas B. Prabhu /* 2d405a98cSShreyas B. Prabhu * PowerNV cpuidle code 3d405a98cSShreyas B. Prabhu * 4d405a98cSShreyas B. Prabhu * Copyright 2015 IBM Corp. 5d405a98cSShreyas B. Prabhu * 6d405a98cSShreyas B. Prabhu * This program is free software; you can redistribute it and/or 7d405a98cSShreyas B. Prabhu * modify it under the terms of the GNU General Public License 8d405a98cSShreyas B. Prabhu * as published by the Free Software Foundation; either version 9d405a98cSShreyas B. Prabhu * 2 of the License, or (at your option) any later version. 10d405a98cSShreyas B. Prabhu */ 11d405a98cSShreyas B. Prabhu 12d405a98cSShreyas B. Prabhu #include <linux/types.h> 13d405a98cSShreyas B. Prabhu #include <linux/mm.h> 14d405a98cSShreyas B. Prabhu #include <linux/slab.h> 15d405a98cSShreyas B. Prabhu #include <linux/of.h> 165703d2f4SShreyas B. Prabhu #include <linux/device.h> 175703d2f4SShreyas B. Prabhu #include <linux/cpu.h> 18d405a98cSShreyas B. Prabhu 19d405a98cSShreyas B. Prabhu #include <asm/firmware.h> 204bece972SMichael Ellerman #include <asm/machdep.h> 21d405a98cSShreyas B. Prabhu #include <asm/opal.h> 22d405a98cSShreyas B. Prabhu #include <asm/cputhreads.h> 23d405a98cSShreyas B. Prabhu #include <asm/cpuidle.h> 24d405a98cSShreyas B. Prabhu #include <asm/code-patching.h> 25d405a98cSShreyas B. Prabhu #include <asm/smp.h> 26d405a98cSShreyas B. Prabhu 27d405a98cSShreyas B. Prabhu #include "powernv.h" 28d405a98cSShreyas B. Prabhu #include "subcore.h" 29d405a98cSShreyas B. Prabhu 30bcef83a0SShreyas B. Prabhu /* Power ISA 3.0 allows for stop states 0x0 - 0xF */ 31bcef83a0SShreyas B. Prabhu #define MAX_STOP_STATE 0xF 32bcef83a0SShreyas B. Prabhu 33d405a98cSShreyas B. Prabhu static u32 supported_cpuidle_states; 34d405a98cSShreyas B. Prabhu 35bcef83a0SShreyas B. Prabhu static int pnv_save_sprs_for_deep_states(void) 36d405a98cSShreyas B. Prabhu { 37d405a98cSShreyas B. Prabhu int cpu; 38d405a98cSShreyas B. Prabhu int rc; 39d405a98cSShreyas B. Prabhu 40d405a98cSShreyas B. Prabhu /* 41446957baSAdam Buchbinder * hid0, hid1, hid4, hid5, hmeer and lpcr values are symmetric across 42d405a98cSShreyas B. Prabhu * all cpus at boot. Get these reg values of current cpu and use the 43446957baSAdam Buchbinder * same across all cpus. 44d405a98cSShreyas B. Prabhu */ 45d405a98cSShreyas B. Prabhu uint64_t lpcr_val = mfspr(SPRN_LPCR) & ~(u64)LPCR_PECE1; 46d405a98cSShreyas B. Prabhu uint64_t hid0_val = mfspr(SPRN_HID0); 47d405a98cSShreyas B. Prabhu uint64_t hid1_val = mfspr(SPRN_HID1); 48d405a98cSShreyas B. Prabhu uint64_t hid4_val = mfspr(SPRN_HID4); 49d405a98cSShreyas B. Prabhu uint64_t hid5_val = mfspr(SPRN_HID5); 50d405a98cSShreyas B. Prabhu uint64_t hmeer_val = mfspr(SPRN_HMEER); 51d405a98cSShreyas B. Prabhu 52d405a98cSShreyas B. Prabhu for_each_possible_cpu(cpu) { 53d405a98cSShreyas B. Prabhu uint64_t pir = get_hard_smp_processor_id(cpu); 54d405a98cSShreyas B. Prabhu uint64_t hsprg0_val = (uint64_t)&paca[cpu]; 55d405a98cSShreyas B. Prabhu 56d405a98cSShreyas B. Prabhu rc = opal_slw_set_reg(pir, SPRN_HSPRG0, hsprg0_val); 57d405a98cSShreyas B. Prabhu if (rc != 0) 58d405a98cSShreyas B. Prabhu return rc; 59d405a98cSShreyas B. Prabhu 60d405a98cSShreyas B. Prabhu rc = opal_slw_set_reg(pir, SPRN_LPCR, lpcr_val); 61d405a98cSShreyas B. Prabhu if (rc != 0) 62d405a98cSShreyas B. Prabhu return rc; 63d405a98cSShreyas B. Prabhu 64d405a98cSShreyas B. Prabhu /* HIDs are per core registers */ 65d405a98cSShreyas B. Prabhu if (cpu_thread_in_core(cpu) == 0) { 66d405a98cSShreyas B. Prabhu 67d405a98cSShreyas B. Prabhu rc = opal_slw_set_reg(pir, SPRN_HMEER, hmeer_val); 68d405a98cSShreyas B. Prabhu if (rc != 0) 69d405a98cSShreyas B. Prabhu return rc; 70d405a98cSShreyas B. Prabhu 71d405a98cSShreyas B. Prabhu rc = opal_slw_set_reg(pir, SPRN_HID0, hid0_val); 72d405a98cSShreyas B. Prabhu if (rc != 0) 73d405a98cSShreyas B. Prabhu return rc; 74d405a98cSShreyas B. Prabhu 75d405a98cSShreyas B. Prabhu rc = opal_slw_set_reg(pir, SPRN_HID1, hid1_val); 76d405a98cSShreyas B. Prabhu if (rc != 0) 77d405a98cSShreyas B. Prabhu return rc; 78d405a98cSShreyas B. Prabhu 79d405a98cSShreyas B. Prabhu rc = opal_slw_set_reg(pir, SPRN_HID4, hid4_val); 80d405a98cSShreyas B. Prabhu if (rc != 0) 81d405a98cSShreyas B. Prabhu return rc; 82d405a98cSShreyas B. Prabhu 83d405a98cSShreyas B. Prabhu rc = opal_slw_set_reg(pir, SPRN_HID5, hid5_val); 84d405a98cSShreyas B. Prabhu if (rc != 0) 85d405a98cSShreyas B. Prabhu return rc; 86d405a98cSShreyas B. Prabhu } 87d405a98cSShreyas B. Prabhu } 88d405a98cSShreyas B. Prabhu 89d405a98cSShreyas B. Prabhu return 0; 90d405a98cSShreyas B. Prabhu } 91d405a98cSShreyas B. Prabhu 92d405a98cSShreyas B. Prabhu static void pnv_alloc_idle_core_states(void) 93d405a98cSShreyas B. Prabhu { 94d405a98cSShreyas B. Prabhu int i, j; 95d405a98cSShreyas B. Prabhu int nr_cores = cpu_nr_cores(); 96d405a98cSShreyas B. Prabhu u32 *core_idle_state; 97d405a98cSShreyas B. Prabhu 98d405a98cSShreyas B. Prabhu /* 99d405a98cSShreyas B. Prabhu * core_idle_state - First 8 bits track the idle state of each thread 100d405a98cSShreyas B. Prabhu * of the core. The 8th bit is the lock bit. Initially all thread bits 101d405a98cSShreyas B. Prabhu * are set. They are cleared when the thread enters deep idle state 102d405a98cSShreyas B. Prabhu * like sleep and winkle. Initially the lock bit is cleared. 103d405a98cSShreyas B. Prabhu * The lock bit has 2 purposes 104d405a98cSShreyas B. Prabhu * a. While the first thread is restoring core state, it prevents 105d405a98cSShreyas B. Prabhu * other threads in the core from switching to process context. 106d405a98cSShreyas B. Prabhu * b. While the last thread in the core is saving the core state, it 107d405a98cSShreyas B. Prabhu * prevents a different thread from waking up. 108d405a98cSShreyas B. Prabhu */ 109d405a98cSShreyas B. Prabhu for (i = 0; i < nr_cores; i++) { 110d405a98cSShreyas B. Prabhu int first_cpu = i * threads_per_core; 111d405a98cSShreyas B. Prabhu int node = cpu_to_node(first_cpu); 11217ed4c8fSGautham R. Shenoy size_t paca_ptr_array_size; 113d405a98cSShreyas B. Prabhu 114d405a98cSShreyas B. Prabhu core_idle_state = kmalloc_node(sizeof(u32), GFP_KERNEL, node); 115d405a98cSShreyas B. Prabhu *core_idle_state = PNV_CORE_IDLE_THREAD_BITS; 11617ed4c8fSGautham R. Shenoy paca_ptr_array_size = (threads_per_core * 11717ed4c8fSGautham R. Shenoy sizeof(struct paca_struct *)); 118d405a98cSShreyas B. Prabhu 119d405a98cSShreyas B. Prabhu for (j = 0; j < threads_per_core; j++) { 120d405a98cSShreyas B. Prabhu int cpu = first_cpu + j; 121d405a98cSShreyas B. Prabhu 122d405a98cSShreyas B. Prabhu paca[cpu].core_idle_state_ptr = core_idle_state; 123d405a98cSShreyas B. Prabhu paca[cpu].thread_idle_state = PNV_THREAD_RUNNING; 124d405a98cSShreyas B. Prabhu paca[cpu].thread_mask = 1 << j; 12517ed4c8fSGautham R. Shenoy if (!cpu_has_feature(CPU_FTR_POWER9_DD1)) 12617ed4c8fSGautham R. Shenoy continue; 12717ed4c8fSGautham R. Shenoy paca[cpu].thread_sibling_pacas = 12817ed4c8fSGautham R. Shenoy kmalloc_node(paca_ptr_array_size, 12917ed4c8fSGautham R. Shenoy GFP_KERNEL, node); 130d405a98cSShreyas B. Prabhu } 131d405a98cSShreyas B. Prabhu } 132d405a98cSShreyas B. Prabhu 133d405a98cSShreyas B. Prabhu update_subcore_sibling_mask(); 134d405a98cSShreyas B. Prabhu 135bcef83a0SShreyas B. Prabhu if (supported_cpuidle_states & OPAL_PM_LOSE_FULL_CONTEXT) 136bcef83a0SShreyas B. Prabhu pnv_save_sprs_for_deep_states(); 137d405a98cSShreyas B. Prabhu } 138d405a98cSShreyas B. Prabhu 139d405a98cSShreyas B. Prabhu u32 pnv_get_supported_cpuidle_states(void) 140d405a98cSShreyas B. Prabhu { 141d405a98cSShreyas B. Prabhu return supported_cpuidle_states; 142d405a98cSShreyas B. Prabhu } 143d405a98cSShreyas B. Prabhu EXPORT_SYMBOL_GPL(pnv_get_supported_cpuidle_states); 144d405a98cSShreyas B. Prabhu 1455703d2f4SShreyas B. Prabhu static void pnv_fastsleep_workaround_apply(void *info) 1465703d2f4SShreyas B. Prabhu 1475703d2f4SShreyas B. Prabhu { 1485703d2f4SShreyas B. Prabhu int rc; 1495703d2f4SShreyas B. Prabhu int *err = info; 1505703d2f4SShreyas B. Prabhu 1515703d2f4SShreyas B. Prabhu rc = opal_config_cpu_idle_state(OPAL_CONFIG_IDLE_FASTSLEEP, 1525703d2f4SShreyas B. Prabhu OPAL_CONFIG_IDLE_APPLY); 1535703d2f4SShreyas B. Prabhu if (rc) 1545703d2f4SShreyas B. Prabhu *err = 1; 1555703d2f4SShreyas B. Prabhu } 1565703d2f4SShreyas B. Prabhu 1575703d2f4SShreyas B. Prabhu /* 1585703d2f4SShreyas B. Prabhu * Used to store fastsleep workaround state 1595703d2f4SShreyas B. Prabhu * 0 - Workaround applied/undone at fastsleep entry/exit path (Default) 1605703d2f4SShreyas B. Prabhu * 1 - Workaround applied once, never undone. 1615703d2f4SShreyas B. Prabhu */ 1625703d2f4SShreyas B. Prabhu static u8 fastsleep_workaround_applyonce; 1635703d2f4SShreyas B. Prabhu 1645703d2f4SShreyas B. Prabhu static ssize_t show_fastsleep_workaround_applyonce(struct device *dev, 1655703d2f4SShreyas B. Prabhu struct device_attribute *attr, char *buf) 1665703d2f4SShreyas B. Prabhu { 1675703d2f4SShreyas B. Prabhu return sprintf(buf, "%u\n", fastsleep_workaround_applyonce); 1685703d2f4SShreyas B. Prabhu } 1695703d2f4SShreyas B. Prabhu 1705703d2f4SShreyas B. Prabhu static ssize_t store_fastsleep_workaround_applyonce(struct device *dev, 1715703d2f4SShreyas B. Prabhu struct device_attribute *attr, const char *buf, 1725703d2f4SShreyas B. Prabhu size_t count) 1735703d2f4SShreyas B. Prabhu { 1745703d2f4SShreyas B. Prabhu cpumask_t primary_thread_mask; 1755703d2f4SShreyas B. Prabhu int err; 1765703d2f4SShreyas B. Prabhu u8 val; 1775703d2f4SShreyas B. Prabhu 1785703d2f4SShreyas B. Prabhu if (kstrtou8(buf, 0, &val) || val != 1) 1795703d2f4SShreyas B. Prabhu return -EINVAL; 1805703d2f4SShreyas B. Prabhu 1815703d2f4SShreyas B. Prabhu if (fastsleep_workaround_applyonce == 1) 1825703d2f4SShreyas B. Prabhu return count; 1835703d2f4SShreyas B. Prabhu 1845703d2f4SShreyas B. Prabhu /* 1855703d2f4SShreyas B. Prabhu * fastsleep_workaround_applyonce = 1 implies 1865703d2f4SShreyas B. Prabhu * fastsleep workaround needs to be left in 'applied' state on all 1875703d2f4SShreyas B. Prabhu * the cores. Do this by- 1885703d2f4SShreyas B. Prabhu * 1. Patching out the call to 'undo' workaround in fastsleep exit path 1895703d2f4SShreyas B. Prabhu * 2. Sending ipi to all the cores which have at least one online thread 1905703d2f4SShreyas B. Prabhu * 3. Patching out the call to 'apply' workaround in fastsleep entry 1915703d2f4SShreyas B. Prabhu * path 1925703d2f4SShreyas B. Prabhu * There is no need to send ipi to cores which have all threads 1935703d2f4SShreyas B. Prabhu * offlined, as last thread of the core entering fastsleep or deeper 1945703d2f4SShreyas B. Prabhu * state would have applied workaround. 1955703d2f4SShreyas B. Prabhu */ 1965703d2f4SShreyas B. Prabhu err = patch_instruction( 1975703d2f4SShreyas B. Prabhu (unsigned int *)pnv_fastsleep_workaround_at_exit, 1985703d2f4SShreyas B. Prabhu PPC_INST_NOP); 1995703d2f4SShreyas B. Prabhu if (err) { 2005703d2f4SShreyas B. Prabhu pr_err("fastsleep_workaround_applyonce change failed while patching pnv_fastsleep_workaround_at_exit"); 2015703d2f4SShreyas B. Prabhu goto fail; 2025703d2f4SShreyas B. Prabhu } 2035703d2f4SShreyas B. Prabhu 2045703d2f4SShreyas B. Prabhu get_online_cpus(); 2055703d2f4SShreyas B. Prabhu primary_thread_mask = cpu_online_cores_map(); 2065703d2f4SShreyas B. Prabhu on_each_cpu_mask(&primary_thread_mask, 2075703d2f4SShreyas B. Prabhu pnv_fastsleep_workaround_apply, 2085703d2f4SShreyas B. Prabhu &err, 1); 2095703d2f4SShreyas B. Prabhu put_online_cpus(); 2105703d2f4SShreyas B. Prabhu if (err) { 2115703d2f4SShreyas B. Prabhu pr_err("fastsleep_workaround_applyonce change failed while running pnv_fastsleep_workaround_apply"); 2125703d2f4SShreyas B. Prabhu goto fail; 2135703d2f4SShreyas B. Prabhu } 2145703d2f4SShreyas B. Prabhu 2155703d2f4SShreyas B. Prabhu err = patch_instruction( 2165703d2f4SShreyas B. Prabhu (unsigned int *)pnv_fastsleep_workaround_at_entry, 2175703d2f4SShreyas B. Prabhu PPC_INST_NOP); 2185703d2f4SShreyas B. Prabhu if (err) { 2195703d2f4SShreyas B. Prabhu pr_err("fastsleep_workaround_applyonce change failed while patching pnv_fastsleep_workaround_at_entry"); 2205703d2f4SShreyas B. Prabhu goto fail; 2215703d2f4SShreyas B. Prabhu } 2225703d2f4SShreyas B. Prabhu 2235703d2f4SShreyas B. Prabhu fastsleep_workaround_applyonce = 1; 2245703d2f4SShreyas B. Prabhu 2255703d2f4SShreyas B. Prabhu return count; 2265703d2f4SShreyas B. Prabhu fail: 2275703d2f4SShreyas B. Prabhu return -EIO; 2285703d2f4SShreyas B. Prabhu } 2295703d2f4SShreyas B. Prabhu 2305703d2f4SShreyas B. Prabhu static DEVICE_ATTR(fastsleep_workaround_applyonce, 0600, 2315703d2f4SShreyas B. Prabhu show_fastsleep_workaround_applyonce, 2325703d2f4SShreyas B. Prabhu store_fastsleep_workaround_applyonce); 2335703d2f4SShreyas B. Prabhu 23409206b60SGautham R. Shenoy /* 23509206b60SGautham R. Shenoy * The default stop state that will be used by ppc_md.power_save 23609206b60SGautham R. Shenoy * function on platforms that support stop instruction. 23709206b60SGautham R. Shenoy */ 238f3b3f284SGautham R. Shenoy static u64 pnv_default_stop_val; 239f3b3f284SGautham R. Shenoy static u64 pnv_default_stop_mask; 240f3b3f284SGautham R. Shenoy static bool default_stop_found; 241bcef83a0SShreyas B. Prabhu 242bcef83a0SShreyas B. Prabhu /* 243bcef83a0SShreyas B. Prabhu * Used for ppc_md.power_save which needs a function with no parameters 244bcef83a0SShreyas B. Prabhu */ 245bcef83a0SShreyas B. Prabhu static void power9_idle(void) 246d405a98cSShreyas B. Prabhu { 24709206b60SGautham R. Shenoy power9_idle_stop(pnv_default_stop_val, pnv_default_stop_mask); 248bcef83a0SShreyas B. Prabhu } 24909206b60SGautham R. Shenoy 250bcef83a0SShreyas B. Prabhu /* 251bcef83a0SShreyas B. Prabhu * First deep stop state. Used to figure out when to save/restore 252bcef83a0SShreyas B. Prabhu * hypervisor context. 253bcef83a0SShreyas B. Prabhu */ 254bcef83a0SShreyas B. Prabhu u64 pnv_first_deep_stop_state = MAX_STOP_STATE; 255bcef83a0SShreyas B. Prabhu 256bcef83a0SShreyas B. Prabhu /* 25709206b60SGautham R. Shenoy * psscr value and mask of the deepest stop idle state. 25809206b60SGautham R. Shenoy * Used when a cpu is offlined. 259c0691f9dSShreyas B. Prabhu */ 260f3b3f284SGautham R. Shenoy static u64 pnv_deepest_stop_psscr_val; 261f3b3f284SGautham R. Shenoy static u64 pnv_deepest_stop_psscr_mask; 262f3b3f284SGautham R. Shenoy static bool deepest_stop_found; 263c0691f9dSShreyas B. Prabhu 26467d20418SNicholas Piggin #ifdef CONFIG_HOTPLUG_CPU 265c0691f9dSShreyas B. Prabhu /* 266a7cd88daSGautham R. Shenoy * pnv_cpu_offline: A function that puts the CPU into the deepest 267a7cd88daSGautham R. Shenoy * available platform idle state on a CPU-Offline. 268a7cd88daSGautham R. Shenoy */ 269a7cd88daSGautham R. Shenoy unsigned long pnv_cpu_offline(unsigned int cpu) 270a7cd88daSGautham R. Shenoy { 271a7cd88daSGautham R. Shenoy unsigned long srr1; 272a7cd88daSGautham R. Shenoy 273a7cd88daSGautham R. Shenoy u32 idle_states = pnv_get_supported_cpuidle_states(); 274a7cd88daSGautham R. Shenoy 275f3b3f284SGautham R. Shenoy if (cpu_has_feature(CPU_FTR_ARCH_300) && deepest_stop_found) { 276a7cd88daSGautham R. Shenoy srr1 = power9_idle_stop(pnv_deepest_stop_psscr_val, 277a7cd88daSGautham R. Shenoy pnv_deepest_stop_psscr_mask); 278a7cd88daSGautham R. Shenoy } else if (idle_states & OPAL_PM_WINKLE_ENABLED) { 279a7cd88daSGautham R. Shenoy srr1 = power7_winkle(); 280a7cd88daSGautham R. Shenoy } else if ((idle_states & OPAL_PM_SLEEP_ENABLED) || 281a7cd88daSGautham R. Shenoy (idle_states & OPAL_PM_SLEEP_ENABLED_ER1)) { 282a7cd88daSGautham R. Shenoy srr1 = power7_sleep(); 28390061231SGautham R. Shenoy } else if (idle_states & OPAL_PM_NAP_ENABLED) { 284a7cd88daSGautham R. Shenoy srr1 = power7_nap(1); 28590061231SGautham R. Shenoy } else { 28690061231SGautham R. Shenoy /* This is the fallback method. We emulate snooze */ 28790061231SGautham R. Shenoy while (!generic_check_cpu_restart(cpu)) { 28890061231SGautham R. Shenoy HMT_low(); 28990061231SGautham R. Shenoy HMT_very_low(); 29090061231SGautham R. Shenoy } 29190061231SGautham R. Shenoy srr1 = 0; 29290061231SGautham R. Shenoy HMT_medium(); 293a7cd88daSGautham R. Shenoy } 294a7cd88daSGautham R. Shenoy 295a7cd88daSGautham R. Shenoy return srr1; 296a7cd88daSGautham R. Shenoy } 29767d20418SNicholas Piggin #endif 298a7cd88daSGautham R. Shenoy 299a7cd88daSGautham R. Shenoy /* 300bcef83a0SShreyas B. Prabhu * Power ISA 3.0 idle initialization. 301bcef83a0SShreyas B. Prabhu * 302bcef83a0SShreyas B. Prabhu * POWER ISA 3.0 defines a new SPR Processor stop Status and Control 303bcef83a0SShreyas B. Prabhu * Register (PSSCR) to control idle behavior. 304bcef83a0SShreyas B. Prabhu * 305bcef83a0SShreyas B. Prabhu * PSSCR layout: 306bcef83a0SShreyas B. Prabhu * ---------------------------------------------------------- 307bcef83a0SShreyas B. Prabhu * | PLS | /// | SD | ESL | EC | PSLL | /// | TR | MTL | RL | 308bcef83a0SShreyas B. Prabhu * ---------------------------------------------------------- 309bcef83a0SShreyas B. Prabhu * 0 4 41 42 43 44 48 54 56 60 310bcef83a0SShreyas B. Prabhu * 311bcef83a0SShreyas B. Prabhu * PSSCR key fields: 312bcef83a0SShreyas B. Prabhu * Bits 0:3 - Power-Saving Level Status (PLS). This field indicates the 313bcef83a0SShreyas B. Prabhu * lowest power-saving state the thread entered since stop instruction was 314bcef83a0SShreyas B. Prabhu * last executed. 315bcef83a0SShreyas B. Prabhu * 316bcef83a0SShreyas B. Prabhu * Bit 41 - Status Disable(SD) 317bcef83a0SShreyas B. Prabhu * 0 - Shows PLS entries 318bcef83a0SShreyas B. Prabhu * 1 - PLS entries are all 0 319bcef83a0SShreyas B. Prabhu * 320bcef83a0SShreyas B. Prabhu * Bit 42 - Enable State Loss 321bcef83a0SShreyas B. Prabhu * 0 - No state is lost irrespective of other fields 322bcef83a0SShreyas B. Prabhu * 1 - Allows state loss 323bcef83a0SShreyas B. Prabhu * 324bcef83a0SShreyas B. Prabhu * Bit 43 - Exit Criterion 325bcef83a0SShreyas B. Prabhu * 0 - Exit from power-save mode on any interrupt 326bcef83a0SShreyas B. Prabhu * 1 - Exit from power-save mode controlled by LPCR's PECE bits 327bcef83a0SShreyas B. Prabhu * 328bcef83a0SShreyas B. Prabhu * Bits 44:47 - Power-Saving Level Limit 329bcef83a0SShreyas B. Prabhu * This limits the power-saving level that can be entered into. 330bcef83a0SShreyas B. Prabhu * 331bcef83a0SShreyas B. Prabhu * Bits 60:63 - Requested Level 332bcef83a0SShreyas B. Prabhu * Used to specify which power-saving level must be entered on executing 333bcef83a0SShreyas B. Prabhu * stop instruction 33409206b60SGautham R. Shenoy */ 33509206b60SGautham R. Shenoy 33609206b60SGautham R. Shenoy int validate_psscr_val_mask(u64 *psscr_val, u64 *psscr_mask, u32 flags) 33709206b60SGautham R. Shenoy { 33809206b60SGautham R. Shenoy int err = 0; 33909206b60SGautham R. Shenoy 34009206b60SGautham R. Shenoy /* 34109206b60SGautham R. Shenoy * psscr_mask == 0xf indicates an older firmware. 34209206b60SGautham R. Shenoy * Set remaining fields of psscr to the default values. 34309206b60SGautham R. Shenoy * See NOTE above definition of PSSCR_HV_DEFAULT_VAL 34409206b60SGautham R. Shenoy */ 34509206b60SGautham R. Shenoy if (*psscr_mask == 0xf) { 34609206b60SGautham R. Shenoy *psscr_val = *psscr_val | PSSCR_HV_DEFAULT_VAL; 34709206b60SGautham R. Shenoy *psscr_mask = PSSCR_HV_DEFAULT_MASK; 34809206b60SGautham R. Shenoy return err; 34909206b60SGautham R. Shenoy } 35009206b60SGautham R. Shenoy 35109206b60SGautham R. Shenoy /* 35209206b60SGautham R. Shenoy * New firmware is expected to set the psscr_val bits correctly. 35309206b60SGautham R. Shenoy * Validate that the following invariants are correctly maintained by 35409206b60SGautham R. Shenoy * the new firmware. 35509206b60SGautham R. Shenoy * - ESL bit value matches the EC bit value. 35609206b60SGautham R. Shenoy * - ESL bit is set for all the deep stop states. 35709206b60SGautham R. Shenoy */ 35809206b60SGautham R. Shenoy if (GET_PSSCR_ESL(*psscr_val) != GET_PSSCR_EC(*psscr_val)) { 35909206b60SGautham R. Shenoy err = ERR_EC_ESL_MISMATCH; 36009206b60SGautham R. Shenoy } else if ((flags & OPAL_PM_LOSE_FULL_CONTEXT) && 36109206b60SGautham R. Shenoy GET_PSSCR_ESL(*psscr_val) == 0) { 36209206b60SGautham R. Shenoy err = ERR_DEEP_STATE_ESL_MISMATCH; 36309206b60SGautham R. Shenoy } 36409206b60SGautham R. Shenoy 36509206b60SGautham R. Shenoy return err; 36609206b60SGautham R. Shenoy } 36709206b60SGautham R. Shenoy 36809206b60SGautham R. Shenoy /* 36909206b60SGautham R. Shenoy * pnv_arch300_idle_init: Initializes the default idle state, first 37009206b60SGautham R. Shenoy * deep idle state and deepest idle state on 37109206b60SGautham R. Shenoy * ISA 3.0 CPUs. 372bcef83a0SShreyas B. Prabhu * 373bcef83a0SShreyas B. Prabhu * @np: /ibm,opal/power-mgt device node 374bcef83a0SShreyas B. Prabhu * @flags: cpu-idle-state-flags array 375bcef83a0SShreyas B. Prabhu * @dt_idle_states: Number of idle state entries 376bcef83a0SShreyas B. Prabhu * Returns 0 on success 377bcef83a0SShreyas B. Prabhu */ 378dd34c74cSGautham R. Shenoy static int __init pnv_power9_idle_init(struct device_node *np, u32 *flags, 379bcef83a0SShreyas B. Prabhu int dt_idle_states) 380bcef83a0SShreyas B. Prabhu { 381bcef83a0SShreyas B. Prabhu u64 *psscr_val = NULL; 38209206b60SGautham R. Shenoy u64 *psscr_mask = NULL; 38309206b60SGautham R. Shenoy u32 *residency_ns = NULL; 38409206b60SGautham R. Shenoy u64 max_residency_ns = 0; 385bcef83a0SShreyas B. Prabhu int rc = 0, i; 386bcef83a0SShreyas B. Prabhu 38709206b60SGautham R. Shenoy psscr_val = kcalloc(dt_idle_states, sizeof(*psscr_val), GFP_KERNEL); 38809206b60SGautham R. Shenoy psscr_mask = kcalloc(dt_idle_states, sizeof(*psscr_mask), GFP_KERNEL); 38909206b60SGautham R. Shenoy residency_ns = kcalloc(dt_idle_states, sizeof(*residency_ns), 390bcef83a0SShreyas B. Prabhu GFP_KERNEL); 39109206b60SGautham R. Shenoy 39209206b60SGautham R. Shenoy if (!psscr_val || !psscr_mask || !residency_ns) { 393bcef83a0SShreyas B. Prabhu rc = -1; 394bcef83a0SShreyas B. Prabhu goto out; 395bcef83a0SShreyas B. Prabhu } 39609206b60SGautham R. Shenoy 397bcef83a0SShreyas B. Prabhu if (of_property_read_u64_array(np, 398bcef83a0SShreyas B. Prabhu "ibm,cpu-idle-state-psscr", 399bcef83a0SShreyas B. Prabhu psscr_val, dt_idle_states)) { 40009206b60SGautham R. Shenoy pr_warn("cpuidle-powernv: missing ibm,cpu-idle-state-psscr in DT\n"); 40109206b60SGautham R. Shenoy rc = -1; 40209206b60SGautham R. Shenoy goto out; 40309206b60SGautham R. Shenoy } 40409206b60SGautham R. Shenoy 40509206b60SGautham R. Shenoy if (of_property_read_u64_array(np, 40609206b60SGautham R. Shenoy "ibm,cpu-idle-state-psscr-mask", 40709206b60SGautham R. Shenoy psscr_mask, dt_idle_states)) { 40809206b60SGautham R. Shenoy pr_warn("cpuidle-powernv: missing ibm,cpu-idle-state-psscr-mask in DT\n"); 40909206b60SGautham R. Shenoy rc = -1; 41009206b60SGautham R. Shenoy goto out; 41109206b60SGautham R. Shenoy } 41209206b60SGautham R. Shenoy 41309206b60SGautham R. Shenoy if (of_property_read_u32_array(np, 41409206b60SGautham R. Shenoy "ibm,cpu-idle-state-residency-ns", 41509206b60SGautham R. Shenoy residency_ns, dt_idle_states)) { 41609206b60SGautham R. Shenoy pr_warn("cpuidle-powernv: missing ibm,cpu-idle-state-residency-ns in DT\n"); 417bcef83a0SShreyas B. Prabhu rc = -1; 418bcef83a0SShreyas B. Prabhu goto out; 419bcef83a0SShreyas B. Prabhu } 420bcef83a0SShreyas B. Prabhu 421bcef83a0SShreyas B. Prabhu /* 42209206b60SGautham R. Shenoy * Set pnv_first_deep_stop_state, pnv_deepest_stop_psscr_{val,mask}, 42309206b60SGautham R. Shenoy * and the pnv_default_stop_{val,mask}. 42409206b60SGautham R. Shenoy * 425c0691f9dSShreyas B. Prabhu * pnv_first_deep_stop_state should be set to the first stop 426c0691f9dSShreyas B. Prabhu * level to cause hypervisor state loss. 42709206b60SGautham R. Shenoy * 42809206b60SGautham R. Shenoy * pnv_deepest_stop_{val,mask} should be set to values corresponding to 42909206b60SGautham R. Shenoy * the deepest stop state. 43009206b60SGautham R. Shenoy * 43109206b60SGautham R. Shenoy * pnv_default_stop_{val,mask} should be set to values corresponding to 43209206b60SGautham R. Shenoy * the shallowest (OPAL_PM_STOP_INST_FAST) loss-less stop state. 433bcef83a0SShreyas B. Prabhu */ 434bcef83a0SShreyas B. Prabhu pnv_first_deep_stop_state = MAX_STOP_STATE; 435bcef83a0SShreyas B. Prabhu for (i = 0; i < dt_idle_states; i++) { 43609206b60SGautham R. Shenoy int err; 437bcef83a0SShreyas B. Prabhu u64 psscr_rl = psscr_val[i] & PSSCR_RL_MASK; 438bcef83a0SShreyas B. Prabhu 439bcef83a0SShreyas B. Prabhu if ((flags[i] & OPAL_PM_LOSE_FULL_CONTEXT) && 440bcef83a0SShreyas B. Prabhu (pnv_first_deep_stop_state > psscr_rl)) 441bcef83a0SShreyas B. Prabhu pnv_first_deep_stop_state = psscr_rl; 442c0691f9dSShreyas B. Prabhu 44309206b60SGautham R. Shenoy err = validate_psscr_val_mask(&psscr_val[i], &psscr_mask[i], 44409206b60SGautham R. Shenoy flags[i]); 44509206b60SGautham R. Shenoy if (err) { 44609206b60SGautham R. Shenoy report_invalid_psscr_val(psscr_val[i], err); 44709206b60SGautham R. Shenoy continue; 44809206b60SGautham R. Shenoy } 44909206b60SGautham R. Shenoy 45009206b60SGautham R. Shenoy if (max_residency_ns < residency_ns[i]) { 45109206b60SGautham R. Shenoy max_residency_ns = residency_ns[i]; 45209206b60SGautham R. Shenoy pnv_deepest_stop_psscr_val = psscr_val[i]; 45309206b60SGautham R. Shenoy pnv_deepest_stop_psscr_mask = psscr_mask[i]; 45409206b60SGautham R. Shenoy deepest_stop_found = true; 45509206b60SGautham R. Shenoy } 45609206b60SGautham R. Shenoy 45709206b60SGautham R. Shenoy if (!default_stop_found && 45809206b60SGautham R. Shenoy (flags[i] & OPAL_PM_STOP_INST_FAST)) { 45909206b60SGautham R. Shenoy pnv_default_stop_val = psscr_val[i]; 46009206b60SGautham R. Shenoy pnv_default_stop_mask = psscr_mask[i]; 46109206b60SGautham R. Shenoy default_stop_found = true; 46209206b60SGautham R. Shenoy } 46309206b60SGautham R. Shenoy } 46409206b60SGautham R. Shenoy 465f3b3f284SGautham R. Shenoy if (unlikely(!default_stop_found)) { 466f3b3f284SGautham R. Shenoy pr_warn("cpuidle-powernv: No suitable default stop state found. Disabling platform idle.\n"); 467f3b3f284SGautham R. Shenoy } else { 468f3b3f284SGautham R. Shenoy ppc_md.power_save = power9_idle; 469f3b3f284SGautham R. Shenoy pr_info("cpuidle-powernv: Default stop: psscr = 0x%016llx,mask=0x%016llx\n", 47009206b60SGautham R. Shenoy pnv_default_stop_val, pnv_default_stop_mask); 47109206b60SGautham R. Shenoy } 47209206b60SGautham R. Shenoy 473f3b3f284SGautham R. Shenoy if (unlikely(!deepest_stop_found)) { 474f3b3f284SGautham R. Shenoy pr_warn("cpuidle-powernv: No suitable stop state for CPU-Hotplug. Offlined CPUs will busy wait"); 475f3b3f284SGautham R. Shenoy } else { 476f3b3f284SGautham R. Shenoy pr_info("cpuidle-powernv: Deepest stop: psscr = 0x%016llx,mask=0x%016llx\n", 47709206b60SGautham R. Shenoy pnv_deepest_stop_psscr_val, 47809206b60SGautham R. Shenoy pnv_deepest_stop_psscr_mask); 479bcef83a0SShreyas B. Prabhu } 480bcef83a0SShreyas B. Prabhu 481f3b3f284SGautham R. Shenoy pr_info("cpuidle-powernv: Requested Level (RL) value of first deep stop = 0x%llx\n", 482f3b3f284SGautham R. Shenoy pnv_first_deep_stop_state); 483bcef83a0SShreyas B. Prabhu out: 484bcef83a0SShreyas B. Prabhu kfree(psscr_val); 48509206b60SGautham R. Shenoy kfree(psscr_mask); 48609206b60SGautham R. Shenoy kfree(residency_ns); 487bcef83a0SShreyas B. Prabhu return rc; 488bcef83a0SShreyas B. Prabhu } 489bcef83a0SShreyas B. Prabhu 490bcef83a0SShreyas B. Prabhu /* 491bcef83a0SShreyas B. Prabhu * Probe device tree for supported idle states 492bcef83a0SShreyas B. Prabhu */ 493bcef83a0SShreyas B. Prabhu static void __init pnv_probe_idle_states(void) 494bcef83a0SShreyas B. Prabhu { 495bcef83a0SShreyas B. Prabhu struct device_node *np; 496d405a98cSShreyas B. Prabhu int dt_idle_states; 497bcef83a0SShreyas B. Prabhu u32 *flags = NULL; 498d405a98cSShreyas B. Prabhu int i; 499d405a98cSShreyas B. Prabhu 500bcef83a0SShreyas B. Prabhu np = of_find_node_by_path("/ibm,opal/power-mgt"); 501bcef83a0SShreyas B. Prabhu if (!np) { 502d405a98cSShreyas B. Prabhu pr_warn("opal: PowerMgmt Node not found\n"); 503d405a98cSShreyas B. Prabhu goto out; 504d405a98cSShreyas B. Prabhu } 505bcef83a0SShreyas B. Prabhu dt_idle_states = of_property_count_u32_elems(np, 506d405a98cSShreyas B. Prabhu "ibm,cpu-idle-state-flags"); 507d405a98cSShreyas B. Prabhu if (dt_idle_states < 0) { 508d405a98cSShreyas B. Prabhu pr_warn("cpuidle-powernv: no idle states found in the DT\n"); 509d405a98cSShreyas B. Prabhu goto out; 510d405a98cSShreyas B. Prabhu } 511d405a98cSShreyas B. Prabhu 512bcef83a0SShreyas B. Prabhu flags = kcalloc(dt_idle_states, sizeof(*flags), GFP_KERNEL); 513bcef83a0SShreyas B. Prabhu 514bcef83a0SShreyas B. Prabhu if (of_property_read_u32_array(np, 515d405a98cSShreyas B. Prabhu "ibm,cpu-idle-state-flags", flags, dt_idle_states)) { 516d405a98cSShreyas B. Prabhu pr_warn("cpuidle-powernv: missing ibm,cpu-idle-state-flags in DT\n"); 517bcef83a0SShreyas B. Prabhu goto out; 518bcef83a0SShreyas B. Prabhu } 519bcef83a0SShreyas B. Prabhu 520bcef83a0SShreyas B. Prabhu if (cpu_has_feature(CPU_FTR_ARCH_300)) { 521dd34c74cSGautham R. Shenoy if (pnv_power9_idle_init(np, flags, dt_idle_states)) 522bcef83a0SShreyas B. Prabhu goto out; 523d405a98cSShreyas B. Prabhu } 524d405a98cSShreyas B. Prabhu 525d405a98cSShreyas B. Prabhu for (i = 0; i < dt_idle_states; i++) 526d405a98cSShreyas B. Prabhu supported_cpuidle_states |= flags[i]; 527d405a98cSShreyas B. Prabhu 528bcef83a0SShreyas B. Prabhu out: 529bcef83a0SShreyas B. Prabhu kfree(flags); 530bcef83a0SShreyas B. Prabhu } 531bcef83a0SShreyas B. Prabhu static int __init pnv_init_idle_states(void) 532bcef83a0SShreyas B. Prabhu { 533bcef83a0SShreyas B. Prabhu 534bcef83a0SShreyas B. Prabhu supported_cpuidle_states = 0; 535bcef83a0SShreyas B. Prabhu 536bcef83a0SShreyas B. Prabhu if (cpuidle_disable != IDLE_NO_OVERRIDE) 537bcef83a0SShreyas B. Prabhu goto out; 538bcef83a0SShreyas B. Prabhu 539bcef83a0SShreyas B. Prabhu pnv_probe_idle_states(); 540bcef83a0SShreyas B. Prabhu 541d405a98cSShreyas B. Prabhu if (!(supported_cpuidle_states & OPAL_PM_SLEEP_ENABLED_ER1)) { 542d405a98cSShreyas B. Prabhu patch_instruction( 543d405a98cSShreyas B. Prabhu (unsigned int *)pnv_fastsleep_workaround_at_entry, 544d405a98cSShreyas B. Prabhu PPC_INST_NOP); 545d405a98cSShreyas B. Prabhu patch_instruction( 546d405a98cSShreyas B. Prabhu (unsigned int *)pnv_fastsleep_workaround_at_exit, 547d405a98cSShreyas B. Prabhu PPC_INST_NOP); 5485703d2f4SShreyas B. Prabhu } else { 5495703d2f4SShreyas B. Prabhu /* 5505703d2f4SShreyas B. Prabhu * OPAL_PM_SLEEP_ENABLED_ER1 is set. It indicates that 5515703d2f4SShreyas B. Prabhu * workaround is needed to use fastsleep. Provide sysfs 5525703d2f4SShreyas B. Prabhu * control to choose how this workaround has to be applied. 5535703d2f4SShreyas B. Prabhu */ 5545703d2f4SShreyas B. Prabhu device_create_file(cpu_subsys.dev_root, 5555703d2f4SShreyas B. Prabhu &dev_attr_fastsleep_workaround_applyonce); 556d405a98cSShreyas B. Prabhu } 5575703d2f4SShreyas B. Prabhu 558d405a98cSShreyas B. Prabhu pnv_alloc_idle_core_states(); 5595593e303SShreyas B. Prabhu 56017ed4c8fSGautham R. Shenoy /* 56117ed4c8fSGautham R. Shenoy * For each CPU, record its PACA address in each of it's 56217ed4c8fSGautham R. Shenoy * sibling thread's PACA at the slot corresponding to this 56317ed4c8fSGautham R. Shenoy * CPU's index in the core. 56417ed4c8fSGautham R. Shenoy */ 56517ed4c8fSGautham R. Shenoy if (cpu_has_feature(CPU_FTR_POWER9_DD1)) { 56617ed4c8fSGautham R. Shenoy int cpu; 56717ed4c8fSGautham R. Shenoy 56817ed4c8fSGautham R. Shenoy pr_info("powernv: idle: Saving PACA pointers of all CPUs in their thread sibling PACA\n"); 56917ed4c8fSGautham R. Shenoy for_each_possible_cpu(cpu) { 57017ed4c8fSGautham R. Shenoy int base_cpu = cpu_first_thread_sibling(cpu); 57117ed4c8fSGautham R. Shenoy int idx = cpu_thread_in_core(cpu); 57217ed4c8fSGautham R. Shenoy int i; 57317ed4c8fSGautham R. Shenoy 57417ed4c8fSGautham R. Shenoy for (i = 0; i < threads_per_core; i++) { 57517ed4c8fSGautham R. Shenoy int j = base_cpu + i; 57617ed4c8fSGautham R. Shenoy 57717ed4c8fSGautham R. Shenoy paca[j].thread_sibling_pacas[idx] = &paca[cpu]; 57817ed4c8fSGautham R. Shenoy } 57917ed4c8fSGautham R. Shenoy } 58017ed4c8fSGautham R. Shenoy } 58117ed4c8fSGautham R. Shenoy 5825593e303SShreyas B. Prabhu if (supported_cpuidle_states & OPAL_PM_NAP_ENABLED) 5835593e303SShreyas B. Prabhu ppc_md.power_save = power7_idle; 584bcef83a0SShreyas B. Prabhu 585d405a98cSShreyas B. Prabhu out: 586d405a98cSShreyas B. Prabhu return 0; 587d405a98cSShreyas B. Prabhu } 5884bece972SMichael Ellerman machine_subsys_initcall(powernv, pnv_init_idle_states); 589