1d405a98cSShreyas B. Prabhu /* 2d405a98cSShreyas B. Prabhu * PowerNV cpuidle code 3d405a98cSShreyas B. Prabhu * 4d405a98cSShreyas B. Prabhu * Copyright 2015 IBM Corp. 5d405a98cSShreyas B. Prabhu * 6d405a98cSShreyas B. Prabhu * This program is free software; you can redistribute it and/or 7d405a98cSShreyas B. Prabhu * modify it under the terms of the GNU General Public License 8d405a98cSShreyas B. Prabhu * as published by the Free Software Foundation; either version 9d405a98cSShreyas B. Prabhu * 2 of the License, or (at your option) any later version. 10d405a98cSShreyas B. Prabhu */ 11d405a98cSShreyas B. Prabhu 12d405a98cSShreyas B. Prabhu #include <linux/types.h> 13d405a98cSShreyas B. Prabhu #include <linux/mm.h> 14d405a98cSShreyas B. Prabhu #include <linux/slab.h> 15d405a98cSShreyas B. Prabhu #include <linux/of.h> 165703d2f4SShreyas B. Prabhu #include <linux/device.h> 175703d2f4SShreyas B. Prabhu #include <linux/cpu.h> 18d405a98cSShreyas B. Prabhu 19d405a98cSShreyas B. Prabhu #include <asm/firmware.h> 20d405a98cSShreyas B. Prabhu #include <asm/opal.h> 21d405a98cSShreyas B. Prabhu #include <asm/cputhreads.h> 22d405a98cSShreyas B. Prabhu #include <asm/cpuidle.h> 23d405a98cSShreyas B. Prabhu #include <asm/code-patching.h> 24d405a98cSShreyas B. Prabhu #include <asm/smp.h> 25d405a98cSShreyas B. Prabhu 26d405a98cSShreyas B. Prabhu #include "powernv.h" 27d405a98cSShreyas B. Prabhu #include "subcore.h" 28d405a98cSShreyas B. Prabhu 29d405a98cSShreyas B. Prabhu static u32 supported_cpuidle_states; 30d405a98cSShreyas B. Prabhu 31d405a98cSShreyas B. Prabhu int pnv_save_sprs_for_winkle(void) 32d405a98cSShreyas B. Prabhu { 33d405a98cSShreyas B. Prabhu int cpu; 34d405a98cSShreyas B. Prabhu int rc; 35d405a98cSShreyas B. Prabhu 36d405a98cSShreyas B. Prabhu /* 37d405a98cSShreyas B. Prabhu * hid0, hid1, hid4, hid5, hmeer and lpcr values are symmetric accross 38d405a98cSShreyas B. Prabhu * all cpus at boot. Get these reg values of current cpu and use the 39d405a98cSShreyas B. Prabhu * same accross all cpus. 40d405a98cSShreyas B. Prabhu */ 41d405a98cSShreyas B. Prabhu uint64_t lpcr_val = mfspr(SPRN_LPCR) & ~(u64)LPCR_PECE1; 42d405a98cSShreyas B. Prabhu uint64_t hid0_val = mfspr(SPRN_HID0); 43d405a98cSShreyas B. Prabhu uint64_t hid1_val = mfspr(SPRN_HID1); 44d405a98cSShreyas B. Prabhu uint64_t hid4_val = mfspr(SPRN_HID4); 45d405a98cSShreyas B. Prabhu uint64_t hid5_val = mfspr(SPRN_HID5); 46d405a98cSShreyas B. Prabhu uint64_t hmeer_val = mfspr(SPRN_HMEER); 47d405a98cSShreyas B. Prabhu 48d405a98cSShreyas B. Prabhu for_each_possible_cpu(cpu) { 49d405a98cSShreyas B. Prabhu uint64_t pir = get_hard_smp_processor_id(cpu); 50d405a98cSShreyas B. Prabhu uint64_t hsprg0_val = (uint64_t)&paca[cpu]; 51d405a98cSShreyas B. Prabhu 52d405a98cSShreyas B. Prabhu /* 53d405a98cSShreyas B. Prabhu * HSPRG0 is used to store the cpu's pointer to paca. Hence last 54d405a98cSShreyas B. Prabhu * 3 bits are guaranteed to be 0. Program slw to restore HSPRG0 55d405a98cSShreyas B. Prabhu * with 63rd bit set, so that when a thread wakes up at 0x100 we 56d405a98cSShreyas B. Prabhu * can use this bit to distinguish between fastsleep and 57d405a98cSShreyas B. Prabhu * deep winkle. 58d405a98cSShreyas B. Prabhu */ 59d405a98cSShreyas B. Prabhu hsprg0_val |= 1; 60d405a98cSShreyas B. Prabhu 61d405a98cSShreyas B. Prabhu rc = opal_slw_set_reg(pir, SPRN_HSPRG0, hsprg0_val); 62d405a98cSShreyas B. Prabhu if (rc != 0) 63d405a98cSShreyas B. Prabhu return rc; 64d405a98cSShreyas B. Prabhu 65d405a98cSShreyas B. Prabhu rc = opal_slw_set_reg(pir, SPRN_LPCR, lpcr_val); 66d405a98cSShreyas B. Prabhu if (rc != 0) 67d405a98cSShreyas B. Prabhu return rc; 68d405a98cSShreyas B. Prabhu 69d405a98cSShreyas B. Prabhu /* HIDs are per core registers */ 70d405a98cSShreyas B. Prabhu if (cpu_thread_in_core(cpu) == 0) { 71d405a98cSShreyas B. Prabhu 72d405a98cSShreyas B. Prabhu rc = opal_slw_set_reg(pir, SPRN_HMEER, hmeer_val); 73d405a98cSShreyas B. Prabhu if (rc != 0) 74d405a98cSShreyas B. Prabhu return rc; 75d405a98cSShreyas B. Prabhu 76d405a98cSShreyas B. Prabhu rc = opal_slw_set_reg(pir, SPRN_HID0, hid0_val); 77d405a98cSShreyas B. Prabhu if (rc != 0) 78d405a98cSShreyas B. Prabhu return rc; 79d405a98cSShreyas B. Prabhu 80d405a98cSShreyas B. Prabhu rc = opal_slw_set_reg(pir, SPRN_HID1, hid1_val); 81d405a98cSShreyas B. Prabhu if (rc != 0) 82d405a98cSShreyas B. Prabhu return rc; 83d405a98cSShreyas B. Prabhu 84d405a98cSShreyas B. Prabhu rc = opal_slw_set_reg(pir, SPRN_HID4, hid4_val); 85d405a98cSShreyas B. Prabhu if (rc != 0) 86d405a98cSShreyas B. Prabhu return rc; 87d405a98cSShreyas B. Prabhu 88d405a98cSShreyas B. Prabhu rc = opal_slw_set_reg(pir, SPRN_HID5, hid5_val); 89d405a98cSShreyas B. Prabhu if (rc != 0) 90d405a98cSShreyas B. Prabhu return rc; 91d405a98cSShreyas B. Prabhu } 92d405a98cSShreyas B. Prabhu } 93d405a98cSShreyas B. Prabhu 94d405a98cSShreyas B. Prabhu return 0; 95d405a98cSShreyas B. Prabhu } 96d405a98cSShreyas B. Prabhu 97d405a98cSShreyas B. Prabhu static void pnv_alloc_idle_core_states(void) 98d405a98cSShreyas B. Prabhu { 99d405a98cSShreyas B. Prabhu int i, j; 100d405a98cSShreyas B. Prabhu int nr_cores = cpu_nr_cores(); 101d405a98cSShreyas B. Prabhu u32 *core_idle_state; 102d405a98cSShreyas B. Prabhu 103d405a98cSShreyas B. Prabhu /* 104d405a98cSShreyas B. Prabhu * core_idle_state - First 8 bits track the idle state of each thread 105d405a98cSShreyas B. Prabhu * of the core. The 8th bit is the lock bit. Initially all thread bits 106d405a98cSShreyas B. Prabhu * are set. They are cleared when the thread enters deep idle state 107d405a98cSShreyas B. Prabhu * like sleep and winkle. Initially the lock bit is cleared. 108d405a98cSShreyas B. Prabhu * The lock bit has 2 purposes 109d405a98cSShreyas B. Prabhu * a. While the first thread is restoring core state, it prevents 110d405a98cSShreyas B. Prabhu * other threads in the core from switching to process context. 111d405a98cSShreyas B. Prabhu * b. While the last thread in the core is saving the core state, it 112d405a98cSShreyas B. Prabhu * prevents a different thread from waking up. 113d405a98cSShreyas B. Prabhu */ 114d405a98cSShreyas B. Prabhu for (i = 0; i < nr_cores; i++) { 115d405a98cSShreyas B. Prabhu int first_cpu = i * threads_per_core; 116d405a98cSShreyas B. Prabhu int node = cpu_to_node(first_cpu); 117d405a98cSShreyas B. Prabhu 118d405a98cSShreyas B. Prabhu core_idle_state = kmalloc_node(sizeof(u32), GFP_KERNEL, node); 119d405a98cSShreyas B. Prabhu *core_idle_state = PNV_CORE_IDLE_THREAD_BITS; 120d405a98cSShreyas B. Prabhu 121d405a98cSShreyas B. Prabhu for (j = 0; j < threads_per_core; j++) { 122d405a98cSShreyas B. Prabhu int cpu = first_cpu + j; 123d405a98cSShreyas B. Prabhu 124d405a98cSShreyas B. Prabhu paca[cpu].core_idle_state_ptr = core_idle_state; 125d405a98cSShreyas B. Prabhu paca[cpu].thread_idle_state = PNV_THREAD_RUNNING; 126d405a98cSShreyas B. Prabhu paca[cpu].thread_mask = 1 << j; 127d405a98cSShreyas B. Prabhu } 128d405a98cSShreyas B. Prabhu } 129d405a98cSShreyas B. Prabhu 130d405a98cSShreyas B. Prabhu update_subcore_sibling_mask(); 131d405a98cSShreyas B. Prabhu 132d405a98cSShreyas B. Prabhu if (supported_cpuidle_states & OPAL_PM_WINKLE_ENABLED) 133d405a98cSShreyas B. Prabhu pnv_save_sprs_for_winkle(); 134d405a98cSShreyas B. Prabhu } 135d405a98cSShreyas B. Prabhu 136d405a98cSShreyas B. Prabhu u32 pnv_get_supported_cpuidle_states(void) 137d405a98cSShreyas B. Prabhu { 138d405a98cSShreyas B. Prabhu return supported_cpuidle_states; 139d405a98cSShreyas B. Prabhu } 140d405a98cSShreyas B. Prabhu EXPORT_SYMBOL_GPL(pnv_get_supported_cpuidle_states); 141d405a98cSShreyas B. Prabhu 1425703d2f4SShreyas B. Prabhu 1435703d2f4SShreyas B. Prabhu static void pnv_fastsleep_workaround_apply(void *info) 1445703d2f4SShreyas B. Prabhu 1455703d2f4SShreyas B. Prabhu { 1465703d2f4SShreyas B. Prabhu int rc; 1475703d2f4SShreyas B. Prabhu int *err = info; 1485703d2f4SShreyas B. Prabhu 1495703d2f4SShreyas B. Prabhu rc = opal_config_cpu_idle_state(OPAL_CONFIG_IDLE_FASTSLEEP, 1505703d2f4SShreyas B. Prabhu OPAL_CONFIG_IDLE_APPLY); 1515703d2f4SShreyas B. Prabhu if (rc) 1525703d2f4SShreyas B. Prabhu *err = 1; 1535703d2f4SShreyas B. Prabhu } 1545703d2f4SShreyas B. Prabhu 1555703d2f4SShreyas B. Prabhu /* 1565703d2f4SShreyas B. Prabhu * Used to store fastsleep workaround state 1575703d2f4SShreyas B. Prabhu * 0 - Workaround applied/undone at fastsleep entry/exit path (Default) 1585703d2f4SShreyas B. Prabhu * 1 - Workaround applied once, never undone. 1595703d2f4SShreyas B. Prabhu */ 1605703d2f4SShreyas B. Prabhu static u8 fastsleep_workaround_applyonce; 1615703d2f4SShreyas B. Prabhu 1625703d2f4SShreyas B. Prabhu static ssize_t show_fastsleep_workaround_applyonce(struct device *dev, 1635703d2f4SShreyas B. Prabhu struct device_attribute *attr, char *buf) 1645703d2f4SShreyas B. Prabhu { 1655703d2f4SShreyas B. Prabhu return sprintf(buf, "%u\n", fastsleep_workaround_applyonce); 1665703d2f4SShreyas B. Prabhu } 1675703d2f4SShreyas B. Prabhu 1685703d2f4SShreyas B. Prabhu static ssize_t store_fastsleep_workaround_applyonce(struct device *dev, 1695703d2f4SShreyas B. Prabhu struct device_attribute *attr, const char *buf, 1705703d2f4SShreyas B. Prabhu size_t count) 1715703d2f4SShreyas B. Prabhu { 1725703d2f4SShreyas B. Prabhu cpumask_t primary_thread_mask; 1735703d2f4SShreyas B. Prabhu int err; 1745703d2f4SShreyas B. Prabhu u8 val; 1755703d2f4SShreyas B. Prabhu 1765703d2f4SShreyas B. Prabhu if (kstrtou8(buf, 0, &val) || val != 1) 1775703d2f4SShreyas B. Prabhu return -EINVAL; 1785703d2f4SShreyas B. Prabhu 1795703d2f4SShreyas B. Prabhu if (fastsleep_workaround_applyonce == 1) 1805703d2f4SShreyas B. Prabhu return count; 1815703d2f4SShreyas B. Prabhu 1825703d2f4SShreyas B. Prabhu /* 1835703d2f4SShreyas B. Prabhu * fastsleep_workaround_applyonce = 1 implies 1845703d2f4SShreyas B. Prabhu * fastsleep workaround needs to be left in 'applied' state on all 1855703d2f4SShreyas B. Prabhu * the cores. Do this by- 1865703d2f4SShreyas B. Prabhu * 1. Patching out the call to 'undo' workaround in fastsleep exit path 1875703d2f4SShreyas B. Prabhu * 2. Sending ipi to all the cores which have atleast one online thread 1885703d2f4SShreyas B. Prabhu * 3. Patching out the call to 'apply' workaround in fastsleep entry 1895703d2f4SShreyas B. Prabhu * path 1905703d2f4SShreyas B. Prabhu * There is no need to send ipi to cores which have all threads 1915703d2f4SShreyas B. Prabhu * offlined, as last thread of the core entering fastsleep or deeper 1925703d2f4SShreyas B. Prabhu * state would have applied workaround. 1935703d2f4SShreyas B. Prabhu */ 1945703d2f4SShreyas B. Prabhu err = patch_instruction( 1955703d2f4SShreyas B. Prabhu (unsigned int *)pnv_fastsleep_workaround_at_exit, 1965703d2f4SShreyas B. Prabhu PPC_INST_NOP); 1975703d2f4SShreyas B. Prabhu if (err) { 1985703d2f4SShreyas B. Prabhu pr_err("fastsleep_workaround_applyonce change failed while patching pnv_fastsleep_workaround_at_exit"); 1995703d2f4SShreyas B. Prabhu goto fail; 2005703d2f4SShreyas B. Prabhu } 2015703d2f4SShreyas B. Prabhu 2025703d2f4SShreyas B. Prabhu get_online_cpus(); 2035703d2f4SShreyas B. Prabhu primary_thread_mask = cpu_online_cores_map(); 2045703d2f4SShreyas B. Prabhu on_each_cpu_mask(&primary_thread_mask, 2055703d2f4SShreyas B. Prabhu pnv_fastsleep_workaround_apply, 2065703d2f4SShreyas B. Prabhu &err, 1); 2075703d2f4SShreyas B. Prabhu put_online_cpus(); 2085703d2f4SShreyas B. Prabhu if (err) { 2095703d2f4SShreyas B. Prabhu pr_err("fastsleep_workaround_applyonce change failed while running pnv_fastsleep_workaround_apply"); 2105703d2f4SShreyas B. Prabhu goto fail; 2115703d2f4SShreyas B. Prabhu } 2125703d2f4SShreyas B. Prabhu 2135703d2f4SShreyas B. Prabhu err = patch_instruction( 2145703d2f4SShreyas B. Prabhu (unsigned int *)pnv_fastsleep_workaround_at_entry, 2155703d2f4SShreyas B. Prabhu PPC_INST_NOP); 2165703d2f4SShreyas B. Prabhu if (err) { 2175703d2f4SShreyas B. Prabhu pr_err("fastsleep_workaround_applyonce change failed while patching pnv_fastsleep_workaround_at_entry"); 2185703d2f4SShreyas B. Prabhu goto fail; 2195703d2f4SShreyas B. Prabhu } 2205703d2f4SShreyas B. Prabhu 2215703d2f4SShreyas B. Prabhu fastsleep_workaround_applyonce = 1; 2225703d2f4SShreyas B. Prabhu 2235703d2f4SShreyas B. Prabhu return count; 2245703d2f4SShreyas B. Prabhu fail: 2255703d2f4SShreyas B. Prabhu return -EIO; 2265703d2f4SShreyas B. Prabhu } 2275703d2f4SShreyas B. Prabhu 2285703d2f4SShreyas B. Prabhu static DEVICE_ATTR(fastsleep_workaround_applyonce, 0600, 2295703d2f4SShreyas B. Prabhu show_fastsleep_workaround_applyonce, 2305703d2f4SShreyas B. Prabhu store_fastsleep_workaround_applyonce); 2315703d2f4SShreyas B. Prabhu 232d405a98cSShreyas B. Prabhu static int __init pnv_init_idle_states(void) 233d405a98cSShreyas B. Prabhu { 234d405a98cSShreyas B. Prabhu struct device_node *power_mgt; 235d405a98cSShreyas B. Prabhu int dt_idle_states; 236d405a98cSShreyas B. Prabhu u32 *flags; 237d405a98cSShreyas B. Prabhu int i; 238d405a98cSShreyas B. Prabhu 239d405a98cSShreyas B. Prabhu supported_cpuidle_states = 0; 240d405a98cSShreyas B. Prabhu 241d405a98cSShreyas B. Prabhu if (cpuidle_disable != IDLE_NO_OVERRIDE) 242d405a98cSShreyas B. Prabhu goto out; 243d405a98cSShreyas B. Prabhu 244d405a98cSShreyas B. Prabhu if (!firmware_has_feature(FW_FEATURE_OPALv3)) 245d405a98cSShreyas B. Prabhu goto out; 246d405a98cSShreyas B. Prabhu 247d405a98cSShreyas B. Prabhu power_mgt = of_find_node_by_path("/ibm,opal/power-mgt"); 248d405a98cSShreyas B. Prabhu if (!power_mgt) { 249d405a98cSShreyas B. Prabhu pr_warn("opal: PowerMgmt Node not found\n"); 250d405a98cSShreyas B. Prabhu goto out; 251d405a98cSShreyas B. Prabhu } 252d405a98cSShreyas B. Prabhu dt_idle_states = of_property_count_u32_elems(power_mgt, 253d405a98cSShreyas B. Prabhu "ibm,cpu-idle-state-flags"); 254d405a98cSShreyas B. Prabhu if (dt_idle_states < 0) { 255d405a98cSShreyas B. Prabhu pr_warn("cpuidle-powernv: no idle states found in the DT\n"); 256d405a98cSShreyas B. Prabhu goto out; 257d405a98cSShreyas B. Prabhu } 258d405a98cSShreyas B. Prabhu 259d405a98cSShreyas B. Prabhu flags = kzalloc(sizeof(*flags) * dt_idle_states, GFP_KERNEL); 260d405a98cSShreyas B. Prabhu if (of_property_read_u32_array(power_mgt, 261d405a98cSShreyas B. Prabhu "ibm,cpu-idle-state-flags", flags, dt_idle_states)) { 262d405a98cSShreyas B. Prabhu pr_warn("cpuidle-powernv: missing ibm,cpu-idle-state-flags in DT\n"); 263d405a98cSShreyas B. Prabhu goto out_free; 264d405a98cSShreyas B. Prabhu } 265d405a98cSShreyas B. Prabhu 266d405a98cSShreyas B. Prabhu for (i = 0; i < dt_idle_states; i++) 267d405a98cSShreyas B. Prabhu supported_cpuidle_states |= flags[i]; 268d405a98cSShreyas B. Prabhu 269d405a98cSShreyas B. Prabhu if (!(supported_cpuidle_states & OPAL_PM_SLEEP_ENABLED_ER1)) { 270d405a98cSShreyas B. Prabhu patch_instruction( 271d405a98cSShreyas B. Prabhu (unsigned int *)pnv_fastsleep_workaround_at_entry, 272d405a98cSShreyas B. Prabhu PPC_INST_NOP); 273d405a98cSShreyas B. Prabhu patch_instruction( 274d405a98cSShreyas B. Prabhu (unsigned int *)pnv_fastsleep_workaround_at_exit, 275d405a98cSShreyas B. Prabhu PPC_INST_NOP); 2765703d2f4SShreyas B. Prabhu } else { 2775703d2f4SShreyas B. Prabhu /* 2785703d2f4SShreyas B. Prabhu * OPAL_PM_SLEEP_ENABLED_ER1 is set. It indicates that 2795703d2f4SShreyas B. Prabhu * workaround is needed to use fastsleep. Provide sysfs 2805703d2f4SShreyas B. Prabhu * control to choose how this workaround has to be applied. 2815703d2f4SShreyas B. Prabhu */ 2825703d2f4SShreyas B. Prabhu device_create_file(cpu_subsys.dev_root, 2835703d2f4SShreyas B. Prabhu &dev_attr_fastsleep_workaround_applyonce); 284d405a98cSShreyas B. Prabhu } 2855703d2f4SShreyas B. Prabhu 286d405a98cSShreyas B. Prabhu pnv_alloc_idle_core_states(); 287d405a98cSShreyas B. Prabhu out_free: 288d405a98cSShreyas B. Prabhu kfree(flags); 289d405a98cSShreyas B. Prabhu out: 290d405a98cSShreyas B. Prabhu return 0; 291d405a98cSShreyas B. Prabhu } 292d405a98cSShreyas B. Prabhu 293d405a98cSShreyas B. Prabhu subsys_initcall(pnv_init_idle_states); 294