126717172SLen Brown /* 226717172SLen Brown * intel_idle.c - native hardware idle loop for modern Intel processors 326717172SLen Brown * 4fab04b22SLen Brown * Copyright (c) 2013, Intel Corporation. 526717172SLen Brown * Len Brown <len.brown@intel.com> 626717172SLen Brown * 726717172SLen Brown * This program is free software; you can redistribute it and/or modify it 826717172SLen Brown * under the terms and conditions of the GNU General Public License, 926717172SLen Brown * version 2, as published by the Free Software Foundation. 1026717172SLen Brown * 1126717172SLen Brown * This program is distributed in the hope it will be useful, but WITHOUT 1226717172SLen Brown * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or 1326717172SLen Brown * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for 1426717172SLen Brown * more details. 1526717172SLen Brown * 1626717172SLen Brown * You should have received a copy of the GNU General Public License along with 1726717172SLen Brown * this program; if not, write to the Free Software Foundation, Inc., 1826717172SLen Brown * 51 Franklin St - Fifth Floor, Boston, MA 02110-1301 USA. 1926717172SLen Brown */ 2026717172SLen Brown 2126717172SLen Brown /* 2226717172SLen Brown * intel_idle is a cpuidle driver that loads on specific Intel processors 2326717172SLen Brown * in lieu of the legacy ACPI processor_idle driver. The intent is to 2426717172SLen Brown * make Linux more efficient on these processors, as intel_idle knows 2526717172SLen Brown * more than ACPI, as well as make Linux more immune to ACPI BIOS bugs. 2626717172SLen Brown */ 2726717172SLen Brown 2826717172SLen Brown /* 2926717172SLen Brown * Design Assumptions 3026717172SLen Brown * 3126717172SLen Brown * All CPUs have same idle states as boot CPU 3226717172SLen Brown * 3326717172SLen Brown * Chipset BM_STS (bus master status) bit is a NOP 3426717172SLen Brown * for preventing entry into deep C-stats 3526717172SLen Brown */ 3626717172SLen Brown 3726717172SLen Brown /* 3826717172SLen Brown * Known limitations 3926717172SLen Brown * 4026717172SLen Brown * The driver currently initializes for_each_online_cpu() upon modprobe. 4126717172SLen Brown * It it unaware of subsequent processors hot-added to the system. 4226717172SLen Brown * This means that if you boot with maxcpus=n and later online 4326717172SLen Brown * processors above n, those processors will use C1 only. 4426717172SLen Brown * 4526717172SLen Brown * ACPI has a .suspend hack to turn off deep c-statees during suspend 4626717172SLen Brown * to avoid complications with the lapic timer workaround. 4726717172SLen Brown * Have not seen issues with suspend, but may need same workaround here. 4826717172SLen Brown * 4926717172SLen Brown */ 5026717172SLen Brown 5126717172SLen Brown /* un-comment DEBUG to enable pr_debug() statements */ 5226717172SLen Brown #define DEBUG 5326717172SLen Brown 5426717172SLen Brown #include <linux/kernel.h> 5526717172SLen Brown #include <linux/cpuidle.h> 5676962caaSThomas Gleixner #include <linux/tick.h> 5726717172SLen Brown #include <trace/events/power.h> 5826717172SLen Brown #include <linux/sched.h> 592a2d31c8SShaohua Li #include <linux/notifier.h> 602a2d31c8SShaohua Li #include <linux/cpu.h> 6102c4fae9SPaul Gortmaker #include <linux/moduleparam.h> 62b66b8b9aSAndi Kleen #include <asm/cpu_device_id.h> 63db73c5a8SDave Hansen #include <asm/intel-family.h> 64bc83ccccSH. Peter Anvin #include <asm/mwait.h> 6514796fcaSLen Brown #include <asm/msr.h> 6626717172SLen Brown 67d70e28f5SLen Brown #define INTEL_IDLE_VERSION "0.4.1" 6826717172SLen Brown #define PREFIX "intel_idle: " 6926717172SLen Brown 7026717172SLen Brown static struct cpuidle_driver intel_idle_driver = { 7126717172SLen Brown .name = "intel_idle", 7226717172SLen Brown .owner = THIS_MODULE, 7326717172SLen Brown }; 7426717172SLen Brown /* intel_idle.max_cstate=0 disables driver */ 75137ecc77SLen Brown static int max_cstate = CPUIDLE_STATE_MAX - 1; 7626717172SLen Brown 77c4236282SLen Brown static unsigned int mwait_substates; 7826717172SLen Brown 792a2d31c8SShaohua Li #define LAPIC_TIMER_ALWAYS_RELIABLE 0xFFFFFFFF 8026717172SLen Brown /* Reliable LAPIC Timer States, bit 1 for C1 etc. */ 81d13780d4SLen Brown static unsigned int lapic_timer_reliable_states = (1 << 1); /* Default to only C1 */ 8226717172SLen Brown 83b66b8b9aSAndi Kleen struct idle_cpu { 84b66b8b9aSAndi Kleen struct cpuidle_state *state_table; 8526717172SLen Brown 8626717172SLen Brown /* 8714796fcaSLen Brown * Hardware C-state auto-demotion may not always be optimal. 8814796fcaSLen Brown * Indicate which enable bits to clear here. 8914796fcaSLen Brown */ 90b66b8b9aSAndi Kleen unsigned long auto_demotion_disable_flags; 918c058d53SLen Brown bool byt_auto_demotion_disable_flag; 9232e95180SLen Brown bool disable_promotion_to_c1e; 93b66b8b9aSAndi Kleen }; 94b66b8b9aSAndi Kleen 95b66b8b9aSAndi Kleen static const struct idle_cpu *icpu; 96b66b8b9aSAndi Kleen static struct cpuidle_device __percpu *intel_idle_cpuidle_devices; 97b66b8b9aSAndi Kleen static int intel_idle(struct cpuidle_device *dev, 98b66b8b9aSAndi Kleen struct cpuidle_driver *drv, int index); 995fe2e527SRafael J. Wysocki static void intel_idle_freeze(struct cpuidle_device *dev, 1005fe2e527SRafael J. Wysocki struct cpuidle_driver *drv, int index); 101b66b8b9aSAndi Kleen static struct cpuidle_state *cpuidle_state_table; 10214796fcaSLen Brown 10314796fcaSLen Brown /* 104956d033fSLen Brown * Set this flag for states where the HW flushes the TLB for us 105956d033fSLen Brown * and so we don't need cross-calls to keep it consistent. 106956d033fSLen Brown * If this flag is set, SW flushes the TLB, so even if the 107956d033fSLen Brown * HW doesn't do the flushing, this flag is safe to use. 108956d033fSLen Brown */ 109956d033fSLen Brown #define CPUIDLE_FLAG_TLB_FLUSHED 0x10000 110956d033fSLen Brown 111956d033fSLen Brown /* 112b1beab48SLen Brown * MWAIT takes an 8-bit "hint" in EAX "suggesting" 113b1beab48SLen Brown * the C-state (top nibble) and sub-state (bottom nibble) 114b1beab48SLen Brown * 0x00 means "MWAIT(C1)", 0x10 means "MWAIT(C2)" etc. 115b1beab48SLen Brown * 116b1beab48SLen Brown * We store the hint at the top of our "flags" for each state. 117b1beab48SLen Brown */ 118b1beab48SLen Brown #define flg2MWAIT(flags) (((flags) >> 24) & 0xFF) 119b1beab48SLen Brown #define MWAIT2flg(eax) ((eax & 0xFF) << 24) 120b1beab48SLen Brown 121b1beab48SLen Brown /* 12226717172SLen Brown * States are indexed by the cstate number, 12326717172SLen Brown * which is also the index into the MWAIT hint array. 12426717172SLen Brown * Thus C0 is a dummy. 12526717172SLen Brown */ 126ba0dc81eSJiang Liu static struct cpuidle_state nehalem_cstates[] = { 127e022e7ebSLen Brown { 128de09cdd0SLen Brown .name = "C1", 12926717172SLen Brown .desc = "MWAIT 0x00", 130b82b6ccaSDaniel Lezcano .flags = MWAIT2flg(0x00), 13126717172SLen Brown .exit_latency = 3, 13226717172SLen Brown .target_residency = 6, 1335fe2e527SRafael J. Wysocki .enter = &intel_idle, 1345fe2e527SRafael J. Wysocki .enter_freeze = intel_idle_freeze, }, 135e022e7ebSLen Brown { 136de09cdd0SLen Brown .name = "C1E", 13732e95180SLen Brown .desc = "MWAIT 0x01", 138b82b6ccaSDaniel Lezcano .flags = MWAIT2flg(0x01), 13932e95180SLen Brown .exit_latency = 10, 14032e95180SLen Brown .target_residency = 20, 1415fe2e527SRafael J. Wysocki .enter = &intel_idle, 1425fe2e527SRafael J. Wysocki .enter_freeze = intel_idle_freeze, }, 14332e95180SLen Brown { 144de09cdd0SLen Brown .name = "C3", 14526717172SLen Brown .desc = "MWAIT 0x10", 146b82b6ccaSDaniel Lezcano .flags = MWAIT2flg(0x10) | CPUIDLE_FLAG_TLB_FLUSHED, 14726717172SLen Brown .exit_latency = 20, 14826717172SLen Brown .target_residency = 80, 1495fe2e527SRafael J. Wysocki .enter = &intel_idle, 1505fe2e527SRafael J. Wysocki .enter_freeze = intel_idle_freeze, }, 151e022e7ebSLen Brown { 152de09cdd0SLen Brown .name = "C6", 15326717172SLen Brown .desc = "MWAIT 0x20", 154b82b6ccaSDaniel Lezcano .flags = MWAIT2flg(0x20) | CPUIDLE_FLAG_TLB_FLUSHED, 15526717172SLen Brown .exit_latency = 200, 15626717172SLen Brown .target_residency = 800, 1575fe2e527SRafael J. Wysocki .enter = &intel_idle, 1585fe2e527SRafael J. Wysocki .enter_freeze = intel_idle_freeze, }, 159e022e7ebSLen Brown { 160e022e7ebSLen Brown .enter = NULL } 16126717172SLen Brown }; 16226717172SLen Brown 163ba0dc81eSJiang Liu static struct cpuidle_state snb_cstates[] = { 164e022e7ebSLen Brown { 165de09cdd0SLen Brown .name = "C1", 166d13780d4SLen Brown .desc = "MWAIT 0x00", 167b82b6ccaSDaniel Lezcano .flags = MWAIT2flg(0x00), 16832e95180SLen Brown .exit_latency = 2, 16932e95180SLen Brown .target_residency = 2, 1705fe2e527SRafael J. Wysocki .enter = &intel_idle, 1715fe2e527SRafael J. Wysocki .enter_freeze = intel_idle_freeze, }, 17232e95180SLen Brown { 173de09cdd0SLen Brown .name = "C1E", 17432e95180SLen Brown .desc = "MWAIT 0x01", 175b82b6ccaSDaniel Lezcano .flags = MWAIT2flg(0x01), 17632e95180SLen Brown .exit_latency = 10, 17732e95180SLen Brown .target_residency = 20, 1785fe2e527SRafael J. Wysocki .enter = &intel_idle, 1795fe2e527SRafael J. Wysocki .enter_freeze = intel_idle_freeze, }, 180e022e7ebSLen Brown { 181de09cdd0SLen Brown .name = "C3", 182d13780d4SLen Brown .desc = "MWAIT 0x10", 183b82b6ccaSDaniel Lezcano .flags = MWAIT2flg(0x10) | CPUIDLE_FLAG_TLB_FLUSHED, 184d13780d4SLen Brown .exit_latency = 80, 185ddbd550dSLen Brown .target_residency = 211, 1865fe2e527SRafael J. Wysocki .enter = &intel_idle, 1875fe2e527SRafael J. Wysocki .enter_freeze = intel_idle_freeze, }, 188e022e7ebSLen Brown { 189de09cdd0SLen Brown .name = "C6", 190d13780d4SLen Brown .desc = "MWAIT 0x20", 191b82b6ccaSDaniel Lezcano .flags = MWAIT2flg(0x20) | CPUIDLE_FLAG_TLB_FLUSHED, 192d13780d4SLen Brown .exit_latency = 104, 193ddbd550dSLen Brown .target_residency = 345, 1945fe2e527SRafael J. Wysocki .enter = &intel_idle, 1955fe2e527SRafael J. Wysocki .enter_freeze = intel_idle_freeze, }, 196e022e7ebSLen Brown { 197de09cdd0SLen Brown .name = "C7", 198d13780d4SLen Brown .desc = "MWAIT 0x30", 199b82b6ccaSDaniel Lezcano .flags = MWAIT2flg(0x30) | CPUIDLE_FLAG_TLB_FLUSHED, 200d13780d4SLen Brown .exit_latency = 109, 201ddbd550dSLen Brown .target_residency = 345, 2025fe2e527SRafael J. Wysocki .enter = &intel_idle, 2035fe2e527SRafael J. Wysocki .enter_freeze = intel_idle_freeze, }, 204e022e7ebSLen Brown { 205e022e7ebSLen Brown .enter = NULL } 206d13780d4SLen Brown }; 207d13780d4SLen Brown 208718987d6SLen Brown static struct cpuidle_state byt_cstates[] = { 209718987d6SLen Brown { 210de09cdd0SLen Brown .name = "C1", 211718987d6SLen Brown .desc = "MWAIT 0x00", 212b82b6ccaSDaniel Lezcano .flags = MWAIT2flg(0x00), 213718987d6SLen Brown .exit_latency = 1, 214718987d6SLen Brown .target_residency = 1, 2155fe2e527SRafael J. Wysocki .enter = &intel_idle, 2165fe2e527SRafael J. Wysocki .enter_freeze = intel_idle_freeze, }, 217718987d6SLen Brown { 218de09cdd0SLen Brown .name = "C6N", 219718987d6SLen Brown .desc = "MWAIT 0x58", 220b82b6ccaSDaniel Lezcano .flags = MWAIT2flg(0x58) | CPUIDLE_FLAG_TLB_FLUSHED, 221d7ef7671SLen Brown .exit_latency = 300, 222718987d6SLen Brown .target_residency = 275, 2235fe2e527SRafael J. Wysocki .enter = &intel_idle, 2245fe2e527SRafael J. Wysocki .enter_freeze = intel_idle_freeze, }, 225718987d6SLen Brown { 226de09cdd0SLen Brown .name = "C6S", 227718987d6SLen Brown .desc = "MWAIT 0x52", 228b82b6ccaSDaniel Lezcano .flags = MWAIT2flg(0x52) | CPUIDLE_FLAG_TLB_FLUSHED, 229d7ef7671SLen Brown .exit_latency = 500, 230718987d6SLen Brown .target_residency = 560, 2315fe2e527SRafael J. Wysocki .enter = &intel_idle, 2325fe2e527SRafael J. Wysocki .enter_freeze = intel_idle_freeze, }, 233718987d6SLen Brown { 234de09cdd0SLen Brown .name = "C7", 235718987d6SLen Brown .desc = "MWAIT 0x60", 236b82b6ccaSDaniel Lezcano .flags = MWAIT2flg(0x60) | CPUIDLE_FLAG_TLB_FLUSHED, 237718987d6SLen Brown .exit_latency = 1200, 238d7ef7671SLen Brown .target_residency = 4000, 2395fe2e527SRafael J. Wysocki .enter = &intel_idle, 2405fe2e527SRafael J. Wysocki .enter_freeze = intel_idle_freeze, }, 241718987d6SLen Brown { 242de09cdd0SLen Brown .name = "C7S", 243718987d6SLen Brown .desc = "MWAIT 0x64", 244b82b6ccaSDaniel Lezcano .flags = MWAIT2flg(0x64) | CPUIDLE_FLAG_TLB_FLUSHED, 245718987d6SLen Brown .exit_latency = 10000, 246718987d6SLen Brown .target_residency = 20000, 2475fe2e527SRafael J. Wysocki .enter = &intel_idle, 2485fe2e527SRafael J. Wysocki .enter_freeze = intel_idle_freeze, }, 249718987d6SLen Brown { 250718987d6SLen Brown .enter = NULL } 251718987d6SLen Brown }; 252718987d6SLen Brown 253cab07a56SLen Brown static struct cpuidle_state cht_cstates[] = { 254cab07a56SLen Brown { 255de09cdd0SLen Brown .name = "C1", 256cab07a56SLen Brown .desc = "MWAIT 0x00", 257cab07a56SLen Brown .flags = MWAIT2flg(0x00), 258cab07a56SLen Brown .exit_latency = 1, 259cab07a56SLen Brown .target_residency = 1, 260cab07a56SLen Brown .enter = &intel_idle, 261cab07a56SLen Brown .enter_freeze = intel_idle_freeze, }, 262cab07a56SLen Brown { 263de09cdd0SLen Brown .name = "C6N", 264cab07a56SLen Brown .desc = "MWAIT 0x58", 265cab07a56SLen Brown .flags = MWAIT2flg(0x58) | CPUIDLE_FLAG_TLB_FLUSHED, 266cab07a56SLen Brown .exit_latency = 80, 267cab07a56SLen Brown .target_residency = 275, 268cab07a56SLen Brown .enter = &intel_idle, 269cab07a56SLen Brown .enter_freeze = intel_idle_freeze, }, 270cab07a56SLen Brown { 271de09cdd0SLen Brown .name = "C6S", 272cab07a56SLen Brown .desc = "MWAIT 0x52", 273cab07a56SLen Brown .flags = MWAIT2flg(0x52) | CPUIDLE_FLAG_TLB_FLUSHED, 274cab07a56SLen Brown .exit_latency = 200, 275cab07a56SLen Brown .target_residency = 560, 276cab07a56SLen Brown .enter = &intel_idle, 277cab07a56SLen Brown .enter_freeze = intel_idle_freeze, }, 278cab07a56SLen Brown { 279de09cdd0SLen Brown .name = "C7", 280cab07a56SLen Brown .desc = "MWAIT 0x60", 281cab07a56SLen Brown .flags = MWAIT2flg(0x60) | CPUIDLE_FLAG_TLB_FLUSHED, 282cab07a56SLen Brown .exit_latency = 1200, 283cab07a56SLen Brown .target_residency = 4000, 284cab07a56SLen Brown .enter = &intel_idle, 285cab07a56SLen Brown .enter_freeze = intel_idle_freeze, }, 286cab07a56SLen Brown { 287de09cdd0SLen Brown .name = "C7S", 288cab07a56SLen Brown .desc = "MWAIT 0x64", 289cab07a56SLen Brown .flags = MWAIT2flg(0x64) | CPUIDLE_FLAG_TLB_FLUSHED, 290cab07a56SLen Brown .exit_latency = 10000, 291cab07a56SLen Brown .target_residency = 20000, 292cab07a56SLen Brown .enter = &intel_idle, 293cab07a56SLen Brown .enter_freeze = intel_idle_freeze, }, 294cab07a56SLen Brown { 295cab07a56SLen Brown .enter = NULL } 296cab07a56SLen Brown }; 297cab07a56SLen Brown 298ba0dc81eSJiang Liu static struct cpuidle_state ivb_cstates[] = { 299e022e7ebSLen Brown { 300de09cdd0SLen Brown .name = "C1", 3016edab08cSLen Brown .desc = "MWAIT 0x00", 302b82b6ccaSDaniel Lezcano .flags = MWAIT2flg(0x00), 3036edab08cSLen Brown .exit_latency = 1, 3046edab08cSLen Brown .target_residency = 1, 3055fe2e527SRafael J. Wysocki .enter = &intel_idle, 3065fe2e527SRafael J. Wysocki .enter_freeze = intel_idle_freeze, }, 307e022e7ebSLen Brown { 308de09cdd0SLen Brown .name = "C1E", 30932e95180SLen Brown .desc = "MWAIT 0x01", 310b82b6ccaSDaniel Lezcano .flags = MWAIT2flg(0x01), 31132e95180SLen Brown .exit_latency = 10, 31232e95180SLen Brown .target_residency = 20, 3135fe2e527SRafael J. Wysocki .enter = &intel_idle, 3145fe2e527SRafael J. Wysocki .enter_freeze = intel_idle_freeze, }, 31532e95180SLen Brown { 316de09cdd0SLen Brown .name = "C3", 3176edab08cSLen Brown .desc = "MWAIT 0x10", 318b82b6ccaSDaniel Lezcano .flags = MWAIT2flg(0x10) | CPUIDLE_FLAG_TLB_FLUSHED, 3196edab08cSLen Brown .exit_latency = 59, 3206edab08cSLen Brown .target_residency = 156, 3215fe2e527SRafael J. Wysocki .enter = &intel_idle, 3225fe2e527SRafael J. Wysocki .enter_freeze = intel_idle_freeze, }, 323e022e7ebSLen Brown { 324de09cdd0SLen Brown .name = "C6", 3256edab08cSLen Brown .desc = "MWAIT 0x20", 326b82b6ccaSDaniel Lezcano .flags = MWAIT2flg(0x20) | CPUIDLE_FLAG_TLB_FLUSHED, 3276edab08cSLen Brown .exit_latency = 80, 3286edab08cSLen Brown .target_residency = 300, 3295fe2e527SRafael J. Wysocki .enter = &intel_idle, 3305fe2e527SRafael J. Wysocki .enter_freeze = intel_idle_freeze, }, 331e022e7ebSLen Brown { 332de09cdd0SLen Brown .name = "C7", 3336edab08cSLen Brown .desc = "MWAIT 0x30", 334b82b6ccaSDaniel Lezcano .flags = MWAIT2flg(0x30) | CPUIDLE_FLAG_TLB_FLUSHED, 3356edab08cSLen Brown .exit_latency = 87, 3366edab08cSLen Brown .target_residency = 300, 3375fe2e527SRafael J. Wysocki .enter = &intel_idle, 3385fe2e527SRafael J. Wysocki .enter_freeze = intel_idle_freeze, }, 339e022e7ebSLen Brown { 340e022e7ebSLen Brown .enter = NULL } 3416edab08cSLen Brown }; 3426edab08cSLen Brown 3430138d8f0SLen Brown static struct cpuidle_state ivt_cstates[] = { 3440138d8f0SLen Brown { 345de09cdd0SLen Brown .name = "C1", 3460138d8f0SLen Brown .desc = "MWAIT 0x00", 347b82b6ccaSDaniel Lezcano .flags = MWAIT2flg(0x00), 3480138d8f0SLen Brown .exit_latency = 1, 3490138d8f0SLen Brown .target_residency = 1, 3505fe2e527SRafael J. Wysocki .enter = &intel_idle, 3515fe2e527SRafael J. Wysocki .enter_freeze = intel_idle_freeze, }, 3520138d8f0SLen Brown { 353de09cdd0SLen Brown .name = "C1E", 3540138d8f0SLen Brown .desc = "MWAIT 0x01", 355b82b6ccaSDaniel Lezcano .flags = MWAIT2flg(0x01), 3560138d8f0SLen Brown .exit_latency = 10, 3570138d8f0SLen Brown .target_residency = 80, 3585fe2e527SRafael J. Wysocki .enter = &intel_idle, 3595fe2e527SRafael J. Wysocki .enter_freeze = intel_idle_freeze, }, 3600138d8f0SLen Brown { 361de09cdd0SLen Brown .name = "C3", 3620138d8f0SLen Brown .desc = "MWAIT 0x10", 363b82b6ccaSDaniel Lezcano .flags = MWAIT2flg(0x10) | CPUIDLE_FLAG_TLB_FLUSHED, 3640138d8f0SLen Brown .exit_latency = 59, 3650138d8f0SLen Brown .target_residency = 156, 3665fe2e527SRafael J. Wysocki .enter = &intel_idle, 3675fe2e527SRafael J. Wysocki .enter_freeze = intel_idle_freeze, }, 3680138d8f0SLen Brown { 369de09cdd0SLen Brown .name = "C6", 3700138d8f0SLen Brown .desc = "MWAIT 0x20", 371b82b6ccaSDaniel Lezcano .flags = MWAIT2flg(0x20) | CPUIDLE_FLAG_TLB_FLUSHED, 3720138d8f0SLen Brown .exit_latency = 82, 3730138d8f0SLen Brown .target_residency = 300, 3745fe2e527SRafael J. Wysocki .enter = &intel_idle, 3755fe2e527SRafael J. Wysocki .enter_freeze = intel_idle_freeze, }, 3760138d8f0SLen Brown { 3770138d8f0SLen Brown .enter = NULL } 3780138d8f0SLen Brown }; 3790138d8f0SLen Brown 3800138d8f0SLen Brown static struct cpuidle_state ivt_cstates_4s[] = { 3810138d8f0SLen Brown { 382de09cdd0SLen Brown .name = "C1", 3830138d8f0SLen Brown .desc = "MWAIT 0x00", 384b82b6ccaSDaniel Lezcano .flags = MWAIT2flg(0x00), 3850138d8f0SLen Brown .exit_latency = 1, 3860138d8f0SLen Brown .target_residency = 1, 3875fe2e527SRafael J. Wysocki .enter = &intel_idle, 3885fe2e527SRafael J. Wysocki .enter_freeze = intel_idle_freeze, }, 3890138d8f0SLen Brown { 390de09cdd0SLen Brown .name = "C1E", 3910138d8f0SLen Brown .desc = "MWAIT 0x01", 392b82b6ccaSDaniel Lezcano .flags = MWAIT2flg(0x01), 3930138d8f0SLen Brown .exit_latency = 10, 3940138d8f0SLen Brown .target_residency = 250, 3955fe2e527SRafael J. Wysocki .enter = &intel_idle, 3965fe2e527SRafael J. Wysocki .enter_freeze = intel_idle_freeze, }, 3970138d8f0SLen Brown { 398de09cdd0SLen Brown .name = "C3", 3990138d8f0SLen Brown .desc = "MWAIT 0x10", 400b82b6ccaSDaniel Lezcano .flags = MWAIT2flg(0x10) | CPUIDLE_FLAG_TLB_FLUSHED, 4010138d8f0SLen Brown .exit_latency = 59, 4020138d8f0SLen Brown .target_residency = 300, 4035fe2e527SRafael J. Wysocki .enter = &intel_idle, 4045fe2e527SRafael J. Wysocki .enter_freeze = intel_idle_freeze, }, 4050138d8f0SLen Brown { 406de09cdd0SLen Brown .name = "C6", 4070138d8f0SLen Brown .desc = "MWAIT 0x20", 408b82b6ccaSDaniel Lezcano .flags = MWAIT2flg(0x20) | CPUIDLE_FLAG_TLB_FLUSHED, 4090138d8f0SLen Brown .exit_latency = 84, 4100138d8f0SLen Brown .target_residency = 400, 4115fe2e527SRafael J. Wysocki .enter = &intel_idle, 4125fe2e527SRafael J. Wysocki .enter_freeze = intel_idle_freeze, }, 4130138d8f0SLen Brown { 4140138d8f0SLen Brown .enter = NULL } 4150138d8f0SLen Brown }; 4160138d8f0SLen Brown 4170138d8f0SLen Brown static struct cpuidle_state ivt_cstates_8s[] = { 4180138d8f0SLen Brown { 419de09cdd0SLen Brown .name = "C1", 4200138d8f0SLen Brown .desc = "MWAIT 0x00", 421b82b6ccaSDaniel Lezcano .flags = MWAIT2flg(0x00), 4220138d8f0SLen Brown .exit_latency = 1, 4230138d8f0SLen Brown .target_residency = 1, 4245fe2e527SRafael J. Wysocki .enter = &intel_idle, 4255fe2e527SRafael J. Wysocki .enter_freeze = intel_idle_freeze, }, 4260138d8f0SLen Brown { 427de09cdd0SLen Brown .name = "C1E", 4280138d8f0SLen Brown .desc = "MWAIT 0x01", 429b82b6ccaSDaniel Lezcano .flags = MWAIT2flg(0x01), 4300138d8f0SLen Brown .exit_latency = 10, 4310138d8f0SLen Brown .target_residency = 500, 4325fe2e527SRafael J. Wysocki .enter = &intel_idle, 4335fe2e527SRafael J. Wysocki .enter_freeze = intel_idle_freeze, }, 4340138d8f0SLen Brown { 435de09cdd0SLen Brown .name = "C3", 4360138d8f0SLen Brown .desc = "MWAIT 0x10", 437b82b6ccaSDaniel Lezcano .flags = MWAIT2flg(0x10) | CPUIDLE_FLAG_TLB_FLUSHED, 4380138d8f0SLen Brown .exit_latency = 59, 4390138d8f0SLen Brown .target_residency = 600, 4405fe2e527SRafael J. Wysocki .enter = &intel_idle, 4415fe2e527SRafael J. Wysocki .enter_freeze = intel_idle_freeze, }, 4420138d8f0SLen Brown { 443de09cdd0SLen Brown .name = "C6", 4440138d8f0SLen Brown .desc = "MWAIT 0x20", 445b82b6ccaSDaniel Lezcano .flags = MWAIT2flg(0x20) | CPUIDLE_FLAG_TLB_FLUSHED, 4460138d8f0SLen Brown .exit_latency = 88, 4470138d8f0SLen Brown .target_residency = 700, 4485fe2e527SRafael J. Wysocki .enter = &intel_idle, 4495fe2e527SRafael J. Wysocki .enter_freeze = intel_idle_freeze, }, 4500138d8f0SLen Brown { 4510138d8f0SLen Brown .enter = NULL } 4520138d8f0SLen Brown }; 4530138d8f0SLen Brown 454ba0dc81eSJiang Liu static struct cpuidle_state hsw_cstates[] = { 455e022e7ebSLen Brown { 456de09cdd0SLen Brown .name = "C1", 45785a4d2d4SLen Brown .desc = "MWAIT 0x00", 458b82b6ccaSDaniel Lezcano .flags = MWAIT2flg(0x00), 45985a4d2d4SLen Brown .exit_latency = 2, 46085a4d2d4SLen Brown .target_residency = 2, 4615fe2e527SRafael J. Wysocki .enter = &intel_idle, 4625fe2e527SRafael J. Wysocki .enter_freeze = intel_idle_freeze, }, 463e022e7ebSLen Brown { 464de09cdd0SLen Brown .name = "C1E", 46532e95180SLen Brown .desc = "MWAIT 0x01", 466b82b6ccaSDaniel Lezcano .flags = MWAIT2flg(0x01), 46732e95180SLen Brown .exit_latency = 10, 46832e95180SLen Brown .target_residency = 20, 4695fe2e527SRafael J. Wysocki .enter = &intel_idle, 4705fe2e527SRafael J. Wysocki .enter_freeze = intel_idle_freeze, }, 47132e95180SLen Brown { 472de09cdd0SLen Brown .name = "C3", 47385a4d2d4SLen Brown .desc = "MWAIT 0x10", 474b82b6ccaSDaniel Lezcano .flags = MWAIT2flg(0x10) | CPUIDLE_FLAG_TLB_FLUSHED, 47585a4d2d4SLen Brown .exit_latency = 33, 47685a4d2d4SLen Brown .target_residency = 100, 4775fe2e527SRafael J. Wysocki .enter = &intel_idle, 4785fe2e527SRafael J. Wysocki .enter_freeze = intel_idle_freeze, }, 479e022e7ebSLen Brown { 480de09cdd0SLen Brown .name = "C6", 48185a4d2d4SLen Brown .desc = "MWAIT 0x20", 482b82b6ccaSDaniel Lezcano .flags = MWAIT2flg(0x20) | CPUIDLE_FLAG_TLB_FLUSHED, 48385a4d2d4SLen Brown .exit_latency = 133, 48485a4d2d4SLen Brown .target_residency = 400, 4855fe2e527SRafael J. Wysocki .enter = &intel_idle, 4865fe2e527SRafael J. Wysocki .enter_freeze = intel_idle_freeze, }, 487e022e7ebSLen Brown { 488de09cdd0SLen Brown .name = "C7s", 48985a4d2d4SLen Brown .desc = "MWAIT 0x32", 490b82b6ccaSDaniel Lezcano .flags = MWAIT2flg(0x32) | CPUIDLE_FLAG_TLB_FLUSHED, 49185a4d2d4SLen Brown .exit_latency = 166, 49285a4d2d4SLen Brown .target_residency = 500, 4935fe2e527SRafael J. Wysocki .enter = &intel_idle, 4945fe2e527SRafael J. Wysocki .enter_freeze = intel_idle_freeze, }, 495e022e7ebSLen Brown { 496de09cdd0SLen Brown .name = "C8", 49786239cebSLen Brown .desc = "MWAIT 0x40", 498b82b6ccaSDaniel Lezcano .flags = MWAIT2flg(0x40) | CPUIDLE_FLAG_TLB_FLUSHED, 49986239cebSLen Brown .exit_latency = 300, 50086239cebSLen Brown .target_residency = 900, 5015fe2e527SRafael J. Wysocki .enter = &intel_idle, 5025fe2e527SRafael J. Wysocki .enter_freeze = intel_idle_freeze, }, 50386239cebSLen Brown { 504de09cdd0SLen Brown .name = "C9", 50586239cebSLen Brown .desc = "MWAIT 0x50", 506b82b6ccaSDaniel Lezcano .flags = MWAIT2flg(0x50) | CPUIDLE_FLAG_TLB_FLUSHED, 50786239cebSLen Brown .exit_latency = 600, 50886239cebSLen Brown .target_residency = 1800, 5095fe2e527SRafael J. Wysocki .enter = &intel_idle, 5105fe2e527SRafael J. Wysocki .enter_freeze = intel_idle_freeze, }, 51186239cebSLen Brown { 512de09cdd0SLen Brown .name = "C10", 51386239cebSLen Brown .desc = "MWAIT 0x60", 514b82b6ccaSDaniel Lezcano .flags = MWAIT2flg(0x60) | CPUIDLE_FLAG_TLB_FLUSHED, 51586239cebSLen Brown .exit_latency = 2600, 51686239cebSLen Brown .target_residency = 7700, 5175fe2e527SRafael J. Wysocki .enter = &intel_idle, 5185fe2e527SRafael J. Wysocki .enter_freeze = intel_idle_freeze, }, 51986239cebSLen Brown { 520e022e7ebSLen Brown .enter = NULL } 52185a4d2d4SLen Brown }; 522a138b568SLen Brown static struct cpuidle_state bdw_cstates[] = { 523a138b568SLen Brown { 524de09cdd0SLen Brown .name = "C1", 525a138b568SLen Brown .desc = "MWAIT 0x00", 526b82b6ccaSDaniel Lezcano .flags = MWAIT2flg(0x00), 527a138b568SLen Brown .exit_latency = 2, 528a138b568SLen Brown .target_residency = 2, 5295fe2e527SRafael J. Wysocki .enter = &intel_idle, 5305fe2e527SRafael J. Wysocki .enter_freeze = intel_idle_freeze, }, 531a138b568SLen Brown { 532de09cdd0SLen Brown .name = "C1E", 533a138b568SLen Brown .desc = "MWAIT 0x01", 534b82b6ccaSDaniel Lezcano .flags = MWAIT2flg(0x01), 535a138b568SLen Brown .exit_latency = 10, 536a138b568SLen Brown .target_residency = 20, 5375fe2e527SRafael J. Wysocki .enter = &intel_idle, 5385fe2e527SRafael J. Wysocki .enter_freeze = intel_idle_freeze, }, 539a138b568SLen Brown { 540de09cdd0SLen Brown .name = "C3", 541a138b568SLen Brown .desc = "MWAIT 0x10", 542b82b6ccaSDaniel Lezcano .flags = MWAIT2flg(0x10) | CPUIDLE_FLAG_TLB_FLUSHED, 543a138b568SLen Brown .exit_latency = 40, 544a138b568SLen Brown .target_residency = 100, 5455fe2e527SRafael J. Wysocki .enter = &intel_idle, 5465fe2e527SRafael J. Wysocki .enter_freeze = intel_idle_freeze, }, 547a138b568SLen Brown { 548de09cdd0SLen Brown .name = "C6", 549a138b568SLen Brown .desc = "MWAIT 0x20", 550b82b6ccaSDaniel Lezcano .flags = MWAIT2flg(0x20) | CPUIDLE_FLAG_TLB_FLUSHED, 551a138b568SLen Brown .exit_latency = 133, 552a138b568SLen Brown .target_residency = 400, 5535fe2e527SRafael J. Wysocki .enter = &intel_idle, 5545fe2e527SRafael J. Wysocki .enter_freeze = intel_idle_freeze, }, 555a138b568SLen Brown { 556de09cdd0SLen Brown .name = "C7s", 557a138b568SLen Brown .desc = "MWAIT 0x32", 558b82b6ccaSDaniel Lezcano .flags = MWAIT2flg(0x32) | CPUIDLE_FLAG_TLB_FLUSHED, 559a138b568SLen Brown .exit_latency = 166, 560a138b568SLen Brown .target_residency = 500, 5615fe2e527SRafael J. Wysocki .enter = &intel_idle, 5625fe2e527SRafael J. Wysocki .enter_freeze = intel_idle_freeze, }, 563a138b568SLen Brown { 564de09cdd0SLen Brown .name = "C8", 565a138b568SLen Brown .desc = "MWAIT 0x40", 566b82b6ccaSDaniel Lezcano .flags = MWAIT2flg(0x40) | CPUIDLE_FLAG_TLB_FLUSHED, 567a138b568SLen Brown .exit_latency = 300, 568a138b568SLen Brown .target_residency = 900, 5695fe2e527SRafael J. Wysocki .enter = &intel_idle, 5705fe2e527SRafael J. Wysocki .enter_freeze = intel_idle_freeze, }, 571a138b568SLen Brown { 572de09cdd0SLen Brown .name = "C9", 573a138b568SLen Brown .desc = "MWAIT 0x50", 574b82b6ccaSDaniel Lezcano .flags = MWAIT2flg(0x50) | CPUIDLE_FLAG_TLB_FLUSHED, 575a138b568SLen Brown .exit_latency = 600, 576a138b568SLen Brown .target_residency = 1800, 5775fe2e527SRafael J. Wysocki .enter = &intel_idle, 5785fe2e527SRafael J. Wysocki .enter_freeze = intel_idle_freeze, }, 579a138b568SLen Brown { 580de09cdd0SLen Brown .name = "C10", 581a138b568SLen Brown .desc = "MWAIT 0x60", 582b82b6ccaSDaniel Lezcano .flags = MWAIT2flg(0x60) | CPUIDLE_FLAG_TLB_FLUSHED, 583a138b568SLen Brown .exit_latency = 2600, 584a138b568SLen Brown .target_residency = 7700, 5855fe2e527SRafael J. Wysocki .enter = &intel_idle, 5865fe2e527SRafael J. Wysocki .enter_freeze = intel_idle_freeze, }, 587a138b568SLen Brown { 588a138b568SLen Brown .enter = NULL } 589a138b568SLen Brown }; 59085a4d2d4SLen Brown 591493f133fSLen Brown static struct cpuidle_state skl_cstates[] = { 592493f133fSLen Brown { 593de09cdd0SLen Brown .name = "C1", 594493f133fSLen Brown .desc = "MWAIT 0x00", 595493f133fSLen Brown .flags = MWAIT2flg(0x00), 596493f133fSLen Brown .exit_latency = 2, 597493f133fSLen Brown .target_residency = 2, 598493f133fSLen Brown .enter = &intel_idle, 599493f133fSLen Brown .enter_freeze = intel_idle_freeze, }, 600493f133fSLen Brown { 601de09cdd0SLen Brown .name = "C1E", 602493f133fSLen Brown .desc = "MWAIT 0x01", 603493f133fSLen Brown .flags = MWAIT2flg(0x01), 604493f133fSLen Brown .exit_latency = 10, 605493f133fSLen Brown .target_residency = 20, 606493f133fSLen Brown .enter = &intel_idle, 607493f133fSLen Brown .enter_freeze = intel_idle_freeze, }, 608493f133fSLen Brown { 609de09cdd0SLen Brown .name = "C3", 610493f133fSLen Brown .desc = "MWAIT 0x10", 611493f133fSLen Brown .flags = MWAIT2flg(0x10) | CPUIDLE_FLAG_TLB_FLUSHED, 612493f133fSLen Brown .exit_latency = 70, 613493f133fSLen Brown .target_residency = 100, 614493f133fSLen Brown .enter = &intel_idle, 615493f133fSLen Brown .enter_freeze = intel_idle_freeze, }, 616493f133fSLen Brown { 617de09cdd0SLen Brown .name = "C6", 618493f133fSLen Brown .desc = "MWAIT 0x20", 619493f133fSLen Brown .flags = MWAIT2flg(0x20) | CPUIDLE_FLAG_TLB_FLUSHED, 620135919a3SLen Brown .exit_latency = 85, 621493f133fSLen Brown .target_residency = 200, 622493f133fSLen Brown .enter = &intel_idle, 623493f133fSLen Brown .enter_freeze = intel_idle_freeze, }, 624493f133fSLen Brown { 625de09cdd0SLen Brown .name = "C7s", 626493f133fSLen Brown .desc = "MWAIT 0x33", 627493f133fSLen Brown .flags = MWAIT2flg(0x33) | CPUIDLE_FLAG_TLB_FLUSHED, 628493f133fSLen Brown .exit_latency = 124, 629493f133fSLen Brown .target_residency = 800, 630493f133fSLen Brown .enter = &intel_idle, 631493f133fSLen Brown .enter_freeze = intel_idle_freeze, }, 632493f133fSLen Brown { 633de09cdd0SLen Brown .name = "C8", 634493f133fSLen Brown .desc = "MWAIT 0x40", 635493f133fSLen Brown .flags = MWAIT2flg(0x40) | CPUIDLE_FLAG_TLB_FLUSHED, 636135919a3SLen Brown .exit_latency = 200, 637493f133fSLen Brown .target_residency = 800, 638493f133fSLen Brown .enter = &intel_idle, 639493f133fSLen Brown .enter_freeze = intel_idle_freeze, }, 640493f133fSLen Brown { 641de09cdd0SLen Brown .name = "C9", 642135919a3SLen Brown .desc = "MWAIT 0x50", 643135919a3SLen Brown .flags = MWAIT2flg(0x50) | CPUIDLE_FLAG_TLB_FLUSHED, 644135919a3SLen Brown .exit_latency = 480, 645135919a3SLen Brown .target_residency = 5000, 646135919a3SLen Brown .enter = &intel_idle, 647135919a3SLen Brown .enter_freeze = intel_idle_freeze, }, 648135919a3SLen Brown { 649de09cdd0SLen Brown .name = "C10", 650493f133fSLen Brown .desc = "MWAIT 0x60", 651493f133fSLen Brown .flags = MWAIT2flg(0x60) | CPUIDLE_FLAG_TLB_FLUSHED, 652493f133fSLen Brown .exit_latency = 890, 653493f133fSLen Brown .target_residency = 5000, 654493f133fSLen Brown .enter = &intel_idle, 655493f133fSLen Brown .enter_freeze = intel_idle_freeze, }, 656493f133fSLen Brown { 657493f133fSLen Brown .enter = NULL } 658493f133fSLen Brown }; 659493f133fSLen Brown 660f9e71657SLen Brown static struct cpuidle_state skx_cstates[] = { 661f9e71657SLen Brown { 662de09cdd0SLen Brown .name = "C1", 663f9e71657SLen Brown .desc = "MWAIT 0x00", 664f9e71657SLen Brown .flags = MWAIT2flg(0x00), 665f9e71657SLen Brown .exit_latency = 2, 666f9e71657SLen Brown .target_residency = 2, 667f9e71657SLen Brown .enter = &intel_idle, 668f9e71657SLen Brown .enter_freeze = intel_idle_freeze, }, 669f9e71657SLen Brown { 670de09cdd0SLen Brown .name = "C1E", 671f9e71657SLen Brown .desc = "MWAIT 0x01", 672f9e71657SLen Brown .flags = MWAIT2flg(0x01), 673f9e71657SLen Brown .exit_latency = 10, 674f9e71657SLen Brown .target_residency = 20, 675f9e71657SLen Brown .enter = &intel_idle, 676f9e71657SLen Brown .enter_freeze = intel_idle_freeze, }, 677f9e71657SLen Brown { 678de09cdd0SLen Brown .name = "C6", 679f9e71657SLen Brown .desc = "MWAIT 0x20", 680f9e71657SLen Brown .flags = MWAIT2flg(0x20) | CPUIDLE_FLAG_TLB_FLUSHED, 681f9e71657SLen Brown .exit_latency = 133, 682f9e71657SLen Brown .target_residency = 600, 683f9e71657SLen Brown .enter = &intel_idle, 684f9e71657SLen Brown .enter_freeze = intel_idle_freeze, }, 685f9e71657SLen Brown { 686f9e71657SLen Brown .enter = NULL } 687f9e71657SLen Brown }; 688f9e71657SLen Brown 689ba0dc81eSJiang Liu static struct cpuidle_state atom_cstates[] = { 690e022e7ebSLen Brown { 691de09cdd0SLen Brown .name = "C1E", 69226717172SLen Brown .desc = "MWAIT 0x00", 693b82b6ccaSDaniel Lezcano .flags = MWAIT2flg(0x00), 69432e95180SLen Brown .exit_latency = 10, 69532e95180SLen Brown .target_residency = 20, 6965fe2e527SRafael J. Wysocki .enter = &intel_idle, 6975fe2e527SRafael J. Wysocki .enter_freeze = intel_idle_freeze, }, 698e022e7ebSLen Brown { 699de09cdd0SLen Brown .name = "C2", 70026717172SLen Brown .desc = "MWAIT 0x10", 701b82b6ccaSDaniel Lezcano .flags = MWAIT2flg(0x10), 70226717172SLen Brown .exit_latency = 20, 70326717172SLen Brown .target_residency = 80, 7045fe2e527SRafael J. Wysocki .enter = &intel_idle, 7055fe2e527SRafael J. Wysocki .enter_freeze = intel_idle_freeze, }, 706e022e7ebSLen Brown { 707de09cdd0SLen Brown .name = "C4", 70826717172SLen Brown .desc = "MWAIT 0x30", 709b82b6ccaSDaniel Lezcano .flags = MWAIT2flg(0x30) | CPUIDLE_FLAG_TLB_FLUSHED, 71026717172SLen Brown .exit_latency = 100, 71126717172SLen Brown .target_residency = 400, 7125fe2e527SRafael J. Wysocki .enter = &intel_idle, 7135fe2e527SRafael J. Wysocki .enter_freeze = intel_idle_freeze, }, 714e022e7ebSLen Brown { 715de09cdd0SLen Brown .name = "C6", 7167fcca7d9SLen Brown .desc = "MWAIT 0x52", 717b82b6ccaSDaniel Lezcano .flags = MWAIT2flg(0x52) | CPUIDLE_FLAG_TLB_FLUSHED, 7187fcca7d9SLen Brown .exit_latency = 140, 7197fcca7d9SLen Brown .target_residency = 560, 7205fe2e527SRafael J. Wysocki .enter = &intel_idle, 7215fe2e527SRafael J. Wysocki .enter_freeze = intel_idle_freeze, }, 722e022e7ebSLen Brown { 723e022e7ebSLen Brown .enter = NULL } 72426717172SLen Brown }; 7255e7ec268SAndy Shevchenko static struct cpuidle_state tangier_cstates[] = { 7265e7ec268SAndy Shevchenko { 727de09cdd0SLen Brown .name = "C1", 7285e7ec268SAndy Shevchenko .desc = "MWAIT 0x00", 7295e7ec268SAndy Shevchenko .flags = MWAIT2flg(0x00), 7305e7ec268SAndy Shevchenko .exit_latency = 1, 7315e7ec268SAndy Shevchenko .target_residency = 4, 7325e7ec268SAndy Shevchenko .enter = &intel_idle, 7335e7ec268SAndy Shevchenko .enter_freeze = intel_idle_freeze, }, 7345e7ec268SAndy Shevchenko { 735de09cdd0SLen Brown .name = "C4", 7365e7ec268SAndy Shevchenko .desc = "MWAIT 0x30", 7375e7ec268SAndy Shevchenko .flags = MWAIT2flg(0x30) | CPUIDLE_FLAG_TLB_FLUSHED, 7385e7ec268SAndy Shevchenko .exit_latency = 100, 7395e7ec268SAndy Shevchenko .target_residency = 400, 7405e7ec268SAndy Shevchenko .enter = &intel_idle, 7415e7ec268SAndy Shevchenko .enter_freeze = intel_idle_freeze, }, 7425e7ec268SAndy Shevchenko { 743de09cdd0SLen Brown .name = "C6", 7445e7ec268SAndy Shevchenko .desc = "MWAIT 0x52", 7455e7ec268SAndy Shevchenko .flags = MWAIT2flg(0x52) | CPUIDLE_FLAG_TLB_FLUSHED, 7465e7ec268SAndy Shevchenko .exit_latency = 140, 7475e7ec268SAndy Shevchenko .target_residency = 560, 7485e7ec268SAndy Shevchenko .enter = &intel_idle, 7495e7ec268SAndy Shevchenko .enter_freeze = intel_idle_freeze, }, 7505e7ec268SAndy Shevchenko { 751de09cdd0SLen Brown .name = "C7", 7525e7ec268SAndy Shevchenko .desc = "MWAIT 0x60", 7535e7ec268SAndy Shevchenko .flags = MWAIT2flg(0x60) | CPUIDLE_FLAG_TLB_FLUSHED, 7545e7ec268SAndy Shevchenko .exit_latency = 1200, 7555e7ec268SAndy Shevchenko .target_residency = 4000, 7565e7ec268SAndy Shevchenko .enter = &intel_idle, 7575e7ec268SAndy Shevchenko .enter_freeze = intel_idle_freeze, }, 7585e7ec268SAndy Shevchenko { 759de09cdd0SLen Brown .name = "C9", 7605e7ec268SAndy Shevchenko .desc = "MWAIT 0x64", 7615e7ec268SAndy Shevchenko .flags = MWAIT2flg(0x64) | CPUIDLE_FLAG_TLB_FLUSHED, 7625e7ec268SAndy Shevchenko .exit_latency = 10000, 7635e7ec268SAndy Shevchenko .target_residency = 20000, 7645e7ec268SAndy Shevchenko .enter = &intel_idle, 7655e7ec268SAndy Shevchenko .enter_freeze = intel_idle_freeze, }, 7665e7ec268SAndy Shevchenko { 7675e7ec268SAndy Shevchenko .enter = NULL } 7685e7ec268SAndy Shevchenko }; 76988390996SJiang Liu static struct cpuidle_state avn_cstates[] = { 770fab04b22SLen Brown { 771de09cdd0SLen Brown .name = "C1", 772fab04b22SLen Brown .desc = "MWAIT 0x00", 773b82b6ccaSDaniel Lezcano .flags = MWAIT2flg(0x00), 774fab04b22SLen Brown .exit_latency = 2, 775fab04b22SLen Brown .target_residency = 2, 7765fe2e527SRafael J. Wysocki .enter = &intel_idle, 7775fe2e527SRafael J. Wysocki .enter_freeze = intel_idle_freeze, }, 778fab04b22SLen Brown { 779de09cdd0SLen Brown .name = "C6", 780fab04b22SLen Brown .desc = "MWAIT 0x51", 781b82b6ccaSDaniel Lezcano .flags = MWAIT2flg(0x51) | CPUIDLE_FLAG_TLB_FLUSHED, 782fab04b22SLen Brown .exit_latency = 15, 783fab04b22SLen Brown .target_residency = 45, 7845fe2e527SRafael J. Wysocki .enter = &intel_idle, 7855fe2e527SRafael J. Wysocki .enter_freeze = intel_idle_freeze, }, 78688390996SJiang Liu { 78788390996SJiang Liu .enter = NULL } 788fab04b22SLen Brown }; 789281baf7aSDasaratharaman Chandramouli static struct cpuidle_state knl_cstates[] = { 790281baf7aSDasaratharaman Chandramouli { 791de09cdd0SLen Brown .name = "C1", 792281baf7aSDasaratharaman Chandramouli .desc = "MWAIT 0x00", 793281baf7aSDasaratharaman Chandramouli .flags = MWAIT2flg(0x00), 794281baf7aSDasaratharaman Chandramouli .exit_latency = 1, 795281baf7aSDasaratharaman Chandramouli .target_residency = 2, 796281baf7aSDasaratharaman Chandramouli .enter = &intel_idle, 797281baf7aSDasaratharaman Chandramouli .enter_freeze = intel_idle_freeze }, 798281baf7aSDasaratharaman Chandramouli { 799de09cdd0SLen Brown .name = "C6", 800281baf7aSDasaratharaman Chandramouli .desc = "MWAIT 0x10", 801281baf7aSDasaratharaman Chandramouli .flags = MWAIT2flg(0x10) | CPUIDLE_FLAG_TLB_FLUSHED, 802281baf7aSDasaratharaman Chandramouli .exit_latency = 120, 803281baf7aSDasaratharaman Chandramouli .target_residency = 500, 804281baf7aSDasaratharaman Chandramouli .enter = &intel_idle, 805281baf7aSDasaratharaman Chandramouli .enter_freeze = intel_idle_freeze }, 806281baf7aSDasaratharaman Chandramouli { 807281baf7aSDasaratharaman Chandramouli .enter = NULL } 808281baf7aSDasaratharaman Chandramouli }; 80926717172SLen Brown 8105dcef694SLen Brown static struct cpuidle_state bxt_cstates[] = { 8115dcef694SLen Brown { 812de09cdd0SLen Brown .name = "C1", 8135dcef694SLen Brown .desc = "MWAIT 0x00", 8145dcef694SLen Brown .flags = MWAIT2flg(0x00), 8155dcef694SLen Brown .exit_latency = 2, 8165dcef694SLen Brown .target_residency = 2, 8175dcef694SLen Brown .enter = &intel_idle, 8185dcef694SLen Brown .enter_freeze = intel_idle_freeze, }, 8195dcef694SLen Brown { 820de09cdd0SLen Brown .name = "C1E", 8215dcef694SLen Brown .desc = "MWAIT 0x01", 8225dcef694SLen Brown .flags = MWAIT2flg(0x01), 8235dcef694SLen Brown .exit_latency = 10, 8245dcef694SLen Brown .target_residency = 20, 8255dcef694SLen Brown .enter = &intel_idle, 8265dcef694SLen Brown .enter_freeze = intel_idle_freeze, }, 8275dcef694SLen Brown { 828de09cdd0SLen Brown .name = "C6", 8295dcef694SLen Brown .desc = "MWAIT 0x20", 8305dcef694SLen Brown .flags = MWAIT2flg(0x20) | CPUIDLE_FLAG_TLB_FLUSHED, 8315dcef694SLen Brown .exit_latency = 133, 8325dcef694SLen Brown .target_residency = 133, 8335dcef694SLen Brown .enter = &intel_idle, 8345dcef694SLen Brown .enter_freeze = intel_idle_freeze, }, 8355dcef694SLen Brown { 836de09cdd0SLen Brown .name = "C7s", 8375dcef694SLen Brown .desc = "MWAIT 0x31", 8385dcef694SLen Brown .flags = MWAIT2flg(0x31) | CPUIDLE_FLAG_TLB_FLUSHED, 8395dcef694SLen Brown .exit_latency = 155, 8405dcef694SLen Brown .target_residency = 155, 8415dcef694SLen Brown .enter = &intel_idle, 8425dcef694SLen Brown .enter_freeze = intel_idle_freeze, }, 8435dcef694SLen Brown { 844de09cdd0SLen Brown .name = "C8", 8455dcef694SLen Brown .desc = "MWAIT 0x40", 8465dcef694SLen Brown .flags = MWAIT2flg(0x40) | CPUIDLE_FLAG_TLB_FLUSHED, 8475dcef694SLen Brown .exit_latency = 1000, 8485dcef694SLen Brown .target_residency = 1000, 8495dcef694SLen Brown .enter = &intel_idle, 8505dcef694SLen Brown .enter_freeze = intel_idle_freeze, }, 8515dcef694SLen Brown { 852de09cdd0SLen Brown .name = "C9", 8535dcef694SLen Brown .desc = "MWAIT 0x50", 8545dcef694SLen Brown .flags = MWAIT2flg(0x50) | CPUIDLE_FLAG_TLB_FLUSHED, 8555dcef694SLen Brown .exit_latency = 2000, 8565dcef694SLen Brown .target_residency = 2000, 8575dcef694SLen Brown .enter = &intel_idle, 8585dcef694SLen Brown .enter_freeze = intel_idle_freeze, }, 8595dcef694SLen Brown { 860de09cdd0SLen Brown .name = "C10", 8615dcef694SLen Brown .desc = "MWAIT 0x60", 8625dcef694SLen Brown .flags = MWAIT2flg(0x60) | CPUIDLE_FLAG_TLB_FLUSHED, 8635dcef694SLen Brown .exit_latency = 10000, 8645dcef694SLen Brown .target_residency = 10000, 8655dcef694SLen Brown .enter = &intel_idle, 8665dcef694SLen Brown .enter_freeze = intel_idle_freeze, }, 8675dcef694SLen Brown { 8685dcef694SLen Brown .enter = NULL } 8695dcef694SLen Brown }; 8705dcef694SLen Brown 8710080d65bSJacob Pan static struct cpuidle_state dnv_cstates[] = { 8720080d65bSJacob Pan { 873de09cdd0SLen Brown .name = "C1", 8740080d65bSJacob Pan .desc = "MWAIT 0x00", 8750080d65bSJacob Pan .flags = MWAIT2flg(0x00), 8760080d65bSJacob Pan .exit_latency = 2, 8770080d65bSJacob Pan .target_residency = 2, 8780080d65bSJacob Pan .enter = &intel_idle, 8790080d65bSJacob Pan .enter_freeze = intel_idle_freeze, }, 8800080d65bSJacob Pan { 881de09cdd0SLen Brown .name = "C1E", 8820080d65bSJacob Pan .desc = "MWAIT 0x01", 8830080d65bSJacob Pan .flags = MWAIT2flg(0x01), 8840080d65bSJacob Pan .exit_latency = 10, 8850080d65bSJacob Pan .target_residency = 20, 8860080d65bSJacob Pan .enter = &intel_idle, 8870080d65bSJacob Pan .enter_freeze = intel_idle_freeze, }, 8880080d65bSJacob Pan { 889de09cdd0SLen Brown .name = "C6", 8900080d65bSJacob Pan .desc = "MWAIT 0x20", 8910080d65bSJacob Pan .flags = MWAIT2flg(0x20) | CPUIDLE_FLAG_TLB_FLUSHED, 8920080d65bSJacob Pan .exit_latency = 50, 8930080d65bSJacob Pan .target_residency = 500, 8940080d65bSJacob Pan .enter = &intel_idle, 8950080d65bSJacob Pan .enter_freeze = intel_idle_freeze, }, 8960080d65bSJacob Pan { 8970080d65bSJacob Pan .enter = NULL } 8980080d65bSJacob Pan }; 8990080d65bSJacob Pan 90026717172SLen Brown /** 90126717172SLen Brown * intel_idle 90226717172SLen Brown * @dev: cpuidle_device 90346bcfad7SDeepthi Dharwar * @drv: cpuidle driver 904e978aa7dSDeepthi Dharwar * @index: index of cpuidle state 90526717172SLen Brown * 90663ff07beSYanmin Zhang * Must be called under local_irq_disable(). 90726717172SLen Brown */ 9086727ad9eSChris Metcalf static __cpuidle int intel_idle(struct cpuidle_device *dev, 90946bcfad7SDeepthi Dharwar struct cpuidle_driver *drv, int index) 91026717172SLen Brown { 91126717172SLen Brown unsigned long ecx = 1; /* break on interrupt flag */ 91246bcfad7SDeepthi Dharwar struct cpuidle_state *state = &drv->states[index]; 913b1beab48SLen Brown unsigned long eax = flg2MWAIT(state->flags); 91426717172SLen Brown unsigned int cstate; 91526717172SLen Brown int cpu = smp_processor_id(); 91626717172SLen Brown 91726717172SLen Brown cstate = (((eax) >> MWAIT_SUBSTATE_SIZE) & MWAIT_CSTATE_MASK) + 1; 91826717172SLen Brown 9196110a1f4SSuresh Siddha /* 920c8381cc3SLen Brown * leave_mm() to avoid costly and often unnecessary wakeups 921c8381cc3SLen Brown * for flushing the user TLB's associated with the active mm. 9226110a1f4SSuresh Siddha */ 923c8381cc3SLen Brown if (state->flags & CPUIDLE_FLAG_TLB_FLUSHED) 9246110a1f4SSuresh Siddha leave_mm(cpu); 9256110a1f4SSuresh Siddha 92626717172SLen Brown if (!(lapic_timer_reliable_states & (1 << (cstate)))) 927f6cee191SThomas Gleixner tick_broadcast_enter(); 92826717172SLen Brown 92916824255SPeter Zijlstra mwait_idle_with_hints(eax, ecx); 93026717172SLen Brown 93126717172SLen Brown if (!(lapic_timer_reliable_states & (1 << (cstate)))) 932f6cee191SThomas Gleixner tick_broadcast_exit(); 93326717172SLen Brown 934e978aa7dSDeepthi Dharwar return index; 93526717172SLen Brown } 93626717172SLen Brown 9375fe2e527SRafael J. Wysocki /** 9385fe2e527SRafael J. Wysocki * intel_idle_freeze - simplified "enter" callback routine for suspend-to-idle 9395fe2e527SRafael J. Wysocki * @dev: cpuidle_device 9405fe2e527SRafael J. Wysocki * @drv: cpuidle driver 9415fe2e527SRafael J. Wysocki * @index: state index 9425fe2e527SRafael J. Wysocki */ 9435fe2e527SRafael J. Wysocki static void intel_idle_freeze(struct cpuidle_device *dev, 9445fe2e527SRafael J. Wysocki struct cpuidle_driver *drv, int index) 9455fe2e527SRafael J. Wysocki { 9465fe2e527SRafael J. Wysocki unsigned long ecx = 1; /* break on interrupt flag */ 9475fe2e527SRafael J. Wysocki unsigned long eax = flg2MWAIT(drv->states[index].flags); 9485fe2e527SRafael J. Wysocki 9495fe2e527SRafael J. Wysocki mwait_idle_with_hints(eax, ecx); 9505fe2e527SRafael J. Wysocki } 9515fe2e527SRafael J. Wysocki 952fb1013a0SSebastian Andrzej Siewior static void __setup_broadcast_timer(bool on) 9532a2d31c8SShaohua Li { 95476962caaSThomas Gleixner if (on) 95576962caaSThomas Gleixner tick_broadcast_enable(); 95676962caaSThomas Gleixner else 95776962caaSThomas Gleixner tick_broadcast_disable(); 9582a2d31c8SShaohua Li } 9592a2d31c8SShaohua Li 960fb1013a0SSebastian Andrzej Siewior static void auto_demotion_disable(void) 96114796fcaSLen Brown { 96214796fcaSLen Brown unsigned long long msr_bits; 96314796fcaSLen Brown 96414796fcaSLen Brown rdmsrl(MSR_NHM_SNB_PKG_CST_CFG_CTL, msr_bits); 965b66b8b9aSAndi Kleen msr_bits &= ~(icpu->auto_demotion_disable_flags); 96614796fcaSLen Brown wrmsrl(MSR_NHM_SNB_PKG_CST_CFG_CTL, msr_bits); 96714796fcaSLen Brown } 968fb1013a0SSebastian Andrzej Siewior static void c1e_promotion_disable(void) 96932e95180SLen Brown { 97032e95180SLen Brown unsigned long long msr_bits; 97132e95180SLen Brown 97232e95180SLen Brown rdmsrl(MSR_IA32_POWER_CTL, msr_bits); 97332e95180SLen Brown msr_bits &= ~0x2; 97432e95180SLen Brown wrmsrl(MSR_IA32_POWER_CTL, msr_bits); 97532e95180SLen Brown } 97614796fcaSLen Brown 977b66b8b9aSAndi Kleen static const struct idle_cpu idle_cpu_nehalem = { 978b66b8b9aSAndi Kleen .state_table = nehalem_cstates, 979b66b8b9aSAndi Kleen .auto_demotion_disable_flags = NHM_C1_AUTO_DEMOTE | NHM_C3_AUTO_DEMOTE, 98032e95180SLen Brown .disable_promotion_to_c1e = true, 981b66b8b9aSAndi Kleen }; 982b66b8b9aSAndi Kleen 983b66b8b9aSAndi Kleen static const struct idle_cpu idle_cpu_atom = { 984b66b8b9aSAndi Kleen .state_table = atom_cstates, 985b66b8b9aSAndi Kleen }; 986b66b8b9aSAndi Kleen 9875e7ec268SAndy Shevchenko static const struct idle_cpu idle_cpu_tangier = { 9885e7ec268SAndy Shevchenko .state_table = tangier_cstates, 9895e7ec268SAndy Shevchenko }; 9905e7ec268SAndy Shevchenko 991b66b8b9aSAndi Kleen static const struct idle_cpu idle_cpu_lincroft = { 992b66b8b9aSAndi Kleen .state_table = atom_cstates, 993b66b8b9aSAndi Kleen .auto_demotion_disable_flags = ATM_LNC_C6_AUTO_DEMOTE, 994b66b8b9aSAndi Kleen }; 995b66b8b9aSAndi Kleen 996b66b8b9aSAndi Kleen static const struct idle_cpu idle_cpu_snb = { 997b66b8b9aSAndi Kleen .state_table = snb_cstates, 99832e95180SLen Brown .disable_promotion_to_c1e = true, 999b66b8b9aSAndi Kleen }; 1000b66b8b9aSAndi Kleen 1001718987d6SLen Brown static const struct idle_cpu idle_cpu_byt = { 1002718987d6SLen Brown .state_table = byt_cstates, 1003718987d6SLen Brown .disable_promotion_to_c1e = true, 10048c058d53SLen Brown .byt_auto_demotion_disable_flag = true, 1005718987d6SLen Brown }; 1006718987d6SLen Brown 1007cab07a56SLen Brown static const struct idle_cpu idle_cpu_cht = { 1008cab07a56SLen Brown .state_table = cht_cstates, 1009cab07a56SLen Brown .disable_promotion_to_c1e = true, 1010cab07a56SLen Brown .byt_auto_demotion_disable_flag = true, 1011cab07a56SLen Brown }; 1012cab07a56SLen Brown 10136edab08cSLen Brown static const struct idle_cpu idle_cpu_ivb = { 10146edab08cSLen Brown .state_table = ivb_cstates, 101532e95180SLen Brown .disable_promotion_to_c1e = true, 10166edab08cSLen Brown }; 10176edab08cSLen Brown 10180138d8f0SLen Brown static const struct idle_cpu idle_cpu_ivt = { 10190138d8f0SLen Brown .state_table = ivt_cstates, 10200138d8f0SLen Brown .disable_promotion_to_c1e = true, 10210138d8f0SLen Brown }; 10220138d8f0SLen Brown 102385a4d2d4SLen Brown static const struct idle_cpu idle_cpu_hsw = { 102485a4d2d4SLen Brown .state_table = hsw_cstates, 102532e95180SLen Brown .disable_promotion_to_c1e = true, 102685a4d2d4SLen Brown }; 102785a4d2d4SLen Brown 1028a138b568SLen Brown static const struct idle_cpu idle_cpu_bdw = { 1029a138b568SLen Brown .state_table = bdw_cstates, 1030a138b568SLen Brown .disable_promotion_to_c1e = true, 1031a138b568SLen Brown }; 1032a138b568SLen Brown 1033493f133fSLen Brown static const struct idle_cpu idle_cpu_skl = { 1034493f133fSLen Brown .state_table = skl_cstates, 1035493f133fSLen Brown .disable_promotion_to_c1e = true, 1036493f133fSLen Brown }; 1037493f133fSLen Brown 1038f9e71657SLen Brown static const struct idle_cpu idle_cpu_skx = { 1039f9e71657SLen Brown .state_table = skx_cstates, 1040f9e71657SLen Brown .disable_promotion_to_c1e = true, 1041f9e71657SLen Brown }; 1042493f133fSLen Brown 1043fab04b22SLen Brown static const struct idle_cpu idle_cpu_avn = { 1044fab04b22SLen Brown .state_table = avn_cstates, 1045fab04b22SLen Brown .disable_promotion_to_c1e = true, 1046fab04b22SLen Brown }; 1047fab04b22SLen Brown 1048281baf7aSDasaratharaman Chandramouli static const struct idle_cpu idle_cpu_knl = { 1049281baf7aSDasaratharaman Chandramouli .state_table = knl_cstates, 1050281baf7aSDasaratharaman Chandramouli }; 1051281baf7aSDasaratharaman Chandramouli 10525dcef694SLen Brown static const struct idle_cpu idle_cpu_bxt = { 10535dcef694SLen Brown .state_table = bxt_cstates, 10545dcef694SLen Brown .disable_promotion_to_c1e = true, 10555dcef694SLen Brown }; 10565dcef694SLen Brown 10570080d65bSJacob Pan static const struct idle_cpu idle_cpu_dnv = { 10580080d65bSJacob Pan .state_table = dnv_cstates, 10590080d65bSJacob Pan .disable_promotion_to_c1e = true, 10600080d65bSJacob Pan }; 10610080d65bSJacob Pan 1062b66b8b9aSAndi Kleen #define ICPU(model, cpu) \ 1063b66b8b9aSAndi Kleen { X86_VENDOR_INTEL, 6, model, X86_FEATURE_MWAIT, (unsigned long)&cpu } 1064b66b8b9aSAndi Kleen 1065d5cdc3c4SMathias Krause static const struct x86_cpu_id intel_idle_ids[] __initconst = { 1066db73c5a8SDave Hansen ICPU(INTEL_FAM6_NEHALEM_EP, idle_cpu_nehalem), 1067db73c5a8SDave Hansen ICPU(INTEL_FAM6_NEHALEM, idle_cpu_nehalem), 10684b3b234fSDave Hansen ICPU(INTEL_FAM6_NEHALEM_G, idle_cpu_nehalem), 1069db73c5a8SDave Hansen ICPU(INTEL_FAM6_WESTMERE, idle_cpu_nehalem), 1070db73c5a8SDave Hansen ICPU(INTEL_FAM6_WESTMERE_EP, idle_cpu_nehalem), 1071db73c5a8SDave Hansen ICPU(INTEL_FAM6_NEHALEM_EX, idle_cpu_nehalem), 1072db73c5a8SDave Hansen ICPU(INTEL_FAM6_ATOM_PINEVIEW, idle_cpu_atom), 1073db73c5a8SDave Hansen ICPU(INTEL_FAM6_ATOM_LINCROFT, idle_cpu_lincroft), 1074db73c5a8SDave Hansen ICPU(INTEL_FAM6_WESTMERE_EX, idle_cpu_nehalem), 1075db73c5a8SDave Hansen ICPU(INTEL_FAM6_SANDYBRIDGE, idle_cpu_snb), 1076db73c5a8SDave Hansen ICPU(INTEL_FAM6_SANDYBRIDGE_X, idle_cpu_snb), 1077db73c5a8SDave Hansen ICPU(INTEL_FAM6_ATOM_CEDARVIEW, idle_cpu_atom), 1078db73c5a8SDave Hansen ICPU(INTEL_FAM6_ATOM_SILVERMONT1, idle_cpu_byt), 10795e7ec268SAndy Shevchenko ICPU(INTEL_FAM6_ATOM_MERRIFIELD, idle_cpu_tangier), 1080db73c5a8SDave Hansen ICPU(INTEL_FAM6_ATOM_AIRMONT, idle_cpu_cht), 1081db73c5a8SDave Hansen ICPU(INTEL_FAM6_IVYBRIDGE, idle_cpu_ivb), 1082db73c5a8SDave Hansen ICPU(INTEL_FAM6_IVYBRIDGE_X, idle_cpu_ivt), 1083db73c5a8SDave Hansen ICPU(INTEL_FAM6_HASWELL_CORE, idle_cpu_hsw), 1084db73c5a8SDave Hansen ICPU(INTEL_FAM6_HASWELL_X, idle_cpu_hsw), 1085db73c5a8SDave Hansen ICPU(INTEL_FAM6_HASWELL_ULT, idle_cpu_hsw), 1086db73c5a8SDave Hansen ICPU(INTEL_FAM6_HASWELL_GT3E, idle_cpu_hsw), 1087db73c5a8SDave Hansen ICPU(INTEL_FAM6_ATOM_SILVERMONT2, idle_cpu_avn), 1088db73c5a8SDave Hansen ICPU(INTEL_FAM6_BROADWELL_CORE, idle_cpu_bdw), 1089db73c5a8SDave Hansen ICPU(INTEL_FAM6_BROADWELL_GT3E, idle_cpu_bdw), 1090db73c5a8SDave Hansen ICPU(INTEL_FAM6_BROADWELL_X, idle_cpu_bdw), 1091db73c5a8SDave Hansen ICPU(INTEL_FAM6_BROADWELL_XEON_D, idle_cpu_bdw), 1092db73c5a8SDave Hansen ICPU(INTEL_FAM6_SKYLAKE_MOBILE, idle_cpu_skl), 1093db73c5a8SDave Hansen ICPU(INTEL_FAM6_SKYLAKE_DESKTOP, idle_cpu_skl), 1094db73c5a8SDave Hansen ICPU(INTEL_FAM6_KABYLAKE_MOBILE, idle_cpu_skl), 1095db73c5a8SDave Hansen ICPU(INTEL_FAM6_KABYLAKE_DESKTOP, idle_cpu_skl), 1096db73c5a8SDave Hansen ICPU(INTEL_FAM6_SKYLAKE_X, idle_cpu_skx), 1097db73c5a8SDave Hansen ICPU(INTEL_FAM6_XEON_PHI_KNL, idle_cpu_knl), 1098a2c1bc64SPiotr Luc ICPU(INTEL_FAM6_XEON_PHI_KNM, idle_cpu_knl), 1099db73c5a8SDave Hansen ICPU(INTEL_FAM6_ATOM_GOLDMONT, idle_cpu_bxt), 11000080d65bSJacob Pan ICPU(INTEL_FAM6_ATOM_DENVERTON, idle_cpu_dnv), 1101b66b8b9aSAndi Kleen {} 1102b66b8b9aSAndi Kleen }; 1103b66b8b9aSAndi Kleen 110426717172SLen Brown /* 110526717172SLen Brown * intel_idle_probe() 110626717172SLen Brown */ 110700f3e755SBartlomiej Zolnierkiewicz static int __init intel_idle_probe(void) 110826717172SLen Brown { 1109c4236282SLen Brown unsigned int eax, ebx, ecx; 1110b66b8b9aSAndi Kleen const struct x86_cpu_id *id; 111126717172SLen Brown 111226717172SLen Brown if (max_cstate == 0) { 111326717172SLen Brown pr_debug(PREFIX "disabled\n"); 111426717172SLen Brown return -EPERM; 111526717172SLen Brown } 111626717172SLen Brown 1117b66b8b9aSAndi Kleen id = x86_match_cpu(intel_idle_ids); 1118b66b8b9aSAndi Kleen if (!id) { 1119b66b8b9aSAndi Kleen if (boot_cpu_data.x86_vendor == X86_VENDOR_INTEL && 1120b66b8b9aSAndi Kleen boot_cpu_data.x86 == 6) 1121b66b8b9aSAndi Kleen pr_debug(PREFIX "does not run on family %d model %d\n", 1122b66b8b9aSAndi Kleen boot_cpu_data.x86, boot_cpu_data.x86_model); 112326717172SLen Brown return -ENODEV; 1124b66b8b9aSAndi Kleen } 112526717172SLen Brown 112626717172SLen Brown if (boot_cpu_data.cpuid_level < CPUID_MWAIT_LEAF) 112726717172SLen Brown return -ENODEV; 112826717172SLen Brown 1129c4236282SLen Brown cpuid(CPUID_MWAIT_LEAF, &eax, &ebx, &ecx, &mwait_substates); 113026717172SLen Brown 113126717172SLen Brown if (!(ecx & CPUID5_ECX_EXTENSIONS_SUPPORTED) || 11325c2a9f06SThomas Renninger !(ecx & CPUID5_ECX_INTERRUPT_BREAK) || 11335c2a9f06SThomas Renninger !mwait_substates) 113426717172SLen Brown return -ENODEV; 113526717172SLen Brown 1136c4236282SLen Brown pr_debug(PREFIX "MWAIT substates: 0x%x\n", mwait_substates); 113726717172SLen Brown 1138b66b8b9aSAndi Kleen icpu = (const struct idle_cpu *)id->driver_data; 1139b66b8b9aSAndi Kleen cpuidle_state_table = icpu->state_table; 114026717172SLen Brown 114126717172SLen Brown pr_debug(PREFIX "v" INTEL_IDLE_VERSION 114226717172SLen Brown " model 0x%X\n", boot_cpu_data.x86_model); 114326717172SLen Brown 114426717172SLen Brown return 0; 114526717172SLen Brown } 114626717172SLen Brown 114726717172SLen Brown /* 114826717172SLen Brown * intel_idle_cpuidle_devices_uninit() 1149ca42489dSRichard Cochran * Unregisters the cpuidle devices. 115026717172SLen Brown */ 115126717172SLen Brown static void intel_idle_cpuidle_devices_uninit(void) 115226717172SLen Brown { 115326717172SLen Brown int i; 115426717172SLen Brown struct cpuidle_device *dev; 115526717172SLen Brown 115626717172SLen Brown for_each_online_cpu(i) { 115726717172SLen Brown dev = per_cpu_ptr(intel_idle_cpuidle_devices, i); 115826717172SLen Brown cpuidle_unregister_device(dev); 115926717172SLen Brown } 116026717172SLen Brown } 11610138d8f0SLen Brown 11620138d8f0SLen Brown /* 1163d70e28f5SLen Brown * ivt_idle_state_table_update(void) 11640138d8f0SLen Brown * 1165d70e28f5SLen Brown * Tune IVT multi-socket targets 11660138d8f0SLen Brown * Assumption: num_sockets == (max_package_num + 1) 11670138d8f0SLen Brown */ 1168d70e28f5SLen Brown static void ivt_idle_state_table_update(void) 11690138d8f0SLen Brown { 11700138d8f0SLen Brown /* IVT uses a different table for 1-2, 3-4, and > 4 sockets */ 11710138d8f0SLen Brown int cpu, package_num, num_sockets = 1; 11720138d8f0SLen Brown 11730138d8f0SLen Brown for_each_online_cpu(cpu) { 11740138d8f0SLen Brown package_num = topology_physical_package_id(cpu); 11750138d8f0SLen Brown if (package_num + 1 > num_sockets) { 11760138d8f0SLen Brown num_sockets = package_num + 1; 11770138d8f0SLen Brown 1178d27dca42SChristoph Jaeger if (num_sockets > 4) { 11790138d8f0SLen Brown cpuidle_state_table = ivt_cstates_8s; 11800138d8f0SLen Brown return; 11810138d8f0SLen Brown } 11820138d8f0SLen Brown } 1183d27dca42SChristoph Jaeger } 11840138d8f0SLen Brown 11850138d8f0SLen Brown if (num_sockets > 2) 11860138d8f0SLen Brown cpuidle_state_table = ivt_cstates_4s; 1187d70e28f5SLen Brown 11880138d8f0SLen Brown /* else, 1 and 2 socket systems use default ivt_cstates */ 11890138d8f0SLen Brown } 11905dcef694SLen Brown 11915dcef694SLen Brown /* 11925dcef694SLen Brown * Translate IRTL (Interrupt Response Time Limit) MSR to usec 11935dcef694SLen Brown */ 11945dcef694SLen Brown 11955dcef694SLen Brown static unsigned int irtl_ns_units[] = { 11965dcef694SLen Brown 1, 32, 1024, 32768, 1048576, 33554432, 0, 0 }; 11975dcef694SLen Brown 11985dcef694SLen Brown static unsigned long long irtl_2_usec(unsigned long long irtl) 11995dcef694SLen Brown { 12005dcef694SLen Brown unsigned long long ns; 12015dcef694SLen Brown 12023451ab3eSJan Beulich if (!irtl) 12033451ab3eSJan Beulich return 0; 12043451ab3eSJan Beulich 1205bef45096SJan Beulich ns = irtl_ns_units[(irtl >> 10) & 0x7]; 12065dcef694SLen Brown 12075dcef694SLen Brown return div64_u64((irtl & 0x3FF) * ns, 1000); 12085dcef694SLen Brown } 12095dcef694SLen Brown /* 12105dcef694SLen Brown * bxt_idle_state_table_update(void) 12115dcef694SLen Brown * 12125dcef694SLen Brown * On BXT, we trust the IRTL to show the definitive maximum latency 12135dcef694SLen Brown * We use the same value for target_residency. 12145dcef694SLen Brown */ 12155dcef694SLen Brown static void bxt_idle_state_table_update(void) 12165dcef694SLen Brown { 12175dcef694SLen Brown unsigned long long msr; 12183451ab3eSJan Beulich unsigned int usec; 12195dcef694SLen Brown 12205dcef694SLen Brown rdmsrl(MSR_PKGC6_IRTL, msr); 12213451ab3eSJan Beulich usec = irtl_2_usec(msr); 12223451ab3eSJan Beulich if (usec) { 12235dcef694SLen Brown bxt_cstates[2].exit_latency = usec; 12245dcef694SLen Brown bxt_cstates[2].target_residency = usec; 12255dcef694SLen Brown } 12265dcef694SLen Brown 12275dcef694SLen Brown rdmsrl(MSR_PKGC7_IRTL, msr); 12283451ab3eSJan Beulich usec = irtl_2_usec(msr); 12293451ab3eSJan Beulich if (usec) { 12305dcef694SLen Brown bxt_cstates[3].exit_latency = usec; 12315dcef694SLen Brown bxt_cstates[3].target_residency = usec; 12325dcef694SLen Brown } 12335dcef694SLen Brown 12345dcef694SLen Brown rdmsrl(MSR_PKGC8_IRTL, msr); 12353451ab3eSJan Beulich usec = irtl_2_usec(msr); 12363451ab3eSJan Beulich if (usec) { 12375dcef694SLen Brown bxt_cstates[4].exit_latency = usec; 12385dcef694SLen Brown bxt_cstates[4].target_residency = usec; 12395dcef694SLen Brown } 12405dcef694SLen Brown 12415dcef694SLen Brown rdmsrl(MSR_PKGC9_IRTL, msr); 12423451ab3eSJan Beulich usec = irtl_2_usec(msr); 12433451ab3eSJan Beulich if (usec) { 12445dcef694SLen Brown bxt_cstates[5].exit_latency = usec; 12455dcef694SLen Brown bxt_cstates[5].target_residency = usec; 12465dcef694SLen Brown } 12475dcef694SLen Brown 12485dcef694SLen Brown rdmsrl(MSR_PKGC10_IRTL, msr); 12493451ab3eSJan Beulich usec = irtl_2_usec(msr); 12503451ab3eSJan Beulich if (usec) { 12515dcef694SLen Brown bxt_cstates[6].exit_latency = usec; 12525dcef694SLen Brown bxt_cstates[6].target_residency = usec; 12535dcef694SLen Brown } 12545dcef694SLen Brown 12555dcef694SLen Brown } 1256d70e28f5SLen Brown /* 1257d70e28f5SLen Brown * sklh_idle_state_table_update(void) 1258d70e28f5SLen Brown * 1259d70e28f5SLen Brown * On SKL-H (model 0x5e) disable C8 and C9 if: 1260d70e28f5SLen Brown * C10 is enabled and SGX disabled 1261d70e28f5SLen Brown */ 1262d70e28f5SLen Brown static void sklh_idle_state_table_update(void) 1263d70e28f5SLen Brown { 1264d70e28f5SLen Brown unsigned long long msr; 1265d70e28f5SLen Brown unsigned int eax, ebx, ecx, edx; 1266d70e28f5SLen Brown 1267d70e28f5SLen Brown 1268d70e28f5SLen Brown /* if PC10 disabled via cmdline intel_idle.max_cstate=7 or shallower */ 1269d70e28f5SLen Brown if (max_cstate <= 7) 12700138d8f0SLen Brown return; 1271d70e28f5SLen Brown 1272d70e28f5SLen Brown /* if PC10 not present in CPUID.MWAIT.EDX */ 1273d70e28f5SLen Brown if ((mwait_substates & (0xF << 28)) == 0) 1274d70e28f5SLen Brown return; 1275d70e28f5SLen Brown 1276d70e28f5SLen Brown rdmsrl(MSR_NHM_SNB_PKG_CST_CFG_CTL, msr); 1277d70e28f5SLen Brown 1278d70e28f5SLen Brown /* PC10 is not enabled in PKG C-state limit */ 1279d70e28f5SLen Brown if ((msr & 0xF) != 8) 1280d70e28f5SLen Brown return; 1281d70e28f5SLen Brown 1282d70e28f5SLen Brown ecx = 0; 1283d70e28f5SLen Brown cpuid(7, &eax, &ebx, &ecx, &edx); 1284d70e28f5SLen Brown 1285d70e28f5SLen Brown /* if SGX is present */ 1286d70e28f5SLen Brown if (ebx & (1 << 2)) { 1287d70e28f5SLen Brown 1288d70e28f5SLen Brown rdmsrl(MSR_IA32_FEATURE_CONTROL, msr); 1289d70e28f5SLen Brown 1290d70e28f5SLen Brown /* if SGX is enabled */ 1291d70e28f5SLen Brown if (msr & (1 << 18)) 1292d70e28f5SLen Brown return; 1293d70e28f5SLen Brown } 1294d70e28f5SLen Brown 1295d70e28f5SLen Brown skl_cstates[5].disabled = 1; /* C8-SKL */ 1296d70e28f5SLen Brown skl_cstates[6].disabled = 1; /* C9-SKL */ 1297d70e28f5SLen Brown } 1298d70e28f5SLen Brown /* 1299d70e28f5SLen Brown * intel_idle_state_table_update() 1300d70e28f5SLen Brown * 1301d70e28f5SLen Brown * Update the default state_table for this CPU-id 1302d70e28f5SLen Brown */ 1303d70e28f5SLen Brown 1304d70e28f5SLen Brown static void intel_idle_state_table_update(void) 1305d70e28f5SLen Brown { 1306d70e28f5SLen Brown switch (boot_cpu_data.x86_model) { 1307d70e28f5SLen Brown 1308db73c5a8SDave Hansen case INTEL_FAM6_IVYBRIDGE_X: 1309d70e28f5SLen Brown ivt_idle_state_table_update(); 1310d70e28f5SLen Brown break; 1311db73c5a8SDave Hansen case INTEL_FAM6_ATOM_GOLDMONT: 13125dcef694SLen Brown bxt_idle_state_table_update(); 13135dcef694SLen Brown break; 1314db73c5a8SDave Hansen case INTEL_FAM6_SKYLAKE_DESKTOP: 1315d70e28f5SLen Brown sklh_idle_state_table_update(); 1316d70e28f5SLen Brown break; 1317d70e28f5SLen Brown } 13180138d8f0SLen Brown } 13190138d8f0SLen Brown 132026717172SLen Brown /* 132146bcfad7SDeepthi Dharwar * intel_idle_cpuidle_driver_init() 132246bcfad7SDeepthi Dharwar * allocate, initialize cpuidle_states 132346bcfad7SDeepthi Dharwar */ 13245469c827SRichard Cochran static void __init intel_idle_cpuidle_driver_init(void) 132546bcfad7SDeepthi Dharwar { 132646bcfad7SDeepthi Dharwar int cstate; 132746bcfad7SDeepthi Dharwar struct cpuidle_driver *drv = &intel_idle_driver; 132846bcfad7SDeepthi Dharwar 13290138d8f0SLen Brown intel_idle_state_table_update(); 13300138d8f0SLen Brown 133146bcfad7SDeepthi Dharwar drv->state_count = 1; 133246bcfad7SDeepthi Dharwar 1333e022e7ebSLen Brown for (cstate = 0; cstate < CPUIDLE_STATE_MAX; ++cstate) { 133424bfa950SLen Brown int num_substates, mwait_hint, mwait_cstate; 133546bcfad7SDeepthi Dharwar 13367dd0e0afSLen Brown if ((cpuidle_state_table[cstate].enter == NULL) && 13377dd0e0afSLen Brown (cpuidle_state_table[cstate].enter_freeze == NULL)) 1338e022e7ebSLen Brown break; 1339e022e7ebSLen Brown 1340e022e7ebSLen Brown if (cstate + 1 > max_cstate) { 134146bcfad7SDeepthi Dharwar printk(PREFIX "max_cstate %d reached\n", 134246bcfad7SDeepthi Dharwar max_cstate); 134346bcfad7SDeepthi Dharwar break; 134446bcfad7SDeepthi Dharwar } 134546bcfad7SDeepthi Dharwar 1346e022e7ebSLen Brown mwait_hint = flg2MWAIT(cpuidle_state_table[cstate].flags); 1347e022e7ebSLen Brown mwait_cstate = MWAIT_HINT2CSTATE(mwait_hint); 134846bcfad7SDeepthi Dharwar 134924bfa950SLen Brown /* number of sub-states for this state in CPUID.MWAIT */ 1350e022e7ebSLen Brown num_substates = (mwait_substates >> ((mwait_cstate + 1) * 4)) 1351e022e7ebSLen Brown & MWAIT_SUBSTATE_MASK; 1352e022e7ebSLen Brown 135324bfa950SLen Brown /* if NO sub-states for this state in CPUID, skip it */ 135424bfa950SLen Brown if (num_substates == 0) 1355e022e7ebSLen Brown continue; 1356e022e7ebSLen Brown 1357d70e28f5SLen Brown /* if state marked as disabled, skip it */ 1358d70e28f5SLen Brown if (cpuidle_state_table[cstate].disabled != 0) { 1359d70e28f5SLen Brown pr_debug(PREFIX "state %s is disabled", 1360d70e28f5SLen Brown cpuidle_state_table[cstate].name); 1361d70e28f5SLen Brown continue; 1362d70e28f5SLen Brown } 1363d70e28f5SLen Brown 1364d70e28f5SLen Brown 1365e022e7ebSLen Brown if (((mwait_cstate + 1) > 2) && 136646bcfad7SDeepthi Dharwar !boot_cpu_has(X86_FEATURE_NONSTOP_TSC)) 136746bcfad7SDeepthi Dharwar mark_tsc_unstable("TSC halts in idle" 136846bcfad7SDeepthi Dharwar " states deeper than C2"); 136946bcfad7SDeepthi Dharwar 137046bcfad7SDeepthi Dharwar drv->states[drv->state_count] = /* structure copy */ 137146bcfad7SDeepthi Dharwar cpuidle_state_table[cstate]; 137246bcfad7SDeepthi Dharwar 137346bcfad7SDeepthi Dharwar drv->state_count += 1; 137446bcfad7SDeepthi Dharwar } 137546bcfad7SDeepthi Dharwar 13768c058d53SLen Brown if (icpu->byt_auto_demotion_disable_flag) { 13778c058d53SLen Brown wrmsrl(MSR_CC6_DEMOTION_POLICY_CONFIG, 0); 13788c058d53SLen Brown wrmsrl(MSR_MC6_DEMOTION_POLICY_CONFIG, 0); 13798c058d53SLen Brown } 138046bcfad7SDeepthi Dharwar } 138146bcfad7SDeepthi Dharwar 138246bcfad7SDeepthi Dharwar 138346bcfad7SDeepthi Dharwar /* 138465b7f839SThomas Renninger * intel_idle_cpu_init() 138526717172SLen Brown * allocate, initialize, register cpuidle_devices 138665b7f839SThomas Renninger * @cpu: cpu/core to initialize 138726717172SLen Brown */ 1388fb1013a0SSebastian Andrzej Siewior static int intel_idle_cpu_init(unsigned int cpu) 138926717172SLen Brown { 139026717172SLen Brown struct cpuidle_device *dev; 139126717172SLen Brown 139265b7f839SThomas Renninger dev = per_cpu_ptr(intel_idle_cpuidle_devices, cpu); 139365b7f839SThomas Renninger dev->cpu = cpu; 139426717172SLen Brown 139526717172SLen Brown if (cpuidle_register_device(dev)) { 139665b7f839SThomas Renninger pr_debug(PREFIX "cpuidle_register_device %d failed!\n", cpu); 139726717172SLen Brown return -EIO; 139826717172SLen Brown } 139965b7f839SThomas Renninger 1400b66b8b9aSAndi Kleen if (icpu->auto_demotion_disable_flags) 1401fb1013a0SSebastian Andrzej Siewior auto_demotion_disable(); 140226717172SLen Brown 1403dbf87ab8SBartlomiej Zolnierkiewicz if (icpu->disable_promotion_to_c1e) 1404fb1013a0SSebastian Andrzej Siewior c1e_promotion_disable(); 1405fb1013a0SSebastian Andrzej Siewior 1406fb1013a0SSebastian Andrzej Siewior return 0; 1407fb1013a0SSebastian Andrzej Siewior } 1408fb1013a0SSebastian Andrzej Siewior 1409fb1013a0SSebastian Andrzej Siewior static int intel_idle_cpu_online(unsigned int cpu) 1410fb1013a0SSebastian Andrzej Siewior { 1411fb1013a0SSebastian Andrzej Siewior struct cpuidle_device *dev; 1412fb1013a0SSebastian Andrzej Siewior 1413fb1013a0SSebastian Andrzej Siewior if (lapic_timer_reliable_states != LAPIC_TIMER_ALWAYS_RELIABLE) 1414fb1013a0SSebastian Andrzej Siewior __setup_broadcast_timer(true); 1415fb1013a0SSebastian Andrzej Siewior 1416fb1013a0SSebastian Andrzej Siewior /* 1417fb1013a0SSebastian Andrzej Siewior * Some systems can hotplug a cpu at runtime after 1418fb1013a0SSebastian Andrzej Siewior * the kernel has booted, we have to initialize the 1419fb1013a0SSebastian Andrzej Siewior * driver in this case 1420fb1013a0SSebastian Andrzej Siewior */ 1421fb1013a0SSebastian Andrzej Siewior dev = per_cpu_ptr(intel_idle_cpuidle_devices, cpu); 1422fb1013a0SSebastian Andrzej Siewior if (!dev->registered) 1423fb1013a0SSebastian Andrzej Siewior return intel_idle_cpu_init(cpu); 1424dbf87ab8SBartlomiej Zolnierkiewicz 142526717172SLen Brown return 0; 142626717172SLen Brown } 142726717172SLen Brown 142826717172SLen Brown static int __init intel_idle_init(void) 142926717172SLen Brown { 1430fb1013a0SSebastian Andrzej Siewior int retval; 143126717172SLen Brown 1432d1896049SThomas Renninger /* Do not load intel_idle at all for now if idle= is passed */ 1433d1896049SThomas Renninger if (boot_option_idle_override != IDLE_NO_OVERRIDE) 1434d1896049SThomas Renninger return -ENODEV; 1435d1896049SThomas Renninger 143626717172SLen Brown retval = intel_idle_probe(); 143726717172SLen Brown if (retval) 143826717172SLen Brown return retval; 143926717172SLen Brown 1440e9df69ccSRichard Cochran intel_idle_cpuidle_devices = alloc_percpu(struct cpuidle_device); 1441e9df69ccSRichard Cochran if (intel_idle_cpuidle_devices == NULL) 1442e9df69ccSRichard Cochran return -ENOMEM; 1443e9df69ccSRichard Cochran 144446bcfad7SDeepthi Dharwar intel_idle_cpuidle_driver_init(); 144526717172SLen Brown retval = cpuidle_register_driver(&intel_idle_driver); 144626717172SLen Brown if (retval) { 14473735d524SKonrad Rzeszutek Wilk struct cpuidle_driver *drv = cpuidle_get_driver(); 144826717172SLen Brown printk(KERN_DEBUG PREFIX "intel_idle yielding to %s", 14493735d524SKonrad Rzeszutek Wilk drv ? drv->name : "none"); 1450fb1013a0SSebastian Andrzej Siewior goto init_driver_fail; 145126717172SLen Brown } 145226717172SLen Brown 14532259a819SRichard Cochran if (boot_cpu_has(X86_FEATURE_ARAT)) /* Always Reliable APIC Timer */ 14542259a819SRichard Cochran lapic_timer_reliable_states = LAPIC_TIMER_ALWAYS_RELIABLE; 14552259a819SRichard Cochran 1456fb1013a0SSebastian Andrzej Siewior retval = cpuhp_setup_state(CPUHP_AP_ONLINE_DYN, "idle/intel:online", 1457fb1013a0SSebastian Andrzej Siewior intel_idle_cpu_online, NULL); 1458fb1013a0SSebastian Andrzej Siewior if (retval < 0) 1459fb1013a0SSebastian Andrzej Siewior goto hp_setup_fail; 146026717172SLen Brown 14612259a819SRichard Cochran pr_debug(PREFIX "lapic_timer_reliable_states 0x%x\n", 14622259a819SRichard Cochran lapic_timer_reliable_states); 14632259a819SRichard Cochran 146426717172SLen Brown return 0; 1465fb1013a0SSebastian Andrzej Siewior 1466fb1013a0SSebastian Andrzej Siewior hp_setup_fail: 1467fb1013a0SSebastian Andrzej Siewior intel_idle_cpuidle_devices_uninit(); 1468fb1013a0SSebastian Andrzej Siewior cpuidle_unregister_driver(&intel_idle_driver); 1469fb1013a0SSebastian Andrzej Siewior init_driver_fail: 1470fb1013a0SSebastian Andrzej Siewior free_percpu(intel_idle_cpuidle_devices); 1471fb1013a0SSebastian Andrzej Siewior return retval; 1472fb1013a0SSebastian Andrzej Siewior 147326717172SLen Brown } 147402c4fae9SPaul Gortmaker device_initcall(intel_idle_init); 147526717172SLen Brown 147602c4fae9SPaul Gortmaker /* 147702c4fae9SPaul Gortmaker * We are not really modular, but we used to support that. Meaning we also 147802c4fae9SPaul Gortmaker * support "intel_idle.max_cstate=..." at boot and also a read-only export of 147902c4fae9SPaul Gortmaker * it at /sys/module/intel_idle/parameters/max_cstate -- so using module_param 148002c4fae9SPaul Gortmaker * is the easiest way (currently) to continue doing that. 148102c4fae9SPaul Gortmaker */ 148226717172SLen Brown module_param(max_cstate, int, 0444); 1483