1a61127c2SThomas Gleixner // SPDX-License-Identifier: GPL-2.0-only 226717172SLen Brown /* 326717172SLen Brown * intel_idle.c - native hardware idle loop for modern Intel processors 426717172SLen Brown * 5fab04b22SLen Brown * Copyright (c) 2013, Intel Corporation. 626717172SLen Brown * Len Brown <len.brown@intel.com> 726717172SLen Brown */ 826717172SLen Brown 926717172SLen Brown /* 1026717172SLen Brown * intel_idle is a cpuidle driver that loads on specific Intel processors 1126717172SLen Brown * in lieu of the legacy ACPI processor_idle driver. The intent is to 1226717172SLen Brown * make Linux more efficient on these processors, as intel_idle knows 1326717172SLen Brown * more than ACPI, as well as make Linux more immune to ACPI BIOS bugs. 1426717172SLen Brown */ 1526717172SLen Brown 1626717172SLen Brown /* 1726717172SLen Brown * Design Assumptions 1826717172SLen Brown * 1926717172SLen Brown * All CPUs have same idle states as boot CPU 2026717172SLen Brown * 2126717172SLen Brown * Chipset BM_STS (bus master status) bit is a NOP 2226717172SLen Brown * for preventing entry into deep C-stats 2326717172SLen Brown */ 2426717172SLen Brown 2526717172SLen Brown /* 2626717172SLen Brown * Known limitations 2726717172SLen Brown * 2826717172SLen Brown * The driver currently initializes for_each_online_cpu() upon modprobe. 2926717172SLen Brown * It it unaware of subsequent processors hot-added to the system. 3026717172SLen Brown * This means that if you boot with maxcpus=n and later online 3126717172SLen Brown * processors above n, those processors will use C1 only. 3226717172SLen Brown * 3326717172SLen Brown * ACPI has a .suspend hack to turn off deep c-statees during suspend 3426717172SLen Brown * to avoid complications with the lapic timer workaround. 3526717172SLen Brown * Have not seen issues with suspend, but may need same workaround here. 3626717172SLen Brown * 3726717172SLen Brown */ 3826717172SLen Brown 3926717172SLen Brown /* un-comment DEBUG to enable pr_debug() statements */ 4026717172SLen Brown #define DEBUG 4126717172SLen Brown 42654d08a4SJoe Perches #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt 43654d08a4SJoe Perches 4418734958SRafael J. Wysocki #include <linux/acpi.h> 4526717172SLen Brown #include <linux/kernel.h> 4626717172SLen Brown #include <linux/cpuidle.h> 4776962caaSThomas Gleixner #include <linux/tick.h> 4826717172SLen Brown #include <trace/events/power.h> 4926717172SLen Brown #include <linux/sched.h> 502a2d31c8SShaohua Li #include <linux/notifier.h> 512a2d31c8SShaohua Li #include <linux/cpu.h> 5202c4fae9SPaul Gortmaker #include <linux/moduleparam.h> 53b66b8b9aSAndi Kleen #include <asm/cpu_device_id.h> 54db73c5a8SDave Hansen #include <asm/intel-family.h> 55bc83ccccSH. Peter Anvin #include <asm/mwait.h> 5614796fcaSLen Brown #include <asm/msr.h> 5726717172SLen Brown 58d70e28f5SLen Brown #define INTEL_IDLE_VERSION "0.4.1" 5926717172SLen Brown 6026717172SLen Brown static struct cpuidle_driver intel_idle_driver = { 6126717172SLen Brown .name = "intel_idle", 6226717172SLen Brown .owner = THIS_MODULE, 6326717172SLen Brown }; 6426717172SLen Brown /* intel_idle.max_cstate=0 disables driver */ 65137ecc77SLen Brown static int max_cstate = CPUIDLE_STATE_MAX - 1; 6626717172SLen Brown 67c4236282SLen Brown static unsigned int mwait_substates; 6826717172SLen Brown 692a2d31c8SShaohua Li #define LAPIC_TIMER_ALWAYS_RELIABLE 0xFFFFFFFF 7026717172SLen Brown /* Reliable LAPIC Timer States, bit 1 for C1 etc. */ 71d13780d4SLen Brown static unsigned int lapic_timer_reliable_states = (1 << 1); /* Default to only C1 */ 7226717172SLen Brown 73b66b8b9aSAndi Kleen struct idle_cpu { 74b66b8b9aSAndi Kleen struct cpuidle_state *state_table; 7526717172SLen Brown 7626717172SLen Brown /* 7714796fcaSLen Brown * Hardware C-state auto-demotion may not always be optimal. 7814796fcaSLen Brown * Indicate which enable bits to clear here. 7914796fcaSLen Brown */ 80b66b8b9aSAndi Kleen unsigned long auto_demotion_disable_flags; 818c058d53SLen Brown bool byt_auto_demotion_disable_flag; 8232e95180SLen Brown bool disable_promotion_to_c1e; 83bff8e60aSRafael J. Wysocki bool use_acpi; 84b66b8b9aSAndi Kleen }; 85b66b8b9aSAndi Kleen 86b66b8b9aSAndi Kleen static const struct idle_cpu *icpu; 87b66b8b9aSAndi Kleen static struct cpuidle_device __percpu *intel_idle_cpuidle_devices; 88b66b8b9aSAndi Kleen static int intel_idle(struct cpuidle_device *dev, 89b66b8b9aSAndi Kleen struct cpuidle_driver *drv, int index); 9028ba086eSRafael J. Wysocki static void intel_idle_s2idle(struct cpuidle_device *dev, 915fe2e527SRafael J. Wysocki struct cpuidle_driver *drv, int index); 92b66b8b9aSAndi Kleen static struct cpuidle_state *cpuidle_state_table; 9314796fcaSLen Brown 9414796fcaSLen Brown /* 95bff8e60aSRafael J. Wysocki * Enable this state by default even if the ACPI _CST does not list it. 96bff8e60aSRafael J. Wysocki */ 97bff8e60aSRafael J. Wysocki #define CPUIDLE_FLAG_ALWAYS_ENABLE BIT(15) 98bff8e60aSRafael J. Wysocki 99bff8e60aSRafael J. Wysocki /* 100956d033fSLen Brown * Set this flag for states where the HW flushes the TLB for us 101956d033fSLen Brown * and so we don't need cross-calls to keep it consistent. 102956d033fSLen Brown * If this flag is set, SW flushes the TLB, so even if the 103956d033fSLen Brown * HW doesn't do the flushing, this flag is safe to use. 104956d033fSLen Brown */ 105956d033fSLen Brown #define CPUIDLE_FLAG_TLB_FLUSHED 0x10000 106956d033fSLen Brown 107956d033fSLen Brown /* 108b1beab48SLen Brown * MWAIT takes an 8-bit "hint" in EAX "suggesting" 109b1beab48SLen Brown * the C-state (top nibble) and sub-state (bottom nibble) 110b1beab48SLen Brown * 0x00 means "MWAIT(C1)", 0x10 means "MWAIT(C2)" etc. 111b1beab48SLen Brown * 112b1beab48SLen Brown * We store the hint at the top of our "flags" for each state. 113b1beab48SLen Brown */ 114b1beab48SLen Brown #define flg2MWAIT(flags) (((flags) >> 24) & 0xFF) 115b1beab48SLen Brown #define MWAIT2flg(eax) ((eax & 0xFF) << 24) 116b1beab48SLen Brown 117b1beab48SLen Brown /* 11826717172SLen Brown * States are indexed by the cstate number, 11926717172SLen Brown * which is also the index into the MWAIT hint array. 12026717172SLen Brown * Thus C0 is a dummy. 12126717172SLen Brown */ 122ba0dc81eSJiang Liu static struct cpuidle_state nehalem_cstates[] = { 123e022e7ebSLen Brown { 124de09cdd0SLen Brown .name = "C1", 12526717172SLen Brown .desc = "MWAIT 0x00", 126b82b6ccaSDaniel Lezcano .flags = MWAIT2flg(0x00), 12726717172SLen Brown .exit_latency = 3, 12826717172SLen Brown .target_residency = 6, 1295fe2e527SRafael J. Wysocki .enter = &intel_idle, 13028ba086eSRafael J. Wysocki .enter_s2idle = intel_idle_s2idle, }, 131e022e7ebSLen Brown { 132de09cdd0SLen Brown .name = "C1E", 13332e95180SLen Brown .desc = "MWAIT 0x01", 134e6d4f08aSRafael J. Wysocki .flags = MWAIT2flg(0x01) | CPUIDLE_FLAG_ALWAYS_ENABLE, 13532e95180SLen Brown .exit_latency = 10, 13632e95180SLen Brown .target_residency = 20, 1375fe2e527SRafael J. Wysocki .enter = &intel_idle, 13828ba086eSRafael J. Wysocki .enter_s2idle = intel_idle_s2idle, }, 13932e95180SLen Brown { 140de09cdd0SLen Brown .name = "C3", 14126717172SLen Brown .desc = "MWAIT 0x10", 142b82b6ccaSDaniel Lezcano .flags = MWAIT2flg(0x10) | CPUIDLE_FLAG_TLB_FLUSHED, 14326717172SLen Brown .exit_latency = 20, 14426717172SLen Brown .target_residency = 80, 1455fe2e527SRafael J. Wysocki .enter = &intel_idle, 14628ba086eSRafael J. Wysocki .enter_s2idle = intel_idle_s2idle, }, 147e022e7ebSLen Brown { 148de09cdd0SLen Brown .name = "C6", 14926717172SLen Brown .desc = "MWAIT 0x20", 150b82b6ccaSDaniel Lezcano .flags = MWAIT2flg(0x20) | CPUIDLE_FLAG_TLB_FLUSHED, 15126717172SLen Brown .exit_latency = 200, 15226717172SLen Brown .target_residency = 800, 1535fe2e527SRafael J. Wysocki .enter = &intel_idle, 15428ba086eSRafael J. Wysocki .enter_s2idle = intel_idle_s2idle, }, 155e022e7ebSLen Brown { 156e022e7ebSLen Brown .enter = NULL } 15726717172SLen Brown }; 15826717172SLen Brown 159ba0dc81eSJiang Liu static struct cpuidle_state snb_cstates[] = { 160e022e7ebSLen Brown { 161de09cdd0SLen Brown .name = "C1", 162d13780d4SLen Brown .desc = "MWAIT 0x00", 163b82b6ccaSDaniel Lezcano .flags = MWAIT2flg(0x00), 16432e95180SLen Brown .exit_latency = 2, 16532e95180SLen Brown .target_residency = 2, 1665fe2e527SRafael J. Wysocki .enter = &intel_idle, 16728ba086eSRafael J. Wysocki .enter_s2idle = intel_idle_s2idle, }, 16832e95180SLen Brown { 169de09cdd0SLen Brown .name = "C1E", 17032e95180SLen Brown .desc = "MWAIT 0x01", 171e6d4f08aSRafael J. Wysocki .flags = MWAIT2flg(0x01) | CPUIDLE_FLAG_ALWAYS_ENABLE, 17232e95180SLen Brown .exit_latency = 10, 17332e95180SLen Brown .target_residency = 20, 1745fe2e527SRafael J. Wysocki .enter = &intel_idle, 17528ba086eSRafael J. Wysocki .enter_s2idle = intel_idle_s2idle, }, 176e022e7ebSLen Brown { 177de09cdd0SLen Brown .name = "C3", 178d13780d4SLen Brown .desc = "MWAIT 0x10", 179b82b6ccaSDaniel Lezcano .flags = MWAIT2flg(0x10) | CPUIDLE_FLAG_TLB_FLUSHED, 180d13780d4SLen Brown .exit_latency = 80, 181ddbd550dSLen Brown .target_residency = 211, 1825fe2e527SRafael J. Wysocki .enter = &intel_idle, 18328ba086eSRafael J. Wysocki .enter_s2idle = intel_idle_s2idle, }, 184e022e7ebSLen Brown { 185de09cdd0SLen Brown .name = "C6", 186d13780d4SLen Brown .desc = "MWAIT 0x20", 187b82b6ccaSDaniel Lezcano .flags = MWAIT2flg(0x20) | CPUIDLE_FLAG_TLB_FLUSHED, 188d13780d4SLen Brown .exit_latency = 104, 189ddbd550dSLen Brown .target_residency = 345, 1905fe2e527SRafael J. Wysocki .enter = &intel_idle, 19128ba086eSRafael J. Wysocki .enter_s2idle = intel_idle_s2idle, }, 192e022e7ebSLen Brown { 193de09cdd0SLen Brown .name = "C7", 194d13780d4SLen Brown .desc = "MWAIT 0x30", 195b82b6ccaSDaniel Lezcano .flags = MWAIT2flg(0x30) | CPUIDLE_FLAG_TLB_FLUSHED, 196d13780d4SLen Brown .exit_latency = 109, 197ddbd550dSLen Brown .target_residency = 345, 1985fe2e527SRafael J. Wysocki .enter = &intel_idle, 19928ba086eSRafael J. Wysocki .enter_s2idle = intel_idle_s2idle, }, 200e022e7ebSLen Brown { 201e022e7ebSLen Brown .enter = NULL } 202d13780d4SLen Brown }; 203d13780d4SLen Brown 204718987d6SLen Brown static struct cpuidle_state byt_cstates[] = { 205718987d6SLen Brown { 206de09cdd0SLen Brown .name = "C1", 207718987d6SLen Brown .desc = "MWAIT 0x00", 208b82b6ccaSDaniel Lezcano .flags = MWAIT2flg(0x00), 209718987d6SLen Brown .exit_latency = 1, 210718987d6SLen Brown .target_residency = 1, 2115fe2e527SRafael J. Wysocki .enter = &intel_idle, 21228ba086eSRafael J. Wysocki .enter_s2idle = intel_idle_s2idle, }, 213718987d6SLen Brown { 214de09cdd0SLen Brown .name = "C6N", 215718987d6SLen Brown .desc = "MWAIT 0x58", 216b82b6ccaSDaniel Lezcano .flags = MWAIT2flg(0x58) | CPUIDLE_FLAG_TLB_FLUSHED, 217d7ef7671SLen Brown .exit_latency = 300, 218718987d6SLen Brown .target_residency = 275, 2195fe2e527SRafael J. Wysocki .enter = &intel_idle, 22028ba086eSRafael J. Wysocki .enter_s2idle = intel_idle_s2idle, }, 221718987d6SLen Brown { 222de09cdd0SLen Brown .name = "C6S", 223718987d6SLen Brown .desc = "MWAIT 0x52", 224b82b6ccaSDaniel Lezcano .flags = MWAIT2flg(0x52) | CPUIDLE_FLAG_TLB_FLUSHED, 225d7ef7671SLen Brown .exit_latency = 500, 226718987d6SLen Brown .target_residency = 560, 2275fe2e527SRafael J. Wysocki .enter = &intel_idle, 22828ba086eSRafael J. Wysocki .enter_s2idle = intel_idle_s2idle, }, 229718987d6SLen Brown { 230de09cdd0SLen Brown .name = "C7", 231718987d6SLen Brown .desc = "MWAIT 0x60", 232b82b6ccaSDaniel Lezcano .flags = MWAIT2flg(0x60) | CPUIDLE_FLAG_TLB_FLUSHED, 233718987d6SLen Brown .exit_latency = 1200, 234d7ef7671SLen Brown .target_residency = 4000, 2355fe2e527SRafael J. Wysocki .enter = &intel_idle, 23628ba086eSRafael J. Wysocki .enter_s2idle = intel_idle_s2idle, }, 237718987d6SLen Brown { 238de09cdd0SLen Brown .name = "C7S", 239718987d6SLen Brown .desc = "MWAIT 0x64", 240b82b6ccaSDaniel Lezcano .flags = MWAIT2flg(0x64) | CPUIDLE_FLAG_TLB_FLUSHED, 241718987d6SLen Brown .exit_latency = 10000, 242718987d6SLen Brown .target_residency = 20000, 2435fe2e527SRafael J. Wysocki .enter = &intel_idle, 24428ba086eSRafael J. Wysocki .enter_s2idle = intel_idle_s2idle, }, 245718987d6SLen Brown { 246718987d6SLen Brown .enter = NULL } 247718987d6SLen Brown }; 248718987d6SLen Brown 249cab07a56SLen Brown static struct cpuidle_state cht_cstates[] = { 250cab07a56SLen Brown { 251de09cdd0SLen Brown .name = "C1", 252cab07a56SLen Brown .desc = "MWAIT 0x00", 253cab07a56SLen Brown .flags = MWAIT2flg(0x00), 254cab07a56SLen Brown .exit_latency = 1, 255cab07a56SLen Brown .target_residency = 1, 256cab07a56SLen Brown .enter = &intel_idle, 25728ba086eSRafael J. Wysocki .enter_s2idle = intel_idle_s2idle, }, 258cab07a56SLen Brown { 259de09cdd0SLen Brown .name = "C6N", 260cab07a56SLen Brown .desc = "MWAIT 0x58", 261cab07a56SLen Brown .flags = MWAIT2flg(0x58) | CPUIDLE_FLAG_TLB_FLUSHED, 262cab07a56SLen Brown .exit_latency = 80, 263cab07a56SLen Brown .target_residency = 275, 264cab07a56SLen Brown .enter = &intel_idle, 26528ba086eSRafael J. Wysocki .enter_s2idle = intel_idle_s2idle, }, 266cab07a56SLen Brown { 267de09cdd0SLen Brown .name = "C6S", 268cab07a56SLen Brown .desc = "MWAIT 0x52", 269cab07a56SLen Brown .flags = MWAIT2flg(0x52) | CPUIDLE_FLAG_TLB_FLUSHED, 270cab07a56SLen Brown .exit_latency = 200, 271cab07a56SLen Brown .target_residency = 560, 272cab07a56SLen Brown .enter = &intel_idle, 27328ba086eSRafael J. Wysocki .enter_s2idle = intel_idle_s2idle, }, 274cab07a56SLen Brown { 275de09cdd0SLen Brown .name = "C7", 276cab07a56SLen Brown .desc = "MWAIT 0x60", 277cab07a56SLen Brown .flags = MWAIT2flg(0x60) | CPUIDLE_FLAG_TLB_FLUSHED, 278cab07a56SLen Brown .exit_latency = 1200, 279cab07a56SLen Brown .target_residency = 4000, 280cab07a56SLen Brown .enter = &intel_idle, 28128ba086eSRafael J. Wysocki .enter_s2idle = intel_idle_s2idle, }, 282cab07a56SLen Brown { 283de09cdd0SLen Brown .name = "C7S", 284cab07a56SLen Brown .desc = "MWAIT 0x64", 285cab07a56SLen Brown .flags = MWAIT2flg(0x64) | CPUIDLE_FLAG_TLB_FLUSHED, 286cab07a56SLen Brown .exit_latency = 10000, 287cab07a56SLen Brown .target_residency = 20000, 288cab07a56SLen Brown .enter = &intel_idle, 28928ba086eSRafael J. Wysocki .enter_s2idle = intel_idle_s2idle, }, 290cab07a56SLen Brown { 291cab07a56SLen Brown .enter = NULL } 292cab07a56SLen Brown }; 293cab07a56SLen Brown 294ba0dc81eSJiang Liu static struct cpuidle_state ivb_cstates[] = { 295e022e7ebSLen Brown { 296de09cdd0SLen Brown .name = "C1", 2976edab08cSLen Brown .desc = "MWAIT 0x00", 298b82b6ccaSDaniel Lezcano .flags = MWAIT2flg(0x00), 2996edab08cSLen Brown .exit_latency = 1, 3006edab08cSLen Brown .target_residency = 1, 3015fe2e527SRafael J. Wysocki .enter = &intel_idle, 30228ba086eSRafael J. Wysocki .enter_s2idle = intel_idle_s2idle, }, 303e022e7ebSLen Brown { 304de09cdd0SLen Brown .name = "C1E", 30532e95180SLen Brown .desc = "MWAIT 0x01", 306e6d4f08aSRafael J. Wysocki .flags = MWAIT2flg(0x01) | CPUIDLE_FLAG_ALWAYS_ENABLE, 30732e95180SLen Brown .exit_latency = 10, 30832e95180SLen Brown .target_residency = 20, 3095fe2e527SRafael J. Wysocki .enter = &intel_idle, 31028ba086eSRafael J. Wysocki .enter_s2idle = intel_idle_s2idle, }, 31132e95180SLen Brown { 312de09cdd0SLen Brown .name = "C3", 3136edab08cSLen Brown .desc = "MWAIT 0x10", 314b82b6ccaSDaniel Lezcano .flags = MWAIT2flg(0x10) | CPUIDLE_FLAG_TLB_FLUSHED, 3156edab08cSLen Brown .exit_latency = 59, 3166edab08cSLen Brown .target_residency = 156, 3175fe2e527SRafael J. Wysocki .enter = &intel_idle, 31828ba086eSRafael J. Wysocki .enter_s2idle = intel_idle_s2idle, }, 319e022e7ebSLen Brown { 320de09cdd0SLen Brown .name = "C6", 3216edab08cSLen Brown .desc = "MWAIT 0x20", 322b82b6ccaSDaniel Lezcano .flags = MWAIT2flg(0x20) | CPUIDLE_FLAG_TLB_FLUSHED, 3236edab08cSLen Brown .exit_latency = 80, 3246edab08cSLen Brown .target_residency = 300, 3255fe2e527SRafael J. Wysocki .enter = &intel_idle, 32628ba086eSRafael J. Wysocki .enter_s2idle = intel_idle_s2idle, }, 327e022e7ebSLen Brown { 328de09cdd0SLen Brown .name = "C7", 3296edab08cSLen Brown .desc = "MWAIT 0x30", 330b82b6ccaSDaniel Lezcano .flags = MWAIT2flg(0x30) | CPUIDLE_FLAG_TLB_FLUSHED, 3316edab08cSLen Brown .exit_latency = 87, 3326edab08cSLen Brown .target_residency = 300, 3335fe2e527SRafael J. Wysocki .enter = &intel_idle, 33428ba086eSRafael J. Wysocki .enter_s2idle = intel_idle_s2idle, }, 335e022e7ebSLen Brown { 336e022e7ebSLen Brown .enter = NULL } 3376edab08cSLen Brown }; 3386edab08cSLen Brown 3390138d8f0SLen Brown static struct cpuidle_state ivt_cstates[] = { 3400138d8f0SLen Brown { 341de09cdd0SLen Brown .name = "C1", 3420138d8f0SLen Brown .desc = "MWAIT 0x00", 343b82b6ccaSDaniel Lezcano .flags = MWAIT2flg(0x00), 3440138d8f0SLen Brown .exit_latency = 1, 3450138d8f0SLen Brown .target_residency = 1, 3465fe2e527SRafael J. Wysocki .enter = &intel_idle, 34728ba086eSRafael J. Wysocki .enter_s2idle = intel_idle_s2idle, }, 3480138d8f0SLen Brown { 349de09cdd0SLen Brown .name = "C1E", 3500138d8f0SLen Brown .desc = "MWAIT 0x01", 351e6d4f08aSRafael J. Wysocki .flags = MWAIT2flg(0x01) | CPUIDLE_FLAG_ALWAYS_ENABLE, 3520138d8f0SLen Brown .exit_latency = 10, 3530138d8f0SLen Brown .target_residency = 80, 3545fe2e527SRafael J. Wysocki .enter = &intel_idle, 35528ba086eSRafael J. Wysocki .enter_s2idle = intel_idle_s2idle, }, 3560138d8f0SLen Brown { 357de09cdd0SLen Brown .name = "C3", 3580138d8f0SLen Brown .desc = "MWAIT 0x10", 359b82b6ccaSDaniel Lezcano .flags = MWAIT2flg(0x10) | CPUIDLE_FLAG_TLB_FLUSHED, 3600138d8f0SLen Brown .exit_latency = 59, 3610138d8f0SLen Brown .target_residency = 156, 3625fe2e527SRafael J. Wysocki .enter = &intel_idle, 36328ba086eSRafael J. Wysocki .enter_s2idle = intel_idle_s2idle, }, 3640138d8f0SLen Brown { 365de09cdd0SLen Brown .name = "C6", 3660138d8f0SLen Brown .desc = "MWAIT 0x20", 367b82b6ccaSDaniel Lezcano .flags = MWAIT2flg(0x20) | CPUIDLE_FLAG_TLB_FLUSHED, 3680138d8f0SLen Brown .exit_latency = 82, 3690138d8f0SLen Brown .target_residency = 300, 3705fe2e527SRafael J. Wysocki .enter = &intel_idle, 37128ba086eSRafael J. Wysocki .enter_s2idle = intel_idle_s2idle, }, 3720138d8f0SLen Brown { 3730138d8f0SLen Brown .enter = NULL } 3740138d8f0SLen Brown }; 3750138d8f0SLen Brown 3760138d8f0SLen Brown static struct cpuidle_state ivt_cstates_4s[] = { 3770138d8f0SLen Brown { 378de09cdd0SLen Brown .name = "C1", 3790138d8f0SLen Brown .desc = "MWAIT 0x00", 380b82b6ccaSDaniel Lezcano .flags = MWAIT2flg(0x00), 3810138d8f0SLen Brown .exit_latency = 1, 3820138d8f0SLen Brown .target_residency = 1, 3835fe2e527SRafael J. Wysocki .enter = &intel_idle, 38428ba086eSRafael J. Wysocki .enter_s2idle = intel_idle_s2idle, }, 3850138d8f0SLen Brown { 386de09cdd0SLen Brown .name = "C1E", 3870138d8f0SLen Brown .desc = "MWAIT 0x01", 388e6d4f08aSRafael J. Wysocki .flags = MWAIT2flg(0x01) | CPUIDLE_FLAG_ALWAYS_ENABLE, 3890138d8f0SLen Brown .exit_latency = 10, 3900138d8f0SLen Brown .target_residency = 250, 3915fe2e527SRafael J. Wysocki .enter = &intel_idle, 39228ba086eSRafael J. Wysocki .enter_s2idle = intel_idle_s2idle, }, 3930138d8f0SLen Brown { 394de09cdd0SLen Brown .name = "C3", 3950138d8f0SLen Brown .desc = "MWAIT 0x10", 396b82b6ccaSDaniel Lezcano .flags = MWAIT2flg(0x10) | CPUIDLE_FLAG_TLB_FLUSHED, 3970138d8f0SLen Brown .exit_latency = 59, 3980138d8f0SLen Brown .target_residency = 300, 3995fe2e527SRafael J. Wysocki .enter = &intel_idle, 40028ba086eSRafael J. Wysocki .enter_s2idle = intel_idle_s2idle, }, 4010138d8f0SLen Brown { 402de09cdd0SLen Brown .name = "C6", 4030138d8f0SLen Brown .desc = "MWAIT 0x20", 404b82b6ccaSDaniel Lezcano .flags = MWAIT2flg(0x20) | CPUIDLE_FLAG_TLB_FLUSHED, 4050138d8f0SLen Brown .exit_latency = 84, 4060138d8f0SLen Brown .target_residency = 400, 4075fe2e527SRafael J. Wysocki .enter = &intel_idle, 40828ba086eSRafael J. Wysocki .enter_s2idle = intel_idle_s2idle, }, 4090138d8f0SLen Brown { 4100138d8f0SLen Brown .enter = NULL } 4110138d8f0SLen Brown }; 4120138d8f0SLen Brown 4130138d8f0SLen Brown static struct cpuidle_state ivt_cstates_8s[] = { 4140138d8f0SLen Brown { 415de09cdd0SLen Brown .name = "C1", 4160138d8f0SLen Brown .desc = "MWAIT 0x00", 417b82b6ccaSDaniel Lezcano .flags = MWAIT2flg(0x00), 4180138d8f0SLen Brown .exit_latency = 1, 4190138d8f0SLen Brown .target_residency = 1, 4205fe2e527SRafael J. Wysocki .enter = &intel_idle, 42128ba086eSRafael J. Wysocki .enter_s2idle = intel_idle_s2idle, }, 4220138d8f0SLen Brown { 423de09cdd0SLen Brown .name = "C1E", 4240138d8f0SLen Brown .desc = "MWAIT 0x01", 425e6d4f08aSRafael J. Wysocki .flags = MWAIT2flg(0x01) | CPUIDLE_FLAG_ALWAYS_ENABLE, 4260138d8f0SLen Brown .exit_latency = 10, 4270138d8f0SLen Brown .target_residency = 500, 4285fe2e527SRafael J. Wysocki .enter = &intel_idle, 42928ba086eSRafael J. Wysocki .enter_s2idle = intel_idle_s2idle, }, 4300138d8f0SLen Brown { 431de09cdd0SLen Brown .name = "C3", 4320138d8f0SLen Brown .desc = "MWAIT 0x10", 433b82b6ccaSDaniel Lezcano .flags = MWAIT2flg(0x10) | CPUIDLE_FLAG_TLB_FLUSHED, 4340138d8f0SLen Brown .exit_latency = 59, 4350138d8f0SLen Brown .target_residency = 600, 4365fe2e527SRafael J. Wysocki .enter = &intel_idle, 43728ba086eSRafael J. Wysocki .enter_s2idle = intel_idle_s2idle, }, 4380138d8f0SLen Brown { 439de09cdd0SLen Brown .name = "C6", 4400138d8f0SLen Brown .desc = "MWAIT 0x20", 441b82b6ccaSDaniel Lezcano .flags = MWAIT2flg(0x20) | CPUIDLE_FLAG_TLB_FLUSHED, 4420138d8f0SLen Brown .exit_latency = 88, 4430138d8f0SLen Brown .target_residency = 700, 4445fe2e527SRafael J. Wysocki .enter = &intel_idle, 44528ba086eSRafael J. Wysocki .enter_s2idle = intel_idle_s2idle, }, 4460138d8f0SLen Brown { 4470138d8f0SLen Brown .enter = NULL } 4480138d8f0SLen Brown }; 4490138d8f0SLen Brown 450ba0dc81eSJiang Liu static struct cpuidle_state hsw_cstates[] = { 451e022e7ebSLen Brown { 452de09cdd0SLen Brown .name = "C1", 45385a4d2d4SLen Brown .desc = "MWAIT 0x00", 454b82b6ccaSDaniel Lezcano .flags = MWAIT2flg(0x00), 45585a4d2d4SLen Brown .exit_latency = 2, 45685a4d2d4SLen Brown .target_residency = 2, 4575fe2e527SRafael J. Wysocki .enter = &intel_idle, 45828ba086eSRafael J. Wysocki .enter_s2idle = intel_idle_s2idle, }, 459e022e7ebSLen Brown { 460de09cdd0SLen Brown .name = "C1E", 46132e95180SLen Brown .desc = "MWAIT 0x01", 462e6d4f08aSRafael J. Wysocki .flags = MWAIT2flg(0x01) | CPUIDLE_FLAG_ALWAYS_ENABLE, 46332e95180SLen Brown .exit_latency = 10, 46432e95180SLen Brown .target_residency = 20, 4655fe2e527SRafael J. Wysocki .enter = &intel_idle, 46628ba086eSRafael J. Wysocki .enter_s2idle = intel_idle_s2idle, }, 46732e95180SLen Brown { 468de09cdd0SLen Brown .name = "C3", 46985a4d2d4SLen Brown .desc = "MWAIT 0x10", 470b82b6ccaSDaniel Lezcano .flags = MWAIT2flg(0x10) | CPUIDLE_FLAG_TLB_FLUSHED, 47185a4d2d4SLen Brown .exit_latency = 33, 47285a4d2d4SLen Brown .target_residency = 100, 4735fe2e527SRafael J. Wysocki .enter = &intel_idle, 47428ba086eSRafael J. Wysocki .enter_s2idle = intel_idle_s2idle, }, 475e022e7ebSLen Brown { 476de09cdd0SLen Brown .name = "C6", 47785a4d2d4SLen Brown .desc = "MWAIT 0x20", 478b82b6ccaSDaniel Lezcano .flags = MWAIT2flg(0x20) | CPUIDLE_FLAG_TLB_FLUSHED, 47985a4d2d4SLen Brown .exit_latency = 133, 48085a4d2d4SLen Brown .target_residency = 400, 4815fe2e527SRafael J. Wysocki .enter = &intel_idle, 48228ba086eSRafael J. Wysocki .enter_s2idle = intel_idle_s2idle, }, 483e022e7ebSLen Brown { 484de09cdd0SLen Brown .name = "C7s", 48585a4d2d4SLen Brown .desc = "MWAIT 0x32", 486b82b6ccaSDaniel Lezcano .flags = MWAIT2flg(0x32) | CPUIDLE_FLAG_TLB_FLUSHED, 48785a4d2d4SLen Brown .exit_latency = 166, 48885a4d2d4SLen Brown .target_residency = 500, 4895fe2e527SRafael J. Wysocki .enter = &intel_idle, 49028ba086eSRafael J. Wysocki .enter_s2idle = intel_idle_s2idle, }, 491e022e7ebSLen Brown { 492de09cdd0SLen Brown .name = "C8", 49386239cebSLen Brown .desc = "MWAIT 0x40", 494b82b6ccaSDaniel Lezcano .flags = MWAIT2flg(0x40) | CPUIDLE_FLAG_TLB_FLUSHED, 49586239cebSLen Brown .exit_latency = 300, 49686239cebSLen Brown .target_residency = 900, 4975fe2e527SRafael J. Wysocki .enter = &intel_idle, 49828ba086eSRafael J. Wysocki .enter_s2idle = intel_idle_s2idle, }, 49986239cebSLen Brown { 500de09cdd0SLen Brown .name = "C9", 50186239cebSLen Brown .desc = "MWAIT 0x50", 502b82b6ccaSDaniel Lezcano .flags = MWAIT2flg(0x50) | CPUIDLE_FLAG_TLB_FLUSHED, 50386239cebSLen Brown .exit_latency = 600, 50486239cebSLen Brown .target_residency = 1800, 5055fe2e527SRafael J. Wysocki .enter = &intel_idle, 50628ba086eSRafael J. Wysocki .enter_s2idle = intel_idle_s2idle, }, 50786239cebSLen Brown { 508de09cdd0SLen Brown .name = "C10", 50986239cebSLen Brown .desc = "MWAIT 0x60", 510b82b6ccaSDaniel Lezcano .flags = MWAIT2flg(0x60) | CPUIDLE_FLAG_TLB_FLUSHED, 51186239cebSLen Brown .exit_latency = 2600, 51286239cebSLen Brown .target_residency = 7700, 5135fe2e527SRafael J. Wysocki .enter = &intel_idle, 51428ba086eSRafael J. Wysocki .enter_s2idle = intel_idle_s2idle, }, 51586239cebSLen Brown { 516e022e7ebSLen Brown .enter = NULL } 51785a4d2d4SLen Brown }; 518a138b568SLen Brown static struct cpuidle_state bdw_cstates[] = { 519a138b568SLen Brown { 520de09cdd0SLen Brown .name = "C1", 521a138b568SLen Brown .desc = "MWAIT 0x00", 522b82b6ccaSDaniel Lezcano .flags = MWAIT2flg(0x00), 523a138b568SLen Brown .exit_latency = 2, 524a138b568SLen Brown .target_residency = 2, 5255fe2e527SRafael J. Wysocki .enter = &intel_idle, 52628ba086eSRafael J. Wysocki .enter_s2idle = intel_idle_s2idle, }, 527a138b568SLen Brown { 528de09cdd0SLen Brown .name = "C1E", 529a138b568SLen Brown .desc = "MWAIT 0x01", 530e6d4f08aSRafael J. Wysocki .flags = MWAIT2flg(0x01) | CPUIDLE_FLAG_ALWAYS_ENABLE, 531a138b568SLen Brown .exit_latency = 10, 532a138b568SLen Brown .target_residency = 20, 5335fe2e527SRafael J. Wysocki .enter = &intel_idle, 53428ba086eSRafael J. Wysocki .enter_s2idle = intel_idle_s2idle, }, 535a138b568SLen Brown { 536de09cdd0SLen Brown .name = "C3", 537a138b568SLen Brown .desc = "MWAIT 0x10", 538b82b6ccaSDaniel Lezcano .flags = MWAIT2flg(0x10) | CPUIDLE_FLAG_TLB_FLUSHED, 539a138b568SLen Brown .exit_latency = 40, 540a138b568SLen Brown .target_residency = 100, 5415fe2e527SRafael J. Wysocki .enter = &intel_idle, 54228ba086eSRafael J. Wysocki .enter_s2idle = intel_idle_s2idle, }, 543a138b568SLen Brown { 544de09cdd0SLen Brown .name = "C6", 545a138b568SLen Brown .desc = "MWAIT 0x20", 546b82b6ccaSDaniel Lezcano .flags = MWAIT2flg(0x20) | CPUIDLE_FLAG_TLB_FLUSHED, 547a138b568SLen Brown .exit_latency = 133, 548a138b568SLen Brown .target_residency = 400, 5495fe2e527SRafael J. Wysocki .enter = &intel_idle, 55028ba086eSRafael J. Wysocki .enter_s2idle = intel_idle_s2idle, }, 551a138b568SLen Brown { 552de09cdd0SLen Brown .name = "C7s", 553a138b568SLen Brown .desc = "MWAIT 0x32", 554b82b6ccaSDaniel Lezcano .flags = MWAIT2flg(0x32) | CPUIDLE_FLAG_TLB_FLUSHED, 555a138b568SLen Brown .exit_latency = 166, 556a138b568SLen Brown .target_residency = 500, 5575fe2e527SRafael J. Wysocki .enter = &intel_idle, 55828ba086eSRafael J. Wysocki .enter_s2idle = intel_idle_s2idle, }, 559a138b568SLen Brown { 560de09cdd0SLen Brown .name = "C8", 561a138b568SLen Brown .desc = "MWAIT 0x40", 562b82b6ccaSDaniel Lezcano .flags = MWAIT2flg(0x40) | CPUIDLE_FLAG_TLB_FLUSHED, 563a138b568SLen Brown .exit_latency = 300, 564a138b568SLen Brown .target_residency = 900, 5655fe2e527SRafael J. Wysocki .enter = &intel_idle, 56628ba086eSRafael J. Wysocki .enter_s2idle = intel_idle_s2idle, }, 567a138b568SLen Brown { 568de09cdd0SLen Brown .name = "C9", 569a138b568SLen Brown .desc = "MWAIT 0x50", 570b82b6ccaSDaniel Lezcano .flags = MWAIT2flg(0x50) | CPUIDLE_FLAG_TLB_FLUSHED, 571a138b568SLen Brown .exit_latency = 600, 572a138b568SLen Brown .target_residency = 1800, 5735fe2e527SRafael J. Wysocki .enter = &intel_idle, 57428ba086eSRafael J. Wysocki .enter_s2idle = intel_idle_s2idle, }, 575a138b568SLen Brown { 576de09cdd0SLen Brown .name = "C10", 577a138b568SLen Brown .desc = "MWAIT 0x60", 578b82b6ccaSDaniel Lezcano .flags = MWAIT2flg(0x60) | CPUIDLE_FLAG_TLB_FLUSHED, 579a138b568SLen Brown .exit_latency = 2600, 580a138b568SLen Brown .target_residency = 7700, 5815fe2e527SRafael J. Wysocki .enter = &intel_idle, 58228ba086eSRafael J. Wysocki .enter_s2idle = intel_idle_s2idle, }, 583a138b568SLen Brown { 584a138b568SLen Brown .enter = NULL } 585a138b568SLen Brown }; 58685a4d2d4SLen Brown 587493f133fSLen Brown static struct cpuidle_state skl_cstates[] = { 588493f133fSLen Brown { 589de09cdd0SLen Brown .name = "C1", 590493f133fSLen Brown .desc = "MWAIT 0x00", 591493f133fSLen Brown .flags = MWAIT2flg(0x00), 592493f133fSLen Brown .exit_latency = 2, 593493f133fSLen Brown .target_residency = 2, 594493f133fSLen Brown .enter = &intel_idle, 59528ba086eSRafael J. Wysocki .enter_s2idle = intel_idle_s2idle, }, 596493f133fSLen Brown { 597de09cdd0SLen Brown .name = "C1E", 598493f133fSLen Brown .desc = "MWAIT 0x01", 599e6d4f08aSRafael J. Wysocki .flags = MWAIT2flg(0x01) | CPUIDLE_FLAG_ALWAYS_ENABLE, 600493f133fSLen Brown .exit_latency = 10, 601493f133fSLen Brown .target_residency = 20, 602493f133fSLen Brown .enter = &intel_idle, 60328ba086eSRafael J. Wysocki .enter_s2idle = intel_idle_s2idle, }, 604493f133fSLen Brown { 605de09cdd0SLen Brown .name = "C3", 606493f133fSLen Brown .desc = "MWAIT 0x10", 607493f133fSLen Brown .flags = MWAIT2flg(0x10) | CPUIDLE_FLAG_TLB_FLUSHED, 608493f133fSLen Brown .exit_latency = 70, 609493f133fSLen Brown .target_residency = 100, 610493f133fSLen Brown .enter = &intel_idle, 61128ba086eSRafael J. Wysocki .enter_s2idle = intel_idle_s2idle, }, 612493f133fSLen Brown { 613de09cdd0SLen Brown .name = "C6", 614493f133fSLen Brown .desc = "MWAIT 0x20", 615493f133fSLen Brown .flags = MWAIT2flg(0x20) | CPUIDLE_FLAG_TLB_FLUSHED, 616135919a3SLen Brown .exit_latency = 85, 617493f133fSLen Brown .target_residency = 200, 618493f133fSLen Brown .enter = &intel_idle, 61928ba086eSRafael J. Wysocki .enter_s2idle = intel_idle_s2idle, }, 620493f133fSLen Brown { 621de09cdd0SLen Brown .name = "C7s", 622493f133fSLen Brown .desc = "MWAIT 0x33", 623493f133fSLen Brown .flags = MWAIT2flg(0x33) | CPUIDLE_FLAG_TLB_FLUSHED, 624493f133fSLen Brown .exit_latency = 124, 625493f133fSLen Brown .target_residency = 800, 626493f133fSLen Brown .enter = &intel_idle, 62728ba086eSRafael J. Wysocki .enter_s2idle = intel_idle_s2idle, }, 628493f133fSLen Brown { 629de09cdd0SLen Brown .name = "C8", 630493f133fSLen Brown .desc = "MWAIT 0x40", 631493f133fSLen Brown .flags = MWAIT2flg(0x40) | CPUIDLE_FLAG_TLB_FLUSHED, 632135919a3SLen Brown .exit_latency = 200, 633493f133fSLen Brown .target_residency = 800, 634493f133fSLen Brown .enter = &intel_idle, 63528ba086eSRafael J. Wysocki .enter_s2idle = intel_idle_s2idle, }, 636493f133fSLen Brown { 637de09cdd0SLen Brown .name = "C9", 638135919a3SLen Brown .desc = "MWAIT 0x50", 639135919a3SLen Brown .flags = MWAIT2flg(0x50) | CPUIDLE_FLAG_TLB_FLUSHED, 640135919a3SLen Brown .exit_latency = 480, 641135919a3SLen Brown .target_residency = 5000, 642135919a3SLen Brown .enter = &intel_idle, 64328ba086eSRafael J. Wysocki .enter_s2idle = intel_idle_s2idle, }, 644135919a3SLen Brown { 645de09cdd0SLen Brown .name = "C10", 646493f133fSLen Brown .desc = "MWAIT 0x60", 647493f133fSLen Brown .flags = MWAIT2flg(0x60) | CPUIDLE_FLAG_TLB_FLUSHED, 648493f133fSLen Brown .exit_latency = 890, 649493f133fSLen Brown .target_residency = 5000, 650493f133fSLen Brown .enter = &intel_idle, 65128ba086eSRafael J. Wysocki .enter_s2idle = intel_idle_s2idle, }, 652493f133fSLen Brown { 653493f133fSLen Brown .enter = NULL } 654493f133fSLen Brown }; 655493f133fSLen Brown 656f9e71657SLen Brown static struct cpuidle_state skx_cstates[] = { 657f9e71657SLen Brown { 658de09cdd0SLen Brown .name = "C1", 659f9e71657SLen Brown .desc = "MWAIT 0x00", 660f9e71657SLen Brown .flags = MWAIT2flg(0x00), 661f9e71657SLen Brown .exit_latency = 2, 662f9e71657SLen Brown .target_residency = 2, 663f9e71657SLen Brown .enter = &intel_idle, 66428ba086eSRafael J. Wysocki .enter_s2idle = intel_idle_s2idle, }, 665f9e71657SLen Brown { 666de09cdd0SLen Brown .name = "C1E", 667f9e71657SLen Brown .desc = "MWAIT 0x01", 668e6d4f08aSRafael J. Wysocki .flags = MWAIT2flg(0x01) | CPUIDLE_FLAG_ALWAYS_ENABLE, 669f9e71657SLen Brown .exit_latency = 10, 670f9e71657SLen Brown .target_residency = 20, 671f9e71657SLen Brown .enter = &intel_idle, 67228ba086eSRafael J. Wysocki .enter_s2idle = intel_idle_s2idle, }, 673f9e71657SLen Brown { 674de09cdd0SLen Brown .name = "C6", 675f9e71657SLen Brown .desc = "MWAIT 0x20", 676f9e71657SLen Brown .flags = MWAIT2flg(0x20) | CPUIDLE_FLAG_TLB_FLUSHED, 677f9e71657SLen Brown .exit_latency = 133, 678f9e71657SLen Brown .target_residency = 600, 679f9e71657SLen Brown .enter = &intel_idle, 68028ba086eSRafael J. Wysocki .enter_s2idle = intel_idle_s2idle, }, 681f9e71657SLen Brown { 682f9e71657SLen Brown .enter = NULL } 683f9e71657SLen Brown }; 684f9e71657SLen Brown 685ba0dc81eSJiang Liu static struct cpuidle_state atom_cstates[] = { 686e022e7ebSLen Brown { 687de09cdd0SLen Brown .name = "C1E", 68826717172SLen Brown .desc = "MWAIT 0x00", 689b82b6ccaSDaniel Lezcano .flags = MWAIT2flg(0x00), 69032e95180SLen Brown .exit_latency = 10, 69132e95180SLen Brown .target_residency = 20, 6925fe2e527SRafael J. Wysocki .enter = &intel_idle, 69328ba086eSRafael J. Wysocki .enter_s2idle = intel_idle_s2idle, }, 694e022e7ebSLen Brown { 695de09cdd0SLen Brown .name = "C2", 69626717172SLen Brown .desc = "MWAIT 0x10", 697b82b6ccaSDaniel Lezcano .flags = MWAIT2flg(0x10), 69826717172SLen Brown .exit_latency = 20, 69926717172SLen Brown .target_residency = 80, 7005fe2e527SRafael J. Wysocki .enter = &intel_idle, 70128ba086eSRafael J. Wysocki .enter_s2idle = intel_idle_s2idle, }, 702e022e7ebSLen Brown { 703de09cdd0SLen Brown .name = "C4", 70426717172SLen Brown .desc = "MWAIT 0x30", 705b82b6ccaSDaniel Lezcano .flags = MWAIT2flg(0x30) | CPUIDLE_FLAG_TLB_FLUSHED, 70626717172SLen Brown .exit_latency = 100, 70726717172SLen Brown .target_residency = 400, 7085fe2e527SRafael J. Wysocki .enter = &intel_idle, 70928ba086eSRafael J. Wysocki .enter_s2idle = intel_idle_s2idle, }, 710e022e7ebSLen Brown { 711de09cdd0SLen Brown .name = "C6", 7127fcca7d9SLen Brown .desc = "MWAIT 0x52", 713b82b6ccaSDaniel Lezcano .flags = MWAIT2flg(0x52) | CPUIDLE_FLAG_TLB_FLUSHED, 7147fcca7d9SLen Brown .exit_latency = 140, 7157fcca7d9SLen Brown .target_residency = 560, 7165fe2e527SRafael J. Wysocki .enter = &intel_idle, 71728ba086eSRafael J. Wysocki .enter_s2idle = intel_idle_s2idle, }, 718e022e7ebSLen Brown { 719e022e7ebSLen Brown .enter = NULL } 72026717172SLen Brown }; 7215e7ec268SAndy Shevchenko static struct cpuidle_state tangier_cstates[] = { 7225e7ec268SAndy Shevchenko { 723de09cdd0SLen Brown .name = "C1", 7245e7ec268SAndy Shevchenko .desc = "MWAIT 0x00", 7255e7ec268SAndy Shevchenko .flags = MWAIT2flg(0x00), 7265e7ec268SAndy Shevchenko .exit_latency = 1, 7275e7ec268SAndy Shevchenko .target_residency = 4, 7285e7ec268SAndy Shevchenko .enter = &intel_idle, 72928ba086eSRafael J. Wysocki .enter_s2idle = intel_idle_s2idle, }, 7305e7ec268SAndy Shevchenko { 731de09cdd0SLen Brown .name = "C4", 7325e7ec268SAndy Shevchenko .desc = "MWAIT 0x30", 7335e7ec268SAndy Shevchenko .flags = MWAIT2flg(0x30) | CPUIDLE_FLAG_TLB_FLUSHED, 7345e7ec268SAndy Shevchenko .exit_latency = 100, 7355e7ec268SAndy Shevchenko .target_residency = 400, 7365e7ec268SAndy Shevchenko .enter = &intel_idle, 73728ba086eSRafael J. Wysocki .enter_s2idle = intel_idle_s2idle, }, 7385e7ec268SAndy Shevchenko { 739de09cdd0SLen Brown .name = "C6", 7405e7ec268SAndy Shevchenko .desc = "MWAIT 0x52", 7415e7ec268SAndy Shevchenko .flags = MWAIT2flg(0x52) | CPUIDLE_FLAG_TLB_FLUSHED, 7425e7ec268SAndy Shevchenko .exit_latency = 140, 7435e7ec268SAndy Shevchenko .target_residency = 560, 7445e7ec268SAndy Shevchenko .enter = &intel_idle, 74528ba086eSRafael J. Wysocki .enter_s2idle = intel_idle_s2idle, }, 7465e7ec268SAndy Shevchenko { 747de09cdd0SLen Brown .name = "C7", 7485e7ec268SAndy Shevchenko .desc = "MWAIT 0x60", 7495e7ec268SAndy Shevchenko .flags = MWAIT2flg(0x60) | CPUIDLE_FLAG_TLB_FLUSHED, 7505e7ec268SAndy Shevchenko .exit_latency = 1200, 7515e7ec268SAndy Shevchenko .target_residency = 4000, 7525e7ec268SAndy Shevchenko .enter = &intel_idle, 75328ba086eSRafael J. Wysocki .enter_s2idle = intel_idle_s2idle, }, 7545e7ec268SAndy Shevchenko { 755de09cdd0SLen Brown .name = "C9", 7565e7ec268SAndy Shevchenko .desc = "MWAIT 0x64", 7575e7ec268SAndy Shevchenko .flags = MWAIT2flg(0x64) | CPUIDLE_FLAG_TLB_FLUSHED, 7585e7ec268SAndy Shevchenko .exit_latency = 10000, 7595e7ec268SAndy Shevchenko .target_residency = 20000, 7605e7ec268SAndy Shevchenko .enter = &intel_idle, 76128ba086eSRafael J. Wysocki .enter_s2idle = intel_idle_s2idle, }, 7625e7ec268SAndy Shevchenko { 7635e7ec268SAndy Shevchenko .enter = NULL } 7645e7ec268SAndy Shevchenko }; 76588390996SJiang Liu static struct cpuidle_state avn_cstates[] = { 766fab04b22SLen Brown { 767de09cdd0SLen Brown .name = "C1", 768fab04b22SLen Brown .desc = "MWAIT 0x00", 769b82b6ccaSDaniel Lezcano .flags = MWAIT2flg(0x00), 770fab04b22SLen Brown .exit_latency = 2, 771fab04b22SLen Brown .target_residency = 2, 7725fe2e527SRafael J. Wysocki .enter = &intel_idle, 77328ba086eSRafael J. Wysocki .enter_s2idle = intel_idle_s2idle, }, 774fab04b22SLen Brown { 775de09cdd0SLen Brown .name = "C6", 776fab04b22SLen Brown .desc = "MWAIT 0x51", 777b82b6ccaSDaniel Lezcano .flags = MWAIT2flg(0x51) | CPUIDLE_FLAG_TLB_FLUSHED, 778fab04b22SLen Brown .exit_latency = 15, 779fab04b22SLen Brown .target_residency = 45, 7805fe2e527SRafael J. Wysocki .enter = &intel_idle, 78128ba086eSRafael J. Wysocki .enter_s2idle = intel_idle_s2idle, }, 78288390996SJiang Liu { 78388390996SJiang Liu .enter = NULL } 784fab04b22SLen Brown }; 785281baf7aSDasaratharaman Chandramouli static struct cpuidle_state knl_cstates[] = { 786281baf7aSDasaratharaman Chandramouli { 787de09cdd0SLen Brown .name = "C1", 788281baf7aSDasaratharaman Chandramouli .desc = "MWAIT 0x00", 789281baf7aSDasaratharaman Chandramouli .flags = MWAIT2flg(0x00), 790281baf7aSDasaratharaman Chandramouli .exit_latency = 1, 791281baf7aSDasaratharaman Chandramouli .target_residency = 2, 792281baf7aSDasaratharaman Chandramouli .enter = &intel_idle, 79328ba086eSRafael J. Wysocki .enter_s2idle = intel_idle_s2idle }, 794281baf7aSDasaratharaman Chandramouli { 795de09cdd0SLen Brown .name = "C6", 796281baf7aSDasaratharaman Chandramouli .desc = "MWAIT 0x10", 797281baf7aSDasaratharaman Chandramouli .flags = MWAIT2flg(0x10) | CPUIDLE_FLAG_TLB_FLUSHED, 798281baf7aSDasaratharaman Chandramouli .exit_latency = 120, 799281baf7aSDasaratharaman Chandramouli .target_residency = 500, 800281baf7aSDasaratharaman Chandramouli .enter = &intel_idle, 80128ba086eSRafael J. Wysocki .enter_s2idle = intel_idle_s2idle }, 802281baf7aSDasaratharaman Chandramouli { 803281baf7aSDasaratharaman Chandramouli .enter = NULL } 804281baf7aSDasaratharaman Chandramouli }; 80526717172SLen Brown 8065dcef694SLen Brown static struct cpuidle_state bxt_cstates[] = { 8075dcef694SLen Brown { 808de09cdd0SLen Brown .name = "C1", 8095dcef694SLen Brown .desc = "MWAIT 0x00", 8105dcef694SLen Brown .flags = MWAIT2flg(0x00), 8115dcef694SLen Brown .exit_latency = 2, 8125dcef694SLen Brown .target_residency = 2, 8135dcef694SLen Brown .enter = &intel_idle, 81428ba086eSRafael J. Wysocki .enter_s2idle = intel_idle_s2idle, }, 8155dcef694SLen Brown { 816de09cdd0SLen Brown .name = "C1E", 8175dcef694SLen Brown .desc = "MWAIT 0x01", 818e6d4f08aSRafael J. Wysocki .flags = MWAIT2flg(0x01) | CPUIDLE_FLAG_ALWAYS_ENABLE, 8195dcef694SLen Brown .exit_latency = 10, 8205dcef694SLen Brown .target_residency = 20, 8215dcef694SLen Brown .enter = &intel_idle, 82228ba086eSRafael J. Wysocki .enter_s2idle = intel_idle_s2idle, }, 8235dcef694SLen Brown { 824de09cdd0SLen Brown .name = "C6", 8255dcef694SLen Brown .desc = "MWAIT 0x20", 8265dcef694SLen Brown .flags = MWAIT2flg(0x20) | CPUIDLE_FLAG_TLB_FLUSHED, 8275dcef694SLen Brown .exit_latency = 133, 8285dcef694SLen Brown .target_residency = 133, 8295dcef694SLen Brown .enter = &intel_idle, 83028ba086eSRafael J. Wysocki .enter_s2idle = intel_idle_s2idle, }, 8315dcef694SLen Brown { 832de09cdd0SLen Brown .name = "C7s", 8335dcef694SLen Brown .desc = "MWAIT 0x31", 8345dcef694SLen Brown .flags = MWAIT2flg(0x31) | CPUIDLE_FLAG_TLB_FLUSHED, 8355dcef694SLen Brown .exit_latency = 155, 8365dcef694SLen Brown .target_residency = 155, 8375dcef694SLen Brown .enter = &intel_idle, 83828ba086eSRafael J. Wysocki .enter_s2idle = intel_idle_s2idle, }, 8395dcef694SLen Brown { 840de09cdd0SLen Brown .name = "C8", 8415dcef694SLen Brown .desc = "MWAIT 0x40", 8425dcef694SLen Brown .flags = MWAIT2flg(0x40) | CPUIDLE_FLAG_TLB_FLUSHED, 8435dcef694SLen Brown .exit_latency = 1000, 8445dcef694SLen Brown .target_residency = 1000, 8455dcef694SLen Brown .enter = &intel_idle, 84628ba086eSRafael J. Wysocki .enter_s2idle = intel_idle_s2idle, }, 8475dcef694SLen Brown { 848de09cdd0SLen Brown .name = "C9", 8495dcef694SLen Brown .desc = "MWAIT 0x50", 8505dcef694SLen Brown .flags = MWAIT2flg(0x50) | CPUIDLE_FLAG_TLB_FLUSHED, 8515dcef694SLen Brown .exit_latency = 2000, 8525dcef694SLen Brown .target_residency = 2000, 8535dcef694SLen Brown .enter = &intel_idle, 85428ba086eSRafael J. Wysocki .enter_s2idle = intel_idle_s2idle, }, 8555dcef694SLen Brown { 856de09cdd0SLen Brown .name = "C10", 8575dcef694SLen Brown .desc = "MWAIT 0x60", 8585dcef694SLen Brown .flags = MWAIT2flg(0x60) | CPUIDLE_FLAG_TLB_FLUSHED, 8595dcef694SLen Brown .exit_latency = 10000, 8605dcef694SLen Brown .target_residency = 10000, 8615dcef694SLen Brown .enter = &intel_idle, 86228ba086eSRafael J. Wysocki .enter_s2idle = intel_idle_s2idle, }, 8635dcef694SLen Brown { 8645dcef694SLen Brown .enter = NULL } 8655dcef694SLen Brown }; 8665dcef694SLen Brown 8670080d65bSJacob Pan static struct cpuidle_state dnv_cstates[] = { 8680080d65bSJacob Pan { 869de09cdd0SLen Brown .name = "C1", 8700080d65bSJacob Pan .desc = "MWAIT 0x00", 8710080d65bSJacob Pan .flags = MWAIT2flg(0x00), 8720080d65bSJacob Pan .exit_latency = 2, 8730080d65bSJacob Pan .target_residency = 2, 8740080d65bSJacob Pan .enter = &intel_idle, 87528ba086eSRafael J. Wysocki .enter_s2idle = intel_idle_s2idle, }, 8760080d65bSJacob Pan { 877de09cdd0SLen Brown .name = "C1E", 8780080d65bSJacob Pan .desc = "MWAIT 0x01", 879e6d4f08aSRafael J. Wysocki .flags = MWAIT2flg(0x01) | CPUIDLE_FLAG_ALWAYS_ENABLE, 8800080d65bSJacob Pan .exit_latency = 10, 8810080d65bSJacob Pan .target_residency = 20, 8820080d65bSJacob Pan .enter = &intel_idle, 88328ba086eSRafael J. Wysocki .enter_s2idle = intel_idle_s2idle, }, 8840080d65bSJacob Pan { 885de09cdd0SLen Brown .name = "C6", 8860080d65bSJacob Pan .desc = "MWAIT 0x20", 8870080d65bSJacob Pan .flags = MWAIT2flg(0x20) | CPUIDLE_FLAG_TLB_FLUSHED, 8880080d65bSJacob Pan .exit_latency = 50, 8890080d65bSJacob Pan .target_residency = 500, 8900080d65bSJacob Pan .enter = &intel_idle, 89128ba086eSRafael J. Wysocki .enter_s2idle = intel_idle_s2idle, }, 8920080d65bSJacob Pan { 8930080d65bSJacob Pan .enter = NULL } 8940080d65bSJacob Pan }; 8950080d65bSJacob Pan 89626717172SLen Brown /** 89726717172SLen Brown * intel_idle 89826717172SLen Brown * @dev: cpuidle_device 89946bcfad7SDeepthi Dharwar * @drv: cpuidle driver 900e978aa7dSDeepthi Dharwar * @index: index of cpuidle state 90126717172SLen Brown * 90263ff07beSYanmin Zhang * Must be called under local_irq_disable(). 90326717172SLen Brown */ 9046727ad9eSChris Metcalf static __cpuidle int intel_idle(struct cpuidle_device *dev, 90546bcfad7SDeepthi Dharwar struct cpuidle_driver *drv, int index) 90626717172SLen Brown { 90726717172SLen Brown unsigned long ecx = 1; /* break on interrupt flag */ 90846bcfad7SDeepthi Dharwar struct cpuidle_state *state = &drv->states[index]; 909b1beab48SLen Brown unsigned long eax = flg2MWAIT(state->flags); 91026717172SLen Brown unsigned int cstate; 9110563bb7bSJason Baron bool uninitialized_var(tick); 91267535736SAndy Lutomirski int cpu = smp_processor_id(); 91326717172SLen Brown 9146110a1f4SSuresh Siddha /* 91567535736SAndy Lutomirski * leave_mm() to avoid costly and often unnecessary wakeups 91667535736SAndy Lutomirski * for flushing the user TLB's associated with the active mm. 9176110a1f4SSuresh Siddha */ 91867535736SAndy Lutomirski if (state->flags & CPUIDLE_FLAG_TLB_FLUSHED) 91967535736SAndy Lutomirski leave_mm(cpu); 9206110a1f4SSuresh Siddha 9210563bb7bSJason Baron if (!static_cpu_has(X86_FEATURE_ARAT)) { 9220563bb7bSJason Baron cstate = (((eax) >> MWAIT_SUBSTATE_SIZE) & 9230563bb7bSJason Baron MWAIT_CSTATE_MASK) + 1; 9240563bb7bSJason Baron tick = false; 9250563bb7bSJason Baron if (!(lapic_timer_reliable_states & (1 << (cstate)))) { 9260563bb7bSJason Baron tick = true; 927f6cee191SThomas Gleixner tick_broadcast_enter(); 9280563bb7bSJason Baron } 9290563bb7bSJason Baron } 93026717172SLen Brown 93116824255SPeter Zijlstra mwait_idle_with_hints(eax, ecx); 93226717172SLen Brown 9330563bb7bSJason Baron if (!static_cpu_has(X86_FEATURE_ARAT) && tick) 934f6cee191SThomas Gleixner tick_broadcast_exit(); 93526717172SLen Brown 936e978aa7dSDeepthi Dharwar return index; 93726717172SLen Brown } 93826717172SLen Brown 9395fe2e527SRafael J. Wysocki /** 94028ba086eSRafael J. Wysocki * intel_idle_s2idle - simplified "enter" callback routine for suspend-to-idle 9415fe2e527SRafael J. Wysocki * @dev: cpuidle_device 9425fe2e527SRafael J. Wysocki * @drv: cpuidle driver 9435fe2e527SRafael J. Wysocki * @index: state index 9445fe2e527SRafael J. Wysocki */ 94528ba086eSRafael J. Wysocki static void intel_idle_s2idle(struct cpuidle_device *dev, 9465fe2e527SRafael J. Wysocki struct cpuidle_driver *drv, int index) 9475fe2e527SRafael J. Wysocki { 9485fe2e527SRafael J. Wysocki unsigned long ecx = 1; /* break on interrupt flag */ 9495fe2e527SRafael J. Wysocki unsigned long eax = flg2MWAIT(drv->states[index].flags); 9505fe2e527SRafael J. Wysocki 9515fe2e527SRafael J. Wysocki mwait_idle_with_hints(eax, ecx); 9525fe2e527SRafael J. Wysocki } 9535fe2e527SRafael J. Wysocki 9549f3d6dafSRafael J. Wysocki static bool intel_idle_verify_cstate(unsigned int mwait_hint) 9559f3d6dafSRafael J. Wysocki { 9569f3d6dafSRafael J. Wysocki unsigned int mwait_cstate = MWAIT_HINT2CSTATE(mwait_hint) + 1; 9579f3d6dafSRafael J. Wysocki unsigned int num_substates = (mwait_substates >> mwait_cstate * 4) & 9589f3d6dafSRafael J. Wysocki MWAIT_SUBSTATE_MASK; 9599f3d6dafSRafael J. Wysocki 9609f3d6dafSRafael J. Wysocki /* Ignore the C-state if there are NO sub-states in CPUID for it. */ 9619f3d6dafSRafael J. Wysocki if (num_substates == 0) 9629f3d6dafSRafael J. Wysocki return false; 9639f3d6dafSRafael J. Wysocki 9649f3d6dafSRafael J. Wysocki if (mwait_cstate > 2 && !boot_cpu_has(X86_FEATURE_NONSTOP_TSC)) 9659f3d6dafSRafael J. Wysocki mark_tsc_unstable("TSC halts in idle states deeper than C2"); 9669f3d6dafSRafael J. Wysocki 9679f3d6dafSRafael J. Wysocki return true; 9689f3d6dafSRafael J. Wysocki } 9699f3d6dafSRafael J. Wysocki 970fb1013a0SSebastian Andrzej Siewior static void auto_demotion_disable(void) 97114796fcaSLen Brown { 97214796fcaSLen Brown unsigned long long msr_bits; 97314796fcaSLen Brown 9746cfb2374SLen Brown rdmsrl(MSR_PKG_CST_CONFIG_CONTROL, msr_bits); 975b66b8b9aSAndi Kleen msr_bits &= ~(icpu->auto_demotion_disable_flags); 9766cfb2374SLen Brown wrmsrl(MSR_PKG_CST_CONFIG_CONTROL, msr_bits); 97714796fcaSLen Brown } 978fb1013a0SSebastian Andrzej Siewior static void c1e_promotion_disable(void) 97932e95180SLen Brown { 98032e95180SLen Brown unsigned long long msr_bits; 98132e95180SLen Brown 98232e95180SLen Brown rdmsrl(MSR_IA32_POWER_CTL, msr_bits); 98332e95180SLen Brown msr_bits &= ~0x2; 98432e95180SLen Brown wrmsrl(MSR_IA32_POWER_CTL, msr_bits); 98532e95180SLen Brown } 98614796fcaSLen Brown 987b66b8b9aSAndi Kleen static const struct idle_cpu idle_cpu_nehalem = { 988b66b8b9aSAndi Kleen .state_table = nehalem_cstates, 989b66b8b9aSAndi Kleen .auto_demotion_disable_flags = NHM_C1_AUTO_DEMOTE | NHM_C3_AUTO_DEMOTE, 99032e95180SLen Brown .disable_promotion_to_c1e = true, 991b66b8b9aSAndi Kleen }; 992b66b8b9aSAndi Kleen 993e6d4f08aSRafael J. Wysocki static const struct idle_cpu idle_cpu_nhx = { 994e6d4f08aSRafael J. Wysocki .state_table = nehalem_cstates, 995e6d4f08aSRafael J. Wysocki .auto_demotion_disable_flags = NHM_C1_AUTO_DEMOTE | NHM_C3_AUTO_DEMOTE, 996e6d4f08aSRafael J. Wysocki .disable_promotion_to_c1e = true, 997e6d4f08aSRafael J. Wysocki .use_acpi = true, 998e6d4f08aSRafael J. Wysocki }; 999e6d4f08aSRafael J. Wysocki 1000b66b8b9aSAndi Kleen static const struct idle_cpu idle_cpu_atom = { 1001b66b8b9aSAndi Kleen .state_table = atom_cstates, 1002b66b8b9aSAndi Kleen }; 1003b66b8b9aSAndi Kleen 10045e7ec268SAndy Shevchenko static const struct idle_cpu idle_cpu_tangier = { 10055e7ec268SAndy Shevchenko .state_table = tangier_cstates, 10065e7ec268SAndy Shevchenko }; 10075e7ec268SAndy Shevchenko 1008b66b8b9aSAndi Kleen static const struct idle_cpu idle_cpu_lincroft = { 1009b66b8b9aSAndi Kleen .state_table = atom_cstates, 1010b66b8b9aSAndi Kleen .auto_demotion_disable_flags = ATM_LNC_C6_AUTO_DEMOTE, 1011b66b8b9aSAndi Kleen }; 1012b66b8b9aSAndi Kleen 1013b66b8b9aSAndi Kleen static const struct idle_cpu idle_cpu_snb = { 1014b66b8b9aSAndi Kleen .state_table = snb_cstates, 101532e95180SLen Brown .disable_promotion_to_c1e = true, 1016b66b8b9aSAndi Kleen }; 1017b66b8b9aSAndi Kleen 1018e6d4f08aSRafael J. Wysocki static const struct idle_cpu idle_cpu_snx = { 1019e6d4f08aSRafael J. Wysocki .state_table = snb_cstates, 1020e6d4f08aSRafael J. Wysocki .disable_promotion_to_c1e = true, 1021e6d4f08aSRafael J. Wysocki .use_acpi = true, 1022e6d4f08aSRafael J. Wysocki }; 1023e6d4f08aSRafael J. Wysocki 1024718987d6SLen Brown static const struct idle_cpu idle_cpu_byt = { 1025718987d6SLen Brown .state_table = byt_cstates, 1026718987d6SLen Brown .disable_promotion_to_c1e = true, 10278c058d53SLen Brown .byt_auto_demotion_disable_flag = true, 1028718987d6SLen Brown }; 1029718987d6SLen Brown 1030cab07a56SLen Brown static const struct idle_cpu idle_cpu_cht = { 1031cab07a56SLen Brown .state_table = cht_cstates, 1032cab07a56SLen Brown .disable_promotion_to_c1e = true, 1033cab07a56SLen Brown .byt_auto_demotion_disable_flag = true, 1034cab07a56SLen Brown }; 1035cab07a56SLen Brown 10366edab08cSLen Brown static const struct idle_cpu idle_cpu_ivb = { 10376edab08cSLen Brown .state_table = ivb_cstates, 103832e95180SLen Brown .disable_promotion_to_c1e = true, 10396edab08cSLen Brown }; 10406edab08cSLen Brown 10410138d8f0SLen Brown static const struct idle_cpu idle_cpu_ivt = { 10420138d8f0SLen Brown .state_table = ivt_cstates, 10430138d8f0SLen Brown .disable_promotion_to_c1e = true, 1044e6d4f08aSRafael J. Wysocki .use_acpi = true, 10450138d8f0SLen Brown }; 10460138d8f0SLen Brown 104785a4d2d4SLen Brown static const struct idle_cpu idle_cpu_hsw = { 104885a4d2d4SLen Brown .state_table = hsw_cstates, 104932e95180SLen Brown .disable_promotion_to_c1e = true, 105085a4d2d4SLen Brown }; 105185a4d2d4SLen Brown 1052e6d4f08aSRafael J. Wysocki static const struct idle_cpu idle_cpu_hsx = { 1053e6d4f08aSRafael J. Wysocki .state_table = hsw_cstates, 1054e6d4f08aSRafael J. Wysocki .disable_promotion_to_c1e = true, 1055e6d4f08aSRafael J. Wysocki .use_acpi = true, 1056e6d4f08aSRafael J. Wysocki }; 1057e6d4f08aSRafael J. Wysocki 1058a138b568SLen Brown static const struct idle_cpu idle_cpu_bdw = { 1059a138b568SLen Brown .state_table = bdw_cstates, 1060a138b568SLen Brown .disable_promotion_to_c1e = true, 1061a138b568SLen Brown }; 1062a138b568SLen Brown 1063e6d4f08aSRafael J. Wysocki static const struct idle_cpu idle_cpu_bdx = { 1064e6d4f08aSRafael J. Wysocki .state_table = bdw_cstates, 1065e6d4f08aSRafael J. Wysocki .disable_promotion_to_c1e = true, 1066e6d4f08aSRafael J. Wysocki .use_acpi = true, 1067e6d4f08aSRafael J. Wysocki }; 1068e6d4f08aSRafael J. Wysocki 1069493f133fSLen Brown static const struct idle_cpu idle_cpu_skl = { 1070493f133fSLen Brown .state_table = skl_cstates, 1071493f133fSLen Brown .disable_promotion_to_c1e = true, 1072493f133fSLen Brown }; 1073493f133fSLen Brown 1074f9e71657SLen Brown static const struct idle_cpu idle_cpu_skx = { 1075f9e71657SLen Brown .state_table = skx_cstates, 1076f9e71657SLen Brown .disable_promotion_to_c1e = true, 1077e6d4f08aSRafael J. Wysocki .use_acpi = true, 1078f9e71657SLen Brown }; 1079493f133fSLen Brown 1080fab04b22SLen Brown static const struct idle_cpu idle_cpu_avn = { 1081fab04b22SLen Brown .state_table = avn_cstates, 1082fab04b22SLen Brown .disable_promotion_to_c1e = true, 1083e6d4f08aSRafael J. Wysocki .use_acpi = true, 1084fab04b22SLen Brown }; 1085fab04b22SLen Brown 1086281baf7aSDasaratharaman Chandramouli static const struct idle_cpu idle_cpu_knl = { 1087281baf7aSDasaratharaman Chandramouli .state_table = knl_cstates, 1088e6d4f08aSRafael J. Wysocki .use_acpi = true, 1089281baf7aSDasaratharaman Chandramouli }; 1090281baf7aSDasaratharaman Chandramouli 10915dcef694SLen Brown static const struct idle_cpu idle_cpu_bxt = { 10925dcef694SLen Brown .state_table = bxt_cstates, 10935dcef694SLen Brown .disable_promotion_to_c1e = true, 10945dcef694SLen Brown }; 10955dcef694SLen Brown 10960080d65bSJacob Pan static const struct idle_cpu idle_cpu_dnv = { 10970080d65bSJacob Pan .state_table = dnv_cstates, 10980080d65bSJacob Pan .disable_promotion_to_c1e = true, 1099e6d4f08aSRafael J. Wysocki .use_acpi = true, 11000080d65bSJacob Pan }; 11010080d65bSJacob Pan 1102d5cdc3c4SMathias Krause static const struct x86_cpu_id intel_idle_ids[] __initconst = { 1103e6d4f08aSRafael J. Wysocki INTEL_CPU_FAM6(NEHALEM_EP, idle_cpu_nhx), 1104a4a008e5SAndy Shevchenko INTEL_CPU_FAM6(NEHALEM, idle_cpu_nehalem), 1105a4a008e5SAndy Shevchenko INTEL_CPU_FAM6(NEHALEM_G, idle_cpu_nehalem), 1106a4a008e5SAndy Shevchenko INTEL_CPU_FAM6(WESTMERE, idle_cpu_nehalem), 1107e6d4f08aSRafael J. Wysocki INTEL_CPU_FAM6(WESTMERE_EP, idle_cpu_nhx), 1108e6d4f08aSRafael J. Wysocki INTEL_CPU_FAM6(NEHALEM_EX, idle_cpu_nhx), 1109c05f3642SLinus Torvalds INTEL_CPU_FAM6(ATOM_BONNELL, idle_cpu_atom), 1110c05f3642SLinus Torvalds INTEL_CPU_FAM6(ATOM_BONNELL_MID, idle_cpu_lincroft), 1111e6d4f08aSRafael J. Wysocki INTEL_CPU_FAM6(WESTMERE_EX, idle_cpu_nhx), 1112a4a008e5SAndy Shevchenko INTEL_CPU_FAM6(SANDYBRIDGE, idle_cpu_snb), 1113e6d4f08aSRafael J. Wysocki INTEL_CPU_FAM6(SANDYBRIDGE_X, idle_cpu_snx), 1114c05f3642SLinus Torvalds INTEL_CPU_FAM6(ATOM_SALTWELL, idle_cpu_atom), 1115c05f3642SLinus Torvalds INTEL_CPU_FAM6(ATOM_SILVERMONT, idle_cpu_byt), 1116c05f3642SLinus Torvalds INTEL_CPU_FAM6(ATOM_SILVERMONT_MID, idle_cpu_tangier), 1117a4a008e5SAndy Shevchenko INTEL_CPU_FAM6(ATOM_AIRMONT, idle_cpu_cht), 1118a4a008e5SAndy Shevchenko INTEL_CPU_FAM6(IVYBRIDGE, idle_cpu_ivb), 1119a4a008e5SAndy Shevchenko INTEL_CPU_FAM6(IVYBRIDGE_X, idle_cpu_ivt), 1120c66f78a6SPeter Zijlstra INTEL_CPU_FAM6(HASWELL, idle_cpu_hsw), 1121e6d4f08aSRafael J. Wysocki INTEL_CPU_FAM6(HASWELL_X, idle_cpu_hsx), 1122af239c44SPeter Zijlstra INTEL_CPU_FAM6(HASWELL_L, idle_cpu_hsw), 11235e741407SPeter Zijlstra INTEL_CPU_FAM6(HASWELL_G, idle_cpu_hsw), 11245ebb34edSPeter Zijlstra INTEL_CPU_FAM6(ATOM_SILVERMONT_D, idle_cpu_avn), 1125c66f78a6SPeter Zijlstra INTEL_CPU_FAM6(BROADWELL, idle_cpu_bdw), 11265e741407SPeter Zijlstra INTEL_CPU_FAM6(BROADWELL_G, idle_cpu_bdw), 1127e6d4f08aSRafael J. Wysocki INTEL_CPU_FAM6(BROADWELL_X, idle_cpu_bdx), 1128e6d4f08aSRafael J. Wysocki INTEL_CPU_FAM6(BROADWELL_D, idle_cpu_bdx), 1129af239c44SPeter Zijlstra INTEL_CPU_FAM6(SKYLAKE_L, idle_cpu_skl), 1130c66f78a6SPeter Zijlstra INTEL_CPU_FAM6(SKYLAKE, idle_cpu_skl), 1131af239c44SPeter Zijlstra INTEL_CPU_FAM6(KABYLAKE_L, idle_cpu_skl), 1132c66f78a6SPeter Zijlstra INTEL_CPU_FAM6(KABYLAKE, idle_cpu_skl), 1133a4a008e5SAndy Shevchenko INTEL_CPU_FAM6(SKYLAKE_X, idle_cpu_skx), 1134a4a008e5SAndy Shevchenko INTEL_CPU_FAM6(XEON_PHI_KNL, idle_cpu_knl), 1135a4a008e5SAndy Shevchenko INTEL_CPU_FAM6(XEON_PHI_KNM, idle_cpu_knl), 1136a4a008e5SAndy Shevchenko INTEL_CPU_FAM6(ATOM_GOLDMONT, idle_cpu_bxt), 1137c05f3642SLinus Torvalds INTEL_CPU_FAM6(ATOM_GOLDMONT_PLUS, idle_cpu_bxt), 11385ebb34edSPeter Zijlstra INTEL_CPU_FAM6(ATOM_GOLDMONT_D, idle_cpu_dnv), 11395ebb34edSPeter Zijlstra INTEL_CPU_FAM6(ATOM_TREMONT_D, idle_cpu_dnv), 1140b66b8b9aSAndi Kleen {} 1141b66b8b9aSAndi Kleen }; 1142b66b8b9aSAndi Kleen 114318734958SRafael J. Wysocki #define INTEL_CPU_FAM6_MWAIT \ 114418734958SRafael J. Wysocki { X86_VENDOR_INTEL, 6, X86_MODEL_ANY, X86_FEATURE_MWAIT, 0 } 114518734958SRafael J. Wysocki 114618734958SRafael J. Wysocki static const struct x86_cpu_id intel_mwait_ids[] __initconst = { 114718734958SRafael J. Wysocki INTEL_CPU_FAM6_MWAIT, 114818734958SRafael J. Wysocki {} 114918734958SRafael J. Wysocki }; 115018734958SRafael J. Wysocki 115118734958SRafael J. Wysocki static bool intel_idle_max_cstate_reached(int cstate) 115218734958SRafael J. Wysocki { 115318734958SRafael J. Wysocki if (cstate + 1 > max_cstate) { 115418734958SRafael J. Wysocki pr_info("max_cstate %d reached\n", max_cstate); 115518734958SRafael J. Wysocki return true; 115618734958SRafael J. Wysocki } 115718734958SRafael J. Wysocki return false; 115818734958SRafael J. Wysocki } 115918734958SRafael J. Wysocki 116018734958SRafael J. Wysocki #ifdef CONFIG_ACPI_PROCESSOR_CSTATE 116118734958SRafael J. Wysocki #include <acpi/processor.h> 116218734958SRafael J. Wysocki 11634ec32d9eSRafael J. Wysocki static bool no_acpi __read_mostly; 11644ec32d9eSRafael J. Wysocki module_param(no_acpi, bool, 0444); 11654ec32d9eSRafael J. Wysocki MODULE_PARM_DESC(no_acpi, "Do not use ACPI _CST for building the idle states list"); 11664ec32d9eSRafael J. Wysocki 116718734958SRafael J. Wysocki static struct acpi_processor_power acpi_state_table; 116818734958SRafael J. Wysocki 116918734958SRafael J. Wysocki /** 117018734958SRafael J. Wysocki * intel_idle_cst_usable - Check if the _CST information can be used. 117118734958SRafael J. Wysocki * 117218734958SRafael J. Wysocki * Check if all of the C-states listed by _CST in the max_cstate range are 117318734958SRafael J. Wysocki * ACPI_CSTATE_FFH, which means that they should be entered via MWAIT. 117418734958SRafael J. Wysocki */ 117518734958SRafael J. Wysocki static bool intel_idle_cst_usable(void) 117618734958SRafael J. Wysocki { 117718734958SRafael J. Wysocki int cstate, limit; 117818734958SRafael J. Wysocki 117918734958SRafael J. Wysocki limit = min_t(int, min_t(int, CPUIDLE_STATE_MAX, max_cstate + 1), 118018734958SRafael J. Wysocki acpi_state_table.count); 118118734958SRafael J. Wysocki 118218734958SRafael J. Wysocki for (cstate = 1; cstate < limit; cstate++) { 118318734958SRafael J. Wysocki struct acpi_processor_cx *cx = &acpi_state_table.states[cstate]; 118418734958SRafael J. Wysocki 118518734958SRafael J. Wysocki if (cx->entry_method != ACPI_CSTATE_FFH) 118618734958SRafael J. Wysocki return false; 118718734958SRafael J. Wysocki } 118818734958SRafael J. Wysocki 118918734958SRafael J. Wysocki return true; 119018734958SRafael J. Wysocki } 119118734958SRafael J. Wysocki 119218734958SRafael J. Wysocki static bool intel_idle_acpi_cst_extract(void) 119318734958SRafael J. Wysocki { 119418734958SRafael J. Wysocki unsigned int cpu; 119518734958SRafael J. Wysocki 11964ec32d9eSRafael J. Wysocki if (no_acpi) { 11974ec32d9eSRafael J. Wysocki pr_debug("Not allowed to use ACPI _CST\n"); 11984ec32d9eSRafael J. Wysocki return false; 11994ec32d9eSRafael J. Wysocki } 12004ec32d9eSRafael J. Wysocki 120118734958SRafael J. Wysocki for_each_possible_cpu(cpu) { 120218734958SRafael J. Wysocki struct acpi_processor *pr = per_cpu(processors, cpu); 120318734958SRafael J. Wysocki 120418734958SRafael J. Wysocki if (!pr) 120518734958SRafael J. Wysocki continue; 120618734958SRafael J. Wysocki 120718734958SRafael J. Wysocki if (acpi_processor_evaluate_cst(pr->handle, cpu, &acpi_state_table)) 120818734958SRafael J. Wysocki continue; 120918734958SRafael J. Wysocki 121018734958SRafael J. Wysocki acpi_state_table.count++; 121118734958SRafael J. Wysocki 121218734958SRafael J. Wysocki if (!intel_idle_cst_usable()) 121318734958SRafael J. Wysocki continue; 121418734958SRafael J. Wysocki 121518734958SRafael J. Wysocki if (!acpi_processor_claim_cst_control()) { 121618734958SRafael J. Wysocki acpi_state_table.count = 0; 121718734958SRafael J. Wysocki return false; 121818734958SRafael J. Wysocki } 121918734958SRafael J. Wysocki 122018734958SRafael J. Wysocki return true; 122118734958SRafael J. Wysocki } 122218734958SRafael J. Wysocki 122318734958SRafael J. Wysocki pr_debug("ACPI _CST not found or not usable\n"); 122418734958SRafael J. Wysocki return false; 122518734958SRafael J. Wysocki } 122618734958SRafael J. Wysocki 122718734958SRafael J. Wysocki static void intel_idle_init_cstates_acpi(struct cpuidle_driver *drv) 122818734958SRafael J. Wysocki { 122918734958SRafael J. Wysocki int cstate, limit = min_t(int, CPUIDLE_STATE_MAX, acpi_state_table.count); 123018734958SRafael J. Wysocki 123118734958SRafael J. Wysocki /* 123218734958SRafael J. Wysocki * If limit > 0, intel_idle_cst_usable() has returned 'true', so all of 123318734958SRafael J. Wysocki * the interesting states are ACPI_CSTATE_FFH. 123418734958SRafael J. Wysocki */ 123518734958SRafael J. Wysocki for (cstate = 1; cstate < limit; cstate++) { 123618734958SRafael J. Wysocki struct acpi_processor_cx *cx; 123718734958SRafael J. Wysocki struct cpuidle_state *state; 123818734958SRafael J. Wysocki 123918734958SRafael J. Wysocki if (intel_idle_max_cstate_reached(cstate)) 124018734958SRafael J. Wysocki break; 124118734958SRafael J. Wysocki 124218734958SRafael J. Wysocki cx = &acpi_state_table.states[cstate]; 124318734958SRafael J. Wysocki 124418734958SRafael J. Wysocki state = &drv->states[drv->state_count++]; 124518734958SRafael J. Wysocki 124618734958SRafael J. Wysocki snprintf(state->name, CPUIDLE_NAME_LEN, "C%d_ACPI", cstate); 124718734958SRafael J. Wysocki strlcpy(state->desc, cx->desc, CPUIDLE_DESC_LEN); 124818734958SRafael J. Wysocki state->exit_latency = cx->latency; 124918734958SRafael J. Wysocki /* 125018734958SRafael J. Wysocki * For C1-type C-states use the same number for both the exit 125118734958SRafael J. Wysocki * latency and target residency, because that is the case for 125218734958SRafael J. Wysocki * C1 in the majority of the static C-states tables above. 125318734958SRafael J. Wysocki * For the other types of C-states, however, set the target 125418734958SRafael J. Wysocki * residency to 3 times the exit latency which should lead to 125518734958SRafael J. Wysocki * a reasonable balance between energy-efficiency and 125618734958SRafael J. Wysocki * performance in the majority of interesting cases. 125718734958SRafael J. Wysocki */ 125818734958SRafael J. Wysocki state->target_residency = cx->latency; 125918734958SRafael J. Wysocki if (cx->type > ACPI_STATE_C1) 126018734958SRafael J. Wysocki state->target_residency *= 3; 126118734958SRafael J. Wysocki 126218734958SRafael J. Wysocki state->flags = MWAIT2flg(cx->address); 126318734958SRafael J. Wysocki if (cx->type > ACPI_STATE_C2) 126418734958SRafael J. Wysocki state->flags |= CPUIDLE_FLAG_TLB_FLUSHED; 126518734958SRafael J. Wysocki 126618734958SRafael J. Wysocki state->enter = intel_idle; 126718734958SRafael J. Wysocki state->enter_s2idle = intel_idle_s2idle; 126818734958SRafael J. Wysocki } 126918734958SRafael J. Wysocki } 1270bff8e60aSRafael J. Wysocki 1271bff8e60aSRafael J. Wysocki static bool intel_idle_off_by_default(u32 mwait_hint) 1272bff8e60aSRafael J. Wysocki { 1273bff8e60aSRafael J. Wysocki int cstate, limit; 1274bff8e60aSRafael J. Wysocki 1275bff8e60aSRafael J. Wysocki /* 1276bff8e60aSRafael J. Wysocki * If there are no _CST C-states, do not disable any C-states by 1277bff8e60aSRafael J. Wysocki * default. 1278bff8e60aSRafael J. Wysocki */ 1279bff8e60aSRafael J. Wysocki if (!acpi_state_table.count) 1280bff8e60aSRafael J. Wysocki return false; 1281bff8e60aSRafael J. Wysocki 1282bff8e60aSRafael J. Wysocki limit = min_t(int, CPUIDLE_STATE_MAX, acpi_state_table.count); 1283bff8e60aSRafael J. Wysocki /* 1284bff8e60aSRafael J. Wysocki * If limit > 0, intel_idle_cst_usable() has returned 'true', so all of 1285bff8e60aSRafael J. Wysocki * the interesting states are ACPI_CSTATE_FFH. 1286bff8e60aSRafael J. Wysocki */ 1287bff8e60aSRafael J. Wysocki for (cstate = 1; cstate < limit; cstate++) { 1288bff8e60aSRafael J. Wysocki if (acpi_state_table.states[cstate].address == mwait_hint) 1289bff8e60aSRafael J. Wysocki return false; 1290bff8e60aSRafael J. Wysocki } 1291bff8e60aSRafael J. Wysocki return true; 1292bff8e60aSRafael J. Wysocki } 129318734958SRafael J. Wysocki #else /* !CONFIG_ACPI_PROCESSOR_CSTATE */ 129418734958SRafael J. Wysocki static inline bool intel_idle_acpi_cst_extract(void) { return false; } 129518734958SRafael J. Wysocki static inline void intel_idle_init_cstates_acpi(struct cpuidle_driver *drv) { } 1296bff8e60aSRafael J. Wysocki static inline bool intel_idle_off_by_default(u32 mwait_hint) { return false; } 129718734958SRafael J. Wysocki #endif /* !CONFIG_ACPI_PROCESSOR_CSTATE */ 129818734958SRafael J. Wysocki 129926717172SLen Brown /* 130026717172SLen Brown * intel_idle_cpuidle_devices_uninit() 1301ca42489dSRichard Cochran * Unregisters the cpuidle devices. 130226717172SLen Brown */ 130326717172SLen Brown static void intel_idle_cpuidle_devices_uninit(void) 130426717172SLen Brown { 130526717172SLen Brown int i; 130626717172SLen Brown struct cpuidle_device *dev; 130726717172SLen Brown 130826717172SLen Brown for_each_online_cpu(i) { 130926717172SLen Brown dev = per_cpu_ptr(intel_idle_cpuidle_devices, i); 131026717172SLen Brown cpuidle_unregister_device(dev); 131126717172SLen Brown } 131226717172SLen Brown } 13130138d8f0SLen Brown 13140138d8f0SLen Brown /* 1315d70e28f5SLen Brown * ivt_idle_state_table_update(void) 13160138d8f0SLen Brown * 1317d70e28f5SLen Brown * Tune IVT multi-socket targets 13180138d8f0SLen Brown * Assumption: num_sockets == (max_package_num + 1) 13190138d8f0SLen Brown */ 1320d70e28f5SLen Brown static void ivt_idle_state_table_update(void) 13210138d8f0SLen Brown { 13220138d8f0SLen Brown /* IVT uses a different table for 1-2, 3-4, and > 4 sockets */ 13230138d8f0SLen Brown int cpu, package_num, num_sockets = 1; 13240138d8f0SLen Brown 13250138d8f0SLen Brown for_each_online_cpu(cpu) { 13260138d8f0SLen Brown package_num = topology_physical_package_id(cpu); 13270138d8f0SLen Brown if (package_num + 1 > num_sockets) { 13280138d8f0SLen Brown num_sockets = package_num + 1; 13290138d8f0SLen Brown 1330d27dca42SChristoph Jaeger if (num_sockets > 4) { 13310138d8f0SLen Brown cpuidle_state_table = ivt_cstates_8s; 13320138d8f0SLen Brown return; 13330138d8f0SLen Brown } 13340138d8f0SLen Brown } 1335d27dca42SChristoph Jaeger } 13360138d8f0SLen Brown 13370138d8f0SLen Brown if (num_sockets > 2) 13380138d8f0SLen Brown cpuidle_state_table = ivt_cstates_4s; 1339d70e28f5SLen Brown 13400138d8f0SLen Brown /* else, 1 and 2 socket systems use default ivt_cstates */ 13410138d8f0SLen Brown } 13425dcef694SLen Brown 13435dcef694SLen Brown /* 13445dcef694SLen Brown * Translate IRTL (Interrupt Response Time Limit) MSR to usec 13455dcef694SLen Brown */ 13465dcef694SLen Brown 13475dcef694SLen Brown static unsigned int irtl_ns_units[] = { 13485dcef694SLen Brown 1, 32, 1024, 32768, 1048576, 33554432, 0, 0 }; 13495dcef694SLen Brown 13505dcef694SLen Brown static unsigned long long irtl_2_usec(unsigned long long irtl) 13515dcef694SLen Brown { 13525dcef694SLen Brown unsigned long long ns; 13535dcef694SLen Brown 13543451ab3eSJan Beulich if (!irtl) 13553451ab3eSJan Beulich return 0; 13563451ab3eSJan Beulich 1357bef45096SJan Beulich ns = irtl_ns_units[(irtl >> 10) & 0x7]; 13585dcef694SLen Brown 13595dcef694SLen Brown return div64_u64((irtl & 0x3FF) * ns, 1000); 13605dcef694SLen Brown } 13615dcef694SLen Brown /* 13625dcef694SLen Brown * bxt_idle_state_table_update(void) 13635dcef694SLen Brown * 13645dcef694SLen Brown * On BXT, we trust the IRTL to show the definitive maximum latency 13655dcef694SLen Brown * We use the same value for target_residency. 13665dcef694SLen Brown */ 13675dcef694SLen Brown static void bxt_idle_state_table_update(void) 13685dcef694SLen Brown { 13695dcef694SLen Brown unsigned long long msr; 13703451ab3eSJan Beulich unsigned int usec; 13715dcef694SLen Brown 13725dcef694SLen Brown rdmsrl(MSR_PKGC6_IRTL, msr); 13733451ab3eSJan Beulich usec = irtl_2_usec(msr); 13743451ab3eSJan Beulich if (usec) { 13755dcef694SLen Brown bxt_cstates[2].exit_latency = usec; 13765dcef694SLen Brown bxt_cstates[2].target_residency = usec; 13775dcef694SLen Brown } 13785dcef694SLen Brown 13795dcef694SLen Brown rdmsrl(MSR_PKGC7_IRTL, msr); 13803451ab3eSJan Beulich usec = irtl_2_usec(msr); 13813451ab3eSJan Beulich if (usec) { 13825dcef694SLen Brown bxt_cstates[3].exit_latency = usec; 13835dcef694SLen Brown bxt_cstates[3].target_residency = usec; 13845dcef694SLen Brown } 13855dcef694SLen Brown 13865dcef694SLen Brown rdmsrl(MSR_PKGC8_IRTL, msr); 13873451ab3eSJan Beulich usec = irtl_2_usec(msr); 13883451ab3eSJan Beulich if (usec) { 13895dcef694SLen Brown bxt_cstates[4].exit_latency = usec; 13905dcef694SLen Brown bxt_cstates[4].target_residency = usec; 13915dcef694SLen Brown } 13925dcef694SLen Brown 13935dcef694SLen Brown rdmsrl(MSR_PKGC9_IRTL, msr); 13943451ab3eSJan Beulich usec = irtl_2_usec(msr); 13953451ab3eSJan Beulich if (usec) { 13965dcef694SLen Brown bxt_cstates[5].exit_latency = usec; 13975dcef694SLen Brown bxt_cstates[5].target_residency = usec; 13985dcef694SLen Brown } 13995dcef694SLen Brown 14005dcef694SLen Brown rdmsrl(MSR_PKGC10_IRTL, msr); 14013451ab3eSJan Beulich usec = irtl_2_usec(msr); 14023451ab3eSJan Beulich if (usec) { 14035dcef694SLen Brown bxt_cstates[6].exit_latency = usec; 14045dcef694SLen Brown bxt_cstates[6].target_residency = usec; 14055dcef694SLen Brown } 14065dcef694SLen Brown 14075dcef694SLen Brown } 1408d70e28f5SLen Brown /* 1409d70e28f5SLen Brown * sklh_idle_state_table_update(void) 1410d70e28f5SLen Brown * 1411d70e28f5SLen Brown * On SKL-H (model 0x5e) disable C8 and C9 if: 1412d70e28f5SLen Brown * C10 is enabled and SGX disabled 1413d70e28f5SLen Brown */ 1414d70e28f5SLen Brown static void sklh_idle_state_table_update(void) 1415d70e28f5SLen Brown { 1416d70e28f5SLen Brown unsigned long long msr; 1417d70e28f5SLen Brown unsigned int eax, ebx, ecx, edx; 1418d70e28f5SLen Brown 1419d70e28f5SLen Brown 1420d70e28f5SLen Brown /* if PC10 disabled via cmdline intel_idle.max_cstate=7 or shallower */ 1421d70e28f5SLen Brown if (max_cstate <= 7) 14220138d8f0SLen Brown return; 1423d70e28f5SLen Brown 1424d70e28f5SLen Brown /* if PC10 not present in CPUID.MWAIT.EDX */ 1425d70e28f5SLen Brown if ((mwait_substates & (0xF << 28)) == 0) 1426d70e28f5SLen Brown return; 1427d70e28f5SLen Brown 14286cfb2374SLen Brown rdmsrl(MSR_PKG_CST_CONFIG_CONTROL, msr); 1429d70e28f5SLen Brown 1430d70e28f5SLen Brown /* PC10 is not enabled in PKG C-state limit */ 1431d70e28f5SLen Brown if ((msr & 0xF) != 8) 1432d70e28f5SLen Brown return; 1433d70e28f5SLen Brown 1434d70e28f5SLen Brown ecx = 0; 1435d70e28f5SLen Brown cpuid(7, &eax, &ebx, &ecx, &edx); 1436d70e28f5SLen Brown 1437d70e28f5SLen Brown /* if SGX is present */ 1438d70e28f5SLen Brown if (ebx & (1 << 2)) { 1439d70e28f5SLen Brown 1440d70e28f5SLen Brown rdmsrl(MSR_IA32_FEATURE_CONTROL, msr); 1441d70e28f5SLen Brown 1442d70e28f5SLen Brown /* if SGX is enabled */ 1443d70e28f5SLen Brown if (msr & (1 << 18)) 1444d70e28f5SLen Brown return; 1445d70e28f5SLen Brown } 1446d70e28f5SLen Brown 1447ba1e78a1SRafael J. Wysocki skl_cstates[5].flags |= CPUIDLE_FLAG_UNUSABLE; /* C8-SKL */ 1448ba1e78a1SRafael J. Wysocki skl_cstates[6].flags |= CPUIDLE_FLAG_UNUSABLE; /* C9-SKL */ 1449d70e28f5SLen Brown } 1450d70e28f5SLen Brown /* 1451d70e28f5SLen Brown * intel_idle_state_table_update() 1452d70e28f5SLen Brown * 1453d70e28f5SLen Brown * Update the default state_table for this CPU-id 1454d70e28f5SLen Brown */ 1455d70e28f5SLen Brown 1456d70e28f5SLen Brown static void intel_idle_state_table_update(void) 1457d70e28f5SLen Brown { 1458d70e28f5SLen Brown switch (boot_cpu_data.x86_model) { 1459d70e28f5SLen Brown 1460db73c5a8SDave Hansen case INTEL_FAM6_IVYBRIDGE_X: 1461d70e28f5SLen Brown ivt_idle_state_table_update(); 1462d70e28f5SLen Brown break; 1463db73c5a8SDave Hansen case INTEL_FAM6_ATOM_GOLDMONT: 1464f2c4db1bSPeter Zijlstra case INTEL_FAM6_ATOM_GOLDMONT_PLUS: 14655dcef694SLen Brown bxt_idle_state_table_update(); 14665dcef694SLen Brown break; 1467c66f78a6SPeter Zijlstra case INTEL_FAM6_SKYLAKE: 1468d70e28f5SLen Brown sklh_idle_state_table_update(); 1469d70e28f5SLen Brown break; 1470d70e28f5SLen Brown } 14710138d8f0SLen Brown } 14720138d8f0SLen Brown 147318734958SRafael J. Wysocki static void intel_idle_init_cstates_icpu(struct cpuidle_driver *drv) 147446bcfad7SDeepthi Dharwar { 147546bcfad7SDeepthi Dharwar int cstate; 147646bcfad7SDeepthi Dharwar 1477e022e7ebSLen Brown for (cstate = 0; cstate < CPUIDLE_STATE_MAX; ++cstate) { 14789f3d6dafSRafael J. Wysocki unsigned int mwait_hint; 147946bcfad7SDeepthi Dharwar 148018734958SRafael J. Wysocki if (intel_idle_max_cstate_reached(cstate)) 148118734958SRafael J. Wysocki break; 148218734958SRafael J. Wysocki 14839f3d6dafSRafael J. Wysocki if (!cpuidle_state_table[cstate].enter && 14849f3d6dafSRafael J. Wysocki !cpuidle_state_table[cstate].enter_s2idle) 1485e022e7ebSLen Brown break; 1486e022e7ebSLen Brown 14879f3d6dafSRafael J. Wysocki /* If marked as unusable, skip this state. */ 1488ba1e78a1SRafael J. Wysocki if (cpuidle_state_table[cstate].flags & CPUIDLE_FLAG_UNUSABLE) { 1489654d08a4SJoe Perches pr_debug("state %s is disabled\n", 1490d70e28f5SLen Brown cpuidle_state_table[cstate].name); 1491d70e28f5SLen Brown continue; 1492d70e28f5SLen Brown } 1493d70e28f5SLen Brown 14949f3d6dafSRafael J. Wysocki mwait_hint = flg2MWAIT(cpuidle_state_table[cstate].flags); 14959f3d6dafSRafael J. Wysocki if (!intel_idle_verify_cstate(mwait_hint)) 14969f3d6dafSRafael J. Wysocki continue; 1497d70e28f5SLen Brown 14989f3d6dafSRafael J. Wysocki /* Structure copy. */ 1499bff8e60aSRafael J. Wysocki drv->states[drv->state_count] = cpuidle_state_table[cstate]; 1500bff8e60aSRafael J. Wysocki 1501bff8e60aSRafael J. Wysocki if (icpu->use_acpi && intel_idle_off_by_default(mwait_hint) && 1502bff8e60aSRafael J. Wysocki !(cpuidle_state_table[cstate].flags & CPUIDLE_FLAG_ALWAYS_ENABLE)) 1503bff8e60aSRafael J. Wysocki drv->states[drv->state_count].flags |= CPUIDLE_FLAG_OFF; 1504bff8e60aSRafael J. Wysocki 1505bff8e60aSRafael J. Wysocki drv->state_count++; 150646bcfad7SDeepthi Dharwar } 150746bcfad7SDeepthi Dharwar 15088c058d53SLen Brown if (icpu->byt_auto_demotion_disable_flag) { 15098c058d53SLen Brown wrmsrl(MSR_CC6_DEMOTION_POLICY_CONFIG, 0); 15108c058d53SLen Brown wrmsrl(MSR_MC6_DEMOTION_POLICY_CONFIG, 0); 15118c058d53SLen Brown } 151246bcfad7SDeepthi Dharwar } 151346bcfad7SDeepthi Dharwar 151418734958SRafael J. Wysocki /* 151518734958SRafael J. Wysocki * intel_idle_cpuidle_driver_init() 151618734958SRafael J. Wysocki * allocate, initialize cpuidle_states 151718734958SRafael J. Wysocki */ 151818734958SRafael J. Wysocki static void __init intel_idle_cpuidle_driver_init(void) 151918734958SRafael J. Wysocki { 152018734958SRafael J. Wysocki struct cpuidle_driver *drv = &intel_idle_driver; 152118734958SRafael J. Wysocki 152218734958SRafael J. Wysocki intel_idle_state_table_update(); 152318734958SRafael J. Wysocki 152418734958SRafael J. Wysocki cpuidle_poll_state_init(drv); 152518734958SRafael J. Wysocki drv->state_count = 1; 152618734958SRafael J. Wysocki 152718734958SRafael J. Wysocki if (icpu) 152818734958SRafael J. Wysocki intel_idle_init_cstates_icpu(drv); 152918734958SRafael J. Wysocki else 153018734958SRafael J. Wysocki intel_idle_init_cstates_acpi(drv); 153118734958SRafael J. Wysocki } 153246bcfad7SDeepthi Dharwar 153346bcfad7SDeepthi Dharwar /* 153465b7f839SThomas Renninger * intel_idle_cpu_init() 153526717172SLen Brown * allocate, initialize, register cpuidle_devices 153665b7f839SThomas Renninger * @cpu: cpu/core to initialize 153726717172SLen Brown */ 1538fb1013a0SSebastian Andrzej Siewior static int intel_idle_cpu_init(unsigned int cpu) 153926717172SLen Brown { 154026717172SLen Brown struct cpuidle_device *dev; 154126717172SLen Brown 154265b7f839SThomas Renninger dev = per_cpu_ptr(intel_idle_cpuidle_devices, cpu); 154365b7f839SThomas Renninger dev->cpu = cpu; 154426717172SLen Brown 154526717172SLen Brown if (cpuidle_register_device(dev)) { 1546654d08a4SJoe Perches pr_debug("cpuidle_register_device %d failed!\n", cpu); 154726717172SLen Brown return -EIO; 154826717172SLen Brown } 154965b7f839SThomas Renninger 155018734958SRafael J. Wysocki if (!icpu) 155118734958SRafael J. Wysocki return 0; 155218734958SRafael J. Wysocki 1553b66b8b9aSAndi Kleen if (icpu->auto_demotion_disable_flags) 1554fb1013a0SSebastian Andrzej Siewior auto_demotion_disable(); 155526717172SLen Brown 1556dbf87ab8SBartlomiej Zolnierkiewicz if (icpu->disable_promotion_to_c1e) 1557fb1013a0SSebastian Andrzej Siewior c1e_promotion_disable(); 1558fb1013a0SSebastian Andrzej Siewior 1559fb1013a0SSebastian Andrzej Siewior return 0; 1560fb1013a0SSebastian Andrzej Siewior } 1561fb1013a0SSebastian Andrzej Siewior 1562fb1013a0SSebastian Andrzej Siewior static int intel_idle_cpu_online(unsigned int cpu) 1563fb1013a0SSebastian Andrzej Siewior { 1564fb1013a0SSebastian Andrzej Siewior struct cpuidle_device *dev; 1565fb1013a0SSebastian Andrzej Siewior 1566fb1013a0SSebastian Andrzej Siewior if (lapic_timer_reliable_states != LAPIC_TIMER_ALWAYS_RELIABLE) 1567cbd2c4c2SRafael J. Wysocki tick_broadcast_enable(); 1568fb1013a0SSebastian Andrzej Siewior 1569fb1013a0SSebastian Andrzej Siewior /* 1570fb1013a0SSebastian Andrzej Siewior * Some systems can hotplug a cpu at runtime after 1571fb1013a0SSebastian Andrzej Siewior * the kernel has booted, we have to initialize the 1572fb1013a0SSebastian Andrzej Siewior * driver in this case 1573fb1013a0SSebastian Andrzej Siewior */ 1574fb1013a0SSebastian Andrzej Siewior dev = per_cpu_ptr(intel_idle_cpuidle_devices, cpu); 1575fb1013a0SSebastian Andrzej Siewior if (!dev->registered) 1576fb1013a0SSebastian Andrzej Siewior return intel_idle_cpu_init(cpu); 1577dbf87ab8SBartlomiej Zolnierkiewicz 157826717172SLen Brown return 0; 157926717172SLen Brown } 158026717172SLen Brown 158126717172SLen Brown static int __init intel_idle_init(void) 158226717172SLen Brown { 1583a6c86e33SRafael J. Wysocki const struct x86_cpu_id *id; 1584a6c86e33SRafael J. Wysocki unsigned int eax, ebx, ecx; 1585fb1013a0SSebastian Andrzej Siewior int retval; 158626717172SLen Brown 1587d1896049SThomas Renninger /* Do not load intel_idle at all for now if idle= is passed */ 1588d1896049SThomas Renninger if (boot_option_idle_override != IDLE_NO_OVERRIDE) 1589d1896049SThomas Renninger return -ENODEV; 1590d1896049SThomas Renninger 1591a6c86e33SRafael J. Wysocki if (max_cstate == 0) { 1592a6c86e33SRafael J. Wysocki pr_debug("disabled\n"); 1593a6c86e33SRafael J. Wysocki return -EPERM; 1594a6c86e33SRafael J. Wysocki } 1595a6c86e33SRafael J. Wysocki 1596a6c86e33SRafael J. Wysocki id = x86_match_cpu(intel_idle_ids); 1597a6c86e33SRafael J. Wysocki if (id) { 1598a6c86e33SRafael J. Wysocki if (!boot_cpu_has(X86_FEATURE_MWAIT)) { 1599a6c86e33SRafael J. Wysocki pr_debug("Please enable MWAIT in BIOS SETUP\n"); 1600a6c86e33SRafael J. Wysocki return -ENODEV; 1601a6c86e33SRafael J. Wysocki } 1602a6c86e33SRafael J. Wysocki } else { 1603a6c86e33SRafael J. Wysocki id = x86_match_cpu(intel_mwait_ids); 1604a6c86e33SRafael J. Wysocki if (!id) 1605a6c86e33SRafael J. Wysocki return -ENODEV; 1606a6c86e33SRafael J. Wysocki } 1607a6c86e33SRafael J. Wysocki 1608a6c86e33SRafael J. Wysocki if (boot_cpu_data.cpuid_level < CPUID_MWAIT_LEAF) 1609a6c86e33SRafael J. Wysocki return -ENODEV; 1610a6c86e33SRafael J. Wysocki 1611a6c86e33SRafael J. Wysocki cpuid(CPUID_MWAIT_LEAF, &eax, &ebx, &ecx, &mwait_substates); 1612a6c86e33SRafael J. Wysocki 1613a6c86e33SRafael J. Wysocki if (!(ecx & CPUID5_ECX_EXTENSIONS_SUPPORTED) || 1614a6c86e33SRafael J. Wysocki !(ecx & CPUID5_ECX_INTERRUPT_BREAK) || 1615a6c86e33SRafael J. Wysocki !mwait_substates) 1616a6c86e33SRafael J. Wysocki return -ENODEV; 1617a6c86e33SRafael J. Wysocki 1618a6c86e33SRafael J. Wysocki pr_debug("MWAIT substates: 0x%x\n", mwait_substates); 1619a6c86e33SRafael J. Wysocki 1620a6c86e33SRafael J. Wysocki icpu = (const struct idle_cpu *)id->driver_data; 1621a6c86e33SRafael J. Wysocki if (icpu) { 1622a6c86e33SRafael J. Wysocki cpuidle_state_table = icpu->state_table; 1623a6c86e33SRafael J. Wysocki if (icpu->use_acpi) 1624a6c86e33SRafael J. Wysocki intel_idle_acpi_cst_extract(); 1625a6c86e33SRafael J. Wysocki } else if (!intel_idle_acpi_cst_extract()) { 1626a6c86e33SRafael J. Wysocki return -ENODEV; 1627a6c86e33SRafael J. Wysocki } 1628a6c86e33SRafael J. Wysocki 1629a6c86e33SRafael J. Wysocki pr_debug("v" INTEL_IDLE_VERSION " model 0x%X\n", 1630a6c86e33SRafael J. Wysocki boot_cpu_data.x86_model); 163126717172SLen Brown 1632e9df69ccSRichard Cochran intel_idle_cpuidle_devices = alloc_percpu(struct cpuidle_device); 1633e9df69ccSRichard Cochran if (intel_idle_cpuidle_devices == NULL) 1634e9df69ccSRichard Cochran return -ENOMEM; 1635e9df69ccSRichard Cochran 163646bcfad7SDeepthi Dharwar intel_idle_cpuidle_driver_init(); 163726717172SLen Brown retval = cpuidle_register_driver(&intel_idle_driver); 163826717172SLen Brown if (retval) { 16393735d524SKonrad Rzeszutek Wilk struct cpuidle_driver *drv = cpuidle_get_driver(); 1640654d08a4SJoe Perches printk(KERN_DEBUG pr_fmt("intel_idle yielding to %s\n"), 16413735d524SKonrad Rzeszutek Wilk drv ? drv->name : "none"); 1642fb1013a0SSebastian Andrzej Siewior goto init_driver_fail; 164326717172SLen Brown } 164426717172SLen Brown 16452259a819SRichard Cochran if (boot_cpu_has(X86_FEATURE_ARAT)) /* Always Reliable APIC Timer */ 16462259a819SRichard Cochran lapic_timer_reliable_states = LAPIC_TIMER_ALWAYS_RELIABLE; 16472259a819SRichard Cochran 1648fb1013a0SSebastian Andrzej Siewior retval = cpuhp_setup_state(CPUHP_AP_ONLINE_DYN, "idle/intel:online", 1649fb1013a0SSebastian Andrzej Siewior intel_idle_cpu_online, NULL); 1650fb1013a0SSebastian Andrzej Siewior if (retval < 0) 1651fb1013a0SSebastian Andrzej Siewior goto hp_setup_fail; 165226717172SLen Brown 1653654d08a4SJoe Perches pr_debug("lapic_timer_reliable_states 0x%x\n", 16542259a819SRichard Cochran lapic_timer_reliable_states); 16552259a819SRichard Cochran 165626717172SLen Brown return 0; 1657fb1013a0SSebastian Andrzej Siewior 1658fb1013a0SSebastian Andrzej Siewior hp_setup_fail: 1659fb1013a0SSebastian Andrzej Siewior intel_idle_cpuidle_devices_uninit(); 1660fb1013a0SSebastian Andrzej Siewior cpuidle_unregister_driver(&intel_idle_driver); 1661fb1013a0SSebastian Andrzej Siewior init_driver_fail: 1662fb1013a0SSebastian Andrzej Siewior free_percpu(intel_idle_cpuidle_devices); 1663fb1013a0SSebastian Andrzej Siewior return retval; 1664fb1013a0SSebastian Andrzej Siewior 166526717172SLen Brown } 166602c4fae9SPaul Gortmaker device_initcall(intel_idle_init); 166726717172SLen Brown 166802c4fae9SPaul Gortmaker /* 166902c4fae9SPaul Gortmaker * We are not really modular, but we used to support that. Meaning we also 167002c4fae9SPaul Gortmaker * support "intel_idle.max_cstate=..." at boot and also a read-only export of 167102c4fae9SPaul Gortmaker * it at /sys/module/intel_idle/parameters/max_cstate -- so using module_param 167202c4fae9SPaul Gortmaker * is the easiest way (currently) to continue doing that. 167302c4fae9SPaul Gortmaker */ 167426717172SLen Brown module_param(max_cstate, int, 0444); 1675