126717172SLen Brown /* 226717172SLen Brown * intel_idle.c - native hardware idle loop for modern Intel processors 326717172SLen Brown * 4fab04b22SLen Brown * Copyright (c) 2013, Intel Corporation. 526717172SLen Brown * Len Brown <len.brown@intel.com> 626717172SLen Brown * 726717172SLen Brown * This program is free software; you can redistribute it and/or modify it 826717172SLen Brown * under the terms and conditions of the GNU General Public License, 926717172SLen Brown * version 2, as published by the Free Software Foundation. 1026717172SLen Brown * 1126717172SLen Brown * This program is distributed in the hope it will be useful, but WITHOUT 1226717172SLen Brown * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or 1326717172SLen Brown * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for 1426717172SLen Brown * more details. 1526717172SLen Brown * 1626717172SLen Brown * You should have received a copy of the GNU General Public License along with 1726717172SLen Brown * this program; if not, write to the Free Software Foundation, Inc., 1826717172SLen Brown * 51 Franklin St - Fifth Floor, Boston, MA 02110-1301 USA. 1926717172SLen Brown */ 2026717172SLen Brown 2126717172SLen Brown /* 2226717172SLen Brown * intel_idle is a cpuidle driver that loads on specific Intel processors 2326717172SLen Brown * in lieu of the legacy ACPI processor_idle driver. The intent is to 2426717172SLen Brown * make Linux more efficient on these processors, as intel_idle knows 2526717172SLen Brown * more than ACPI, as well as make Linux more immune to ACPI BIOS bugs. 2626717172SLen Brown */ 2726717172SLen Brown 2826717172SLen Brown /* 2926717172SLen Brown * Design Assumptions 3026717172SLen Brown * 3126717172SLen Brown * All CPUs have same idle states as boot CPU 3226717172SLen Brown * 3326717172SLen Brown * Chipset BM_STS (bus master status) bit is a NOP 3426717172SLen Brown * for preventing entry into deep C-stats 3526717172SLen Brown */ 3626717172SLen Brown 3726717172SLen Brown /* 3826717172SLen Brown * Known limitations 3926717172SLen Brown * 4026717172SLen Brown * The driver currently initializes for_each_online_cpu() upon modprobe. 4126717172SLen Brown * It it unaware of subsequent processors hot-added to the system. 4226717172SLen Brown * This means that if you boot with maxcpus=n and later online 4326717172SLen Brown * processors above n, those processors will use C1 only. 4426717172SLen Brown * 4526717172SLen Brown * ACPI has a .suspend hack to turn off deep c-statees during suspend 4626717172SLen Brown * to avoid complications with the lapic timer workaround. 4726717172SLen Brown * Have not seen issues with suspend, but may need same workaround here. 4826717172SLen Brown * 4926717172SLen Brown */ 5026717172SLen Brown 5126717172SLen Brown /* un-comment DEBUG to enable pr_debug() statements */ 5226717172SLen Brown #define DEBUG 5326717172SLen Brown 54654d08a4SJoe Perches #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt 55654d08a4SJoe Perches 5626717172SLen Brown #include <linux/kernel.h> 5726717172SLen Brown #include <linux/cpuidle.h> 5876962caaSThomas Gleixner #include <linux/tick.h> 5926717172SLen Brown #include <trace/events/power.h> 6026717172SLen Brown #include <linux/sched.h> 612a2d31c8SShaohua Li #include <linux/notifier.h> 622a2d31c8SShaohua Li #include <linux/cpu.h> 6302c4fae9SPaul Gortmaker #include <linux/moduleparam.h> 64b66b8b9aSAndi Kleen #include <asm/cpu_device_id.h> 65db73c5a8SDave Hansen #include <asm/intel-family.h> 66bc83ccccSH. Peter Anvin #include <asm/mwait.h> 6714796fcaSLen Brown #include <asm/msr.h> 6826717172SLen Brown 69d70e28f5SLen Brown #define INTEL_IDLE_VERSION "0.4.1" 7026717172SLen Brown 7126717172SLen Brown static struct cpuidle_driver intel_idle_driver = { 7226717172SLen Brown .name = "intel_idle", 7326717172SLen Brown .owner = THIS_MODULE, 7426717172SLen Brown }; 7526717172SLen Brown /* intel_idle.max_cstate=0 disables driver */ 76137ecc77SLen Brown static int max_cstate = CPUIDLE_STATE_MAX - 1; 7726717172SLen Brown 78c4236282SLen Brown static unsigned int mwait_substates; 7926717172SLen Brown 802a2d31c8SShaohua Li #define LAPIC_TIMER_ALWAYS_RELIABLE 0xFFFFFFFF 8126717172SLen Brown /* Reliable LAPIC Timer States, bit 1 for C1 etc. */ 82d13780d4SLen Brown static unsigned int lapic_timer_reliable_states = (1 << 1); /* Default to only C1 */ 8326717172SLen Brown 84b66b8b9aSAndi Kleen struct idle_cpu { 85b66b8b9aSAndi Kleen struct cpuidle_state *state_table; 8626717172SLen Brown 8726717172SLen Brown /* 8814796fcaSLen Brown * Hardware C-state auto-demotion may not always be optimal. 8914796fcaSLen Brown * Indicate which enable bits to clear here. 9014796fcaSLen Brown */ 91b66b8b9aSAndi Kleen unsigned long auto_demotion_disable_flags; 928c058d53SLen Brown bool byt_auto_demotion_disable_flag; 9332e95180SLen Brown bool disable_promotion_to_c1e; 94b66b8b9aSAndi Kleen }; 95b66b8b9aSAndi Kleen 96b66b8b9aSAndi Kleen static const struct idle_cpu *icpu; 97b66b8b9aSAndi Kleen static struct cpuidle_device __percpu *intel_idle_cpuidle_devices; 98b66b8b9aSAndi Kleen static int intel_idle(struct cpuidle_device *dev, 99b66b8b9aSAndi Kleen struct cpuidle_driver *drv, int index); 1005fe2e527SRafael J. Wysocki static void intel_idle_freeze(struct cpuidle_device *dev, 1015fe2e527SRafael J. Wysocki struct cpuidle_driver *drv, int index); 102b66b8b9aSAndi Kleen static struct cpuidle_state *cpuidle_state_table; 10314796fcaSLen Brown 10414796fcaSLen Brown /* 105956d033fSLen Brown * Set this flag for states where the HW flushes the TLB for us 106956d033fSLen Brown * and so we don't need cross-calls to keep it consistent. 107956d033fSLen Brown * If this flag is set, SW flushes the TLB, so even if the 108956d033fSLen Brown * HW doesn't do the flushing, this flag is safe to use. 109956d033fSLen Brown */ 110956d033fSLen Brown #define CPUIDLE_FLAG_TLB_FLUSHED 0x10000 111956d033fSLen Brown 112956d033fSLen Brown /* 113b1beab48SLen Brown * MWAIT takes an 8-bit "hint" in EAX "suggesting" 114b1beab48SLen Brown * the C-state (top nibble) and sub-state (bottom nibble) 115b1beab48SLen Brown * 0x00 means "MWAIT(C1)", 0x10 means "MWAIT(C2)" etc. 116b1beab48SLen Brown * 117b1beab48SLen Brown * We store the hint at the top of our "flags" for each state. 118b1beab48SLen Brown */ 119b1beab48SLen Brown #define flg2MWAIT(flags) (((flags) >> 24) & 0xFF) 120b1beab48SLen Brown #define MWAIT2flg(eax) ((eax & 0xFF) << 24) 121b1beab48SLen Brown 122b1beab48SLen Brown /* 12326717172SLen Brown * States are indexed by the cstate number, 12426717172SLen Brown * which is also the index into the MWAIT hint array. 12526717172SLen Brown * Thus C0 is a dummy. 12626717172SLen Brown */ 127ba0dc81eSJiang Liu static struct cpuidle_state nehalem_cstates[] = { 128e022e7ebSLen Brown { 129de09cdd0SLen Brown .name = "C1", 13026717172SLen Brown .desc = "MWAIT 0x00", 131b82b6ccaSDaniel Lezcano .flags = MWAIT2flg(0x00), 13226717172SLen Brown .exit_latency = 3, 13326717172SLen Brown .target_residency = 6, 1345fe2e527SRafael J. Wysocki .enter = &intel_idle, 1355fe2e527SRafael J. Wysocki .enter_freeze = intel_idle_freeze, }, 136e022e7ebSLen Brown { 137de09cdd0SLen Brown .name = "C1E", 13832e95180SLen Brown .desc = "MWAIT 0x01", 139b82b6ccaSDaniel Lezcano .flags = MWAIT2flg(0x01), 14032e95180SLen Brown .exit_latency = 10, 14132e95180SLen Brown .target_residency = 20, 1425fe2e527SRafael J. Wysocki .enter = &intel_idle, 1435fe2e527SRafael J. Wysocki .enter_freeze = intel_idle_freeze, }, 14432e95180SLen Brown { 145de09cdd0SLen Brown .name = "C3", 14626717172SLen Brown .desc = "MWAIT 0x10", 147b82b6ccaSDaniel Lezcano .flags = MWAIT2flg(0x10) | CPUIDLE_FLAG_TLB_FLUSHED, 14826717172SLen Brown .exit_latency = 20, 14926717172SLen Brown .target_residency = 80, 1505fe2e527SRafael J. Wysocki .enter = &intel_idle, 1515fe2e527SRafael J. Wysocki .enter_freeze = intel_idle_freeze, }, 152e022e7ebSLen Brown { 153de09cdd0SLen Brown .name = "C6", 15426717172SLen Brown .desc = "MWAIT 0x20", 155b82b6ccaSDaniel Lezcano .flags = MWAIT2flg(0x20) | CPUIDLE_FLAG_TLB_FLUSHED, 15626717172SLen Brown .exit_latency = 200, 15726717172SLen Brown .target_residency = 800, 1585fe2e527SRafael J. Wysocki .enter = &intel_idle, 1595fe2e527SRafael J. Wysocki .enter_freeze = intel_idle_freeze, }, 160e022e7ebSLen Brown { 161e022e7ebSLen Brown .enter = NULL } 16226717172SLen Brown }; 16326717172SLen Brown 164ba0dc81eSJiang Liu static struct cpuidle_state snb_cstates[] = { 165e022e7ebSLen Brown { 166de09cdd0SLen Brown .name = "C1", 167d13780d4SLen Brown .desc = "MWAIT 0x00", 168b82b6ccaSDaniel Lezcano .flags = MWAIT2flg(0x00), 16932e95180SLen Brown .exit_latency = 2, 17032e95180SLen Brown .target_residency = 2, 1715fe2e527SRafael J. Wysocki .enter = &intel_idle, 1725fe2e527SRafael J. Wysocki .enter_freeze = intel_idle_freeze, }, 17332e95180SLen Brown { 174de09cdd0SLen Brown .name = "C1E", 17532e95180SLen Brown .desc = "MWAIT 0x01", 176b82b6ccaSDaniel Lezcano .flags = MWAIT2flg(0x01), 17732e95180SLen Brown .exit_latency = 10, 17832e95180SLen Brown .target_residency = 20, 1795fe2e527SRafael J. Wysocki .enter = &intel_idle, 1805fe2e527SRafael J. Wysocki .enter_freeze = intel_idle_freeze, }, 181e022e7ebSLen Brown { 182de09cdd0SLen Brown .name = "C3", 183d13780d4SLen Brown .desc = "MWAIT 0x10", 184b82b6ccaSDaniel Lezcano .flags = MWAIT2flg(0x10) | CPUIDLE_FLAG_TLB_FLUSHED, 185d13780d4SLen Brown .exit_latency = 80, 186ddbd550dSLen Brown .target_residency = 211, 1875fe2e527SRafael J. Wysocki .enter = &intel_idle, 1885fe2e527SRafael J. Wysocki .enter_freeze = intel_idle_freeze, }, 189e022e7ebSLen Brown { 190de09cdd0SLen Brown .name = "C6", 191d13780d4SLen Brown .desc = "MWAIT 0x20", 192b82b6ccaSDaniel Lezcano .flags = MWAIT2flg(0x20) | CPUIDLE_FLAG_TLB_FLUSHED, 193d13780d4SLen Brown .exit_latency = 104, 194ddbd550dSLen Brown .target_residency = 345, 1955fe2e527SRafael J. Wysocki .enter = &intel_idle, 1965fe2e527SRafael J. Wysocki .enter_freeze = intel_idle_freeze, }, 197e022e7ebSLen Brown { 198de09cdd0SLen Brown .name = "C7", 199d13780d4SLen Brown .desc = "MWAIT 0x30", 200b82b6ccaSDaniel Lezcano .flags = MWAIT2flg(0x30) | CPUIDLE_FLAG_TLB_FLUSHED, 201d13780d4SLen Brown .exit_latency = 109, 202ddbd550dSLen Brown .target_residency = 345, 2035fe2e527SRafael J. Wysocki .enter = &intel_idle, 2045fe2e527SRafael J. Wysocki .enter_freeze = intel_idle_freeze, }, 205e022e7ebSLen Brown { 206e022e7ebSLen Brown .enter = NULL } 207d13780d4SLen Brown }; 208d13780d4SLen Brown 209718987d6SLen Brown static struct cpuidle_state byt_cstates[] = { 210718987d6SLen Brown { 211de09cdd0SLen Brown .name = "C1", 212718987d6SLen Brown .desc = "MWAIT 0x00", 213b82b6ccaSDaniel Lezcano .flags = MWAIT2flg(0x00), 214718987d6SLen Brown .exit_latency = 1, 215718987d6SLen Brown .target_residency = 1, 2165fe2e527SRafael J. Wysocki .enter = &intel_idle, 2175fe2e527SRafael J. Wysocki .enter_freeze = intel_idle_freeze, }, 218718987d6SLen Brown { 219de09cdd0SLen Brown .name = "C6N", 220718987d6SLen Brown .desc = "MWAIT 0x58", 221b82b6ccaSDaniel Lezcano .flags = MWAIT2flg(0x58) | CPUIDLE_FLAG_TLB_FLUSHED, 222d7ef7671SLen Brown .exit_latency = 300, 223718987d6SLen Brown .target_residency = 275, 2245fe2e527SRafael J. Wysocki .enter = &intel_idle, 2255fe2e527SRafael J. Wysocki .enter_freeze = intel_idle_freeze, }, 226718987d6SLen Brown { 227de09cdd0SLen Brown .name = "C6S", 228718987d6SLen Brown .desc = "MWAIT 0x52", 229b82b6ccaSDaniel Lezcano .flags = MWAIT2flg(0x52) | CPUIDLE_FLAG_TLB_FLUSHED, 230d7ef7671SLen Brown .exit_latency = 500, 231718987d6SLen Brown .target_residency = 560, 2325fe2e527SRafael J. Wysocki .enter = &intel_idle, 2335fe2e527SRafael J. Wysocki .enter_freeze = intel_idle_freeze, }, 234718987d6SLen Brown { 235de09cdd0SLen Brown .name = "C7", 236718987d6SLen Brown .desc = "MWAIT 0x60", 237b82b6ccaSDaniel Lezcano .flags = MWAIT2flg(0x60) | CPUIDLE_FLAG_TLB_FLUSHED, 238718987d6SLen Brown .exit_latency = 1200, 239d7ef7671SLen Brown .target_residency = 4000, 2405fe2e527SRafael J. Wysocki .enter = &intel_idle, 2415fe2e527SRafael J. Wysocki .enter_freeze = intel_idle_freeze, }, 242718987d6SLen Brown { 243de09cdd0SLen Brown .name = "C7S", 244718987d6SLen Brown .desc = "MWAIT 0x64", 245b82b6ccaSDaniel Lezcano .flags = MWAIT2flg(0x64) | CPUIDLE_FLAG_TLB_FLUSHED, 246718987d6SLen Brown .exit_latency = 10000, 247718987d6SLen Brown .target_residency = 20000, 2485fe2e527SRafael J. Wysocki .enter = &intel_idle, 2495fe2e527SRafael J. Wysocki .enter_freeze = intel_idle_freeze, }, 250718987d6SLen Brown { 251718987d6SLen Brown .enter = NULL } 252718987d6SLen Brown }; 253718987d6SLen Brown 254cab07a56SLen Brown static struct cpuidle_state cht_cstates[] = { 255cab07a56SLen Brown { 256de09cdd0SLen Brown .name = "C1", 257cab07a56SLen Brown .desc = "MWAIT 0x00", 258cab07a56SLen Brown .flags = MWAIT2flg(0x00), 259cab07a56SLen Brown .exit_latency = 1, 260cab07a56SLen Brown .target_residency = 1, 261cab07a56SLen Brown .enter = &intel_idle, 262cab07a56SLen Brown .enter_freeze = intel_idle_freeze, }, 263cab07a56SLen Brown { 264de09cdd0SLen Brown .name = "C6N", 265cab07a56SLen Brown .desc = "MWAIT 0x58", 266cab07a56SLen Brown .flags = MWAIT2flg(0x58) | CPUIDLE_FLAG_TLB_FLUSHED, 267cab07a56SLen Brown .exit_latency = 80, 268cab07a56SLen Brown .target_residency = 275, 269cab07a56SLen Brown .enter = &intel_idle, 270cab07a56SLen Brown .enter_freeze = intel_idle_freeze, }, 271cab07a56SLen Brown { 272de09cdd0SLen Brown .name = "C6S", 273cab07a56SLen Brown .desc = "MWAIT 0x52", 274cab07a56SLen Brown .flags = MWAIT2flg(0x52) | CPUIDLE_FLAG_TLB_FLUSHED, 275cab07a56SLen Brown .exit_latency = 200, 276cab07a56SLen Brown .target_residency = 560, 277cab07a56SLen Brown .enter = &intel_idle, 278cab07a56SLen Brown .enter_freeze = intel_idle_freeze, }, 279cab07a56SLen Brown { 280de09cdd0SLen Brown .name = "C7", 281cab07a56SLen Brown .desc = "MWAIT 0x60", 282cab07a56SLen Brown .flags = MWAIT2flg(0x60) | CPUIDLE_FLAG_TLB_FLUSHED, 283cab07a56SLen Brown .exit_latency = 1200, 284cab07a56SLen Brown .target_residency = 4000, 285cab07a56SLen Brown .enter = &intel_idle, 286cab07a56SLen Brown .enter_freeze = intel_idle_freeze, }, 287cab07a56SLen Brown { 288de09cdd0SLen Brown .name = "C7S", 289cab07a56SLen Brown .desc = "MWAIT 0x64", 290cab07a56SLen Brown .flags = MWAIT2flg(0x64) | CPUIDLE_FLAG_TLB_FLUSHED, 291cab07a56SLen Brown .exit_latency = 10000, 292cab07a56SLen Brown .target_residency = 20000, 293cab07a56SLen Brown .enter = &intel_idle, 294cab07a56SLen Brown .enter_freeze = intel_idle_freeze, }, 295cab07a56SLen Brown { 296cab07a56SLen Brown .enter = NULL } 297cab07a56SLen Brown }; 298cab07a56SLen Brown 299ba0dc81eSJiang Liu static struct cpuidle_state ivb_cstates[] = { 300e022e7ebSLen Brown { 301de09cdd0SLen Brown .name = "C1", 3026edab08cSLen Brown .desc = "MWAIT 0x00", 303b82b6ccaSDaniel Lezcano .flags = MWAIT2flg(0x00), 3046edab08cSLen Brown .exit_latency = 1, 3056edab08cSLen Brown .target_residency = 1, 3065fe2e527SRafael J. Wysocki .enter = &intel_idle, 3075fe2e527SRafael J. Wysocki .enter_freeze = intel_idle_freeze, }, 308e022e7ebSLen Brown { 309de09cdd0SLen Brown .name = "C1E", 31032e95180SLen Brown .desc = "MWAIT 0x01", 311b82b6ccaSDaniel Lezcano .flags = MWAIT2flg(0x01), 31232e95180SLen Brown .exit_latency = 10, 31332e95180SLen Brown .target_residency = 20, 3145fe2e527SRafael J. Wysocki .enter = &intel_idle, 3155fe2e527SRafael J. Wysocki .enter_freeze = intel_idle_freeze, }, 31632e95180SLen Brown { 317de09cdd0SLen Brown .name = "C3", 3186edab08cSLen Brown .desc = "MWAIT 0x10", 319b82b6ccaSDaniel Lezcano .flags = MWAIT2flg(0x10) | CPUIDLE_FLAG_TLB_FLUSHED, 3206edab08cSLen Brown .exit_latency = 59, 3216edab08cSLen Brown .target_residency = 156, 3225fe2e527SRafael J. Wysocki .enter = &intel_idle, 3235fe2e527SRafael J. Wysocki .enter_freeze = intel_idle_freeze, }, 324e022e7ebSLen Brown { 325de09cdd0SLen Brown .name = "C6", 3266edab08cSLen Brown .desc = "MWAIT 0x20", 327b82b6ccaSDaniel Lezcano .flags = MWAIT2flg(0x20) | CPUIDLE_FLAG_TLB_FLUSHED, 3286edab08cSLen Brown .exit_latency = 80, 3296edab08cSLen Brown .target_residency = 300, 3305fe2e527SRafael J. Wysocki .enter = &intel_idle, 3315fe2e527SRafael J. Wysocki .enter_freeze = intel_idle_freeze, }, 332e022e7ebSLen Brown { 333de09cdd0SLen Brown .name = "C7", 3346edab08cSLen Brown .desc = "MWAIT 0x30", 335b82b6ccaSDaniel Lezcano .flags = MWAIT2flg(0x30) | CPUIDLE_FLAG_TLB_FLUSHED, 3366edab08cSLen Brown .exit_latency = 87, 3376edab08cSLen Brown .target_residency = 300, 3385fe2e527SRafael J. Wysocki .enter = &intel_idle, 3395fe2e527SRafael J. Wysocki .enter_freeze = intel_idle_freeze, }, 340e022e7ebSLen Brown { 341e022e7ebSLen Brown .enter = NULL } 3426edab08cSLen Brown }; 3436edab08cSLen Brown 3440138d8f0SLen Brown static struct cpuidle_state ivt_cstates[] = { 3450138d8f0SLen Brown { 346de09cdd0SLen Brown .name = "C1", 3470138d8f0SLen Brown .desc = "MWAIT 0x00", 348b82b6ccaSDaniel Lezcano .flags = MWAIT2flg(0x00), 3490138d8f0SLen Brown .exit_latency = 1, 3500138d8f0SLen Brown .target_residency = 1, 3515fe2e527SRafael J. Wysocki .enter = &intel_idle, 3525fe2e527SRafael J. Wysocki .enter_freeze = intel_idle_freeze, }, 3530138d8f0SLen Brown { 354de09cdd0SLen Brown .name = "C1E", 3550138d8f0SLen Brown .desc = "MWAIT 0x01", 356b82b6ccaSDaniel Lezcano .flags = MWAIT2flg(0x01), 3570138d8f0SLen Brown .exit_latency = 10, 3580138d8f0SLen Brown .target_residency = 80, 3595fe2e527SRafael J. Wysocki .enter = &intel_idle, 3605fe2e527SRafael J. Wysocki .enter_freeze = intel_idle_freeze, }, 3610138d8f0SLen Brown { 362de09cdd0SLen Brown .name = "C3", 3630138d8f0SLen Brown .desc = "MWAIT 0x10", 364b82b6ccaSDaniel Lezcano .flags = MWAIT2flg(0x10) | CPUIDLE_FLAG_TLB_FLUSHED, 3650138d8f0SLen Brown .exit_latency = 59, 3660138d8f0SLen Brown .target_residency = 156, 3675fe2e527SRafael J. Wysocki .enter = &intel_idle, 3685fe2e527SRafael J. Wysocki .enter_freeze = intel_idle_freeze, }, 3690138d8f0SLen Brown { 370de09cdd0SLen Brown .name = "C6", 3710138d8f0SLen Brown .desc = "MWAIT 0x20", 372b82b6ccaSDaniel Lezcano .flags = MWAIT2flg(0x20) | CPUIDLE_FLAG_TLB_FLUSHED, 3730138d8f0SLen Brown .exit_latency = 82, 3740138d8f0SLen Brown .target_residency = 300, 3755fe2e527SRafael J. Wysocki .enter = &intel_idle, 3765fe2e527SRafael J. Wysocki .enter_freeze = intel_idle_freeze, }, 3770138d8f0SLen Brown { 3780138d8f0SLen Brown .enter = NULL } 3790138d8f0SLen Brown }; 3800138d8f0SLen Brown 3810138d8f0SLen Brown static struct cpuidle_state ivt_cstates_4s[] = { 3820138d8f0SLen Brown { 383de09cdd0SLen Brown .name = "C1", 3840138d8f0SLen Brown .desc = "MWAIT 0x00", 385b82b6ccaSDaniel Lezcano .flags = MWAIT2flg(0x00), 3860138d8f0SLen Brown .exit_latency = 1, 3870138d8f0SLen Brown .target_residency = 1, 3885fe2e527SRafael J. Wysocki .enter = &intel_idle, 3895fe2e527SRafael J. Wysocki .enter_freeze = intel_idle_freeze, }, 3900138d8f0SLen Brown { 391de09cdd0SLen Brown .name = "C1E", 3920138d8f0SLen Brown .desc = "MWAIT 0x01", 393b82b6ccaSDaniel Lezcano .flags = MWAIT2flg(0x01), 3940138d8f0SLen Brown .exit_latency = 10, 3950138d8f0SLen Brown .target_residency = 250, 3965fe2e527SRafael J. Wysocki .enter = &intel_idle, 3975fe2e527SRafael J. Wysocki .enter_freeze = intel_idle_freeze, }, 3980138d8f0SLen Brown { 399de09cdd0SLen Brown .name = "C3", 4000138d8f0SLen Brown .desc = "MWAIT 0x10", 401b82b6ccaSDaniel Lezcano .flags = MWAIT2flg(0x10) | CPUIDLE_FLAG_TLB_FLUSHED, 4020138d8f0SLen Brown .exit_latency = 59, 4030138d8f0SLen Brown .target_residency = 300, 4045fe2e527SRafael J. Wysocki .enter = &intel_idle, 4055fe2e527SRafael J. Wysocki .enter_freeze = intel_idle_freeze, }, 4060138d8f0SLen Brown { 407de09cdd0SLen Brown .name = "C6", 4080138d8f0SLen Brown .desc = "MWAIT 0x20", 409b82b6ccaSDaniel Lezcano .flags = MWAIT2flg(0x20) | CPUIDLE_FLAG_TLB_FLUSHED, 4100138d8f0SLen Brown .exit_latency = 84, 4110138d8f0SLen Brown .target_residency = 400, 4125fe2e527SRafael J. Wysocki .enter = &intel_idle, 4135fe2e527SRafael J. Wysocki .enter_freeze = intel_idle_freeze, }, 4140138d8f0SLen Brown { 4150138d8f0SLen Brown .enter = NULL } 4160138d8f0SLen Brown }; 4170138d8f0SLen Brown 4180138d8f0SLen Brown static struct cpuidle_state ivt_cstates_8s[] = { 4190138d8f0SLen Brown { 420de09cdd0SLen Brown .name = "C1", 4210138d8f0SLen Brown .desc = "MWAIT 0x00", 422b82b6ccaSDaniel Lezcano .flags = MWAIT2flg(0x00), 4230138d8f0SLen Brown .exit_latency = 1, 4240138d8f0SLen Brown .target_residency = 1, 4255fe2e527SRafael J. Wysocki .enter = &intel_idle, 4265fe2e527SRafael J. Wysocki .enter_freeze = intel_idle_freeze, }, 4270138d8f0SLen Brown { 428de09cdd0SLen Brown .name = "C1E", 4290138d8f0SLen Brown .desc = "MWAIT 0x01", 430b82b6ccaSDaniel Lezcano .flags = MWAIT2flg(0x01), 4310138d8f0SLen Brown .exit_latency = 10, 4320138d8f0SLen Brown .target_residency = 500, 4335fe2e527SRafael J. Wysocki .enter = &intel_idle, 4345fe2e527SRafael J. Wysocki .enter_freeze = intel_idle_freeze, }, 4350138d8f0SLen Brown { 436de09cdd0SLen Brown .name = "C3", 4370138d8f0SLen Brown .desc = "MWAIT 0x10", 438b82b6ccaSDaniel Lezcano .flags = MWAIT2flg(0x10) | CPUIDLE_FLAG_TLB_FLUSHED, 4390138d8f0SLen Brown .exit_latency = 59, 4400138d8f0SLen Brown .target_residency = 600, 4415fe2e527SRafael J. Wysocki .enter = &intel_idle, 4425fe2e527SRafael J. Wysocki .enter_freeze = intel_idle_freeze, }, 4430138d8f0SLen Brown { 444de09cdd0SLen Brown .name = "C6", 4450138d8f0SLen Brown .desc = "MWAIT 0x20", 446b82b6ccaSDaniel Lezcano .flags = MWAIT2flg(0x20) | CPUIDLE_FLAG_TLB_FLUSHED, 4470138d8f0SLen Brown .exit_latency = 88, 4480138d8f0SLen Brown .target_residency = 700, 4495fe2e527SRafael J. Wysocki .enter = &intel_idle, 4505fe2e527SRafael J. Wysocki .enter_freeze = intel_idle_freeze, }, 4510138d8f0SLen Brown { 4520138d8f0SLen Brown .enter = NULL } 4530138d8f0SLen Brown }; 4540138d8f0SLen Brown 455ba0dc81eSJiang Liu static struct cpuidle_state hsw_cstates[] = { 456e022e7ebSLen Brown { 457de09cdd0SLen Brown .name = "C1", 45885a4d2d4SLen Brown .desc = "MWAIT 0x00", 459b82b6ccaSDaniel Lezcano .flags = MWAIT2flg(0x00), 46085a4d2d4SLen Brown .exit_latency = 2, 46185a4d2d4SLen Brown .target_residency = 2, 4625fe2e527SRafael J. Wysocki .enter = &intel_idle, 4635fe2e527SRafael J. Wysocki .enter_freeze = intel_idle_freeze, }, 464e022e7ebSLen Brown { 465de09cdd0SLen Brown .name = "C1E", 46632e95180SLen Brown .desc = "MWAIT 0x01", 467b82b6ccaSDaniel Lezcano .flags = MWAIT2flg(0x01), 46832e95180SLen Brown .exit_latency = 10, 46932e95180SLen Brown .target_residency = 20, 4705fe2e527SRafael J. Wysocki .enter = &intel_idle, 4715fe2e527SRafael J. Wysocki .enter_freeze = intel_idle_freeze, }, 47232e95180SLen Brown { 473de09cdd0SLen Brown .name = "C3", 47485a4d2d4SLen Brown .desc = "MWAIT 0x10", 475b82b6ccaSDaniel Lezcano .flags = MWAIT2flg(0x10) | CPUIDLE_FLAG_TLB_FLUSHED, 47685a4d2d4SLen Brown .exit_latency = 33, 47785a4d2d4SLen Brown .target_residency = 100, 4785fe2e527SRafael J. Wysocki .enter = &intel_idle, 4795fe2e527SRafael J. Wysocki .enter_freeze = intel_idle_freeze, }, 480e022e7ebSLen Brown { 481de09cdd0SLen Brown .name = "C6", 48285a4d2d4SLen Brown .desc = "MWAIT 0x20", 483b82b6ccaSDaniel Lezcano .flags = MWAIT2flg(0x20) | CPUIDLE_FLAG_TLB_FLUSHED, 48485a4d2d4SLen Brown .exit_latency = 133, 48585a4d2d4SLen Brown .target_residency = 400, 4865fe2e527SRafael J. Wysocki .enter = &intel_idle, 4875fe2e527SRafael J. Wysocki .enter_freeze = intel_idle_freeze, }, 488e022e7ebSLen Brown { 489de09cdd0SLen Brown .name = "C7s", 49085a4d2d4SLen Brown .desc = "MWAIT 0x32", 491b82b6ccaSDaniel Lezcano .flags = MWAIT2flg(0x32) | CPUIDLE_FLAG_TLB_FLUSHED, 49285a4d2d4SLen Brown .exit_latency = 166, 49385a4d2d4SLen Brown .target_residency = 500, 4945fe2e527SRafael J. Wysocki .enter = &intel_idle, 4955fe2e527SRafael J. Wysocki .enter_freeze = intel_idle_freeze, }, 496e022e7ebSLen Brown { 497de09cdd0SLen Brown .name = "C8", 49886239cebSLen Brown .desc = "MWAIT 0x40", 499b82b6ccaSDaniel Lezcano .flags = MWAIT2flg(0x40) | CPUIDLE_FLAG_TLB_FLUSHED, 50086239cebSLen Brown .exit_latency = 300, 50186239cebSLen Brown .target_residency = 900, 5025fe2e527SRafael J. Wysocki .enter = &intel_idle, 5035fe2e527SRafael J. Wysocki .enter_freeze = intel_idle_freeze, }, 50486239cebSLen Brown { 505de09cdd0SLen Brown .name = "C9", 50686239cebSLen Brown .desc = "MWAIT 0x50", 507b82b6ccaSDaniel Lezcano .flags = MWAIT2flg(0x50) | CPUIDLE_FLAG_TLB_FLUSHED, 50886239cebSLen Brown .exit_latency = 600, 50986239cebSLen Brown .target_residency = 1800, 5105fe2e527SRafael J. Wysocki .enter = &intel_idle, 5115fe2e527SRafael J. Wysocki .enter_freeze = intel_idle_freeze, }, 51286239cebSLen Brown { 513de09cdd0SLen Brown .name = "C10", 51486239cebSLen Brown .desc = "MWAIT 0x60", 515b82b6ccaSDaniel Lezcano .flags = MWAIT2flg(0x60) | CPUIDLE_FLAG_TLB_FLUSHED, 51686239cebSLen Brown .exit_latency = 2600, 51786239cebSLen Brown .target_residency = 7700, 5185fe2e527SRafael J. Wysocki .enter = &intel_idle, 5195fe2e527SRafael J. Wysocki .enter_freeze = intel_idle_freeze, }, 52086239cebSLen Brown { 521e022e7ebSLen Brown .enter = NULL } 52285a4d2d4SLen Brown }; 523a138b568SLen Brown static struct cpuidle_state bdw_cstates[] = { 524a138b568SLen Brown { 525de09cdd0SLen Brown .name = "C1", 526a138b568SLen Brown .desc = "MWAIT 0x00", 527b82b6ccaSDaniel Lezcano .flags = MWAIT2flg(0x00), 528a138b568SLen Brown .exit_latency = 2, 529a138b568SLen Brown .target_residency = 2, 5305fe2e527SRafael J. Wysocki .enter = &intel_idle, 5315fe2e527SRafael J. Wysocki .enter_freeze = intel_idle_freeze, }, 532a138b568SLen Brown { 533de09cdd0SLen Brown .name = "C1E", 534a138b568SLen Brown .desc = "MWAIT 0x01", 535b82b6ccaSDaniel Lezcano .flags = MWAIT2flg(0x01), 536a138b568SLen Brown .exit_latency = 10, 537a138b568SLen Brown .target_residency = 20, 5385fe2e527SRafael J. Wysocki .enter = &intel_idle, 5395fe2e527SRafael J. Wysocki .enter_freeze = intel_idle_freeze, }, 540a138b568SLen Brown { 541de09cdd0SLen Brown .name = "C3", 542a138b568SLen Brown .desc = "MWAIT 0x10", 543b82b6ccaSDaniel Lezcano .flags = MWAIT2flg(0x10) | CPUIDLE_FLAG_TLB_FLUSHED, 544a138b568SLen Brown .exit_latency = 40, 545a138b568SLen Brown .target_residency = 100, 5465fe2e527SRafael J. Wysocki .enter = &intel_idle, 5475fe2e527SRafael J. Wysocki .enter_freeze = intel_idle_freeze, }, 548a138b568SLen Brown { 549de09cdd0SLen Brown .name = "C6", 550a138b568SLen Brown .desc = "MWAIT 0x20", 551b82b6ccaSDaniel Lezcano .flags = MWAIT2flg(0x20) | CPUIDLE_FLAG_TLB_FLUSHED, 552a138b568SLen Brown .exit_latency = 133, 553a138b568SLen Brown .target_residency = 400, 5545fe2e527SRafael J. Wysocki .enter = &intel_idle, 5555fe2e527SRafael J. Wysocki .enter_freeze = intel_idle_freeze, }, 556a138b568SLen Brown { 557de09cdd0SLen Brown .name = "C7s", 558a138b568SLen Brown .desc = "MWAIT 0x32", 559b82b6ccaSDaniel Lezcano .flags = MWAIT2flg(0x32) | CPUIDLE_FLAG_TLB_FLUSHED, 560a138b568SLen Brown .exit_latency = 166, 561a138b568SLen Brown .target_residency = 500, 5625fe2e527SRafael J. Wysocki .enter = &intel_idle, 5635fe2e527SRafael J. Wysocki .enter_freeze = intel_idle_freeze, }, 564a138b568SLen Brown { 565de09cdd0SLen Brown .name = "C8", 566a138b568SLen Brown .desc = "MWAIT 0x40", 567b82b6ccaSDaniel Lezcano .flags = MWAIT2flg(0x40) | CPUIDLE_FLAG_TLB_FLUSHED, 568a138b568SLen Brown .exit_latency = 300, 569a138b568SLen Brown .target_residency = 900, 5705fe2e527SRafael J. Wysocki .enter = &intel_idle, 5715fe2e527SRafael J. Wysocki .enter_freeze = intel_idle_freeze, }, 572a138b568SLen Brown { 573de09cdd0SLen Brown .name = "C9", 574a138b568SLen Brown .desc = "MWAIT 0x50", 575b82b6ccaSDaniel Lezcano .flags = MWAIT2flg(0x50) | CPUIDLE_FLAG_TLB_FLUSHED, 576a138b568SLen Brown .exit_latency = 600, 577a138b568SLen Brown .target_residency = 1800, 5785fe2e527SRafael J. Wysocki .enter = &intel_idle, 5795fe2e527SRafael J. Wysocki .enter_freeze = intel_idle_freeze, }, 580a138b568SLen Brown { 581de09cdd0SLen Brown .name = "C10", 582a138b568SLen Brown .desc = "MWAIT 0x60", 583b82b6ccaSDaniel Lezcano .flags = MWAIT2flg(0x60) | CPUIDLE_FLAG_TLB_FLUSHED, 584a138b568SLen Brown .exit_latency = 2600, 585a138b568SLen Brown .target_residency = 7700, 5865fe2e527SRafael J. Wysocki .enter = &intel_idle, 5875fe2e527SRafael J. Wysocki .enter_freeze = intel_idle_freeze, }, 588a138b568SLen Brown { 589a138b568SLen Brown .enter = NULL } 590a138b568SLen Brown }; 59185a4d2d4SLen Brown 592493f133fSLen Brown static struct cpuidle_state skl_cstates[] = { 593493f133fSLen Brown { 594de09cdd0SLen Brown .name = "C1", 595493f133fSLen Brown .desc = "MWAIT 0x00", 596493f133fSLen Brown .flags = MWAIT2flg(0x00), 597493f133fSLen Brown .exit_latency = 2, 598493f133fSLen Brown .target_residency = 2, 599493f133fSLen Brown .enter = &intel_idle, 600493f133fSLen Brown .enter_freeze = intel_idle_freeze, }, 601493f133fSLen Brown { 602de09cdd0SLen Brown .name = "C1E", 603493f133fSLen Brown .desc = "MWAIT 0x01", 604493f133fSLen Brown .flags = MWAIT2flg(0x01), 605493f133fSLen Brown .exit_latency = 10, 606493f133fSLen Brown .target_residency = 20, 607493f133fSLen Brown .enter = &intel_idle, 608493f133fSLen Brown .enter_freeze = intel_idle_freeze, }, 609493f133fSLen Brown { 610de09cdd0SLen Brown .name = "C3", 611493f133fSLen Brown .desc = "MWAIT 0x10", 612493f133fSLen Brown .flags = MWAIT2flg(0x10) | CPUIDLE_FLAG_TLB_FLUSHED, 613493f133fSLen Brown .exit_latency = 70, 614493f133fSLen Brown .target_residency = 100, 615493f133fSLen Brown .enter = &intel_idle, 616493f133fSLen Brown .enter_freeze = intel_idle_freeze, }, 617493f133fSLen Brown { 618de09cdd0SLen Brown .name = "C6", 619493f133fSLen Brown .desc = "MWAIT 0x20", 620493f133fSLen Brown .flags = MWAIT2flg(0x20) | CPUIDLE_FLAG_TLB_FLUSHED, 621135919a3SLen Brown .exit_latency = 85, 622493f133fSLen Brown .target_residency = 200, 623493f133fSLen Brown .enter = &intel_idle, 624493f133fSLen Brown .enter_freeze = intel_idle_freeze, }, 625493f133fSLen Brown { 626de09cdd0SLen Brown .name = "C7s", 627493f133fSLen Brown .desc = "MWAIT 0x33", 628493f133fSLen Brown .flags = MWAIT2flg(0x33) | CPUIDLE_FLAG_TLB_FLUSHED, 629493f133fSLen Brown .exit_latency = 124, 630493f133fSLen Brown .target_residency = 800, 631493f133fSLen Brown .enter = &intel_idle, 632493f133fSLen Brown .enter_freeze = intel_idle_freeze, }, 633493f133fSLen Brown { 634de09cdd0SLen Brown .name = "C8", 635493f133fSLen Brown .desc = "MWAIT 0x40", 636493f133fSLen Brown .flags = MWAIT2flg(0x40) | CPUIDLE_FLAG_TLB_FLUSHED, 637135919a3SLen Brown .exit_latency = 200, 638493f133fSLen Brown .target_residency = 800, 639493f133fSLen Brown .enter = &intel_idle, 640493f133fSLen Brown .enter_freeze = intel_idle_freeze, }, 641493f133fSLen Brown { 642de09cdd0SLen Brown .name = "C9", 643135919a3SLen Brown .desc = "MWAIT 0x50", 644135919a3SLen Brown .flags = MWAIT2flg(0x50) | CPUIDLE_FLAG_TLB_FLUSHED, 645135919a3SLen Brown .exit_latency = 480, 646135919a3SLen Brown .target_residency = 5000, 647135919a3SLen Brown .enter = &intel_idle, 648135919a3SLen Brown .enter_freeze = intel_idle_freeze, }, 649135919a3SLen Brown { 650de09cdd0SLen Brown .name = "C10", 651493f133fSLen Brown .desc = "MWAIT 0x60", 652493f133fSLen Brown .flags = MWAIT2flg(0x60) | CPUIDLE_FLAG_TLB_FLUSHED, 653493f133fSLen Brown .exit_latency = 890, 654493f133fSLen Brown .target_residency = 5000, 655493f133fSLen Brown .enter = &intel_idle, 656493f133fSLen Brown .enter_freeze = intel_idle_freeze, }, 657493f133fSLen Brown { 658493f133fSLen Brown .enter = NULL } 659493f133fSLen Brown }; 660493f133fSLen Brown 661f9e71657SLen Brown static struct cpuidle_state skx_cstates[] = { 662f9e71657SLen Brown { 663de09cdd0SLen Brown .name = "C1", 664f9e71657SLen Brown .desc = "MWAIT 0x00", 665f9e71657SLen Brown .flags = MWAIT2flg(0x00), 666f9e71657SLen Brown .exit_latency = 2, 667f9e71657SLen Brown .target_residency = 2, 668f9e71657SLen Brown .enter = &intel_idle, 669f9e71657SLen Brown .enter_freeze = intel_idle_freeze, }, 670f9e71657SLen Brown { 671de09cdd0SLen Brown .name = "C1E", 672f9e71657SLen Brown .desc = "MWAIT 0x01", 673f9e71657SLen Brown .flags = MWAIT2flg(0x01), 674f9e71657SLen Brown .exit_latency = 10, 675f9e71657SLen Brown .target_residency = 20, 676f9e71657SLen Brown .enter = &intel_idle, 677f9e71657SLen Brown .enter_freeze = intel_idle_freeze, }, 678f9e71657SLen Brown { 679de09cdd0SLen Brown .name = "C6", 680f9e71657SLen Brown .desc = "MWAIT 0x20", 681f9e71657SLen Brown .flags = MWAIT2flg(0x20) | CPUIDLE_FLAG_TLB_FLUSHED, 682f9e71657SLen Brown .exit_latency = 133, 683f9e71657SLen Brown .target_residency = 600, 684f9e71657SLen Brown .enter = &intel_idle, 685f9e71657SLen Brown .enter_freeze = intel_idle_freeze, }, 686f9e71657SLen Brown { 687f9e71657SLen Brown .enter = NULL } 688f9e71657SLen Brown }; 689f9e71657SLen Brown 690ba0dc81eSJiang Liu static struct cpuidle_state atom_cstates[] = { 691e022e7ebSLen Brown { 692de09cdd0SLen Brown .name = "C1E", 69326717172SLen Brown .desc = "MWAIT 0x00", 694b82b6ccaSDaniel Lezcano .flags = MWAIT2flg(0x00), 69532e95180SLen Brown .exit_latency = 10, 69632e95180SLen Brown .target_residency = 20, 6975fe2e527SRafael J. Wysocki .enter = &intel_idle, 6985fe2e527SRafael J. Wysocki .enter_freeze = intel_idle_freeze, }, 699e022e7ebSLen Brown { 700de09cdd0SLen Brown .name = "C2", 70126717172SLen Brown .desc = "MWAIT 0x10", 702b82b6ccaSDaniel Lezcano .flags = MWAIT2flg(0x10), 70326717172SLen Brown .exit_latency = 20, 70426717172SLen Brown .target_residency = 80, 7055fe2e527SRafael J. Wysocki .enter = &intel_idle, 7065fe2e527SRafael J. Wysocki .enter_freeze = intel_idle_freeze, }, 707e022e7ebSLen Brown { 708de09cdd0SLen Brown .name = "C4", 70926717172SLen Brown .desc = "MWAIT 0x30", 710b82b6ccaSDaniel Lezcano .flags = MWAIT2flg(0x30) | CPUIDLE_FLAG_TLB_FLUSHED, 71126717172SLen Brown .exit_latency = 100, 71226717172SLen Brown .target_residency = 400, 7135fe2e527SRafael J. Wysocki .enter = &intel_idle, 7145fe2e527SRafael J. Wysocki .enter_freeze = intel_idle_freeze, }, 715e022e7ebSLen Brown { 716de09cdd0SLen Brown .name = "C6", 7177fcca7d9SLen Brown .desc = "MWAIT 0x52", 718b82b6ccaSDaniel Lezcano .flags = MWAIT2flg(0x52) | CPUIDLE_FLAG_TLB_FLUSHED, 7197fcca7d9SLen Brown .exit_latency = 140, 7207fcca7d9SLen Brown .target_residency = 560, 7215fe2e527SRafael J. Wysocki .enter = &intel_idle, 7225fe2e527SRafael J. Wysocki .enter_freeze = intel_idle_freeze, }, 723e022e7ebSLen Brown { 724e022e7ebSLen Brown .enter = NULL } 72526717172SLen Brown }; 7265e7ec268SAndy Shevchenko static struct cpuidle_state tangier_cstates[] = { 7275e7ec268SAndy Shevchenko { 728de09cdd0SLen Brown .name = "C1", 7295e7ec268SAndy Shevchenko .desc = "MWAIT 0x00", 7305e7ec268SAndy Shevchenko .flags = MWAIT2flg(0x00), 7315e7ec268SAndy Shevchenko .exit_latency = 1, 7325e7ec268SAndy Shevchenko .target_residency = 4, 7335e7ec268SAndy Shevchenko .enter = &intel_idle, 7345e7ec268SAndy Shevchenko .enter_freeze = intel_idle_freeze, }, 7355e7ec268SAndy Shevchenko { 736de09cdd0SLen Brown .name = "C4", 7375e7ec268SAndy Shevchenko .desc = "MWAIT 0x30", 7385e7ec268SAndy Shevchenko .flags = MWAIT2flg(0x30) | CPUIDLE_FLAG_TLB_FLUSHED, 7395e7ec268SAndy Shevchenko .exit_latency = 100, 7405e7ec268SAndy Shevchenko .target_residency = 400, 7415e7ec268SAndy Shevchenko .enter = &intel_idle, 7425e7ec268SAndy Shevchenko .enter_freeze = intel_idle_freeze, }, 7435e7ec268SAndy Shevchenko { 744de09cdd0SLen Brown .name = "C6", 7455e7ec268SAndy Shevchenko .desc = "MWAIT 0x52", 7465e7ec268SAndy Shevchenko .flags = MWAIT2flg(0x52) | CPUIDLE_FLAG_TLB_FLUSHED, 7475e7ec268SAndy Shevchenko .exit_latency = 140, 7485e7ec268SAndy Shevchenko .target_residency = 560, 7495e7ec268SAndy Shevchenko .enter = &intel_idle, 7505e7ec268SAndy Shevchenko .enter_freeze = intel_idle_freeze, }, 7515e7ec268SAndy Shevchenko { 752de09cdd0SLen Brown .name = "C7", 7535e7ec268SAndy Shevchenko .desc = "MWAIT 0x60", 7545e7ec268SAndy Shevchenko .flags = MWAIT2flg(0x60) | CPUIDLE_FLAG_TLB_FLUSHED, 7555e7ec268SAndy Shevchenko .exit_latency = 1200, 7565e7ec268SAndy Shevchenko .target_residency = 4000, 7575e7ec268SAndy Shevchenko .enter = &intel_idle, 7585e7ec268SAndy Shevchenko .enter_freeze = intel_idle_freeze, }, 7595e7ec268SAndy Shevchenko { 760de09cdd0SLen Brown .name = "C9", 7615e7ec268SAndy Shevchenko .desc = "MWAIT 0x64", 7625e7ec268SAndy Shevchenko .flags = MWAIT2flg(0x64) | CPUIDLE_FLAG_TLB_FLUSHED, 7635e7ec268SAndy Shevchenko .exit_latency = 10000, 7645e7ec268SAndy Shevchenko .target_residency = 20000, 7655e7ec268SAndy Shevchenko .enter = &intel_idle, 7665e7ec268SAndy Shevchenko .enter_freeze = intel_idle_freeze, }, 7675e7ec268SAndy Shevchenko { 7685e7ec268SAndy Shevchenko .enter = NULL } 7695e7ec268SAndy Shevchenko }; 77088390996SJiang Liu static struct cpuidle_state avn_cstates[] = { 771fab04b22SLen Brown { 772de09cdd0SLen Brown .name = "C1", 773fab04b22SLen Brown .desc = "MWAIT 0x00", 774b82b6ccaSDaniel Lezcano .flags = MWAIT2flg(0x00), 775fab04b22SLen Brown .exit_latency = 2, 776fab04b22SLen Brown .target_residency = 2, 7775fe2e527SRafael J. Wysocki .enter = &intel_idle, 7785fe2e527SRafael J. Wysocki .enter_freeze = intel_idle_freeze, }, 779fab04b22SLen Brown { 780de09cdd0SLen Brown .name = "C6", 781fab04b22SLen Brown .desc = "MWAIT 0x51", 782b82b6ccaSDaniel Lezcano .flags = MWAIT2flg(0x51) | CPUIDLE_FLAG_TLB_FLUSHED, 783fab04b22SLen Brown .exit_latency = 15, 784fab04b22SLen Brown .target_residency = 45, 7855fe2e527SRafael J. Wysocki .enter = &intel_idle, 7865fe2e527SRafael J. Wysocki .enter_freeze = intel_idle_freeze, }, 78788390996SJiang Liu { 78888390996SJiang Liu .enter = NULL } 789fab04b22SLen Brown }; 790281baf7aSDasaratharaman Chandramouli static struct cpuidle_state knl_cstates[] = { 791281baf7aSDasaratharaman Chandramouli { 792de09cdd0SLen Brown .name = "C1", 793281baf7aSDasaratharaman Chandramouli .desc = "MWAIT 0x00", 794281baf7aSDasaratharaman Chandramouli .flags = MWAIT2flg(0x00), 795281baf7aSDasaratharaman Chandramouli .exit_latency = 1, 796281baf7aSDasaratharaman Chandramouli .target_residency = 2, 797281baf7aSDasaratharaman Chandramouli .enter = &intel_idle, 798281baf7aSDasaratharaman Chandramouli .enter_freeze = intel_idle_freeze }, 799281baf7aSDasaratharaman Chandramouli { 800de09cdd0SLen Brown .name = "C6", 801281baf7aSDasaratharaman Chandramouli .desc = "MWAIT 0x10", 802281baf7aSDasaratharaman Chandramouli .flags = MWAIT2flg(0x10) | CPUIDLE_FLAG_TLB_FLUSHED, 803281baf7aSDasaratharaman Chandramouli .exit_latency = 120, 804281baf7aSDasaratharaman Chandramouli .target_residency = 500, 805281baf7aSDasaratharaman Chandramouli .enter = &intel_idle, 806281baf7aSDasaratharaman Chandramouli .enter_freeze = intel_idle_freeze }, 807281baf7aSDasaratharaman Chandramouli { 808281baf7aSDasaratharaman Chandramouli .enter = NULL } 809281baf7aSDasaratharaman Chandramouli }; 81026717172SLen Brown 8115dcef694SLen Brown static struct cpuidle_state bxt_cstates[] = { 8125dcef694SLen Brown { 813de09cdd0SLen Brown .name = "C1", 8145dcef694SLen Brown .desc = "MWAIT 0x00", 8155dcef694SLen Brown .flags = MWAIT2flg(0x00), 8165dcef694SLen Brown .exit_latency = 2, 8175dcef694SLen Brown .target_residency = 2, 8185dcef694SLen Brown .enter = &intel_idle, 8195dcef694SLen Brown .enter_freeze = intel_idle_freeze, }, 8205dcef694SLen Brown { 821de09cdd0SLen Brown .name = "C1E", 8225dcef694SLen Brown .desc = "MWAIT 0x01", 8235dcef694SLen Brown .flags = MWAIT2flg(0x01), 8245dcef694SLen Brown .exit_latency = 10, 8255dcef694SLen Brown .target_residency = 20, 8265dcef694SLen Brown .enter = &intel_idle, 8275dcef694SLen Brown .enter_freeze = intel_idle_freeze, }, 8285dcef694SLen Brown { 829de09cdd0SLen Brown .name = "C6", 8305dcef694SLen Brown .desc = "MWAIT 0x20", 8315dcef694SLen Brown .flags = MWAIT2flg(0x20) | CPUIDLE_FLAG_TLB_FLUSHED, 8325dcef694SLen Brown .exit_latency = 133, 8335dcef694SLen Brown .target_residency = 133, 8345dcef694SLen Brown .enter = &intel_idle, 8355dcef694SLen Brown .enter_freeze = intel_idle_freeze, }, 8365dcef694SLen Brown { 837de09cdd0SLen Brown .name = "C7s", 8385dcef694SLen Brown .desc = "MWAIT 0x31", 8395dcef694SLen Brown .flags = MWAIT2flg(0x31) | CPUIDLE_FLAG_TLB_FLUSHED, 8405dcef694SLen Brown .exit_latency = 155, 8415dcef694SLen Brown .target_residency = 155, 8425dcef694SLen Brown .enter = &intel_idle, 8435dcef694SLen Brown .enter_freeze = intel_idle_freeze, }, 8445dcef694SLen Brown { 845de09cdd0SLen Brown .name = "C8", 8465dcef694SLen Brown .desc = "MWAIT 0x40", 8475dcef694SLen Brown .flags = MWAIT2flg(0x40) | CPUIDLE_FLAG_TLB_FLUSHED, 8485dcef694SLen Brown .exit_latency = 1000, 8495dcef694SLen Brown .target_residency = 1000, 8505dcef694SLen Brown .enter = &intel_idle, 8515dcef694SLen Brown .enter_freeze = intel_idle_freeze, }, 8525dcef694SLen Brown { 853de09cdd0SLen Brown .name = "C9", 8545dcef694SLen Brown .desc = "MWAIT 0x50", 8555dcef694SLen Brown .flags = MWAIT2flg(0x50) | CPUIDLE_FLAG_TLB_FLUSHED, 8565dcef694SLen Brown .exit_latency = 2000, 8575dcef694SLen Brown .target_residency = 2000, 8585dcef694SLen Brown .enter = &intel_idle, 8595dcef694SLen Brown .enter_freeze = intel_idle_freeze, }, 8605dcef694SLen Brown { 861de09cdd0SLen Brown .name = "C10", 8625dcef694SLen Brown .desc = "MWAIT 0x60", 8635dcef694SLen Brown .flags = MWAIT2flg(0x60) | CPUIDLE_FLAG_TLB_FLUSHED, 8645dcef694SLen Brown .exit_latency = 10000, 8655dcef694SLen Brown .target_residency = 10000, 8665dcef694SLen Brown .enter = &intel_idle, 8675dcef694SLen Brown .enter_freeze = intel_idle_freeze, }, 8685dcef694SLen Brown { 8695dcef694SLen Brown .enter = NULL } 8705dcef694SLen Brown }; 8715dcef694SLen Brown 8720080d65bSJacob Pan static struct cpuidle_state dnv_cstates[] = { 8730080d65bSJacob Pan { 874de09cdd0SLen Brown .name = "C1", 8750080d65bSJacob Pan .desc = "MWAIT 0x00", 8760080d65bSJacob Pan .flags = MWAIT2flg(0x00), 8770080d65bSJacob Pan .exit_latency = 2, 8780080d65bSJacob Pan .target_residency = 2, 8790080d65bSJacob Pan .enter = &intel_idle, 8800080d65bSJacob Pan .enter_freeze = intel_idle_freeze, }, 8810080d65bSJacob Pan { 882de09cdd0SLen Brown .name = "C1E", 8830080d65bSJacob Pan .desc = "MWAIT 0x01", 8840080d65bSJacob Pan .flags = MWAIT2flg(0x01), 8850080d65bSJacob Pan .exit_latency = 10, 8860080d65bSJacob Pan .target_residency = 20, 8870080d65bSJacob Pan .enter = &intel_idle, 8880080d65bSJacob Pan .enter_freeze = intel_idle_freeze, }, 8890080d65bSJacob Pan { 890de09cdd0SLen Brown .name = "C6", 8910080d65bSJacob Pan .desc = "MWAIT 0x20", 8920080d65bSJacob Pan .flags = MWAIT2flg(0x20) | CPUIDLE_FLAG_TLB_FLUSHED, 8930080d65bSJacob Pan .exit_latency = 50, 8940080d65bSJacob Pan .target_residency = 500, 8950080d65bSJacob Pan .enter = &intel_idle, 8960080d65bSJacob Pan .enter_freeze = intel_idle_freeze, }, 8970080d65bSJacob Pan { 8980080d65bSJacob Pan .enter = NULL } 8990080d65bSJacob Pan }; 9000080d65bSJacob Pan 90126717172SLen Brown /** 90226717172SLen Brown * intel_idle 90326717172SLen Brown * @dev: cpuidle_device 90446bcfad7SDeepthi Dharwar * @drv: cpuidle driver 905e978aa7dSDeepthi Dharwar * @index: index of cpuidle state 90626717172SLen Brown * 90763ff07beSYanmin Zhang * Must be called under local_irq_disable(). 90826717172SLen Brown */ 9096727ad9eSChris Metcalf static __cpuidle int intel_idle(struct cpuidle_device *dev, 91046bcfad7SDeepthi Dharwar struct cpuidle_driver *drv, int index) 91126717172SLen Brown { 91226717172SLen Brown unsigned long ecx = 1; /* break on interrupt flag */ 91346bcfad7SDeepthi Dharwar struct cpuidle_state *state = &drv->states[index]; 914b1beab48SLen Brown unsigned long eax = flg2MWAIT(state->flags); 91526717172SLen Brown unsigned int cstate; 91626717172SLen Brown int cpu = smp_processor_id(); 91726717172SLen Brown 91826717172SLen Brown cstate = (((eax) >> MWAIT_SUBSTATE_SIZE) & MWAIT_CSTATE_MASK) + 1; 91926717172SLen Brown 9206110a1f4SSuresh Siddha /* 921c8381cc3SLen Brown * leave_mm() to avoid costly and often unnecessary wakeups 922c8381cc3SLen Brown * for flushing the user TLB's associated with the active mm. 9236110a1f4SSuresh Siddha */ 924c8381cc3SLen Brown if (state->flags & CPUIDLE_FLAG_TLB_FLUSHED) 9256110a1f4SSuresh Siddha leave_mm(cpu); 9266110a1f4SSuresh Siddha 92726717172SLen Brown if (!(lapic_timer_reliable_states & (1 << (cstate)))) 928f6cee191SThomas Gleixner tick_broadcast_enter(); 92926717172SLen Brown 93016824255SPeter Zijlstra mwait_idle_with_hints(eax, ecx); 93126717172SLen Brown 93226717172SLen Brown if (!(lapic_timer_reliable_states & (1 << (cstate)))) 933f6cee191SThomas Gleixner tick_broadcast_exit(); 93426717172SLen Brown 935e978aa7dSDeepthi Dharwar return index; 93626717172SLen Brown } 93726717172SLen Brown 9385fe2e527SRafael J. Wysocki /** 9395fe2e527SRafael J. Wysocki * intel_idle_freeze - simplified "enter" callback routine for suspend-to-idle 9405fe2e527SRafael J. Wysocki * @dev: cpuidle_device 9415fe2e527SRafael J. Wysocki * @drv: cpuidle driver 9425fe2e527SRafael J. Wysocki * @index: state index 9435fe2e527SRafael J. Wysocki */ 9445fe2e527SRafael J. Wysocki static void intel_idle_freeze(struct cpuidle_device *dev, 9455fe2e527SRafael J. Wysocki struct cpuidle_driver *drv, int index) 9465fe2e527SRafael J. Wysocki { 9475fe2e527SRafael J. Wysocki unsigned long ecx = 1; /* break on interrupt flag */ 9485fe2e527SRafael J. Wysocki unsigned long eax = flg2MWAIT(drv->states[index].flags); 9495fe2e527SRafael J. Wysocki 9505fe2e527SRafael J. Wysocki mwait_idle_with_hints(eax, ecx); 9515fe2e527SRafael J. Wysocki } 9525fe2e527SRafael J. Wysocki 953fb1013a0SSebastian Andrzej Siewior static void __setup_broadcast_timer(bool on) 9542a2d31c8SShaohua Li { 95576962caaSThomas Gleixner if (on) 95676962caaSThomas Gleixner tick_broadcast_enable(); 95776962caaSThomas Gleixner else 95876962caaSThomas Gleixner tick_broadcast_disable(); 9592a2d31c8SShaohua Li } 9602a2d31c8SShaohua Li 961fb1013a0SSebastian Andrzej Siewior static void auto_demotion_disable(void) 96214796fcaSLen Brown { 96314796fcaSLen Brown unsigned long long msr_bits; 96414796fcaSLen Brown 9656cfb2374SLen Brown rdmsrl(MSR_PKG_CST_CONFIG_CONTROL, msr_bits); 966b66b8b9aSAndi Kleen msr_bits &= ~(icpu->auto_demotion_disable_flags); 9676cfb2374SLen Brown wrmsrl(MSR_PKG_CST_CONFIG_CONTROL, msr_bits); 96814796fcaSLen Brown } 969fb1013a0SSebastian Andrzej Siewior static void c1e_promotion_disable(void) 97032e95180SLen Brown { 97132e95180SLen Brown unsigned long long msr_bits; 97232e95180SLen Brown 97332e95180SLen Brown rdmsrl(MSR_IA32_POWER_CTL, msr_bits); 97432e95180SLen Brown msr_bits &= ~0x2; 97532e95180SLen Brown wrmsrl(MSR_IA32_POWER_CTL, msr_bits); 97632e95180SLen Brown } 97714796fcaSLen Brown 978b66b8b9aSAndi Kleen static const struct idle_cpu idle_cpu_nehalem = { 979b66b8b9aSAndi Kleen .state_table = nehalem_cstates, 980b66b8b9aSAndi Kleen .auto_demotion_disable_flags = NHM_C1_AUTO_DEMOTE | NHM_C3_AUTO_DEMOTE, 98132e95180SLen Brown .disable_promotion_to_c1e = true, 982b66b8b9aSAndi Kleen }; 983b66b8b9aSAndi Kleen 984b66b8b9aSAndi Kleen static const struct idle_cpu idle_cpu_atom = { 985b66b8b9aSAndi Kleen .state_table = atom_cstates, 986b66b8b9aSAndi Kleen }; 987b66b8b9aSAndi Kleen 9885e7ec268SAndy Shevchenko static const struct idle_cpu idle_cpu_tangier = { 9895e7ec268SAndy Shevchenko .state_table = tangier_cstates, 9905e7ec268SAndy Shevchenko }; 9915e7ec268SAndy Shevchenko 992b66b8b9aSAndi Kleen static const struct idle_cpu idle_cpu_lincroft = { 993b66b8b9aSAndi Kleen .state_table = atom_cstates, 994b66b8b9aSAndi Kleen .auto_demotion_disable_flags = ATM_LNC_C6_AUTO_DEMOTE, 995b66b8b9aSAndi Kleen }; 996b66b8b9aSAndi Kleen 997b66b8b9aSAndi Kleen static const struct idle_cpu idle_cpu_snb = { 998b66b8b9aSAndi Kleen .state_table = snb_cstates, 99932e95180SLen Brown .disable_promotion_to_c1e = true, 1000b66b8b9aSAndi Kleen }; 1001b66b8b9aSAndi Kleen 1002718987d6SLen Brown static const struct idle_cpu idle_cpu_byt = { 1003718987d6SLen Brown .state_table = byt_cstates, 1004718987d6SLen Brown .disable_promotion_to_c1e = true, 10058c058d53SLen Brown .byt_auto_demotion_disable_flag = true, 1006718987d6SLen Brown }; 1007718987d6SLen Brown 1008cab07a56SLen Brown static const struct idle_cpu idle_cpu_cht = { 1009cab07a56SLen Brown .state_table = cht_cstates, 1010cab07a56SLen Brown .disable_promotion_to_c1e = true, 1011cab07a56SLen Brown .byt_auto_demotion_disable_flag = true, 1012cab07a56SLen Brown }; 1013cab07a56SLen Brown 10146edab08cSLen Brown static const struct idle_cpu idle_cpu_ivb = { 10156edab08cSLen Brown .state_table = ivb_cstates, 101632e95180SLen Brown .disable_promotion_to_c1e = true, 10176edab08cSLen Brown }; 10186edab08cSLen Brown 10190138d8f0SLen Brown static const struct idle_cpu idle_cpu_ivt = { 10200138d8f0SLen Brown .state_table = ivt_cstates, 10210138d8f0SLen Brown .disable_promotion_to_c1e = true, 10220138d8f0SLen Brown }; 10230138d8f0SLen Brown 102485a4d2d4SLen Brown static const struct idle_cpu idle_cpu_hsw = { 102585a4d2d4SLen Brown .state_table = hsw_cstates, 102632e95180SLen Brown .disable_promotion_to_c1e = true, 102785a4d2d4SLen Brown }; 102885a4d2d4SLen Brown 1029a138b568SLen Brown static const struct idle_cpu idle_cpu_bdw = { 1030a138b568SLen Brown .state_table = bdw_cstates, 1031a138b568SLen Brown .disable_promotion_to_c1e = true, 1032a138b568SLen Brown }; 1033a138b568SLen Brown 1034493f133fSLen Brown static const struct idle_cpu idle_cpu_skl = { 1035493f133fSLen Brown .state_table = skl_cstates, 1036493f133fSLen Brown .disable_promotion_to_c1e = true, 1037493f133fSLen Brown }; 1038493f133fSLen Brown 1039f9e71657SLen Brown static const struct idle_cpu idle_cpu_skx = { 1040f9e71657SLen Brown .state_table = skx_cstates, 1041f9e71657SLen Brown .disable_promotion_to_c1e = true, 1042f9e71657SLen Brown }; 1043493f133fSLen Brown 1044fab04b22SLen Brown static const struct idle_cpu idle_cpu_avn = { 1045fab04b22SLen Brown .state_table = avn_cstates, 1046fab04b22SLen Brown .disable_promotion_to_c1e = true, 1047fab04b22SLen Brown }; 1048fab04b22SLen Brown 1049281baf7aSDasaratharaman Chandramouli static const struct idle_cpu idle_cpu_knl = { 1050281baf7aSDasaratharaman Chandramouli .state_table = knl_cstates, 1051281baf7aSDasaratharaman Chandramouli }; 1052281baf7aSDasaratharaman Chandramouli 10535dcef694SLen Brown static const struct idle_cpu idle_cpu_bxt = { 10545dcef694SLen Brown .state_table = bxt_cstates, 10555dcef694SLen Brown .disable_promotion_to_c1e = true, 10565dcef694SLen Brown }; 10575dcef694SLen Brown 10580080d65bSJacob Pan static const struct idle_cpu idle_cpu_dnv = { 10590080d65bSJacob Pan .state_table = dnv_cstates, 10600080d65bSJacob Pan .disable_promotion_to_c1e = true, 10610080d65bSJacob Pan }; 10620080d65bSJacob Pan 1063b66b8b9aSAndi Kleen #define ICPU(model, cpu) \ 1064b66b8b9aSAndi Kleen { X86_VENDOR_INTEL, 6, model, X86_FEATURE_MWAIT, (unsigned long)&cpu } 1065b66b8b9aSAndi Kleen 1066d5cdc3c4SMathias Krause static const struct x86_cpu_id intel_idle_ids[] __initconst = { 1067db73c5a8SDave Hansen ICPU(INTEL_FAM6_NEHALEM_EP, idle_cpu_nehalem), 1068db73c5a8SDave Hansen ICPU(INTEL_FAM6_NEHALEM, idle_cpu_nehalem), 10694b3b234fSDave Hansen ICPU(INTEL_FAM6_NEHALEM_G, idle_cpu_nehalem), 1070db73c5a8SDave Hansen ICPU(INTEL_FAM6_WESTMERE, idle_cpu_nehalem), 1071db73c5a8SDave Hansen ICPU(INTEL_FAM6_WESTMERE_EP, idle_cpu_nehalem), 1072db73c5a8SDave Hansen ICPU(INTEL_FAM6_NEHALEM_EX, idle_cpu_nehalem), 1073db73c5a8SDave Hansen ICPU(INTEL_FAM6_ATOM_PINEVIEW, idle_cpu_atom), 1074db73c5a8SDave Hansen ICPU(INTEL_FAM6_ATOM_LINCROFT, idle_cpu_lincroft), 1075db73c5a8SDave Hansen ICPU(INTEL_FAM6_WESTMERE_EX, idle_cpu_nehalem), 1076db73c5a8SDave Hansen ICPU(INTEL_FAM6_SANDYBRIDGE, idle_cpu_snb), 1077db73c5a8SDave Hansen ICPU(INTEL_FAM6_SANDYBRIDGE_X, idle_cpu_snb), 1078db73c5a8SDave Hansen ICPU(INTEL_FAM6_ATOM_CEDARVIEW, idle_cpu_atom), 1079db73c5a8SDave Hansen ICPU(INTEL_FAM6_ATOM_SILVERMONT1, idle_cpu_byt), 10805e7ec268SAndy Shevchenko ICPU(INTEL_FAM6_ATOM_MERRIFIELD, idle_cpu_tangier), 1081db73c5a8SDave Hansen ICPU(INTEL_FAM6_ATOM_AIRMONT, idle_cpu_cht), 1082db73c5a8SDave Hansen ICPU(INTEL_FAM6_IVYBRIDGE, idle_cpu_ivb), 1083db73c5a8SDave Hansen ICPU(INTEL_FAM6_IVYBRIDGE_X, idle_cpu_ivt), 1084db73c5a8SDave Hansen ICPU(INTEL_FAM6_HASWELL_CORE, idle_cpu_hsw), 1085db73c5a8SDave Hansen ICPU(INTEL_FAM6_HASWELL_X, idle_cpu_hsw), 1086db73c5a8SDave Hansen ICPU(INTEL_FAM6_HASWELL_ULT, idle_cpu_hsw), 1087db73c5a8SDave Hansen ICPU(INTEL_FAM6_HASWELL_GT3E, idle_cpu_hsw), 1088db73c5a8SDave Hansen ICPU(INTEL_FAM6_ATOM_SILVERMONT2, idle_cpu_avn), 1089db73c5a8SDave Hansen ICPU(INTEL_FAM6_BROADWELL_CORE, idle_cpu_bdw), 1090db73c5a8SDave Hansen ICPU(INTEL_FAM6_BROADWELL_GT3E, idle_cpu_bdw), 1091db73c5a8SDave Hansen ICPU(INTEL_FAM6_BROADWELL_X, idle_cpu_bdw), 1092db73c5a8SDave Hansen ICPU(INTEL_FAM6_BROADWELL_XEON_D, idle_cpu_bdw), 1093db73c5a8SDave Hansen ICPU(INTEL_FAM6_SKYLAKE_MOBILE, idle_cpu_skl), 1094db73c5a8SDave Hansen ICPU(INTEL_FAM6_SKYLAKE_DESKTOP, idle_cpu_skl), 1095db73c5a8SDave Hansen ICPU(INTEL_FAM6_KABYLAKE_MOBILE, idle_cpu_skl), 1096db73c5a8SDave Hansen ICPU(INTEL_FAM6_KABYLAKE_DESKTOP, idle_cpu_skl), 1097db73c5a8SDave Hansen ICPU(INTEL_FAM6_SKYLAKE_X, idle_cpu_skx), 1098db73c5a8SDave Hansen ICPU(INTEL_FAM6_XEON_PHI_KNL, idle_cpu_knl), 1099a2c1bc64SPiotr Luc ICPU(INTEL_FAM6_XEON_PHI_KNM, idle_cpu_knl), 1100db73c5a8SDave Hansen ICPU(INTEL_FAM6_ATOM_GOLDMONT, idle_cpu_bxt), 11011b2e8768SDavid E. Box ICPU(INTEL_FAM6_ATOM_GEMINI_LAKE, idle_cpu_bxt), 11020080d65bSJacob Pan ICPU(INTEL_FAM6_ATOM_DENVERTON, idle_cpu_dnv), 1103b66b8b9aSAndi Kleen {} 1104b66b8b9aSAndi Kleen }; 1105b66b8b9aSAndi Kleen 110626717172SLen Brown /* 110726717172SLen Brown * intel_idle_probe() 110826717172SLen Brown */ 110900f3e755SBartlomiej Zolnierkiewicz static int __init intel_idle_probe(void) 111026717172SLen Brown { 1111c4236282SLen Brown unsigned int eax, ebx, ecx; 1112b66b8b9aSAndi Kleen const struct x86_cpu_id *id; 111326717172SLen Brown 111426717172SLen Brown if (max_cstate == 0) { 1115654d08a4SJoe Perches pr_debug("disabled\n"); 111626717172SLen Brown return -EPERM; 111726717172SLen Brown } 111826717172SLen Brown 1119b66b8b9aSAndi Kleen id = x86_match_cpu(intel_idle_ids); 1120b66b8b9aSAndi Kleen if (!id) { 1121b66b8b9aSAndi Kleen if (boot_cpu_data.x86_vendor == X86_VENDOR_INTEL && 1122b66b8b9aSAndi Kleen boot_cpu_data.x86 == 6) 1123654d08a4SJoe Perches pr_debug("does not run on family %d model %d\n", 1124b66b8b9aSAndi Kleen boot_cpu_data.x86, boot_cpu_data.x86_model); 112526717172SLen Brown return -ENODEV; 1126b66b8b9aSAndi Kleen } 112726717172SLen Brown 112826717172SLen Brown if (boot_cpu_data.cpuid_level < CPUID_MWAIT_LEAF) 112926717172SLen Brown return -ENODEV; 113026717172SLen Brown 1131c4236282SLen Brown cpuid(CPUID_MWAIT_LEAF, &eax, &ebx, &ecx, &mwait_substates); 113226717172SLen Brown 113326717172SLen Brown if (!(ecx & CPUID5_ECX_EXTENSIONS_SUPPORTED) || 11345c2a9f06SThomas Renninger !(ecx & CPUID5_ECX_INTERRUPT_BREAK) || 11355c2a9f06SThomas Renninger !mwait_substates) 113626717172SLen Brown return -ENODEV; 113726717172SLen Brown 1138654d08a4SJoe Perches pr_debug("MWAIT substates: 0x%x\n", mwait_substates); 113926717172SLen Brown 1140b66b8b9aSAndi Kleen icpu = (const struct idle_cpu *)id->driver_data; 1141b66b8b9aSAndi Kleen cpuidle_state_table = icpu->state_table; 114226717172SLen Brown 1143654d08a4SJoe Perches pr_debug("v" INTEL_IDLE_VERSION " model 0x%X\n", 1144654d08a4SJoe Perches boot_cpu_data.x86_model); 114526717172SLen Brown 114626717172SLen Brown return 0; 114726717172SLen Brown } 114826717172SLen Brown 114926717172SLen Brown /* 115026717172SLen Brown * intel_idle_cpuidle_devices_uninit() 1151ca42489dSRichard Cochran * Unregisters the cpuidle devices. 115226717172SLen Brown */ 115326717172SLen Brown static void intel_idle_cpuidle_devices_uninit(void) 115426717172SLen Brown { 115526717172SLen Brown int i; 115626717172SLen Brown struct cpuidle_device *dev; 115726717172SLen Brown 115826717172SLen Brown for_each_online_cpu(i) { 115926717172SLen Brown dev = per_cpu_ptr(intel_idle_cpuidle_devices, i); 116026717172SLen Brown cpuidle_unregister_device(dev); 116126717172SLen Brown } 116226717172SLen Brown } 11630138d8f0SLen Brown 11640138d8f0SLen Brown /* 1165d70e28f5SLen Brown * ivt_idle_state_table_update(void) 11660138d8f0SLen Brown * 1167d70e28f5SLen Brown * Tune IVT multi-socket targets 11680138d8f0SLen Brown * Assumption: num_sockets == (max_package_num + 1) 11690138d8f0SLen Brown */ 1170d70e28f5SLen Brown static void ivt_idle_state_table_update(void) 11710138d8f0SLen Brown { 11720138d8f0SLen Brown /* IVT uses a different table for 1-2, 3-4, and > 4 sockets */ 11730138d8f0SLen Brown int cpu, package_num, num_sockets = 1; 11740138d8f0SLen Brown 11750138d8f0SLen Brown for_each_online_cpu(cpu) { 11760138d8f0SLen Brown package_num = topology_physical_package_id(cpu); 11770138d8f0SLen Brown if (package_num + 1 > num_sockets) { 11780138d8f0SLen Brown num_sockets = package_num + 1; 11790138d8f0SLen Brown 1180d27dca42SChristoph Jaeger if (num_sockets > 4) { 11810138d8f0SLen Brown cpuidle_state_table = ivt_cstates_8s; 11820138d8f0SLen Brown return; 11830138d8f0SLen Brown } 11840138d8f0SLen Brown } 1185d27dca42SChristoph Jaeger } 11860138d8f0SLen Brown 11870138d8f0SLen Brown if (num_sockets > 2) 11880138d8f0SLen Brown cpuidle_state_table = ivt_cstates_4s; 1189d70e28f5SLen Brown 11900138d8f0SLen Brown /* else, 1 and 2 socket systems use default ivt_cstates */ 11910138d8f0SLen Brown } 11925dcef694SLen Brown 11935dcef694SLen Brown /* 11945dcef694SLen Brown * Translate IRTL (Interrupt Response Time Limit) MSR to usec 11955dcef694SLen Brown */ 11965dcef694SLen Brown 11975dcef694SLen Brown static unsigned int irtl_ns_units[] = { 11985dcef694SLen Brown 1, 32, 1024, 32768, 1048576, 33554432, 0, 0 }; 11995dcef694SLen Brown 12005dcef694SLen Brown static unsigned long long irtl_2_usec(unsigned long long irtl) 12015dcef694SLen Brown { 12025dcef694SLen Brown unsigned long long ns; 12035dcef694SLen Brown 12043451ab3eSJan Beulich if (!irtl) 12053451ab3eSJan Beulich return 0; 12063451ab3eSJan Beulich 1207bef45096SJan Beulich ns = irtl_ns_units[(irtl >> 10) & 0x7]; 12085dcef694SLen Brown 12095dcef694SLen Brown return div64_u64((irtl & 0x3FF) * ns, 1000); 12105dcef694SLen Brown } 12115dcef694SLen Brown /* 12125dcef694SLen Brown * bxt_idle_state_table_update(void) 12135dcef694SLen Brown * 12145dcef694SLen Brown * On BXT, we trust the IRTL to show the definitive maximum latency 12155dcef694SLen Brown * We use the same value for target_residency. 12165dcef694SLen Brown */ 12175dcef694SLen Brown static void bxt_idle_state_table_update(void) 12185dcef694SLen Brown { 12195dcef694SLen Brown unsigned long long msr; 12203451ab3eSJan Beulich unsigned int usec; 12215dcef694SLen Brown 12225dcef694SLen Brown rdmsrl(MSR_PKGC6_IRTL, msr); 12233451ab3eSJan Beulich usec = irtl_2_usec(msr); 12243451ab3eSJan Beulich if (usec) { 12255dcef694SLen Brown bxt_cstates[2].exit_latency = usec; 12265dcef694SLen Brown bxt_cstates[2].target_residency = usec; 12275dcef694SLen Brown } 12285dcef694SLen Brown 12295dcef694SLen Brown rdmsrl(MSR_PKGC7_IRTL, msr); 12303451ab3eSJan Beulich usec = irtl_2_usec(msr); 12313451ab3eSJan Beulich if (usec) { 12325dcef694SLen Brown bxt_cstates[3].exit_latency = usec; 12335dcef694SLen Brown bxt_cstates[3].target_residency = usec; 12345dcef694SLen Brown } 12355dcef694SLen Brown 12365dcef694SLen Brown rdmsrl(MSR_PKGC8_IRTL, msr); 12373451ab3eSJan Beulich usec = irtl_2_usec(msr); 12383451ab3eSJan Beulich if (usec) { 12395dcef694SLen Brown bxt_cstates[4].exit_latency = usec; 12405dcef694SLen Brown bxt_cstates[4].target_residency = usec; 12415dcef694SLen Brown } 12425dcef694SLen Brown 12435dcef694SLen Brown rdmsrl(MSR_PKGC9_IRTL, msr); 12443451ab3eSJan Beulich usec = irtl_2_usec(msr); 12453451ab3eSJan Beulich if (usec) { 12465dcef694SLen Brown bxt_cstates[5].exit_latency = usec; 12475dcef694SLen Brown bxt_cstates[5].target_residency = usec; 12485dcef694SLen Brown } 12495dcef694SLen Brown 12505dcef694SLen Brown rdmsrl(MSR_PKGC10_IRTL, msr); 12513451ab3eSJan Beulich usec = irtl_2_usec(msr); 12523451ab3eSJan Beulich if (usec) { 12535dcef694SLen Brown bxt_cstates[6].exit_latency = usec; 12545dcef694SLen Brown bxt_cstates[6].target_residency = usec; 12555dcef694SLen Brown } 12565dcef694SLen Brown 12575dcef694SLen Brown } 1258d70e28f5SLen Brown /* 1259d70e28f5SLen Brown * sklh_idle_state_table_update(void) 1260d70e28f5SLen Brown * 1261d70e28f5SLen Brown * On SKL-H (model 0x5e) disable C8 and C9 if: 1262d70e28f5SLen Brown * C10 is enabled and SGX disabled 1263d70e28f5SLen Brown */ 1264d70e28f5SLen Brown static void sklh_idle_state_table_update(void) 1265d70e28f5SLen Brown { 1266d70e28f5SLen Brown unsigned long long msr; 1267d70e28f5SLen Brown unsigned int eax, ebx, ecx, edx; 1268d70e28f5SLen Brown 1269d70e28f5SLen Brown 1270d70e28f5SLen Brown /* if PC10 disabled via cmdline intel_idle.max_cstate=7 or shallower */ 1271d70e28f5SLen Brown if (max_cstate <= 7) 12720138d8f0SLen Brown return; 1273d70e28f5SLen Brown 1274d70e28f5SLen Brown /* if PC10 not present in CPUID.MWAIT.EDX */ 1275d70e28f5SLen Brown if ((mwait_substates & (0xF << 28)) == 0) 1276d70e28f5SLen Brown return; 1277d70e28f5SLen Brown 12786cfb2374SLen Brown rdmsrl(MSR_PKG_CST_CONFIG_CONTROL, msr); 1279d70e28f5SLen Brown 1280d70e28f5SLen Brown /* PC10 is not enabled in PKG C-state limit */ 1281d70e28f5SLen Brown if ((msr & 0xF) != 8) 1282d70e28f5SLen Brown return; 1283d70e28f5SLen Brown 1284d70e28f5SLen Brown ecx = 0; 1285d70e28f5SLen Brown cpuid(7, &eax, &ebx, &ecx, &edx); 1286d70e28f5SLen Brown 1287d70e28f5SLen Brown /* if SGX is present */ 1288d70e28f5SLen Brown if (ebx & (1 << 2)) { 1289d70e28f5SLen Brown 1290d70e28f5SLen Brown rdmsrl(MSR_IA32_FEATURE_CONTROL, msr); 1291d70e28f5SLen Brown 1292d70e28f5SLen Brown /* if SGX is enabled */ 1293d70e28f5SLen Brown if (msr & (1 << 18)) 1294d70e28f5SLen Brown return; 1295d70e28f5SLen Brown } 1296d70e28f5SLen Brown 1297d70e28f5SLen Brown skl_cstates[5].disabled = 1; /* C8-SKL */ 1298d70e28f5SLen Brown skl_cstates[6].disabled = 1; /* C9-SKL */ 1299d70e28f5SLen Brown } 1300d70e28f5SLen Brown /* 1301d70e28f5SLen Brown * intel_idle_state_table_update() 1302d70e28f5SLen Brown * 1303d70e28f5SLen Brown * Update the default state_table for this CPU-id 1304d70e28f5SLen Brown */ 1305d70e28f5SLen Brown 1306d70e28f5SLen Brown static void intel_idle_state_table_update(void) 1307d70e28f5SLen Brown { 1308d70e28f5SLen Brown switch (boot_cpu_data.x86_model) { 1309d70e28f5SLen Brown 1310db73c5a8SDave Hansen case INTEL_FAM6_IVYBRIDGE_X: 1311d70e28f5SLen Brown ivt_idle_state_table_update(); 1312d70e28f5SLen Brown break; 1313db73c5a8SDave Hansen case INTEL_FAM6_ATOM_GOLDMONT: 13141b2e8768SDavid E. Box case INTEL_FAM6_ATOM_GEMINI_LAKE: 13155dcef694SLen Brown bxt_idle_state_table_update(); 13165dcef694SLen Brown break; 1317db73c5a8SDave Hansen case INTEL_FAM6_SKYLAKE_DESKTOP: 1318d70e28f5SLen Brown sklh_idle_state_table_update(); 1319d70e28f5SLen Brown break; 1320d70e28f5SLen Brown } 13210138d8f0SLen Brown } 13220138d8f0SLen Brown 132326717172SLen Brown /* 132446bcfad7SDeepthi Dharwar * intel_idle_cpuidle_driver_init() 132546bcfad7SDeepthi Dharwar * allocate, initialize cpuidle_states 132646bcfad7SDeepthi Dharwar */ 13275469c827SRichard Cochran static void __init intel_idle_cpuidle_driver_init(void) 132846bcfad7SDeepthi Dharwar { 132946bcfad7SDeepthi Dharwar int cstate; 133046bcfad7SDeepthi Dharwar struct cpuidle_driver *drv = &intel_idle_driver; 133146bcfad7SDeepthi Dharwar 13320138d8f0SLen Brown intel_idle_state_table_update(); 13330138d8f0SLen Brown 13341b39e3f8SRafael J. Wysocki cpuidle_poll_state_init(drv); 133546bcfad7SDeepthi Dharwar drv->state_count = 1; 133646bcfad7SDeepthi Dharwar 1337e022e7ebSLen Brown for (cstate = 0; cstate < CPUIDLE_STATE_MAX; ++cstate) { 133824bfa950SLen Brown int num_substates, mwait_hint, mwait_cstate; 133946bcfad7SDeepthi Dharwar 13407dd0e0afSLen Brown if ((cpuidle_state_table[cstate].enter == NULL) && 13417dd0e0afSLen Brown (cpuidle_state_table[cstate].enter_freeze == NULL)) 1342e022e7ebSLen Brown break; 1343e022e7ebSLen Brown 1344e022e7ebSLen Brown if (cstate + 1 > max_cstate) { 1345654d08a4SJoe Perches pr_info("max_cstate %d reached\n", max_cstate); 134646bcfad7SDeepthi Dharwar break; 134746bcfad7SDeepthi Dharwar } 134846bcfad7SDeepthi Dharwar 1349e022e7ebSLen Brown mwait_hint = flg2MWAIT(cpuidle_state_table[cstate].flags); 1350e022e7ebSLen Brown mwait_cstate = MWAIT_HINT2CSTATE(mwait_hint); 135146bcfad7SDeepthi Dharwar 135224bfa950SLen Brown /* number of sub-states for this state in CPUID.MWAIT */ 1353e022e7ebSLen Brown num_substates = (mwait_substates >> ((mwait_cstate + 1) * 4)) 1354e022e7ebSLen Brown & MWAIT_SUBSTATE_MASK; 1355e022e7ebSLen Brown 135624bfa950SLen Brown /* if NO sub-states for this state in CPUID, skip it */ 135724bfa950SLen Brown if (num_substates == 0) 1358e022e7ebSLen Brown continue; 1359e022e7ebSLen Brown 1360d70e28f5SLen Brown /* if state marked as disabled, skip it */ 1361d70e28f5SLen Brown if (cpuidle_state_table[cstate].disabled != 0) { 1362654d08a4SJoe Perches pr_debug("state %s is disabled\n", 1363d70e28f5SLen Brown cpuidle_state_table[cstate].name); 1364d70e28f5SLen Brown continue; 1365d70e28f5SLen Brown } 1366d70e28f5SLen Brown 1367d70e28f5SLen Brown 1368e022e7ebSLen Brown if (((mwait_cstate + 1) > 2) && 136946bcfad7SDeepthi Dharwar !boot_cpu_has(X86_FEATURE_NONSTOP_TSC)) 137046bcfad7SDeepthi Dharwar mark_tsc_unstable("TSC halts in idle" 137146bcfad7SDeepthi Dharwar " states deeper than C2"); 137246bcfad7SDeepthi Dharwar 137346bcfad7SDeepthi Dharwar drv->states[drv->state_count] = /* structure copy */ 137446bcfad7SDeepthi Dharwar cpuidle_state_table[cstate]; 137546bcfad7SDeepthi Dharwar 137646bcfad7SDeepthi Dharwar drv->state_count += 1; 137746bcfad7SDeepthi Dharwar } 137846bcfad7SDeepthi Dharwar 13798c058d53SLen Brown if (icpu->byt_auto_demotion_disable_flag) { 13808c058d53SLen Brown wrmsrl(MSR_CC6_DEMOTION_POLICY_CONFIG, 0); 13818c058d53SLen Brown wrmsrl(MSR_MC6_DEMOTION_POLICY_CONFIG, 0); 13828c058d53SLen Brown } 138346bcfad7SDeepthi Dharwar } 138446bcfad7SDeepthi Dharwar 138546bcfad7SDeepthi Dharwar 138646bcfad7SDeepthi Dharwar /* 138765b7f839SThomas Renninger * intel_idle_cpu_init() 138826717172SLen Brown * allocate, initialize, register cpuidle_devices 138965b7f839SThomas Renninger * @cpu: cpu/core to initialize 139026717172SLen Brown */ 1391fb1013a0SSebastian Andrzej Siewior static int intel_idle_cpu_init(unsigned int cpu) 139226717172SLen Brown { 139326717172SLen Brown struct cpuidle_device *dev; 139426717172SLen Brown 139565b7f839SThomas Renninger dev = per_cpu_ptr(intel_idle_cpuidle_devices, cpu); 139665b7f839SThomas Renninger dev->cpu = cpu; 139726717172SLen Brown 139826717172SLen Brown if (cpuidle_register_device(dev)) { 1399654d08a4SJoe Perches pr_debug("cpuidle_register_device %d failed!\n", cpu); 140026717172SLen Brown return -EIO; 140126717172SLen Brown } 140265b7f839SThomas Renninger 1403b66b8b9aSAndi Kleen if (icpu->auto_demotion_disable_flags) 1404fb1013a0SSebastian Andrzej Siewior auto_demotion_disable(); 140526717172SLen Brown 1406dbf87ab8SBartlomiej Zolnierkiewicz if (icpu->disable_promotion_to_c1e) 1407fb1013a0SSebastian Andrzej Siewior c1e_promotion_disable(); 1408fb1013a0SSebastian Andrzej Siewior 1409fb1013a0SSebastian Andrzej Siewior return 0; 1410fb1013a0SSebastian Andrzej Siewior } 1411fb1013a0SSebastian Andrzej Siewior 1412fb1013a0SSebastian Andrzej Siewior static int intel_idle_cpu_online(unsigned int cpu) 1413fb1013a0SSebastian Andrzej Siewior { 1414fb1013a0SSebastian Andrzej Siewior struct cpuidle_device *dev; 1415fb1013a0SSebastian Andrzej Siewior 1416fb1013a0SSebastian Andrzej Siewior if (lapic_timer_reliable_states != LAPIC_TIMER_ALWAYS_RELIABLE) 1417fb1013a0SSebastian Andrzej Siewior __setup_broadcast_timer(true); 1418fb1013a0SSebastian Andrzej Siewior 1419fb1013a0SSebastian Andrzej Siewior /* 1420fb1013a0SSebastian Andrzej Siewior * Some systems can hotplug a cpu at runtime after 1421fb1013a0SSebastian Andrzej Siewior * the kernel has booted, we have to initialize the 1422fb1013a0SSebastian Andrzej Siewior * driver in this case 1423fb1013a0SSebastian Andrzej Siewior */ 1424fb1013a0SSebastian Andrzej Siewior dev = per_cpu_ptr(intel_idle_cpuidle_devices, cpu); 1425fb1013a0SSebastian Andrzej Siewior if (!dev->registered) 1426fb1013a0SSebastian Andrzej Siewior return intel_idle_cpu_init(cpu); 1427dbf87ab8SBartlomiej Zolnierkiewicz 142826717172SLen Brown return 0; 142926717172SLen Brown } 143026717172SLen Brown 143126717172SLen Brown static int __init intel_idle_init(void) 143226717172SLen Brown { 1433fb1013a0SSebastian Andrzej Siewior int retval; 143426717172SLen Brown 1435d1896049SThomas Renninger /* Do not load intel_idle at all for now if idle= is passed */ 1436d1896049SThomas Renninger if (boot_option_idle_override != IDLE_NO_OVERRIDE) 1437d1896049SThomas Renninger return -ENODEV; 1438d1896049SThomas Renninger 143926717172SLen Brown retval = intel_idle_probe(); 144026717172SLen Brown if (retval) 144126717172SLen Brown return retval; 144226717172SLen Brown 1443e9df69ccSRichard Cochran intel_idle_cpuidle_devices = alloc_percpu(struct cpuidle_device); 1444e9df69ccSRichard Cochran if (intel_idle_cpuidle_devices == NULL) 1445e9df69ccSRichard Cochran return -ENOMEM; 1446e9df69ccSRichard Cochran 144746bcfad7SDeepthi Dharwar intel_idle_cpuidle_driver_init(); 144826717172SLen Brown retval = cpuidle_register_driver(&intel_idle_driver); 144926717172SLen Brown if (retval) { 14503735d524SKonrad Rzeszutek Wilk struct cpuidle_driver *drv = cpuidle_get_driver(); 1451654d08a4SJoe Perches printk(KERN_DEBUG pr_fmt("intel_idle yielding to %s\n"), 14523735d524SKonrad Rzeszutek Wilk drv ? drv->name : "none"); 1453fb1013a0SSebastian Andrzej Siewior goto init_driver_fail; 145426717172SLen Brown } 145526717172SLen Brown 14562259a819SRichard Cochran if (boot_cpu_has(X86_FEATURE_ARAT)) /* Always Reliable APIC Timer */ 14572259a819SRichard Cochran lapic_timer_reliable_states = LAPIC_TIMER_ALWAYS_RELIABLE; 14582259a819SRichard Cochran 1459fb1013a0SSebastian Andrzej Siewior retval = cpuhp_setup_state(CPUHP_AP_ONLINE_DYN, "idle/intel:online", 1460fb1013a0SSebastian Andrzej Siewior intel_idle_cpu_online, NULL); 1461fb1013a0SSebastian Andrzej Siewior if (retval < 0) 1462fb1013a0SSebastian Andrzej Siewior goto hp_setup_fail; 146326717172SLen Brown 1464654d08a4SJoe Perches pr_debug("lapic_timer_reliable_states 0x%x\n", 14652259a819SRichard Cochran lapic_timer_reliable_states); 14662259a819SRichard Cochran 146726717172SLen Brown return 0; 1468fb1013a0SSebastian Andrzej Siewior 1469fb1013a0SSebastian Andrzej Siewior hp_setup_fail: 1470fb1013a0SSebastian Andrzej Siewior intel_idle_cpuidle_devices_uninit(); 1471fb1013a0SSebastian Andrzej Siewior cpuidle_unregister_driver(&intel_idle_driver); 1472fb1013a0SSebastian Andrzej Siewior init_driver_fail: 1473fb1013a0SSebastian Andrzej Siewior free_percpu(intel_idle_cpuidle_devices); 1474fb1013a0SSebastian Andrzej Siewior return retval; 1475fb1013a0SSebastian Andrzej Siewior 147626717172SLen Brown } 147702c4fae9SPaul Gortmaker device_initcall(intel_idle_init); 147826717172SLen Brown 147902c4fae9SPaul Gortmaker /* 148002c4fae9SPaul Gortmaker * We are not really modular, but we used to support that. Meaning we also 148102c4fae9SPaul Gortmaker * support "intel_idle.max_cstate=..." at boot and also a read-only export of 148202c4fae9SPaul Gortmaker * it at /sys/module/intel_idle/parameters/max_cstate -- so using module_param 148302c4fae9SPaul Gortmaker * is the easiest way (currently) to continue doing that. 148402c4fae9SPaul Gortmaker */ 148526717172SLen Brown module_param(max_cstate, int, 0444); 1486