126717172SLen Brown /* 226717172SLen Brown * intel_idle.c - native hardware idle loop for modern Intel processors 326717172SLen Brown * 4fab04b22SLen Brown * Copyright (c) 2013, Intel Corporation. 526717172SLen Brown * Len Brown <len.brown@intel.com> 626717172SLen Brown * 726717172SLen Brown * This program is free software; you can redistribute it and/or modify it 826717172SLen Brown * under the terms and conditions of the GNU General Public License, 926717172SLen Brown * version 2, as published by the Free Software Foundation. 1026717172SLen Brown * 1126717172SLen Brown * This program is distributed in the hope it will be useful, but WITHOUT 1226717172SLen Brown * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or 1326717172SLen Brown * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for 1426717172SLen Brown * more details. 1526717172SLen Brown * 1626717172SLen Brown * You should have received a copy of the GNU General Public License along with 1726717172SLen Brown * this program; if not, write to the Free Software Foundation, Inc., 1826717172SLen Brown * 51 Franklin St - Fifth Floor, Boston, MA 02110-1301 USA. 1926717172SLen Brown */ 2026717172SLen Brown 2126717172SLen Brown /* 2226717172SLen Brown * intel_idle is a cpuidle driver that loads on specific Intel processors 2326717172SLen Brown * in lieu of the legacy ACPI processor_idle driver. The intent is to 2426717172SLen Brown * make Linux more efficient on these processors, as intel_idle knows 2526717172SLen Brown * more than ACPI, as well as make Linux more immune to ACPI BIOS bugs. 2626717172SLen Brown */ 2726717172SLen Brown 2826717172SLen Brown /* 2926717172SLen Brown * Design Assumptions 3026717172SLen Brown * 3126717172SLen Brown * All CPUs have same idle states as boot CPU 3226717172SLen Brown * 3326717172SLen Brown * Chipset BM_STS (bus master status) bit is a NOP 3426717172SLen Brown * for preventing entry into deep C-stats 3526717172SLen Brown */ 3626717172SLen Brown 3726717172SLen Brown /* 3826717172SLen Brown * Known limitations 3926717172SLen Brown * 4026717172SLen Brown * The driver currently initializes for_each_online_cpu() upon modprobe. 4126717172SLen Brown * It it unaware of subsequent processors hot-added to the system. 4226717172SLen Brown * This means that if you boot with maxcpus=n and later online 4326717172SLen Brown * processors above n, those processors will use C1 only. 4426717172SLen Brown * 4526717172SLen Brown * ACPI has a .suspend hack to turn off deep c-statees during suspend 4626717172SLen Brown * to avoid complications with the lapic timer workaround. 4726717172SLen Brown * Have not seen issues with suspend, but may need same workaround here. 4826717172SLen Brown * 4926717172SLen Brown */ 5026717172SLen Brown 5126717172SLen Brown /* un-comment DEBUG to enable pr_debug() statements */ 5226717172SLen Brown #define DEBUG 5326717172SLen Brown 5426717172SLen Brown #include <linux/kernel.h> 5526717172SLen Brown #include <linux/cpuidle.h> 5676962caaSThomas Gleixner #include <linux/tick.h> 5726717172SLen Brown #include <trace/events/power.h> 5826717172SLen Brown #include <linux/sched.h> 592a2d31c8SShaohua Li #include <linux/notifier.h> 602a2d31c8SShaohua Li #include <linux/cpu.h> 6102c4fae9SPaul Gortmaker #include <linux/moduleparam.h> 62b66b8b9aSAndi Kleen #include <asm/cpu_device_id.h> 63db73c5a8SDave Hansen #include <asm/intel-family.h> 64bc83ccccSH. Peter Anvin #include <asm/mwait.h> 6514796fcaSLen Brown #include <asm/msr.h> 6626717172SLen Brown 67d70e28f5SLen Brown #define INTEL_IDLE_VERSION "0.4.1" 6826717172SLen Brown #define PREFIX "intel_idle: " 6926717172SLen Brown 7026717172SLen Brown static struct cpuidle_driver intel_idle_driver = { 7126717172SLen Brown .name = "intel_idle", 7226717172SLen Brown .owner = THIS_MODULE, 7326717172SLen Brown }; 7426717172SLen Brown /* intel_idle.max_cstate=0 disables driver */ 75137ecc77SLen Brown static int max_cstate = CPUIDLE_STATE_MAX - 1; 7626717172SLen Brown 77c4236282SLen Brown static unsigned int mwait_substates; 7826717172SLen Brown 792a2d31c8SShaohua Li #define LAPIC_TIMER_ALWAYS_RELIABLE 0xFFFFFFFF 8026717172SLen Brown /* Reliable LAPIC Timer States, bit 1 for C1 etc. */ 81d13780d4SLen Brown static unsigned int lapic_timer_reliable_states = (1 << 1); /* Default to only C1 */ 8226717172SLen Brown 83b66b8b9aSAndi Kleen struct idle_cpu { 84b66b8b9aSAndi Kleen struct cpuidle_state *state_table; 8526717172SLen Brown 8626717172SLen Brown /* 8714796fcaSLen Brown * Hardware C-state auto-demotion may not always be optimal. 8814796fcaSLen Brown * Indicate which enable bits to clear here. 8914796fcaSLen Brown */ 90b66b8b9aSAndi Kleen unsigned long auto_demotion_disable_flags; 918c058d53SLen Brown bool byt_auto_demotion_disable_flag; 9232e95180SLen Brown bool disable_promotion_to_c1e; 93b66b8b9aSAndi Kleen }; 94b66b8b9aSAndi Kleen 95b66b8b9aSAndi Kleen static const struct idle_cpu *icpu; 96b66b8b9aSAndi Kleen static struct cpuidle_device __percpu *intel_idle_cpuidle_devices; 97b66b8b9aSAndi Kleen static int intel_idle(struct cpuidle_device *dev, 98b66b8b9aSAndi Kleen struct cpuidle_driver *drv, int index); 995fe2e527SRafael J. Wysocki static void intel_idle_freeze(struct cpuidle_device *dev, 1005fe2e527SRafael J. Wysocki struct cpuidle_driver *drv, int index); 10125ac7761SDaniel Lezcano static int intel_idle_cpu_init(int cpu); 102b66b8b9aSAndi Kleen 103b66b8b9aSAndi Kleen static struct cpuidle_state *cpuidle_state_table; 10414796fcaSLen Brown 10514796fcaSLen Brown /* 106956d033fSLen Brown * Set this flag for states where the HW flushes the TLB for us 107956d033fSLen Brown * and so we don't need cross-calls to keep it consistent. 108956d033fSLen Brown * If this flag is set, SW flushes the TLB, so even if the 109956d033fSLen Brown * HW doesn't do the flushing, this flag is safe to use. 110956d033fSLen Brown */ 111956d033fSLen Brown #define CPUIDLE_FLAG_TLB_FLUSHED 0x10000 112956d033fSLen Brown 113956d033fSLen Brown /* 114b1beab48SLen Brown * MWAIT takes an 8-bit "hint" in EAX "suggesting" 115b1beab48SLen Brown * the C-state (top nibble) and sub-state (bottom nibble) 116b1beab48SLen Brown * 0x00 means "MWAIT(C1)", 0x10 means "MWAIT(C2)" etc. 117b1beab48SLen Brown * 118b1beab48SLen Brown * We store the hint at the top of our "flags" for each state. 119b1beab48SLen Brown */ 120b1beab48SLen Brown #define flg2MWAIT(flags) (((flags) >> 24) & 0xFF) 121b1beab48SLen Brown #define MWAIT2flg(eax) ((eax & 0xFF) << 24) 122b1beab48SLen Brown 123b1beab48SLen Brown /* 12426717172SLen Brown * States are indexed by the cstate number, 12526717172SLen Brown * which is also the index into the MWAIT hint array. 12626717172SLen Brown * Thus C0 is a dummy. 12726717172SLen Brown */ 128ba0dc81eSJiang Liu static struct cpuidle_state nehalem_cstates[] = { 129e022e7ebSLen Brown { 13015e123e5SThomas Renninger .name = "C1-NHM", 13126717172SLen Brown .desc = "MWAIT 0x00", 132b82b6ccaSDaniel Lezcano .flags = MWAIT2flg(0x00), 13326717172SLen Brown .exit_latency = 3, 13426717172SLen Brown .target_residency = 6, 1355fe2e527SRafael J. Wysocki .enter = &intel_idle, 1365fe2e527SRafael J. Wysocki .enter_freeze = intel_idle_freeze, }, 137e022e7ebSLen Brown { 13832e95180SLen Brown .name = "C1E-NHM", 13932e95180SLen Brown .desc = "MWAIT 0x01", 140b82b6ccaSDaniel Lezcano .flags = MWAIT2flg(0x01), 14132e95180SLen Brown .exit_latency = 10, 14232e95180SLen Brown .target_residency = 20, 1435fe2e527SRafael J. Wysocki .enter = &intel_idle, 1445fe2e527SRafael J. Wysocki .enter_freeze = intel_idle_freeze, }, 14532e95180SLen Brown { 14615e123e5SThomas Renninger .name = "C3-NHM", 14726717172SLen Brown .desc = "MWAIT 0x10", 148b82b6ccaSDaniel Lezcano .flags = MWAIT2flg(0x10) | CPUIDLE_FLAG_TLB_FLUSHED, 14926717172SLen Brown .exit_latency = 20, 15026717172SLen Brown .target_residency = 80, 1515fe2e527SRafael J. Wysocki .enter = &intel_idle, 1525fe2e527SRafael J. Wysocki .enter_freeze = intel_idle_freeze, }, 153e022e7ebSLen Brown { 15415e123e5SThomas Renninger .name = "C6-NHM", 15526717172SLen Brown .desc = "MWAIT 0x20", 156b82b6ccaSDaniel Lezcano .flags = MWAIT2flg(0x20) | CPUIDLE_FLAG_TLB_FLUSHED, 15726717172SLen Brown .exit_latency = 200, 15826717172SLen Brown .target_residency = 800, 1595fe2e527SRafael J. Wysocki .enter = &intel_idle, 1605fe2e527SRafael J. Wysocki .enter_freeze = intel_idle_freeze, }, 161e022e7ebSLen Brown { 162e022e7ebSLen Brown .enter = NULL } 16326717172SLen Brown }; 16426717172SLen Brown 165ba0dc81eSJiang Liu static struct cpuidle_state snb_cstates[] = { 166e022e7ebSLen Brown { 16715e123e5SThomas Renninger .name = "C1-SNB", 168d13780d4SLen Brown .desc = "MWAIT 0x00", 169b82b6ccaSDaniel Lezcano .flags = MWAIT2flg(0x00), 17032e95180SLen Brown .exit_latency = 2, 17132e95180SLen Brown .target_residency = 2, 1725fe2e527SRafael J. Wysocki .enter = &intel_idle, 1735fe2e527SRafael J. Wysocki .enter_freeze = intel_idle_freeze, }, 17432e95180SLen Brown { 17532e95180SLen Brown .name = "C1E-SNB", 17632e95180SLen Brown .desc = "MWAIT 0x01", 177b82b6ccaSDaniel Lezcano .flags = MWAIT2flg(0x01), 17832e95180SLen Brown .exit_latency = 10, 17932e95180SLen Brown .target_residency = 20, 1805fe2e527SRafael J. Wysocki .enter = &intel_idle, 1815fe2e527SRafael J. Wysocki .enter_freeze = intel_idle_freeze, }, 182e022e7ebSLen Brown { 18315e123e5SThomas Renninger .name = "C3-SNB", 184d13780d4SLen Brown .desc = "MWAIT 0x10", 185b82b6ccaSDaniel Lezcano .flags = MWAIT2flg(0x10) | CPUIDLE_FLAG_TLB_FLUSHED, 186d13780d4SLen Brown .exit_latency = 80, 187ddbd550dSLen Brown .target_residency = 211, 1885fe2e527SRafael J. Wysocki .enter = &intel_idle, 1895fe2e527SRafael J. Wysocki .enter_freeze = intel_idle_freeze, }, 190e022e7ebSLen Brown { 19115e123e5SThomas Renninger .name = "C6-SNB", 192d13780d4SLen Brown .desc = "MWAIT 0x20", 193b82b6ccaSDaniel Lezcano .flags = MWAIT2flg(0x20) | CPUIDLE_FLAG_TLB_FLUSHED, 194d13780d4SLen Brown .exit_latency = 104, 195ddbd550dSLen Brown .target_residency = 345, 1965fe2e527SRafael J. Wysocki .enter = &intel_idle, 1975fe2e527SRafael J. Wysocki .enter_freeze = intel_idle_freeze, }, 198e022e7ebSLen Brown { 19915e123e5SThomas Renninger .name = "C7-SNB", 200d13780d4SLen Brown .desc = "MWAIT 0x30", 201b82b6ccaSDaniel Lezcano .flags = MWAIT2flg(0x30) | CPUIDLE_FLAG_TLB_FLUSHED, 202d13780d4SLen Brown .exit_latency = 109, 203ddbd550dSLen Brown .target_residency = 345, 2045fe2e527SRafael J. Wysocki .enter = &intel_idle, 2055fe2e527SRafael J. Wysocki .enter_freeze = intel_idle_freeze, }, 206e022e7ebSLen Brown { 207e022e7ebSLen Brown .enter = NULL } 208d13780d4SLen Brown }; 209d13780d4SLen Brown 210718987d6SLen Brown static struct cpuidle_state byt_cstates[] = { 211718987d6SLen Brown { 212718987d6SLen Brown .name = "C1-BYT", 213718987d6SLen Brown .desc = "MWAIT 0x00", 214b82b6ccaSDaniel Lezcano .flags = MWAIT2flg(0x00), 215718987d6SLen Brown .exit_latency = 1, 216718987d6SLen Brown .target_residency = 1, 2175fe2e527SRafael J. Wysocki .enter = &intel_idle, 2185fe2e527SRafael J. Wysocki .enter_freeze = intel_idle_freeze, }, 219718987d6SLen Brown { 220718987d6SLen Brown .name = "C6N-BYT", 221718987d6SLen Brown .desc = "MWAIT 0x58", 222b82b6ccaSDaniel Lezcano .flags = MWAIT2flg(0x58) | CPUIDLE_FLAG_TLB_FLUSHED, 223d7ef7671SLen Brown .exit_latency = 300, 224718987d6SLen Brown .target_residency = 275, 2255fe2e527SRafael J. Wysocki .enter = &intel_idle, 2265fe2e527SRafael J. Wysocki .enter_freeze = intel_idle_freeze, }, 227718987d6SLen Brown { 228718987d6SLen Brown .name = "C6S-BYT", 229718987d6SLen Brown .desc = "MWAIT 0x52", 230b82b6ccaSDaniel Lezcano .flags = MWAIT2flg(0x52) | CPUIDLE_FLAG_TLB_FLUSHED, 231d7ef7671SLen Brown .exit_latency = 500, 232718987d6SLen Brown .target_residency = 560, 2335fe2e527SRafael J. Wysocki .enter = &intel_idle, 2345fe2e527SRafael J. Wysocki .enter_freeze = intel_idle_freeze, }, 235718987d6SLen Brown { 236718987d6SLen Brown .name = "C7-BYT", 237718987d6SLen Brown .desc = "MWAIT 0x60", 238b82b6ccaSDaniel Lezcano .flags = MWAIT2flg(0x60) | CPUIDLE_FLAG_TLB_FLUSHED, 239718987d6SLen Brown .exit_latency = 1200, 240d7ef7671SLen Brown .target_residency = 4000, 2415fe2e527SRafael J. Wysocki .enter = &intel_idle, 2425fe2e527SRafael J. Wysocki .enter_freeze = intel_idle_freeze, }, 243718987d6SLen Brown { 244718987d6SLen Brown .name = "C7S-BYT", 245718987d6SLen Brown .desc = "MWAIT 0x64", 246b82b6ccaSDaniel Lezcano .flags = MWAIT2flg(0x64) | CPUIDLE_FLAG_TLB_FLUSHED, 247718987d6SLen Brown .exit_latency = 10000, 248718987d6SLen Brown .target_residency = 20000, 2495fe2e527SRafael J. Wysocki .enter = &intel_idle, 2505fe2e527SRafael J. Wysocki .enter_freeze = intel_idle_freeze, }, 251718987d6SLen Brown { 252718987d6SLen Brown .enter = NULL } 253718987d6SLen Brown }; 254718987d6SLen Brown 255cab07a56SLen Brown static struct cpuidle_state cht_cstates[] = { 256cab07a56SLen Brown { 257cab07a56SLen Brown .name = "C1-CHT", 258cab07a56SLen Brown .desc = "MWAIT 0x00", 259cab07a56SLen Brown .flags = MWAIT2flg(0x00), 260cab07a56SLen Brown .exit_latency = 1, 261cab07a56SLen Brown .target_residency = 1, 262cab07a56SLen Brown .enter = &intel_idle, 263cab07a56SLen Brown .enter_freeze = intel_idle_freeze, }, 264cab07a56SLen Brown { 265cab07a56SLen Brown .name = "C6N-CHT", 266cab07a56SLen Brown .desc = "MWAIT 0x58", 267cab07a56SLen Brown .flags = MWAIT2flg(0x58) | CPUIDLE_FLAG_TLB_FLUSHED, 268cab07a56SLen Brown .exit_latency = 80, 269cab07a56SLen Brown .target_residency = 275, 270cab07a56SLen Brown .enter = &intel_idle, 271cab07a56SLen Brown .enter_freeze = intel_idle_freeze, }, 272cab07a56SLen Brown { 273cab07a56SLen Brown .name = "C6S-CHT", 274cab07a56SLen Brown .desc = "MWAIT 0x52", 275cab07a56SLen Brown .flags = MWAIT2flg(0x52) | CPUIDLE_FLAG_TLB_FLUSHED, 276cab07a56SLen Brown .exit_latency = 200, 277cab07a56SLen Brown .target_residency = 560, 278cab07a56SLen Brown .enter = &intel_idle, 279cab07a56SLen Brown .enter_freeze = intel_idle_freeze, }, 280cab07a56SLen Brown { 281cab07a56SLen Brown .name = "C7-CHT", 282cab07a56SLen Brown .desc = "MWAIT 0x60", 283cab07a56SLen Brown .flags = MWAIT2flg(0x60) | CPUIDLE_FLAG_TLB_FLUSHED, 284cab07a56SLen Brown .exit_latency = 1200, 285cab07a56SLen Brown .target_residency = 4000, 286cab07a56SLen Brown .enter = &intel_idle, 287cab07a56SLen Brown .enter_freeze = intel_idle_freeze, }, 288cab07a56SLen Brown { 289cab07a56SLen Brown .name = "C7S-CHT", 290cab07a56SLen Brown .desc = "MWAIT 0x64", 291cab07a56SLen Brown .flags = MWAIT2flg(0x64) | CPUIDLE_FLAG_TLB_FLUSHED, 292cab07a56SLen Brown .exit_latency = 10000, 293cab07a56SLen Brown .target_residency = 20000, 294cab07a56SLen Brown .enter = &intel_idle, 295cab07a56SLen Brown .enter_freeze = intel_idle_freeze, }, 296cab07a56SLen Brown { 297cab07a56SLen Brown .enter = NULL } 298cab07a56SLen Brown }; 299cab07a56SLen Brown 300ba0dc81eSJiang Liu static struct cpuidle_state ivb_cstates[] = { 301e022e7ebSLen Brown { 3026edab08cSLen Brown .name = "C1-IVB", 3036edab08cSLen Brown .desc = "MWAIT 0x00", 304b82b6ccaSDaniel Lezcano .flags = MWAIT2flg(0x00), 3056edab08cSLen Brown .exit_latency = 1, 3066edab08cSLen Brown .target_residency = 1, 3075fe2e527SRafael J. Wysocki .enter = &intel_idle, 3085fe2e527SRafael J. Wysocki .enter_freeze = intel_idle_freeze, }, 309e022e7ebSLen Brown { 31032e95180SLen Brown .name = "C1E-IVB", 31132e95180SLen Brown .desc = "MWAIT 0x01", 312b82b6ccaSDaniel Lezcano .flags = MWAIT2flg(0x01), 31332e95180SLen Brown .exit_latency = 10, 31432e95180SLen Brown .target_residency = 20, 3155fe2e527SRafael J. Wysocki .enter = &intel_idle, 3165fe2e527SRafael J. Wysocki .enter_freeze = intel_idle_freeze, }, 31732e95180SLen Brown { 3186edab08cSLen Brown .name = "C3-IVB", 3196edab08cSLen Brown .desc = "MWAIT 0x10", 320b82b6ccaSDaniel Lezcano .flags = MWAIT2flg(0x10) | CPUIDLE_FLAG_TLB_FLUSHED, 3216edab08cSLen Brown .exit_latency = 59, 3226edab08cSLen Brown .target_residency = 156, 3235fe2e527SRafael J. Wysocki .enter = &intel_idle, 3245fe2e527SRafael J. Wysocki .enter_freeze = intel_idle_freeze, }, 325e022e7ebSLen Brown { 3266edab08cSLen Brown .name = "C6-IVB", 3276edab08cSLen Brown .desc = "MWAIT 0x20", 328b82b6ccaSDaniel Lezcano .flags = MWAIT2flg(0x20) | CPUIDLE_FLAG_TLB_FLUSHED, 3296edab08cSLen Brown .exit_latency = 80, 3306edab08cSLen Brown .target_residency = 300, 3315fe2e527SRafael J. Wysocki .enter = &intel_idle, 3325fe2e527SRafael J. Wysocki .enter_freeze = intel_idle_freeze, }, 333e022e7ebSLen Brown { 3346edab08cSLen Brown .name = "C7-IVB", 3356edab08cSLen Brown .desc = "MWAIT 0x30", 336b82b6ccaSDaniel Lezcano .flags = MWAIT2flg(0x30) | CPUIDLE_FLAG_TLB_FLUSHED, 3376edab08cSLen Brown .exit_latency = 87, 3386edab08cSLen Brown .target_residency = 300, 3395fe2e527SRafael J. Wysocki .enter = &intel_idle, 3405fe2e527SRafael J. Wysocki .enter_freeze = intel_idle_freeze, }, 341e022e7ebSLen Brown { 342e022e7ebSLen Brown .enter = NULL } 3436edab08cSLen Brown }; 3446edab08cSLen Brown 3450138d8f0SLen Brown static struct cpuidle_state ivt_cstates[] = { 3460138d8f0SLen Brown { 3470138d8f0SLen Brown .name = "C1-IVT", 3480138d8f0SLen Brown .desc = "MWAIT 0x00", 349b82b6ccaSDaniel Lezcano .flags = MWAIT2flg(0x00), 3500138d8f0SLen Brown .exit_latency = 1, 3510138d8f0SLen Brown .target_residency = 1, 3525fe2e527SRafael J. Wysocki .enter = &intel_idle, 3535fe2e527SRafael J. Wysocki .enter_freeze = intel_idle_freeze, }, 3540138d8f0SLen Brown { 3550138d8f0SLen Brown .name = "C1E-IVT", 3560138d8f0SLen Brown .desc = "MWAIT 0x01", 357b82b6ccaSDaniel Lezcano .flags = MWAIT2flg(0x01), 3580138d8f0SLen Brown .exit_latency = 10, 3590138d8f0SLen Brown .target_residency = 80, 3605fe2e527SRafael J. Wysocki .enter = &intel_idle, 3615fe2e527SRafael J. Wysocki .enter_freeze = intel_idle_freeze, }, 3620138d8f0SLen Brown { 3630138d8f0SLen Brown .name = "C3-IVT", 3640138d8f0SLen Brown .desc = "MWAIT 0x10", 365b82b6ccaSDaniel Lezcano .flags = MWAIT2flg(0x10) | CPUIDLE_FLAG_TLB_FLUSHED, 3660138d8f0SLen Brown .exit_latency = 59, 3670138d8f0SLen Brown .target_residency = 156, 3685fe2e527SRafael J. Wysocki .enter = &intel_idle, 3695fe2e527SRafael J. Wysocki .enter_freeze = intel_idle_freeze, }, 3700138d8f0SLen Brown { 3710138d8f0SLen Brown .name = "C6-IVT", 3720138d8f0SLen Brown .desc = "MWAIT 0x20", 373b82b6ccaSDaniel Lezcano .flags = MWAIT2flg(0x20) | CPUIDLE_FLAG_TLB_FLUSHED, 3740138d8f0SLen Brown .exit_latency = 82, 3750138d8f0SLen Brown .target_residency = 300, 3765fe2e527SRafael J. Wysocki .enter = &intel_idle, 3775fe2e527SRafael J. Wysocki .enter_freeze = intel_idle_freeze, }, 3780138d8f0SLen Brown { 3790138d8f0SLen Brown .enter = NULL } 3800138d8f0SLen Brown }; 3810138d8f0SLen Brown 3820138d8f0SLen Brown static struct cpuidle_state ivt_cstates_4s[] = { 3830138d8f0SLen Brown { 3840138d8f0SLen Brown .name = "C1-IVT-4S", 3850138d8f0SLen Brown .desc = "MWAIT 0x00", 386b82b6ccaSDaniel Lezcano .flags = MWAIT2flg(0x00), 3870138d8f0SLen Brown .exit_latency = 1, 3880138d8f0SLen Brown .target_residency = 1, 3895fe2e527SRafael J. Wysocki .enter = &intel_idle, 3905fe2e527SRafael J. Wysocki .enter_freeze = intel_idle_freeze, }, 3910138d8f0SLen Brown { 3920138d8f0SLen Brown .name = "C1E-IVT-4S", 3930138d8f0SLen Brown .desc = "MWAIT 0x01", 394b82b6ccaSDaniel Lezcano .flags = MWAIT2flg(0x01), 3950138d8f0SLen Brown .exit_latency = 10, 3960138d8f0SLen Brown .target_residency = 250, 3975fe2e527SRafael J. Wysocki .enter = &intel_idle, 3985fe2e527SRafael J. Wysocki .enter_freeze = intel_idle_freeze, }, 3990138d8f0SLen Brown { 4000138d8f0SLen Brown .name = "C3-IVT-4S", 4010138d8f0SLen Brown .desc = "MWAIT 0x10", 402b82b6ccaSDaniel Lezcano .flags = MWAIT2flg(0x10) | CPUIDLE_FLAG_TLB_FLUSHED, 4030138d8f0SLen Brown .exit_latency = 59, 4040138d8f0SLen Brown .target_residency = 300, 4055fe2e527SRafael J. Wysocki .enter = &intel_idle, 4065fe2e527SRafael J. Wysocki .enter_freeze = intel_idle_freeze, }, 4070138d8f0SLen Brown { 4080138d8f0SLen Brown .name = "C6-IVT-4S", 4090138d8f0SLen Brown .desc = "MWAIT 0x20", 410b82b6ccaSDaniel Lezcano .flags = MWAIT2flg(0x20) | CPUIDLE_FLAG_TLB_FLUSHED, 4110138d8f0SLen Brown .exit_latency = 84, 4120138d8f0SLen Brown .target_residency = 400, 4135fe2e527SRafael J. Wysocki .enter = &intel_idle, 4145fe2e527SRafael J. Wysocki .enter_freeze = intel_idle_freeze, }, 4150138d8f0SLen Brown { 4160138d8f0SLen Brown .enter = NULL } 4170138d8f0SLen Brown }; 4180138d8f0SLen Brown 4190138d8f0SLen Brown static struct cpuidle_state ivt_cstates_8s[] = { 4200138d8f0SLen Brown { 4210138d8f0SLen Brown .name = "C1-IVT-8S", 4220138d8f0SLen Brown .desc = "MWAIT 0x00", 423b82b6ccaSDaniel Lezcano .flags = MWAIT2flg(0x00), 4240138d8f0SLen Brown .exit_latency = 1, 4250138d8f0SLen Brown .target_residency = 1, 4265fe2e527SRafael J. Wysocki .enter = &intel_idle, 4275fe2e527SRafael J. Wysocki .enter_freeze = intel_idle_freeze, }, 4280138d8f0SLen Brown { 4290138d8f0SLen Brown .name = "C1E-IVT-8S", 4300138d8f0SLen Brown .desc = "MWAIT 0x01", 431b82b6ccaSDaniel Lezcano .flags = MWAIT2flg(0x01), 4320138d8f0SLen Brown .exit_latency = 10, 4330138d8f0SLen Brown .target_residency = 500, 4345fe2e527SRafael J. Wysocki .enter = &intel_idle, 4355fe2e527SRafael J. Wysocki .enter_freeze = intel_idle_freeze, }, 4360138d8f0SLen Brown { 4370138d8f0SLen Brown .name = "C3-IVT-8S", 4380138d8f0SLen Brown .desc = "MWAIT 0x10", 439b82b6ccaSDaniel Lezcano .flags = MWAIT2flg(0x10) | CPUIDLE_FLAG_TLB_FLUSHED, 4400138d8f0SLen Brown .exit_latency = 59, 4410138d8f0SLen Brown .target_residency = 600, 4425fe2e527SRafael J. Wysocki .enter = &intel_idle, 4435fe2e527SRafael J. Wysocki .enter_freeze = intel_idle_freeze, }, 4440138d8f0SLen Brown { 4450138d8f0SLen Brown .name = "C6-IVT-8S", 4460138d8f0SLen Brown .desc = "MWAIT 0x20", 447b82b6ccaSDaniel Lezcano .flags = MWAIT2flg(0x20) | CPUIDLE_FLAG_TLB_FLUSHED, 4480138d8f0SLen Brown .exit_latency = 88, 4490138d8f0SLen Brown .target_residency = 700, 4505fe2e527SRafael J. Wysocki .enter = &intel_idle, 4515fe2e527SRafael J. Wysocki .enter_freeze = intel_idle_freeze, }, 4520138d8f0SLen Brown { 4530138d8f0SLen Brown .enter = NULL } 4540138d8f0SLen Brown }; 4550138d8f0SLen Brown 456ba0dc81eSJiang Liu static struct cpuidle_state hsw_cstates[] = { 457e022e7ebSLen Brown { 45885a4d2d4SLen Brown .name = "C1-HSW", 45985a4d2d4SLen Brown .desc = "MWAIT 0x00", 460b82b6ccaSDaniel Lezcano .flags = MWAIT2flg(0x00), 46185a4d2d4SLen Brown .exit_latency = 2, 46285a4d2d4SLen Brown .target_residency = 2, 4635fe2e527SRafael J. Wysocki .enter = &intel_idle, 4645fe2e527SRafael J. Wysocki .enter_freeze = intel_idle_freeze, }, 465e022e7ebSLen Brown { 46632e95180SLen Brown .name = "C1E-HSW", 46732e95180SLen Brown .desc = "MWAIT 0x01", 468b82b6ccaSDaniel Lezcano .flags = MWAIT2flg(0x01), 46932e95180SLen Brown .exit_latency = 10, 47032e95180SLen Brown .target_residency = 20, 4715fe2e527SRafael J. Wysocki .enter = &intel_idle, 4725fe2e527SRafael J. Wysocki .enter_freeze = intel_idle_freeze, }, 47332e95180SLen Brown { 47485a4d2d4SLen Brown .name = "C3-HSW", 47585a4d2d4SLen Brown .desc = "MWAIT 0x10", 476b82b6ccaSDaniel Lezcano .flags = MWAIT2flg(0x10) | CPUIDLE_FLAG_TLB_FLUSHED, 47785a4d2d4SLen Brown .exit_latency = 33, 47885a4d2d4SLen Brown .target_residency = 100, 4795fe2e527SRafael J. Wysocki .enter = &intel_idle, 4805fe2e527SRafael J. Wysocki .enter_freeze = intel_idle_freeze, }, 481e022e7ebSLen Brown { 48285a4d2d4SLen Brown .name = "C6-HSW", 48385a4d2d4SLen Brown .desc = "MWAIT 0x20", 484b82b6ccaSDaniel Lezcano .flags = MWAIT2flg(0x20) | CPUIDLE_FLAG_TLB_FLUSHED, 48585a4d2d4SLen Brown .exit_latency = 133, 48685a4d2d4SLen Brown .target_residency = 400, 4875fe2e527SRafael J. Wysocki .enter = &intel_idle, 4885fe2e527SRafael J. Wysocki .enter_freeze = intel_idle_freeze, }, 489e022e7ebSLen Brown { 49085a4d2d4SLen Brown .name = "C7s-HSW", 49185a4d2d4SLen Brown .desc = "MWAIT 0x32", 492b82b6ccaSDaniel Lezcano .flags = MWAIT2flg(0x32) | CPUIDLE_FLAG_TLB_FLUSHED, 49385a4d2d4SLen Brown .exit_latency = 166, 49485a4d2d4SLen Brown .target_residency = 500, 4955fe2e527SRafael J. Wysocki .enter = &intel_idle, 4965fe2e527SRafael J. Wysocki .enter_freeze = intel_idle_freeze, }, 497e022e7ebSLen Brown { 49886239cebSLen Brown .name = "C8-HSW", 49986239cebSLen Brown .desc = "MWAIT 0x40", 500b82b6ccaSDaniel Lezcano .flags = MWAIT2flg(0x40) | CPUIDLE_FLAG_TLB_FLUSHED, 50186239cebSLen Brown .exit_latency = 300, 50286239cebSLen Brown .target_residency = 900, 5035fe2e527SRafael J. Wysocki .enter = &intel_idle, 5045fe2e527SRafael J. Wysocki .enter_freeze = intel_idle_freeze, }, 50586239cebSLen Brown { 50686239cebSLen Brown .name = "C9-HSW", 50786239cebSLen Brown .desc = "MWAIT 0x50", 508b82b6ccaSDaniel Lezcano .flags = MWAIT2flg(0x50) | CPUIDLE_FLAG_TLB_FLUSHED, 50986239cebSLen Brown .exit_latency = 600, 51086239cebSLen Brown .target_residency = 1800, 5115fe2e527SRafael J. Wysocki .enter = &intel_idle, 5125fe2e527SRafael J. Wysocki .enter_freeze = intel_idle_freeze, }, 51386239cebSLen Brown { 51486239cebSLen Brown .name = "C10-HSW", 51586239cebSLen Brown .desc = "MWAIT 0x60", 516b82b6ccaSDaniel Lezcano .flags = MWAIT2flg(0x60) | CPUIDLE_FLAG_TLB_FLUSHED, 51786239cebSLen Brown .exit_latency = 2600, 51886239cebSLen Brown .target_residency = 7700, 5195fe2e527SRafael J. Wysocki .enter = &intel_idle, 5205fe2e527SRafael J. Wysocki .enter_freeze = intel_idle_freeze, }, 52186239cebSLen Brown { 522e022e7ebSLen Brown .enter = NULL } 52385a4d2d4SLen Brown }; 524a138b568SLen Brown static struct cpuidle_state bdw_cstates[] = { 525a138b568SLen Brown { 526a138b568SLen Brown .name = "C1-BDW", 527a138b568SLen Brown .desc = "MWAIT 0x00", 528b82b6ccaSDaniel Lezcano .flags = MWAIT2flg(0x00), 529a138b568SLen Brown .exit_latency = 2, 530a138b568SLen Brown .target_residency = 2, 5315fe2e527SRafael J. Wysocki .enter = &intel_idle, 5325fe2e527SRafael J. Wysocki .enter_freeze = intel_idle_freeze, }, 533a138b568SLen Brown { 534a138b568SLen Brown .name = "C1E-BDW", 535a138b568SLen Brown .desc = "MWAIT 0x01", 536b82b6ccaSDaniel Lezcano .flags = MWAIT2flg(0x01), 537a138b568SLen Brown .exit_latency = 10, 538a138b568SLen Brown .target_residency = 20, 5395fe2e527SRafael J. Wysocki .enter = &intel_idle, 5405fe2e527SRafael J. Wysocki .enter_freeze = intel_idle_freeze, }, 541a138b568SLen Brown { 542a138b568SLen Brown .name = "C3-BDW", 543a138b568SLen Brown .desc = "MWAIT 0x10", 544b82b6ccaSDaniel Lezcano .flags = MWAIT2flg(0x10) | CPUIDLE_FLAG_TLB_FLUSHED, 545a138b568SLen Brown .exit_latency = 40, 546a138b568SLen Brown .target_residency = 100, 5475fe2e527SRafael J. Wysocki .enter = &intel_idle, 5485fe2e527SRafael J. Wysocki .enter_freeze = intel_idle_freeze, }, 549a138b568SLen Brown { 550a138b568SLen Brown .name = "C6-BDW", 551a138b568SLen Brown .desc = "MWAIT 0x20", 552b82b6ccaSDaniel Lezcano .flags = MWAIT2flg(0x20) | CPUIDLE_FLAG_TLB_FLUSHED, 553a138b568SLen Brown .exit_latency = 133, 554a138b568SLen Brown .target_residency = 400, 5555fe2e527SRafael J. Wysocki .enter = &intel_idle, 5565fe2e527SRafael J. Wysocki .enter_freeze = intel_idle_freeze, }, 557a138b568SLen Brown { 558a138b568SLen Brown .name = "C7s-BDW", 559a138b568SLen Brown .desc = "MWAIT 0x32", 560b82b6ccaSDaniel Lezcano .flags = MWAIT2flg(0x32) | CPUIDLE_FLAG_TLB_FLUSHED, 561a138b568SLen Brown .exit_latency = 166, 562a138b568SLen Brown .target_residency = 500, 5635fe2e527SRafael J. Wysocki .enter = &intel_idle, 5645fe2e527SRafael J. Wysocki .enter_freeze = intel_idle_freeze, }, 565a138b568SLen Brown { 566a138b568SLen Brown .name = "C8-BDW", 567a138b568SLen Brown .desc = "MWAIT 0x40", 568b82b6ccaSDaniel Lezcano .flags = MWAIT2flg(0x40) | CPUIDLE_FLAG_TLB_FLUSHED, 569a138b568SLen Brown .exit_latency = 300, 570a138b568SLen Brown .target_residency = 900, 5715fe2e527SRafael J. Wysocki .enter = &intel_idle, 5725fe2e527SRafael J. Wysocki .enter_freeze = intel_idle_freeze, }, 573a138b568SLen Brown { 574a138b568SLen Brown .name = "C9-BDW", 575a138b568SLen Brown .desc = "MWAIT 0x50", 576b82b6ccaSDaniel Lezcano .flags = MWAIT2flg(0x50) | CPUIDLE_FLAG_TLB_FLUSHED, 577a138b568SLen Brown .exit_latency = 600, 578a138b568SLen Brown .target_residency = 1800, 5795fe2e527SRafael J. Wysocki .enter = &intel_idle, 5805fe2e527SRafael J. Wysocki .enter_freeze = intel_idle_freeze, }, 581a138b568SLen Brown { 582a138b568SLen Brown .name = "C10-BDW", 583a138b568SLen Brown .desc = "MWAIT 0x60", 584b82b6ccaSDaniel Lezcano .flags = MWAIT2flg(0x60) | CPUIDLE_FLAG_TLB_FLUSHED, 585a138b568SLen Brown .exit_latency = 2600, 586a138b568SLen Brown .target_residency = 7700, 5875fe2e527SRafael J. Wysocki .enter = &intel_idle, 5885fe2e527SRafael J. Wysocki .enter_freeze = intel_idle_freeze, }, 589a138b568SLen Brown { 590a138b568SLen Brown .enter = NULL } 591a138b568SLen Brown }; 59285a4d2d4SLen Brown 593493f133fSLen Brown static struct cpuidle_state skl_cstates[] = { 594493f133fSLen Brown { 595493f133fSLen Brown .name = "C1-SKL", 596493f133fSLen Brown .desc = "MWAIT 0x00", 597493f133fSLen Brown .flags = MWAIT2flg(0x00), 598493f133fSLen Brown .exit_latency = 2, 599493f133fSLen Brown .target_residency = 2, 600493f133fSLen Brown .enter = &intel_idle, 601493f133fSLen Brown .enter_freeze = intel_idle_freeze, }, 602493f133fSLen Brown { 603493f133fSLen Brown .name = "C1E-SKL", 604493f133fSLen Brown .desc = "MWAIT 0x01", 605493f133fSLen Brown .flags = MWAIT2flg(0x01), 606493f133fSLen Brown .exit_latency = 10, 607493f133fSLen Brown .target_residency = 20, 608493f133fSLen Brown .enter = &intel_idle, 609493f133fSLen Brown .enter_freeze = intel_idle_freeze, }, 610493f133fSLen Brown { 611493f133fSLen Brown .name = "C3-SKL", 612493f133fSLen Brown .desc = "MWAIT 0x10", 613493f133fSLen Brown .flags = MWAIT2flg(0x10) | CPUIDLE_FLAG_TLB_FLUSHED, 614493f133fSLen Brown .exit_latency = 70, 615493f133fSLen Brown .target_residency = 100, 616493f133fSLen Brown .enter = &intel_idle, 617493f133fSLen Brown .enter_freeze = intel_idle_freeze, }, 618493f133fSLen Brown { 619493f133fSLen Brown .name = "C6-SKL", 620493f133fSLen Brown .desc = "MWAIT 0x20", 621493f133fSLen Brown .flags = MWAIT2flg(0x20) | CPUIDLE_FLAG_TLB_FLUSHED, 622135919a3SLen Brown .exit_latency = 85, 623493f133fSLen Brown .target_residency = 200, 624493f133fSLen Brown .enter = &intel_idle, 625493f133fSLen Brown .enter_freeze = intel_idle_freeze, }, 626493f133fSLen Brown { 627493f133fSLen Brown .name = "C7s-SKL", 628493f133fSLen Brown .desc = "MWAIT 0x33", 629493f133fSLen Brown .flags = MWAIT2flg(0x33) | CPUIDLE_FLAG_TLB_FLUSHED, 630493f133fSLen Brown .exit_latency = 124, 631493f133fSLen Brown .target_residency = 800, 632493f133fSLen Brown .enter = &intel_idle, 633493f133fSLen Brown .enter_freeze = intel_idle_freeze, }, 634493f133fSLen Brown { 635493f133fSLen Brown .name = "C8-SKL", 636493f133fSLen Brown .desc = "MWAIT 0x40", 637493f133fSLen Brown .flags = MWAIT2flg(0x40) | CPUIDLE_FLAG_TLB_FLUSHED, 638135919a3SLen Brown .exit_latency = 200, 639493f133fSLen Brown .target_residency = 800, 640493f133fSLen Brown .enter = &intel_idle, 641493f133fSLen Brown .enter_freeze = intel_idle_freeze, }, 642493f133fSLen Brown { 643135919a3SLen Brown .name = "C9-SKL", 644135919a3SLen Brown .desc = "MWAIT 0x50", 645135919a3SLen Brown .flags = MWAIT2flg(0x50) | CPUIDLE_FLAG_TLB_FLUSHED, 646135919a3SLen Brown .exit_latency = 480, 647135919a3SLen Brown .target_residency = 5000, 648135919a3SLen Brown .enter = &intel_idle, 649135919a3SLen Brown .enter_freeze = intel_idle_freeze, }, 650135919a3SLen Brown { 651493f133fSLen Brown .name = "C10-SKL", 652493f133fSLen Brown .desc = "MWAIT 0x60", 653493f133fSLen Brown .flags = MWAIT2flg(0x60) | CPUIDLE_FLAG_TLB_FLUSHED, 654493f133fSLen Brown .exit_latency = 890, 655493f133fSLen Brown .target_residency = 5000, 656493f133fSLen Brown .enter = &intel_idle, 657493f133fSLen Brown .enter_freeze = intel_idle_freeze, }, 658493f133fSLen Brown { 659493f133fSLen Brown .enter = NULL } 660493f133fSLen Brown }; 661493f133fSLen Brown 662f9e71657SLen Brown static struct cpuidle_state skx_cstates[] = { 663f9e71657SLen Brown { 664f9e71657SLen Brown .name = "C1-SKX", 665f9e71657SLen Brown .desc = "MWAIT 0x00", 666f9e71657SLen Brown .flags = MWAIT2flg(0x00), 667f9e71657SLen Brown .exit_latency = 2, 668f9e71657SLen Brown .target_residency = 2, 669f9e71657SLen Brown .enter = &intel_idle, 670f9e71657SLen Brown .enter_freeze = intel_idle_freeze, }, 671f9e71657SLen Brown { 672f9e71657SLen Brown .name = "C1E-SKX", 673f9e71657SLen Brown .desc = "MWAIT 0x01", 674f9e71657SLen Brown .flags = MWAIT2flg(0x01), 675f9e71657SLen Brown .exit_latency = 10, 676f9e71657SLen Brown .target_residency = 20, 677f9e71657SLen Brown .enter = &intel_idle, 678f9e71657SLen Brown .enter_freeze = intel_idle_freeze, }, 679f9e71657SLen Brown { 680f9e71657SLen Brown .name = "C6-SKX", 681f9e71657SLen Brown .desc = "MWAIT 0x20", 682f9e71657SLen Brown .flags = MWAIT2flg(0x20) | CPUIDLE_FLAG_TLB_FLUSHED, 683f9e71657SLen Brown .exit_latency = 133, 684f9e71657SLen Brown .target_residency = 600, 685f9e71657SLen Brown .enter = &intel_idle, 686f9e71657SLen Brown .enter_freeze = intel_idle_freeze, }, 687f9e71657SLen Brown { 688f9e71657SLen Brown .enter = NULL } 689f9e71657SLen Brown }; 690f9e71657SLen Brown 691ba0dc81eSJiang Liu static struct cpuidle_state atom_cstates[] = { 692e022e7ebSLen Brown { 69332e95180SLen Brown .name = "C1E-ATM", 69426717172SLen Brown .desc = "MWAIT 0x00", 695b82b6ccaSDaniel Lezcano .flags = MWAIT2flg(0x00), 69632e95180SLen Brown .exit_latency = 10, 69732e95180SLen Brown .target_residency = 20, 6985fe2e527SRafael J. Wysocki .enter = &intel_idle, 6995fe2e527SRafael J. Wysocki .enter_freeze = intel_idle_freeze, }, 700e022e7ebSLen Brown { 70115e123e5SThomas Renninger .name = "C2-ATM", 70226717172SLen Brown .desc = "MWAIT 0x10", 703b82b6ccaSDaniel Lezcano .flags = MWAIT2flg(0x10), 70426717172SLen Brown .exit_latency = 20, 70526717172SLen Brown .target_residency = 80, 7065fe2e527SRafael J. Wysocki .enter = &intel_idle, 7075fe2e527SRafael J. Wysocki .enter_freeze = intel_idle_freeze, }, 708e022e7ebSLen Brown { 70915e123e5SThomas Renninger .name = "C4-ATM", 71026717172SLen Brown .desc = "MWAIT 0x30", 711b82b6ccaSDaniel Lezcano .flags = MWAIT2flg(0x30) | CPUIDLE_FLAG_TLB_FLUSHED, 71226717172SLen Brown .exit_latency = 100, 71326717172SLen Brown .target_residency = 400, 7145fe2e527SRafael J. Wysocki .enter = &intel_idle, 7155fe2e527SRafael J. Wysocki .enter_freeze = intel_idle_freeze, }, 716e022e7ebSLen Brown { 71715e123e5SThomas Renninger .name = "C6-ATM", 7187fcca7d9SLen Brown .desc = "MWAIT 0x52", 719b82b6ccaSDaniel Lezcano .flags = MWAIT2flg(0x52) | CPUIDLE_FLAG_TLB_FLUSHED, 7207fcca7d9SLen Brown .exit_latency = 140, 7217fcca7d9SLen Brown .target_residency = 560, 7225fe2e527SRafael J. Wysocki .enter = &intel_idle, 7235fe2e527SRafael J. Wysocki .enter_freeze = intel_idle_freeze, }, 724e022e7ebSLen Brown { 725e022e7ebSLen Brown .enter = NULL } 72626717172SLen Brown }; 72788390996SJiang Liu static struct cpuidle_state avn_cstates[] = { 728fab04b22SLen Brown { 729fab04b22SLen Brown .name = "C1-AVN", 730fab04b22SLen Brown .desc = "MWAIT 0x00", 731b82b6ccaSDaniel Lezcano .flags = MWAIT2flg(0x00), 732fab04b22SLen Brown .exit_latency = 2, 733fab04b22SLen Brown .target_residency = 2, 7345fe2e527SRafael J. Wysocki .enter = &intel_idle, 7355fe2e527SRafael J. Wysocki .enter_freeze = intel_idle_freeze, }, 736fab04b22SLen Brown { 737fab04b22SLen Brown .name = "C6-AVN", 738fab04b22SLen Brown .desc = "MWAIT 0x51", 739b82b6ccaSDaniel Lezcano .flags = MWAIT2flg(0x51) | CPUIDLE_FLAG_TLB_FLUSHED, 740fab04b22SLen Brown .exit_latency = 15, 741fab04b22SLen Brown .target_residency = 45, 7425fe2e527SRafael J. Wysocki .enter = &intel_idle, 7435fe2e527SRafael J. Wysocki .enter_freeze = intel_idle_freeze, }, 74488390996SJiang Liu { 74588390996SJiang Liu .enter = NULL } 746fab04b22SLen Brown }; 747281baf7aSDasaratharaman Chandramouli static struct cpuidle_state knl_cstates[] = { 748281baf7aSDasaratharaman Chandramouli { 749281baf7aSDasaratharaman Chandramouli .name = "C1-KNL", 750281baf7aSDasaratharaman Chandramouli .desc = "MWAIT 0x00", 751281baf7aSDasaratharaman Chandramouli .flags = MWAIT2flg(0x00), 752281baf7aSDasaratharaman Chandramouli .exit_latency = 1, 753281baf7aSDasaratharaman Chandramouli .target_residency = 2, 754281baf7aSDasaratharaman Chandramouli .enter = &intel_idle, 755281baf7aSDasaratharaman Chandramouli .enter_freeze = intel_idle_freeze }, 756281baf7aSDasaratharaman Chandramouli { 757281baf7aSDasaratharaman Chandramouli .name = "C6-KNL", 758281baf7aSDasaratharaman Chandramouli .desc = "MWAIT 0x10", 759281baf7aSDasaratharaman Chandramouli .flags = MWAIT2flg(0x10) | CPUIDLE_FLAG_TLB_FLUSHED, 760281baf7aSDasaratharaman Chandramouli .exit_latency = 120, 761281baf7aSDasaratharaman Chandramouli .target_residency = 500, 762281baf7aSDasaratharaman Chandramouli .enter = &intel_idle, 763281baf7aSDasaratharaman Chandramouli .enter_freeze = intel_idle_freeze }, 764281baf7aSDasaratharaman Chandramouli { 765281baf7aSDasaratharaman Chandramouli .enter = NULL } 766281baf7aSDasaratharaman Chandramouli }; 76726717172SLen Brown 7685dcef694SLen Brown static struct cpuidle_state bxt_cstates[] = { 7695dcef694SLen Brown { 7705dcef694SLen Brown .name = "C1-BXT", 7715dcef694SLen Brown .desc = "MWAIT 0x00", 7725dcef694SLen Brown .flags = MWAIT2flg(0x00), 7735dcef694SLen Brown .exit_latency = 2, 7745dcef694SLen Brown .target_residency = 2, 7755dcef694SLen Brown .enter = &intel_idle, 7765dcef694SLen Brown .enter_freeze = intel_idle_freeze, }, 7775dcef694SLen Brown { 7785dcef694SLen Brown .name = "C1E-BXT", 7795dcef694SLen Brown .desc = "MWAIT 0x01", 7805dcef694SLen Brown .flags = MWAIT2flg(0x01), 7815dcef694SLen Brown .exit_latency = 10, 7825dcef694SLen Brown .target_residency = 20, 7835dcef694SLen Brown .enter = &intel_idle, 7845dcef694SLen Brown .enter_freeze = intel_idle_freeze, }, 7855dcef694SLen Brown { 7865dcef694SLen Brown .name = "C6-BXT", 7875dcef694SLen Brown .desc = "MWAIT 0x20", 7885dcef694SLen Brown .flags = MWAIT2flg(0x20) | CPUIDLE_FLAG_TLB_FLUSHED, 7895dcef694SLen Brown .exit_latency = 133, 7905dcef694SLen Brown .target_residency = 133, 7915dcef694SLen Brown .enter = &intel_idle, 7925dcef694SLen Brown .enter_freeze = intel_idle_freeze, }, 7935dcef694SLen Brown { 7945dcef694SLen Brown .name = "C7s-BXT", 7955dcef694SLen Brown .desc = "MWAIT 0x31", 7965dcef694SLen Brown .flags = MWAIT2flg(0x31) | CPUIDLE_FLAG_TLB_FLUSHED, 7975dcef694SLen Brown .exit_latency = 155, 7985dcef694SLen Brown .target_residency = 155, 7995dcef694SLen Brown .enter = &intel_idle, 8005dcef694SLen Brown .enter_freeze = intel_idle_freeze, }, 8015dcef694SLen Brown { 8025dcef694SLen Brown .name = "C8-BXT", 8035dcef694SLen Brown .desc = "MWAIT 0x40", 8045dcef694SLen Brown .flags = MWAIT2flg(0x40) | CPUIDLE_FLAG_TLB_FLUSHED, 8055dcef694SLen Brown .exit_latency = 1000, 8065dcef694SLen Brown .target_residency = 1000, 8075dcef694SLen Brown .enter = &intel_idle, 8085dcef694SLen Brown .enter_freeze = intel_idle_freeze, }, 8095dcef694SLen Brown { 8105dcef694SLen Brown .name = "C9-BXT", 8115dcef694SLen Brown .desc = "MWAIT 0x50", 8125dcef694SLen Brown .flags = MWAIT2flg(0x50) | CPUIDLE_FLAG_TLB_FLUSHED, 8135dcef694SLen Brown .exit_latency = 2000, 8145dcef694SLen Brown .target_residency = 2000, 8155dcef694SLen Brown .enter = &intel_idle, 8165dcef694SLen Brown .enter_freeze = intel_idle_freeze, }, 8175dcef694SLen Brown { 8185dcef694SLen Brown .name = "C10-BXT", 8195dcef694SLen Brown .desc = "MWAIT 0x60", 8205dcef694SLen Brown .flags = MWAIT2flg(0x60) | CPUIDLE_FLAG_TLB_FLUSHED, 8215dcef694SLen Brown .exit_latency = 10000, 8225dcef694SLen Brown .target_residency = 10000, 8235dcef694SLen Brown .enter = &intel_idle, 8245dcef694SLen Brown .enter_freeze = intel_idle_freeze, }, 8255dcef694SLen Brown { 8265dcef694SLen Brown .enter = NULL } 8275dcef694SLen Brown }; 8285dcef694SLen Brown 8290080d65bSJacob Pan static struct cpuidle_state dnv_cstates[] = { 8300080d65bSJacob Pan { 8310080d65bSJacob Pan .name = "C1-DNV", 8320080d65bSJacob Pan .desc = "MWAIT 0x00", 8330080d65bSJacob Pan .flags = MWAIT2flg(0x00), 8340080d65bSJacob Pan .exit_latency = 2, 8350080d65bSJacob Pan .target_residency = 2, 8360080d65bSJacob Pan .enter = &intel_idle, 8370080d65bSJacob Pan .enter_freeze = intel_idle_freeze, }, 8380080d65bSJacob Pan { 8390080d65bSJacob Pan .name = "C1E-DNV", 8400080d65bSJacob Pan .desc = "MWAIT 0x01", 8410080d65bSJacob Pan .flags = MWAIT2flg(0x01), 8420080d65bSJacob Pan .exit_latency = 10, 8430080d65bSJacob Pan .target_residency = 20, 8440080d65bSJacob Pan .enter = &intel_idle, 8450080d65bSJacob Pan .enter_freeze = intel_idle_freeze, }, 8460080d65bSJacob Pan { 8470080d65bSJacob Pan .name = "C6-DNV", 8480080d65bSJacob Pan .desc = "MWAIT 0x20", 8490080d65bSJacob Pan .flags = MWAIT2flg(0x20) | CPUIDLE_FLAG_TLB_FLUSHED, 8500080d65bSJacob Pan .exit_latency = 50, 8510080d65bSJacob Pan .target_residency = 500, 8520080d65bSJacob Pan .enter = &intel_idle, 8530080d65bSJacob Pan .enter_freeze = intel_idle_freeze, }, 8540080d65bSJacob Pan { 8550080d65bSJacob Pan .enter = NULL } 8560080d65bSJacob Pan }; 8570080d65bSJacob Pan 85826717172SLen Brown /** 85926717172SLen Brown * intel_idle 86026717172SLen Brown * @dev: cpuidle_device 86146bcfad7SDeepthi Dharwar * @drv: cpuidle driver 862e978aa7dSDeepthi Dharwar * @index: index of cpuidle state 86326717172SLen Brown * 86463ff07beSYanmin Zhang * Must be called under local_irq_disable(). 86526717172SLen Brown */ 8666727ad9eSChris Metcalf static __cpuidle int intel_idle(struct cpuidle_device *dev, 86746bcfad7SDeepthi Dharwar struct cpuidle_driver *drv, int index) 86826717172SLen Brown { 86926717172SLen Brown unsigned long ecx = 1; /* break on interrupt flag */ 87046bcfad7SDeepthi Dharwar struct cpuidle_state *state = &drv->states[index]; 871b1beab48SLen Brown unsigned long eax = flg2MWAIT(state->flags); 87226717172SLen Brown unsigned int cstate; 87326717172SLen Brown int cpu = smp_processor_id(); 87426717172SLen Brown 87526717172SLen Brown cstate = (((eax) >> MWAIT_SUBSTATE_SIZE) & MWAIT_CSTATE_MASK) + 1; 87626717172SLen Brown 8776110a1f4SSuresh Siddha /* 878c8381cc3SLen Brown * leave_mm() to avoid costly and often unnecessary wakeups 879c8381cc3SLen Brown * for flushing the user TLB's associated with the active mm. 8806110a1f4SSuresh Siddha */ 881c8381cc3SLen Brown if (state->flags & CPUIDLE_FLAG_TLB_FLUSHED) 8826110a1f4SSuresh Siddha leave_mm(cpu); 8836110a1f4SSuresh Siddha 88426717172SLen Brown if (!(lapic_timer_reliable_states & (1 << (cstate)))) 885f6cee191SThomas Gleixner tick_broadcast_enter(); 88626717172SLen Brown 88716824255SPeter Zijlstra mwait_idle_with_hints(eax, ecx); 88826717172SLen Brown 88926717172SLen Brown if (!(lapic_timer_reliable_states & (1 << (cstate)))) 890f6cee191SThomas Gleixner tick_broadcast_exit(); 89126717172SLen Brown 892e978aa7dSDeepthi Dharwar return index; 89326717172SLen Brown } 89426717172SLen Brown 8955fe2e527SRafael J. Wysocki /** 8965fe2e527SRafael J. Wysocki * intel_idle_freeze - simplified "enter" callback routine for suspend-to-idle 8975fe2e527SRafael J. Wysocki * @dev: cpuidle_device 8985fe2e527SRafael J. Wysocki * @drv: cpuidle driver 8995fe2e527SRafael J. Wysocki * @index: state index 9005fe2e527SRafael J. Wysocki */ 9015fe2e527SRafael J. Wysocki static void intel_idle_freeze(struct cpuidle_device *dev, 9025fe2e527SRafael J. Wysocki struct cpuidle_driver *drv, int index) 9035fe2e527SRafael J. Wysocki { 9045fe2e527SRafael J. Wysocki unsigned long ecx = 1; /* break on interrupt flag */ 9055fe2e527SRafael J. Wysocki unsigned long eax = flg2MWAIT(drv->states[index].flags); 9065fe2e527SRafael J. Wysocki 9075fe2e527SRafael J. Wysocki mwait_idle_with_hints(eax, ecx); 9085fe2e527SRafael J. Wysocki } 9095fe2e527SRafael J. Wysocki 9102a2d31c8SShaohua Li static void __setup_broadcast_timer(void *arg) 9112a2d31c8SShaohua Li { 91276962caaSThomas Gleixner unsigned long on = (unsigned long)arg; 9132a2d31c8SShaohua Li 91476962caaSThomas Gleixner if (on) 91576962caaSThomas Gleixner tick_broadcast_enable(); 91676962caaSThomas Gleixner else 91776962caaSThomas Gleixner tick_broadcast_disable(); 9182a2d31c8SShaohua Li } 9192a2d31c8SShaohua Li 92025ac7761SDaniel Lezcano static int cpu_hotplug_notify(struct notifier_block *n, 9212a2d31c8SShaohua Li unsigned long action, void *hcpu) 9222a2d31c8SShaohua Li { 9232a2d31c8SShaohua Li int hotcpu = (unsigned long)hcpu; 92425ac7761SDaniel Lezcano struct cpuidle_device *dev; 9252a2d31c8SShaohua Li 926e2401453SPrarit Bhargava switch (action & ~CPU_TASKS_FROZEN) { 9272a2d31c8SShaohua Li case CPU_ONLINE: 92825ac7761SDaniel Lezcano 92925ac7761SDaniel Lezcano if (lapic_timer_reliable_states != LAPIC_TIMER_ALWAYS_RELIABLE) 9302a2d31c8SShaohua Li smp_call_function_single(hotcpu, __setup_broadcast_timer, 9312a2d31c8SShaohua Li (void *)true, 1); 93225ac7761SDaniel Lezcano 93325ac7761SDaniel Lezcano /* 93425ac7761SDaniel Lezcano * Some systems can hotplug a cpu at runtime after 93525ac7761SDaniel Lezcano * the kernel has booted, we have to initialize the 93625ac7761SDaniel Lezcano * driver in this case 93725ac7761SDaniel Lezcano */ 93825ac7761SDaniel Lezcano dev = per_cpu_ptr(intel_idle_cpuidle_devices, hotcpu); 93908820546SRichard Cochran if (dev->registered) 94008820546SRichard Cochran break; 94108820546SRichard Cochran 94208820546SRichard Cochran if (intel_idle_cpu_init(hotcpu)) 94308820546SRichard Cochran return NOTIFY_BAD; 94425ac7761SDaniel Lezcano 9452a2d31c8SShaohua Li break; 9462a2d31c8SShaohua Li } 9472a2d31c8SShaohua Li return NOTIFY_OK; 9482a2d31c8SShaohua Li } 9492a2d31c8SShaohua Li 95025ac7761SDaniel Lezcano static struct notifier_block cpu_hotplug_notifier = { 95125ac7761SDaniel Lezcano .notifier_call = cpu_hotplug_notify, 9522a2d31c8SShaohua Li }; 9532a2d31c8SShaohua Li 95414796fcaSLen Brown static void auto_demotion_disable(void *dummy) 95514796fcaSLen Brown { 95614796fcaSLen Brown unsigned long long msr_bits; 95714796fcaSLen Brown 95814796fcaSLen Brown rdmsrl(MSR_NHM_SNB_PKG_CST_CFG_CTL, msr_bits); 959b66b8b9aSAndi Kleen msr_bits &= ~(icpu->auto_demotion_disable_flags); 96014796fcaSLen Brown wrmsrl(MSR_NHM_SNB_PKG_CST_CFG_CTL, msr_bits); 96114796fcaSLen Brown } 96232e95180SLen Brown static void c1e_promotion_disable(void *dummy) 96332e95180SLen Brown { 96432e95180SLen Brown unsigned long long msr_bits; 96532e95180SLen Brown 96632e95180SLen Brown rdmsrl(MSR_IA32_POWER_CTL, msr_bits); 96732e95180SLen Brown msr_bits &= ~0x2; 96832e95180SLen Brown wrmsrl(MSR_IA32_POWER_CTL, msr_bits); 96932e95180SLen Brown } 97014796fcaSLen Brown 971b66b8b9aSAndi Kleen static const struct idle_cpu idle_cpu_nehalem = { 972b66b8b9aSAndi Kleen .state_table = nehalem_cstates, 973b66b8b9aSAndi Kleen .auto_demotion_disable_flags = NHM_C1_AUTO_DEMOTE | NHM_C3_AUTO_DEMOTE, 97432e95180SLen Brown .disable_promotion_to_c1e = true, 975b66b8b9aSAndi Kleen }; 976b66b8b9aSAndi Kleen 977b66b8b9aSAndi Kleen static const struct idle_cpu idle_cpu_atom = { 978b66b8b9aSAndi Kleen .state_table = atom_cstates, 979b66b8b9aSAndi Kleen }; 980b66b8b9aSAndi Kleen 981b66b8b9aSAndi Kleen static const struct idle_cpu idle_cpu_lincroft = { 982b66b8b9aSAndi Kleen .state_table = atom_cstates, 983b66b8b9aSAndi Kleen .auto_demotion_disable_flags = ATM_LNC_C6_AUTO_DEMOTE, 984b66b8b9aSAndi Kleen }; 985b66b8b9aSAndi Kleen 986b66b8b9aSAndi Kleen static const struct idle_cpu idle_cpu_snb = { 987b66b8b9aSAndi Kleen .state_table = snb_cstates, 98832e95180SLen Brown .disable_promotion_to_c1e = true, 989b66b8b9aSAndi Kleen }; 990b66b8b9aSAndi Kleen 991718987d6SLen Brown static const struct idle_cpu idle_cpu_byt = { 992718987d6SLen Brown .state_table = byt_cstates, 993718987d6SLen Brown .disable_promotion_to_c1e = true, 9948c058d53SLen Brown .byt_auto_demotion_disable_flag = true, 995718987d6SLen Brown }; 996718987d6SLen Brown 997cab07a56SLen Brown static const struct idle_cpu idle_cpu_cht = { 998cab07a56SLen Brown .state_table = cht_cstates, 999cab07a56SLen Brown .disable_promotion_to_c1e = true, 1000cab07a56SLen Brown .byt_auto_demotion_disable_flag = true, 1001cab07a56SLen Brown }; 1002cab07a56SLen Brown 10036edab08cSLen Brown static const struct idle_cpu idle_cpu_ivb = { 10046edab08cSLen Brown .state_table = ivb_cstates, 100532e95180SLen Brown .disable_promotion_to_c1e = true, 10066edab08cSLen Brown }; 10076edab08cSLen Brown 10080138d8f0SLen Brown static const struct idle_cpu idle_cpu_ivt = { 10090138d8f0SLen Brown .state_table = ivt_cstates, 10100138d8f0SLen Brown .disable_promotion_to_c1e = true, 10110138d8f0SLen Brown }; 10120138d8f0SLen Brown 101385a4d2d4SLen Brown static const struct idle_cpu idle_cpu_hsw = { 101485a4d2d4SLen Brown .state_table = hsw_cstates, 101532e95180SLen Brown .disable_promotion_to_c1e = true, 101685a4d2d4SLen Brown }; 101785a4d2d4SLen Brown 1018a138b568SLen Brown static const struct idle_cpu idle_cpu_bdw = { 1019a138b568SLen Brown .state_table = bdw_cstates, 1020a138b568SLen Brown .disable_promotion_to_c1e = true, 1021a138b568SLen Brown }; 1022a138b568SLen Brown 1023493f133fSLen Brown static const struct idle_cpu idle_cpu_skl = { 1024493f133fSLen Brown .state_table = skl_cstates, 1025493f133fSLen Brown .disable_promotion_to_c1e = true, 1026493f133fSLen Brown }; 1027493f133fSLen Brown 1028f9e71657SLen Brown static const struct idle_cpu idle_cpu_skx = { 1029f9e71657SLen Brown .state_table = skx_cstates, 1030f9e71657SLen Brown .disable_promotion_to_c1e = true, 1031f9e71657SLen Brown }; 1032493f133fSLen Brown 1033fab04b22SLen Brown static const struct idle_cpu idle_cpu_avn = { 1034fab04b22SLen Brown .state_table = avn_cstates, 1035fab04b22SLen Brown .disable_promotion_to_c1e = true, 1036fab04b22SLen Brown }; 1037fab04b22SLen Brown 1038281baf7aSDasaratharaman Chandramouli static const struct idle_cpu idle_cpu_knl = { 1039281baf7aSDasaratharaman Chandramouli .state_table = knl_cstates, 1040281baf7aSDasaratharaman Chandramouli }; 1041281baf7aSDasaratharaman Chandramouli 10425dcef694SLen Brown static const struct idle_cpu idle_cpu_bxt = { 10435dcef694SLen Brown .state_table = bxt_cstates, 10445dcef694SLen Brown .disable_promotion_to_c1e = true, 10455dcef694SLen Brown }; 10465dcef694SLen Brown 10470080d65bSJacob Pan static const struct idle_cpu idle_cpu_dnv = { 10480080d65bSJacob Pan .state_table = dnv_cstates, 10490080d65bSJacob Pan .disable_promotion_to_c1e = true, 10500080d65bSJacob Pan }; 10510080d65bSJacob Pan 1052b66b8b9aSAndi Kleen #define ICPU(model, cpu) \ 1053b66b8b9aSAndi Kleen { X86_VENDOR_INTEL, 6, model, X86_FEATURE_MWAIT, (unsigned long)&cpu } 1054b66b8b9aSAndi Kleen 1055d5cdc3c4SMathias Krause static const struct x86_cpu_id intel_idle_ids[] __initconst = { 1056db73c5a8SDave Hansen ICPU(INTEL_FAM6_NEHALEM_EP, idle_cpu_nehalem), 1057db73c5a8SDave Hansen ICPU(INTEL_FAM6_NEHALEM, idle_cpu_nehalem), 10584b3b234fSDave Hansen ICPU(INTEL_FAM6_NEHALEM_G, idle_cpu_nehalem), 1059db73c5a8SDave Hansen ICPU(INTEL_FAM6_WESTMERE, idle_cpu_nehalem), 1060db73c5a8SDave Hansen ICPU(INTEL_FAM6_WESTMERE_EP, idle_cpu_nehalem), 1061db73c5a8SDave Hansen ICPU(INTEL_FAM6_NEHALEM_EX, idle_cpu_nehalem), 1062db73c5a8SDave Hansen ICPU(INTEL_FAM6_ATOM_PINEVIEW, idle_cpu_atom), 1063db73c5a8SDave Hansen ICPU(INTEL_FAM6_ATOM_LINCROFT, idle_cpu_lincroft), 1064db73c5a8SDave Hansen ICPU(INTEL_FAM6_WESTMERE_EX, idle_cpu_nehalem), 1065db73c5a8SDave Hansen ICPU(INTEL_FAM6_SANDYBRIDGE, idle_cpu_snb), 1066db73c5a8SDave Hansen ICPU(INTEL_FAM6_SANDYBRIDGE_X, idle_cpu_snb), 1067db73c5a8SDave Hansen ICPU(INTEL_FAM6_ATOM_CEDARVIEW, idle_cpu_atom), 1068db73c5a8SDave Hansen ICPU(INTEL_FAM6_ATOM_SILVERMONT1, idle_cpu_byt), 1069db73c5a8SDave Hansen ICPU(INTEL_FAM6_ATOM_AIRMONT, idle_cpu_cht), 1070db73c5a8SDave Hansen ICPU(INTEL_FAM6_IVYBRIDGE, idle_cpu_ivb), 1071db73c5a8SDave Hansen ICPU(INTEL_FAM6_IVYBRIDGE_X, idle_cpu_ivt), 1072db73c5a8SDave Hansen ICPU(INTEL_FAM6_HASWELL_CORE, idle_cpu_hsw), 1073db73c5a8SDave Hansen ICPU(INTEL_FAM6_HASWELL_X, idle_cpu_hsw), 1074db73c5a8SDave Hansen ICPU(INTEL_FAM6_HASWELL_ULT, idle_cpu_hsw), 1075db73c5a8SDave Hansen ICPU(INTEL_FAM6_HASWELL_GT3E, idle_cpu_hsw), 1076db73c5a8SDave Hansen ICPU(INTEL_FAM6_ATOM_SILVERMONT2, idle_cpu_avn), 1077db73c5a8SDave Hansen ICPU(INTEL_FAM6_BROADWELL_CORE, idle_cpu_bdw), 1078db73c5a8SDave Hansen ICPU(INTEL_FAM6_BROADWELL_GT3E, idle_cpu_bdw), 1079db73c5a8SDave Hansen ICPU(INTEL_FAM6_BROADWELL_X, idle_cpu_bdw), 1080db73c5a8SDave Hansen ICPU(INTEL_FAM6_BROADWELL_XEON_D, idle_cpu_bdw), 1081db73c5a8SDave Hansen ICPU(INTEL_FAM6_SKYLAKE_MOBILE, idle_cpu_skl), 1082db73c5a8SDave Hansen ICPU(INTEL_FAM6_SKYLAKE_DESKTOP, idle_cpu_skl), 1083db73c5a8SDave Hansen ICPU(INTEL_FAM6_KABYLAKE_MOBILE, idle_cpu_skl), 1084db73c5a8SDave Hansen ICPU(INTEL_FAM6_KABYLAKE_DESKTOP, idle_cpu_skl), 1085db73c5a8SDave Hansen ICPU(INTEL_FAM6_SKYLAKE_X, idle_cpu_skx), 1086db73c5a8SDave Hansen ICPU(INTEL_FAM6_XEON_PHI_KNL, idle_cpu_knl), 1087db73c5a8SDave Hansen ICPU(INTEL_FAM6_ATOM_GOLDMONT, idle_cpu_bxt), 10880080d65bSJacob Pan ICPU(INTEL_FAM6_ATOM_DENVERTON, idle_cpu_dnv), 1089b66b8b9aSAndi Kleen {} 1090b66b8b9aSAndi Kleen }; 1091b66b8b9aSAndi Kleen 109226717172SLen Brown /* 109326717172SLen Brown * intel_idle_probe() 109426717172SLen Brown */ 109500f3e755SBartlomiej Zolnierkiewicz static int __init intel_idle_probe(void) 109626717172SLen Brown { 1097c4236282SLen Brown unsigned int eax, ebx, ecx; 1098b66b8b9aSAndi Kleen const struct x86_cpu_id *id; 109926717172SLen Brown 110026717172SLen Brown if (max_cstate == 0) { 110126717172SLen Brown pr_debug(PREFIX "disabled\n"); 110226717172SLen Brown return -EPERM; 110326717172SLen Brown } 110426717172SLen Brown 1105b66b8b9aSAndi Kleen id = x86_match_cpu(intel_idle_ids); 1106b66b8b9aSAndi Kleen if (!id) { 1107b66b8b9aSAndi Kleen if (boot_cpu_data.x86_vendor == X86_VENDOR_INTEL && 1108b66b8b9aSAndi Kleen boot_cpu_data.x86 == 6) 1109b66b8b9aSAndi Kleen pr_debug(PREFIX "does not run on family %d model %d\n", 1110b66b8b9aSAndi Kleen boot_cpu_data.x86, boot_cpu_data.x86_model); 111126717172SLen Brown return -ENODEV; 1112b66b8b9aSAndi Kleen } 111326717172SLen Brown 111426717172SLen Brown if (boot_cpu_data.cpuid_level < CPUID_MWAIT_LEAF) 111526717172SLen Brown return -ENODEV; 111626717172SLen Brown 1117c4236282SLen Brown cpuid(CPUID_MWAIT_LEAF, &eax, &ebx, &ecx, &mwait_substates); 111826717172SLen Brown 111926717172SLen Brown if (!(ecx & CPUID5_ECX_EXTENSIONS_SUPPORTED) || 11205c2a9f06SThomas Renninger !(ecx & CPUID5_ECX_INTERRUPT_BREAK) || 11215c2a9f06SThomas Renninger !mwait_substates) 112226717172SLen Brown return -ENODEV; 112326717172SLen Brown 1124c4236282SLen Brown pr_debug(PREFIX "MWAIT substates: 0x%x\n", mwait_substates); 112526717172SLen Brown 1126b66b8b9aSAndi Kleen icpu = (const struct idle_cpu *)id->driver_data; 1127b66b8b9aSAndi Kleen cpuidle_state_table = icpu->state_table; 112826717172SLen Brown 112926717172SLen Brown pr_debug(PREFIX "v" INTEL_IDLE_VERSION 113026717172SLen Brown " model 0x%X\n", boot_cpu_data.x86_model); 113126717172SLen Brown 113226717172SLen Brown return 0; 113326717172SLen Brown } 113426717172SLen Brown 113526717172SLen Brown /* 113626717172SLen Brown * intel_idle_cpuidle_devices_uninit() 1137ca42489dSRichard Cochran * Unregisters the cpuidle devices. 113826717172SLen Brown */ 113926717172SLen Brown static void intel_idle_cpuidle_devices_uninit(void) 114026717172SLen Brown { 114126717172SLen Brown int i; 114226717172SLen Brown struct cpuidle_device *dev; 114326717172SLen Brown 114426717172SLen Brown for_each_online_cpu(i) { 114526717172SLen Brown dev = per_cpu_ptr(intel_idle_cpuidle_devices, i); 114626717172SLen Brown cpuidle_unregister_device(dev); 114726717172SLen Brown } 114826717172SLen Brown } 11490138d8f0SLen Brown 11500138d8f0SLen Brown /* 1151d70e28f5SLen Brown * ivt_idle_state_table_update(void) 11520138d8f0SLen Brown * 1153d70e28f5SLen Brown * Tune IVT multi-socket targets 11540138d8f0SLen Brown * Assumption: num_sockets == (max_package_num + 1) 11550138d8f0SLen Brown */ 1156d70e28f5SLen Brown static void ivt_idle_state_table_update(void) 11570138d8f0SLen Brown { 11580138d8f0SLen Brown /* IVT uses a different table for 1-2, 3-4, and > 4 sockets */ 11590138d8f0SLen Brown int cpu, package_num, num_sockets = 1; 11600138d8f0SLen Brown 11610138d8f0SLen Brown for_each_online_cpu(cpu) { 11620138d8f0SLen Brown package_num = topology_physical_package_id(cpu); 11630138d8f0SLen Brown if (package_num + 1 > num_sockets) { 11640138d8f0SLen Brown num_sockets = package_num + 1; 11650138d8f0SLen Brown 1166d27dca42SChristoph Jaeger if (num_sockets > 4) { 11670138d8f0SLen Brown cpuidle_state_table = ivt_cstates_8s; 11680138d8f0SLen Brown return; 11690138d8f0SLen Brown } 11700138d8f0SLen Brown } 1171d27dca42SChristoph Jaeger } 11720138d8f0SLen Brown 11730138d8f0SLen Brown if (num_sockets > 2) 11740138d8f0SLen Brown cpuidle_state_table = ivt_cstates_4s; 1175d70e28f5SLen Brown 11760138d8f0SLen Brown /* else, 1 and 2 socket systems use default ivt_cstates */ 11770138d8f0SLen Brown } 11785dcef694SLen Brown 11795dcef694SLen Brown /* 11805dcef694SLen Brown * Translate IRTL (Interrupt Response Time Limit) MSR to usec 11815dcef694SLen Brown */ 11825dcef694SLen Brown 11835dcef694SLen Brown static unsigned int irtl_ns_units[] = { 11845dcef694SLen Brown 1, 32, 1024, 32768, 1048576, 33554432, 0, 0 }; 11855dcef694SLen Brown 11865dcef694SLen Brown static unsigned long long irtl_2_usec(unsigned long long irtl) 11875dcef694SLen Brown { 11885dcef694SLen Brown unsigned long long ns; 11895dcef694SLen Brown 11903451ab3eSJan Beulich if (!irtl) 11913451ab3eSJan Beulich return 0; 11923451ab3eSJan Beulich 1193bef45096SJan Beulich ns = irtl_ns_units[(irtl >> 10) & 0x7]; 11945dcef694SLen Brown 11955dcef694SLen Brown return div64_u64((irtl & 0x3FF) * ns, 1000); 11965dcef694SLen Brown } 11975dcef694SLen Brown /* 11985dcef694SLen Brown * bxt_idle_state_table_update(void) 11995dcef694SLen Brown * 12005dcef694SLen Brown * On BXT, we trust the IRTL to show the definitive maximum latency 12015dcef694SLen Brown * We use the same value for target_residency. 12025dcef694SLen Brown */ 12035dcef694SLen Brown static void bxt_idle_state_table_update(void) 12045dcef694SLen Brown { 12055dcef694SLen Brown unsigned long long msr; 12063451ab3eSJan Beulich unsigned int usec; 12075dcef694SLen Brown 12085dcef694SLen Brown rdmsrl(MSR_PKGC6_IRTL, msr); 12093451ab3eSJan Beulich usec = irtl_2_usec(msr); 12103451ab3eSJan Beulich if (usec) { 12115dcef694SLen Brown bxt_cstates[2].exit_latency = usec; 12125dcef694SLen Brown bxt_cstates[2].target_residency = usec; 12135dcef694SLen Brown } 12145dcef694SLen Brown 12155dcef694SLen Brown rdmsrl(MSR_PKGC7_IRTL, msr); 12163451ab3eSJan Beulich usec = irtl_2_usec(msr); 12173451ab3eSJan Beulich if (usec) { 12185dcef694SLen Brown bxt_cstates[3].exit_latency = usec; 12195dcef694SLen Brown bxt_cstates[3].target_residency = usec; 12205dcef694SLen Brown } 12215dcef694SLen Brown 12225dcef694SLen Brown rdmsrl(MSR_PKGC8_IRTL, msr); 12233451ab3eSJan Beulich usec = irtl_2_usec(msr); 12243451ab3eSJan Beulich if (usec) { 12255dcef694SLen Brown bxt_cstates[4].exit_latency = usec; 12265dcef694SLen Brown bxt_cstates[4].target_residency = usec; 12275dcef694SLen Brown } 12285dcef694SLen Brown 12295dcef694SLen Brown rdmsrl(MSR_PKGC9_IRTL, msr); 12303451ab3eSJan Beulich usec = irtl_2_usec(msr); 12313451ab3eSJan Beulich if (usec) { 12325dcef694SLen Brown bxt_cstates[5].exit_latency = usec; 12335dcef694SLen Brown bxt_cstates[5].target_residency = usec; 12345dcef694SLen Brown } 12355dcef694SLen Brown 12365dcef694SLen Brown rdmsrl(MSR_PKGC10_IRTL, msr); 12373451ab3eSJan Beulich usec = irtl_2_usec(msr); 12383451ab3eSJan Beulich if (usec) { 12395dcef694SLen Brown bxt_cstates[6].exit_latency = usec; 12405dcef694SLen Brown bxt_cstates[6].target_residency = usec; 12415dcef694SLen Brown } 12425dcef694SLen Brown 12435dcef694SLen Brown } 1244d70e28f5SLen Brown /* 1245d70e28f5SLen Brown * sklh_idle_state_table_update(void) 1246d70e28f5SLen Brown * 1247d70e28f5SLen Brown * On SKL-H (model 0x5e) disable C8 and C9 if: 1248d70e28f5SLen Brown * C10 is enabled and SGX disabled 1249d70e28f5SLen Brown */ 1250d70e28f5SLen Brown static void sklh_idle_state_table_update(void) 1251d70e28f5SLen Brown { 1252d70e28f5SLen Brown unsigned long long msr; 1253d70e28f5SLen Brown unsigned int eax, ebx, ecx, edx; 1254d70e28f5SLen Brown 1255d70e28f5SLen Brown 1256d70e28f5SLen Brown /* if PC10 disabled via cmdline intel_idle.max_cstate=7 or shallower */ 1257d70e28f5SLen Brown if (max_cstate <= 7) 12580138d8f0SLen Brown return; 1259d70e28f5SLen Brown 1260d70e28f5SLen Brown /* if PC10 not present in CPUID.MWAIT.EDX */ 1261d70e28f5SLen Brown if ((mwait_substates & (0xF << 28)) == 0) 1262d70e28f5SLen Brown return; 1263d70e28f5SLen Brown 1264d70e28f5SLen Brown rdmsrl(MSR_NHM_SNB_PKG_CST_CFG_CTL, msr); 1265d70e28f5SLen Brown 1266d70e28f5SLen Brown /* PC10 is not enabled in PKG C-state limit */ 1267d70e28f5SLen Brown if ((msr & 0xF) != 8) 1268d70e28f5SLen Brown return; 1269d70e28f5SLen Brown 1270d70e28f5SLen Brown ecx = 0; 1271d70e28f5SLen Brown cpuid(7, &eax, &ebx, &ecx, &edx); 1272d70e28f5SLen Brown 1273d70e28f5SLen Brown /* if SGX is present */ 1274d70e28f5SLen Brown if (ebx & (1 << 2)) { 1275d70e28f5SLen Brown 1276d70e28f5SLen Brown rdmsrl(MSR_IA32_FEATURE_CONTROL, msr); 1277d70e28f5SLen Brown 1278d70e28f5SLen Brown /* if SGX is enabled */ 1279d70e28f5SLen Brown if (msr & (1 << 18)) 1280d70e28f5SLen Brown return; 1281d70e28f5SLen Brown } 1282d70e28f5SLen Brown 1283d70e28f5SLen Brown skl_cstates[5].disabled = 1; /* C8-SKL */ 1284d70e28f5SLen Brown skl_cstates[6].disabled = 1; /* C9-SKL */ 1285d70e28f5SLen Brown } 1286d70e28f5SLen Brown /* 1287d70e28f5SLen Brown * intel_idle_state_table_update() 1288d70e28f5SLen Brown * 1289d70e28f5SLen Brown * Update the default state_table for this CPU-id 1290d70e28f5SLen Brown */ 1291d70e28f5SLen Brown 1292d70e28f5SLen Brown static void intel_idle_state_table_update(void) 1293d70e28f5SLen Brown { 1294d70e28f5SLen Brown switch (boot_cpu_data.x86_model) { 1295d70e28f5SLen Brown 1296db73c5a8SDave Hansen case INTEL_FAM6_IVYBRIDGE_X: 1297d70e28f5SLen Brown ivt_idle_state_table_update(); 1298d70e28f5SLen Brown break; 1299db73c5a8SDave Hansen case INTEL_FAM6_ATOM_GOLDMONT: 13005dcef694SLen Brown bxt_idle_state_table_update(); 13015dcef694SLen Brown break; 1302db73c5a8SDave Hansen case INTEL_FAM6_SKYLAKE_DESKTOP: 1303d70e28f5SLen Brown sklh_idle_state_table_update(); 1304d70e28f5SLen Brown break; 1305d70e28f5SLen Brown } 13060138d8f0SLen Brown } 13070138d8f0SLen Brown 130826717172SLen Brown /* 130946bcfad7SDeepthi Dharwar * intel_idle_cpuidle_driver_init() 131046bcfad7SDeepthi Dharwar * allocate, initialize cpuidle_states 131146bcfad7SDeepthi Dharwar */ 13125469c827SRichard Cochran static void __init intel_idle_cpuidle_driver_init(void) 131346bcfad7SDeepthi Dharwar { 131446bcfad7SDeepthi Dharwar int cstate; 131546bcfad7SDeepthi Dharwar struct cpuidle_driver *drv = &intel_idle_driver; 131646bcfad7SDeepthi Dharwar 13170138d8f0SLen Brown intel_idle_state_table_update(); 13180138d8f0SLen Brown 131946bcfad7SDeepthi Dharwar drv->state_count = 1; 132046bcfad7SDeepthi Dharwar 1321e022e7ebSLen Brown for (cstate = 0; cstate < CPUIDLE_STATE_MAX; ++cstate) { 132224bfa950SLen Brown int num_substates, mwait_hint, mwait_cstate; 132346bcfad7SDeepthi Dharwar 13247dd0e0afSLen Brown if ((cpuidle_state_table[cstate].enter == NULL) && 13257dd0e0afSLen Brown (cpuidle_state_table[cstate].enter_freeze == NULL)) 1326e022e7ebSLen Brown break; 1327e022e7ebSLen Brown 1328e022e7ebSLen Brown if (cstate + 1 > max_cstate) { 132946bcfad7SDeepthi Dharwar printk(PREFIX "max_cstate %d reached\n", 133046bcfad7SDeepthi Dharwar max_cstate); 133146bcfad7SDeepthi Dharwar break; 133246bcfad7SDeepthi Dharwar } 133346bcfad7SDeepthi Dharwar 1334e022e7ebSLen Brown mwait_hint = flg2MWAIT(cpuidle_state_table[cstate].flags); 1335e022e7ebSLen Brown mwait_cstate = MWAIT_HINT2CSTATE(mwait_hint); 133646bcfad7SDeepthi Dharwar 133724bfa950SLen Brown /* number of sub-states for this state in CPUID.MWAIT */ 1338e022e7ebSLen Brown num_substates = (mwait_substates >> ((mwait_cstate + 1) * 4)) 1339e022e7ebSLen Brown & MWAIT_SUBSTATE_MASK; 1340e022e7ebSLen Brown 134124bfa950SLen Brown /* if NO sub-states for this state in CPUID, skip it */ 134224bfa950SLen Brown if (num_substates == 0) 1343e022e7ebSLen Brown continue; 1344e022e7ebSLen Brown 1345d70e28f5SLen Brown /* if state marked as disabled, skip it */ 1346d70e28f5SLen Brown if (cpuidle_state_table[cstate].disabled != 0) { 1347d70e28f5SLen Brown pr_debug(PREFIX "state %s is disabled", 1348d70e28f5SLen Brown cpuidle_state_table[cstate].name); 1349d70e28f5SLen Brown continue; 1350d70e28f5SLen Brown } 1351d70e28f5SLen Brown 1352d70e28f5SLen Brown 1353e022e7ebSLen Brown if (((mwait_cstate + 1) > 2) && 135446bcfad7SDeepthi Dharwar !boot_cpu_has(X86_FEATURE_NONSTOP_TSC)) 135546bcfad7SDeepthi Dharwar mark_tsc_unstable("TSC halts in idle" 135646bcfad7SDeepthi Dharwar " states deeper than C2"); 135746bcfad7SDeepthi Dharwar 135846bcfad7SDeepthi Dharwar drv->states[drv->state_count] = /* structure copy */ 135946bcfad7SDeepthi Dharwar cpuidle_state_table[cstate]; 136046bcfad7SDeepthi Dharwar 136146bcfad7SDeepthi Dharwar drv->state_count += 1; 136246bcfad7SDeepthi Dharwar } 136346bcfad7SDeepthi Dharwar 13648c058d53SLen Brown if (icpu->byt_auto_demotion_disable_flag) { 13658c058d53SLen Brown wrmsrl(MSR_CC6_DEMOTION_POLICY_CONFIG, 0); 13668c058d53SLen Brown wrmsrl(MSR_MC6_DEMOTION_POLICY_CONFIG, 0); 13678c058d53SLen Brown } 136846bcfad7SDeepthi Dharwar } 136946bcfad7SDeepthi Dharwar 137046bcfad7SDeepthi Dharwar 137146bcfad7SDeepthi Dharwar /* 137265b7f839SThomas Renninger * intel_idle_cpu_init() 137326717172SLen Brown * allocate, initialize, register cpuidle_devices 137465b7f839SThomas Renninger * @cpu: cpu/core to initialize 137526717172SLen Brown */ 137625ac7761SDaniel Lezcano static int intel_idle_cpu_init(int cpu) 137726717172SLen Brown { 137826717172SLen Brown struct cpuidle_device *dev; 137926717172SLen Brown 138065b7f839SThomas Renninger dev = per_cpu_ptr(intel_idle_cpuidle_devices, cpu); 138126717172SLen Brown 138265b7f839SThomas Renninger dev->cpu = cpu; 138326717172SLen Brown 138426717172SLen Brown if (cpuidle_register_device(dev)) { 138565b7f839SThomas Renninger pr_debug(PREFIX "cpuidle_register_device %d failed!\n", cpu); 138626717172SLen Brown return -EIO; 138726717172SLen Brown } 138865b7f839SThomas Renninger 1389b66b8b9aSAndi Kleen if (icpu->auto_demotion_disable_flags) 139065b7f839SThomas Renninger smp_call_function_single(cpu, auto_demotion_disable, NULL, 1); 139126717172SLen Brown 1392dbf87ab8SBartlomiej Zolnierkiewicz if (icpu->disable_promotion_to_c1e) 1393dbf87ab8SBartlomiej Zolnierkiewicz smp_call_function_single(cpu, c1e_promotion_disable, NULL, 1); 1394dbf87ab8SBartlomiej Zolnierkiewicz 139526717172SLen Brown return 0; 139626717172SLen Brown } 139726717172SLen Brown 139826717172SLen Brown static int __init intel_idle_init(void) 139926717172SLen Brown { 140065b7f839SThomas Renninger int retval, i; 140126717172SLen Brown 1402d1896049SThomas Renninger /* Do not load intel_idle at all for now if idle= is passed */ 1403d1896049SThomas Renninger if (boot_option_idle_override != IDLE_NO_OVERRIDE) 1404d1896049SThomas Renninger return -ENODEV; 1405d1896049SThomas Renninger 140626717172SLen Brown retval = intel_idle_probe(); 140726717172SLen Brown if (retval) 140826717172SLen Brown return retval; 140926717172SLen Brown 1410e9df69ccSRichard Cochran intel_idle_cpuidle_devices = alloc_percpu(struct cpuidle_device); 1411e9df69ccSRichard Cochran if (intel_idle_cpuidle_devices == NULL) 1412e9df69ccSRichard Cochran return -ENOMEM; 1413e9df69ccSRichard Cochran 141446bcfad7SDeepthi Dharwar intel_idle_cpuidle_driver_init(); 141526717172SLen Brown retval = cpuidle_register_driver(&intel_idle_driver); 141626717172SLen Brown if (retval) { 14173735d524SKonrad Rzeszutek Wilk struct cpuidle_driver *drv = cpuidle_get_driver(); 141826717172SLen Brown printk(KERN_DEBUG PREFIX "intel_idle yielding to %s", 14193735d524SKonrad Rzeszutek Wilk drv ? drv->name : "none"); 1420e9df69ccSRichard Cochran free_percpu(intel_idle_cpuidle_devices); 142126717172SLen Brown return retval; 142226717172SLen Brown } 142326717172SLen Brown 142407494d54SSrivatsa S. Bhat cpu_notifier_register_begin(); 142507494d54SSrivatsa S. Bhat 142665b7f839SThomas Renninger for_each_online_cpu(i) { 142765b7f839SThomas Renninger retval = intel_idle_cpu_init(i); 142826717172SLen Brown if (retval) { 1429b69ef2c0SRichard Cochran intel_idle_cpuidle_devices_uninit(); 143007494d54SSrivatsa S. Bhat cpu_notifier_register_done(); 143126717172SLen Brown cpuidle_unregister_driver(&intel_idle_driver); 1432ca42489dSRichard Cochran free_percpu(intel_idle_cpuidle_devices); 143326717172SLen Brown return retval; 143426717172SLen Brown } 143565b7f839SThomas Renninger } 143607494d54SSrivatsa S. Bhat __register_cpu_notifier(&cpu_hotplug_notifier); 143707494d54SSrivatsa S. Bhat 14382259a819SRichard Cochran if (boot_cpu_has(X86_FEATURE_ARAT)) /* Always Reliable APIC Timer */ 14392259a819SRichard Cochran lapic_timer_reliable_states = LAPIC_TIMER_ALWAYS_RELIABLE; 14402259a819SRichard Cochran else 14412259a819SRichard Cochran on_each_cpu(__setup_broadcast_timer, (void *)true, 1); 14422259a819SRichard Cochran 144307494d54SSrivatsa S. Bhat cpu_notifier_register_done(); 144426717172SLen Brown 14452259a819SRichard Cochran pr_debug(PREFIX "lapic_timer_reliable_states 0x%x\n", 14462259a819SRichard Cochran lapic_timer_reliable_states); 14472259a819SRichard Cochran 144826717172SLen Brown return 0; 144926717172SLen Brown } 145002c4fae9SPaul Gortmaker device_initcall(intel_idle_init); 145126717172SLen Brown 145202c4fae9SPaul Gortmaker /* 145302c4fae9SPaul Gortmaker * We are not really modular, but we used to support that. Meaning we also 145402c4fae9SPaul Gortmaker * support "intel_idle.max_cstate=..." at boot and also a read-only export of 145502c4fae9SPaul Gortmaker * it at /sys/module/intel_idle/parameters/max_cstate -- so using module_param 145602c4fae9SPaul Gortmaker * is the easiest way (currently) to continue doing that. 145702c4fae9SPaul Gortmaker */ 145826717172SLen Brown module_param(max_cstate, int, 0444); 1459