126717172SLen Brown /* 226717172SLen Brown * intel_idle.c - native hardware idle loop for modern Intel processors 326717172SLen Brown * 4fab04b22SLen Brown * Copyright (c) 2013, Intel Corporation. 526717172SLen Brown * Len Brown <len.brown@intel.com> 626717172SLen Brown * 726717172SLen Brown * This program is free software; you can redistribute it and/or modify it 826717172SLen Brown * under the terms and conditions of the GNU General Public License, 926717172SLen Brown * version 2, as published by the Free Software Foundation. 1026717172SLen Brown * 1126717172SLen Brown * This program is distributed in the hope it will be useful, but WITHOUT 1226717172SLen Brown * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or 1326717172SLen Brown * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for 1426717172SLen Brown * more details. 1526717172SLen Brown * 1626717172SLen Brown * You should have received a copy of the GNU General Public License along with 1726717172SLen Brown * this program; if not, write to the Free Software Foundation, Inc., 1826717172SLen Brown * 51 Franklin St - Fifth Floor, Boston, MA 02110-1301 USA. 1926717172SLen Brown */ 2026717172SLen Brown 2126717172SLen Brown /* 2226717172SLen Brown * intel_idle is a cpuidle driver that loads on specific Intel processors 2326717172SLen Brown * in lieu of the legacy ACPI processor_idle driver. The intent is to 2426717172SLen Brown * make Linux more efficient on these processors, as intel_idle knows 2526717172SLen Brown * more than ACPI, as well as make Linux more immune to ACPI BIOS bugs. 2626717172SLen Brown */ 2726717172SLen Brown 2826717172SLen Brown /* 2926717172SLen Brown * Design Assumptions 3026717172SLen Brown * 3126717172SLen Brown * All CPUs have same idle states as boot CPU 3226717172SLen Brown * 3326717172SLen Brown * Chipset BM_STS (bus master status) bit is a NOP 3426717172SLen Brown * for preventing entry into deep C-stats 3526717172SLen Brown */ 3626717172SLen Brown 3726717172SLen Brown /* 3826717172SLen Brown * Known limitations 3926717172SLen Brown * 4026717172SLen Brown * The driver currently initializes for_each_online_cpu() upon modprobe. 4126717172SLen Brown * It it unaware of subsequent processors hot-added to the system. 4226717172SLen Brown * This means that if you boot with maxcpus=n and later online 4326717172SLen Brown * processors above n, those processors will use C1 only. 4426717172SLen Brown * 4526717172SLen Brown * ACPI has a .suspend hack to turn off deep c-statees during suspend 4626717172SLen Brown * to avoid complications with the lapic timer workaround. 4726717172SLen Brown * Have not seen issues with suspend, but may need same workaround here. 4826717172SLen Brown * 4926717172SLen Brown * There is currently no kernel-based automatic probing/loading mechanism 5026717172SLen Brown * if the driver is built as a module. 5126717172SLen Brown */ 5226717172SLen Brown 5326717172SLen Brown /* un-comment DEBUG to enable pr_debug() statements */ 5426717172SLen Brown #define DEBUG 5526717172SLen Brown 5626717172SLen Brown #include <linux/kernel.h> 5726717172SLen Brown #include <linux/cpuidle.h> 5876962caaSThomas Gleixner #include <linux/tick.h> 5926717172SLen Brown #include <trace/events/power.h> 6026717172SLen Brown #include <linux/sched.h> 612a2d31c8SShaohua Li #include <linux/notifier.h> 622a2d31c8SShaohua Li #include <linux/cpu.h> 637c52d551SPaul Gortmaker #include <linux/module.h> 64b66b8b9aSAndi Kleen #include <asm/cpu_device_id.h> 65bc83ccccSH. Peter Anvin #include <asm/mwait.h> 6614796fcaSLen Brown #include <asm/msr.h> 6726717172SLen Brown 68d70e28f5SLen Brown #define INTEL_IDLE_VERSION "0.4.1" 6926717172SLen Brown #define PREFIX "intel_idle: " 7026717172SLen Brown 7126717172SLen Brown static struct cpuidle_driver intel_idle_driver = { 7226717172SLen Brown .name = "intel_idle", 7326717172SLen Brown .owner = THIS_MODULE, 7426717172SLen Brown }; 7526717172SLen Brown /* intel_idle.max_cstate=0 disables driver */ 76137ecc77SLen Brown static int max_cstate = CPUIDLE_STATE_MAX - 1; 7726717172SLen Brown 78c4236282SLen Brown static unsigned int mwait_substates; 7926717172SLen Brown 802a2d31c8SShaohua Li #define LAPIC_TIMER_ALWAYS_RELIABLE 0xFFFFFFFF 8126717172SLen Brown /* Reliable LAPIC Timer States, bit 1 for C1 etc. */ 82d13780d4SLen Brown static unsigned int lapic_timer_reliable_states = (1 << 1); /* Default to only C1 */ 8326717172SLen Brown 84b66b8b9aSAndi Kleen struct idle_cpu { 85b66b8b9aSAndi Kleen struct cpuidle_state *state_table; 8626717172SLen Brown 8726717172SLen Brown /* 8814796fcaSLen Brown * Hardware C-state auto-demotion may not always be optimal. 8914796fcaSLen Brown * Indicate which enable bits to clear here. 9014796fcaSLen Brown */ 91b66b8b9aSAndi Kleen unsigned long auto_demotion_disable_flags; 928c058d53SLen Brown bool byt_auto_demotion_disable_flag; 9332e95180SLen Brown bool disable_promotion_to_c1e; 94b66b8b9aSAndi Kleen }; 95b66b8b9aSAndi Kleen 96b66b8b9aSAndi Kleen static const struct idle_cpu *icpu; 97b66b8b9aSAndi Kleen static struct cpuidle_device __percpu *intel_idle_cpuidle_devices; 98b66b8b9aSAndi Kleen static int intel_idle(struct cpuidle_device *dev, 99b66b8b9aSAndi Kleen struct cpuidle_driver *drv, int index); 1005fe2e527SRafael J. Wysocki static void intel_idle_freeze(struct cpuidle_device *dev, 1015fe2e527SRafael J. Wysocki struct cpuidle_driver *drv, int index); 10225ac7761SDaniel Lezcano static int intel_idle_cpu_init(int cpu); 103b66b8b9aSAndi Kleen 104b66b8b9aSAndi Kleen static struct cpuidle_state *cpuidle_state_table; 10514796fcaSLen Brown 10614796fcaSLen Brown /* 107956d033fSLen Brown * Set this flag for states where the HW flushes the TLB for us 108956d033fSLen Brown * and so we don't need cross-calls to keep it consistent. 109956d033fSLen Brown * If this flag is set, SW flushes the TLB, so even if the 110956d033fSLen Brown * HW doesn't do the flushing, this flag is safe to use. 111956d033fSLen Brown */ 112956d033fSLen Brown #define CPUIDLE_FLAG_TLB_FLUSHED 0x10000 113956d033fSLen Brown 114956d033fSLen Brown /* 115b1beab48SLen Brown * MWAIT takes an 8-bit "hint" in EAX "suggesting" 116b1beab48SLen Brown * the C-state (top nibble) and sub-state (bottom nibble) 117b1beab48SLen Brown * 0x00 means "MWAIT(C1)", 0x10 means "MWAIT(C2)" etc. 118b1beab48SLen Brown * 119b1beab48SLen Brown * We store the hint at the top of our "flags" for each state. 120b1beab48SLen Brown */ 121b1beab48SLen Brown #define flg2MWAIT(flags) (((flags) >> 24) & 0xFF) 122b1beab48SLen Brown #define MWAIT2flg(eax) ((eax & 0xFF) << 24) 123b1beab48SLen Brown 124b1beab48SLen Brown /* 12526717172SLen Brown * States are indexed by the cstate number, 12626717172SLen Brown * which is also the index into the MWAIT hint array. 12726717172SLen Brown * Thus C0 is a dummy. 12826717172SLen Brown */ 129ba0dc81eSJiang Liu static struct cpuidle_state nehalem_cstates[] = { 130e022e7ebSLen Brown { 13115e123e5SThomas Renninger .name = "C1-NHM", 13226717172SLen Brown .desc = "MWAIT 0x00", 133b82b6ccaSDaniel Lezcano .flags = MWAIT2flg(0x00), 13426717172SLen Brown .exit_latency = 3, 13526717172SLen Brown .target_residency = 6, 1365fe2e527SRafael J. Wysocki .enter = &intel_idle, 1375fe2e527SRafael J. Wysocki .enter_freeze = intel_idle_freeze, }, 138e022e7ebSLen Brown { 13932e95180SLen Brown .name = "C1E-NHM", 14032e95180SLen Brown .desc = "MWAIT 0x01", 141b82b6ccaSDaniel Lezcano .flags = MWAIT2flg(0x01), 14232e95180SLen Brown .exit_latency = 10, 14332e95180SLen Brown .target_residency = 20, 1445fe2e527SRafael J. Wysocki .enter = &intel_idle, 1455fe2e527SRafael J. Wysocki .enter_freeze = intel_idle_freeze, }, 14632e95180SLen Brown { 14715e123e5SThomas Renninger .name = "C3-NHM", 14826717172SLen Brown .desc = "MWAIT 0x10", 149b82b6ccaSDaniel Lezcano .flags = MWAIT2flg(0x10) | CPUIDLE_FLAG_TLB_FLUSHED, 15026717172SLen Brown .exit_latency = 20, 15126717172SLen Brown .target_residency = 80, 1525fe2e527SRafael J. Wysocki .enter = &intel_idle, 1535fe2e527SRafael J. Wysocki .enter_freeze = intel_idle_freeze, }, 154e022e7ebSLen Brown { 15515e123e5SThomas Renninger .name = "C6-NHM", 15626717172SLen Brown .desc = "MWAIT 0x20", 157b82b6ccaSDaniel Lezcano .flags = MWAIT2flg(0x20) | CPUIDLE_FLAG_TLB_FLUSHED, 15826717172SLen Brown .exit_latency = 200, 15926717172SLen Brown .target_residency = 800, 1605fe2e527SRafael J. Wysocki .enter = &intel_idle, 1615fe2e527SRafael J. Wysocki .enter_freeze = intel_idle_freeze, }, 162e022e7ebSLen Brown { 163e022e7ebSLen Brown .enter = NULL } 16426717172SLen Brown }; 16526717172SLen Brown 166ba0dc81eSJiang Liu static struct cpuidle_state snb_cstates[] = { 167e022e7ebSLen Brown { 16815e123e5SThomas Renninger .name = "C1-SNB", 169d13780d4SLen Brown .desc = "MWAIT 0x00", 170b82b6ccaSDaniel Lezcano .flags = MWAIT2flg(0x00), 17132e95180SLen Brown .exit_latency = 2, 17232e95180SLen Brown .target_residency = 2, 1735fe2e527SRafael J. Wysocki .enter = &intel_idle, 1745fe2e527SRafael J. Wysocki .enter_freeze = intel_idle_freeze, }, 17532e95180SLen Brown { 17632e95180SLen Brown .name = "C1E-SNB", 17732e95180SLen Brown .desc = "MWAIT 0x01", 178b82b6ccaSDaniel Lezcano .flags = MWAIT2flg(0x01), 17932e95180SLen Brown .exit_latency = 10, 18032e95180SLen Brown .target_residency = 20, 1815fe2e527SRafael J. Wysocki .enter = &intel_idle, 1825fe2e527SRafael J. Wysocki .enter_freeze = intel_idle_freeze, }, 183e022e7ebSLen Brown { 18415e123e5SThomas Renninger .name = "C3-SNB", 185d13780d4SLen Brown .desc = "MWAIT 0x10", 186b82b6ccaSDaniel Lezcano .flags = MWAIT2flg(0x10) | CPUIDLE_FLAG_TLB_FLUSHED, 187d13780d4SLen Brown .exit_latency = 80, 188ddbd550dSLen Brown .target_residency = 211, 1895fe2e527SRafael J. Wysocki .enter = &intel_idle, 1905fe2e527SRafael J. Wysocki .enter_freeze = intel_idle_freeze, }, 191e022e7ebSLen Brown { 19215e123e5SThomas Renninger .name = "C6-SNB", 193d13780d4SLen Brown .desc = "MWAIT 0x20", 194b82b6ccaSDaniel Lezcano .flags = MWAIT2flg(0x20) | CPUIDLE_FLAG_TLB_FLUSHED, 195d13780d4SLen Brown .exit_latency = 104, 196ddbd550dSLen Brown .target_residency = 345, 1975fe2e527SRafael J. Wysocki .enter = &intel_idle, 1985fe2e527SRafael J. Wysocki .enter_freeze = intel_idle_freeze, }, 199e022e7ebSLen Brown { 20015e123e5SThomas Renninger .name = "C7-SNB", 201d13780d4SLen Brown .desc = "MWAIT 0x30", 202b82b6ccaSDaniel Lezcano .flags = MWAIT2flg(0x30) | CPUIDLE_FLAG_TLB_FLUSHED, 203d13780d4SLen Brown .exit_latency = 109, 204ddbd550dSLen Brown .target_residency = 345, 2055fe2e527SRafael J. Wysocki .enter = &intel_idle, 2065fe2e527SRafael J. Wysocki .enter_freeze = intel_idle_freeze, }, 207e022e7ebSLen Brown { 208e022e7ebSLen Brown .enter = NULL } 209d13780d4SLen Brown }; 210d13780d4SLen Brown 211718987d6SLen Brown static struct cpuidle_state byt_cstates[] = { 212718987d6SLen Brown { 213718987d6SLen Brown .name = "C1-BYT", 214718987d6SLen Brown .desc = "MWAIT 0x00", 215b82b6ccaSDaniel Lezcano .flags = MWAIT2flg(0x00), 216718987d6SLen Brown .exit_latency = 1, 217718987d6SLen Brown .target_residency = 1, 2185fe2e527SRafael J. Wysocki .enter = &intel_idle, 2195fe2e527SRafael J. Wysocki .enter_freeze = intel_idle_freeze, }, 220718987d6SLen Brown { 221718987d6SLen Brown .name = "C6N-BYT", 222718987d6SLen Brown .desc = "MWAIT 0x58", 223b82b6ccaSDaniel Lezcano .flags = MWAIT2flg(0x58) | CPUIDLE_FLAG_TLB_FLUSHED, 224d7ef7671SLen Brown .exit_latency = 300, 225718987d6SLen Brown .target_residency = 275, 2265fe2e527SRafael J. Wysocki .enter = &intel_idle, 2275fe2e527SRafael J. Wysocki .enter_freeze = intel_idle_freeze, }, 228718987d6SLen Brown { 229718987d6SLen Brown .name = "C6S-BYT", 230718987d6SLen Brown .desc = "MWAIT 0x52", 231b82b6ccaSDaniel Lezcano .flags = MWAIT2flg(0x52) | CPUIDLE_FLAG_TLB_FLUSHED, 232d7ef7671SLen Brown .exit_latency = 500, 233718987d6SLen Brown .target_residency = 560, 2345fe2e527SRafael J. Wysocki .enter = &intel_idle, 2355fe2e527SRafael J. Wysocki .enter_freeze = intel_idle_freeze, }, 236718987d6SLen Brown { 237718987d6SLen Brown .name = "C7-BYT", 238718987d6SLen Brown .desc = "MWAIT 0x60", 239b82b6ccaSDaniel Lezcano .flags = MWAIT2flg(0x60) | CPUIDLE_FLAG_TLB_FLUSHED, 240718987d6SLen Brown .exit_latency = 1200, 241d7ef7671SLen Brown .target_residency = 4000, 2425fe2e527SRafael J. Wysocki .enter = &intel_idle, 2435fe2e527SRafael J. Wysocki .enter_freeze = intel_idle_freeze, }, 244718987d6SLen Brown { 245718987d6SLen Brown .name = "C7S-BYT", 246718987d6SLen Brown .desc = "MWAIT 0x64", 247b82b6ccaSDaniel Lezcano .flags = MWAIT2flg(0x64) | CPUIDLE_FLAG_TLB_FLUSHED, 248718987d6SLen Brown .exit_latency = 10000, 249718987d6SLen Brown .target_residency = 20000, 2505fe2e527SRafael J. Wysocki .enter = &intel_idle, 2515fe2e527SRafael J. Wysocki .enter_freeze = intel_idle_freeze, }, 252718987d6SLen Brown { 253718987d6SLen Brown .enter = NULL } 254718987d6SLen Brown }; 255718987d6SLen Brown 256cab07a56SLen Brown static struct cpuidle_state cht_cstates[] = { 257cab07a56SLen Brown { 258cab07a56SLen Brown .name = "C1-CHT", 259cab07a56SLen Brown .desc = "MWAIT 0x00", 260cab07a56SLen Brown .flags = MWAIT2flg(0x00), 261cab07a56SLen Brown .exit_latency = 1, 262cab07a56SLen Brown .target_residency = 1, 263cab07a56SLen Brown .enter = &intel_idle, 264cab07a56SLen Brown .enter_freeze = intel_idle_freeze, }, 265cab07a56SLen Brown { 266cab07a56SLen Brown .name = "C6N-CHT", 267cab07a56SLen Brown .desc = "MWAIT 0x58", 268cab07a56SLen Brown .flags = MWAIT2flg(0x58) | CPUIDLE_FLAG_TLB_FLUSHED, 269cab07a56SLen Brown .exit_latency = 80, 270cab07a56SLen Brown .target_residency = 275, 271cab07a56SLen Brown .enter = &intel_idle, 272cab07a56SLen Brown .enter_freeze = intel_idle_freeze, }, 273cab07a56SLen Brown { 274cab07a56SLen Brown .name = "C6S-CHT", 275cab07a56SLen Brown .desc = "MWAIT 0x52", 276cab07a56SLen Brown .flags = MWAIT2flg(0x52) | CPUIDLE_FLAG_TLB_FLUSHED, 277cab07a56SLen Brown .exit_latency = 200, 278cab07a56SLen Brown .target_residency = 560, 279cab07a56SLen Brown .enter = &intel_idle, 280cab07a56SLen Brown .enter_freeze = intel_idle_freeze, }, 281cab07a56SLen Brown { 282cab07a56SLen Brown .name = "C7-CHT", 283cab07a56SLen Brown .desc = "MWAIT 0x60", 284cab07a56SLen Brown .flags = MWAIT2flg(0x60) | CPUIDLE_FLAG_TLB_FLUSHED, 285cab07a56SLen Brown .exit_latency = 1200, 286cab07a56SLen Brown .target_residency = 4000, 287cab07a56SLen Brown .enter = &intel_idle, 288cab07a56SLen Brown .enter_freeze = intel_idle_freeze, }, 289cab07a56SLen Brown { 290cab07a56SLen Brown .name = "C7S-CHT", 291cab07a56SLen Brown .desc = "MWAIT 0x64", 292cab07a56SLen Brown .flags = MWAIT2flg(0x64) | CPUIDLE_FLAG_TLB_FLUSHED, 293cab07a56SLen Brown .exit_latency = 10000, 294cab07a56SLen Brown .target_residency = 20000, 295cab07a56SLen Brown .enter = &intel_idle, 296cab07a56SLen Brown .enter_freeze = intel_idle_freeze, }, 297cab07a56SLen Brown { 298cab07a56SLen Brown .enter = NULL } 299cab07a56SLen Brown }; 300cab07a56SLen Brown 301ba0dc81eSJiang Liu static struct cpuidle_state ivb_cstates[] = { 302e022e7ebSLen Brown { 3036edab08cSLen Brown .name = "C1-IVB", 3046edab08cSLen Brown .desc = "MWAIT 0x00", 305b82b6ccaSDaniel Lezcano .flags = MWAIT2flg(0x00), 3066edab08cSLen Brown .exit_latency = 1, 3076edab08cSLen Brown .target_residency = 1, 3085fe2e527SRafael J. Wysocki .enter = &intel_idle, 3095fe2e527SRafael J. Wysocki .enter_freeze = intel_idle_freeze, }, 310e022e7ebSLen Brown { 31132e95180SLen Brown .name = "C1E-IVB", 31232e95180SLen Brown .desc = "MWAIT 0x01", 313b82b6ccaSDaniel Lezcano .flags = MWAIT2flg(0x01), 31432e95180SLen Brown .exit_latency = 10, 31532e95180SLen Brown .target_residency = 20, 3165fe2e527SRafael J. Wysocki .enter = &intel_idle, 3175fe2e527SRafael J. Wysocki .enter_freeze = intel_idle_freeze, }, 31832e95180SLen Brown { 3196edab08cSLen Brown .name = "C3-IVB", 3206edab08cSLen Brown .desc = "MWAIT 0x10", 321b82b6ccaSDaniel Lezcano .flags = MWAIT2flg(0x10) | CPUIDLE_FLAG_TLB_FLUSHED, 3226edab08cSLen Brown .exit_latency = 59, 3236edab08cSLen Brown .target_residency = 156, 3245fe2e527SRafael J. Wysocki .enter = &intel_idle, 3255fe2e527SRafael J. Wysocki .enter_freeze = intel_idle_freeze, }, 326e022e7ebSLen Brown { 3276edab08cSLen Brown .name = "C6-IVB", 3286edab08cSLen Brown .desc = "MWAIT 0x20", 329b82b6ccaSDaniel Lezcano .flags = MWAIT2flg(0x20) | CPUIDLE_FLAG_TLB_FLUSHED, 3306edab08cSLen Brown .exit_latency = 80, 3316edab08cSLen Brown .target_residency = 300, 3325fe2e527SRafael J. Wysocki .enter = &intel_idle, 3335fe2e527SRafael J. Wysocki .enter_freeze = intel_idle_freeze, }, 334e022e7ebSLen Brown { 3356edab08cSLen Brown .name = "C7-IVB", 3366edab08cSLen Brown .desc = "MWAIT 0x30", 337b82b6ccaSDaniel Lezcano .flags = MWAIT2flg(0x30) | CPUIDLE_FLAG_TLB_FLUSHED, 3386edab08cSLen Brown .exit_latency = 87, 3396edab08cSLen Brown .target_residency = 300, 3405fe2e527SRafael J. Wysocki .enter = &intel_idle, 3415fe2e527SRafael J. Wysocki .enter_freeze = intel_idle_freeze, }, 342e022e7ebSLen Brown { 343e022e7ebSLen Brown .enter = NULL } 3446edab08cSLen Brown }; 3456edab08cSLen Brown 3460138d8f0SLen Brown static struct cpuidle_state ivt_cstates[] = { 3470138d8f0SLen Brown { 3480138d8f0SLen Brown .name = "C1-IVT", 3490138d8f0SLen Brown .desc = "MWAIT 0x00", 350b82b6ccaSDaniel Lezcano .flags = MWAIT2flg(0x00), 3510138d8f0SLen Brown .exit_latency = 1, 3520138d8f0SLen Brown .target_residency = 1, 3535fe2e527SRafael J. Wysocki .enter = &intel_idle, 3545fe2e527SRafael J. Wysocki .enter_freeze = intel_idle_freeze, }, 3550138d8f0SLen Brown { 3560138d8f0SLen Brown .name = "C1E-IVT", 3570138d8f0SLen Brown .desc = "MWAIT 0x01", 358b82b6ccaSDaniel Lezcano .flags = MWAIT2flg(0x01), 3590138d8f0SLen Brown .exit_latency = 10, 3600138d8f0SLen Brown .target_residency = 80, 3615fe2e527SRafael J. Wysocki .enter = &intel_idle, 3625fe2e527SRafael J. Wysocki .enter_freeze = intel_idle_freeze, }, 3630138d8f0SLen Brown { 3640138d8f0SLen Brown .name = "C3-IVT", 3650138d8f0SLen Brown .desc = "MWAIT 0x10", 366b82b6ccaSDaniel Lezcano .flags = MWAIT2flg(0x10) | CPUIDLE_FLAG_TLB_FLUSHED, 3670138d8f0SLen Brown .exit_latency = 59, 3680138d8f0SLen Brown .target_residency = 156, 3695fe2e527SRafael J. Wysocki .enter = &intel_idle, 3705fe2e527SRafael J. Wysocki .enter_freeze = intel_idle_freeze, }, 3710138d8f0SLen Brown { 3720138d8f0SLen Brown .name = "C6-IVT", 3730138d8f0SLen Brown .desc = "MWAIT 0x20", 374b82b6ccaSDaniel Lezcano .flags = MWAIT2flg(0x20) | CPUIDLE_FLAG_TLB_FLUSHED, 3750138d8f0SLen Brown .exit_latency = 82, 3760138d8f0SLen Brown .target_residency = 300, 3775fe2e527SRafael J. Wysocki .enter = &intel_idle, 3785fe2e527SRafael J. Wysocki .enter_freeze = intel_idle_freeze, }, 3790138d8f0SLen Brown { 3800138d8f0SLen Brown .enter = NULL } 3810138d8f0SLen Brown }; 3820138d8f0SLen Brown 3830138d8f0SLen Brown static struct cpuidle_state ivt_cstates_4s[] = { 3840138d8f0SLen Brown { 3850138d8f0SLen Brown .name = "C1-IVT-4S", 3860138d8f0SLen Brown .desc = "MWAIT 0x00", 387b82b6ccaSDaniel Lezcano .flags = MWAIT2flg(0x00), 3880138d8f0SLen Brown .exit_latency = 1, 3890138d8f0SLen Brown .target_residency = 1, 3905fe2e527SRafael J. Wysocki .enter = &intel_idle, 3915fe2e527SRafael J. Wysocki .enter_freeze = intel_idle_freeze, }, 3920138d8f0SLen Brown { 3930138d8f0SLen Brown .name = "C1E-IVT-4S", 3940138d8f0SLen Brown .desc = "MWAIT 0x01", 395b82b6ccaSDaniel Lezcano .flags = MWAIT2flg(0x01), 3960138d8f0SLen Brown .exit_latency = 10, 3970138d8f0SLen Brown .target_residency = 250, 3985fe2e527SRafael J. Wysocki .enter = &intel_idle, 3995fe2e527SRafael J. Wysocki .enter_freeze = intel_idle_freeze, }, 4000138d8f0SLen Brown { 4010138d8f0SLen Brown .name = "C3-IVT-4S", 4020138d8f0SLen Brown .desc = "MWAIT 0x10", 403b82b6ccaSDaniel Lezcano .flags = MWAIT2flg(0x10) | CPUIDLE_FLAG_TLB_FLUSHED, 4040138d8f0SLen Brown .exit_latency = 59, 4050138d8f0SLen Brown .target_residency = 300, 4065fe2e527SRafael J. Wysocki .enter = &intel_idle, 4075fe2e527SRafael J. Wysocki .enter_freeze = intel_idle_freeze, }, 4080138d8f0SLen Brown { 4090138d8f0SLen Brown .name = "C6-IVT-4S", 4100138d8f0SLen Brown .desc = "MWAIT 0x20", 411b82b6ccaSDaniel Lezcano .flags = MWAIT2flg(0x20) | CPUIDLE_FLAG_TLB_FLUSHED, 4120138d8f0SLen Brown .exit_latency = 84, 4130138d8f0SLen Brown .target_residency = 400, 4145fe2e527SRafael J. Wysocki .enter = &intel_idle, 4155fe2e527SRafael J. Wysocki .enter_freeze = intel_idle_freeze, }, 4160138d8f0SLen Brown { 4170138d8f0SLen Brown .enter = NULL } 4180138d8f0SLen Brown }; 4190138d8f0SLen Brown 4200138d8f0SLen Brown static struct cpuidle_state ivt_cstates_8s[] = { 4210138d8f0SLen Brown { 4220138d8f0SLen Brown .name = "C1-IVT-8S", 4230138d8f0SLen Brown .desc = "MWAIT 0x00", 424b82b6ccaSDaniel Lezcano .flags = MWAIT2flg(0x00), 4250138d8f0SLen Brown .exit_latency = 1, 4260138d8f0SLen Brown .target_residency = 1, 4275fe2e527SRafael J. Wysocki .enter = &intel_idle, 4285fe2e527SRafael J. Wysocki .enter_freeze = intel_idle_freeze, }, 4290138d8f0SLen Brown { 4300138d8f0SLen Brown .name = "C1E-IVT-8S", 4310138d8f0SLen Brown .desc = "MWAIT 0x01", 432b82b6ccaSDaniel Lezcano .flags = MWAIT2flg(0x01), 4330138d8f0SLen Brown .exit_latency = 10, 4340138d8f0SLen Brown .target_residency = 500, 4355fe2e527SRafael J. Wysocki .enter = &intel_idle, 4365fe2e527SRafael J. Wysocki .enter_freeze = intel_idle_freeze, }, 4370138d8f0SLen Brown { 4380138d8f0SLen Brown .name = "C3-IVT-8S", 4390138d8f0SLen Brown .desc = "MWAIT 0x10", 440b82b6ccaSDaniel Lezcano .flags = MWAIT2flg(0x10) | CPUIDLE_FLAG_TLB_FLUSHED, 4410138d8f0SLen Brown .exit_latency = 59, 4420138d8f0SLen Brown .target_residency = 600, 4435fe2e527SRafael J. Wysocki .enter = &intel_idle, 4445fe2e527SRafael J. Wysocki .enter_freeze = intel_idle_freeze, }, 4450138d8f0SLen Brown { 4460138d8f0SLen Brown .name = "C6-IVT-8S", 4470138d8f0SLen Brown .desc = "MWAIT 0x20", 448b82b6ccaSDaniel Lezcano .flags = MWAIT2flg(0x20) | CPUIDLE_FLAG_TLB_FLUSHED, 4490138d8f0SLen Brown .exit_latency = 88, 4500138d8f0SLen Brown .target_residency = 700, 4515fe2e527SRafael J. Wysocki .enter = &intel_idle, 4525fe2e527SRafael J. Wysocki .enter_freeze = intel_idle_freeze, }, 4530138d8f0SLen Brown { 4540138d8f0SLen Brown .enter = NULL } 4550138d8f0SLen Brown }; 4560138d8f0SLen Brown 457ba0dc81eSJiang Liu static struct cpuidle_state hsw_cstates[] = { 458e022e7ebSLen Brown { 45985a4d2d4SLen Brown .name = "C1-HSW", 46085a4d2d4SLen Brown .desc = "MWAIT 0x00", 461b82b6ccaSDaniel Lezcano .flags = MWAIT2flg(0x00), 46285a4d2d4SLen Brown .exit_latency = 2, 46385a4d2d4SLen Brown .target_residency = 2, 4645fe2e527SRafael J. Wysocki .enter = &intel_idle, 4655fe2e527SRafael J. Wysocki .enter_freeze = intel_idle_freeze, }, 466e022e7ebSLen Brown { 46732e95180SLen Brown .name = "C1E-HSW", 46832e95180SLen Brown .desc = "MWAIT 0x01", 469b82b6ccaSDaniel Lezcano .flags = MWAIT2flg(0x01), 47032e95180SLen Brown .exit_latency = 10, 47132e95180SLen Brown .target_residency = 20, 4725fe2e527SRafael J. Wysocki .enter = &intel_idle, 4735fe2e527SRafael J. Wysocki .enter_freeze = intel_idle_freeze, }, 47432e95180SLen Brown { 47585a4d2d4SLen Brown .name = "C3-HSW", 47685a4d2d4SLen Brown .desc = "MWAIT 0x10", 477b82b6ccaSDaniel Lezcano .flags = MWAIT2flg(0x10) | CPUIDLE_FLAG_TLB_FLUSHED, 47885a4d2d4SLen Brown .exit_latency = 33, 47985a4d2d4SLen Brown .target_residency = 100, 4805fe2e527SRafael J. Wysocki .enter = &intel_idle, 4815fe2e527SRafael J. Wysocki .enter_freeze = intel_idle_freeze, }, 482e022e7ebSLen Brown { 48385a4d2d4SLen Brown .name = "C6-HSW", 48485a4d2d4SLen Brown .desc = "MWAIT 0x20", 485b82b6ccaSDaniel Lezcano .flags = MWAIT2flg(0x20) | CPUIDLE_FLAG_TLB_FLUSHED, 48685a4d2d4SLen Brown .exit_latency = 133, 48785a4d2d4SLen Brown .target_residency = 400, 4885fe2e527SRafael J. Wysocki .enter = &intel_idle, 4895fe2e527SRafael J. Wysocki .enter_freeze = intel_idle_freeze, }, 490e022e7ebSLen Brown { 49185a4d2d4SLen Brown .name = "C7s-HSW", 49285a4d2d4SLen Brown .desc = "MWAIT 0x32", 493b82b6ccaSDaniel Lezcano .flags = MWAIT2flg(0x32) | CPUIDLE_FLAG_TLB_FLUSHED, 49485a4d2d4SLen Brown .exit_latency = 166, 49585a4d2d4SLen Brown .target_residency = 500, 4965fe2e527SRafael J. Wysocki .enter = &intel_idle, 4975fe2e527SRafael J. Wysocki .enter_freeze = intel_idle_freeze, }, 498e022e7ebSLen Brown { 49986239cebSLen Brown .name = "C8-HSW", 50086239cebSLen Brown .desc = "MWAIT 0x40", 501b82b6ccaSDaniel Lezcano .flags = MWAIT2flg(0x40) | CPUIDLE_FLAG_TLB_FLUSHED, 50286239cebSLen Brown .exit_latency = 300, 50386239cebSLen Brown .target_residency = 900, 5045fe2e527SRafael J. Wysocki .enter = &intel_idle, 5055fe2e527SRafael J. Wysocki .enter_freeze = intel_idle_freeze, }, 50686239cebSLen Brown { 50786239cebSLen Brown .name = "C9-HSW", 50886239cebSLen Brown .desc = "MWAIT 0x50", 509b82b6ccaSDaniel Lezcano .flags = MWAIT2flg(0x50) | CPUIDLE_FLAG_TLB_FLUSHED, 51086239cebSLen Brown .exit_latency = 600, 51186239cebSLen Brown .target_residency = 1800, 5125fe2e527SRafael J. Wysocki .enter = &intel_idle, 5135fe2e527SRafael J. Wysocki .enter_freeze = intel_idle_freeze, }, 51486239cebSLen Brown { 51586239cebSLen Brown .name = "C10-HSW", 51686239cebSLen Brown .desc = "MWAIT 0x60", 517b82b6ccaSDaniel Lezcano .flags = MWAIT2flg(0x60) | CPUIDLE_FLAG_TLB_FLUSHED, 51886239cebSLen Brown .exit_latency = 2600, 51986239cebSLen Brown .target_residency = 7700, 5205fe2e527SRafael J. Wysocki .enter = &intel_idle, 5215fe2e527SRafael J. Wysocki .enter_freeze = intel_idle_freeze, }, 52286239cebSLen Brown { 523e022e7ebSLen Brown .enter = NULL } 52485a4d2d4SLen Brown }; 525a138b568SLen Brown static struct cpuidle_state bdw_cstates[] = { 526a138b568SLen Brown { 527a138b568SLen Brown .name = "C1-BDW", 528a138b568SLen Brown .desc = "MWAIT 0x00", 529b82b6ccaSDaniel Lezcano .flags = MWAIT2flg(0x00), 530a138b568SLen Brown .exit_latency = 2, 531a138b568SLen Brown .target_residency = 2, 5325fe2e527SRafael J. Wysocki .enter = &intel_idle, 5335fe2e527SRafael J. Wysocki .enter_freeze = intel_idle_freeze, }, 534a138b568SLen Brown { 535a138b568SLen Brown .name = "C1E-BDW", 536a138b568SLen Brown .desc = "MWAIT 0x01", 537b82b6ccaSDaniel Lezcano .flags = MWAIT2flg(0x01), 538a138b568SLen Brown .exit_latency = 10, 539a138b568SLen Brown .target_residency = 20, 5405fe2e527SRafael J. Wysocki .enter = &intel_idle, 5415fe2e527SRafael J. Wysocki .enter_freeze = intel_idle_freeze, }, 542a138b568SLen Brown { 543a138b568SLen Brown .name = "C3-BDW", 544a138b568SLen Brown .desc = "MWAIT 0x10", 545b82b6ccaSDaniel Lezcano .flags = MWAIT2flg(0x10) | CPUIDLE_FLAG_TLB_FLUSHED, 546a138b568SLen Brown .exit_latency = 40, 547a138b568SLen Brown .target_residency = 100, 5485fe2e527SRafael J. Wysocki .enter = &intel_idle, 5495fe2e527SRafael J. Wysocki .enter_freeze = intel_idle_freeze, }, 550a138b568SLen Brown { 551a138b568SLen Brown .name = "C6-BDW", 552a138b568SLen Brown .desc = "MWAIT 0x20", 553b82b6ccaSDaniel Lezcano .flags = MWAIT2flg(0x20) | CPUIDLE_FLAG_TLB_FLUSHED, 554a138b568SLen Brown .exit_latency = 133, 555a138b568SLen Brown .target_residency = 400, 5565fe2e527SRafael J. Wysocki .enter = &intel_idle, 5575fe2e527SRafael J. Wysocki .enter_freeze = intel_idle_freeze, }, 558a138b568SLen Brown { 559a138b568SLen Brown .name = "C7s-BDW", 560a138b568SLen Brown .desc = "MWAIT 0x32", 561b82b6ccaSDaniel Lezcano .flags = MWAIT2flg(0x32) | CPUIDLE_FLAG_TLB_FLUSHED, 562a138b568SLen Brown .exit_latency = 166, 563a138b568SLen Brown .target_residency = 500, 5645fe2e527SRafael J. Wysocki .enter = &intel_idle, 5655fe2e527SRafael J. Wysocki .enter_freeze = intel_idle_freeze, }, 566a138b568SLen Brown { 567a138b568SLen Brown .name = "C8-BDW", 568a138b568SLen Brown .desc = "MWAIT 0x40", 569b82b6ccaSDaniel Lezcano .flags = MWAIT2flg(0x40) | CPUIDLE_FLAG_TLB_FLUSHED, 570a138b568SLen Brown .exit_latency = 300, 571a138b568SLen Brown .target_residency = 900, 5725fe2e527SRafael J. Wysocki .enter = &intel_idle, 5735fe2e527SRafael J. Wysocki .enter_freeze = intel_idle_freeze, }, 574a138b568SLen Brown { 575a138b568SLen Brown .name = "C9-BDW", 576a138b568SLen Brown .desc = "MWAIT 0x50", 577b82b6ccaSDaniel Lezcano .flags = MWAIT2flg(0x50) | CPUIDLE_FLAG_TLB_FLUSHED, 578a138b568SLen Brown .exit_latency = 600, 579a138b568SLen Brown .target_residency = 1800, 5805fe2e527SRafael J. Wysocki .enter = &intel_idle, 5815fe2e527SRafael J. Wysocki .enter_freeze = intel_idle_freeze, }, 582a138b568SLen Brown { 583a138b568SLen Brown .name = "C10-BDW", 584a138b568SLen Brown .desc = "MWAIT 0x60", 585b82b6ccaSDaniel Lezcano .flags = MWAIT2flg(0x60) | CPUIDLE_FLAG_TLB_FLUSHED, 586a138b568SLen Brown .exit_latency = 2600, 587a138b568SLen Brown .target_residency = 7700, 5885fe2e527SRafael J. Wysocki .enter = &intel_idle, 5895fe2e527SRafael J. Wysocki .enter_freeze = intel_idle_freeze, }, 590a138b568SLen Brown { 591a138b568SLen Brown .enter = NULL } 592a138b568SLen Brown }; 59385a4d2d4SLen Brown 594493f133fSLen Brown static struct cpuidle_state skl_cstates[] = { 595493f133fSLen Brown { 596493f133fSLen Brown .name = "C1-SKL", 597493f133fSLen Brown .desc = "MWAIT 0x00", 598493f133fSLen Brown .flags = MWAIT2flg(0x00), 599493f133fSLen Brown .exit_latency = 2, 600493f133fSLen Brown .target_residency = 2, 601493f133fSLen Brown .enter = &intel_idle, 602493f133fSLen Brown .enter_freeze = intel_idle_freeze, }, 603493f133fSLen Brown { 604493f133fSLen Brown .name = "C1E-SKL", 605493f133fSLen Brown .desc = "MWAIT 0x01", 606493f133fSLen Brown .flags = MWAIT2flg(0x01), 607493f133fSLen Brown .exit_latency = 10, 608493f133fSLen Brown .target_residency = 20, 609493f133fSLen Brown .enter = &intel_idle, 610493f133fSLen Brown .enter_freeze = intel_idle_freeze, }, 611493f133fSLen Brown { 612493f133fSLen Brown .name = "C3-SKL", 613493f133fSLen Brown .desc = "MWAIT 0x10", 614493f133fSLen Brown .flags = MWAIT2flg(0x10) | CPUIDLE_FLAG_TLB_FLUSHED, 615493f133fSLen Brown .exit_latency = 70, 616493f133fSLen Brown .target_residency = 100, 617493f133fSLen Brown .enter = &intel_idle, 618493f133fSLen Brown .enter_freeze = intel_idle_freeze, }, 619493f133fSLen Brown { 620493f133fSLen Brown .name = "C6-SKL", 621493f133fSLen Brown .desc = "MWAIT 0x20", 622493f133fSLen Brown .flags = MWAIT2flg(0x20) | CPUIDLE_FLAG_TLB_FLUSHED, 623135919a3SLen Brown .exit_latency = 85, 624493f133fSLen Brown .target_residency = 200, 625493f133fSLen Brown .enter = &intel_idle, 626493f133fSLen Brown .enter_freeze = intel_idle_freeze, }, 627493f133fSLen Brown { 628493f133fSLen Brown .name = "C7s-SKL", 629493f133fSLen Brown .desc = "MWAIT 0x33", 630493f133fSLen Brown .flags = MWAIT2flg(0x33) | CPUIDLE_FLAG_TLB_FLUSHED, 631493f133fSLen Brown .exit_latency = 124, 632493f133fSLen Brown .target_residency = 800, 633493f133fSLen Brown .enter = &intel_idle, 634493f133fSLen Brown .enter_freeze = intel_idle_freeze, }, 635493f133fSLen Brown { 636493f133fSLen Brown .name = "C8-SKL", 637493f133fSLen Brown .desc = "MWAIT 0x40", 638493f133fSLen Brown .flags = MWAIT2flg(0x40) | CPUIDLE_FLAG_TLB_FLUSHED, 639135919a3SLen Brown .exit_latency = 200, 640493f133fSLen Brown .target_residency = 800, 641493f133fSLen Brown .enter = &intel_idle, 642493f133fSLen Brown .enter_freeze = intel_idle_freeze, }, 643493f133fSLen Brown { 644135919a3SLen Brown .name = "C9-SKL", 645135919a3SLen Brown .desc = "MWAIT 0x50", 646135919a3SLen Brown .flags = MWAIT2flg(0x50) | CPUIDLE_FLAG_TLB_FLUSHED, 647135919a3SLen Brown .exit_latency = 480, 648135919a3SLen Brown .target_residency = 5000, 649135919a3SLen Brown .enter = &intel_idle, 650135919a3SLen Brown .enter_freeze = intel_idle_freeze, }, 651135919a3SLen Brown { 652493f133fSLen Brown .name = "C10-SKL", 653493f133fSLen Brown .desc = "MWAIT 0x60", 654493f133fSLen Brown .flags = MWAIT2flg(0x60) | CPUIDLE_FLAG_TLB_FLUSHED, 655493f133fSLen Brown .exit_latency = 890, 656493f133fSLen Brown .target_residency = 5000, 657493f133fSLen Brown .enter = &intel_idle, 658493f133fSLen Brown .enter_freeze = intel_idle_freeze, }, 659493f133fSLen Brown { 660493f133fSLen Brown .enter = NULL } 661493f133fSLen Brown }; 662493f133fSLen Brown 663ba0dc81eSJiang Liu static struct cpuidle_state atom_cstates[] = { 664e022e7ebSLen Brown { 66532e95180SLen Brown .name = "C1E-ATM", 66626717172SLen Brown .desc = "MWAIT 0x00", 667b82b6ccaSDaniel Lezcano .flags = MWAIT2flg(0x00), 66832e95180SLen Brown .exit_latency = 10, 66932e95180SLen Brown .target_residency = 20, 6705fe2e527SRafael J. Wysocki .enter = &intel_idle, 6715fe2e527SRafael J. Wysocki .enter_freeze = intel_idle_freeze, }, 672e022e7ebSLen Brown { 67315e123e5SThomas Renninger .name = "C2-ATM", 67426717172SLen Brown .desc = "MWAIT 0x10", 675b82b6ccaSDaniel Lezcano .flags = MWAIT2flg(0x10), 67626717172SLen Brown .exit_latency = 20, 67726717172SLen Brown .target_residency = 80, 6785fe2e527SRafael J. Wysocki .enter = &intel_idle, 6795fe2e527SRafael J. Wysocki .enter_freeze = intel_idle_freeze, }, 680e022e7ebSLen Brown { 68115e123e5SThomas Renninger .name = "C4-ATM", 68226717172SLen Brown .desc = "MWAIT 0x30", 683b82b6ccaSDaniel Lezcano .flags = MWAIT2flg(0x30) | CPUIDLE_FLAG_TLB_FLUSHED, 68426717172SLen Brown .exit_latency = 100, 68526717172SLen Brown .target_residency = 400, 6865fe2e527SRafael J. Wysocki .enter = &intel_idle, 6875fe2e527SRafael J. Wysocki .enter_freeze = intel_idle_freeze, }, 688e022e7ebSLen Brown { 68915e123e5SThomas Renninger .name = "C6-ATM", 6907fcca7d9SLen Brown .desc = "MWAIT 0x52", 691b82b6ccaSDaniel Lezcano .flags = MWAIT2flg(0x52) | CPUIDLE_FLAG_TLB_FLUSHED, 6927fcca7d9SLen Brown .exit_latency = 140, 6937fcca7d9SLen Brown .target_residency = 560, 6945fe2e527SRafael J. Wysocki .enter = &intel_idle, 6955fe2e527SRafael J. Wysocki .enter_freeze = intel_idle_freeze, }, 696e022e7ebSLen Brown { 697e022e7ebSLen Brown .enter = NULL } 69826717172SLen Brown }; 69988390996SJiang Liu static struct cpuidle_state avn_cstates[] = { 700fab04b22SLen Brown { 701fab04b22SLen Brown .name = "C1-AVN", 702fab04b22SLen Brown .desc = "MWAIT 0x00", 703b82b6ccaSDaniel Lezcano .flags = MWAIT2flg(0x00), 704fab04b22SLen Brown .exit_latency = 2, 705fab04b22SLen Brown .target_residency = 2, 7065fe2e527SRafael J. Wysocki .enter = &intel_idle, 7075fe2e527SRafael J. Wysocki .enter_freeze = intel_idle_freeze, }, 708fab04b22SLen Brown { 709fab04b22SLen Brown .name = "C6-AVN", 710fab04b22SLen Brown .desc = "MWAIT 0x51", 711b82b6ccaSDaniel Lezcano .flags = MWAIT2flg(0x51) | CPUIDLE_FLAG_TLB_FLUSHED, 712fab04b22SLen Brown .exit_latency = 15, 713fab04b22SLen Brown .target_residency = 45, 7145fe2e527SRafael J. Wysocki .enter = &intel_idle, 7155fe2e527SRafael J. Wysocki .enter_freeze = intel_idle_freeze, }, 71688390996SJiang Liu { 71788390996SJiang Liu .enter = NULL } 718fab04b22SLen Brown }; 71926717172SLen Brown 72026717172SLen Brown /** 72126717172SLen Brown * intel_idle 72226717172SLen Brown * @dev: cpuidle_device 72346bcfad7SDeepthi Dharwar * @drv: cpuidle driver 724e978aa7dSDeepthi Dharwar * @index: index of cpuidle state 72526717172SLen Brown * 72663ff07beSYanmin Zhang * Must be called under local_irq_disable(). 72726717172SLen Brown */ 72846bcfad7SDeepthi Dharwar static int intel_idle(struct cpuidle_device *dev, 72946bcfad7SDeepthi Dharwar struct cpuidle_driver *drv, int index) 73026717172SLen Brown { 73126717172SLen Brown unsigned long ecx = 1; /* break on interrupt flag */ 73246bcfad7SDeepthi Dharwar struct cpuidle_state *state = &drv->states[index]; 733b1beab48SLen Brown unsigned long eax = flg2MWAIT(state->flags); 73426717172SLen Brown unsigned int cstate; 73526717172SLen Brown int cpu = smp_processor_id(); 73626717172SLen Brown 73726717172SLen Brown cstate = (((eax) >> MWAIT_SUBSTATE_SIZE) & MWAIT_CSTATE_MASK) + 1; 73826717172SLen Brown 7396110a1f4SSuresh Siddha /* 740c8381cc3SLen Brown * leave_mm() to avoid costly and often unnecessary wakeups 741c8381cc3SLen Brown * for flushing the user TLB's associated with the active mm. 7426110a1f4SSuresh Siddha */ 743c8381cc3SLen Brown if (state->flags & CPUIDLE_FLAG_TLB_FLUSHED) 7446110a1f4SSuresh Siddha leave_mm(cpu); 7456110a1f4SSuresh Siddha 74626717172SLen Brown if (!(lapic_timer_reliable_states & (1 << (cstate)))) 747f6cee191SThomas Gleixner tick_broadcast_enter(); 74826717172SLen Brown 74916824255SPeter Zijlstra mwait_idle_with_hints(eax, ecx); 75026717172SLen Brown 75126717172SLen Brown if (!(lapic_timer_reliable_states & (1 << (cstate)))) 752f6cee191SThomas Gleixner tick_broadcast_exit(); 75326717172SLen Brown 754e978aa7dSDeepthi Dharwar return index; 75526717172SLen Brown } 75626717172SLen Brown 7575fe2e527SRafael J. Wysocki /** 7585fe2e527SRafael J. Wysocki * intel_idle_freeze - simplified "enter" callback routine for suspend-to-idle 7595fe2e527SRafael J. Wysocki * @dev: cpuidle_device 7605fe2e527SRafael J. Wysocki * @drv: cpuidle driver 7615fe2e527SRafael J. Wysocki * @index: state index 7625fe2e527SRafael J. Wysocki */ 7635fe2e527SRafael J. Wysocki static void intel_idle_freeze(struct cpuidle_device *dev, 7645fe2e527SRafael J. Wysocki struct cpuidle_driver *drv, int index) 7655fe2e527SRafael J. Wysocki { 7665fe2e527SRafael J. Wysocki unsigned long ecx = 1; /* break on interrupt flag */ 7675fe2e527SRafael J. Wysocki unsigned long eax = flg2MWAIT(drv->states[index].flags); 7685fe2e527SRafael J. Wysocki 7695fe2e527SRafael J. Wysocki mwait_idle_with_hints(eax, ecx); 7705fe2e527SRafael J. Wysocki } 7715fe2e527SRafael J. Wysocki 7722a2d31c8SShaohua Li static void __setup_broadcast_timer(void *arg) 7732a2d31c8SShaohua Li { 77476962caaSThomas Gleixner unsigned long on = (unsigned long)arg; 7752a2d31c8SShaohua Li 77676962caaSThomas Gleixner if (on) 77776962caaSThomas Gleixner tick_broadcast_enable(); 77876962caaSThomas Gleixner else 77976962caaSThomas Gleixner tick_broadcast_disable(); 7802a2d31c8SShaohua Li } 7812a2d31c8SShaohua Li 78225ac7761SDaniel Lezcano static int cpu_hotplug_notify(struct notifier_block *n, 7832a2d31c8SShaohua Li unsigned long action, void *hcpu) 7842a2d31c8SShaohua Li { 7852a2d31c8SShaohua Li int hotcpu = (unsigned long)hcpu; 78625ac7761SDaniel Lezcano struct cpuidle_device *dev; 7872a2d31c8SShaohua Li 788e2401453SPrarit Bhargava switch (action & ~CPU_TASKS_FROZEN) { 7892a2d31c8SShaohua Li case CPU_ONLINE: 79025ac7761SDaniel Lezcano 79125ac7761SDaniel Lezcano if (lapic_timer_reliable_states != LAPIC_TIMER_ALWAYS_RELIABLE) 7922a2d31c8SShaohua Li smp_call_function_single(hotcpu, __setup_broadcast_timer, 7932a2d31c8SShaohua Li (void *)true, 1); 79425ac7761SDaniel Lezcano 79525ac7761SDaniel Lezcano /* 79625ac7761SDaniel Lezcano * Some systems can hotplug a cpu at runtime after 79725ac7761SDaniel Lezcano * the kernel has booted, we have to initialize the 79825ac7761SDaniel Lezcano * driver in this case 79925ac7761SDaniel Lezcano */ 80025ac7761SDaniel Lezcano dev = per_cpu_ptr(intel_idle_cpuidle_devices, hotcpu); 80125ac7761SDaniel Lezcano if (!dev->registered) 80225ac7761SDaniel Lezcano intel_idle_cpu_init(hotcpu); 80325ac7761SDaniel Lezcano 8042a2d31c8SShaohua Li break; 8052a2d31c8SShaohua Li } 8062a2d31c8SShaohua Li return NOTIFY_OK; 8072a2d31c8SShaohua Li } 8082a2d31c8SShaohua Li 80925ac7761SDaniel Lezcano static struct notifier_block cpu_hotplug_notifier = { 81025ac7761SDaniel Lezcano .notifier_call = cpu_hotplug_notify, 8112a2d31c8SShaohua Li }; 8122a2d31c8SShaohua Li 81314796fcaSLen Brown static void auto_demotion_disable(void *dummy) 81414796fcaSLen Brown { 81514796fcaSLen Brown unsigned long long msr_bits; 81614796fcaSLen Brown 81714796fcaSLen Brown rdmsrl(MSR_NHM_SNB_PKG_CST_CFG_CTL, msr_bits); 818b66b8b9aSAndi Kleen msr_bits &= ~(icpu->auto_demotion_disable_flags); 81914796fcaSLen Brown wrmsrl(MSR_NHM_SNB_PKG_CST_CFG_CTL, msr_bits); 82014796fcaSLen Brown } 82132e95180SLen Brown static void c1e_promotion_disable(void *dummy) 82232e95180SLen Brown { 82332e95180SLen Brown unsigned long long msr_bits; 82432e95180SLen Brown 82532e95180SLen Brown rdmsrl(MSR_IA32_POWER_CTL, msr_bits); 82632e95180SLen Brown msr_bits &= ~0x2; 82732e95180SLen Brown wrmsrl(MSR_IA32_POWER_CTL, msr_bits); 82832e95180SLen Brown } 82914796fcaSLen Brown 830b66b8b9aSAndi Kleen static const struct idle_cpu idle_cpu_nehalem = { 831b66b8b9aSAndi Kleen .state_table = nehalem_cstates, 832b66b8b9aSAndi Kleen .auto_demotion_disable_flags = NHM_C1_AUTO_DEMOTE | NHM_C3_AUTO_DEMOTE, 83332e95180SLen Brown .disable_promotion_to_c1e = true, 834b66b8b9aSAndi Kleen }; 835b66b8b9aSAndi Kleen 836b66b8b9aSAndi Kleen static const struct idle_cpu idle_cpu_atom = { 837b66b8b9aSAndi Kleen .state_table = atom_cstates, 838b66b8b9aSAndi Kleen }; 839b66b8b9aSAndi Kleen 840b66b8b9aSAndi Kleen static const struct idle_cpu idle_cpu_lincroft = { 841b66b8b9aSAndi Kleen .state_table = atom_cstates, 842b66b8b9aSAndi Kleen .auto_demotion_disable_flags = ATM_LNC_C6_AUTO_DEMOTE, 843b66b8b9aSAndi Kleen }; 844b66b8b9aSAndi Kleen 845b66b8b9aSAndi Kleen static const struct idle_cpu idle_cpu_snb = { 846b66b8b9aSAndi Kleen .state_table = snb_cstates, 84732e95180SLen Brown .disable_promotion_to_c1e = true, 848b66b8b9aSAndi Kleen }; 849b66b8b9aSAndi Kleen 850718987d6SLen Brown static const struct idle_cpu idle_cpu_byt = { 851718987d6SLen Brown .state_table = byt_cstates, 852718987d6SLen Brown .disable_promotion_to_c1e = true, 8538c058d53SLen Brown .byt_auto_demotion_disable_flag = true, 854718987d6SLen Brown }; 855718987d6SLen Brown 856cab07a56SLen Brown static const struct idle_cpu idle_cpu_cht = { 857cab07a56SLen Brown .state_table = cht_cstates, 858cab07a56SLen Brown .disable_promotion_to_c1e = true, 859cab07a56SLen Brown .byt_auto_demotion_disable_flag = true, 860cab07a56SLen Brown }; 861cab07a56SLen Brown 8626edab08cSLen Brown static const struct idle_cpu idle_cpu_ivb = { 8636edab08cSLen Brown .state_table = ivb_cstates, 86432e95180SLen Brown .disable_promotion_to_c1e = true, 8656edab08cSLen Brown }; 8666edab08cSLen Brown 8670138d8f0SLen Brown static const struct idle_cpu idle_cpu_ivt = { 8680138d8f0SLen Brown .state_table = ivt_cstates, 8690138d8f0SLen Brown .disable_promotion_to_c1e = true, 8700138d8f0SLen Brown }; 8710138d8f0SLen Brown 87285a4d2d4SLen Brown static const struct idle_cpu idle_cpu_hsw = { 87385a4d2d4SLen Brown .state_table = hsw_cstates, 87432e95180SLen Brown .disable_promotion_to_c1e = true, 87585a4d2d4SLen Brown }; 87685a4d2d4SLen Brown 877a138b568SLen Brown static const struct idle_cpu idle_cpu_bdw = { 878a138b568SLen Brown .state_table = bdw_cstates, 879a138b568SLen Brown .disable_promotion_to_c1e = true, 880a138b568SLen Brown }; 881a138b568SLen Brown 882493f133fSLen Brown static const struct idle_cpu idle_cpu_skl = { 883493f133fSLen Brown .state_table = skl_cstates, 884493f133fSLen Brown .disable_promotion_to_c1e = true, 885493f133fSLen Brown }; 886493f133fSLen Brown 887493f133fSLen Brown 888fab04b22SLen Brown static const struct idle_cpu idle_cpu_avn = { 889fab04b22SLen Brown .state_table = avn_cstates, 890fab04b22SLen Brown .disable_promotion_to_c1e = true, 891fab04b22SLen Brown }; 892fab04b22SLen Brown 893b66b8b9aSAndi Kleen #define ICPU(model, cpu) \ 894b66b8b9aSAndi Kleen { X86_VENDOR_INTEL, 6, model, X86_FEATURE_MWAIT, (unsigned long)&cpu } 895b66b8b9aSAndi Kleen 896d5cdc3c4SMathias Krause static const struct x86_cpu_id intel_idle_ids[] __initconst = { 897b66b8b9aSAndi Kleen ICPU(0x1a, idle_cpu_nehalem), 898b66b8b9aSAndi Kleen ICPU(0x1e, idle_cpu_nehalem), 899b66b8b9aSAndi Kleen ICPU(0x1f, idle_cpu_nehalem), 9008bf11938SBen Hutchings ICPU(0x25, idle_cpu_nehalem), 9018bf11938SBen Hutchings ICPU(0x2c, idle_cpu_nehalem), 9028bf11938SBen Hutchings ICPU(0x2e, idle_cpu_nehalem), 903b66b8b9aSAndi Kleen ICPU(0x1c, idle_cpu_atom), 904b66b8b9aSAndi Kleen ICPU(0x26, idle_cpu_lincroft), 9058bf11938SBen Hutchings ICPU(0x2f, idle_cpu_nehalem), 906b66b8b9aSAndi Kleen ICPU(0x2a, idle_cpu_snb), 907b66b8b9aSAndi Kleen ICPU(0x2d, idle_cpu_snb), 908acead1b0SJan Kiszka ICPU(0x36, idle_cpu_atom), 909718987d6SLen Brown ICPU(0x37, idle_cpu_byt), 910cab07a56SLen Brown ICPU(0x4c, idle_cpu_cht), 9116edab08cSLen Brown ICPU(0x3a, idle_cpu_ivb), 9120138d8f0SLen Brown ICPU(0x3e, idle_cpu_ivt), 91385a4d2d4SLen Brown ICPU(0x3c, idle_cpu_hsw), 91485a4d2d4SLen Brown ICPU(0x3f, idle_cpu_hsw), 91585a4d2d4SLen Brown ICPU(0x45, idle_cpu_hsw), 9160b15841bSLen Brown ICPU(0x46, idle_cpu_hsw), 917a138b568SLen Brown ICPU(0x4d, idle_cpu_avn), 918a138b568SLen Brown ICPU(0x3d, idle_cpu_bdw), 919bea57077SLen Brown ICPU(0x47, idle_cpu_bdw), 920a138b568SLen Brown ICPU(0x4f, idle_cpu_bdw), 921a138b568SLen Brown ICPU(0x56, idle_cpu_bdw), 922493f133fSLen Brown ICPU(0x4e, idle_cpu_skl), 923493f133fSLen Brown ICPU(0x5e, idle_cpu_skl), 924b66b8b9aSAndi Kleen {} 925b66b8b9aSAndi Kleen }; 926b66b8b9aSAndi Kleen MODULE_DEVICE_TABLE(x86cpu, intel_idle_ids); 927b66b8b9aSAndi Kleen 92826717172SLen Brown /* 92926717172SLen Brown * intel_idle_probe() 93026717172SLen Brown */ 93100f3e755SBartlomiej Zolnierkiewicz static int __init intel_idle_probe(void) 93226717172SLen Brown { 933c4236282SLen Brown unsigned int eax, ebx, ecx; 934b66b8b9aSAndi Kleen const struct x86_cpu_id *id; 93526717172SLen Brown 93626717172SLen Brown if (max_cstate == 0) { 93726717172SLen Brown pr_debug(PREFIX "disabled\n"); 93826717172SLen Brown return -EPERM; 93926717172SLen Brown } 94026717172SLen Brown 941b66b8b9aSAndi Kleen id = x86_match_cpu(intel_idle_ids); 942b66b8b9aSAndi Kleen if (!id) { 943b66b8b9aSAndi Kleen if (boot_cpu_data.x86_vendor == X86_VENDOR_INTEL && 944b66b8b9aSAndi Kleen boot_cpu_data.x86 == 6) 945b66b8b9aSAndi Kleen pr_debug(PREFIX "does not run on family %d model %d\n", 946b66b8b9aSAndi Kleen boot_cpu_data.x86, boot_cpu_data.x86_model); 94726717172SLen Brown return -ENODEV; 948b66b8b9aSAndi Kleen } 94926717172SLen Brown 95026717172SLen Brown if (boot_cpu_data.cpuid_level < CPUID_MWAIT_LEAF) 95126717172SLen Brown return -ENODEV; 95226717172SLen Brown 953c4236282SLen Brown cpuid(CPUID_MWAIT_LEAF, &eax, &ebx, &ecx, &mwait_substates); 95426717172SLen Brown 95526717172SLen Brown if (!(ecx & CPUID5_ECX_EXTENSIONS_SUPPORTED) || 9565c2a9f06SThomas Renninger !(ecx & CPUID5_ECX_INTERRUPT_BREAK) || 9575c2a9f06SThomas Renninger !mwait_substates) 95826717172SLen Brown return -ENODEV; 95926717172SLen Brown 960c4236282SLen Brown pr_debug(PREFIX "MWAIT substates: 0x%x\n", mwait_substates); 96126717172SLen Brown 962b66b8b9aSAndi Kleen icpu = (const struct idle_cpu *)id->driver_data; 963b66b8b9aSAndi Kleen cpuidle_state_table = icpu->state_table; 96426717172SLen Brown 96556b9aea3SLen Brown if (boot_cpu_has(X86_FEATURE_ARAT)) /* Always Reliable APIC Timer */ 9662a2d31c8SShaohua Li lapic_timer_reliable_states = LAPIC_TIMER_ALWAYS_RELIABLE; 96725ac7761SDaniel Lezcano else 96839a74fdeSShaohua Li on_each_cpu(__setup_broadcast_timer, (void *)true, 1); 96925ac7761SDaniel Lezcano 97026717172SLen Brown pr_debug(PREFIX "v" INTEL_IDLE_VERSION 97126717172SLen Brown " model 0x%X\n", boot_cpu_data.x86_model); 97226717172SLen Brown 97326717172SLen Brown pr_debug(PREFIX "lapic_timer_reliable_states 0x%x\n", 97426717172SLen Brown lapic_timer_reliable_states); 97526717172SLen Brown return 0; 97626717172SLen Brown } 97726717172SLen Brown 97826717172SLen Brown /* 97926717172SLen Brown * intel_idle_cpuidle_devices_uninit() 98026717172SLen Brown * unregister, free cpuidle_devices 98126717172SLen Brown */ 98226717172SLen Brown static void intel_idle_cpuidle_devices_uninit(void) 98326717172SLen Brown { 98426717172SLen Brown int i; 98526717172SLen Brown struct cpuidle_device *dev; 98626717172SLen Brown 98726717172SLen Brown for_each_online_cpu(i) { 98826717172SLen Brown dev = per_cpu_ptr(intel_idle_cpuidle_devices, i); 98926717172SLen Brown cpuidle_unregister_device(dev); 99026717172SLen Brown } 99126717172SLen Brown 99226717172SLen Brown free_percpu(intel_idle_cpuidle_devices); 99326717172SLen Brown return; 99426717172SLen Brown } 9950138d8f0SLen Brown 9960138d8f0SLen Brown /* 997d70e28f5SLen Brown * ivt_idle_state_table_update(void) 9980138d8f0SLen Brown * 999d70e28f5SLen Brown * Tune IVT multi-socket targets 10000138d8f0SLen Brown * Assumption: num_sockets == (max_package_num + 1) 10010138d8f0SLen Brown */ 1002d70e28f5SLen Brown static void ivt_idle_state_table_update(void) 10030138d8f0SLen Brown { 10040138d8f0SLen Brown /* IVT uses a different table for 1-2, 3-4, and > 4 sockets */ 10050138d8f0SLen Brown int cpu, package_num, num_sockets = 1; 10060138d8f0SLen Brown 10070138d8f0SLen Brown for_each_online_cpu(cpu) { 10080138d8f0SLen Brown package_num = topology_physical_package_id(cpu); 10090138d8f0SLen Brown if (package_num + 1 > num_sockets) { 10100138d8f0SLen Brown num_sockets = package_num + 1; 10110138d8f0SLen Brown 1012d27dca42SChristoph Jaeger if (num_sockets > 4) { 10130138d8f0SLen Brown cpuidle_state_table = ivt_cstates_8s; 10140138d8f0SLen Brown return; 10150138d8f0SLen Brown } 10160138d8f0SLen Brown } 1017d27dca42SChristoph Jaeger } 10180138d8f0SLen Brown 10190138d8f0SLen Brown if (num_sockets > 2) 10200138d8f0SLen Brown cpuidle_state_table = ivt_cstates_4s; 1021d70e28f5SLen Brown 10220138d8f0SLen Brown /* else, 1 and 2 socket systems use default ivt_cstates */ 10230138d8f0SLen Brown } 1024d70e28f5SLen Brown /* 1025d70e28f5SLen Brown * sklh_idle_state_table_update(void) 1026d70e28f5SLen Brown * 1027d70e28f5SLen Brown * On SKL-H (model 0x5e) disable C8 and C9 if: 1028d70e28f5SLen Brown * C10 is enabled and SGX disabled 1029d70e28f5SLen Brown */ 1030d70e28f5SLen Brown static void sklh_idle_state_table_update(void) 1031d70e28f5SLen Brown { 1032d70e28f5SLen Brown unsigned long long msr; 1033d70e28f5SLen Brown unsigned int eax, ebx, ecx, edx; 1034d70e28f5SLen Brown 1035d70e28f5SLen Brown 1036d70e28f5SLen Brown /* if PC10 disabled via cmdline intel_idle.max_cstate=7 or shallower */ 1037d70e28f5SLen Brown if (max_cstate <= 7) 10380138d8f0SLen Brown return; 1039d70e28f5SLen Brown 1040d70e28f5SLen Brown /* if PC10 not present in CPUID.MWAIT.EDX */ 1041d70e28f5SLen Brown if ((mwait_substates & (0xF << 28)) == 0) 1042d70e28f5SLen Brown return; 1043d70e28f5SLen Brown 1044d70e28f5SLen Brown rdmsrl(MSR_NHM_SNB_PKG_CST_CFG_CTL, msr); 1045d70e28f5SLen Brown 1046d70e28f5SLen Brown /* PC10 is not enabled in PKG C-state limit */ 1047d70e28f5SLen Brown if ((msr & 0xF) != 8) 1048d70e28f5SLen Brown return; 1049d70e28f5SLen Brown 1050d70e28f5SLen Brown ecx = 0; 1051d70e28f5SLen Brown cpuid(7, &eax, &ebx, &ecx, &edx); 1052d70e28f5SLen Brown 1053d70e28f5SLen Brown /* if SGX is present */ 1054d70e28f5SLen Brown if (ebx & (1 << 2)) { 1055d70e28f5SLen Brown 1056d70e28f5SLen Brown rdmsrl(MSR_IA32_FEATURE_CONTROL, msr); 1057d70e28f5SLen Brown 1058d70e28f5SLen Brown /* if SGX is enabled */ 1059d70e28f5SLen Brown if (msr & (1 << 18)) 1060d70e28f5SLen Brown return; 1061d70e28f5SLen Brown } 1062d70e28f5SLen Brown 1063d70e28f5SLen Brown skl_cstates[5].disabled = 1; /* C8-SKL */ 1064d70e28f5SLen Brown skl_cstates[6].disabled = 1; /* C9-SKL */ 1065d70e28f5SLen Brown } 1066d70e28f5SLen Brown /* 1067d70e28f5SLen Brown * intel_idle_state_table_update() 1068d70e28f5SLen Brown * 1069d70e28f5SLen Brown * Update the default state_table for this CPU-id 1070d70e28f5SLen Brown */ 1071d70e28f5SLen Brown 1072d70e28f5SLen Brown static void intel_idle_state_table_update(void) 1073d70e28f5SLen Brown { 1074d70e28f5SLen Brown switch (boot_cpu_data.x86_model) { 1075d70e28f5SLen Brown 1076d70e28f5SLen Brown case 0x3e: /* IVT */ 1077d70e28f5SLen Brown ivt_idle_state_table_update(); 1078d70e28f5SLen Brown break; 1079d70e28f5SLen Brown case 0x5e: /* SKL-H */ 1080d70e28f5SLen Brown sklh_idle_state_table_update(); 1081d70e28f5SLen Brown break; 1082d70e28f5SLen Brown } 10830138d8f0SLen Brown } 10840138d8f0SLen Brown 108526717172SLen Brown /* 108646bcfad7SDeepthi Dharwar * intel_idle_cpuidle_driver_init() 108746bcfad7SDeepthi Dharwar * allocate, initialize cpuidle_states 108846bcfad7SDeepthi Dharwar */ 108900f3e755SBartlomiej Zolnierkiewicz static int __init intel_idle_cpuidle_driver_init(void) 109046bcfad7SDeepthi Dharwar { 109146bcfad7SDeepthi Dharwar int cstate; 109246bcfad7SDeepthi Dharwar struct cpuidle_driver *drv = &intel_idle_driver; 109346bcfad7SDeepthi Dharwar 10940138d8f0SLen Brown intel_idle_state_table_update(); 10950138d8f0SLen Brown 109646bcfad7SDeepthi Dharwar drv->state_count = 1; 109746bcfad7SDeepthi Dharwar 1098e022e7ebSLen Brown for (cstate = 0; cstate < CPUIDLE_STATE_MAX; ++cstate) { 109924bfa950SLen Brown int num_substates, mwait_hint, mwait_cstate; 110046bcfad7SDeepthi Dharwar 11017dd0e0afSLen Brown if ((cpuidle_state_table[cstate].enter == NULL) && 11027dd0e0afSLen Brown (cpuidle_state_table[cstate].enter_freeze == NULL)) 1103e022e7ebSLen Brown break; 1104e022e7ebSLen Brown 1105e022e7ebSLen Brown if (cstate + 1 > max_cstate) { 110646bcfad7SDeepthi Dharwar printk(PREFIX "max_cstate %d reached\n", 110746bcfad7SDeepthi Dharwar max_cstate); 110846bcfad7SDeepthi Dharwar break; 110946bcfad7SDeepthi Dharwar } 111046bcfad7SDeepthi Dharwar 1111e022e7ebSLen Brown mwait_hint = flg2MWAIT(cpuidle_state_table[cstate].flags); 1112e022e7ebSLen Brown mwait_cstate = MWAIT_HINT2CSTATE(mwait_hint); 111346bcfad7SDeepthi Dharwar 111424bfa950SLen Brown /* number of sub-states for this state in CPUID.MWAIT */ 1115e022e7ebSLen Brown num_substates = (mwait_substates >> ((mwait_cstate + 1) * 4)) 1116e022e7ebSLen Brown & MWAIT_SUBSTATE_MASK; 1117e022e7ebSLen Brown 111824bfa950SLen Brown /* if NO sub-states for this state in CPUID, skip it */ 111924bfa950SLen Brown if (num_substates == 0) 1120e022e7ebSLen Brown continue; 1121e022e7ebSLen Brown 1122d70e28f5SLen Brown /* if state marked as disabled, skip it */ 1123d70e28f5SLen Brown if (cpuidle_state_table[cstate].disabled != 0) { 1124d70e28f5SLen Brown pr_debug(PREFIX "state %s is disabled", 1125d70e28f5SLen Brown cpuidle_state_table[cstate].name); 1126d70e28f5SLen Brown continue; 1127d70e28f5SLen Brown } 1128d70e28f5SLen Brown 1129d70e28f5SLen Brown 1130e022e7ebSLen Brown if (((mwait_cstate + 1) > 2) && 113146bcfad7SDeepthi Dharwar !boot_cpu_has(X86_FEATURE_NONSTOP_TSC)) 113246bcfad7SDeepthi Dharwar mark_tsc_unstable("TSC halts in idle" 113346bcfad7SDeepthi Dharwar " states deeper than C2"); 113446bcfad7SDeepthi Dharwar 113546bcfad7SDeepthi Dharwar drv->states[drv->state_count] = /* structure copy */ 113646bcfad7SDeepthi Dharwar cpuidle_state_table[cstate]; 113746bcfad7SDeepthi Dharwar 113846bcfad7SDeepthi Dharwar drv->state_count += 1; 113946bcfad7SDeepthi Dharwar } 114046bcfad7SDeepthi Dharwar 1141b66b8b9aSAndi Kleen if (icpu->auto_demotion_disable_flags) 114239a74fdeSShaohua Li on_each_cpu(auto_demotion_disable, NULL, 1); 114346bcfad7SDeepthi Dharwar 11448c058d53SLen Brown if (icpu->byt_auto_demotion_disable_flag) { 11458c058d53SLen Brown wrmsrl(MSR_CC6_DEMOTION_POLICY_CONFIG, 0); 11468c058d53SLen Brown wrmsrl(MSR_MC6_DEMOTION_POLICY_CONFIG, 0); 11478c058d53SLen Brown } 11488c058d53SLen Brown 114932e95180SLen Brown if (icpu->disable_promotion_to_c1e) /* each-cpu is redundant */ 115032e95180SLen Brown on_each_cpu(c1e_promotion_disable, NULL, 1); 115132e95180SLen Brown 115246bcfad7SDeepthi Dharwar return 0; 115346bcfad7SDeepthi Dharwar } 115446bcfad7SDeepthi Dharwar 115546bcfad7SDeepthi Dharwar 115646bcfad7SDeepthi Dharwar /* 115765b7f839SThomas Renninger * intel_idle_cpu_init() 115826717172SLen Brown * allocate, initialize, register cpuidle_devices 115965b7f839SThomas Renninger * @cpu: cpu/core to initialize 116026717172SLen Brown */ 116125ac7761SDaniel Lezcano static int intel_idle_cpu_init(int cpu) 116226717172SLen Brown { 116326717172SLen Brown struct cpuidle_device *dev; 116426717172SLen Brown 116565b7f839SThomas Renninger dev = per_cpu_ptr(intel_idle_cpuidle_devices, cpu); 116626717172SLen Brown 116765b7f839SThomas Renninger dev->cpu = cpu; 116826717172SLen Brown 116926717172SLen Brown if (cpuidle_register_device(dev)) { 117065b7f839SThomas Renninger pr_debug(PREFIX "cpuidle_register_device %d failed!\n", cpu); 117126717172SLen Brown intel_idle_cpuidle_devices_uninit(); 117226717172SLen Brown return -EIO; 117326717172SLen Brown } 117465b7f839SThomas Renninger 1175b66b8b9aSAndi Kleen if (icpu->auto_demotion_disable_flags) 117665b7f839SThomas Renninger smp_call_function_single(cpu, auto_demotion_disable, NULL, 1); 117726717172SLen Brown 1178dbf87ab8SBartlomiej Zolnierkiewicz if (icpu->disable_promotion_to_c1e) 1179dbf87ab8SBartlomiej Zolnierkiewicz smp_call_function_single(cpu, c1e_promotion_disable, NULL, 1); 1180dbf87ab8SBartlomiej Zolnierkiewicz 118126717172SLen Brown return 0; 118226717172SLen Brown } 118326717172SLen Brown 118426717172SLen Brown static int __init intel_idle_init(void) 118526717172SLen Brown { 118665b7f839SThomas Renninger int retval, i; 118726717172SLen Brown 1188d1896049SThomas Renninger /* Do not load intel_idle at all for now if idle= is passed */ 1189d1896049SThomas Renninger if (boot_option_idle_override != IDLE_NO_OVERRIDE) 1190d1896049SThomas Renninger return -ENODEV; 1191d1896049SThomas Renninger 119226717172SLen Brown retval = intel_idle_probe(); 119326717172SLen Brown if (retval) 119426717172SLen Brown return retval; 119526717172SLen Brown 119646bcfad7SDeepthi Dharwar intel_idle_cpuidle_driver_init(); 119726717172SLen Brown retval = cpuidle_register_driver(&intel_idle_driver); 119826717172SLen Brown if (retval) { 11993735d524SKonrad Rzeszutek Wilk struct cpuidle_driver *drv = cpuidle_get_driver(); 120026717172SLen Brown printk(KERN_DEBUG PREFIX "intel_idle yielding to %s", 12013735d524SKonrad Rzeszutek Wilk drv ? drv->name : "none"); 120226717172SLen Brown return retval; 120326717172SLen Brown } 120426717172SLen Brown 120565b7f839SThomas Renninger intel_idle_cpuidle_devices = alloc_percpu(struct cpuidle_device); 120665b7f839SThomas Renninger if (intel_idle_cpuidle_devices == NULL) 120765b7f839SThomas Renninger return -ENOMEM; 120865b7f839SThomas Renninger 120907494d54SSrivatsa S. Bhat cpu_notifier_register_begin(); 121007494d54SSrivatsa S. Bhat 121165b7f839SThomas Renninger for_each_online_cpu(i) { 121265b7f839SThomas Renninger retval = intel_idle_cpu_init(i); 121326717172SLen Brown if (retval) { 121407494d54SSrivatsa S. Bhat cpu_notifier_register_done(); 121526717172SLen Brown cpuidle_unregister_driver(&intel_idle_driver); 121626717172SLen Brown return retval; 121726717172SLen Brown } 121865b7f839SThomas Renninger } 121907494d54SSrivatsa S. Bhat __register_cpu_notifier(&cpu_hotplug_notifier); 122007494d54SSrivatsa S. Bhat 122107494d54SSrivatsa S. Bhat cpu_notifier_register_done(); 122226717172SLen Brown 122326717172SLen Brown return 0; 122426717172SLen Brown } 122526717172SLen Brown 122626717172SLen Brown static void __exit intel_idle_exit(void) 122726717172SLen Brown { 122826717172SLen Brown intel_idle_cpuidle_devices_uninit(); 122926717172SLen Brown cpuidle_unregister_driver(&intel_idle_driver); 123026717172SLen Brown 123107494d54SSrivatsa S. Bhat cpu_notifier_register_begin(); 123225ac7761SDaniel Lezcano 123325ac7761SDaniel Lezcano if (lapic_timer_reliable_states != LAPIC_TIMER_ALWAYS_RELIABLE) 123439a74fdeSShaohua Li on_each_cpu(__setup_broadcast_timer, (void *)false, 1); 123507494d54SSrivatsa S. Bhat __unregister_cpu_notifier(&cpu_hotplug_notifier); 123607494d54SSrivatsa S. Bhat 123707494d54SSrivatsa S. Bhat cpu_notifier_register_done(); 12382a2d31c8SShaohua Li 123926717172SLen Brown return; 124026717172SLen Brown } 124126717172SLen Brown 124226717172SLen Brown module_init(intel_idle_init); 124326717172SLen Brown module_exit(intel_idle_exit); 124426717172SLen Brown 124526717172SLen Brown module_param(max_cstate, int, 0444); 124626717172SLen Brown 124726717172SLen Brown MODULE_AUTHOR("Len Brown <len.brown@intel.com>"); 124826717172SLen Brown MODULE_DESCRIPTION("Cpuidle driver for Intel Hardware v" INTEL_IDLE_VERSION); 124926717172SLen Brown MODULE_LICENSE("GPL"); 1250