1a61127c2SThomas Gleixner // SPDX-License-Identifier: GPL-2.0-only 226717172SLen Brown /* 326717172SLen Brown * intel_idle.c - native hardware idle loop for modern Intel processors 426717172SLen Brown * 5fab04b22SLen Brown * Copyright (c) 2013, Intel Corporation. 626717172SLen Brown * Len Brown <len.brown@intel.com> 726717172SLen Brown */ 826717172SLen Brown 926717172SLen Brown /* 1026717172SLen Brown * intel_idle is a cpuidle driver that loads on specific Intel processors 1126717172SLen Brown * in lieu of the legacy ACPI processor_idle driver. The intent is to 1226717172SLen Brown * make Linux more efficient on these processors, as intel_idle knows 1326717172SLen Brown * more than ACPI, as well as make Linux more immune to ACPI BIOS bugs. 1426717172SLen Brown */ 1526717172SLen Brown 1626717172SLen Brown /* 1726717172SLen Brown * Design Assumptions 1826717172SLen Brown * 1926717172SLen Brown * All CPUs have same idle states as boot CPU 2026717172SLen Brown * 2126717172SLen Brown * Chipset BM_STS (bus master status) bit is a NOP 2226717172SLen Brown * for preventing entry into deep C-stats 2326717172SLen Brown */ 2426717172SLen Brown 2526717172SLen Brown /* 2626717172SLen Brown * Known limitations 2726717172SLen Brown * 2826717172SLen Brown * The driver currently initializes for_each_online_cpu() upon modprobe. 2926717172SLen Brown * It it unaware of subsequent processors hot-added to the system. 3026717172SLen Brown * This means that if you boot with maxcpus=n and later online 3126717172SLen Brown * processors above n, those processors will use C1 only. 3226717172SLen Brown * 3326717172SLen Brown * ACPI has a .suspend hack to turn off deep c-statees during suspend 3426717172SLen Brown * to avoid complications with the lapic timer workaround. 3526717172SLen Brown * Have not seen issues with suspend, but may need same workaround here. 3626717172SLen Brown * 3726717172SLen Brown */ 3826717172SLen Brown 3926717172SLen Brown /* un-comment DEBUG to enable pr_debug() statements */ 4026717172SLen Brown #define DEBUG 4126717172SLen Brown 42654d08a4SJoe Perches #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt 43654d08a4SJoe Perches 4418734958SRafael J. Wysocki #include <linux/acpi.h> 4526717172SLen Brown #include <linux/kernel.h> 4626717172SLen Brown #include <linux/cpuidle.h> 4776962caaSThomas Gleixner #include <linux/tick.h> 4826717172SLen Brown #include <trace/events/power.h> 4926717172SLen Brown #include <linux/sched.h> 502a2d31c8SShaohua Li #include <linux/notifier.h> 512a2d31c8SShaohua Li #include <linux/cpu.h> 5202c4fae9SPaul Gortmaker #include <linux/moduleparam.h> 53b66b8b9aSAndi Kleen #include <asm/cpu_device_id.h> 54db73c5a8SDave Hansen #include <asm/intel-family.h> 55bc83ccccSH. Peter Anvin #include <asm/mwait.h> 5614796fcaSLen Brown #include <asm/msr.h> 5726717172SLen Brown 58d70e28f5SLen Brown #define INTEL_IDLE_VERSION "0.4.1" 5926717172SLen Brown 6026717172SLen Brown static struct cpuidle_driver intel_idle_driver = { 6126717172SLen Brown .name = "intel_idle", 6226717172SLen Brown .owner = THIS_MODULE, 6326717172SLen Brown }; 6426717172SLen Brown /* intel_idle.max_cstate=0 disables driver */ 65137ecc77SLen Brown static int max_cstate = CPUIDLE_STATE_MAX - 1; 664dcb78eeSRafael J. Wysocki static unsigned int disabled_states_mask; 6726717172SLen Brown 686eb0443aSRafael J. Wysocki static struct cpuidle_device __percpu *intel_idle_cpuidle_devices; 697f843dd7SRafael J. Wysocki 707f843dd7SRafael J. Wysocki static unsigned long auto_demotion_disable_flags; 717f843dd7SRafael J. Wysocki static bool disable_promotion_to_c1e; 727f843dd7SRafael J. Wysocki 7340ab82e0SRafael J. Wysocki static bool lapic_timer_always_reliable; 7426717172SLen Brown 75b66b8b9aSAndi Kleen struct idle_cpu { 76b66b8b9aSAndi Kleen struct cpuidle_state *state_table; 7726717172SLen Brown 7826717172SLen Brown /* 7914796fcaSLen Brown * Hardware C-state auto-demotion may not always be optimal. 8014796fcaSLen Brown * Indicate which enable bits to clear here. 8114796fcaSLen Brown */ 82b66b8b9aSAndi Kleen unsigned long auto_demotion_disable_flags; 838c058d53SLen Brown bool byt_auto_demotion_disable_flag; 8432e95180SLen Brown bool disable_promotion_to_c1e; 85bff8e60aSRafael J. Wysocki bool use_acpi; 86b66b8b9aSAndi Kleen }; 87b66b8b9aSAndi Kleen 887f843dd7SRafael J. Wysocki static const struct idle_cpu *icpu __initdata; 897f843dd7SRafael J. Wysocki static struct cpuidle_state *cpuidle_state_table __initdata; 9014796fcaSLen Brown 916eb0443aSRafael J. Wysocki static unsigned int mwait_substates __initdata; 926eb0443aSRafael J. Wysocki 9314796fcaSLen Brown /* 94bff8e60aSRafael J. Wysocki * Enable this state by default even if the ACPI _CST does not list it. 95bff8e60aSRafael J. Wysocki */ 96bff8e60aSRafael J. Wysocki #define CPUIDLE_FLAG_ALWAYS_ENABLE BIT(15) 97bff8e60aSRafael J. Wysocki 98bff8e60aSRafael J. Wysocki /* 99956d033fSLen Brown * Set this flag for states where the HW flushes the TLB for us 100956d033fSLen Brown * and so we don't need cross-calls to keep it consistent. 101956d033fSLen Brown * If this flag is set, SW flushes the TLB, so even if the 102956d033fSLen Brown * HW doesn't do the flushing, this flag is safe to use. 103956d033fSLen Brown */ 104956d033fSLen Brown #define CPUIDLE_FLAG_TLB_FLUSHED 0x10000 105956d033fSLen Brown 106956d033fSLen Brown /* 107b1beab48SLen Brown * MWAIT takes an 8-bit "hint" in EAX "suggesting" 108b1beab48SLen Brown * the C-state (top nibble) and sub-state (bottom nibble) 109b1beab48SLen Brown * 0x00 means "MWAIT(C1)", 0x10 means "MWAIT(C2)" etc. 110b1beab48SLen Brown * 111b1beab48SLen Brown * We store the hint at the top of our "flags" for each state. 112b1beab48SLen Brown */ 113b1beab48SLen Brown #define flg2MWAIT(flags) (((flags) >> 24) & 0xFF) 114b1beab48SLen Brown #define MWAIT2flg(eax) ((eax & 0xFF) << 24) 115b1beab48SLen Brown 11630a996fbSRafael J. Wysocki /** 11730a996fbSRafael J. Wysocki * intel_idle - Ask the processor to enter the given idle state. 11830a996fbSRafael J. Wysocki * @dev: cpuidle device of the target CPU. 11930a996fbSRafael J. Wysocki * @drv: cpuidle driver (assumed to point to intel_idle_driver). 12030a996fbSRafael J. Wysocki * @index: Target idle state index. 12130a996fbSRafael J. Wysocki * 12230a996fbSRafael J. Wysocki * Use the MWAIT instruction to notify the processor that the CPU represented by 12330a996fbSRafael J. Wysocki * @dev is idle and it can try to enter the idle state corresponding to @index. 12430a996fbSRafael J. Wysocki * 12530a996fbSRafael J. Wysocki * If the local APIC timer is not known to be reliable in the target idle state, 12630a996fbSRafael J. Wysocki * enable one-shot tick broadcasting for the target CPU before executing MWAIT. 12730a996fbSRafael J. Wysocki * 12830a996fbSRafael J. Wysocki * Optionally call leave_mm() for the target CPU upfront to avoid wakeups due to 12930a996fbSRafael J. Wysocki * flushing user TLBs. 13030a996fbSRafael J. Wysocki * 13130a996fbSRafael J. Wysocki * Must be called under local_irq_disable(). 13230a996fbSRafael J. Wysocki */ 13330a996fbSRafael J. Wysocki static __cpuidle int intel_idle(struct cpuidle_device *dev, 13430a996fbSRafael J. Wysocki struct cpuidle_driver *drv, int index) 13530a996fbSRafael J. Wysocki { 13630a996fbSRafael J. Wysocki struct cpuidle_state *state = &drv->states[index]; 13730a996fbSRafael J. Wysocki unsigned long eax = flg2MWAIT(state->flags); 13830a996fbSRafael J. Wysocki unsigned long ecx = 1; /* break on interrupt flag */ 13930a996fbSRafael J. Wysocki bool uninitialized_var(tick); 14030a996fbSRafael J. Wysocki int cpu = smp_processor_id(); 14130a996fbSRafael J. Wysocki 14230a996fbSRafael J. Wysocki /* 14330a996fbSRafael J. Wysocki * leave_mm() to avoid costly and often unnecessary wakeups 14430a996fbSRafael J. Wysocki * for flushing the user TLB's associated with the active mm. 14530a996fbSRafael J. Wysocki */ 14630a996fbSRafael J. Wysocki if (state->flags & CPUIDLE_FLAG_TLB_FLUSHED) 14730a996fbSRafael J. Wysocki leave_mm(cpu); 14830a996fbSRafael J. Wysocki 14930a996fbSRafael J. Wysocki if (!static_cpu_has(X86_FEATURE_ARAT) && !lapic_timer_always_reliable) { 15030a996fbSRafael J. Wysocki /* 15130a996fbSRafael J. Wysocki * Switch over to one-shot tick broadcast if the target C-state 15230a996fbSRafael J. Wysocki * is deeper than C1. 15330a996fbSRafael J. Wysocki */ 15430a996fbSRafael J. Wysocki if ((eax >> MWAIT_SUBSTATE_SIZE) & MWAIT_CSTATE_MASK) { 15530a996fbSRafael J. Wysocki tick = true; 15630a996fbSRafael J. Wysocki tick_broadcast_enter(); 15730a996fbSRafael J. Wysocki } else { 15830a996fbSRafael J. Wysocki tick = false; 15930a996fbSRafael J. Wysocki } 16030a996fbSRafael J. Wysocki } 16130a996fbSRafael J. Wysocki 16230a996fbSRafael J. Wysocki mwait_idle_with_hints(eax, ecx); 16330a996fbSRafael J. Wysocki 16430a996fbSRafael J. Wysocki if (!static_cpu_has(X86_FEATURE_ARAT) && tick) 16530a996fbSRafael J. Wysocki tick_broadcast_exit(); 16630a996fbSRafael J. Wysocki 16730a996fbSRafael J. Wysocki return index; 16830a996fbSRafael J. Wysocki } 16930a996fbSRafael J. Wysocki 17030a996fbSRafael J. Wysocki /** 17130a996fbSRafael J. Wysocki * intel_idle_s2idle - Ask the processor to enter the given idle state. 17230a996fbSRafael J. Wysocki * @dev: cpuidle device of the target CPU. 17330a996fbSRafael J. Wysocki * @drv: cpuidle driver (assumed to point to intel_idle_driver). 17430a996fbSRafael J. Wysocki * @index: Target idle state index. 17530a996fbSRafael J. Wysocki * 17630a996fbSRafael J. Wysocki * Use the MWAIT instruction to notify the processor that the CPU represented by 17730a996fbSRafael J. Wysocki * @dev is idle and it can try to enter the idle state corresponding to @index. 17830a996fbSRafael J. Wysocki * 17930a996fbSRafael J. Wysocki * Invoked as a suspend-to-idle callback routine with frozen user space, frozen 18030a996fbSRafael J. Wysocki * scheduler tick and suspended scheduler clock on the target CPU. 18130a996fbSRafael J. Wysocki */ 18230a996fbSRafael J. Wysocki static __cpuidle void intel_idle_s2idle(struct cpuidle_device *dev, 18330a996fbSRafael J. Wysocki struct cpuidle_driver *drv, int index) 18430a996fbSRafael J. Wysocki { 18530a996fbSRafael J. Wysocki unsigned long eax = flg2MWAIT(drv->states[index].flags); 18630a996fbSRafael J. Wysocki unsigned long ecx = 1; /* break on interrupt flag */ 18730a996fbSRafael J. Wysocki 18830a996fbSRafael J. Wysocki mwait_idle_with_hints(eax, ecx); 18930a996fbSRafael J. Wysocki } 19030a996fbSRafael J. Wysocki 191b1beab48SLen Brown /* 19226717172SLen Brown * States are indexed by the cstate number, 19326717172SLen Brown * which is also the index into the MWAIT hint array. 19426717172SLen Brown * Thus C0 is a dummy. 19526717172SLen Brown */ 196ab1a8522SRafael J. Wysocki static struct cpuidle_state nehalem_cstates[] __initdata = { 197e022e7ebSLen Brown { 198de09cdd0SLen Brown .name = "C1", 19926717172SLen Brown .desc = "MWAIT 0x00", 200b82b6ccaSDaniel Lezcano .flags = MWAIT2flg(0x00), 20126717172SLen Brown .exit_latency = 3, 20226717172SLen Brown .target_residency = 6, 2035fe2e527SRafael J. Wysocki .enter = &intel_idle, 20428ba086eSRafael J. Wysocki .enter_s2idle = intel_idle_s2idle, }, 205e022e7ebSLen Brown { 206de09cdd0SLen Brown .name = "C1E", 20732e95180SLen Brown .desc = "MWAIT 0x01", 208e6d4f08aSRafael J. Wysocki .flags = MWAIT2flg(0x01) | CPUIDLE_FLAG_ALWAYS_ENABLE, 20932e95180SLen Brown .exit_latency = 10, 21032e95180SLen Brown .target_residency = 20, 2115fe2e527SRafael J. Wysocki .enter = &intel_idle, 21228ba086eSRafael J. Wysocki .enter_s2idle = intel_idle_s2idle, }, 21332e95180SLen Brown { 214de09cdd0SLen Brown .name = "C3", 21526717172SLen Brown .desc = "MWAIT 0x10", 216b82b6ccaSDaniel Lezcano .flags = MWAIT2flg(0x10) | CPUIDLE_FLAG_TLB_FLUSHED, 21726717172SLen Brown .exit_latency = 20, 21826717172SLen Brown .target_residency = 80, 2195fe2e527SRafael J. Wysocki .enter = &intel_idle, 22028ba086eSRafael J. Wysocki .enter_s2idle = intel_idle_s2idle, }, 221e022e7ebSLen Brown { 222de09cdd0SLen Brown .name = "C6", 22326717172SLen Brown .desc = "MWAIT 0x20", 224b82b6ccaSDaniel Lezcano .flags = MWAIT2flg(0x20) | CPUIDLE_FLAG_TLB_FLUSHED, 22526717172SLen Brown .exit_latency = 200, 22626717172SLen Brown .target_residency = 800, 2275fe2e527SRafael J. Wysocki .enter = &intel_idle, 22828ba086eSRafael J. Wysocki .enter_s2idle = intel_idle_s2idle, }, 229e022e7ebSLen Brown { 230e022e7ebSLen Brown .enter = NULL } 23126717172SLen Brown }; 23226717172SLen Brown 233ab1a8522SRafael J. Wysocki static struct cpuidle_state snb_cstates[] __initdata = { 234e022e7ebSLen Brown { 235de09cdd0SLen Brown .name = "C1", 236d13780d4SLen Brown .desc = "MWAIT 0x00", 237b82b6ccaSDaniel Lezcano .flags = MWAIT2flg(0x00), 23832e95180SLen Brown .exit_latency = 2, 23932e95180SLen Brown .target_residency = 2, 2405fe2e527SRafael J. Wysocki .enter = &intel_idle, 24128ba086eSRafael J. Wysocki .enter_s2idle = intel_idle_s2idle, }, 24232e95180SLen Brown { 243de09cdd0SLen Brown .name = "C1E", 24432e95180SLen Brown .desc = "MWAIT 0x01", 245e6d4f08aSRafael J. Wysocki .flags = MWAIT2flg(0x01) | CPUIDLE_FLAG_ALWAYS_ENABLE, 24632e95180SLen Brown .exit_latency = 10, 24732e95180SLen Brown .target_residency = 20, 2485fe2e527SRafael J. Wysocki .enter = &intel_idle, 24928ba086eSRafael J. Wysocki .enter_s2idle = intel_idle_s2idle, }, 250e022e7ebSLen Brown { 251de09cdd0SLen Brown .name = "C3", 252d13780d4SLen Brown .desc = "MWAIT 0x10", 253b82b6ccaSDaniel Lezcano .flags = MWAIT2flg(0x10) | CPUIDLE_FLAG_TLB_FLUSHED, 254d13780d4SLen Brown .exit_latency = 80, 255ddbd550dSLen Brown .target_residency = 211, 2565fe2e527SRafael J. Wysocki .enter = &intel_idle, 25728ba086eSRafael J. Wysocki .enter_s2idle = intel_idle_s2idle, }, 258e022e7ebSLen Brown { 259de09cdd0SLen Brown .name = "C6", 260d13780d4SLen Brown .desc = "MWAIT 0x20", 261b82b6ccaSDaniel Lezcano .flags = MWAIT2flg(0x20) | CPUIDLE_FLAG_TLB_FLUSHED, 262d13780d4SLen Brown .exit_latency = 104, 263ddbd550dSLen Brown .target_residency = 345, 2645fe2e527SRafael J. Wysocki .enter = &intel_idle, 26528ba086eSRafael J. Wysocki .enter_s2idle = intel_idle_s2idle, }, 266e022e7ebSLen Brown { 267de09cdd0SLen Brown .name = "C7", 268d13780d4SLen Brown .desc = "MWAIT 0x30", 269b82b6ccaSDaniel Lezcano .flags = MWAIT2flg(0x30) | CPUIDLE_FLAG_TLB_FLUSHED, 270d13780d4SLen Brown .exit_latency = 109, 271ddbd550dSLen Brown .target_residency = 345, 2725fe2e527SRafael J. Wysocki .enter = &intel_idle, 27328ba086eSRafael J. Wysocki .enter_s2idle = intel_idle_s2idle, }, 274e022e7ebSLen Brown { 275e022e7ebSLen Brown .enter = NULL } 276d13780d4SLen Brown }; 277d13780d4SLen Brown 278ab1a8522SRafael J. Wysocki static struct cpuidle_state byt_cstates[] __initdata = { 279718987d6SLen Brown { 280de09cdd0SLen Brown .name = "C1", 281718987d6SLen Brown .desc = "MWAIT 0x00", 282b82b6ccaSDaniel Lezcano .flags = MWAIT2flg(0x00), 283718987d6SLen Brown .exit_latency = 1, 284718987d6SLen Brown .target_residency = 1, 2855fe2e527SRafael J. Wysocki .enter = &intel_idle, 28628ba086eSRafael J. Wysocki .enter_s2idle = intel_idle_s2idle, }, 287718987d6SLen Brown { 288de09cdd0SLen Brown .name = "C6N", 289718987d6SLen Brown .desc = "MWAIT 0x58", 290b82b6ccaSDaniel Lezcano .flags = MWAIT2flg(0x58) | CPUIDLE_FLAG_TLB_FLUSHED, 291d7ef7671SLen Brown .exit_latency = 300, 292718987d6SLen Brown .target_residency = 275, 2935fe2e527SRafael J. Wysocki .enter = &intel_idle, 29428ba086eSRafael J. Wysocki .enter_s2idle = intel_idle_s2idle, }, 295718987d6SLen Brown { 296de09cdd0SLen Brown .name = "C6S", 297718987d6SLen Brown .desc = "MWAIT 0x52", 298b82b6ccaSDaniel Lezcano .flags = MWAIT2flg(0x52) | CPUIDLE_FLAG_TLB_FLUSHED, 299d7ef7671SLen Brown .exit_latency = 500, 300718987d6SLen Brown .target_residency = 560, 3015fe2e527SRafael J. Wysocki .enter = &intel_idle, 30228ba086eSRafael J. Wysocki .enter_s2idle = intel_idle_s2idle, }, 303718987d6SLen Brown { 304de09cdd0SLen Brown .name = "C7", 305718987d6SLen Brown .desc = "MWAIT 0x60", 306b82b6ccaSDaniel Lezcano .flags = MWAIT2flg(0x60) | CPUIDLE_FLAG_TLB_FLUSHED, 307718987d6SLen Brown .exit_latency = 1200, 308d7ef7671SLen Brown .target_residency = 4000, 3095fe2e527SRafael J. Wysocki .enter = &intel_idle, 31028ba086eSRafael J. Wysocki .enter_s2idle = intel_idle_s2idle, }, 311718987d6SLen Brown { 312de09cdd0SLen Brown .name = "C7S", 313718987d6SLen Brown .desc = "MWAIT 0x64", 314b82b6ccaSDaniel Lezcano .flags = MWAIT2flg(0x64) | CPUIDLE_FLAG_TLB_FLUSHED, 315718987d6SLen Brown .exit_latency = 10000, 316718987d6SLen Brown .target_residency = 20000, 3175fe2e527SRafael J. Wysocki .enter = &intel_idle, 31828ba086eSRafael J. Wysocki .enter_s2idle = intel_idle_s2idle, }, 319718987d6SLen Brown { 320718987d6SLen Brown .enter = NULL } 321718987d6SLen Brown }; 322718987d6SLen Brown 323ab1a8522SRafael J. Wysocki static struct cpuidle_state cht_cstates[] __initdata = { 324cab07a56SLen Brown { 325de09cdd0SLen Brown .name = "C1", 326cab07a56SLen Brown .desc = "MWAIT 0x00", 327cab07a56SLen Brown .flags = MWAIT2flg(0x00), 328cab07a56SLen Brown .exit_latency = 1, 329cab07a56SLen Brown .target_residency = 1, 330cab07a56SLen Brown .enter = &intel_idle, 33128ba086eSRafael J. Wysocki .enter_s2idle = intel_idle_s2idle, }, 332cab07a56SLen Brown { 333de09cdd0SLen Brown .name = "C6N", 334cab07a56SLen Brown .desc = "MWAIT 0x58", 335cab07a56SLen Brown .flags = MWAIT2flg(0x58) | CPUIDLE_FLAG_TLB_FLUSHED, 336cab07a56SLen Brown .exit_latency = 80, 337cab07a56SLen Brown .target_residency = 275, 338cab07a56SLen Brown .enter = &intel_idle, 33928ba086eSRafael J. Wysocki .enter_s2idle = intel_idle_s2idle, }, 340cab07a56SLen Brown { 341de09cdd0SLen Brown .name = "C6S", 342cab07a56SLen Brown .desc = "MWAIT 0x52", 343cab07a56SLen Brown .flags = MWAIT2flg(0x52) | CPUIDLE_FLAG_TLB_FLUSHED, 344cab07a56SLen Brown .exit_latency = 200, 345cab07a56SLen Brown .target_residency = 560, 346cab07a56SLen Brown .enter = &intel_idle, 34728ba086eSRafael J. Wysocki .enter_s2idle = intel_idle_s2idle, }, 348cab07a56SLen Brown { 349de09cdd0SLen Brown .name = "C7", 350cab07a56SLen Brown .desc = "MWAIT 0x60", 351cab07a56SLen Brown .flags = MWAIT2flg(0x60) | CPUIDLE_FLAG_TLB_FLUSHED, 352cab07a56SLen Brown .exit_latency = 1200, 353cab07a56SLen Brown .target_residency = 4000, 354cab07a56SLen Brown .enter = &intel_idle, 35528ba086eSRafael J. Wysocki .enter_s2idle = intel_idle_s2idle, }, 356cab07a56SLen Brown { 357de09cdd0SLen Brown .name = "C7S", 358cab07a56SLen Brown .desc = "MWAIT 0x64", 359cab07a56SLen Brown .flags = MWAIT2flg(0x64) | CPUIDLE_FLAG_TLB_FLUSHED, 360cab07a56SLen Brown .exit_latency = 10000, 361cab07a56SLen Brown .target_residency = 20000, 362cab07a56SLen Brown .enter = &intel_idle, 36328ba086eSRafael J. Wysocki .enter_s2idle = intel_idle_s2idle, }, 364cab07a56SLen Brown { 365cab07a56SLen Brown .enter = NULL } 366cab07a56SLen Brown }; 367cab07a56SLen Brown 368ab1a8522SRafael J. Wysocki static struct cpuidle_state ivb_cstates[] __initdata = { 369e022e7ebSLen Brown { 370de09cdd0SLen Brown .name = "C1", 3716edab08cSLen Brown .desc = "MWAIT 0x00", 372b82b6ccaSDaniel Lezcano .flags = MWAIT2flg(0x00), 3736edab08cSLen Brown .exit_latency = 1, 3746edab08cSLen Brown .target_residency = 1, 3755fe2e527SRafael J. Wysocki .enter = &intel_idle, 37628ba086eSRafael J. Wysocki .enter_s2idle = intel_idle_s2idle, }, 377e022e7ebSLen Brown { 378de09cdd0SLen Brown .name = "C1E", 37932e95180SLen Brown .desc = "MWAIT 0x01", 380e6d4f08aSRafael J. Wysocki .flags = MWAIT2flg(0x01) | CPUIDLE_FLAG_ALWAYS_ENABLE, 38132e95180SLen Brown .exit_latency = 10, 38232e95180SLen Brown .target_residency = 20, 3835fe2e527SRafael J. Wysocki .enter = &intel_idle, 38428ba086eSRafael J. Wysocki .enter_s2idle = intel_idle_s2idle, }, 38532e95180SLen Brown { 386de09cdd0SLen Brown .name = "C3", 3876edab08cSLen Brown .desc = "MWAIT 0x10", 388b82b6ccaSDaniel Lezcano .flags = MWAIT2flg(0x10) | CPUIDLE_FLAG_TLB_FLUSHED, 3896edab08cSLen Brown .exit_latency = 59, 3906edab08cSLen Brown .target_residency = 156, 3915fe2e527SRafael J. Wysocki .enter = &intel_idle, 39228ba086eSRafael J. Wysocki .enter_s2idle = intel_idle_s2idle, }, 393e022e7ebSLen Brown { 394de09cdd0SLen Brown .name = "C6", 3956edab08cSLen Brown .desc = "MWAIT 0x20", 396b82b6ccaSDaniel Lezcano .flags = MWAIT2flg(0x20) | CPUIDLE_FLAG_TLB_FLUSHED, 3976edab08cSLen Brown .exit_latency = 80, 3986edab08cSLen Brown .target_residency = 300, 3995fe2e527SRafael J. Wysocki .enter = &intel_idle, 40028ba086eSRafael J. Wysocki .enter_s2idle = intel_idle_s2idle, }, 401e022e7ebSLen Brown { 402de09cdd0SLen Brown .name = "C7", 4036edab08cSLen Brown .desc = "MWAIT 0x30", 404b82b6ccaSDaniel Lezcano .flags = MWAIT2flg(0x30) | CPUIDLE_FLAG_TLB_FLUSHED, 4056edab08cSLen Brown .exit_latency = 87, 4066edab08cSLen Brown .target_residency = 300, 4075fe2e527SRafael J. Wysocki .enter = &intel_idle, 40828ba086eSRafael J. Wysocki .enter_s2idle = intel_idle_s2idle, }, 409e022e7ebSLen Brown { 410e022e7ebSLen Brown .enter = NULL } 4116edab08cSLen Brown }; 4126edab08cSLen Brown 413ab1a8522SRafael J. Wysocki static struct cpuidle_state ivt_cstates[] __initdata = { 4140138d8f0SLen Brown { 415de09cdd0SLen Brown .name = "C1", 4160138d8f0SLen Brown .desc = "MWAIT 0x00", 417b82b6ccaSDaniel Lezcano .flags = MWAIT2flg(0x00), 4180138d8f0SLen Brown .exit_latency = 1, 4190138d8f0SLen Brown .target_residency = 1, 4205fe2e527SRafael J. Wysocki .enter = &intel_idle, 42128ba086eSRafael J. Wysocki .enter_s2idle = intel_idle_s2idle, }, 4220138d8f0SLen Brown { 423de09cdd0SLen Brown .name = "C1E", 4240138d8f0SLen Brown .desc = "MWAIT 0x01", 425e6d4f08aSRafael J. Wysocki .flags = MWAIT2flg(0x01) | CPUIDLE_FLAG_ALWAYS_ENABLE, 4260138d8f0SLen Brown .exit_latency = 10, 4270138d8f0SLen Brown .target_residency = 80, 4285fe2e527SRafael J. Wysocki .enter = &intel_idle, 42928ba086eSRafael J. Wysocki .enter_s2idle = intel_idle_s2idle, }, 4300138d8f0SLen Brown { 431de09cdd0SLen Brown .name = "C3", 4320138d8f0SLen Brown .desc = "MWAIT 0x10", 433b82b6ccaSDaniel Lezcano .flags = MWAIT2flg(0x10) | CPUIDLE_FLAG_TLB_FLUSHED, 4340138d8f0SLen Brown .exit_latency = 59, 4350138d8f0SLen Brown .target_residency = 156, 4365fe2e527SRafael J. Wysocki .enter = &intel_idle, 43728ba086eSRafael J. Wysocki .enter_s2idle = intel_idle_s2idle, }, 4380138d8f0SLen Brown { 439de09cdd0SLen Brown .name = "C6", 4400138d8f0SLen Brown .desc = "MWAIT 0x20", 441b82b6ccaSDaniel Lezcano .flags = MWAIT2flg(0x20) | CPUIDLE_FLAG_TLB_FLUSHED, 4420138d8f0SLen Brown .exit_latency = 82, 4430138d8f0SLen Brown .target_residency = 300, 4445fe2e527SRafael J. Wysocki .enter = &intel_idle, 44528ba086eSRafael J. Wysocki .enter_s2idle = intel_idle_s2idle, }, 4460138d8f0SLen Brown { 4470138d8f0SLen Brown .enter = NULL } 4480138d8f0SLen Brown }; 4490138d8f0SLen Brown 450ab1a8522SRafael J. Wysocki static struct cpuidle_state ivt_cstates_4s[] __initdata = { 4510138d8f0SLen Brown { 452de09cdd0SLen Brown .name = "C1", 4530138d8f0SLen Brown .desc = "MWAIT 0x00", 454b82b6ccaSDaniel Lezcano .flags = MWAIT2flg(0x00), 4550138d8f0SLen Brown .exit_latency = 1, 4560138d8f0SLen Brown .target_residency = 1, 4575fe2e527SRafael J. Wysocki .enter = &intel_idle, 45828ba086eSRafael J. Wysocki .enter_s2idle = intel_idle_s2idle, }, 4590138d8f0SLen Brown { 460de09cdd0SLen Brown .name = "C1E", 4610138d8f0SLen Brown .desc = "MWAIT 0x01", 462e6d4f08aSRafael J. Wysocki .flags = MWAIT2flg(0x01) | CPUIDLE_FLAG_ALWAYS_ENABLE, 4630138d8f0SLen Brown .exit_latency = 10, 4640138d8f0SLen Brown .target_residency = 250, 4655fe2e527SRafael J. Wysocki .enter = &intel_idle, 46628ba086eSRafael J. Wysocki .enter_s2idle = intel_idle_s2idle, }, 4670138d8f0SLen Brown { 468de09cdd0SLen Brown .name = "C3", 4690138d8f0SLen Brown .desc = "MWAIT 0x10", 470b82b6ccaSDaniel Lezcano .flags = MWAIT2flg(0x10) | CPUIDLE_FLAG_TLB_FLUSHED, 4710138d8f0SLen Brown .exit_latency = 59, 4720138d8f0SLen Brown .target_residency = 300, 4735fe2e527SRafael J. Wysocki .enter = &intel_idle, 47428ba086eSRafael J. Wysocki .enter_s2idle = intel_idle_s2idle, }, 4750138d8f0SLen Brown { 476de09cdd0SLen Brown .name = "C6", 4770138d8f0SLen Brown .desc = "MWAIT 0x20", 478b82b6ccaSDaniel Lezcano .flags = MWAIT2flg(0x20) | CPUIDLE_FLAG_TLB_FLUSHED, 4790138d8f0SLen Brown .exit_latency = 84, 4800138d8f0SLen Brown .target_residency = 400, 4815fe2e527SRafael J. Wysocki .enter = &intel_idle, 48228ba086eSRafael J. Wysocki .enter_s2idle = intel_idle_s2idle, }, 4830138d8f0SLen Brown { 4840138d8f0SLen Brown .enter = NULL } 4850138d8f0SLen Brown }; 4860138d8f0SLen Brown 487ab1a8522SRafael J. Wysocki static struct cpuidle_state ivt_cstates_8s[] __initdata = { 4880138d8f0SLen Brown { 489de09cdd0SLen Brown .name = "C1", 4900138d8f0SLen Brown .desc = "MWAIT 0x00", 491b82b6ccaSDaniel Lezcano .flags = MWAIT2flg(0x00), 4920138d8f0SLen Brown .exit_latency = 1, 4930138d8f0SLen Brown .target_residency = 1, 4945fe2e527SRafael J. Wysocki .enter = &intel_idle, 49528ba086eSRafael J. Wysocki .enter_s2idle = intel_idle_s2idle, }, 4960138d8f0SLen Brown { 497de09cdd0SLen Brown .name = "C1E", 4980138d8f0SLen Brown .desc = "MWAIT 0x01", 499e6d4f08aSRafael J. Wysocki .flags = MWAIT2flg(0x01) | CPUIDLE_FLAG_ALWAYS_ENABLE, 5000138d8f0SLen Brown .exit_latency = 10, 5010138d8f0SLen Brown .target_residency = 500, 5025fe2e527SRafael J. Wysocki .enter = &intel_idle, 50328ba086eSRafael J. Wysocki .enter_s2idle = intel_idle_s2idle, }, 5040138d8f0SLen Brown { 505de09cdd0SLen Brown .name = "C3", 5060138d8f0SLen Brown .desc = "MWAIT 0x10", 507b82b6ccaSDaniel Lezcano .flags = MWAIT2flg(0x10) | CPUIDLE_FLAG_TLB_FLUSHED, 5080138d8f0SLen Brown .exit_latency = 59, 5090138d8f0SLen Brown .target_residency = 600, 5105fe2e527SRafael J. Wysocki .enter = &intel_idle, 51128ba086eSRafael J. Wysocki .enter_s2idle = intel_idle_s2idle, }, 5120138d8f0SLen Brown { 513de09cdd0SLen Brown .name = "C6", 5140138d8f0SLen Brown .desc = "MWAIT 0x20", 515b82b6ccaSDaniel Lezcano .flags = MWAIT2flg(0x20) | CPUIDLE_FLAG_TLB_FLUSHED, 5160138d8f0SLen Brown .exit_latency = 88, 5170138d8f0SLen Brown .target_residency = 700, 5185fe2e527SRafael J. Wysocki .enter = &intel_idle, 51928ba086eSRafael J. Wysocki .enter_s2idle = intel_idle_s2idle, }, 5200138d8f0SLen Brown { 5210138d8f0SLen Brown .enter = NULL } 5220138d8f0SLen Brown }; 5230138d8f0SLen Brown 524ab1a8522SRafael J. Wysocki static struct cpuidle_state hsw_cstates[] __initdata = { 525e022e7ebSLen Brown { 526de09cdd0SLen Brown .name = "C1", 52785a4d2d4SLen Brown .desc = "MWAIT 0x00", 528b82b6ccaSDaniel Lezcano .flags = MWAIT2flg(0x00), 52985a4d2d4SLen Brown .exit_latency = 2, 53085a4d2d4SLen Brown .target_residency = 2, 5315fe2e527SRafael J. Wysocki .enter = &intel_idle, 53228ba086eSRafael J. Wysocki .enter_s2idle = intel_idle_s2idle, }, 533e022e7ebSLen Brown { 534de09cdd0SLen Brown .name = "C1E", 53532e95180SLen Brown .desc = "MWAIT 0x01", 536e6d4f08aSRafael J. Wysocki .flags = MWAIT2flg(0x01) | CPUIDLE_FLAG_ALWAYS_ENABLE, 53732e95180SLen Brown .exit_latency = 10, 53832e95180SLen Brown .target_residency = 20, 5395fe2e527SRafael J. Wysocki .enter = &intel_idle, 54028ba086eSRafael J. Wysocki .enter_s2idle = intel_idle_s2idle, }, 54132e95180SLen Brown { 542de09cdd0SLen Brown .name = "C3", 54385a4d2d4SLen Brown .desc = "MWAIT 0x10", 544b82b6ccaSDaniel Lezcano .flags = MWAIT2flg(0x10) | CPUIDLE_FLAG_TLB_FLUSHED, 54585a4d2d4SLen Brown .exit_latency = 33, 54685a4d2d4SLen Brown .target_residency = 100, 5475fe2e527SRafael J. Wysocki .enter = &intel_idle, 54828ba086eSRafael J. Wysocki .enter_s2idle = intel_idle_s2idle, }, 549e022e7ebSLen Brown { 550de09cdd0SLen Brown .name = "C6", 55185a4d2d4SLen Brown .desc = "MWAIT 0x20", 552b82b6ccaSDaniel Lezcano .flags = MWAIT2flg(0x20) | CPUIDLE_FLAG_TLB_FLUSHED, 55385a4d2d4SLen Brown .exit_latency = 133, 55485a4d2d4SLen Brown .target_residency = 400, 5555fe2e527SRafael J. Wysocki .enter = &intel_idle, 55628ba086eSRafael J. Wysocki .enter_s2idle = intel_idle_s2idle, }, 557e022e7ebSLen Brown { 558de09cdd0SLen Brown .name = "C7s", 55985a4d2d4SLen Brown .desc = "MWAIT 0x32", 560b82b6ccaSDaniel Lezcano .flags = MWAIT2flg(0x32) | CPUIDLE_FLAG_TLB_FLUSHED, 56185a4d2d4SLen Brown .exit_latency = 166, 56285a4d2d4SLen Brown .target_residency = 500, 5635fe2e527SRafael J. Wysocki .enter = &intel_idle, 56428ba086eSRafael J. Wysocki .enter_s2idle = intel_idle_s2idle, }, 565e022e7ebSLen Brown { 566de09cdd0SLen Brown .name = "C8", 56786239cebSLen Brown .desc = "MWAIT 0x40", 568b82b6ccaSDaniel Lezcano .flags = MWAIT2flg(0x40) | CPUIDLE_FLAG_TLB_FLUSHED, 56986239cebSLen Brown .exit_latency = 300, 57086239cebSLen Brown .target_residency = 900, 5715fe2e527SRafael J. Wysocki .enter = &intel_idle, 57228ba086eSRafael J. Wysocki .enter_s2idle = intel_idle_s2idle, }, 57386239cebSLen Brown { 574de09cdd0SLen Brown .name = "C9", 57586239cebSLen Brown .desc = "MWAIT 0x50", 576b82b6ccaSDaniel Lezcano .flags = MWAIT2flg(0x50) | CPUIDLE_FLAG_TLB_FLUSHED, 57786239cebSLen Brown .exit_latency = 600, 57886239cebSLen Brown .target_residency = 1800, 5795fe2e527SRafael J. Wysocki .enter = &intel_idle, 58028ba086eSRafael J. Wysocki .enter_s2idle = intel_idle_s2idle, }, 58186239cebSLen Brown { 582de09cdd0SLen Brown .name = "C10", 58386239cebSLen Brown .desc = "MWAIT 0x60", 584b82b6ccaSDaniel Lezcano .flags = MWAIT2flg(0x60) | CPUIDLE_FLAG_TLB_FLUSHED, 58586239cebSLen Brown .exit_latency = 2600, 58686239cebSLen Brown .target_residency = 7700, 5875fe2e527SRafael J. Wysocki .enter = &intel_idle, 58828ba086eSRafael J. Wysocki .enter_s2idle = intel_idle_s2idle, }, 58986239cebSLen Brown { 590e022e7ebSLen Brown .enter = NULL } 59185a4d2d4SLen Brown }; 592ab1a8522SRafael J. Wysocki static struct cpuidle_state bdw_cstates[] __initdata = { 593a138b568SLen Brown { 594de09cdd0SLen Brown .name = "C1", 595a138b568SLen Brown .desc = "MWAIT 0x00", 596b82b6ccaSDaniel Lezcano .flags = MWAIT2flg(0x00), 597a138b568SLen Brown .exit_latency = 2, 598a138b568SLen Brown .target_residency = 2, 5995fe2e527SRafael J. Wysocki .enter = &intel_idle, 60028ba086eSRafael J. Wysocki .enter_s2idle = intel_idle_s2idle, }, 601a138b568SLen Brown { 602de09cdd0SLen Brown .name = "C1E", 603a138b568SLen Brown .desc = "MWAIT 0x01", 604e6d4f08aSRafael J. Wysocki .flags = MWAIT2flg(0x01) | CPUIDLE_FLAG_ALWAYS_ENABLE, 605a138b568SLen Brown .exit_latency = 10, 606a138b568SLen Brown .target_residency = 20, 6075fe2e527SRafael J. Wysocki .enter = &intel_idle, 60828ba086eSRafael J. Wysocki .enter_s2idle = intel_idle_s2idle, }, 609a138b568SLen Brown { 610de09cdd0SLen Brown .name = "C3", 611a138b568SLen Brown .desc = "MWAIT 0x10", 612b82b6ccaSDaniel Lezcano .flags = MWAIT2flg(0x10) | CPUIDLE_FLAG_TLB_FLUSHED, 613a138b568SLen Brown .exit_latency = 40, 614a138b568SLen Brown .target_residency = 100, 6155fe2e527SRafael J. Wysocki .enter = &intel_idle, 61628ba086eSRafael J. Wysocki .enter_s2idle = intel_idle_s2idle, }, 617a138b568SLen Brown { 618de09cdd0SLen Brown .name = "C6", 619a138b568SLen Brown .desc = "MWAIT 0x20", 620b82b6ccaSDaniel Lezcano .flags = MWAIT2flg(0x20) | CPUIDLE_FLAG_TLB_FLUSHED, 621a138b568SLen Brown .exit_latency = 133, 622a138b568SLen Brown .target_residency = 400, 6235fe2e527SRafael J. Wysocki .enter = &intel_idle, 62428ba086eSRafael J. Wysocki .enter_s2idle = intel_idle_s2idle, }, 625a138b568SLen Brown { 626de09cdd0SLen Brown .name = "C7s", 627a138b568SLen Brown .desc = "MWAIT 0x32", 628b82b6ccaSDaniel Lezcano .flags = MWAIT2flg(0x32) | CPUIDLE_FLAG_TLB_FLUSHED, 629a138b568SLen Brown .exit_latency = 166, 630a138b568SLen Brown .target_residency = 500, 6315fe2e527SRafael J. Wysocki .enter = &intel_idle, 63228ba086eSRafael J. Wysocki .enter_s2idle = intel_idle_s2idle, }, 633a138b568SLen Brown { 634de09cdd0SLen Brown .name = "C8", 635a138b568SLen Brown .desc = "MWAIT 0x40", 636b82b6ccaSDaniel Lezcano .flags = MWAIT2flg(0x40) | CPUIDLE_FLAG_TLB_FLUSHED, 637a138b568SLen Brown .exit_latency = 300, 638a138b568SLen Brown .target_residency = 900, 6395fe2e527SRafael J. Wysocki .enter = &intel_idle, 64028ba086eSRafael J. Wysocki .enter_s2idle = intel_idle_s2idle, }, 641a138b568SLen Brown { 642de09cdd0SLen Brown .name = "C9", 643a138b568SLen Brown .desc = "MWAIT 0x50", 644b82b6ccaSDaniel Lezcano .flags = MWAIT2flg(0x50) | CPUIDLE_FLAG_TLB_FLUSHED, 645a138b568SLen Brown .exit_latency = 600, 646a138b568SLen Brown .target_residency = 1800, 6475fe2e527SRafael J. Wysocki .enter = &intel_idle, 64828ba086eSRafael J. Wysocki .enter_s2idle = intel_idle_s2idle, }, 649a138b568SLen Brown { 650de09cdd0SLen Brown .name = "C10", 651a138b568SLen Brown .desc = "MWAIT 0x60", 652b82b6ccaSDaniel Lezcano .flags = MWAIT2flg(0x60) | CPUIDLE_FLAG_TLB_FLUSHED, 653a138b568SLen Brown .exit_latency = 2600, 654a138b568SLen Brown .target_residency = 7700, 6555fe2e527SRafael J. Wysocki .enter = &intel_idle, 65628ba086eSRafael J. Wysocki .enter_s2idle = intel_idle_s2idle, }, 657a138b568SLen Brown { 658a138b568SLen Brown .enter = NULL } 659a138b568SLen Brown }; 66085a4d2d4SLen Brown 661ab1a8522SRafael J. Wysocki static struct cpuidle_state skl_cstates[] __initdata = { 662493f133fSLen Brown { 663de09cdd0SLen Brown .name = "C1", 664493f133fSLen Brown .desc = "MWAIT 0x00", 665493f133fSLen Brown .flags = MWAIT2flg(0x00), 666493f133fSLen Brown .exit_latency = 2, 667493f133fSLen Brown .target_residency = 2, 668493f133fSLen Brown .enter = &intel_idle, 66928ba086eSRafael J. Wysocki .enter_s2idle = intel_idle_s2idle, }, 670493f133fSLen Brown { 671de09cdd0SLen Brown .name = "C1E", 672493f133fSLen Brown .desc = "MWAIT 0x01", 673e6d4f08aSRafael J. Wysocki .flags = MWAIT2flg(0x01) | CPUIDLE_FLAG_ALWAYS_ENABLE, 674493f133fSLen Brown .exit_latency = 10, 675493f133fSLen Brown .target_residency = 20, 676493f133fSLen Brown .enter = &intel_idle, 67728ba086eSRafael J. Wysocki .enter_s2idle = intel_idle_s2idle, }, 678493f133fSLen Brown { 679de09cdd0SLen Brown .name = "C3", 680493f133fSLen Brown .desc = "MWAIT 0x10", 681493f133fSLen Brown .flags = MWAIT2flg(0x10) | CPUIDLE_FLAG_TLB_FLUSHED, 682493f133fSLen Brown .exit_latency = 70, 683493f133fSLen Brown .target_residency = 100, 684493f133fSLen Brown .enter = &intel_idle, 68528ba086eSRafael J. Wysocki .enter_s2idle = intel_idle_s2idle, }, 686493f133fSLen Brown { 687de09cdd0SLen Brown .name = "C6", 688493f133fSLen Brown .desc = "MWAIT 0x20", 689493f133fSLen Brown .flags = MWAIT2flg(0x20) | CPUIDLE_FLAG_TLB_FLUSHED, 690135919a3SLen Brown .exit_latency = 85, 691493f133fSLen Brown .target_residency = 200, 692493f133fSLen Brown .enter = &intel_idle, 69328ba086eSRafael J. Wysocki .enter_s2idle = intel_idle_s2idle, }, 694493f133fSLen Brown { 695de09cdd0SLen Brown .name = "C7s", 696493f133fSLen Brown .desc = "MWAIT 0x33", 697493f133fSLen Brown .flags = MWAIT2flg(0x33) | CPUIDLE_FLAG_TLB_FLUSHED, 698493f133fSLen Brown .exit_latency = 124, 699493f133fSLen Brown .target_residency = 800, 700493f133fSLen Brown .enter = &intel_idle, 70128ba086eSRafael J. Wysocki .enter_s2idle = intel_idle_s2idle, }, 702493f133fSLen Brown { 703de09cdd0SLen Brown .name = "C8", 704493f133fSLen Brown .desc = "MWAIT 0x40", 705493f133fSLen Brown .flags = MWAIT2flg(0x40) | CPUIDLE_FLAG_TLB_FLUSHED, 706135919a3SLen Brown .exit_latency = 200, 707493f133fSLen Brown .target_residency = 800, 708493f133fSLen Brown .enter = &intel_idle, 70928ba086eSRafael J. Wysocki .enter_s2idle = intel_idle_s2idle, }, 710493f133fSLen Brown { 711de09cdd0SLen Brown .name = "C9", 712135919a3SLen Brown .desc = "MWAIT 0x50", 713135919a3SLen Brown .flags = MWAIT2flg(0x50) | CPUIDLE_FLAG_TLB_FLUSHED, 714135919a3SLen Brown .exit_latency = 480, 715135919a3SLen Brown .target_residency = 5000, 716135919a3SLen Brown .enter = &intel_idle, 71728ba086eSRafael J. Wysocki .enter_s2idle = intel_idle_s2idle, }, 718135919a3SLen Brown { 719de09cdd0SLen Brown .name = "C10", 720493f133fSLen Brown .desc = "MWAIT 0x60", 721493f133fSLen Brown .flags = MWAIT2flg(0x60) | CPUIDLE_FLAG_TLB_FLUSHED, 722493f133fSLen Brown .exit_latency = 890, 723493f133fSLen Brown .target_residency = 5000, 724493f133fSLen Brown .enter = &intel_idle, 72528ba086eSRafael J. Wysocki .enter_s2idle = intel_idle_s2idle, }, 726493f133fSLen Brown { 727493f133fSLen Brown .enter = NULL } 728493f133fSLen Brown }; 729493f133fSLen Brown 730ab1a8522SRafael J. Wysocki static struct cpuidle_state skx_cstates[] __initdata = { 731f9e71657SLen Brown { 732de09cdd0SLen Brown .name = "C1", 733f9e71657SLen Brown .desc = "MWAIT 0x00", 734f9e71657SLen Brown .flags = MWAIT2flg(0x00), 735f9e71657SLen Brown .exit_latency = 2, 736f9e71657SLen Brown .target_residency = 2, 737f9e71657SLen Brown .enter = &intel_idle, 73828ba086eSRafael J. Wysocki .enter_s2idle = intel_idle_s2idle, }, 739f9e71657SLen Brown { 740de09cdd0SLen Brown .name = "C1E", 741f9e71657SLen Brown .desc = "MWAIT 0x01", 742e6d4f08aSRafael J. Wysocki .flags = MWAIT2flg(0x01) | CPUIDLE_FLAG_ALWAYS_ENABLE, 743f9e71657SLen Brown .exit_latency = 10, 744f9e71657SLen Brown .target_residency = 20, 745f9e71657SLen Brown .enter = &intel_idle, 74628ba086eSRafael J. Wysocki .enter_s2idle = intel_idle_s2idle, }, 747f9e71657SLen Brown { 748de09cdd0SLen Brown .name = "C6", 749f9e71657SLen Brown .desc = "MWAIT 0x20", 750f9e71657SLen Brown .flags = MWAIT2flg(0x20) | CPUIDLE_FLAG_TLB_FLUSHED, 751f9e71657SLen Brown .exit_latency = 133, 752f9e71657SLen Brown .target_residency = 600, 753f9e71657SLen Brown .enter = &intel_idle, 75428ba086eSRafael J. Wysocki .enter_s2idle = intel_idle_s2idle, }, 755f9e71657SLen Brown { 756f9e71657SLen Brown .enter = NULL } 757f9e71657SLen Brown }; 758f9e71657SLen Brown 759ab1a8522SRafael J. Wysocki static struct cpuidle_state atom_cstates[] __initdata = { 760e022e7ebSLen Brown { 761de09cdd0SLen Brown .name = "C1E", 76226717172SLen Brown .desc = "MWAIT 0x00", 763b82b6ccaSDaniel Lezcano .flags = MWAIT2flg(0x00), 76432e95180SLen Brown .exit_latency = 10, 76532e95180SLen Brown .target_residency = 20, 7665fe2e527SRafael J. Wysocki .enter = &intel_idle, 76728ba086eSRafael J. Wysocki .enter_s2idle = intel_idle_s2idle, }, 768e022e7ebSLen Brown { 769de09cdd0SLen Brown .name = "C2", 77026717172SLen Brown .desc = "MWAIT 0x10", 771b82b6ccaSDaniel Lezcano .flags = MWAIT2flg(0x10), 77226717172SLen Brown .exit_latency = 20, 77326717172SLen Brown .target_residency = 80, 7745fe2e527SRafael J. Wysocki .enter = &intel_idle, 77528ba086eSRafael J. Wysocki .enter_s2idle = intel_idle_s2idle, }, 776e022e7ebSLen Brown { 777de09cdd0SLen Brown .name = "C4", 77826717172SLen Brown .desc = "MWAIT 0x30", 779b82b6ccaSDaniel Lezcano .flags = MWAIT2flg(0x30) | CPUIDLE_FLAG_TLB_FLUSHED, 78026717172SLen Brown .exit_latency = 100, 78126717172SLen Brown .target_residency = 400, 7825fe2e527SRafael J. Wysocki .enter = &intel_idle, 78328ba086eSRafael J. Wysocki .enter_s2idle = intel_idle_s2idle, }, 784e022e7ebSLen Brown { 785de09cdd0SLen Brown .name = "C6", 7867fcca7d9SLen Brown .desc = "MWAIT 0x52", 787b82b6ccaSDaniel Lezcano .flags = MWAIT2flg(0x52) | CPUIDLE_FLAG_TLB_FLUSHED, 7887fcca7d9SLen Brown .exit_latency = 140, 7897fcca7d9SLen Brown .target_residency = 560, 7905fe2e527SRafael J. Wysocki .enter = &intel_idle, 79128ba086eSRafael J. Wysocki .enter_s2idle = intel_idle_s2idle, }, 792e022e7ebSLen Brown { 793e022e7ebSLen Brown .enter = NULL } 79426717172SLen Brown }; 795ab1a8522SRafael J. Wysocki static struct cpuidle_state tangier_cstates[] __initdata = { 7965e7ec268SAndy Shevchenko { 797de09cdd0SLen Brown .name = "C1", 7985e7ec268SAndy Shevchenko .desc = "MWAIT 0x00", 7995e7ec268SAndy Shevchenko .flags = MWAIT2flg(0x00), 8005e7ec268SAndy Shevchenko .exit_latency = 1, 8015e7ec268SAndy Shevchenko .target_residency = 4, 8025e7ec268SAndy Shevchenko .enter = &intel_idle, 80328ba086eSRafael J. Wysocki .enter_s2idle = intel_idle_s2idle, }, 8045e7ec268SAndy Shevchenko { 805de09cdd0SLen Brown .name = "C4", 8065e7ec268SAndy Shevchenko .desc = "MWAIT 0x30", 8075e7ec268SAndy Shevchenko .flags = MWAIT2flg(0x30) | CPUIDLE_FLAG_TLB_FLUSHED, 8085e7ec268SAndy Shevchenko .exit_latency = 100, 8095e7ec268SAndy Shevchenko .target_residency = 400, 8105e7ec268SAndy Shevchenko .enter = &intel_idle, 81128ba086eSRafael J. Wysocki .enter_s2idle = intel_idle_s2idle, }, 8125e7ec268SAndy Shevchenko { 813de09cdd0SLen Brown .name = "C6", 8145e7ec268SAndy Shevchenko .desc = "MWAIT 0x52", 8155e7ec268SAndy Shevchenko .flags = MWAIT2flg(0x52) | CPUIDLE_FLAG_TLB_FLUSHED, 8165e7ec268SAndy Shevchenko .exit_latency = 140, 8175e7ec268SAndy Shevchenko .target_residency = 560, 8185e7ec268SAndy Shevchenko .enter = &intel_idle, 81928ba086eSRafael J. Wysocki .enter_s2idle = intel_idle_s2idle, }, 8205e7ec268SAndy Shevchenko { 821de09cdd0SLen Brown .name = "C7", 8225e7ec268SAndy Shevchenko .desc = "MWAIT 0x60", 8235e7ec268SAndy Shevchenko .flags = MWAIT2flg(0x60) | CPUIDLE_FLAG_TLB_FLUSHED, 8245e7ec268SAndy Shevchenko .exit_latency = 1200, 8255e7ec268SAndy Shevchenko .target_residency = 4000, 8265e7ec268SAndy Shevchenko .enter = &intel_idle, 82728ba086eSRafael J. Wysocki .enter_s2idle = intel_idle_s2idle, }, 8285e7ec268SAndy Shevchenko { 829de09cdd0SLen Brown .name = "C9", 8305e7ec268SAndy Shevchenko .desc = "MWAIT 0x64", 8315e7ec268SAndy Shevchenko .flags = MWAIT2flg(0x64) | CPUIDLE_FLAG_TLB_FLUSHED, 8325e7ec268SAndy Shevchenko .exit_latency = 10000, 8335e7ec268SAndy Shevchenko .target_residency = 20000, 8345e7ec268SAndy Shevchenko .enter = &intel_idle, 83528ba086eSRafael J. Wysocki .enter_s2idle = intel_idle_s2idle, }, 8365e7ec268SAndy Shevchenko { 8375e7ec268SAndy Shevchenko .enter = NULL } 8385e7ec268SAndy Shevchenko }; 839ab1a8522SRafael J. Wysocki static struct cpuidle_state avn_cstates[] __initdata = { 840fab04b22SLen Brown { 841de09cdd0SLen Brown .name = "C1", 842fab04b22SLen Brown .desc = "MWAIT 0x00", 843b82b6ccaSDaniel Lezcano .flags = MWAIT2flg(0x00), 844fab04b22SLen Brown .exit_latency = 2, 845fab04b22SLen Brown .target_residency = 2, 8465fe2e527SRafael J. Wysocki .enter = &intel_idle, 84728ba086eSRafael J. Wysocki .enter_s2idle = intel_idle_s2idle, }, 848fab04b22SLen Brown { 849de09cdd0SLen Brown .name = "C6", 850fab04b22SLen Brown .desc = "MWAIT 0x51", 851b82b6ccaSDaniel Lezcano .flags = MWAIT2flg(0x51) | CPUIDLE_FLAG_TLB_FLUSHED, 852fab04b22SLen Brown .exit_latency = 15, 853fab04b22SLen Brown .target_residency = 45, 8545fe2e527SRafael J. Wysocki .enter = &intel_idle, 85528ba086eSRafael J. Wysocki .enter_s2idle = intel_idle_s2idle, }, 85688390996SJiang Liu { 85788390996SJiang Liu .enter = NULL } 858fab04b22SLen Brown }; 859ab1a8522SRafael J. Wysocki static struct cpuidle_state knl_cstates[] __initdata = { 860281baf7aSDasaratharaman Chandramouli { 861de09cdd0SLen Brown .name = "C1", 862281baf7aSDasaratharaman Chandramouli .desc = "MWAIT 0x00", 863281baf7aSDasaratharaman Chandramouli .flags = MWAIT2flg(0x00), 864281baf7aSDasaratharaman Chandramouli .exit_latency = 1, 865281baf7aSDasaratharaman Chandramouli .target_residency = 2, 866281baf7aSDasaratharaman Chandramouli .enter = &intel_idle, 86728ba086eSRafael J. Wysocki .enter_s2idle = intel_idle_s2idle }, 868281baf7aSDasaratharaman Chandramouli { 869de09cdd0SLen Brown .name = "C6", 870281baf7aSDasaratharaman Chandramouli .desc = "MWAIT 0x10", 871281baf7aSDasaratharaman Chandramouli .flags = MWAIT2flg(0x10) | CPUIDLE_FLAG_TLB_FLUSHED, 872281baf7aSDasaratharaman Chandramouli .exit_latency = 120, 873281baf7aSDasaratharaman Chandramouli .target_residency = 500, 874281baf7aSDasaratharaman Chandramouli .enter = &intel_idle, 87528ba086eSRafael J. Wysocki .enter_s2idle = intel_idle_s2idle }, 876281baf7aSDasaratharaman Chandramouli { 877281baf7aSDasaratharaman Chandramouli .enter = NULL } 878281baf7aSDasaratharaman Chandramouli }; 87926717172SLen Brown 880ab1a8522SRafael J. Wysocki static struct cpuidle_state bxt_cstates[] __initdata = { 8815dcef694SLen Brown { 882de09cdd0SLen Brown .name = "C1", 8835dcef694SLen Brown .desc = "MWAIT 0x00", 8845dcef694SLen Brown .flags = MWAIT2flg(0x00), 8855dcef694SLen Brown .exit_latency = 2, 8865dcef694SLen Brown .target_residency = 2, 8875dcef694SLen Brown .enter = &intel_idle, 88828ba086eSRafael J. Wysocki .enter_s2idle = intel_idle_s2idle, }, 8895dcef694SLen Brown { 890de09cdd0SLen Brown .name = "C1E", 8915dcef694SLen Brown .desc = "MWAIT 0x01", 892e6d4f08aSRafael J. Wysocki .flags = MWAIT2flg(0x01) | CPUIDLE_FLAG_ALWAYS_ENABLE, 8935dcef694SLen Brown .exit_latency = 10, 8945dcef694SLen Brown .target_residency = 20, 8955dcef694SLen Brown .enter = &intel_idle, 89628ba086eSRafael J. Wysocki .enter_s2idle = intel_idle_s2idle, }, 8975dcef694SLen Brown { 898de09cdd0SLen Brown .name = "C6", 8995dcef694SLen Brown .desc = "MWAIT 0x20", 9005dcef694SLen Brown .flags = MWAIT2flg(0x20) | CPUIDLE_FLAG_TLB_FLUSHED, 9015dcef694SLen Brown .exit_latency = 133, 9025dcef694SLen Brown .target_residency = 133, 9035dcef694SLen Brown .enter = &intel_idle, 90428ba086eSRafael J. Wysocki .enter_s2idle = intel_idle_s2idle, }, 9055dcef694SLen Brown { 906de09cdd0SLen Brown .name = "C7s", 9075dcef694SLen Brown .desc = "MWAIT 0x31", 9085dcef694SLen Brown .flags = MWAIT2flg(0x31) | CPUIDLE_FLAG_TLB_FLUSHED, 9095dcef694SLen Brown .exit_latency = 155, 9105dcef694SLen Brown .target_residency = 155, 9115dcef694SLen Brown .enter = &intel_idle, 91228ba086eSRafael J. Wysocki .enter_s2idle = intel_idle_s2idle, }, 9135dcef694SLen Brown { 914de09cdd0SLen Brown .name = "C8", 9155dcef694SLen Brown .desc = "MWAIT 0x40", 9165dcef694SLen Brown .flags = MWAIT2flg(0x40) | CPUIDLE_FLAG_TLB_FLUSHED, 9175dcef694SLen Brown .exit_latency = 1000, 9185dcef694SLen Brown .target_residency = 1000, 9195dcef694SLen Brown .enter = &intel_idle, 92028ba086eSRafael J. Wysocki .enter_s2idle = intel_idle_s2idle, }, 9215dcef694SLen Brown { 922de09cdd0SLen Brown .name = "C9", 9235dcef694SLen Brown .desc = "MWAIT 0x50", 9245dcef694SLen Brown .flags = MWAIT2flg(0x50) | CPUIDLE_FLAG_TLB_FLUSHED, 9255dcef694SLen Brown .exit_latency = 2000, 9265dcef694SLen Brown .target_residency = 2000, 9275dcef694SLen Brown .enter = &intel_idle, 92828ba086eSRafael J. Wysocki .enter_s2idle = intel_idle_s2idle, }, 9295dcef694SLen Brown { 930de09cdd0SLen Brown .name = "C10", 9315dcef694SLen Brown .desc = "MWAIT 0x60", 9325dcef694SLen Brown .flags = MWAIT2flg(0x60) | CPUIDLE_FLAG_TLB_FLUSHED, 9335dcef694SLen Brown .exit_latency = 10000, 9345dcef694SLen Brown .target_residency = 10000, 9355dcef694SLen Brown .enter = &intel_idle, 93628ba086eSRafael J. Wysocki .enter_s2idle = intel_idle_s2idle, }, 9375dcef694SLen Brown { 9385dcef694SLen Brown .enter = NULL } 9395dcef694SLen Brown }; 9405dcef694SLen Brown 941ab1a8522SRafael J. Wysocki static struct cpuidle_state dnv_cstates[] __initdata = { 9420080d65bSJacob Pan { 943de09cdd0SLen Brown .name = "C1", 9440080d65bSJacob Pan .desc = "MWAIT 0x00", 9450080d65bSJacob Pan .flags = MWAIT2flg(0x00), 9460080d65bSJacob Pan .exit_latency = 2, 9470080d65bSJacob Pan .target_residency = 2, 9480080d65bSJacob Pan .enter = &intel_idle, 94928ba086eSRafael J. Wysocki .enter_s2idle = intel_idle_s2idle, }, 9500080d65bSJacob Pan { 951de09cdd0SLen Brown .name = "C1E", 9520080d65bSJacob Pan .desc = "MWAIT 0x01", 953e6d4f08aSRafael J. Wysocki .flags = MWAIT2flg(0x01) | CPUIDLE_FLAG_ALWAYS_ENABLE, 9540080d65bSJacob Pan .exit_latency = 10, 9550080d65bSJacob Pan .target_residency = 20, 9560080d65bSJacob Pan .enter = &intel_idle, 95728ba086eSRafael J. Wysocki .enter_s2idle = intel_idle_s2idle, }, 9580080d65bSJacob Pan { 959de09cdd0SLen Brown .name = "C6", 9600080d65bSJacob Pan .desc = "MWAIT 0x20", 9610080d65bSJacob Pan .flags = MWAIT2flg(0x20) | CPUIDLE_FLAG_TLB_FLUSHED, 9620080d65bSJacob Pan .exit_latency = 50, 9630080d65bSJacob Pan .target_residency = 500, 9640080d65bSJacob Pan .enter = &intel_idle, 96528ba086eSRafael J. Wysocki .enter_s2idle = intel_idle_s2idle, }, 9660080d65bSJacob Pan { 9670080d65bSJacob Pan .enter = NULL } 9680080d65bSJacob Pan }; 9690080d65bSJacob Pan 970ab1a8522SRafael J. Wysocki static const struct idle_cpu idle_cpu_nehalem __initconst = { 971b66b8b9aSAndi Kleen .state_table = nehalem_cstates, 972b66b8b9aSAndi Kleen .auto_demotion_disable_flags = NHM_C1_AUTO_DEMOTE | NHM_C3_AUTO_DEMOTE, 97332e95180SLen Brown .disable_promotion_to_c1e = true, 974b66b8b9aSAndi Kleen }; 975b66b8b9aSAndi Kleen 976ab1a8522SRafael J. Wysocki static const struct idle_cpu idle_cpu_nhx __initconst = { 977e6d4f08aSRafael J. Wysocki .state_table = nehalem_cstates, 978e6d4f08aSRafael J. Wysocki .auto_demotion_disable_flags = NHM_C1_AUTO_DEMOTE | NHM_C3_AUTO_DEMOTE, 979e6d4f08aSRafael J. Wysocki .disable_promotion_to_c1e = true, 980e6d4f08aSRafael J. Wysocki .use_acpi = true, 981e6d4f08aSRafael J. Wysocki }; 982e6d4f08aSRafael J. Wysocki 983ab1a8522SRafael J. Wysocki static const struct idle_cpu idle_cpu_atom __initconst = { 984b66b8b9aSAndi Kleen .state_table = atom_cstates, 985b66b8b9aSAndi Kleen }; 986b66b8b9aSAndi Kleen 987ab1a8522SRafael J. Wysocki static const struct idle_cpu idle_cpu_tangier __initconst = { 9885e7ec268SAndy Shevchenko .state_table = tangier_cstates, 9895e7ec268SAndy Shevchenko }; 9905e7ec268SAndy Shevchenko 991ab1a8522SRafael J. Wysocki static const struct idle_cpu idle_cpu_lincroft __initconst = { 992b66b8b9aSAndi Kleen .state_table = atom_cstates, 993b66b8b9aSAndi Kleen .auto_demotion_disable_flags = ATM_LNC_C6_AUTO_DEMOTE, 994b66b8b9aSAndi Kleen }; 995b66b8b9aSAndi Kleen 996ab1a8522SRafael J. Wysocki static const struct idle_cpu idle_cpu_snb __initconst = { 997b66b8b9aSAndi Kleen .state_table = snb_cstates, 99832e95180SLen Brown .disable_promotion_to_c1e = true, 999b66b8b9aSAndi Kleen }; 1000b66b8b9aSAndi Kleen 1001ab1a8522SRafael J. Wysocki static const struct idle_cpu idle_cpu_snx __initconst = { 1002e6d4f08aSRafael J. Wysocki .state_table = snb_cstates, 1003e6d4f08aSRafael J. Wysocki .disable_promotion_to_c1e = true, 1004e6d4f08aSRafael J. Wysocki .use_acpi = true, 1005e6d4f08aSRafael J. Wysocki }; 1006e6d4f08aSRafael J. Wysocki 1007ab1a8522SRafael J. Wysocki static const struct idle_cpu idle_cpu_byt __initconst = { 1008718987d6SLen Brown .state_table = byt_cstates, 1009718987d6SLen Brown .disable_promotion_to_c1e = true, 10108c058d53SLen Brown .byt_auto_demotion_disable_flag = true, 1011718987d6SLen Brown }; 1012718987d6SLen Brown 1013ab1a8522SRafael J. Wysocki static const struct idle_cpu idle_cpu_cht __initconst = { 1014cab07a56SLen Brown .state_table = cht_cstates, 1015cab07a56SLen Brown .disable_promotion_to_c1e = true, 1016cab07a56SLen Brown .byt_auto_demotion_disable_flag = true, 1017cab07a56SLen Brown }; 1018cab07a56SLen Brown 1019ab1a8522SRafael J. Wysocki static const struct idle_cpu idle_cpu_ivb __initconst = { 10206edab08cSLen Brown .state_table = ivb_cstates, 102132e95180SLen Brown .disable_promotion_to_c1e = true, 10226edab08cSLen Brown }; 10236edab08cSLen Brown 1024ab1a8522SRafael J. Wysocki static const struct idle_cpu idle_cpu_ivt __initconst = { 10250138d8f0SLen Brown .state_table = ivt_cstates, 10260138d8f0SLen Brown .disable_promotion_to_c1e = true, 1027e6d4f08aSRafael J. Wysocki .use_acpi = true, 10280138d8f0SLen Brown }; 10290138d8f0SLen Brown 1030ab1a8522SRafael J. Wysocki static const struct idle_cpu idle_cpu_hsw __initconst = { 103185a4d2d4SLen Brown .state_table = hsw_cstates, 103232e95180SLen Brown .disable_promotion_to_c1e = true, 103385a4d2d4SLen Brown }; 103485a4d2d4SLen Brown 1035ab1a8522SRafael J. Wysocki static const struct idle_cpu idle_cpu_hsx __initconst = { 1036e6d4f08aSRafael J. Wysocki .state_table = hsw_cstates, 1037e6d4f08aSRafael J. Wysocki .disable_promotion_to_c1e = true, 1038e6d4f08aSRafael J. Wysocki .use_acpi = true, 1039e6d4f08aSRafael J. Wysocki }; 1040e6d4f08aSRafael J. Wysocki 1041ab1a8522SRafael J. Wysocki static const struct idle_cpu idle_cpu_bdw __initconst = { 1042a138b568SLen Brown .state_table = bdw_cstates, 1043a138b568SLen Brown .disable_promotion_to_c1e = true, 1044a138b568SLen Brown }; 1045a138b568SLen Brown 1046ab1a8522SRafael J. Wysocki static const struct idle_cpu idle_cpu_bdx __initconst = { 1047e6d4f08aSRafael J. Wysocki .state_table = bdw_cstates, 1048e6d4f08aSRafael J. Wysocki .disable_promotion_to_c1e = true, 1049e6d4f08aSRafael J. Wysocki .use_acpi = true, 1050e6d4f08aSRafael J. Wysocki }; 1051e6d4f08aSRafael J. Wysocki 1052ab1a8522SRafael J. Wysocki static const struct idle_cpu idle_cpu_skl __initconst = { 1053493f133fSLen Brown .state_table = skl_cstates, 1054493f133fSLen Brown .disable_promotion_to_c1e = true, 1055493f133fSLen Brown }; 1056493f133fSLen Brown 1057ab1a8522SRafael J. Wysocki static const struct idle_cpu idle_cpu_skx __initconst = { 1058f9e71657SLen Brown .state_table = skx_cstates, 1059f9e71657SLen Brown .disable_promotion_to_c1e = true, 1060e6d4f08aSRafael J. Wysocki .use_acpi = true, 1061f9e71657SLen Brown }; 1062493f133fSLen Brown 1063ab1a8522SRafael J. Wysocki static const struct idle_cpu idle_cpu_avn __initconst = { 1064fab04b22SLen Brown .state_table = avn_cstates, 1065fab04b22SLen Brown .disable_promotion_to_c1e = true, 1066e6d4f08aSRafael J. Wysocki .use_acpi = true, 1067fab04b22SLen Brown }; 1068fab04b22SLen Brown 1069ab1a8522SRafael J. Wysocki static const struct idle_cpu idle_cpu_knl __initconst = { 1070281baf7aSDasaratharaman Chandramouli .state_table = knl_cstates, 1071e6d4f08aSRafael J. Wysocki .use_acpi = true, 1072281baf7aSDasaratharaman Chandramouli }; 1073281baf7aSDasaratharaman Chandramouli 1074ab1a8522SRafael J. Wysocki static const struct idle_cpu idle_cpu_bxt __initconst = { 10755dcef694SLen Brown .state_table = bxt_cstates, 10765dcef694SLen Brown .disable_promotion_to_c1e = true, 10775dcef694SLen Brown }; 10785dcef694SLen Brown 1079ab1a8522SRafael J. Wysocki static const struct idle_cpu idle_cpu_dnv __initconst = { 10800080d65bSJacob Pan .state_table = dnv_cstates, 10810080d65bSJacob Pan .disable_promotion_to_c1e = true, 1082e6d4f08aSRafael J. Wysocki .use_acpi = true, 10830080d65bSJacob Pan }; 10840080d65bSJacob Pan 1085d5cdc3c4SMathias Krause static const struct x86_cpu_id intel_idle_ids[] __initconst = { 1086e6d4f08aSRafael J. Wysocki INTEL_CPU_FAM6(NEHALEM_EP, idle_cpu_nhx), 1087a4a008e5SAndy Shevchenko INTEL_CPU_FAM6(NEHALEM, idle_cpu_nehalem), 1088a4a008e5SAndy Shevchenko INTEL_CPU_FAM6(NEHALEM_G, idle_cpu_nehalem), 1089a4a008e5SAndy Shevchenko INTEL_CPU_FAM6(WESTMERE, idle_cpu_nehalem), 1090e6d4f08aSRafael J. Wysocki INTEL_CPU_FAM6(WESTMERE_EP, idle_cpu_nhx), 1091e6d4f08aSRafael J. Wysocki INTEL_CPU_FAM6(NEHALEM_EX, idle_cpu_nhx), 1092c05f3642SLinus Torvalds INTEL_CPU_FAM6(ATOM_BONNELL, idle_cpu_atom), 1093c05f3642SLinus Torvalds INTEL_CPU_FAM6(ATOM_BONNELL_MID, idle_cpu_lincroft), 1094e6d4f08aSRafael J. Wysocki INTEL_CPU_FAM6(WESTMERE_EX, idle_cpu_nhx), 1095a4a008e5SAndy Shevchenko INTEL_CPU_FAM6(SANDYBRIDGE, idle_cpu_snb), 1096e6d4f08aSRafael J. Wysocki INTEL_CPU_FAM6(SANDYBRIDGE_X, idle_cpu_snx), 1097c05f3642SLinus Torvalds INTEL_CPU_FAM6(ATOM_SALTWELL, idle_cpu_atom), 1098c05f3642SLinus Torvalds INTEL_CPU_FAM6(ATOM_SILVERMONT, idle_cpu_byt), 1099c05f3642SLinus Torvalds INTEL_CPU_FAM6(ATOM_SILVERMONT_MID, idle_cpu_tangier), 1100a4a008e5SAndy Shevchenko INTEL_CPU_FAM6(ATOM_AIRMONT, idle_cpu_cht), 1101a4a008e5SAndy Shevchenko INTEL_CPU_FAM6(IVYBRIDGE, idle_cpu_ivb), 1102a4a008e5SAndy Shevchenko INTEL_CPU_FAM6(IVYBRIDGE_X, idle_cpu_ivt), 1103c66f78a6SPeter Zijlstra INTEL_CPU_FAM6(HASWELL, idle_cpu_hsw), 1104e6d4f08aSRafael J. Wysocki INTEL_CPU_FAM6(HASWELL_X, idle_cpu_hsx), 1105af239c44SPeter Zijlstra INTEL_CPU_FAM6(HASWELL_L, idle_cpu_hsw), 11065e741407SPeter Zijlstra INTEL_CPU_FAM6(HASWELL_G, idle_cpu_hsw), 11075ebb34edSPeter Zijlstra INTEL_CPU_FAM6(ATOM_SILVERMONT_D, idle_cpu_avn), 1108c66f78a6SPeter Zijlstra INTEL_CPU_FAM6(BROADWELL, idle_cpu_bdw), 11095e741407SPeter Zijlstra INTEL_CPU_FAM6(BROADWELL_G, idle_cpu_bdw), 1110e6d4f08aSRafael J. Wysocki INTEL_CPU_FAM6(BROADWELL_X, idle_cpu_bdx), 1111e6d4f08aSRafael J. Wysocki INTEL_CPU_FAM6(BROADWELL_D, idle_cpu_bdx), 1112af239c44SPeter Zijlstra INTEL_CPU_FAM6(SKYLAKE_L, idle_cpu_skl), 1113c66f78a6SPeter Zijlstra INTEL_CPU_FAM6(SKYLAKE, idle_cpu_skl), 1114af239c44SPeter Zijlstra INTEL_CPU_FAM6(KABYLAKE_L, idle_cpu_skl), 1115c66f78a6SPeter Zijlstra INTEL_CPU_FAM6(KABYLAKE, idle_cpu_skl), 1116a4a008e5SAndy Shevchenko INTEL_CPU_FAM6(SKYLAKE_X, idle_cpu_skx), 1117a4a008e5SAndy Shevchenko INTEL_CPU_FAM6(XEON_PHI_KNL, idle_cpu_knl), 1118a4a008e5SAndy Shevchenko INTEL_CPU_FAM6(XEON_PHI_KNM, idle_cpu_knl), 1119a4a008e5SAndy Shevchenko INTEL_CPU_FAM6(ATOM_GOLDMONT, idle_cpu_bxt), 1120c05f3642SLinus Torvalds INTEL_CPU_FAM6(ATOM_GOLDMONT_PLUS, idle_cpu_bxt), 11215ebb34edSPeter Zijlstra INTEL_CPU_FAM6(ATOM_GOLDMONT_D, idle_cpu_dnv), 11225ebb34edSPeter Zijlstra INTEL_CPU_FAM6(ATOM_TREMONT_D, idle_cpu_dnv), 1123b66b8b9aSAndi Kleen {} 1124b66b8b9aSAndi Kleen }; 1125b66b8b9aSAndi Kleen 112618734958SRafael J. Wysocki #define INTEL_CPU_FAM6_MWAIT \ 112718734958SRafael J. Wysocki { X86_VENDOR_INTEL, 6, X86_MODEL_ANY, X86_FEATURE_MWAIT, 0 } 112818734958SRafael J. Wysocki 112918734958SRafael J. Wysocki static const struct x86_cpu_id intel_mwait_ids[] __initconst = { 113018734958SRafael J. Wysocki INTEL_CPU_FAM6_MWAIT, 113118734958SRafael J. Wysocki {} 113218734958SRafael J. Wysocki }; 113318734958SRafael J. Wysocki 1134095928aeSRafael J. Wysocki static bool __init intel_idle_max_cstate_reached(int cstate) 113518734958SRafael J. Wysocki { 113618734958SRafael J. Wysocki if (cstate + 1 > max_cstate) { 113718734958SRafael J. Wysocki pr_info("max_cstate %d reached\n", max_cstate); 113818734958SRafael J. Wysocki return true; 113918734958SRafael J. Wysocki } 114018734958SRafael J. Wysocki return false; 114118734958SRafael J. Wysocki } 114218734958SRafael J. Wysocki 114318734958SRafael J. Wysocki #ifdef CONFIG_ACPI_PROCESSOR_CSTATE 114418734958SRafael J. Wysocki #include <acpi/processor.h> 114518734958SRafael J. Wysocki 11464ec32d9eSRafael J. Wysocki static bool no_acpi __read_mostly; 11474ec32d9eSRafael J. Wysocki module_param(no_acpi, bool, 0444); 11484ec32d9eSRafael J. Wysocki MODULE_PARM_DESC(no_acpi, "Do not use ACPI _CST for building the idle states list"); 11494ec32d9eSRafael J. Wysocki 11503a5be9b8SRafael J. Wysocki static bool force_use_acpi __read_mostly; /* No effect if no_acpi is set. */ 11513a5be9b8SRafael J. Wysocki module_param_named(use_acpi, force_use_acpi, bool, 0444); 11523a5be9b8SRafael J. Wysocki MODULE_PARM_DESC(use_acpi, "Use ACPI _CST for building the idle states list"); 11533a5be9b8SRafael J. Wysocki 1154095928aeSRafael J. Wysocki static struct acpi_processor_power acpi_state_table __initdata; 115518734958SRafael J. Wysocki 115618734958SRafael J. Wysocki /** 115718734958SRafael J. Wysocki * intel_idle_cst_usable - Check if the _CST information can be used. 115818734958SRafael J. Wysocki * 115918734958SRafael J. Wysocki * Check if all of the C-states listed by _CST in the max_cstate range are 116018734958SRafael J. Wysocki * ACPI_CSTATE_FFH, which means that they should be entered via MWAIT. 116118734958SRafael J. Wysocki */ 1162095928aeSRafael J. Wysocki static bool __init intel_idle_cst_usable(void) 116318734958SRafael J. Wysocki { 116418734958SRafael J. Wysocki int cstate, limit; 116518734958SRafael J. Wysocki 116618734958SRafael J. Wysocki limit = min_t(int, min_t(int, CPUIDLE_STATE_MAX, max_cstate + 1), 116718734958SRafael J. Wysocki acpi_state_table.count); 116818734958SRafael J. Wysocki 116918734958SRafael J. Wysocki for (cstate = 1; cstate < limit; cstate++) { 117018734958SRafael J. Wysocki struct acpi_processor_cx *cx = &acpi_state_table.states[cstate]; 117118734958SRafael J. Wysocki 117218734958SRafael J. Wysocki if (cx->entry_method != ACPI_CSTATE_FFH) 117318734958SRafael J. Wysocki return false; 117418734958SRafael J. Wysocki } 117518734958SRafael J. Wysocki 117618734958SRafael J. Wysocki return true; 117718734958SRafael J. Wysocki } 117818734958SRafael J. Wysocki 1179095928aeSRafael J. Wysocki static bool __init intel_idle_acpi_cst_extract(void) 118018734958SRafael J. Wysocki { 118118734958SRafael J. Wysocki unsigned int cpu; 118218734958SRafael J. Wysocki 11834ec32d9eSRafael J. Wysocki if (no_acpi) { 11844ec32d9eSRafael J. Wysocki pr_debug("Not allowed to use ACPI _CST\n"); 11854ec32d9eSRafael J. Wysocki return false; 11864ec32d9eSRafael J. Wysocki } 11874ec32d9eSRafael J. Wysocki 118818734958SRafael J. Wysocki for_each_possible_cpu(cpu) { 118918734958SRafael J. Wysocki struct acpi_processor *pr = per_cpu(processors, cpu); 119018734958SRafael J. Wysocki 119118734958SRafael J. Wysocki if (!pr) 119218734958SRafael J. Wysocki continue; 119318734958SRafael J. Wysocki 119418734958SRafael J. Wysocki if (acpi_processor_evaluate_cst(pr->handle, cpu, &acpi_state_table)) 119518734958SRafael J. Wysocki continue; 119618734958SRafael J. Wysocki 119718734958SRafael J. Wysocki acpi_state_table.count++; 119818734958SRafael J. Wysocki 119918734958SRafael J. Wysocki if (!intel_idle_cst_usable()) 120018734958SRafael J. Wysocki continue; 120118734958SRafael J. Wysocki 120218734958SRafael J. Wysocki if (!acpi_processor_claim_cst_control()) { 120318734958SRafael J. Wysocki acpi_state_table.count = 0; 120418734958SRafael J. Wysocki return false; 120518734958SRafael J. Wysocki } 120618734958SRafael J. Wysocki 120718734958SRafael J. Wysocki return true; 120818734958SRafael J. Wysocki } 120918734958SRafael J. Wysocki 121018734958SRafael J. Wysocki pr_debug("ACPI _CST not found or not usable\n"); 121118734958SRafael J. Wysocki return false; 121218734958SRafael J. Wysocki } 121318734958SRafael J. Wysocki 1214095928aeSRafael J. Wysocki static void __init intel_idle_init_cstates_acpi(struct cpuidle_driver *drv) 121518734958SRafael J. Wysocki { 121618734958SRafael J. Wysocki int cstate, limit = min_t(int, CPUIDLE_STATE_MAX, acpi_state_table.count); 121718734958SRafael J. Wysocki 121818734958SRafael J. Wysocki /* 121918734958SRafael J. Wysocki * If limit > 0, intel_idle_cst_usable() has returned 'true', so all of 122018734958SRafael J. Wysocki * the interesting states are ACPI_CSTATE_FFH. 122118734958SRafael J. Wysocki */ 122218734958SRafael J. Wysocki for (cstate = 1; cstate < limit; cstate++) { 122318734958SRafael J. Wysocki struct acpi_processor_cx *cx; 122418734958SRafael J. Wysocki struct cpuidle_state *state; 122518734958SRafael J. Wysocki 122618734958SRafael J. Wysocki if (intel_idle_max_cstate_reached(cstate)) 122718734958SRafael J. Wysocki break; 122818734958SRafael J. Wysocki 122918734958SRafael J. Wysocki cx = &acpi_state_table.states[cstate]; 123018734958SRafael J. Wysocki 123118734958SRafael J. Wysocki state = &drv->states[drv->state_count++]; 123218734958SRafael J. Wysocki 123318734958SRafael J. Wysocki snprintf(state->name, CPUIDLE_NAME_LEN, "C%d_ACPI", cstate); 123418734958SRafael J. Wysocki strlcpy(state->desc, cx->desc, CPUIDLE_DESC_LEN); 123518734958SRafael J. Wysocki state->exit_latency = cx->latency; 123618734958SRafael J. Wysocki /* 123718734958SRafael J. Wysocki * For C1-type C-states use the same number for both the exit 123818734958SRafael J. Wysocki * latency and target residency, because that is the case for 123918734958SRafael J. Wysocki * C1 in the majority of the static C-states tables above. 124018734958SRafael J. Wysocki * For the other types of C-states, however, set the target 124118734958SRafael J. Wysocki * residency to 3 times the exit latency which should lead to 124218734958SRafael J. Wysocki * a reasonable balance between energy-efficiency and 124318734958SRafael J. Wysocki * performance in the majority of interesting cases. 124418734958SRafael J. Wysocki */ 124518734958SRafael J. Wysocki state->target_residency = cx->latency; 124618734958SRafael J. Wysocki if (cx->type > ACPI_STATE_C1) 124718734958SRafael J. Wysocki state->target_residency *= 3; 124818734958SRafael J. Wysocki 124918734958SRafael J. Wysocki state->flags = MWAIT2flg(cx->address); 125018734958SRafael J. Wysocki if (cx->type > ACPI_STATE_C2) 125118734958SRafael J. Wysocki state->flags |= CPUIDLE_FLAG_TLB_FLUSHED; 125218734958SRafael J. Wysocki 12534dcb78eeSRafael J. Wysocki if (disabled_states_mask & BIT(cstate)) 12544dcb78eeSRafael J. Wysocki state->flags |= CPUIDLE_FLAG_OFF; 12554dcb78eeSRafael J. Wysocki 125618734958SRafael J. Wysocki state->enter = intel_idle; 125718734958SRafael J. Wysocki state->enter_s2idle = intel_idle_s2idle; 125818734958SRafael J. Wysocki } 125918734958SRafael J. Wysocki } 1260bff8e60aSRafael J. Wysocki 1261095928aeSRafael J. Wysocki static bool __init intel_idle_off_by_default(u32 mwait_hint) 1262bff8e60aSRafael J. Wysocki { 1263bff8e60aSRafael J. Wysocki int cstate, limit; 1264bff8e60aSRafael J. Wysocki 1265bff8e60aSRafael J. Wysocki /* 1266bff8e60aSRafael J. Wysocki * If there are no _CST C-states, do not disable any C-states by 1267bff8e60aSRafael J. Wysocki * default. 1268bff8e60aSRafael J. Wysocki */ 1269bff8e60aSRafael J. Wysocki if (!acpi_state_table.count) 1270bff8e60aSRafael J. Wysocki return false; 1271bff8e60aSRafael J. Wysocki 1272bff8e60aSRafael J. Wysocki limit = min_t(int, CPUIDLE_STATE_MAX, acpi_state_table.count); 1273bff8e60aSRafael J. Wysocki /* 1274bff8e60aSRafael J. Wysocki * If limit > 0, intel_idle_cst_usable() has returned 'true', so all of 1275bff8e60aSRafael J. Wysocki * the interesting states are ACPI_CSTATE_FFH. 1276bff8e60aSRafael J. Wysocki */ 1277bff8e60aSRafael J. Wysocki for (cstate = 1; cstate < limit; cstate++) { 1278bff8e60aSRafael J. Wysocki if (acpi_state_table.states[cstate].address == mwait_hint) 1279bff8e60aSRafael J. Wysocki return false; 1280bff8e60aSRafael J. Wysocki } 1281bff8e60aSRafael J. Wysocki return true; 1282bff8e60aSRafael J. Wysocki } 128318734958SRafael J. Wysocki #else /* !CONFIG_ACPI_PROCESSOR_CSTATE */ 12843a5be9b8SRafael J. Wysocki #define force_use_acpi (false) 12853a5be9b8SRafael J. Wysocki 128618734958SRafael J. Wysocki static inline bool intel_idle_acpi_cst_extract(void) { return false; } 128718734958SRafael J. Wysocki static inline void intel_idle_init_cstates_acpi(struct cpuidle_driver *drv) { } 1288bff8e60aSRafael J. Wysocki static inline bool intel_idle_off_by_default(u32 mwait_hint) { return false; } 128918734958SRafael J. Wysocki #endif /* !CONFIG_ACPI_PROCESSOR_CSTATE */ 129018734958SRafael J. Wysocki 129126717172SLen Brown /* 1292d70e28f5SLen Brown * ivt_idle_state_table_update(void) 12930138d8f0SLen Brown * 1294d70e28f5SLen Brown * Tune IVT multi-socket targets 12950138d8f0SLen Brown * Assumption: num_sockets == (max_package_num + 1) 12960138d8f0SLen Brown */ 1297095928aeSRafael J. Wysocki static void __init ivt_idle_state_table_update(void) 12980138d8f0SLen Brown { 12990138d8f0SLen Brown /* IVT uses a different table for 1-2, 3-4, and > 4 sockets */ 13000138d8f0SLen Brown int cpu, package_num, num_sockets = 1; 13010138d8f0SLen Brown 13020138d8f0SLen Brown for_each_online_cpu(cpu) { 13030138d8f0SLen Brown package_num = topology_physical_package_id(cpu); 13040138d8f0SLen Brown if (package_num + 1 > num_sockets) { 13050138d8f0SLen Brown num_sockets = package_num + 1; 13060138d8f0SLen Brown 1307d27dca42SChristoph Jaeger if (num_sockets > 4) { 13080138d8f0SLen Brown cpuidle_state_table = ivt_cstates_8s; 13090138d8f0SLen Brown return; 13100138d8f0SLen Brown } 13110138d8f0SLen Brown } 1312d27dca42SChristoph Jaeger } 13130138d8f0SLen Brown 13140138d8f0SLen Brown if (num_sockets > 2) 13150138d8f0SLen Brown cpuidle_state_table = ivt_cstates_4s; 1316d70e28f5SLen Brown 13170138d8f0SLen Brown /* else, 1 and 2 socket systems use default ivt_cstates */ 13180138d8f0SLen Brown } 13195dcef694SLen Brown 132086e9466aSRafael J. Wysocki /** 132186e9466aSRafael J. Wysocki * irtl_2_usec - IRTL to microseconds conversion. 132286e9466aSRafael J. Wysocki * @irtl: IRTL MSR value. 132386e9466aSRafael J. Wysocki * 132486e9466aSRafael J. Wysocki * Translate the IRTL (Interrupt Response Time Limit) MSR value to microseconds. 13255dcef694SLen Brown */ 132686e9466aSRafael J. Wysocki static unsigned long long __init irtl_2_usec(unsigned long long irtl) 132786e9466aSRafael J. Wysocki { 1328095928aeSRafael J. Wysocki static const unsigned int irtl_ns_units[] __initconst = { 1329095928aeSRafael J. Wysocki 1, 32, 1024, 32768, 1048576, 33554432, 0, 0 1330095928aeSRafael J. Wysocki }; 13315dcef694SLen Brown unsigned long long ns; 13325dcef694SLen Brown 13333451ab3eSJan Beulich if (!irtl) 13343451ab3eSJan Beulich return 0; 13353451ab3eSJan Beulich 1336bef45096SJan Beulich ns = irtl_ns_units[(irtl >> 10) & 0x7]; 13375dcef694SLen Brown 133886e9466aSRafael J. Wysocki return div_u64((irtl & 0x3FF) * ns, NSEC_PER_USEC); 13395dcef694SLen Brown } 134086e9466aSRafael J. Wysocki 13415dcef694SLen Brown /* 13425dcef694SLen Brown * bxt_idle_state_table_update(void) 13435dcef694SLen Brown * 13445dcef694SLen Brown * On BXT, we trust the IRTL to show the definitive maximum latency 13455dcef694SLen Brown * We use the same value for target_residency. 13465dcef694SLen Brown */ 1347095928aeSRafael J. Wysocki static void __init bxt_idle_state_table_update(void) 13485dcef694SLen Brown { 13495dcef694SLen Brown unsigned long long msr; 13503451ab3eSJan Beulich unsigned int usec; 13515dcef694SLen Brown 13525dcef694SLen Brown rdmsrl(MSR_PKGC6_IRTL, msr); 13533451ab3eSJan Beulich usec = irtl_2_usec(msr); 13543451ab3eSJan Beulich if (usec) { 13555dcef694SLen Brown bxt_cstates[2].exit_latency = usec; 13565dcef694SLen Brown bxt_cstates[2].target_residency = usec; 13575dcef694SLen Brown } 13585dcef694SLen Brown 13595dcef694SLen Brown rdmsrl(MSR_PKGC7_IRTL, msr); 13603451ab3eSJan Beulich usec = irtl_2_usec(msr); 13613451ab3eSJan Beulich if (usec) { 13625dcef694SLen Brown bxt_cstates[3].exit_latency = usec; 13635dcef694SLen Brown bxt_cstates[3].target_residency = usec; 13645dcef694SLen Brown } 13655dcef694SLen Brown 13665dcef694SLen Brown rdmsrl(MSR_PKGC8_IRTL, msr); 13673451ab3eSJan Beulich usec = irtl_2_usec(msr); 13683451ab3eSJan Beulich if (usec) { 13695dcef694SLen Brown bxt_cstates[4].exit_latency = usec; 13705dcef694SLen Brown bxt_cstates[4].target_residency = usec; 13715dcef694SLen Brown } 13725dcef694SLen Brown 13735dcef694SLen Brown rdmsrl(MSR_PKGC9_IRTL, msr); 13743451ab3eSJan Beulich usec = irtl_2_usec(msr); 13753451ab3eSJan Beulich if (usec) { 13765dcef694SLen Brown bxt_cstates[5].exit_latency = usec; 13775dcef694SLen Brown bxt_cstates[5].target_residency = usec; 13785dcef694SLen Brown } 13795dcef694SLen Brown 13805dcef694SLen Brown rdmsrl(MSR_PKGC10_IRTL, msr); 13813451ab3eSJan Beulich usec = irtl_2_usec(msr); 13823451ab3eSJan Beulich if (usec) { 13835dcef694SLen Brown bxt_cstates[6].exit_latency = usec; 13845dcef694SLen Brown bxt_cstates[6].target_residency = usec; 13855dcef694SLen Brown } 13865dcef694SLen Brown 13875dcef694SLen Brown } 1388d70e28f5SLen Brown /* 1389d70e28f5SLen Brown * sklh_idle_state_table_update(void) 1390d70e28f5SLen Brown * 1391d70e28f5SLen Brown * On SKL-H (model 0x5e) disable C8 and C9 if: 1392d70e28f5SLen Brown * C10 is enabled and SGX disabled 1393d70e28f5SLen Brown */ 1394095928aeSRafael J. Wysocki static void __init sklh_idle_state_table_update(void) 1395d70e28f5SLen Brown { 1396d70e28f5SLen Brown unsigned long long msr; 1397d70e28f5SLen Brown unsigned int eax, ebx, ecx, edx; 1398d70e28f5SLen Brown 1399d70e28f5SLen Brown 1400d70e28f5SLen Brown /* if PC10 disabled via cmdline intel_idle.max_cstate=7 or shallower */ 1401d70e28f5SLen Brown if (max_cstate <= 7) 14020138d8f0SLen Brown return; 1403d70e28f5SLen Brown 1404d70e28f5SLen Brown /* if PC10 not present in CPUID.MWAIT.EDX */ 1405d70e28f5SLen Brown if ((mwait_substates & (0xF << 28)) == 0) 1406d70e28f5SLen Brown return; 1407d70e28f5SLen Brown 14086cfb2374SLen Brown rdmsrl(MSR_PKG_CST_CONFIG_CONTROL, msr); 1409d70e28f5SLen Brown 1410d70e28f5SLen Brown /* PC10 is not enabled in PKG C-state limit */ 1411d70e28f5SLen Brown if ((msr & 0xF) != 8) 1412d70e28f5SLen Brown return; 1413d70e28f5SLen Brown 1414d70e28f5SLen Brown ecx = 0; 1415d70e28f5SLen Brown cpuid(7, &eax, &ebx, &ecx, &edx); 1416d70e28f5SLen Brown 1417d70e28f5SLen Brown /* if SGX is present */ 1418d70e28f5SLen Brown if (ebx & (1 << 2)) { 1419d70e28f5SLen Brown 142032ad73dbSSean Christopherson rdmsrl(MSR_IA32_FEAT_CTL, msr); 1421d70e28f5SLen Brown 1422d70e28f5SLen Brown /* if SGX is enabled */ 1423d70e28f5SLen Brown if (msr & (1 << 18)) 1424d70e28f5SLen Brown return; 1425d70e28f5SLen Brown } 1426d70e28f5SLen Brown 1427ba1e78a1SRafael J. Wysocki skl_cstates[5].flags |= CPUIDLE_FLAG_UNUSABLE; /* C8-SKL */ 1428ba1e78a1SRafael J. Wysocki skl_cstates[6].flags |= CPUIDLE_FLAG_UNUSABLE; /* C9-SKL */ 1429d70e28f5SLen Brown } 1430d70e28f5SLen Brown 14311aefbd7aSRafael J. Wysocki static bool __init intel_idle_verify_cstate(unsigned int mwait_hint) 14321aefbd7aSRafael J. Wysocki { 14331aefbd7aSRafael J. Wysocki unsigned int mwait_cstate = MWAIT_HINT2CSTATE(mwait_hint) + 1; 14341aefbd7aSRafael J. Wysocki unsigned int num_substates = (mwait_substates >> mwait_cstate * 4) & 14351aefbd7aSRafael J. Wysocki MWAIT_SUBSTATE_MASK; 14361aefbd7aSRafael J. Wysocki 14371aefbd7aSRafael J. Wysocki /* Ignore the C-state if there are NO sub-states in CPUID for it. */ 14381aefbd7aSRafael J. Wysocki if (num_substates == 0) 14391aefbd7aSRafael J. Wysocki return false; 14401aefbd7aSRafael J. Wysocki 14411aefbd7aSRafael J. Wysocki if (mwait_cstate > 2 && !boot_cpu_has(X86_FEATURE_NONSTOP_TSC)) 14421aefbd7aSRafael J. Wysocki mark_tsc_unstable("TSC halts in idle states deeper than C2"); 14431aefbd7aSRafael J. Wysocki 14441aefbd7aSRafael J. Wysocki return true; 14451aefbd7aSRafael J. Wysocki } 14461aefbd7aSRafael J. Wysocki 1447095928aeSRafael J. Wysocki static void __init intel_idle_init_cstates_icpu(struct cpuidle_driver *drv) 1448d70e28f5SLen Brown { 14493d3a1ae9SRafael J. Wysocki int cstate; 1450d70e28f5SLen Brown 14513d3a1ae9SRafael J. Wysocki switch (boot_cpu_data.x86_model) { 1452db73c5a8SDave Hansen case INTEL_FAM6_IVYBRIDGE_X: 1453d70e28f5SLen Brown ivt_idle_state_table_update(); 1454d70e28f5SLen Brown break; 1455db73c5a8SDave Hansen case INTEL_FAM6_ATOM_GOLDMONT: 1456f2c4db1bSPeter Zijlstra case INTEL_FAM6_ATOM_GOLDMONT_PLUS: 14575dcef694SLen Brown bxt_idle_state_table_update(); 14585dcef694SLen Brown break; 1459c66f78a6SPeter Zijlstra case INTEL_FAM6_SKYLAKE: 1460d70e28f5SLen Brown sklh_idle_state_table_update(); 1461d70e28f5SLen Brown break; 1462d70e28f5SLen Brown } 146346bcfad7SDeepthi Dharwar 1464e022e7ebSLen Brown for (cstate = 0; cstate < CPUIDLE_STATE_MAX; ++cstate) { 14659f3d6dafSRafael J. Wysocki unsigned int mwait_hint; 146646bcfad7SDeepthi Dharwar 146718734958SRafael J. Wysocki if (intel_idle_max_cstate_reached(cstate)) 146818734958SRafael J. Wysocki break; 146918734958SRafael J. Wysocki 14709f3d6dafSRafael J. Wysocki if (!cpuidle_state_table[cstate].enter && 14719f3d6dafSRafael J. Wysocki !cpuidle_state_table[cstate].enter_s2idle) 1472e022e7ebSLen Brown break; 1473e022e7ebSLen Brown 14749f3d6dafSRafael J. Wysocki /* If marked as unusable, skip this state. */ 1475ba1e78a1SRafael J. Wysocki if (cpuidle_state_table[cstate].flags & CPUIDLE_FLAG_UNUSABLE) { 1476654d08a4SJoe Perches pr_debug("state %s is disabled\n", 1477d70e28f5SLen Brown cpuidle_state_table[cstate].name); 1478d70e28f5SLen Brown continue; 1479d70e28f5SLen Brown } 1480d70e28f5SLen Brown 14819f3d6dafSRafael J. Wysocki mwait_hint = flg2MWAIT(cpuidle_state_table[cstate].flags); 14829f3d6dafSRafael J. Wysocki if (!intel_idle_verify_cstate(mwait_hint)) 14839f3d6dafSRafael J. Wysocki continue; 1484d70e28f5SLen Brown 14859f3d6dafSRafael J. Wysocki /* Structure copy. */ 1486bff8e60aSRafael J. Wysocki drv->states[drv->state_count] = cpuidle_state_table[cstate]; 1487bff8e60aSRafael J. Wysocki 14884dcb78eeSRafael J. Wysocki if ((disabled_states_mask & BIT(drv->state_count)) || 14894dcb78eeSRafael J. Wysocki ((icpu->use_acpi || force_use_acpi) && 14903a5be9b8SRafael J. Wysocki intel_idle_off_by_default(mwait_hint) && 14914dcb78eeSRafael J. Wysocki !(cpuidle_state_table[cstate].flags & CPUIDLE_FLAG_ALWAYS_ENABLE))) 1492bff8e60aSRafael J. Wysocki drv->states[drv->state_count].flags |= CPUIDLE_FLAG_OFF; 1493bff8e60aSRafael J. Wysocki 1494bff8e60aSRafael J. Wysocki drv->state_count++; 149546bcfad7SDeepthi Dharwar } 149646bcfad7SDeepthi Dharwar 14978c058d53SLen Brown if (icpu->byt_auto_demotion_disable_flag) { 14988c058d53SLen Brown wrmsrl(MSR_CC6_DEMOTION_POLICY_CONFIG, 0); 14998c058d53SLen Brown wrmsrl(MSR_MC6_DEMOTION_POLICY_CONFIG, 0); 15008c058d53SLen Brown } 150146bcfad7SDeepthi Dharwar } 150246bcfad7SDeepthi Dharwar 150318734958SRafael J. Wysocki /* 150418734958SRafael J. Wysocki * intel_idle_cpuidle_driver_init() 150518734958SRafael J. Wysocki * allocate, initialize cpuidle_states 150618734958SRafael J. Wysocki */ 15073d3a1ae9SRafael J. Wysocki static void __init intel_idle_cpuidle_driver_init(struct cpuidle_driver *drv) 150818734958SRafael J. Wysocki { 150918734958SRafael J. Wysocki cpuidle_poll_state_init(drv); 15104dcb78eeSRafael J. Wysocki 15114dcb78eeSRafael J. Wysocki if (disabled_states_mask & BIT(0)) 15124dcb78eeSRafael J. Wysocki drv->states[0].flags |= CPUIDLE_FLAG_OFF; 15134dcb78eeSRafael J. Wysocki 151418734958SRafael J. Wysocki drv->state_count = 1; 151518734958SRafael J. Wysocki 151618734958SRafael J. Wysocki if (icpu) 151718734958SRafael J. Wysocki intel_idle_init_cstates_icpu(drv); 151818734958SRafael J. Wysocki else 151918734958SRafael J. Wysocki intel_idle_init_cstates_acpi(drv); 152018734958SRafael J. Wysocki } 152146bcfad7SDeepthi Dharwar 15221aefbd7aSRafael J. Wysocki static void auto_demotion_disable(void) 15231aefbd7aSRafael J. Wysocki { 15241aefbd7aSRafael J. Wysocki unsigned long long msr_bits; 15251aefbd7aSRafael J. Wysocki 15261aefbd7aSRafael J. Wysocki rdmsrl(MSR_PKG_CST_CONFIG_CONTROL, msr_bits); 15277f843dd7SRafael J. Wysocki msr_bits &= ~auto_demotion_disable_flags; 15281aefbd7aSRafael J. Wysocki wrmsrl(MSR_PKG_CST_CONFIG_CONTROL, msr_bits); 15291aefbd7aSRafael J. Wysocki } 15301aefbd7aSRafael J. Wysocki 15311aefbd7aSRafael J. Wysocki static void c1e_promotion_disable(void) 15321aefbd7aSRafael J. Wysocki { 15331aefbd7aSRafael J. Wysocki unsigned long long msr_bits; 15341aefbd7aSRafael J. Wysocki 15351aefbd7aSRafael J. Wysocki rdmsrl(MSR_IA32_POWER_CTL, msr_bits); 15361aefbd7aSRafael J. Wysocki msr_bits &= ~0x2; 15371aefbd7aSRafael J. Wysocki wrmsrl(MSR_IA32_POWER_CTL, msr_bits); 15381aefbd7aSRafael J. Wysocki } 15391aefbd7aSRafael J. Wysocki 154046bcfad7SDeepthi Dharwar /* 154165b7f839SThomas Renninger * intel_idle_cpu_init() 154226717172SLen Brown * allocate, initialize, register cpuidle_devices 154365b7f839SThomas Renninger * @cpu: cpu/core to initialize 154426717172SLen Brown */ 1545fb1013a0SSebastian Andrzej Siewior static int intel_idle_cpu_init(unsigned int cpu) 154626717172SLen Brown { 154726717172SLen Brown struct cpuidle_device *dev; 154826717172SLen Brown 154965b7f839SThomas Renninger dev = per_cpu_ptr(intel_idle_cpuidle_devices, cpu); 155065b7f839SThomas Renninger dev->cpu = cpu; 155126717172SLen Brown 155226717172SLen Brown if (cpuidle_register_device(dev)) { 1553654d08a4SJoe Perches pr_debug("cpuidle_register_device %d failed!\n", cpu); 155426717172SLen Brown return -EIO; 155526717172SLen Brown } 155665b7f839SThomas Renninger 15577f843dd7SRafael J. Wysocki if (auto_demotion_disable_flags) 1558fb1013a0SSebastian Andrzej Siewior auto_demotion_disable(); 155926717172SLen Brown 15607f843dd7SRafael J. Wysocki if (disable_promotion_to_c1e) 1561fb1013a0SSebastian Andrzej Siewior c1e_promotion_disable(); 1562fb1013a0SSebastian Andrzej Siewior 1563fb1013a0SSebastian Andrzej Siewior return 0; 1564fb1013a0SSebastian Andrzej Siewior } 1565fb1013a0SSebastian Andrzej Siewior 1566fb1013a0SSebastian Andrzej Siewior static int intel_idle_cpu_online(unsigned int cpu) 1567fb1013a0SSebastian Andrzej Siewior { 1568fb1013a0SSebastian Andrzej Siewior struct cpuidle_device *dev; 1569fb1013a0SSebastian Andrzej Siewior 157040ab82e0SRafael J. Wysocki if (!lapic_timer_always_reliable) 1571cbd2c4c2SRafael J. Wysocki tick_broadcast_enable(); 1572fb1013a0SSebastian Andrzej Siewior 1573fb1013a0SSebastian Andrzej Siewior /* 1574fb1013a0SSebastian Andrzej Siewior * Some systems can hotplug a cpu at runtime after 1575fb1013a0SSebastian Andrzej Siewior * the kernel has booted, we have to initialize the 1576fb1013a0SSebastian Andrzej Siewior * driver in this case 1577fb1013a0SSebastian Andrzej Siewior */ 1578fb1013a0SSebastian Andrzej Siewior dev = per_cpu_ptr(intel_idle_cpuidle_devices, cpu); 1579fb1013a0SSebastian Andrzej Siewior if (!dev->registered) 1580fb1013a0SSebastian Andrzej Siewior return intel_idle_cpu_init(cpu); 1581dbf87ab8SBartlomiej Zolnierkiewicz 158226717172SLen Brown return 0; 158326717172SLen Brown } 158426717172SLen Brown 15850755a9bdSRafael J. Wysocki /** 15860755a9bdSRafael J. Wysocki * intel_idle_cpuidle_devices_uninit - Unregister all cpuidle devices. 15870755a9bdSRafael J. Wysocki */ 15880755a9bdSRafael J. Wysocki static void __init intel_idle_cpuidle_devices_uninit(void) 15890755a9bdSRafael J. Wysocki { 15900755a9bdSRafael J. Wysocki int i; 15910755a9bdSRafael J. Wysocki 15920755a9bdSRafael J. Wysocki for_each_online_cpu(i) 15930755a9bdSRafael J. Wysocki cpuidle_unregister_device(per_cpu_ptr(intel_idle_cpuidle_devices, i)); 15940755a9bdSRafael J. Wysocki } 15950755a9bdSRafael J. Wysocki 159626717172SLen Brown static int __init intel_idle_init(void) 159726717172SLen Brown { 1598a6c86e33SRafael J. Wysocki const struct x86_cpu_id *id; 1599a6c86e33SRafael J. Wysocki unsigned int eax, ebx, ecx; 1600fb1013a0SSebastian Andrzej Siewior int retval; 160126717172SLen Brown 1602d1896049SThomas Renninger /* Do not load intel_idle at all for now if idle= is passed */ 1603d1896049SThomas Renninger if (boot_option_idle_override != IDLE_NO_OVERRIDE) 1604d1896049SThomas Renninger return -ENODEV; 1605d1896049SThomas Renninger 1606a6c86e33SRafael J. Wysocki if (max_cstate == 0) { 1607a6c86e33SRafael J. Wysocki pr_debug("disabled\n"); 1608a6c86e33SRafael J. Wysocki return -EPERM; 1609a6c86e33SRafael J. Wysocki } 1610a6c86e33SRafael J. Wysocki 1611a6c86e33SRafael J. Wysocki id = x86_match_cpu(intel_idle_ids); 1612a6c86e33SRafael J. Wysocki if (id) { 1613a6c86e33SRafael J. Wysocki if (!boot_cpu_has(X86_FEATURE_MWAIT)) { 1614a6c86e33SRafael J. Wysocki pr_debug("Please enable MWAIT in BIOS SETUP\n"); 1615a6c86e33SRafael J. Wysocki return -ENODEV; 1616a6c86e33SRafael J. Wysocki } 1617a6c86e33SRafael J. Wysocki } else { 1618a6c86e33SRafael J. Wysocki id = x86_match_cpu(intel_mwait_ids); 1619a6c86e33SRafael J. Wysocki if (!id) 1620a6c86e33SRafael J. Wysocki return -ENODEV; 1621a6c86e33SRafael J. Wysocki } 1622a6c86e33SRafael J. Wysocki 1623a6c86e33SRafael J. Wysocki if (boot_cpu_data.cpuid_level < CPUID_MWAIT_LEAF) 1624a6c86e33SRafael J. Wysocki return -ENODEV; 1625a6c86e33SRafael J. Wysocki 1626a6c86e33SRafael J. Wysocki cpuid(CPUID_MWAIT_LEAF, &eax, &ebx, &ecx, &mwait_substates); 1627a6c86e33SRafael J. Wysocki 1628a6c86e33SRafael J. Wysocki if (!(ecx & CPUID5_ECX_EXTENSIONS_SUPPORTED) || 1629a6c86e33SRafael J. Wysocki !(ecx & CPUID5_ECX_INTERRUPT_BREAK) || 1630a6c86e33SRafael J. Wysocki !mwait_substates) 1631a6c86e33SRafael J. Wysocki return -ENODEV; 1632a6c86e33SRafael J. Wysocki 1633a6c86e33SRafael J. Wysocki pr_debug("MWAIT substates: 0x%x\n", mwait_substates); 1634a6c86e33SRafael J. Wysocki 1635a6c86e33SRafael J. Wysocki icpu = (const struct idle_cpu *)id->driver_data; 1636a6c86e33SRafael J. Wysocki if (icpu) { 1637a6c86e33SRafael J. Wysocki cpuidle_state_table = icpu->state_table; 16387f843dd7SRafael J. Wysocki auto_demotion_disable_flags = icpu->auto_demotion_disable_flags; 16397f843dd7SRafael J. Wysocki disable_promotion_to_c1e = icpu->disable_promotion_to_c1e; 16403a5be9b8SRafael J. Wysocki if (icpu->use_acpi || force_use_acpi) 1641a6c86e33SRafael J. Wysocki intel_idle_acpi_cst_extract(); 1642a6c86e33SRafael J. Wysocki } else if (!intel_idle_acpi_cst_extract()) { 1643a6c86e33SRafael J. Wysocki return -ENODEV; 1644a6c86e33SRafael J. Wysocki } 1645a6c86e33SRafael J. Wysocki 1646a6c86e33SRafael J. Wysocki pr_debug("v" INTEL_IDLE_VERSION " model 0x%X\n", 1647a6c86e33SRafael J. Wysocki boot_cpu_data.x86_model); 164826717172SLen Brown 1649e9df69ccSRichard Cochran intel_idle_cpuidle_devices = alloc_percpu(struct cpuidle_device); 1650533da74aSRafael J. Wysocki if (!intel_idle_cpuidle_devices) 1651e9df69ccSRichard Cochran return -ENOMEM; 1652e9df69ccSRichard Cochran 16533d3a1ae9SRafael J. Wysocki intel_idle_cpuidle_driver_init(&intel_idle_driver); 16543d3a1ae9SRafael J. Wysocki 165526717172SLen Brown retval = cpuidle_register_driver(&intel_idle_driver); 165626717172SLen Brown if (retval) { 16573735d524SKonrad Rzeszutek Wilk struct cpuidle_driver *drv = cpuidle_get_driver(); 1658654d08a4SJoe Perches printk(KERN_DEBUG pr_fmt("intel_idle yielding to %s\n"), 16593735d524SKonrad Rzeszutek Wilk drv ? drv->name : "none"); 1660fb1013a0SSebastian Andrzej Siewior goto init_driver_fail; 166126717172SLen Brown } 166226717172SLen Brown 16632259a819SRichard Cochran if (boot_cpu_has(X86_FEATURE_ARAT)) /* Always Reliable APIC Timer */ 166440ab82e0SRafael J. Wysocki lapic_timer_always_reliable = true; 16652259a819SRichard Cochran 1666fb1013a0SSebastian Andrzej Siewior retval = cpuhp_setup_state(CPUHP_AP_ONLINE_DYN, "idle/intel:online", 1667fb1013a0SSebastian Andrzej Siewior intel_idle_cpu_online, NULL); 1668fb1013a0SSebastian Andrzej Siewior if (retval < 0) 1669fb1013a0SSebastian Andrzej Siewior goto hp_setup_fail; 167026717172SLen Brown 167140ab82e0SRafael J. Wysocki pr_debug("Local APIC timer is reliable in %s\n", 167240ab82e0SRafael J. Wysocki lapic_timer_always_reliable ? "all C-states" : "C1"); 16732259a819SRichard Cochran 167426717172SLen Brown return 0; 1675fb1013a0SSebastian Andrzej Siewior 1676fb1013a0SSebastian Andrzej Siewior hp_setup_fail: 1677fb1013a0SSebastian Andrzej Siewior intel_idle_cpuidle_devices_uninit(); 1678fb1013a0SSebastian Andrzej Siewior cpuidle_unregister_driver(&intel_idle_driver); 1679fb1013a0SSebastian Andrzej Siewior init_driver_fail: 1680fb1013a0SSebastian Andrzej Siewior free_percpu(intel_idle_cpuidle_devices); 1681fb1013a0SSebastian Andrzej Siewior return retval; 1682fb1013a0SSebastian Andrzej Siewior 168326717172SLen Brown } 168402c4fae9SPaul Gortmaker device_initcall(intel_idle_init); 168526717172SLen Brown 168602c4fae9SPaul Gortmaker /* 168702c4fae9SPaul Gortmaker * We are not really modular, but we used to support that. Meaning we also 168802c4fae9SPaul Gortmaker * support "intel_idle.max_cstate=..." at boot and also a read-only export of 168902c4fae9SPaul Gortmaker * it at /sys/module/intel_idle/parameters/max_cstate -- so using module_param 169002c4fae9SPaul Gortmaker * is the easiest way (currently) to continue doing that. 169102c4fae9SPaul Gortmaker */ 169226717172SLen Brown module_param(max_cstate, int, 0444); 16934dcb78eeSRafael J. Wysocki /* 16944dcb78eeSRafael J. Wysocki * The positions of the bits that are set in this number are the indices of the 16954dcb78eeSRafael J. Wysocki * idle states to be disabled by default (as reflected by the names of the 16964dcb78eeSRafael J. Wysocki * corresponding idle state directories in sysfs, "state0", "state1" ... 16974dcb78eeSRafael J. Wysocki * "state<i>" ..., where <i> is the index of the given state). 16984dcb78eeSRafael J. Wysocki */ 16994dcb78eeSRafael J. Wysocki module_param_named(states_off, disabled_states_mask, uint, 0444); 17004dcb78eeSRafael J. Wysocki MODULE_PARM_DESC(states_off, "Mask of disabled idle states"); 1701