1a61127c2SThomas Gleixner // SPDX-License-Identifier: GPL-2.0-only 226717172SLen Brown /* 326717172SLen Brown * intel_idle.c - native hardware idle loop for modern Intel processors 426717172SLen Brown * 5317e5ec3SRafael J. Wysocki * Copyright (c) 2013 - 2020, Intel Corporation. 626717172SLen Brown * Len Brown <len.brown@intel.com> 7317e5ec3SRafael J. Wysocki * Rafael J. Wysocki <rafael.j.wysocki@intel.com> 826717172SLen Brown */ 926717172SLen Brown 1026717172SLen Brown /* 1126717172SLen Brown * intel_idle is a cpuidle driver that loads on specific Intel processors 1226717172SLen Brown * in lieu of the legacy ACPI processor_idle driver. The intent is to 1326717172SLen Brown * make Linux more efficient on these processors, as intel_idle knows 1426717172SLen Brown * more than ACPI, as well as make Linux more immune to ACPI BIOS bugs. 1526717172SLen Brown */ 1626717172SLen Brown 1726717172SLen Brown /* 1826717172SLen Brown * Design Assumptions 1926717172SLen Brown * 2026717172SLen Brown * All CPUs have same idle states as boot CPU 2126717172SLen Brown * 2226717172SLen Brown * Chipset BM_STS (bus master status) bit is a NOP 2326717172SLen Brown * for preventing entry into deep C-stats 2426717172SLen Brown */ 2526717172SLen Brown 2626717172SLen Brown /* 2726717172SLen Brown * Known limitations 2826717172SLen Brown * 2926717172SLen Brown * ACPI has a .suspend hack to turn off deep c-statees during suspend 3026717172SLen Brown * to avoid complications with the lapic timer workaround. 3126717172SLen Brown * Have not seen issues with suspend, but may need same workaround here. 3226717172SLen Brown * 3326717172SLen Brown */ 3426717172SLen Brown 3526717172SLen Brown /* un-comment DEBUG to enable pr_debug() statements */ 3626717172SLen Brown #define DEBUG 3726717172SLen Brown 38654d08a4SJoe Perches #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt 39654d08a4SJoe Perches 4018734958SRafael J. Wysocki #include <linux/acpi.h> 4126717172SLen Brown #include <linux/kernel.h> 4226717172SLen Brown #include <linux/cpuidle.h> 4376962caaSThomas Gleixner #include <linux/tick.h> 4426717172SLen Brown #include <trace/events/power.h> 4526717172SLen Brown #include <linux/sched.h> 462a2d31c8SShaohua Li #include <linux/notifier.h> 472a2d31c8SShaohua Li #include <linux/cpu.h> 4802c4fae9SPaul Gortmaker #include <linux/moduleparam.h> 49b66b8b9aSAndi Kleen #include <asm/cpu_device_id.h> 50db73c5a8SDave Hansen #include <asm/intel-family.h> 51bc83ccccSH. Peter Anvin #include <asm/mwait.h> 5214796fcaSLen Brown #include <asm/msr.h> 5326717172SLen Brown 54317e5ec3SRafael J. Wysocki #define INTEL_IDLE_VERSION "0.5.1" 5526717172SLen Brown 5626717172SLen Brown static struct cpuidle_driver intel_idle_driver = { 5726717172SLen Brown .name = "intel_idle", 5826717172SLen Brown .owner = THIS_MODULE, 5926717172SLen Brown }; 6026717172SLen Brown /* intel_idle.max_cstate=0 disables driver */ 61137ecc77SLen Brown static int max_cstate = CPUIDLE_STATE_MAX - 1; 624dcb78eeSRafael J. Wysocki static unsigned int disabled_states_mask; 6326717172SLen Brown 646eb0443aSRafael J. Wysocki static struct cpuidle_device __percpu *intel_idle_cpuidle_devices; 657f843dd7SRafael J. Wysocki 667f843dd7SRafael J. Wysocki static unsigned long auto_demotion_disable_flags; 677f843dd7SRafael J. Wysocki static bool disable_promotion_to_c1e; 687f843dd7SRafael J. Wysocki 69b66b8b9aSAndi Kleen struct idle_cpu { 70b66b8b9aSAndi Kleen struct cpuidle_state *state_table; 7126717172SLen Brown 7226717172SLen Brown /* 7314796fcaSLen Brown * Hardware C-state auto-demotion may not always be optimal. 7414796fcaSLen Brown * Indicate which enable bits to clear here. 7514796fcaSLen Brown */ 76b66b8b9aSAndi Kleen unsigned long auto_demotion_disable_flags; 778c058d53SLen Brown bool byt_auto_demotion_disable_flag; 7832e95180SLen Brown bool disable_promotion_to_c1e; 79bff8e60aSRafael J. Wysocki bool use_acpi; 80b66b8b9aSAndi Kleen }; 81b66b8b9aSAndi Kleen 827f843dd7SRafael J. Wysocki static const struct idle_cpu *icpu __initdata; 837f843dd7SRafael J. Wysocki static struct cpuidle_state *cpuidle_state_table __initdata; 8414796fcaSLen Brown 856eb0443aSRafael J. Wysocki static unsigned int mwait_substates __initdata; 866eb0443aSRafael J. Wysocki 8714796fcaSLen Brown /* 88bff8e60aSRafael J. Wysocki * Enable this state by default even if the ACPI _CST does not list it. 89bff8e60aSRafael J. Wysocki */ 90bff8e60aSRafael J. Wysocki #define CPUIDLE_FLAG_ALWAYS_ENABLE BIT(15) 91bff8e60aSRafael J. Wysocki 92bff8e60aSRafael J. Wysocki /* 93956d033fSLen Brown * Set this flag for states where the HW flushes the TLB for us 94956d033fSLen Brown * and so we don't need cross-calls to keep it consistent. 95956d033fSLen Brown * If this flag is set, SW flushes the TLB, so even if the 96956d033fSLen Brown * HW doesn't do the flushing, this flag is safe to use. 97956d033fSLen Brown */ 98a472e4b5SRafael J. Wysocki #define CPUIDLE_FLAG_TLB_FLUSHED BIT(16) 99956d033fSLen Brown 100956d033fSLen Brown /* 101b1beab48SLen Brown * MWAIT takes an 8-bit "hint" in EAX "suggesting" 102b1beab48SLen Brown * the C-state (top nibble) and sub-state (bottom nibble) 103b1beab48SLen Brown * 0x00 means "MWAIT(C1)", 0x10 means "MWAIT(C2)" etc. 104b1beab48SLen Brown * 105b1beab48SLen Brown * We store the hint at the top of our "flags" for each state. 106b1beab48SLen Brown */ 107b1beab48SLen Brown #define flg2MWAIT(flags) (((flags) >> 24) & 0xFF) 108b1beab48SLen Brown #define MWAIT2flg(eax) ((eax & 0xFF) << 24) 109b1beab48SLen Brown 11030a996fbSRafael J. Wysocki /** 11130a996fbSRafael J. Wysocki * intel_idle - Ask the processor to enter the given idle state. 11230a996fbSRafael J. Wysocki * @dev: cpuidle device of the target CPU. 11330a996fbSRafael J. Wysocki * @drv: cpuidle driver (assumed to point to intel_idle_driver). 11430a996fbSRafael J. Wysocki * @index: Target idle state index. 11530a996fbSRafael J. Wysocki * 11630a996fbSRafael J. Wysocki * Use the MWAIT instruction to notify the processor that the CPU represented by 11730a996fbSRafael J. Wysocki * @dev is idle and it can try to enter the idle state corresponding to @index. 11830a996fbSRafael J. Wysocki * 11930a996fbSRafael J. Wysocki * If the local APIC timer is not known to be reliable in the target idle state, 12030a996fbSRafael J. Wysocki * enable one-shot tick broadcasting for the target CPU before executing MWAIT. 12130a996fbSRafael J. Wysocki * 12230a996fbSRafael J. Wysocki * Optionally call leave_mm() for the target CPU upfront to avoid wakeups due to 12330a996fbSRafael J. Wysocki * flushing user TLBs. 12430a996fbSRafael J. Wysocki * 12530a996fbSRafael J. Wysocki * Must be called under local_irq_disable(). 12630a996fbSRafael J. Wysocki */ 12730a996fbSRafael J. Wysocki static __cpuidle int intel_idle(struct cpuidle_device *dev, 12830a996fbSRafael J. Wysocki struct cpuidle_driver *drv, int index) 12930a996fbSRafael J. Wysocki { 13030a996fbSRafael J. Wysocki struct cpuidle_state *state = &drv->states[index]; 13130a996fbSRafael J. Wysocki unsigned long eax = flg2MWAIT(state->flags); 13230a996fbSRafael J. Wysocki unsigned long ecx = 1; /* break on interrupt flag */ 13330a996fbSRafael J. Wysocki bool uninitialized_var(tick); 13430a996fbSRafael J. Wysocki int cpu = smp_processor_id(); 13530a996fbSRafael J. Wysocki 13630a996fbSRafael J. Wysocki /* 13730a996fbSRafael J. Wysocki * leave_mm() to avoid costly and often unnecessary wakeups 13830a996fbSRafael J. Wysocki * for flushing the user TLB's associated with the active mm. 13930a996fbSRafael J. Wysocki */ 14030a996fbSRafael J. Wysocki if (state->flags & CPUIDLE_FLAG_TLB_FLUSHED) 14130a996fbSRafael J. Wysocki leave_mm(cpu); 14230a996fbSRafael J. Wysocki 143dab20177SRafael J. Wysocki if (!static_cpu_has(X86_FEATURE_ARAT)) { 14430a996fbSRafael J. Wysocki /* 14530a996fbSRafael J. Wysocki * Switch over to one-shot tick broadcast if the target C-state 14630a996fbSRafael J. Wysocki * is deeper than C1. 14730a996fbSRafael J. Wysocki */ 14830a996fbSRafael J. Wysocki if ((eax >> MWAIT_SUBSTATE_SIZE) & MWAIT_CSTATE_MASK) { 14930a996fbSRafael J. Wysocki tick = true; 15030a996fbSRafael J. Wysocki tick_broadcast_enter(); 15130a996fbSRafael J. Wysocki } else { 15230a996fbSRafael J. Wysocki tick = false; 15330a996fbSRafael J. Wysocki } 15430a996fbSRafael J. Wysocki } 15530a996fbSRafael J. Wysocki 15630a996fbSRafael J. Wysocki mwait_idle_with_hints(eax, ecx); 15730a996fbSRafael J. Wysocki 15830a996fbSRafael J. Wysocki if (!static_cpu_has(X86_FEATURE_ARAT) && tick) 15930a996fbSRafael J. Wysocki tick_broadcast_exit(); 16030a996fbSRafael J. Wysocki 16130a996fbSRafael J. Wysocki return index; 16230a996fbSRafael J. Wysocki } 16330a996fbSRafael J. Wysocki 16430a996fbSRafael J. Wysocki /** 16530a996fbSRafael J. Wysocki * intel_idle_s2idle - Ask the processor to enter the given idle state. 16630a996fbSRafael J. Wysocki * @dev: cpuidle device of the target CPU. 16730a996fbSRafael J. Wysocki * @drv: cpuidle driver (assumed to point to intel_idle_driver). 16830a996fbSRafael J. Wysocki * @index: Target idle state index. 16930a996fbSRafael J. Wysocki * 17030a996fbSRafael J. Wysocki * Use the MWAIT instruction to notify the processor that the CPU represented by 17130a996fbSRafael J. Wysocki * @dev is idle and it can try to enter the idle state corresponding to @index. 17230a996fbSRafael J. Wysocki * 17330a996fbSRafael J. Wysocki * Invoked as a suspend-to-idle callback routine with frozen user space, frozen 17430a996fbSRafael J. Wysocki * scheduler tick and suspended scheduler clock on the target CPU. 17530a996fbSRafael J. Wysocki */ 176efe97112SNeal Liu static __cpuidle int intel_idle_s2idle(struct cpuidle_device *dev, 17730a996fbSRafael J. Wysocki struct cpuidle_driver *drv, int index) 17830a996fbSRafael J. Wysocki { 17930a996fbSRafael J. Wysocki unsigned long eax = flg2MWAIT(drv->states[index].flags); 18030a996fbSRafael J. Wysocki unsigned long ecx = 1; /* break on interrupt flag */ 18130a996fbSRafael J. Wysocki 18230a996fbSRafael J. Wysocki mwait_idle_with_hints(eax, ecx); 183efe97112SNeal Liu 184efe97112SNeal Liu return 0; 18530a996fbSRafael J. Wysocki } 18630a996fbSRafael J. Wysocki 187b1beab48SLen Brown /* 18826717172SLen Brown * States are indexed by the cstate number, 18926717172SLen Brown * which is also the index into the MWAIT hint array. 19026717172SLen Brown * Thus C0 is a dummy. 19126717172SLen Brown */ 192ab1a8522SRafael J. Wysocki static struct cpuidle_state nehalem_cstates[] __initdata = { 193e022e7ebSLen Brown { 194de09cdd0SLen Brown .name = "C1", 19526717172SLen Brown .desc = "MWAIT 0x00", 196b82b6ccaSDaniel Lezcano .flags = MWAIT2flg(0x00), 19726717172SLen Brown .exit_latency = 3, 19826717172SLen Brown .target_residency = 6, 1995fe2e527SRafael J. Wysocki .enter = &intel_idle, 20028ba086eSRafael J. Wysocki .enter_s2idle = intel_idle_s2idle, }, 201e022e7ebSLen Brown { 202de09cdd0SLen Brown .name = "C1E", 20332e95180SLen Brown .desc = "MWAIT 0x01", 204e6d4f08aSRafael J. Wysocki .flags = MWAIT2flg(0x01) | CPUIDLE_FLAG_ALWAYS_ENABLE, 20532e95180SLen Brown .exit_latency = 10, 20632e95180SLen Brown .target_residency = 20, 2075fe2e527SRafael J. Wysocki .enter = &intel_idle, 20828ba086eSRafael J. Wysocki .enter_s2idle = intel_idle_s2idle, }, 20932e95180SLen Brown { 210de09cdd0SLen Brown .name = "C3", 21126717172SLen Brown .desc = "MWAIT 0x10", 212b82b6ccaSDaniel Lezcano .flags = MWAIT2flg(0x10) | CPUIDLE_FLAG_TLB_FLUSHED, 21326717172SLen Brown .exit_latency = 20, 21426717172SLen Brown .target_residency = 80, 2155fe2e527SRafael J. Wysocki .enter = &intel_idle, 21628ba086eSRafael J. Wysocki .enter_s2idle = intel_idle_s2idle, }, 217e022e7ebSLen Brown { 218de09cdd0SLen Brown .name = "C6", 21926717172SLen Brown .desc = "MWAIT 0x20", 220b82b6ccaSDaniel Lezcano .flags = MWAIT2flg(0x20) | CPUIDLE_FLAG_TLB_FLUSHED, 22126717172SLen Brown .exit_latency = 200, 22226717172SLen Brown .target_residency = 800, 2235fe2e527SRafael J. Wysocki .enter = &intel_idle, 22428ba086eSRafael J. Wysocki .enter_s2idle = intel_idle_s2idle, }, 225e022e7ebSLen Brown { 226e022e7ebSLen Brown .enter = NULL } 22726717172SLen Brown }; 22826717172SLen Brown 229ab1a8522SRafael J. Wysocki static struct cpuidle_state snb_cstates[] __initdata = { 230e022e7ebSLen Brown { 231de09cdd0SLen Brown .name = "C1", 232d13780d4SLen Brown .desc = "MWAIT 0x00", 233b82b6ccaSDaniel Lezcano .flags = MWAIT2flg(0x00), 23432e95180SLen Brown .exit_latency = 2, 23532e95180SLen Brown .target_residency = 2, 2365fe2e527SRafael J. Wysocki .enter = &intel_idle, 23728ba086eSRafael J. Wysocki .enter_s2idle = intel_idle_s2idle, }, 23832e95180SLen Brown { 239de09cdd0SLen Brown .name = "C1E", 24032e95180SLen Brown .desc = "MWAIT 0x01", 241e6d4f08aSRafael J. Wysocki .flags = MWAIT2flg(0x01) | CPUIDLE_FLAG_ALWAYS_ENABLE, 24232e95180SLen Brown .exit_latency = 10, 24332e95180SLen Brown .target_residency = 20, 2445fe2e527SRafael J. Wysocki .enter = &intel_idle, 24528ba086eSRafael J. Wysocki .enter_s2idle = intel_idle_s2idle, }, 246e022e7ebSLen Brown { 247de09cdd0SLen Brown .name = "C3", 248d13780d4SLen Brown .desc = "MWAIT 0x10", 249b82b6ccaSDaniel Lezcano .flags = MWAIT2flg(0x10) | CPUIDLE_FLAG_TLB_FLUSHED, 250d13780d4SLen Brown .exit_latency = 80, 251ddbd550dSLen Brown .target_residency = 211, 2525fe2e527SRafael J. Wysocki .enter = &intel_idle, 25328ba086eSRafael J. Wysocki .enter_s2idle = intel_idle_s2idle, }, 254e022e7ebSLen Brown { 255de09cdd0SLen Brown .name = "C6", 256d13780d4SLen Brown .desc = "MWAIT 0x20", 257b82b6ccaSDaniel Lezcano .flags = MWAIT2flg(0x20) | CPUIDLE_FLAG_TLB_FLUSHED, 258d13780d4SLen Brown .exit_latency = 104, 259ddbd550dSLen Brown .target_residency = 345, 2605fe2e527SRafael J. Wysocki .enter = &intel_idle, 26128ba086eSRafael J. Wysocki .enter_s2idle = intel_idle_s2idle, }, 262e022e7ebSLen Brown { 263de09cdd0SLen Brown .name = "C7", 264d13780d4SLen Brown .desc = "MWAIT 0x30", 265b82b6ccaSDaniel Lezcano .flags = MWAIT2flg(0x30) | CPUIDLE_FLAG_TLB_FLUSHED, 266d13780d4SLen Brown .exit_latency = 109, 267ddbd550dSLen Brown .target_residency = 345, 2685fe2e527SRafael J. Wysocki .enter = &intel_idle, 26928ba086eSRafael J. Wysocki .enter_s2idle = intel_idle_s2idle, }, 270e022e7ebSLen Brown { 271e022e7ebSLen Brown .enter = NULL } 272d13780d4SLen Brown }; 273d13780d4SLen Brown 274ab1a8522SRafael J. Wysocki static struct cpuidle_state byt_cstates[] __initdata = { 275718987d6SLen Brown { 276de09cdd0SLen Brown .name = "C1", 277718987d6SLen Brown .desc = "MWAIT 0x00", 278b82b6ccaSDaniel Lezcano .flags = MWAIT2flg(0x00), 279718987d6SLen Brown .exit_latency = 1, 280718987d6SLen Brown .target_residency = 1, 2815fe2e527SRafael J. Wysocki .enter = &intel_idle, 28228ba086eSRafael J. Wysocki .enter_s2idle = intel_idle_s2idle, }, 283718987d6SLen Brown { 284de09cdd0SLen Brown .name = "C6N", 285718987d6SLen Brown .desc = "MWAIT 0x58", 286b82b6ccaSDaniel Lezcano .flags = MWAIT2flg(0x58) | CPUIDLE_FLAG_TLB_FLUSHED, 287d7ef7671SLen Brown .exit_latency = 300, 288718987d6SLen Brown .target_residency = 275, 2895fe2e527SRafael J. Wysocki .enter = &intel_idle, 29028ba086eSRafael J. Wysocki .enter_s2idle = intel_idle_s2idle, }, 291718987d6SLen Brown { 292de09cdd0SLen Brown .name = "C6S", 293718987d6SLen Brown .desc = "MWAIT 0x52", 294b82b6ccaSDaniel Lezcano .flags = MWAIT2flg(0x52) | CPUIDLE_FLAG_TLB_FLUSHED, 295d7ef7671SLen Brown .exit_latency = 500, 296718987d6SLen Brown .target_residency = 560, 2975fe2e527SRafael J. Wysocki .enter = &intel_idle, 29828ba086eSRafael J. Wysocki .enter_s2idle = intel_idle_s2idle, }, 299718987d6SLen Brown { 300de09cdd0SLen Brown .name = "C7", 301718987d6SLen Brown .desc = "MWAIT 0x60", 302b82b6ccaSDaniel Lezcano .flags = MWAIT2flg(0x60) | CPUIDLE_FLAG_TLB_FLUSHED, 303718987d6SLen Brown .exit_latency = 1200, 304d7ef7671SLen Brown .target_residency = 4000, 3055fe2e527SRafael J. Wysocki .enter = &intel_idle, 30628ba086eSRafael J. Wysocki .enter_s2idle = intel_idle_s2idle, }, 307718987d6SLen Brown { 308de09cdd0SLen Brown .name = "C7S", 309718987d6SLen Brown .desc = "MWAIT 0x64", 310b82b6ccaSDaniel Lezcano .flags = MWAIT2flg(0x64) | CPUIDLE_FLAG_TLB_FLUSHED, 311718987d6SLen Brown .exit_latency = 10000, 312718987d6SLen Brown .target_residency = 20000, 3135fe2e527SRafael J. Wysocki .enter = &intel_idle, 31428ba086eSRafael J. Wysocki .enter_s2idle = intel_idle_s2idle, }, 315718987d6SLen Brown { 316718987d6SLen Brown .enter = NULL } 317718987d6SLen Brown }; 318718987d6SLen Brown 319ab1a8522SRafael J. Wysocki static struct cpuidle_state cht_cstates[] __initdata = { 320cab07a56SLen Brown { 321de09cdd0SLen Brown .name = "C1", 322cab07a56SLen Brown .desc = "MWAIT 0x00", 323cab07a56SLen Brown .flags = MWAIT2flg(0x00), 324cab07a56SLen Brown .exit_latency = 1, 325cab07a56SLen Brown .target_residency = 1, 326cab07a56SLen Brown .enter = &intel_idle, 32728ba086eSRafael J. Wysocki .enter_s2idle = intel_idle_s2idle, }, 328cab07a56SLen Brown { 329de09cdd0SLen Brown .name = "C6N", 330cab07a56SLen Brown .desc = "MWAIT 0x58", 331cab07a56SLen Brown .flags = MWAIT2flg(0x58) | CPUIDLE_FLAG_TLB_FLUSHED, 332cab07a56SLen Brown .exit_latency = 80, 333cab07a56SLen Brown .target_residency = 275, 334cab07a56SLen Brown .enter = &intel_idle, 33528ba086eSRafael J. Wysocki .enter_s2idle = intel_idle_s2idle, }, 336cab07a56SLen Brown { 337de09cdd0SLen Brown .name = "C6S", 338cab07a56SLen Brown .desc = "MWAIT 0x52", 339cab07a56SLen Brown .flags = MWAIT2flg(0x52) | CPUIDLE_FLAG_TLB_FLUSHED, 340cab07a56SLen Brown .exit_latency = 200, 341cab07a56SLen Brown .target_residency = 560, 342cab07a56SLen Brown .enter = &intel_idle, 34328ba086eSRafael J. Wysocki .enter_s2idle = intel_idle_s2idle, }, 344cab07a56SLen Brown { 345de09cdd0SLen Brown .name = "C7", 346cab07a56SLen Brown .desc = "MWAIT 0x60", 347cab07a56SLen Brown .flags = MWAIT2flg(0x60) | CPUIDLE_FLAG_TLB_FLUSHED, 348cab07a56SLen Brown .exit_latency = 1200, 349cab07a56SLen Brown .target_residency = 4000, 350cab07a56SLen Brown .enter = &intel_idle, 35128ba086eSRafael J. Wysocki .enter_s2idle = intel_idle_s2idle, }, 352cab07a56SLen Brown { 353de09cdd0SLen Brown .name = "C7S", 354cab07a56SLen Brown .desc = "MWAIT 0x64", 355cab07a56SLen Brown .flags = MWAIT2flg(0x64) | CPUIDLE_FLAG_TLB_FLUSHED, 356cab07a56SLen Brown .exit_latency = 10000, 357cab07a56SLen Brown .target_residency = 20000, 358cab07a56SLen Brown .enter = &intel_idle, 35928ba086eSRafael J. Wysocki .enter_s2idle = intel_idle_s2idle, }, 360cab07a56SLen Brown { 361cab07a56SLen Brown .enter = NULL } 362cab07a56SLen Brown }; 363cab07a56SLen Brown 364ab1a8522SRafael J. Wysocki static struct cpuidle_state ivb_cstates[] __initdata = { 365e022e7ebSLen Brown { 366de09cdd0SLen Brown .name = "C1", 3676edab08cSLen Brown .desc = "MWAIT 0x00", 368b82b6ccaSDaniel Lezcano .flags = MWAIT2flg(0x00), 3696edab08cSLen Brown .exit_latency = 1, 3706edab08cSLen Brown .target_residency = 1, 3715fe2e527SRafael J. Wysocki .enter = &intel_idle, 37228ba086eSRafael J. Wysocki .enter_s2idle = intel_idle_s2idle, }, 373e022e7ebSLen Brown { 374de09cdd0SLen Brown .name = "C1E", 37532e95180SLen Brown .desc = "MWAIT 0x01", 376e6d4f08aSRafael J. Wysocki .flags = MWAIT2flg(0x01) | CPUIDLE_FLAG_ALWAYS_ENABLE, 37732e95180SLen Brown .exit_latency = 10, 37832e95180SLen Brown .target_residency = 20, 3795fe2e527SRafael J. Wysocki .enter = &intel_idle, 38028ba086eSRafael J. Wysocki .enter_s2idle = intel_idle_s2idle, }, 38132e95180SLen Brown { 382de09cdd0SLen Brown .name = "C3", 3836edab08cSLen Brown .desc = "MWAIT 0x10", 384b82b6ccaSDaniel Lezcano .flags = MWAIT2flg(0x10) | CPUIDLE_FLAG_TLB_FLUSHED, 3856edab08cSLen Brown .exit_latency = 59, 3866edab08cSLen Brown .target_residency = 156, 3875fe2e527SRafael J. Wysocki .enter = &intel_idle, 38828ba086eSRafael J. Wysocki .enter_s2idle = intel_idle_s2idle, }, 389e022e7ebSLen Brown { 390de09cdd0SLen Brown .name = "C6", 3916edab08cSLen Brown .desc = "MWAIT 0x20", 392b82b6ccaSDaniel Lezcano .flags = MWAIT2flg(0x20) | CPUIDLE_FLAG_TLB_FLUSHED, 3936edab08cSLen Brown .exit_latency = 80, 3946edab08cSLen Brown .target_residency = 300, 3955fe2e527SRafael J. Wysocki .enter = &intel_idle, 39628ba086eSRafael J. Wysocki .enter_s2idle = intel_idle_s2idle, }, 397e022e7ebSLen Brown { 398de09cdd0SLen Brown .name = "C7", 3996edab08cSLen Brown .desc = "MWAIT 0x30", 400b82b6ccaSDaniel Lezcano .flags = MWAIT2flg(0x30) | CPUIDLE_FLAG_TLB_FLUSHED, 4016edab08cSLen Brown .exit_latency = 87, 4026edab08cSLen Brown .target_residency = 300, 4035fe2e527SRafael J. Wysocki .enter = &intel_idle, 40428ba086eSRafael J. Wysocki .enter_s2idle = intel_idle_s2idle, }, 405e022e7ebSLen Brown { 406e022e7ebSLen Brown .enter = NULL } 4076edab08cSLen Brown }; 4086edab08cSLen Brown 409ab1a8522SRafael J. Wysocki static struct cpuidle_state ivt_cstates[] __initdata = { 4100138d8f0SLen Brown { 411de09cdd0SLen Brown .name = "C1", 4120138d8f0SLen Brown .desc = "MWAIT 0x00", 413b82b6ccaSDaniel Lezcano .flags = MWAIT2flg(0x00), 4140138d8f0SLen Brown .exit_latency = 1, 4150138d8f0SLen Brown .target_residency = 1, 4165fe2e527SRafael J. Wysocki .enter = &intel_idle, 41728ba086eSRafael J. Wysocki .enter_s2idle = intel_idle_s2idle, }, 4180138d8f0SLen Brown { 419de09cdd0SLen Brown .name = "C1E", 4200138d8f0SLen Brown .desc = "MWAIT 0x01", 421e6d4f08aSRafael J. Wysocki .flags = MWAIT2flg(0x01) | CPUIDLE_FLAG_ALWAYS_ENABLE, 4220138d8f0SLen Brown .exit_latency = 10, 4230138d8f0SLen Brown .target_residency = 80, 4245fe2e527SRafael J. Wysocki .enter = &intel_idle, 42528ba086eSRafael J. Wysocki .enter_s2idle = intel_idle_s2idle, }, 4260138d8f0SLen Brown { 427de09cdd0SLen Brown .name = "C3", 4280138d8f0SLen Brown .desc = "MWAIT 0x10", 429b82b6ccaSDaniel Lezcano .flags = MWAIT2flg(0x10) | CPUIDLE_FLAG_TLB_FLUSHED, 4300138d8f0SLen Brown .exit_latency = 59, 4310138d8f0SLen Brown .target_residency = 156, 4325fe2e527SRafael J. Wysocki .enter = &intel_idle, 43328ba086eSRafael J. Wysocki .enter_s2idle = intel_idle_s2idle, }, 4340138d8f0SLen Brown { 435de09cdd0SLen Brown .name = "C6", 4360138d8f0SLen Brown .desc = "MWAIT 0x20", 437b82b6ccaSDaniel Lezcano .flags = MWAIT2flg(0x20) | CPUIDLE_FLAG_TLB_FLUSHED, 4380138d8f0SLen Brown .exit_latency = 82, 4390138d8f0SLen Brown .target_residency = 300, 4405fe2e527SRafael J. Wysocki .enter = &intel_idle, 44128ba086eSRafael J. Wysocki .enter_s2idle = intel_idle_s2idle, }, 4420138d8f0SLen Brown { 4430138d8f0SLen Brown .enter = NULL } 4440138d8f0SLen Brown }; 4450138d8f0SLen Brown 446ab1a8522SRafael J. Wysocki static struct cpuidle_state ivt_cstates_4s[] __initdata = { 4470138d8f0SLen Brown { 448de09cdd0SLen Brown .name = "C1", 4490138d8f0SLen Brown .desc = "MWAIT 0x00", 450b82b6ccaSDaniel Lezcano .flags = MWAIT2flg(0x00), 4510138d8f0SLen Brown .exit_latency = 1, 4520138d8f0SLen Brown .target_residency = 1, 4535fe2e527SRafael J. Wysocki .enter = &intel_idle, 45428ba086eSRafael J. Wysocki .enter_s2idle = intel_idle_s2idle, }, 4550138d8f0SLen Brown { 456de09cdd0SLen Brown .name = "C1E", 4570138d8f0SLen Brown .desc = "MWAIT 0x01", 458e6d4f08aSRafael J. Wysocki .flags = MWAIT2flg(0x01) | CPUIDLE_FLAG_ALWAYS_ENABLE, 4590138d8f0SLen Brown .exit_latency = 10, 4600138d8f0SLen Brown .target_residency = 250, 4615fe2e527SRafael J. Wysocki .enter = &intel_idle, 46228ba086eSRafael J. Wysocki .enter_s2idle = intel_idle_s2idle, }, 4630138d8f0SLen Brown { 464de09cdd0SLen Brown .name = "C3", 4650138d8f0SLen Brown .desc = "MWAIT 0x10", 466b82b6ccaSDaniel Lezcano .flags = MWAIT2flg(0x10) | CPUIDLE_FLAG_TLB_FLUSHED, 4670138d8f0SLen Brown .exit_latency = 59, 4680138d8f0SLen Brown .target_residency = 300, 4695fe2e527SRafael J. Wysocki .enter = &intel_idle, 47028ba086eSRafael J. Wysocki .enter_s2idle = intel_idle_s2idle, }, 4710138d8f0SLen Brown { 472de09cdd0SLen Brown .name = "C6", 4730138d8f0SLen Brown .desc = "MWAIT 0x20", 474b82b6ccaSDaniel Lezcano .flags = MWAIT2flg(0x20) | CPUIDLE_FLAG_TLB_FLUSHED, 4750138d8f0SLen Brown .exit_latency = 84, 4760138d8f0SLen Brown .target_residency = 400, 4775fe2e527SRafael J. Wysocki .enter = &intel_idle, 47828ba086eSRafael J. Wysocki .enter_s2idle = intel_idle_s2idle, }, 4790138d8f0SLen Brown { 4800138d8f0SLen Brown .enter = NULL } 4810138d8f0SLen Brown }; 4820138d8f0SLen Brown 483ab1a8522SRafael J. Wysocki static struct cpuidle_state ivt_cstates_8s[] __initdata = { 4840138d8f0SLen Brown { 485de09cdd0SLen Brown .name = "C1", 4860138d8f0SLen Brown .desc = "MWAIT 0x00", 487b82b6ccaSDaniel Lezcano .flags = MWAIT2flg(0x00), 4880138d8f0SLen Brown .exit_latency = 1, 4890138d8f0SLen Brown .target_residency = 1, 4905fe2e527SRafael J. Wysocki .enter = &intel_idle, 49128ba086eSRafael J. Wysocki .enter_s2idle = intel_idle_s2idle, }, 4920138d8f0SLen Brown { 493de09cdd0SLen Brown .name = "C1E", 4940138d8f0SLen Brown .desc = "MWAIT 0x01", 495e6d4f08aSRafael J. Wysocki .flags = MWAIT2flg(0x01) | CPUIDLE_FLAG_ALWAYS_ENABLE, 4960138d8f0SLen Brown .exit_latency = 10, 4970138d8f0SLen Brown .target_residency = 500, 4985fe2e527SRafael J. Wysocki .enter = &intel_idle, 49928ba086eSRafael J. Wysocki .enter_s2idle = intel_idle_s2idle, }, 5000138d8f0SLen Brown { 501de09cdd0SLen Brown .name = "C3", 5020138d8f0SLen Brown .desc = "MWAIT 0x10", 503b82b6ccaSDaniel Lezcano .flags = MWAIT2flg(0x10) | CPUIDLE_FLAG_TLB_FLUSHED, 5040138d8f0SLen Brown .exit_latency = 59, 5050138d8f0SLen Brown .target_residency = 600, 5065fe2e527SRafael J. Wysocki .enter = &intel_idle, 50728ba086eSRafael J. Wysocki .enter_s2idle = intel_idle_s2idle, }, 5080138d8f0SLen Brown { 509de09cdd0SLen Brown .name = "C6", 5100138d8f0SLen Brown .desc = "MWAIT 0x20", 511b82b6ccaSDaniel Lezcano .flags = MWAIT2flg(0x20) | CPUIDLE_FLAG_TLB_FLUSHED, 5120138d8f0SLen Brown .exit_latency = 88, 5130138d8f0SLen Brown .target_residency = 700, 5145fe2e527SRafael J. Wysocki .enter = &intel_idle, 51528ba086eSRafael J. Wysocki .enter_s2idle = intel_idle_s2idle, }, 5160138d8f0SLen Brown { 5170138d8f0SLen Brown .enter = NULL } 5180138d8f0SLen Brown }; 5190138d8f0SLen Brown 520ab1a8522SRafael J. Wysocki static struct cpuidle_state hsw_cstates[] __initdata = { 521e022e7ebSLen Brown { 522de09cdd0SLen Brown .name = "C1", 52385a4d2d4SLen Brown .desc = "MWAIT 0x00", 524b82b6ccaSDaniel Lezcano .flags = MWAIT2flg(0x00), 52585a4d2d4SLen Brown .exit_latency = 2, 52685a4d2d4SLen Brown .target_residency = 2, 5275fe2e527SRafael J. Wysocki .enter = &intel_idle, 52828ba086eSRafael J. Wysocki .enter_s2idle = intel_idle_s2idle, }, 529e022e7ebSLen Brown { 530de09cdd0SLen Brown .name = "C1E", 53132e95180SLen Brown .desc = "MWAIT 0x01", 532e6d4f08aSRafael J. Wysocki .flags = MWAIT2flg(0x01) | CPUIDLE_FLAG_ALWAYS_ENABLE, 53332e95180SLen Brown .exit_latency = 10, 53432e95180SLen Brown .target_residency = 20, 5355fe2e527SRafael J. Wysocki .enter = &intel_idle, 53628ba086eSRafael J. Wysocki .enter_s2idle = intel_idle_s2idle, }, 53732e95180SLen Brown { 538de09cdd0SLen Brown .name = "C3", 53985a4d2d4SLen Brown .desc = "MWAIT 0x10", 540b82b6ccaSDaniel Lezcano .flags = MWAIT2flg(0x10) | CPUIDLE_FLAG_TLB_FLUSHED, 54185a4d2d4SLen Brown .exit_latency = 33, 54285a4d2d4SLen Brown .target_residency = 100, 5435fe2e527SRafael J. Wysocki .enter = &intel_idle, 54428ba086eSRafael J. Wysocki .enter_s2idle = intel_idle_s2idle, }, 545e022e7ebSLen Brown { 546de09cdd0SLen Brown .name = "C6", 54785a4d2d4SLen Brown .desc = "MWAIT 0x20", 548b82b6ccaSDaniel Lezcano .flags = MWAIT2flg(0x20) | CPUIDLE_FLAG_TLB_FLUSHED, 54985a4d2d4SLen Brown .exit_latency = 133, 55085a4d2d4SLen Brown .target_residency = 400, 5515fe2e527SRafael J. Wysocki .enter = &intel_idle, 55228ba086eSRafael J. Wysocki .enter_s2idle = intel_idle_s2idle, }, 553e022e7ebSLen Brown { 554de09cdd0SLen Brown .name = "C7s", 55585a4d2d4SLen Brown .desc = "MWAIT 0x32", 556b82b6ccaSDaniel Lezcano .flags = MWAIT2flg(0x32) | CPUIDLE_FLAG_TLB_FLUSHED, 55785a4d2d4SLen Brown .exit_latency = 166, 55885a4d2d4SLen Brown .target_residency = 500, 5595fe2e527SRafael J. Wysocki .enter = &intel_idle, 56028ba086eSRafael J. Wysocki .enter_s2idle = intel_idle_s2idle, }, 561e022e7ebSLen Brown { 562de09cdd0SLen Brown .name = "C8", 56386239cebSLen Brown .desc = "MWAIT 0x40", 564b82b6ccaSDaniel Lezcano .flags = MWAIT2flg(0x40) | CPUIDLE_FLAG_TLB_FLUSHED, 56586239cebSLen Brown .exit_latency = 300, 56686239cebSLen Brown .target_residency = 900, 5675fe2e527SRafael J. Wysocki .enter = &intel_idle, 56828ba086eSRafael J. Wysocki .enter_s2idle = intel_idle_s2idle, }, 56986239cebSLen Brown { 570de09cdd0SLen Brown .name = "C9", 57186239cebSLen Brown .desc = "MWAIT 0x50", 572b82b6ccaSDaniel Lezcano .flags = MWAIT2flg(0x50) | CPUIDLE_FLAG_TLB_FLUSHED, 57386239cebSLen Brown .exit_latency = 600, 57486239cebSLen Brown .target_residency = 1800, 5755fe2e527SRafael J. Wysocki .enter = &intel_idle, 57628ba086eSRafael J. Wysocki .enter_s2idle = intel_idle_s2idle, }, 57786239cebSLen Brown { 578de09cdd0SLen Brown .name = "C10", 57986239cebSLen Brown .desc = "MWAIT 0x60", 580b82b6ccaSDaniel Lezcano .flags = MWAIT2flg(0x60) | CPUIDLE_FLAG_TLB_FLUSHED, 58186239cebSLen Brown .exit_latency = 2600, 58286239cebSLen Brown .target_residency = 7700, 5835fe2e527SRafael J. Wysocki .enter = &intel_idle, 58428ba086eSRafael J. Wysocki .enter_s2idle = intel_idle_s2idle, }, 58586239cebSLen Brown { 586e022e7ebSLen Brown .enter = NULL } 58785a4d2d4SLen Brown }; 588ab1a8522SRafael J. Wysocki static struct cpuidle_state bdw_cstates[] __initdata = { 589a138b568SLen Brown { 590de09cdd0SLen Brown .name = "C1", 591a138b568SLen Brown .desc = "MWAIT 0x00", 592b82b6ccaSDaniel Lezcano .flags = MWAIT2flg(0x00), 593a138b568SLen Brown .exit_latency = 2, 594a138b568SLen Brown .target_residency = 2, 5955fe2e527SRafael J. Wysocki .enter = &intel_idle, 59628ba086eSRafael J. Wysocki .enter_s2idle = intel_idle_s2idle, }, 597a138b568SLen Brown { 598de09cdd0SLen Brown .name = "C1E", 599a138b568SLen Brown .desc = "MWAIT 0x01", 600e6d4f08aSRafael J. Wysocki .flags = MWAIT2flg(0x01) | CPUIDLE_FLAG_ALWAYS_ENABLE, 601a138b568SLen Brown .exit_latency = 10, 602a138b568SLen Brown .target_residency = 20, 6035fe2e527SRafael J. Wysocki .enter = &intel_idle, 60428ba086eSRafael J. Wysocki .enter_s2idle = intel_idle_s2idle, }, 605a138b568SLen Brown { 606de09cdd0SLen Brown .name = "C3", 607a138b568SLen Brown .desc = "MWAIT 0x10", 608b82b6ccaSDaniel Lezcano .flags = MWAIT2flg(0x10) | CPUIDLE_FLAG_TLB_FLUSHED, 609a138b568SLen Brown .exit_latency = 40, 610a138b568SLen Brown .target_residency = 100, 6115fe2e527SRafael J. Wysocki .enter = &intel_idle, 61228ba086eSRafael J. Wysocki .enter_s2idle = intel_idle_s2idle, }, 613a138b568SLen Brown { 614de09cdd0SLen Brown .name = "C6", 615a138b568SLen Brown .desc = "MWAIT 0x20", 616b82b6ccaSDaniel Lezcano .flags = MWAIT2flg(0x20) | CPUIDLE_FLAG_TLB_FLUSHED, 617a138b568SLen Brown .exit_latency = 133, 618a138b568SLen Brown .target_residency = 400, 6195fe2e527SRafael J. Wysocki .enter = &intel_idle, 62028ba086eSRafael J. Wysocki .enter_s2idle = intel_idle_s2idle, }, 621a138b568SLen Brown { 622de09cdd0SLen Brown .name = "C7s", 623a138b568SLen Brown .desc = "MWAIT 0x32", 624b82b6ccaSDaniel Lezcano .flags = MWAIT2flg(0x32) | CPUIDLE_FLAG_TLB_FLUSHED, 625a138b568SLen Brown .exit_latency = 166, 626a138b568SLen Brown .target_residency = 500, 6275fe2e527SRafael J. Wysocki .enter = &intel_idle, 62828ba086eSRafael J. Wysocki .enter_s2idle = intel_idle_s2idle, }, 629a138b568SLen Brown { 630de09cdd0SLen Brown .name = "C8", 631a138b568SLen Brown .desc = "MWAIT 0x40", 632b82b6ccaSDaniel Lezcano .flags = MWAIT2flg(0x40) | CPUIDLE_FLAG_TLB_FLUSHED, 633a138b568SLen Brown .exit_latency = 300, 634a138b568SLen Brown .target_residency = 900, 6355fe2e527SRafael J. Wysocki .enter = &intel_idle, 63628ba086eSRafael J. Wysocki .enter_s2idle = intel_idle_s2idle, }, 637a138b568SLen Brown { 638de09cdd0SLen Brown .name = "C9", 639a138b568SLen Brown .desc = "MWAIT 0x50", 640b82b6ccaSDaniel Lezcano .flags = MWAIT2flg(0x50) | CPUIDLE_FLAG_TLB_FLUSHED, 641a138b568SLen Brown .exit_latency = 600, 642a138b568SLen Brown .target_residency = 1800, 6435fe2e527SRafael J. Wysocki .enter = &intel_idle, 64428ba086eSRafael J. Wysocki .enter_s2idle = intel_idle_s2idle, }, 645a138b568SLen Brown { 646de09cdd0SLen Brown .name = "C10", 647a138b568SLen Brown .desc = "MWAIT 0x60", 648b82b6ccaSDaniel Lezcano .flags = MWAIT2flg(0x60) | CPUIDLE_FLAG_TLB_FLUSHED, 649a138b568SLen Brown .exit_latency = 2600, 650a138b568SLen Brown .target_residency = 7700, 6515fe2e527SRafael J. Wysocki .enter = &intel_idle, 65228ba086eSRafael J. Wysocki .enter_s2idle = intel_idle_s2idle, }, 653a138b568SLen Brown { 654a138b568SLen Brown .enter = NULL } 655a138b568SLen Brown }; 65685a4d2d4SLen Brown 657ab1a8522SRafael J. Wysocki static struct cpuidle_state skl_cstates[] __initdata = { 658493f133fSLen Brown { 659de09cdd0SLen Brown .name = "C1", 660493f133fSLen Brown .desc = "MWAIT 0x00", 661493f133fSLen Brown .flags = MWAIT2flg(0x00), 662493f133fSLen Brown .exit_latency = 2, 663493f133fSLen Brown .target_residency = 2, 664493f133fSLen Brown .enter = &intel_idle, 66528ba086eSRafael J. Wysocki .enter_s2idle = intel_idle_s2idle, }, 666493f133fSLen Brown { 667de09cdd0SLen Brown .name = "C1E", 668493f133fSLen Brown .desc = "MWAIT 0x01", 669e6d4f08aSRafael J. Wysocki .flags = MWAIT2flg(0x01) | CPUIDLE_FLAG_ALWAYS_ENABLE, 670493f133fSLen Brown .exit_latency = 10, 671493f133fSLen Brown .target_residency = 20, 672493f133fSLen Brown .enter = &intel_idle, 67328ba086eSRafael J. Wysocki .enter_s2idle = intel_idle_s2idle, }, 674493f133fSLen Brown { 675de09cdd0SLen Brown .name = "C3", 676493f133fSLen Brown .desc = "MWAIT 0x10", 677493f133fSLen Brown .flags = MWAIT2flg(0x10) | CPUIDLE_FLAG_TLB_FLUSHED, 678493f133fSLen Brown .exit_latency = 70, 679493f133fSLen Brown .target_residency = 100, 680493f133fSLen Brown .enter = &intel_idle, 68128ba086eSRafael J. Wysocki .enter_s2idle = intel_idle_s2idle, }, 682493f133fSLen Brown { 683de09cdd0SLen Brown .name = "C6", 684493f133fSLen Brown .desc = "MWAIT 0x20", 685493f133fSLen Brown .flags = MWAIT2flg(0x20) | CPUIDLE_FLAG_TLB_FLUSHED, 686135919a3SLen Brown .exit_latency = 85, 687493f133fSLen Brown .target_residency = 200, 688493f133fSLen Brown .enter = &intel_idle, 68928ba086eSRafael J. Wysocki .enter_s2idle = intel_idle_s2idle, }, 690493f133fSLen Brown { 691de09cdd0SLen Brown .name = "C7s", 692493f133fSLen Brown .desc = "MWAIT 0x33", 693493f133fSLen Brown .flags = MWAIT2flg(0x33) | CPUIDLE_FLAG_TLB_FLUSHED, 694493f133fSLen Brown .exit_latency = 124, 695493f133fSLen Brown .target_residency = 800, 696493f133fSLen Brown .enter = &intel_idle, 69728ba086eSRafael J. Wysocki .enter_s2idle = intel_idle_s2idle, }, 698493f133fSLen Brown { 699de09cdd0SLen Brown .name = "C8", 700493f133fSLen Brown .desc = "MWAIT 0x40", 701493f133fSLen Brown .flags = MWAIT2flg(0x40) | CPUIDLE_FLAG_TLB_FLUSHED, 702135919a3SLen Brown .exit_latency = 200, 703493f133fSLen Brown .target_residency = 800, 704493f133fSLen Brown .enter = &intel_idle, 70528ba086eSRafael J. Wysocki .enter_s2idle = intel_idle_s2idle, }, 706493f133fSLen Brown { 707de09cdd0SLen Brown .name = "C9", 708135919a3SLen Brown .desc = "MWAIT 0x50", 709135919a3SLen Brown .flags = MWAIT2flg(0x50) | CPUIDLE_FLAG_TLB_FLUSHED, 710135919a3SLen Brown .exit_latency = 480, 711135919a3SLen Brown .target_residency = 5000, 712135919a3SLen Brown .enter = &intel_idle, 71328ba086eSRafael J. Wysocki .enter_s2idle = intel_idle_s2idle, }, 714135919a3SLen Brown { 715de09cdd0SLen Brown .name = "C10", 716493f133fSLen Brown .desc = "MWAIT 0x60", 717493f133fSLen Brown .flags = MWAIT2flg(0x60) | CPUIDLE_FLAG_TLB_FLUSHED, 718493f133fSLen Brown .exit_latency = 890, 719493f133fSLen Brown .target_residency = 5000, 720493f133fSLen Brown .enter = &intel_idle, 72128ba086eSRafael J. Wysocki .enter_s2idle = intel_idle_s2idle, }, 722493f133fSLen Brown { 723493f133fSLen Brown .enter = NULL } 724493f133fSLen Brown }; 725493f133fSLen Brown 726ab1a8522SRafael J. Wysocki static struct cpuidle_state skx_cstates[] __initdata = { 727f9e71657SLen Brown { 728de09cdd0SLen Brown .name = "C1", 729f9e71657SLen Brown .desc = "MWAIT 0x00", 730f9e71657SLen Brown .flags = MWAIT2flg(0x00), 731f9e71657SLen Brown .exit_latency = 2, 732f9e71657SLen Brown .target_residency = 2, 733f9e71657SLen Brown .enter = &intel_idle, 73428ba086eSRafael J. Wysocki .enter_s2idle = intel_idle_s2idle, }, 735f9e71657SLen Brown { 736de09cdd0SLen Brown .name = "C1E", 737f9e71657SLen Brown .desc = "MWAIT 0x01", 738e6d4f08aSRafael J. Wysocki .flags = MWAIT2flg(0x01) | CPUIDLE_FLAG_ALWAYS_ENABLE, 739f9e71657SLen Brown .exit_latency = 10, 740f9e71657SLen Brown .target_residency = 20, 741f9e71657SLen Brown .enter = &intel_idle, 74228ba086eSRafael J. Wysocki .enter_s2idle = intel_idle_s2idle, }, 743f9e71657SLen Brown { 744de09cdd0SLen Brown .name = "C6", 745f9e71657SLen Brown .desc = "MWAIT 0x20", 746f9e71657SLen Brown .flags = MWAIT2flg(0x20) | CPUIDLE_FLAG_TLB_FLUSHED, 747f9e71657SLen Brown .exit_latency = 133, 748f9e71657SLen Brown .target_residency = 600, 749f9e71657SLen Brown .enter = &intel_idle, 75028ba086eSRafael J. Wysocki .enter_s2idle = intel_idle_s2idle, }, 751f9e71657SLen Brown { 752f9e71657SLen Brown .enter = NULL } 753f9e71657SLen Brown }; 754f9e71657SLen Brown 755a472ad2bSChen Yu static struct cpuidle_state icx_cstates[] __initdata = { 756a472ad2bSChen Yu { 757a472ad2bSChen Yu .name = "C1", 758a472ad2bSChen Yu .desc = "MWAIT 0x00", 759a472ad2bSChen Yu .flags = MWAIT2flg(0x00), 760a472ad2bSChen Yu .exit_latency = 1, 761a472ad2bSChen Yu .target_residency = 1, 762a472ad2bSChen Yu .enter = &intel_idle, 763a472ad2bSChen Yu .enter_s2idle = intel_idle_s2idle, }, 764a472ad2bSChen Yu { 765a472ad2bSChen Yu .name = "C1E", 766a472ad2bSChen Yu .desc = "MWAIT 0x01", 767a472ad2bSChen Yu .flags = MWAIT2flg(0x01) | CPUIDLE_FLAG_ALWAYS_ENABLE, 768a472ad2bSChen Yu .exit_latency = 4, 769a472ad2bSChen Yu .target_residency = 4, 770a472ad2bSChen Yu .enter = &intel_idle, 771a472ad2bSChen Yu .enter_s2idle = intel_idle_s2idle, }, 772a472ad2bSChen Yu { 773a472ad2bSChen Yu .name = "C6", 774a472ad2bSChen Yu .desc = "MWAIT 0x20", 775a472ad2bSChen Yu .flags = MWAIT2flg(0x20) | CPUIDLE_FLAG_TLB_FLUSHED, 776a472ad2bSChen Yu .exit_latency = 128, 777a472ad2bSChen Yu .target_residency = 384, 778a472ad2bSChen Yu .enter = &intel_idle, 779a472ad2bSChen Yu .enter_s2idle = intel_idle_s2idle, }, 780a472ad2bSChen Yu { 781a472ad2bSChen Yu .enter = NULL } 782a472ad2bSChen Yu }; 783a472ad2bSChen Yu 784ab1a8522SRafael J. Wysocki static struct cpuidle_state atom_cstates[] __initdata = { 785e022e7ebSLen Brown { 786de09cdd0SLen Brown .name = "C1E", 78726717172SLen Brown .desc = "MWAIT 0x00", 788b82b6ccaSDaniel Lezcano .flags = MWAIT2flg(0x00), 78932e95180SLen Brown .exit_latency = 10, 79032e95180SLen Brown .target_residency = 20, 7915fe2e527SRafael J. Wysocki .enter = &intel_idle, 79228ba086eSRafael J. Wysocki .enter_s2idle = intel_idle_s2idle, }, 793e022e7ebSLen Brown { 794de09cdd0SLen Brown .name = "C2", 79526717172SLen Brown .desc = "MWAIT 0x10", 796b82b6ccaSDaniel Lezcano .flags = MWAIT2flg(0x10), 79726717172SLen Brown .exit_latency = 20, 79826717172SLen Brown .target_residency = 80, 7995fe2e527SRafael J. Wysocki .enter = &intel_idle, 80028ba086eSRafael J. Wysocki .enter_s2idle = intel_idle_s2idle, }, 801e022e7ebSLen Brown { 802de09cdd0SLen Brown .name = "C4", 80326717172SLen Brown .desc = "MWAIT 0x30", 804b82b6ccaSDaniel Lezcano .flags = MWAIT2flg(0x30) | CPUIDLE_FLAG_TLB_FLUSHED, 80526717172SLen Brown .exit_latency = 100, 80626717172SLen Brown .target_residency = 400, 8075fe2e527SRafael J. Wysocki .enter = &intel_idle, 80828ba086eSRafael J. Wysocki .enter_s2idle = intel_idle_s2idle, }, 809e022e7ebSLen Brown { 810de09cdd0SLen Brown .name = "C6", 8117fcca7d9SLen Brown .desc = "MWAIT 0x52", 812b82b6ccaSDaniel Lezcano .flags = MWAIT2flg(0x52) | CPUIDLE_FLAG_TLB_FLUSHED, 8137fcca7d9SLen Brown .exit_latency = 140, 8147fcca7d9SLen Brown .target_residency = 560, 8155fe2e527SRafael J. Wysocki .enter = &intel_idle, 81628ba086eSRafael J. Wysocki .enter_s2idle = intel_idle_s2idle, }, 817e022e7ebSLen Brown { 818e022e7ebSLen Brown .enter = NULL } 81926717172SLen Brown }; 820ab1a8522SRafael J. Wysocki static struct cpuidle_state tangier_cstates[] __initdata = { 8215e7ec268SAndy Shevchenko { 822de09cdd0SLen Brown .name = "C1", 8235e7ec268SAndy Shevchenko .desc = "MWAIT 0x00", 8245e7ec268SAndy Shevchenko .flags = MWAIT2flg(0x00), 8255e7ec268SAndy Shevchenko .exit_latency = 1, 8265e7ec268SAndy Shevchenko .target_residency = 4, 8275e7ec268SAndy Shevchenko .enter = &intel_idle, 82828ba086eSRafael J. Wysocki .enter_s2idle = intel_idle_s2idle, }, 8295e7ec268SAndy Shevchenko { 830de09cdd0SLen Brown .name = "C4", 8315e7ec268SAndy Shevchenko .desc = "MWAIT 0x30", 8325e7ec268SAndy Shevchenko .flags = MWAIT2flg(0x30) | CPUIDLE_FLAG_TLB_FLUSHED, 8335e7ec268SAndy Shevchenko .exit_latency = 100, 8345e7ec268SAndy Shevchenko .target_residency = 400, 8355e7ec268SAndy Shevchenko .enter = &intel_idle, 83628ba086eSRafael J. Wysocki .enter_s2idle = intel_idle_s2idle, }, 8375e7ec268SAndy Shevchenko { 838de09cdd0SLen Brown .name = "C6", 8395e7ec268SAndy Shevchenko .desc = "MWAIT 0x52", 8405e7ec268SAndy Shevchenko .flags = MWAIT2flg(0x52) | CPUIDLE_FLAG_TLB_FLUSHED, 8415e7ec268SAndy Shevchenko .exit_latency = 140, 8425e7ec268SAndy Shevchenko .target_residency = 560, 8435e7ec268SAndy Shevchenko .enter = &intel_idle, 84428ba086eSRafael J. Wysocki .enter_s2idle = intel_idle_s2idle, }, 8455e7ec268SAndy Shevchenko { 846de09cdd0SLen Brown .name = "C7", 8475e7ec268SAndy Shevchenko .desc = "MWAIT 0x60", 8485e7ec268SAndy Shevchenko .flags = MWAIT2flg(0x60) | CPUIDLE_FLAG_TLB_FLUSHED, 8495e7ec268SAndy Shevchenko .exit_latency = 1200, 8505e7ec268SAndy Shevchenko .target_residency = 4000, 8515e7ec268SAndy Shevchenko .enter = &intel_idle, 85228ba086eSRafael J. Wysocki .enter_s2idle = intel_idle_s2idle, }, 8535e7ec268SAndy Shevchenko { 854de09cdd0SLen Brown .name = "C9", 8555e7ec268SAndy Shevchenko .desc = "MWAIT 0x64", 8565e7ec268SAndy Shevchenko .flags = MWAIT2flg(0x64) | CPUIDLE_FLAG_TLB_FLUSHED, 8575e7ec268SAndy Shevchenko .exit_latency = 10000, 8585e7ec268SAndy Shevchenko .target_residency = 20000, 8595e7ec268SAndy Shevchenko .enter = &intel_idle, 86028ba086eSRafael J. Wysocki .enter_s2idle = intel_idle_s2idle, }, 8615e7ec268SAndy Shevchenko { 8625e7ec268SAndy Shevchenko .enter = NULL } 8635e7ec268SAndy Shevchenko }; 864ab1a8522SRafael J. Wysocki static struct cpuidle_state avn_cstates[] __initdata = { 865fab04b22SLen Brown { 866de09cdd0SLen Brown .name = "C1", 867fab04b22SLen Brown .desc = "MWAIT 0x00", 868b82b6ccaSDaniel Lezcano .flags = MWAIT2flg(0x00), 869fab04b22SLen Brown .exit_latency = 2, 870fab04b22SLen Brown .target_residency = 2, 8715fe2e527SRafael J. Wysocki .enter = &intel_idle, 87228ba086eSRafael J. Wysocki .enter_s2idle = intel_idle_s2idle, }, 873fab04b22SLen Brown { 874de09cdd0SLen Brown .name = "C6", 875fab04b22SLen Brown .desc = "MWAIT 0x51", 876b82b6ccaSDaniel Lezcano .flags = MWAIT2flg(0x51) | CPUIDLE_FLAG_TLB_FLUSHED, 877fab04b22SLen Brown .exit_latency = 15, 878fab04b22SLen Brown .target_residency = 45, 8795fe2e527SRafael J. Wysocki .enter = &intel_idle, 88028ba086eSRafael J. Wysocki .enter_s2idle = intel_idle_s2idle, }, 88188390996SJiang Liu { 88288390996SJiang Liu .enter = NULL } 883fab04b22SLen Brown }; 884ab1a8522SRafael J. Wysocki static struct cpuidle_state knl_cstates[] __initdata = { 885281baf7aSDasaratharaman Chandramouli { 886de09cdd0SLen Brown .name = "C1", 887281baf7aSDasaratharaman Chandramouli .desc = "MWAIT 0x00", 888281baf7aSDasaratharaman Chandramouli .flags = MWAIT2flg(0x00), 889281baf7aSDasaratharaman Chandramouli .exit_latency = 1, 890281baf7aSDasaratharaman Chandramouli .target_residency = 2, 891281baf7aSDasaratharaman Chandramouli .enter = &intel_idle, 89228ba086eSRafael J. Wysocki .enter_s2idle = intel_idle_s2idle }, 893281baf7aSDasaratharaman Chandramouli { 894de09cdd0SLen Brown .name = "C6", 895281baf7aSDasaratharaman Chandramouli .desc = "MWAIT 0x10", 896281baf7aSDasaratharaman Chandramouli .flags = MWAIT2flg(0x10) | CPUIDLE_FLAG_TLB_FLUSHED, 897281baf7aSDasaratharaman Chandramouli .exit_latency = 120, 898281baf7aSDasaratharaman Chandramouli .target_residency = 500, 899281baf7aSDasaratharaman Chandramouli .enter = &intel_idle, 90028ba086eSRafael J. Wysocki .enter_s2idle = intel_idle_s2idle }, 901281baf7aSDasaratharaman Chandramouli { 902281baf7aSDasaratharaman Chandramouli .enter = NULL } 903281baf7aSDasaratharaman Chandramouli }; 90426717172SLen Brown 905ab1a8522SRafael J. Wysocki static struct cpuidle_state bxt_cstates[] __initdata = { 9065dcef694SLen Brown { 907de09cdd0SLen Brown .name = "C1", 9085dcef694SLen Brown .desc = "MWAIT 0x00", 9095dcef694SLen Brown .flags = MWAIT2flg(0x00), 9105dcef694SLen Brown .exit_latency = 2, 9115dcef694SLen Brown .target_residency = 2, 9125dcef694SLen Brown .enter = &intel_idle, 91328ba086eSRafael J. Wysocki .enter_s2idle = intel_idle_s2idle, }, 9145dcef694SLen Brown { 915de09cdd0SLen Brown .name = "C1E", 9165dcef694SLen Brown .desc = "MWAIT 0x01", 917e6d4f08aSRafael J. Wysocki .flags = MWAIT2flg(0x01) | CPUIDLE_FLAG_ALWAYS_ENABLE, 9185dcef694SLen Brown .exit_latency = 10, 9195dcef694SLen Brown .target_residency = 20, 9205dcef694SLen Brown .enter = &intel_idle, 92128ba086eSRafael J. Wysocki .enter_s2idle = intel_idle_s2idle, }, 9225dcef694SLen Brown { 923de09cdd0SLen Brown .name = "C6", 9245dcef694SLen Brown .desc = "MWAIT 0x20", 9255dcef694SLen Brown .flags = MWAIT2flg(0x20) | CPUIDLE_FLAG_TLB_FLUSHED, 9265dcef694SLen Brown .exit_latency = 133, 9275dcef694SLen Brown .target_residency = 133, 9285dcef694SLen Brown .enter = &intel_idle, 92928ba086eSRafael J. Wysocki .enter_s2idle = intel_idle_s2idle, }, 9305dcef694SLen Brown { 931de09cdd0SLen Brown .name = "C7s", 9325dcef694SLen Brown .desc = "MWAIT 0x31", 9335dcef694SLen Brown .flags = MWAIT2flg(0x31) | CPUIDLE_FLAG_TLB_FLUSHED, 9345dcef694SLen Brown .exit_latency = 155, 9355dcef694SLen Brown .target_residency = 155, 9365dcef694SLen Brown .enter = &intel_idle, 93728ba086eSRafael J. Wysocki .enter_s2idle = intel_idle_s2idle, }, 9385dcef694SLen Brown { 939de09cdd0SLen Brown .name = "C8", 9405dcef694SLen Brown .desc = "MWAIT 0x40", 9415dcef694SLen Brown .flags = MWAIT2flg(0x40) | CPUIDLE_FLAG_TLB_FLUSHED, 9425dcef694SLen Brown .exit_latency = 1000, 9435dcef694SLen Brown .target_residency = 1000, 9445dcef694SLen Brown .enter = &intel_idle, 94528ba086eSRafael J. Wysocki .enter_s2idle = intel_idle_s2idle, }, 9465dcef694SLen Brown { 947de09cdd0SLen Brown .name = "C9", 9485dcef694SLen Brown .desc = "MWAIT 0x50", 9495dcef694SLen Brown .flags = MWAIT2flg(0x50) | CPUIDLE_FLAG_TLB_FLUSHED, 9505dcef694SLen Brown .exit_latency = 2000, 9515dcef694SLen Brown .target_residency = 2000, 9525dcef694SLen Brown .enter = &intel_idle, 95328ba086eSRafael J. Wysocki .enter_s2idle = intel_idle_s2idle, }, 9545dcef694SLen Brown { 955de09cdd0SLen Brown .name = "C10", 9565dcef694SLen Brown .desc = "MWAIT 0x60", 9575dcef694SLen Brown .flags = MWAIT2flg(0x60) | CPUIDLE_FLAG_TLB_FLUSHED, 9585dcef694SLen Brown .exit_latency = 10000, 9595dcef694SLen Brown .target_residency = 10000, 9605dcef694SLen Brown .enter = &intel_idle, 96128ba086eSRafael J. Wysocki .enter_s2idle = intel_idle_s2idle, }, 9625dcef694SLen Brown { 9635dcef694SLen Brown .enter = NULL } 9645dcef694SLen Brown }; 9655dcef694SLen Brown 966ab1a8522SRafael J. Wysocki static struct cpuidle_state dnv_cstates[] __initdata = { 9670080d65bSJacob Pan { 968de09cdd0SLen Brown .name = "C1", 9690080d65bSJacob Pan .desc = "MWAIT 0x00", 9700080d65bSJacob Pan .flags = MWAIT2flg(0x00), 9710080d65bSJacob Pan .exit_latency = 2, 9720080d65bSJacob Pan .target_residency = 2, 9730080d65bSJacob Pan .enter = &intel_idle, 97428ba086eSRafael J. Wysocki .enter_s2idle = intel_idle_s2idle, }, 9750080d65bSJacob Pan { 976de09cdd0SLen Brown .name = "C1E", 9770080d65bSJacob Pan .desc = "MWAIT 0x01", 978e6d4f08aSRafael J. Wysocki .flags = MWAIT2flg(0x01) | CPUIDLE_FLAG_ALWAYS_ENABLE, 9790080d65bSJacob Pan .exit_latency = 10, 9800080d65bSJacob Pan .target_residency = 20, 9810080d65bSJacob Pan .enter = &intel_idle, 98228ba086eSRafael J. Wysocki .enter_s2idle = intel_idle_s2idle, }, 9830080d65bSJacob Pan { 984de09cdd0SLen Brown .name = "C6", 9850080d65bSJacob Pan .desc = "MWAIT 0x20", 9860080d65bSJacob Pan .flags = MWAIT2flg(0x20) | CPUIDLE_FLAG_TLB_FLUSHED, 9870080d65bSJacob Pan .exit_latency = 50, 9880080d65bSJacob Pan .target_residency = 500, 9890080d65bSJacob Pan .enter = &intel_idle, 99028ba086eSRafael J. Wysocki .enter_s2idle = intel_idle_s2idle, }, 9910080d65bSJacob Pan { 9920080d65bSJacob Pan .enter = NULL } 9930080d65bSJacob Pan }; 9940080d65bSJacob Pan 995ab1a8522SRafael J. Wysocki static const struct idle_cpu idle_cpu_nehalem __initconst = { 996b66b8b9aSAndi Kleen .state_table = nehalem_cstates, 997b66b8b9aSAndi Kleen .auto_demotion_disable_flags = NHM_C1_AUTO_DEMOTE | NHM_C3_AUTO_DEMOTE, 99832e95180SLen Brown .disable_promotion_to_c1e = true, 999b66b8b9aSAndi Kleen }; 1000b66b8b9aSAndi Kleen 1001ab1a8522SRafael J. Wysocki static const struct idle_cpu idle_cpu_nhx __initconst = { 1002e6d4f08aSRafael J. Wysocki .state_table = nehalem_cstates, 1003e6d4f08aSRafael J. Wysocki .auto_demotion_disable_flags = NHM_C1_AUTO_DEMOTE | NHM_C3_AUTO_DEMOTE, 1004e6d4f08aSRafael J. Wysocki .disable_promotion_to_c1e = true, 1005e6d4f08aSRafael J. Wysocki .use_acpi = true, 1006e6d4f08aSRafael J. Wysocki }; 1007e6d4f08aSRafael J. Wysocki 1008ab1a8522SRafael J. Wysocki static const struct idle_cpu idle_cpu_atom __initconst = { 1009b66b8b9aSAndi Kleen .state_table = atom_cstates, 1010b66b8b9aSAndi Kleen }; 1011b66b8b9aSAndi Kleen 1012ab1a8522SRafael J. Wysocki static const struct idle_cpu idle_cpu_tangier __initconst = { 10135e7ec268SAndy Shevchenko .state_table = tangier_cstates, 10145e7ec268SAndy Shevchenko }; 10155e7ec268SAndy Shevchenko 1016ab1a8522SRafael J. Wysocki static const struct idle_cpu idle_cpu_lincroft __initconst = { 1017b66b8b9aSAndi Kleen .state_table = atom_cstates, 1018b66b8b9aSAndi Kleen .auto_demotion_disable_flags = ATM_LNC_C6_AUTO_DEMOTE, 1019b66b8b9aSAndi Kleen }; 1020b66b8b9aSAndi Kleen 1021ab1a8522SRafael J. Wysocki static const struct idle_cpu idle_cpu_snb __initconst = { 1022b66b8b9aSAndi Kleen .state_table = snb_cstates, 102332e95180SLen Brown .disable_promotion_to_c1e = true, 1024b66b8b9aSAndi Kleen }; 1025b66b8b9aSAndi Kleen 1026ab1a8522SRafael J. Wysocki static const struct idle_cpu idle_cpu_snx __initconst = { 1027e6d4f08aSRafael J. Wysocki .state_table = snb_cstates, 1028e6d4f08aSRafael J. Wysocki .disable_promotion_to_c1e = true, 1029e6d4f08aSRafael J. Wysocki .use_acpi = true, 1030e6d4f08aSRafael J. Wysocki }; 1031e6d4f08aSRafael J. Wysocki 1032ab1a8522SRafael J. Wysocki static const struct idle_cpu idle_cpu_byt __initconst = { 1033718987d6SLen Brown .state_table = byt_cstates, 1034718987d6SLen Brown .disable_promotion_to_c1e = true, 10358c058d53SLen Brown .byt_auto_demotion_disable_flag = true, 1036718987d6SLen Brown }; 1037718987d6SLen Brown 1038ab1a8522SRafael J. Wysocki static const struct idle_cpu idle_cpu_cht __initconst = { 1039cab07a56SLen Brown .state_table = cht_cstates, 1040cab07a56SLen Brown .disable_promotion_to_c1e = true, 1041cab07a56SLen Brown .byt_auto_demotion_disable_flag = true, 1042cab07a56SLen Brown }; 1043cab07a56SLen Brown 1044ab1a8522SRafael J. Wysocki static const struct idle_cpu idle_cpu_ivb __initconst = { 10456edab08cSLen Brown .state_table = ivb_cstates, 104632e95180SLen Brown .disable_promotion_to_c1e = true, 10476edab08cSLen Brown }; 10486edab08cSLen Brown 1049ab1a8522SRafael J. Wysocki static const struct idle_cpu idle_cpu_ivt __initconst = { 10500138d8f0SLen Brown .state_table = ivt_cstates, 10510138d8f0SLen Brown .disable_promotion_to_c1e = true, 1052e6d4f08aSRafael J. Wysocki .use_acpi = true, 10530138d8f0SLen Brown }; 10540138d8f0SLen Brown 1055ab1a8522SRafael J. Wysocki static const struct idle_cpu idle_cpu_hsw __initconst = { 105685a4d2d4SLen Brown .state_table = hsw_cstates, 105732e95180SLen Brown .disable_promotion_to_c1e = true, 105885a4d2d4SLen Brown }; 105985a4d2d4SLen Brown 1060ab1a8522SRafael J. Wysocki static const struct idle_cpu idle_cpu_hsx __initconst = { 1061e6d4f08aSRafael J. Wysocki .state_table = hsw_cstates, 1062e6d4f08aSRafael J. Wysocki .disable_promotion_to_c1e = true, 1063e6d4f08aSRafael J. Wysocki .use_acpi = true, 1064e6d4f08aSRafael J. Wysocki }; 1065e6d4f08aSRafael J. Wysocki 1066ab1a8522SRafael J. Wysocki static const struct idle_cpu idle_cpu_bdw __initconst = { 1067a138b568SLen Brown .state_table = bdw_cstates, 1068a138b568SLen Brown .disable_promotion_to_c1e = true, 1069a138b568SLen Brown }; 1070a138b568SLen Brown 1071ab1a8522SRafael J. Wysocki static const struct idle_cpu idle_cpu_bdx __initconst = { 1072e6d4f08aSRafael J. Wysocki .state_table = bdw_cstates, 1073e6d4f08aSRafael J. Wysocki .disable_promotion_to_c1e = true, 1074e6d4f08aSRafael J. Wysocki .use_acpi = true, 1075e6d4f08aSRafael J. Wysocki }; 1076e6d4f08aSRafael J. Wysocki 1077ab1a8522SRafael J. Wysocki static const struct idle_cpu idle_cpu_skl __initconst = { 1078493f133fSLen Brown .state_table = skl_cstates, 1079493f133fSLen Brown .disable_promotion_to_c1e = true, 1080493f133fSLen Brown }; 1081493f133fSLen Brown 1082ab1a8522SRafael J. Wysocki static const struct idle_cpu idle_cpu_skx __initconst = { 1083f9e71657SLen Brown .state_table = skx_cstates, 1084f9e71657SLen Brown .disable_promotion_to_c1e = true, 1085e6d4f08aSRafael J. Wysocki .use_acpi = true, 1086f9e71657SLen Brown }; 1087493f133fSLen Brown 1088a472ad2bSChen Yu static const struct idle_cpu idle_cpu_icx __initconst = { 1089a472ad2bSChen Yu .state_table = icx_cstates, 1090a472ad2bSChen Yu .disable_promotion_to_c1e = true, 1091a472ad2bSChen Yu .use_acpi = true, 1092a472ad2bSChen Yu }; 1093a472ad2bSChen Yu 1094ab1a8522SRafael J. Wysocki static const struct idle_cpu idle_cpu_avn __initconst = { 1095fab04b22SLen Brown .state_table = avn_cstates, 1096fab04b22SLen Brown .disable_promotion_to_c1e = true, 1097e6d4f08aSRafael J. Wysocki .use_acpi = true, 1098fab04b22SLen Brown }; 1099fab04b22SLen Brown 1100ab1a8522SRafael J. Wysocki static const struct idle_cpu idle_cpu_knl __initconst = { 1101281baf7aSDasaratharaman Chandramouli .state_table = knl_cstates, 1102e6d4f08aSRafael J. Wysocki .use_acpi = true, 1103281baf7aSDasaratharaman Chandramouli }; 1104281baf7aSDasaratharaman Chandramouli 1105ab1a8522SRafael J. Wysocki static const struct idle_cpu idle_cpu_bxt __initconst = { 11065dcef694SLen Brown .state_table = bxt_cstates, 11075dcef694SLen Brown .disable_promotion_to_c1e = true, 11085dcef694SLen Brown }; 11095dcef694SLen Brown 1110ab1a8522SRafael J. Wysocki static const struct idle_cpu idle_cpu_dnv __initconst = { 11110080d65bSJacob Pan .state_table = dnv_cstates, 11120080d65bSJacob Pan .disable_promotion_to_c1e = true, 1113e6d4f08aSRafael J. Wysocki .use_acpi = true, 11140080d65bSJacob Pan }; 11150080d65bSJacob Pan 1116d5cdc3c4SMathias Krause static const struct x86_cpu_id intel_idle_ids[] __initconst = { 11174a9f45a0SThomas Gleixner X86_MATCH_INTEL_FAM6_MODEL(NEHALEM_EP, &idle_cpu_nhx), 11184a9f45a0SThomas Gleixner X86_MATCH_INTEL_FAM6_MODEL(NEHALEM, &idle_cpu_nehalem), 11194a9f45a0SThomas Gleixner X86_MATCH_INTEL_FAM6_MODEL(NEHALEM_G, &idle_cpu_nehalem), 11204a9f45a0SThomas Gleixner X86_MATCH_INTEL_FAM6_MODEL(WESTMERE, &idle_cpu_nehalem), 11214a9f45a0SThomas Gleixner X86_MATCH_INTEL_FAM6_MODEL(WESTMERE_EP, &idle_cpu_nhx), 11224a9f45a0SThomas Gleixner X86_MATCH_INTEL_FAM6_MODEL(NEHALEM_EX, &idle_cpu_nhx), 11234a9f45a0SThomas Gleixner X86_MATCH_INTEL_FAM6_MODEL(ATOM_BONNELL, &idle_cpu_atom), 11244a9f45a0SThomas Gleixner X86_MATCH_INTEL_FAM6_MODEL(ATOM_BONNELL_MID, &idle_cpu_lincroft), 11254a9f45a0SThomas Gleixner X86_MATCH_INTEL_FAM6_MODEL(WESTMERE_EX, &idle_cpu_nhx), 11264a9f45a0SThomas Gleixner X86_MATCH_INTEL_FAM6_MODEL(SANDYBRIDGE, &idle_cpu_snb), 11274a9f45a0SThomas Gleixner X86_MATCH_INTEL_FAM6_MODEL(SANDYBRIDGE_X, &idle_cpu_snx), 11284a9f45a0SThomas Gleixner X86_MATCH_INTEL_FAM6_MODEL(ATOM_SALTWELL, &idle_cpu_atom), 11294a9f45a0SThomas Gleixner X86_MATCH_INTEL_FAM6_MODEL(ATOM_SILVERMONT, &idle_cpu_byt), 11304a9f45a0SThomas Gleixner X86_MATCH_INTEL_FAM6_MODEL(ATOM_SILVERMONT_MID, &idle_cpu_tangier), 11314a9f45a0SThomas Gleixner X86_MATCH_INTEL_FAM6_MODEL(ATOM_AIRMONT, &idle_cpu_cht), 11324a9f45a0SThomas Gleixner X86_MATCH_INTEL_FAM6_MODEL(IVYBRIDGE, &idle_cpu_ivb), 11334a9f45a0SThomas Gleixner X86_MATCH_INTEL_FAM6_MODEL(IVYBRIDGE_X, &idle_cpu_ivt), 11344a9f45a0SThomas Gleixner X86_MATCH_INTEL_FAM6_MODEL(HASWELL, &idle_cpu_hsw), 11354a9f45a0SThomas Gleixner X86_MATCH_INTEL_FAM6_MODEL(HASWELL_X, &idle_cpu_hsx), 11364a9f45a0SThomas Gleixner X86_MATCH_INTEL_FAM6_MODEL(HASWELL_L, &idle_cpu_hsw), 11374a9f45a0SThomas Gleixner X86_MATCH_INTEL_FAM6_MODEL(HASWELL_G, &idle_cpu_hsw), 11384a9f45a0SThomas Gleixner X86_MATCH_INTEL_FAM6_MODEL(ATOM_SILVERMONT_D, &idle_cpu_avn), 11394a9f45a0SThomas Gleixner X86_MATCH_INTEL_FAM6_MODEL(BROADWELL, &idle_cpu_bdw), 11404a9f45a0SThomas Gleixner X86_MATCH_INTEL_FAM6_MODEL(BROADWELL_G, &idle_cpu_bdw), 11414a9f45a0SThomas Gleixner X86_MATCH_INTEL_FAM6_MODEL(BROADWELL_X, &idle_cpu_bdx), 11424a9f45a0SThomas Gleixner X86_MATCH_INTEL_FAM6_MODEL(BROADWELL_D, &idle_cpu_bdx), 11434a9f45a0SThomas Gleixner X86_MATCH_INTEL_FAM6_MODEL(SKYLAKE_L, &idle_cpu_skl), 11444a9f45a0SThomas Gleixner X86_MATCH_INTEL_FAM6_MODEL(SKYLAKE, &idle_cpu_skl), 11454a9f45a0SThomas Gleixner X86_MATCH_INTEL_FAM6_MODEL(KABYLAKE_L, &idle_cpu_skl), 11464a9f45a0SThomas Gleixner X86_MATCH_INTEL_FAM6_MODEL(KABYLAKE, &idle_cpu_skl), 11474a9f45a0SThomas Gleixner X86_MATCH_INTEL_FAM6_MODEL(SKYLAKE_X, &idle_cpu_skx), 1148a472ad2bSChen Yu X86_MATCH_INTEL_FAM6_MODEL(ICELAKE_X, &idle_cpu_icx), 11494a9f45a0SThomas Gleixner X86_MATCH_INTEL_FAM6_MODEL(XEON_PHI_KNL, &idle_cpu_knl), 11504a9f45a0SThomas Gleixner X86_MATCH_INTEL_FAM6_MODEL(XEON_PHI_KNM, &idle_cpu_knl), 11514a9f45a0SThomas Gleixner X86_MATCH_INTEL_FAM6_MODEL(ATOM_GOLDMONT, &idle_cpu_bxt), 11524a9f45a0SThomas Gleixner X86_MATCH_INTEL_FAM6_MODEL(ATOM_GOLDMONT_PLUS, &idle_cpu_bxt), 11534a9f45a0SThomas Gleixner X86_MATCH_INTEL_FAM6_MODEL(ATOM_GOLDMONT_D, &idle_cpu_dnv), 11544a9f45a0SThomas Gleixner X86_MATCH_INTEL_FAM6_MODEL(ATOM_TREMONT_D, &idle_cpu_dnv), 1155b66b8b9aSAndi Kleen {} 1156b66b8b9aSAndi Kleen }; 1157b66b8b9aSAndi Kleen 115818734958SRafael J. Wysocki static const struct x86_cpu_id intel_mwait_ids[] __initconst = { 11594a9f45a0SThomas Gleixner X86_MATCH_VENDOR_FAM_FEATURE(INTEL, 6, X86_FEATURE_MWAIT, NULL), 116018734958SRafael J. Wysocki {} 116118734958SRafael J. Wysocki }; 116218734958SRafael J. Wysocki 1163095928aeSRafael J. Wysocki static bool __init intel_idle_max_cstate_reached(int cstate) 116418734958SRafael J. Wysocki { 116518734958SRafael J. Wysocki if (cstate + 1 > max_cstate) { 116618734958SRafael J. Wysocki pr_info("max_cstate %d reached\n", max_cstate); 116718734958SRafael J. Wysocki return true; 116818734958SRafael J. Wysocki } 116918734958SRafael J. Wysocki return false; 117018734958SRafael J. Wysocki } 117118734958SRafael J. Wysocki 117218734958SRafael J. Wysocki #ifdef CONFIG_ACPI_PROCESSOR_CSTATE 117318734958SRafael J. Wysocki #include <acpi/processor.h> 117418734958SRafael J. Wysocki 11754ec32d9eSRafael J. Wysocki static bool no_acpi __read_mostly; 11764ec32d9eSRafael J. Wysocki module_param(no_acpi, bool, 0444); 11774ec32d9eSRafael J. Wysocki MODULE_PARM_DESC(no_acpi, "Do not use ACPI _CST for building the idle states list"); 11784ec32d9eSRafael J. Wysocki 11793a5be9b8SRafael J. Wysocki static bool force_use_acpi __read_mostly; /* No effect if no_acpi is set. */ 11803a5be9b8SRafael J. Wysocki module_param_named(use_acpi, force_use_acpi, bool, 0444); 11813a5be9b8SRafael J. Wysocki MODULE_PARM_DESC(use_acpi, "Use ACPI _CST for building the idle states list"); 11823a5be9b8SRafael J. Wysocki 1183095928aeSRafael J. Wysocki static struct acpi_processor_power acpi_state_table __initdata; 118418734958SRafael J. Wysocki 118518734958SRafael J. Wysocki /** 118618734958SRafael J. Wysocki * intel_idle_cst_usable - Check if the _CST information can be used. 118718734958SRafael J. Wysocki * 118818734958SRafael J. Wysocki * Check if all of the C-states listed by _CST in the max_cstate range are 118918734958SRafael J. Wysocki * ACPI_CSTATE_FFH, which means that they should be entered via MWAIT. 119018734958SRafael J. Wysocki */ 1191095928aeSRafael J. Wysocki static bool __init intel_idle_cst_usable(void) 119218734958SRafael J. Wysocki { 119318734958SRafael J. Wysocki int cstate, limit; 119418734958SRafael J. Wysocki 119518734958SRafael J. Wysocki limit = min_t(int, min_t(int, CPUIDLE_STATE_MAX, max_cstate + 1), 119618734958SRafael J. Wysocki acpi_state_table.count); 119718734958SRafael J. Wysocki 119818734958SRafael J. Wysocki for (cstate = 1; cstate < limit; cstate++) { 119918734958SRafael J. Wysocki struct acpi_processor_cx *cx = &acpi_state_table.states[cstate]; 120018734958SRafael J. Wysocki 120118734958SRafael J. Wysocki if (cx->entry_method != ACPI_CSTATE_FFH) 120218734958SRafael J. Wysocki return false; 120318734958SRafael J. Wysocki } 120418734958SRafael J. Wysocki 120518734958SRafael J. Wysocki return true; 120618734958SRafael J. Wysocki } 120718734958SRafael J. Wysocki 1208095928aeSRafael J. Wysocki static bool __init intel_idle_acpi_cst_extract(void) 120918734958SRafael J. Wysocki { 121018734958SRafael J. Wysocki unsigned int cpu; 121118734958SRafael J. Wysocki 12124ec32d9eSRafael J. Wysocki if (no_acpi) { 12134ec32d9eSRafael J. Wysocki pr_debug("Not allowed to use ACPI _CST\n"); 12144ec32d9eSRafael J. Wysocki return false; 12154ec32d9eSRafael J. Wysocki } 12164ec32d9eSRafael J. Wysocki 121718734958SRafael J. Wysocki for_each_possible_cpu(cpu) { 121818734958SRafael J. Wysocki struct acpi_processor *pr = per_cpu(processors, cpu); 121918734958SRafael J. Wysocki 122018734958SRafael J. Wysocki if (!pr) 122118734958SRafael J. Wysocki continue; 122218734958SRafael J. Wysocki 122318734958SRafael J. Wysocki if (acpi_processor_evaluate_cst(pr->handle, cpu, &acpi_state_table)) 122418734958SRafael J. Wysocki continue; 122518734958SRafael J. Wysocki 122618734958SRafael J. Wysocki acpi_state_table.count++; 122718734958SRafael J. Wysocki 122818734958SRafael J. Wysocki if (!intel_idle_cst_usable()) 122918734958SRafael J. Wysocki continue; 123018734958SRafael J. Wysocki 123118734958SRafael J. Wysocki if (!acpi_processor_claim_cst_control()) { 123218734958SRafael J. Wysocki acpi_state_table.count = 0; 123318734958SRafael J. Wysocki return false; 123418734958SRafael J. Wysocki } 123518734958SRafael J. Wysocki 123618734958SRafael J. Wysocki return true; 123718734958SRafael J. Wysocki } 123818734958SRafael J. Wysocki 123918734958SRafael J. Wysocki pr_debug("ACPI _CST not found or not usable\n"); 124018734958SRafael J. Wysocki return false; 124118734958SRafael J. Wysocki } 124218734958SRafael J. Wysocki 1243095928aeSRafael J. Wysocki static void __init intel_idle_init_cstates_acpi(struct cpuidle_driver *drv) 124418734958SRafael J. Wysocki { 124518734958SRafael J. Wysocki int cstate, limit = min_t(int, CPUIDLE_STATE_MAX, acpi_state_table.count); 124618734958SRafael J. Wysocki 124718734958SRafael J. Wysocki /* 124818734958SRafael J. Wysocki * If limit > 0, intel_idle_cst_usable() has returned 'true', so all of 124918734958SRafael J. Wysocki * the interesting states are ACPI_CSTATE_FFH. 125018734958SRafael J. Wysocki */ 125118734958SRafael J. Wysocki for (cstate = 1; cstate < limit; cstate++) { 125218734958SRafael J. Wysocki struct acpi_processor_cx *cx; 125318734958SRafael J. Wysocki struct cpuidle_state *state; 125418734958SRafael J. Wysocki 125518734958SRafael J. Wysocki if (intel_idle_max_cstate_reached(cstate)) 125618734958SRafael J. Wysocki break; 125718734958SRafael J. Wysocki 125818734958SRafael J. Wysocki cx = &acpi_state_table.states[cstate]; 125918734958SRafael J. Wysocki 126018734958SRafael J. Wysocki state = &drv->states[drv->state_count++]; 126118734958SRafael J. Wysocki 126218734958SRafael J. Wysocki snprintf(state->name, CPUIDLE_NAME_LEN, "C%d_ACPI", cstate); 126318734958SRafael J. Wysocki strlcpy(state->desc, cx->desc, CPUIDLE_DESC_LEN); 126418734958SRafael J. Wysocki state->exit_latency = cx->latency; 126518734958SRafael J. Wysocki /* 126618734958SRafael J. Wysocki * For C1-type C-states use the same number for both the exit 126718734958SRafael J. Wysocki * latency and target residency, because that is the case for 126818734958SRafael J. Wysocki * C1 in the majority of the static C-states tables above. 126918734958SRafael J. Wysocki * For the other types of C-states, however, set the target 127018734958SRafael J. Wysocki * residency to 3 times the exit latency which should lead to 127118734958SRafael J. Wysocki * a reasonable balance between energy-efficiency and 127218734958SRafael J. Wysocki * performance in the majority of interesting cases. 127318734958SRafael J. Wysocki */ 127418734958SRafael J. Wysocki state->target_residency = cx->latency; 127518734958SRafael J. Wysocki if (cx->type > ACPI_STATE_C1) 127618734958SRafael J. Wysocki state->target_residency *= 3; 127718734958SRafael J. Wysocki 127818734958SRafael J. Wysocki state->flags = MWAIT2flg(cx->address); 127918734958SRafael J. Wysocki if (cx->type > ACPI_STATE_C2) 128018734958SRafael J. Wysocki state->flags |= CPUIDLE_FLAG_TLB_FLUSHED; 128118734958SRafael J. Wysocki 12824dcb78eeSRafael J. Wysocki if (disabled_states_mask & BIT(cstate)) 12834dcb78eeSRafael J. Wysocki state->flags |= CPUIDLE_FLAG_OFF; 12844dcb78eeSRafael J. Wysocki 128518734958SRafael J. Wysocki state->enter = intel_idle; 128618734958SRafael J. Wysocki state->enter_s2idle = intel_idle_s2idle; 128718734958SRafael J. Wysocki } 128818734958SRafael J. Wysocki } 1289bff8e60aSRafael J. Wysocki 1290095928aeSRafael J. Wysocki static bool __init intel_idle_off_by_default(u32 mwait_hint) 1291bff8e60aSRafael J. Wysocki { 1292bff8e60aSRafael J. Wysocki int cstate, limit; 1293bff8e60aSRafael J. Wysocki 1294bff8e60aSRafael J. Wysocki /* 1295bff8e60aSRafael J. Wysocki * If there are no _CST C-states, do not disable any C-states by 1296bff8e60aSRafael J. Wysocki * default. 1297bff8e60aSRafael J. Wysocki */ 1298bff8e60aSRafael J. Wysocki if (!acpi_state_table.count) 1299bff8e60aSRafael J. Wysocki return false; 1300bff8e60aSRafael J. Wysocki 1301bff8e60aSRafael J. Wysocki limit = min_t(int, CPUIDLE_STATE_MAX, acpi_state_table.count); 1302bff8e60aSRafael J. Wysocki /* 1303bff8e60aSRafael J. Wysocki * If limit > 0, intel_idle_cst_usable() has returned 'true', so all of 1304bff8e60aSRafael J. Wysocki * the interesting states are ACPI_CSTATE_FFH. 1305bff8e60aSRafael J. Wysocki */ 1306bff8e60aSRafael J. Wysocki for (cstate = 1; cstate < limit; cstate++) { 1307bff8e60aSRafael J. Wysocki if (acpi_state_table.states[cstate].address == mwait_hint) 1308bff8e60aSRafael J. Wysocki return false; 1309bff8e60aSRafael J. Wysocki } 1310bff8e60aSRafael J. Wysocki return true; 1311bff8e60aSRafael J. Wysocki } 131218734958SRafael J. Wysocki #else /* !CONFIG_ACPI_PROCESSOR_CSTATE */ 13133a5be9b8SRafael J. Wysocki #define force_use_acpi (false) 13143a5be9b8SRafael J. Wysocki 131518734958SRafael J. Wysocki static inline bool intel_idle_acpi_cst_extract(void) { return false; } 131618734958SRafael J. Wysocki static inline void intel_idle_init_cstates_acpi(struct cpuidle_driver *drv) { } 1317bff8e60aSRafael J. Wysocki static inline bool intel_idle_off_by_default(u32 mwait_hint) { return false; } 131818734958SRafael J. Wysocki #endif /* !CONFIG_ACPI_PROCESSOR_CSTATE */ 131918734958SRafael J. Wysocki 13206eacb15fSRafael J. Wysocki /** 13216eacb15fSRafael J. Wysocki * ivt_idle_state_table_update - Tune the idle states table for Ivy Town. 13220138d8f0SLen Brown * 13236eacb15fSRafael J. Wysocki * Tune IVT multi-socket targets. 13246eacb15fSRafael J. Wysocki * Assumption: num_sockets == (max_package_num + 1). 13250138d8f0SLen Brown */ 1326095928aeSRafael J. Wysocki static void __init ivt_idle_state_table_update(void) 13270138d8f0SLen Brown { 13280138d8f0SLen Brown /* IVT uses a different table for 1-2, 3-4, and > 4 sockets */ 13290138d8f0SLen Brown int cpu, package_num, num_sockets = 1; 13300138d8f0SLen Brown 13310138d8f0SLen Brown for_each_online_cpu(cpu) { 13320138d8f0SLen Brown package_num = topology_physical_package_id(cpu); 13330138d8f0SLen Brown if (package_num + 1 > num_sockets) { 13340138d8f0SLen Brown num_sockets = package_num + 1; 13350138d8f0SLen Brown 1336d27dca42SChristoph Jaeger if (num_sockets > 4) { 13370138d8f0SLen Brown cpuidle_state_table = ivt_cstates_8s; 13380138d8f0SLen Brown return; 13390138d8f0SLen Brown } 13400138d8f0SLen Brown } 1341d27dca42SChristoph Jaeger } 13420138d8f0SLen Brown 13430138d8f0SLen Brown if (num_sockets > 2) 13440138d8f0SLen Brown cpuidle_state_table = ivt_cstates_4s; 1345d70e28f5SLen Brown 13460138d8f0SLen Brown /* else, 1 and 2 socket systems use default ivt_cstates */ 13470138d8f0SLen Brown } 13485dcef694SLen Brown 134986e9466aSRafael J. Wysocki /** 135086e9466aSRafael J. Wysocki * irtl_2_usec - IRTL to microseconds conversion. 135186e9466aSRafael J. Wysocki * @irtl: IRTL MSR value. 135286e9466aSRafael J. Wysocki * 135386e9466aSRafael J. Wysocki * Translate the IRTL (Interrupt Response Time Limit) MSR value to microseconds. 13545dcef694SLen Brown */ 135586e9466aSRafael J. Wysocki static unsigned long long __init irtl_2_usec(unsigned long long irtl) 135686e9466aSRafael J. Wysocki { 1357095928aeSRafael J. Wysocki static const unsigned int irtl_ns_units[] __initconst = { 1358095928aeSRafael J. Wysocki 1, 32, 1024, 32768, 1048576, 33554432, 0, 0 1359095928aeSRafael J. Wysocki }; 13605dcef694SLen Brown unsigned long long ns; 13615dcef694SLen Brown 13623451ab3eSJan Beulich if (!irtl) 13633451ab3eSJan Beulich return 0; 13643451ab3eSJan Beulich 1365bef45096SJan Beulich ns = irtl_ns_units[(irtl >> 10) & 0x7]; 13665dcef694SLen Brown 136786e9466aSRafael J. Wysocki return div_u64((irtl & 0x3FF) * ns, NSEC_PER_USEC); 13685dcef694SLen Brown } 136986e9466aSRafael J. Wysocki 13706eacb15fSRafael J. Wysocki /** 13716eacb15fSRafael J. Wysocki * bxt_idle_state_table_update - Fix up the Broxton idle states table. 13725dcef694SLen Brown * 13736eacb15fSRafael J. Wysocki * On BXT, trust the IRTL (Interrupt Response Time Limit) MSR to show the 13746eacb15fSRafael J. Wysocki * definitive maximum latency and use the same value for target_residency. 13755dcef694SLen Brown */ 1376095928aeSRafael J. Wysocki static void __init bxt_idle_state_table_update(void) 13775dcef694SLen Brown { 13785dcef694SLen Brown unsigned long long msr; 13793451ab3eSJan Beulich unsigned int usec; 13805dcef694SLen Brown 13815dcef694SLen Brown rdmsrl(MSR_PKGC6_IRTL, msr); 13823451ab3eSJan Beulich usec = irtl_2_usec(msr); 13833451ab3eSJan Beulich if (usec) { 13845dcef694SLen Brown bxt_cstates[2].exit_latency = usec; 13855dcef694SLen Brown bxt_cstates[2].target_residency = usec; 13865dcef694SLen Brown } 13875dcef694SLen Brown 13885dcef694SLen Brown rdmsrl(MSR_PKGC7_IRTL, msr); 13893451ab3eSJan Beulich usec = irtl_2_usec(msr); 13903451ab3eSJan Beulich if (usec) { 13915dcef694SLen Brown bxt_cstates[3].exit_latency = usec; 13925dcef694SLen Brown bxt_cstates[3].target_residency = usec; 13935dcef694SLen Brown } 13945dcef694SLen Brown 13955dcef694SLen Brown rdmsrl(MSR_PKGC8_IRTL, msr); 13963451ab3eSJan Beulich usec = irtl_2_usec(msr); 13973451ab3eSJan Beulich if (usec) { 13985dcef694SLen Brown bxt_cstates[4].exit_latency = usec; 13995dcef694SLen Brown bxt_cstates[4].target_residency = usec; 14005dcef694SLen Brown } 14015dcef694SLen Brown 14025dcef694SLen Brown rdmsrl(MSR_PKGC9_IRTL, msr); 14033451ab3eSJan Beulich usec = irtl_2_usec(msr); 14043451ab3eSJan Beulich if (usec) { 14055dcef694SLen Brown bxt_cstates[5].exit_latency = usec; 14065dcef694SLen Brown bxt_cstates[5].target_residency = usec; 14075dcef694SLen Brown } 14085dcef694SLen Brown 14095dcef694SLen Brown rdmsrl(MSR_PKGC10_IRTL, msr); 14103451ab3eSJan Beulich usec = irtl_2_usec(msr); 14113451ab3eSJan Beulich if (usec) { 14125dcef694SLen Brown bxt_cstates[6].exit_latency = usec; 14135dcef694SLen Brown bxt_cstates[6].target_residency = usec; 14145dcef694SLen Brown } 14155dcef694SLen Brown 14165dcef694SLen Brown } 14176eacb15fSRafael J. Wysocki 14186eacb15fSRafael J. Wysocki /** 14196eacb15fSRafael J. Wysocki * sklh_idle_state_table_update - Fix up the Sky Lake idle states table. 1420d70e28f5SLen Brown * 14216eacb15fSRafael J. Wysocki * On SKL-H (model 0x5e) skip C8 and C9 if C10 is enabled and SGX disabled. 1422d70e28f5SLen Brown */ 1423095928aeSRafael J. Wysocki static void __init sklh_idle_state_table_update(void) 1424d70e28f5SLen Brown { 1425d70e28f5SLen Brown unsigned long long msr; 1426d70e28f5SLen Brown unsigned int eax, ebx, ecx, edx; 1427d70e28f5SLen Brown 1428d70e28f5SLen Brown 1429d70e28f5SLen Brown /* if PC10 disabled via cmdline intel_idle.max_cstate=7 or shallower */ 1430d70e28f5SLen Brown if (max_cstate <= 7) 14310138d8f0SLen Brown return; 1432d70e28f5SLen Brown 1433d70e28f5SLen Brown /* if PC10 not present in CPUID.MWAIT.EDX */ 1434d70e28f5SLen Brown if ((mwait_substates & (0xF << 28)) == 0) 1435d70e28f5SLen Brown return; 1436d70e28f5SLen Brown 14376cfb2374SLen Brown rdmsrl(MSR_PKG_CST_CONFIG_CONTROL, msr); 1438d70e28f5SLen Brown 1439d70e28f5SLen Brown /* PC10 is not enabled in PKG C-state limit */ 1440d70e28f5SLen Brown if ((msr & 0xF) != 8) 1441d70e28f5SLen Brown return; 1442d70e28f5SLen Brown 1443d70e28f5SLen Brown ecx = 0; 1444d70e28f5SLen Brown cpuid(7, &eax, &ebx, &ecx, &edx); 1445d70e28f5SLen Brown 1446d70e28f5SLen Brown /* if SGX is present */ 1447d70e28f5SLen Brown if (ebx & (1 << 2)) { 1448d70e28f5SLen Brown 144932ad73dbSSean Christopherson rdmsrl(MSR_IA32_FEAT_CTL, msr); 1450d70e28f5SLen Brown 1451d70e28f5SLen Brown /* if SGX is enabled */ 1452d70e28f5SLen Brown if (msr & (1 << 18)) 1453d70e28f5SLen Brown return; 1454d70e28f5SLen Brown } 1455d70e28f5SLen Brown 1456ba1e78a1SRafael J. Wysocki skl_cstates[5].flags |= CPUIDLE_FLAG_UNUSABLE; /* C8-SKL */ 1457ba1e78a1SRafael J. Wysocki skl_cstates[6].flags |= CPUIDLE_FLAG_UNUSABLE; /* C9-SKL */ 1458d70e28f5SLen Brown } 1459d70e28f5SLen Brown 14601aefbd7aSRafael J. Wysocki static bool __init intel_idle_verify_cstate(unsigned int mwait_hint) 14611aefbd7aSRafael J. Wysocki { 14621aefbd7aSRafael J. Wysocki unsigned int mwait_cstate = MWAIT_HINT2CSTATE(mwait_hint) + 1; 14631aefbd7aSRafael J. Wysocki unsigned int num_substates = (mwait_substates >> mwait_cstate * 4) & 14641aefbd7aSRafael J. Wysocki MWAIT_SUBSTATE_MASK; 14651aefbd7aSRafael J. Wysocki 14661aefbd7aSRafael J. Wysocki /* Ignore the C-state if there are NO sub-states in CPUID for it. */ 14671aefbd7aSRafael J. Wysocki if (num_substates == 0) 14681aefbd7aSRafael J. Wysocki return false; 14691aefbd7aSRafael J. Wysocki 14701aefbd7aSRafael J. Wysocki if (mwait_cstate > 2 && !boot_cpu_has(X86_FEATURE_NONSTOP_TSC)) 14711aefbd7aSRafael J. Wysocki mark_tsc_unstable("TSC halts in idle states deeper than C2"); 14721aefbd7aSRafael J. Wysocki 14731aefbd7aSRafael J. Wysocki return true; 14741aefbd7aSRafael J. Wysocki } 14751aefbd7aSRafael J. Wysocki 1476095928aeSRafael J. Wysocki static void __init intel_idle_init_cstates_icpu(struct cpuidle_driver *drv) 1477d70e28f5SLen Brown { 14783d3a1ae9SRafael J. Wysocki int cstate; 1479d70e28f5SLen Brown 14803d3a1ae9SRafael J. Wysocki switch (boot_cpu_data.x86_model) { 1481db73c5a8SDave Hansen case INTEL_FAM6_IVYBRIDGE_X: 1482d70e28f5SLen Brown ivt_idle_state_table_update(); 1483d70e28f5SLen Brown break; 1484db73c5a8SDave Hansen case INTEL_FAM6_ATOM_GOLDMONT: 1485f2c4db1bSPeter Zijlstra case INTEL_FAM6_ATOM_GOLDMONT_PLUS: 14865dcef694SLen Brown bxt_idle_state_table_update(); 14875dcef694SLen Brown break; 1488c66f78a6SPeter Zijlstra case INTEL_FAM6_SKYLAKE: 1489d70e28f5SLen Brown sklh_idle_state_table_update(); 1490d70e28f5SLen Brown break; 1491d70e28f5SLen Brown } 149246bcfad7SDeepthi Dharwar 1493e022e7ebSLen Brown for (cstate = 0; cstate < CPUIDLE_STATE_MAX; ++cstate) { 14949f3d6dafSRafael J. Wysocki unsigned int mwait_hint; 149546bcfad7SDeepthi Dharwar 149618734958SRafael J. Wysocki if (intel_idle_max_cstate_reached(cstate)) 149718734958SRafael J. Wysocki break; 149818734958SRafael J. Wysocki 14999f3d6dafSRafael J. Wysocki if (!cpuidle_state_table[cstate].enter && 15009f3d6dafSRafael J. Wysocki !cpuidle_state_table[cstate].enter_s2idle) 1501e022e7ebSLen Brown break; 1502e022e7ebSLen Brown 15039f3d6dafSRafael J. Wysocki /* If marked as unusable, skip this state. */ 1504ba1e78a1SRafael J. Wysocki if (cpuidle_state_table[cstate].flags & CPUIDLE_FLAG_UNUSABLE) { 1505654d08a4SJoe Perches pr_debug("state %s is disabled\n", 1506d70e28f5SLen Brown cpuidle_state_table[cstate].name); 1507d70e28f5SLen Brown continue; 1508d70e28f5SLen Brown } 1509d70e28f5SLen Brown 15109f3d6dafSRafael J. Wysocki mwait_hint = flg2MWAIT(cpuidle_state_table[cstate].flags); 15119f3d6dafSRafael J. Wysocki if (!intel_idle_verify_cstate(mwait_hint)) 15129f3d6dafSRafael J. Wysocki continue; 1513d70e28f5SLen Brown 15149f3d6dafSRafael J. Wysocki /* Structure copy. */ 1515bff8e60aSRafael J. Wysocki drv->states[drv->state_count] = cpuidle_state_table[cstate]; 1516bff8e60aSRafael J. Wysocki 15174dcb78eeSRafael J. Wysocki if ((disabled_states_mask & BIT(drv->state_count)) || 15184dcb78eeSRafael J. Wysocki ((icpu->use_acpi || force_use_acpi) && 15193a5be9b8SRafael J. Wysocki intel_idle_off_by_default(mwait_hint) && 15204dcb78eeSRafael J. Wysocki !(cpuidle_state_table[cstate].flags & CPUIDLE_FLAG_ALWAYS_ENABLE))) 1521bff8e60aSRafael J. Wysocki drv->states[drv->state_count].flags |= CPUIDLE_FLAG_OFF; 1522bff8e60aSRafael J. Wysocki 1523bff8e60aSRafael J. Wysocki drv->state_count++; 152446bcfad7SDeepthi Dharwar } 152546bcfad7SDeepthi Dharwar 15268c058d53SLen Brown if (icpu->byt_auto_demotion_disable_flag) { 15278c058d53SLen Brown wrmsrl(MSR_CC6_DEMOTION_POLICY_CONFIG, 0); 15288c058d53SLen Brown wrmsrl(MSR_MC6_DEMOTION_POLICY_CONFIG, 0); 15298c058d53SLen Brown } 153046bcfad7SDeepthi Dharwar } 153146bcfad7SDeepthi Dharwar 15326eacb15fSRafael J. Wysocki /** 15336eacb15fSRafael J. Wysocki * intel_idle_cpuidle_driver_init - Create the list of available idle states. 15346eacb15fSRafael J. Wysocki * @drv: cpuidle driver structure to initialize. 153518734958SRafael J. Wysocki */ 15363d3a1ae9SRafael J. Wysocki static void __init intel_idle_cpuidle_driver_init(struct cpuidle_driver *drv) 153718734958SRafael J. Wysocki { 153818734958SRafael J. Wysocki cpuidle_poll_state_init(drv); 15394dcb78eeSRafael J. Wysocki 15404dcb78eeSRafael J. Wysocki if (disabled_states_mask & BIT(0)) 15414dcb78eeSRafael J. Wysocki drv->states[0].flags |= CPUIDLE_FLAG_OFF; 15424dcb78eeSRafael J. Wysocki 154318734958SRafael J. Wysocki drv->state_count = 1; 154418734958SRafael J. Wysocki 154518734958SRafael J. Wysocki if (icpu) 154618734958SRafael J. Wysocki intel_idle_init_cstates_icpu(drv); 154718734958SRafael J. Wysocki else 154818734958SRafael J. Wysocki intel_idle_init_cstates_acpi(drv); 154918734958SRafael J. Wysocki } 155046bcfad7SDeepthi Dharwar 15511aefbd7aSRafael J. Wysocki static void auto_demotion_disable(void) 15521aefbd7aSRafael J. Wysocki { 15531aefbd7aSRafael J. Wysocki unsigned long long msr_bits; 15541aefbd7aSRafael J. Wysocki 15551aefbd7aSRafael J. Wysocki rdmsrl(MSR_PKG_CST_CONFIG_CONTROL, msr_bits); 15567f843dd7SRafael J. Wysocki msr_bits &= ~auto_demotion_disable_flags; 15571aefbd7aSRafael J. Wysocki wrmsrl(MSR_PKG_CST_CONFIG_CONTROL, msr_bits); 15581aefbd7aSRafael J. Wysocki } 15591aefbd7aSRafael J. Wysocki 15601aefbd7aSRafael J. Wysocki static void c1e_promotion_disable(void) 15611aefbd7aSRafael J. Wysocki { 15621aefbd7aSRafael J. Wysocki unsigned long long msr_bits; 15631aefbd7aSRafael J. Wysocki 15641aefbd7aSRafael J. Wysocki rdmsrl(MSR_IA32_POWER_CTL, msr_bits); 15651aefbd7aSRafael J. Wysocki msr_bits &= ~0x2; 15661aefbd7aSRafael J. Wysocki wrmsrl(MSR_IA32_POWER_CTL, msr_bits); 15671aefbd7aSRafael J. Wysocki } 15681aefbd7aSRafael J. Wysocki 15696eacb15fSRafael J. Wysocki /** 15706eacb15fSRafael J. Wysocki * intel_idle_cpu_init - Register the target CPU with the cpuidle core. 15716eacb15fSRafael J. Wysocki * @cpu: CPU to initialize. 15726eacb15fSRafael J. Wysocki * 15736eacb15fSRafael J. Wysocki * Register a cpuidle device object for @cpu and update its MSRs in accordance 15746eacb15fSRafael J. Wysocki * with the processor model flags. 157526717172SLen Brown */ 1576fb1013a0SSebastian Andrzej Siewior static int intel_idle_cpu_init(unsigned int cpu) 157726717172SLen Brown { 157826717172SLen Brown struct cpuidle_device *dev; 157926717172SLen Brown 158065b7f839SThomas Renninger dev = per_cpu_ptr(intel_idle_cpuidle_devices, cpu); 158165b7f839SThomas Renninger dev->cpu = cpu; 158226717172SLen Brown 158326717172SLen Brown if (cpuidle_register_device(dev)) { 1584654d08a4SJoe Perches pr_debug("cpuidle_register_device %d failed!\n", cpu); 158526717172SLen Brown return -EIO; 158626717172SLen Brown } 158765b7f839SThomas Renninger 15887f843dd7SRafael J. Wysocki if (auto_demotion_disable_flags) 1589fb1013a0SSebastian Andrzej Siewior auto_demotion_disable(); 159026717172SLen Brown 15917f843dd7SRafael J. Wysocki if (disable_promotion_to_c1e) 1592fb1013a0SSebastian Andrzej Siewior c1e_promotion_disable(); 1593fb1013a0SSebastian Andrzej Siewior 1594fb1013a0SSebastian Andrzej Siewior return 0; 1595fb1013a0SSebastian Andrzej Siewior } 1596fb1013a0SSebastian Andrzej Siewior 1597fb1013a0SSebastian Andrzej Siewior static int intel_idle_cpu_online(unsigned int cpu) 1598fb1013a0SSebastian Andrzej Siewior { 1599fb1013a0SSebastian Andrzej Siewior struct cpuidle_device *dev; 1600fb1013a0SSebastian Andrzej Siewior 1601dab20177SRafael J. Wysocki if (!boot_cpu_has(X86_FEATURE_ARAT)) 1602cbd2c4c2SRafael J. Wysocki tick_broadcast_enable(); 1603fb1013a0SSebastian Andrzej Siewior 1604fb1013a0SSebastian Andrzej Siewior /* 1605fb1013a0SSebastian Andrzej Siewior * Some systems can hotplug a cpu at runtime after 1606fb1013a0SSebastian Andrzej Siewior * the kernel has booted, we have to initialize the 1607fb1013a0SSebastian Andrzej Siewior * driver in this case 1608fb1013a0SSebastian Andrzej Siewior */ 1609fb1013a0SSebastian Andrzej Siewior dev = per_cpu_ptr(intel_idle_cpuidle_devices, cpu); 1610fb1013a0SSebastian Andrzej Siewior if (!dev->registered) 1611fb1013a0SSebastian Andrzej Siewior return intel_idle_cpu_init(cpu); 1612dbf87ab8SBartlomiej Zolnierkiewicz 161326717172SLen Brown return 0; 161426717172SLen Brown } 161526717172SLen Brown 16160755a9bdSRafael J. Wysocki /** 16170755a9bdSRafael J. Wysocki * intel_idle_cpuidle_devices_uninit - Unregister all cpuidle devices. 16180755a9bdSRafael J. Wysocki */ 16190755a9bdSRafael J. Wysocki static void __init intel_idle_cpuidle_devices_uninit(void) 16200755a9bdSRafael J. Wysocki { 16210755a9bdSRafael J. Wysocki int i; 16220755a9bdSRafael J. Wysocki 16230755a9bdSRafael J. Wysocki for_each_online_cpu(i) 16240755a9bdSRafael J. Wysocki cpuidle_unregister_device(per_cpu_ptr(intel_idle_cpuidle_devices, i)); 16250755a9bdSRafael J. Wysocki } 16260755a9bdSRafael J. Wysocki 162726717172SLen Brown static int __init intel_idle_init(void) 162826717172SLen Brown { 1629a6c86e33SRafael J. Wysocki const struct x86_cpu_id *id; 1630a6c86e33SRafael J. Wysocki unsigned int eax, ebx, ecx; 1631fb1013a0SSebastian Andrzej Siewior int retval; 163226717172SLen Brown 1633d1896049SThomas Renninger /* Do not load intel_idle at all for now if idle= is passed */ 1634d1896049SThomas Renninger if (boot_option_idle_override != IDLE_NO_OVERRIDE) 1635d1896049SThomas Renninger return -ENODEV; 1636d1896049SThomas Renninger 1637a6c86e33SRafael J. Wysocki if (max_cstate == 0) { 1638a6c86e33SRafael J. Wysocki pr_debug("disabled\n"); 1639a6c86e33SRafael J. Wysocki return -EPERM; 1640a6c86e33SRafael J. Wysocki } 1641a6c86e33SRafael J. Wysocki 1642a6c86e33SRafael J. Wysocki id = x86_match_cpu(intel_idle_ids); 1643a6c86e33SRafael J. Wysocki if (id) { 1644a6c86e33SRafael J. Wysocki if (!boot_cpu_has(X86_FEATURE_MWAIT)) { 1645a6c86e33SRafael J. Wysocki pr_debug("Please enable MWAIT in BIOS SETUP\n"); 1646a6c86e33SRafael J. Wysocki return -ENODEV; 1647a6c86e33SRafael J. Wysocki } 1648a6c86e33SRafael J. Wysocki } else { 1649a6c86e33SRafael J. Wysocki id = x86_match_cpu(intel_mwait_ids); 1650a6c86e33SRafael J. Wysocki if (!id) 1651a6c86e33SRafael J. Wysocki return -ENODEV; 1652a6c86e33SRafael J. Wysocki } 1653a6c86e33SRafael J. Wysocki 1654a6c86e33SRafael J. Wysocki if (boot_cpu_data.cpuid_level < CPUID_MWAIT_LEAF) 1655a6c86e33SRafael J. Wysocki return -ENODEV; 1656a6c86e33SRafael J. Wysocki 1657a6c86e33SRafael J. Wysocki cpuid(CPUID_MWAIT_LEAF, &eax, &ebx, &ecx, &mwait_substates); 1658a6c86e33SRafael J. Wysocki 1659a6c86e33SRafael J. Wysocki if (!(ecx & CPUID5_ECX_EXTENSIONS_SUPPORTED) || 1660a6c86e33SRafael J. Wysocki !(ecx & CPUID5_ECX_INTERRUPT_BREAK) || 1661a6c86e33SRafael J. Wysocki !mwait_substates) 1662a6c86e33SRafael J. Wysocki return -ENODEV; 1663a6c86e33SRafael J. Wysocki 1664a6c86e33SRafael J. Wysocki pr_debug("MWAIT substates: 0x%x\n", mwait_substates); 1665a6c86e33SRafael J. Wysocki 1666a6c86e33SRafael J. Wysocki icpu = (const struct idle_cpu *)id->driver_data; 1667a6c86e33SRafael J. Wysocki if (icpu) { 1668a6c86e33SRafael J. Wysocki cpuidle_state_table = icpu->state_table; 16697f843dd7SRafael J. Wysocki auto_demotion_disable_flags = icpu->auto_demotion_disable_flags; 16707f843dd7SRafael J. Wysocki disable_promotion_to_c1e = icpu->disable_promotion_to_c1e; 16713a5be9b8SRafael J. Wysocki if (icpu->use_acpi || force_use_acpi) 1672a6c86e33SRafael J. Wysocki intel_idle_acpi_cst_extract(); 1673a6c86e33SRafael J. Wysocki } else if (!intel_idle_acpi_cst_extract()) { 1674a6c86e33SRafael J. Wysocki return -ENODEV; 1675a6c86e33SRafael J. Wysocki } 1676a6c86e33SRafael J. Wysocki 1677a6c86e33SRafael J. Wysocki pr_debug("v" INTEL_IDLE_VERSION " model 0x%X\n", 1678a6c86e33SRafael J. Wysocki boot_cpu_data.x86_model); 167926717172SLen Brown 1680e9df69ccSRichard Cochran intel_idle_cpuidle_devices = alloc_percpu(struct cpuidle_device); 1681533da74aSRafael J. Wysocki if (!intel_idle_cpuidle_devices) 1682e9df69ccSRichard Cochran return -ENOMEM; 1683e9df69ccSRichard Cochran 16843d3a1ae9SRafael J. Wysocki intel_idle_cpuidle_driver_init(&intel_idle_driver); 16853d3a1ae9SRafael J. Wysocki 168626717172SLen Brown retval = cpuidle_register_driver(&intel_idle_driver); 168726717172SLen Brown if (retval) { 16883735d524SKonrad Rzeszutek Wilk struct cpuidle_driver *drv = cpuidle_get_driver(); 1689654d08a4SJoe Perches printk(KERN_DEBUG pr_fmt("intel_idle yielding to %s\n"), 16903735d524SKonrad Rzeszutek Wilk drv ? drv->name : "none"); 1691fb1013a0SSebastian Andrzej Siewior goto init_driver_fail; 169226717172SLen Brown } 169326717172SLen Brown 1694fb1013a0SSebastian Andrzej Siewior retval = cpuhp_setup_state(CPUHP_AP_ONLINE_DYN, "idle/intel:online", 1695fb1013a0SSebastian Andrzej Siewior intel_idle_cpu_online, NULL); 1696fb1013a0SSebastian Andrzej Siewior if (retval < 0) 1697fb1013a0SSebastian Andrzej Siewior goto hp_setup_fail; 169826717172SLen Brown 169940ab82e0SRafael J. Wysocki pr_debug("Local APIC timer is reliable in %s\n", 1700dab20177SRafael J. Wysocki boot_cpu_has(X86_FEATURE_ARAT) ? "all C-states" : "C1"); 17012259a819SRichard Cochran 170226717172SLen Brown return 0; 1703fb1013a0SSebastian Andrzej Siewior 1704fb1013a0SSebastian Andrzej Siewior hp_setup_fail: 1705fb1013a0SSebastian Andrzej Siewior intel_idle_cpuidle_devices_uninit(); 1706fb1013a0SSebastian Andrzej Siewior cpuidle_unregister_driver(&intel_idle_driver); 1707fb1013a0SSebastian Andrzej Siewior init_driver_fail: 1708fb1013a0SSebastian Andrzej Siewior free_percpu(intel_idle_cpuidle_devices); 1709fb1013a0SSebastian Andrzej Siewior return retval; 1710fb1013a0SSebastian Andrzej Siewior 171126717172SLen Brown } 171202c4fae9SPaul Gortmaker device_initcall(intel_idle_init); 171326717172SLen Brown 171402c4fae9SPaul Gortmaker /* 171502c4fae9SPaul Gortmaker * We are not really modular, but we used to support that. Meaning we also 171602c4fae9SPaul Gortmaker * support "intel_idle.max_cstate=..." at boot and also a read-only export of 171702c4fae9SPaul Gortmaker * it at /sys/module/intel_idle/parameters/max_cstate -- so using module_param 171802c4fae9SPaul Gortmaker * is the easiest way (currently) to continue doing that. 171902c4fae9SPaul Gortmaker */ 172026717172SLen Brown module_param(max_cstate, int, 0444); 17214dcb78eeSRafael J. Wysocki /* 17224dcb78eeSRafael J. Wysocki * The positions of the bits that are set in this number are the indices of the 17234dcb78eeSRafael J. Wysocki * idle states to be disabled by default (as reflected by the names of the 17244dcb78eeSRafael J. Wysocki * corresponding idle state directories in sysfs, "state0", "state1" ... 17254dcb78eeSRafael J. Wysocki * "state<i>" ..., where <i> is the index of the given state). 17264dcb78eeSRafael J. Wysocki */ 17274dcb78eeSRafael J. Wysocki module_param_named(states_off, disabled_states_mask, uint, 0444); 17284dcb78eeSRafael J. Wysocki MODULE_PARM_DESC(states_off, "Mask of disabled idle states"); 1729