1a61127c2SThomas Gleixner // SPDX-License-Identifier: GPL-2.0-only 226717172SLen Brown /* 326717172SLen Brown * intel_idle.c - native hardware idle loop for modern Intel processors 426717172SLen Brown * 5317e5ec3SRafael J. Wysocki * Copyright (c) 2013 - 2020, Intel Corporation. 626717172SLen Brown * Len Brown <len.brown@intel.com> 7317e5ec3SRafael J. Wysocki * Rafael J. Wysocki <rafael.j.wysocki@intel.com> 826717172SLen Brown */ 926717172SLen Brown 1026717172SLen Brown /* 118bb2e2a8SAlexander Monakov * intel_idle is a cpuidle driver that loads on all Intel CPUs with MWAIT 1226717172SLen Brown * in lieu of the legacy ACPI processor_idle driver. The intent is to 1326717172SLen Brown * make Linux more efficient on these processors, as intel_idle knows 1426717172SLen Brown * more than ACPI, as well as make Linux more immune to ACPI BIOS bugs. 1526717172SLen Brown */ 1626717172SLen Brown 1726717172SLen Brown /* 1826717172SLen Brown * Design Assumptions 1926717172SLen Brown * 2026717172SLen Brown * All CPUs have same idle states as boot CPU 2126717172SLen Brown * 2226717172SLen Brown * Chipset BM_STS (bus master status) bit is a NOP 238bb2e2a8SAlexander Monakov * for preventing entry into deep C-states 248bb2e2a8SAlexander Monakov * 258bb2e2a8SAlexander Monakov * CPU will flush caches as needed when entering a C-state via MWAIT 268bb2e2a8SAlexander Monakov * (in contrast to entering ACPI C3, in which case the WBINVD 278bb2e2a8SAlexander Monakov * instruction needs to be executed to flush the caches) 2826717172SLen Brown */ 2926717172SLen Brown 3026717172SLen Brown /* 3126717172SLen Brown * Known limitations 3226717172SLen Brown * 3326717172SLen Brown * ACPI has a .suspend hack to turn off deep c-statees during suspend 3426717172SLen Brown * to avoid complications with the lapic timer workaround. 3526717172SLen Brown * Have not seen issues with suspend, but may need same workaround here. 3626717172SLen Brown * 3726717172SLen Brown */ 3826717172SLen Brown 3926717172SLen Brown /* un-comment DEBUG to enable pr_debug() statements */ 40651bc581STom Rix /* #define DEBUG */ 4126717172SLen Brown 42654d08a4SJoe Perches #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt 43654d08a4SJoe Perches 4418734958SRafael J. Wysocki #include <linux/acpi.h> 4526717172SLen Brown #include <linux/kernel.h> 4626717172SLen Brown #include <linux/cpuidle.h> 4776962caaSThomas Gleixner #include <linux/tick.h> 4826717172SLen Brown #include <trace/events/power.h> 4926717172SLen Brown #include <linux/sched.h> 502a2d31c8SShaohua Li #include <linux/notifier.h> 512a2d31c8SShaohua Li #include <linux/cpu.h> 5202c4fae9SPaul Gortmaker #include <linux/moduleparam.h> 53b66b8b9aSAndi Kleen #include <asm/cpu_device_id.h> 54db73c5a8SDave Hansen #include <asm/intel-family.h> 55bc83ccccSH. Peter Anvin #include <asm/mwait.h> 5614796fcaSLen Brown #include <asm/msr.h> 5726717172SLen Brown 58317e5ec3SRafael J. Wysocki #define INTEL_IDLE_VERSION "0.5.1" 5926717172SLen Brown 6026717172SLen Brown static struct cpuidle_driver intel_idle_driver = { 6126717172SLen Brown .name = "intel_idle", 6226717172SLen Brown .owner = THIS_MODULE, 6326717172SLen Brown }; 6426717172SLen Brown /* intel_idle.max_cstate=0 disables driver */ 65137ecc77SLen Brown static int max_cstate = CPUIDLE_STATE_MAX - 1; 664dcb78eeSRafael J. Wysocki static unsigned int disabled_states_mask; 67da0e58c0SArtem Bityutskiy static unsigned int preferred_states_mask; 6826717172SLen Brown 696eb0443aSRafael J. Wysocki static struct cpuidle_device __percpu *intel_idle_cpuidle_devices; 707f843dd7SRafael J. Wysocki 717f843dd7SRafael J. Wysocki static unsigned long auto_demotion_disable_flags; 727f843dd7SRafael J. Wysocki static bool disable_promotion_to_c1e; 737f843dd7SRafael J. Wysocki 74b66b8b9aSAndi Kleen struct idle_cpu { 75b66b8b9aSAndi Kleen struct cpuidle_state *state_table; 7626717172SLen Brown 7726717172SLen Brown /* 7814796fcaSLen Brown * Hardware C-state auto-demotion may not always be optimal. 7914796fcaSLen Brown * Indicate which enable bits to clear here. 8014796fcaSLen Brown */ 81b66b8b9aSAndi Kleen unsigned long auto_demotion_disable_flags; 828c058d53SLen Brown bool byt_auto_demotion_disable_flag; 8332e95180SLen Brown bool disable_promotion_to_c1e; 84bff8e60aSRafael J. Wysocki bool use_acpi; 85b66b8b9aSAndi Kleen }; 86b66b8b9aSAndi Kleen 877f843dd7SRafael J. Wysocki static const struct idle_cpu *icpu __initdata; 887f843dd7SRafael J. Wysocki static struct cpuidle_state *cpuidle_state_table __initdata; 8914796fcaSLen Brown 906eb0443aSRafael J. Wysocki static unsigned int mwait_substates __initdata; 916eb0443aSRafael J. Wysocki 9214796fcaSLen Brown /* 93c227233aSArtem Bityutskiy * Enable interrupts before entering the C-state. On some platforms and for 94c227233aSArtem Bityutskiy * some C-states, this may measurably decrease interrupt latency. 95c227233aSArtem Bityutskiy */ 96c227233aSArtem Bityutskiy #define CPUIDLE_FLAG_IRQ_ENABLE BIT(14) 97c227233aSArtem Bityutskiy 98c227233aSArtem Bityutskiy /* 99bff8e60aSRafael J. Wysocki * Enable this state by default even if the ACPI _CST does not list it. 100bff8e60aSRafael J. Wysocki */ 101bff8e60aSRafael J. Wysocki #define CPUIDLE_FLAG_ALWAYS_ENABLE BIT(15) 102bff8e60aSRafael J. Wysocki 103bff8e60aSRafael J. Wysocki /* 104b1beab48SLen Brown * MWAIT takes an 8-bit "hint" in EAX "suggesting" 105b1beab48SLen Brown * the C-state (top nibble) and sub-state (bottom nibble) 106b1beab48SLen Brown * 0x00 means "MWAIT(C1)", 0x10 means "MWAIT(C2)" etc. 107b1beab48SLen Brown * 108b1beab48SLen Brown * We store the hint at the top of our "flags" for each state. 109b1beab48SLen Brown */ 110b1beab48SLen Brown #define flg2MWAIT(flags) (((flags) >> 24) & 0xFF) 111b1beab48SLen Brown #define MWAIT2flg(eax) ((eax & 0xFF) << 24) 112b1beab48SLen Brown 11330a996fbSRafael J. Wysocki /** 11430a996fbSRafael J. Wysocki * intel_idle - Ask the processor to enter the given idle state. 11530a996fbSRafael J. Wysocki * @dev: cpuidle device of the target CPU. 11630a996fbSRafael J. Wysocki * @drv: cpuidle driver (assumed to point to intel_idle_driver). 11730a996fbSRafael J. Wysocki * @index: Target idle state index. 11830a996fbSRafael J. Wysocki * 11930a996fbSRafael J. Wysocki * Use the MWAIT instruction to notify the processor that the CPU represented by 12030a996fbSRafael J. Wysocki * @dev is idle and it can try to enter the idle state corresponding to @index. 12130a996fbSRafael J. Wysocki * 12230a996fbSRafael J. Wysocki * If the local APIC timer is not known to be reliable in the target idle state, 12330a996fbSRafael J. Wysocki * enable one-shot tick broadcasting for the target CPU before executing MWAIT. 12430a996fbSRafael J. Wysocki * 12530a996fbSRafael J. Wysocki * Must be called under local_irq_disable(). 12630a996fbSRafael J. Wysocki */ 12730a996fbSRafael J. Wysocki static __cpuidle int intel_idle(struct cpuidle_device *dev, 12830a996fbSRafael J. Wysocki struct cpuidle_driver *drv, int index) 12930a996fbSRafael J. Wysocki { 13030a996fbSRafael J. Wysocki struct cpuidle_state *state = &drv->states[index]; 13130a996fbSRafael J. Wysocki unsigned long eax = flg2MWAIT(state->flags); 13230a996fbSRafael J. Wysocki unsigned long ecx = 1; /* break on interrupt flag */ 13330a996fbSRafael J. Wysocki 134c227233aSArtem Bityutskiy if (state->flags & CPUIDLE_FLAG_IRQ_ENABLE) 135c227233aSArtem Bityutskiy local_irq_enable(); 136c227233aSArtem Bityutskiy 13730a996fbSRafael J. Wysocki mwait_idle_with_hints(eax, ecx); 13830a996fbSRafael J. Wysocki 13930a996fbSRafael J. Wysocki return index; 14030a996fbSRafael J. Wysocki } 14130a996fbSRafael J. Wysocki 14230a996fbSRafael J. Wysocki /** 14330a996fbSRafael J. Wysocki * intel_idle_s2idle - Ask the processor to enter the given idle state. 14430a996fbSRafael J. Wysocki * @dev: cpuidle device of the target CPU. 14530a996fbSRafael J. Wysocki * @drv: cpuidle driver (assumed to point to intel_idle_driver). 14630a996fbSRafael J. Wysocki * @index: Target idle state index. 14730a996fbSRafael J. Wysocki * 14830a996fbSRafael J. Wysocki * Use the MWAIT instruction to notify the processor that the CPU represented by 14930a996fbSRafael J. Wysocki * @dev is idle and it can try to enter the idle state corresponding to @index. 15030a996fbSRafael J. Wysocki * 15130a996fbSRafael J. Wysocki * Invoked as a suspend-to-idle callback routine with frozen user space, frozen 15230a996fbSRafael J. Wysocki * scheduler tick and suspended scheduler clock on the target CPU. 15330a996fbSRafael J. Wysocki */ 154efe97112SNeal Liu static __cpuidle int intel_idle_s2idle(struct cpuidle_device *dev, 15530a996fbSRafael J. Wysocki struct cpuidle_driver *drv, int index) 15630a996fbSRafael J. Wysocki { 15730a996fbSRafael J. Wysocki unsigned long eax = flg2MWAIT(drv->states[index].flags); 15830a996fbSRafael J. Wysocki unsigned long ecx = 1; /* break on interrupt flag */ 15930a996fbSRafael J. Wysocki 16030a996fbSRafael J. Wysocki mwait_idle_with_hints(eax, ecx); 161efe97112SNeal Liu 162efe97112SNeal Liu return 0; 16330a996fbSRafael J. Wysocki } 16430a996fbSRafael J. Wysocki 165b1beab48SLen Brown /* 16626717172SLen Brown * States are indexed by the cstate number, 16726717172SLen Brown * which is also the index into the MWAIT hint array. 16826717172SLen Brown * Thus C0 is a dummy. 16926717172SLen Brown */ 170ab1a8522SRafael J. Wysocki static struct cpuidle_state nehalem_cstates[] __initdata = { 171e022e7ebSLen Brown { 172de09cdd0SLen Brown .name = "C1", 17326717172SLen Brown .desc = "MWAIT 0x00", 174b82b6ccaSDaniel Lezcano .flags = MWAIT2flg(0x00), 17526717172SLen Brown .exit_latency = 3, 17626717172SLen Brown .target_residency = 6, 1775fe2e527SRafael J. Wysocki .enter = &intel_idle, 17828ba086eSRafael J. Wysocki .enter_s2idle = intel_idle_s2idle, }, 179e022e7ebSLen Brown { 180de09cdd0SLen Brown .name = "C1E", 18132e95180SLen Brown .desc = "MWAIT 0x01", 182e6d4f08aSRafael J. Wysocki .flags = MWAIT2flg(0x01) | CPUIDLE_FLAG_ALWAYS_ENABLE, 18332e95180SLen Brown .exit_latency = 10, 18432e95180SLen Brown .target_residency = 20, 1855fe2e527SRafael J. Wysocki .enter = &intel_idle, 18628ba086eSRafael J. Wysocki .enter_s2idle = intel_idle_s2idle, }, 18732e95180SLen Brown { 188de09cdd0SLen Brown .name = "C3", 18926717172SLen Brown .desc = "MWAIT 0x10", 190b82b6ccaSDaniel Lezcano .flags = MWAIT2flg(0x10) | CPUIDLE_FLAG_TLB_FLUSHED, 19126717172SLen Brown .exit_latency = 20, 19226717172SLen Brown .target_residency = 80, 1935fe2e527SRafael J. Wysocki .enter = &intel_idle, 19428ba086eSRafael J. Wysocki .enter_s2idle = intel_idle_s2idle, }, 195e022e7ebSLen Brown { 196de09cdd0SLen Brown .name = "C6", 19726717172SLen Brown .desc = "MWAIT 0x20", 198b82b6ccaSDaniel Lezcano .flags = MWAIT2flg(0x20) | CPUIDLE_FLAG_TLB_FLUSHED, 19926717172SLen Brown .exit_latency = 200, 20026717172SLen Brown .target_residency = 800, 2015fe2e527SRafael J. Wysocki .enter = &intel_idle, 20228ba086eSRafael J. Wysocki .enter_s2idle = intel_idle_s2idle, }, 203e022e7ebSLen Brown { 204e022e7ebSLen Brown .enter = NULL } 20526717172SLen Brown }; 20626717172SLen Brown 207ab1a8522SRafael J. Wysocki static struct cpuidle_state snb_cstates[] __initdata = { 208e022e7ebSLen Brown { 209de09cdd0SLen Brown .name = "C1", 210d13780d4SLen Brown .desc = "MWAIT 0x00", 211b82b6ccaSDaniel Lezcano .flags = MWAIT2flg(0x00), 21232e95180SLen Brown .exit_latency = 2, 21332e95180SLen Brown .target_residency = 2, 2145fe2e527SRafael J. Wysocki .enter = &intel_idle, 21528ba086eSRafael J. Wysocki .enter_s2idle = intel_idle_s2idle, }, 21632e95180SLen Brown { 217de09cdd0SLen Brown .name = "C1E", 21832e95180SLen Brown .desc = "MWAIT 0x01", 219e6d4f08aSRafael J. Wysocki .flags = MWAIT2flg(0x01) | CPUIDLE_FLAG_ALWAYS_ENABLE, 22032e95180SLen Brown .exit_latency = 10, 22132e95180SLen Brown .target_residency = 20, 2225fe2e527SRafael J. Wysocki .enter = &intel_idle, 22328ba086eSRafael J. Wysocki .enter_s2idle = intel_idle_s2idle, }, 224e022e7ebSLen Brown { 225de09cdd0SLen Brown .name = "C3", 226d13780d4SLen Brown .desc = "MWAIT 0x10", 227b82b6ccaSDaniel Lezcano .flags = MWAIT2flg(0x10) | CPUIDLE_FLAG_TLB_FLUSHED, 228d13780d4SLen Brown .exit_latency = 80, 229ddbd550dSLen Brown .target_residency = 211, 2305fe2e527SRafael J. Wysocki .enter = &intel_idle, 23128ba086eSRafael J. Wysocki .enter_s2idle = intel_idle_s2idle, }, 232e022e7ebSLen Brown { 233de09cdd0SLen Brown .name = "C6", 234d13780d4SLen Brown .desc = "MWAIT 0x20", 235b82b6ccaSDaniel Lezcano .flags = MWAIT2flg(0x20) | CPUIDLE_FLAG_TLB_FLUSHED, 236d13780d4SLen Brown .exit_latency = 104, 237ddbd550dSLen Brown .target_residency = 345, 2385fe2e527SRafael J. Wysocki .enter = &intel_idle, 23928ba086eSRafael J. Wysocki .enter_s2idle = intel_idle_s2idle, }, 240e022e7ebSLen Brown { 241de09cdd0SLen Brown .name = "C7", 242d13780d4SLen Brown .desc = "MWAIT 0x30", 243b82b6ccaSDaniel Lezcano .flags = MWAIT2flg(0x30) | CPUIDLE_FLAG_TLB_FLUSHED, 244d13780d4SLen Brown .exit_latency = 109, 245ddbd550dSLen Brown .target_residency = 345, 2465fe2e527SRafael J. Wysocki .enter = &intel_idle, 24728ba086eSRafael J. Wysocki .enter_s2idle = intel_idle_s2idle, }, 248e022e7ebSLen Brown { 249e022e7ebSLen Brown .enter = NULL } 250d13780d4SLen Brown }; 251d13780d4SLen Brown 252ab1a8522SRafael J. Wysocki static struct cpuidle_state byt_cstates[] __initdata = { 253718987d6SLen Brown { 254de09cdd0SLen Brown .name = "C1", 255718987d6SLen Brown .desc = "MWAIT 0x00", 256b82b6ccaSDaniel Lezcano .flags = MWAIT2flg(0x00), 257718987d6SLen Brown .exit_latency = 1, 258718987d6SLen Brown .target_residency = 1, 2595fe2e527SRafael J. Wysocki .enter = &intel_idle, 26028ba086eSRafael J. Wysocki .enter_s2idle = intel_idle_s2idle, }, 261718987d6SLen Brown { 262de09cdd0SLen Brown .name = "C6N", 263718987d6SLen Brown .desc = "MWAIT 0x58", 264b82b6ccaSDaniel Lezcano .flags = MWAIT2flg(0x58) | CPUIDLE_FLAG_TLB_FLUSHED, 265d7ef7671SLen Brown .exit_latency = 300, 266718987d6SLen Brown .target_residency = 275, 2675fe2e527SRafael J. Wysocki .enter = &intel_idle, 26828ba086eSRafael J. Wysocki .enter_s2idle = intel_idle_s2idle, }, 269718987d6SLen Brown { 270de09cdd0SLen Brown .name = "C6S", 271718987d6SLen Brown .desc = "MWAIT 0x52", 272b82b6ccaSDaniel Lezcano .flags = MWAIT2flg(0x52) | CPUIDLE_FLAG_TLB_FLUSHED, 273d7ef7671SLen Brown .exit_latency = 500, 274718987d6SLen Brown .target_residency = 560, 2755fe2e527SRafael J. Wysocki .enter = &intel_idle, 27628ba086eSRafael J. Wysocki .enter_s2idle = intel_idle_s2idle, }, 277718987d6SLen Brown { 278de09cdd0SLen Brown .name = "C7", 279718987d6SLen Brown .desc = "MWAIT 0x60", 280b82b6ccaSDaniel Lezcano .flags = MWAIT2flg(0x60) | CPUIDLE_FLAG_TLB_FLUSHED, 281718987d6SLen Brown .exit_latency = 1200, 282d7ef7671SLen Brown .target_residency = 4000, 2835fe2e527SRafael J. Wysocki .enter = &intel_idle, 28428ba086eSRafael J. Wysocki .enter_s2idle = intel_idle_s2idle, }, 285718987d6SLen Brown { 286de09cdd0SLen Brown .name = "C7S", 287718987d6SLen Brown .desc = "MWAIT 0x64", 288b82b6ccaSDaniel Lezcano .flags = MWAIT2flg(0x64) | CPUIDLE_FLAG_TLB_FLUSHED, 289718987d6SLen Brown .exit_latency = 10000, 290718987d6SLen Brown .target_residency = 20000, 2915fe2e527SRafael J. Wysocki .enter = &intel_idle, 29228ba086eSRafael J. Wysocki .enter_s2idle = intel_idle_s2idle, }, 293718987d6SLen Brown { 294718987d6SLen Brown .enter = NULL } 295718987d6SLen Brown }; 296718987d6SLen Brown 297ab1a8522SRafael J. Wysocki static struct cpuidle_state cht_cstates[] __initdata = { 298cab07a56SLen Brown { 299de09cdd0SLen Brown .name = "C1", 300cab07a56SLen Brown .desc = "MWAIT 0x00", 301cab07a56SLen Brown .flags = MWAIT2flg(0x00), 302cab07a56SLen Brown .exit_latency = 1, 303cab07a56SLen Brown .target_residency = 1, 304cab07a56SLen Brown .enter = &intel_idle, 30528ba086eSRafael J. Wysocki .enter_s2idle = intel_idle_s2idle, }, 306cab07a56SLen Brown { 307de09cdd0SLen Brown .name = "C6N", 308cab07a56SLen Brown .desc = "MWAIT 0x58", 309cab07a56SLen Brown .flags = MWAIT2flg(0x58) | CPUIDLE_FLAG_TLB_FLUSHED, 310cab07a56SLen Brown .exit_latency = 80, 311cab07a56SLen Brown .target_residency = 275, 312cab07a56SLen Brown .enter = &intel_idle, 31328ba086eSRafael J. Wysocki .enter_s2idle = intel_idle_s2idle, }, 314cab07a56SLen Brown { 315de09cdd0SLen Brown .name = "C6S", 316cab07a56SLen Brown .desc = "MWAIT 0x52", 317cab07a56SLen Brown .flags = MWAIT2flg(0x52) | CPUIDLE_FLAG_TLB_FLUSHED, 318cab07a56SLen Brown .exit_latency = 200, 319cab07a56SLen Brown .target_residency = 560, 320cab07a56SLen Brown .enter = &intel_idle, 32128ba086eSRafael J. Wysocki .enter_s2idle = intel_idle_s2idle, }, 322cab07a56SLen Brown { 323de09cdd0SLen Brown .name = "C7", 324cab07a56SLen Brown .desc = "MWAIT 0x60", 325cab07a56SLen Brown .flags = MWAIT2flg(0x60) | CPUIDLE_FLAG_TLB_FLUSHED, 326cab07a56SLen Brown .exit_latency = 1200, 327cab07a56SLen Brown .target_residency = 4000, 328cab07a56SLen Brown .enter = &intel_idle, 32928ba086eSRafael J. Wysocki .enter_s2idle = intel_idle_s2idle, }, 330cab07a56SLen Brown { 331de09cdd0SLen Brown .name = "C7S", 332cab07a56SLen Brown .desc = "MWAIT 0x64", 333cab07a56SLen Brown .flags = MWAIT2flg(0x64) | CPUIDLE_FLAG_TLB_FLUSHED, 334cab07a56SLen Brown .exit_latency = 10000, 335cab07a56SLen Brown .target_residency = 20000, 336cab07a56SLen Brown .enter = &intel_idle, 33728ba086eSRafael J. Wysocki .enter_s2idle = intel_idle_s2idle, }, 338cab07a56SLen Brown { 339cab07a56SLen Brown .enter = NULL } 340cab07a56SLen Brown }; 341cab07a56SLen Brown 342ab1a8522SRafael J. Wysocki static struct cpuidle_state ivb_cstates[] __initdata = { 343e022e7ebSLen Brown { 344de09cdd0SLen Brown .name = "C1", 3456edab08cSLen Brown .desc = "MWAIT 0x00", 346b82b6ccaSDaniel Lezcano .flags = MWAIT2flg(0x00), 3476edab08cSLen Brown .exit_latency = 1, 3486edab08cSLen Brown .target_residency = 1, 3495fe2e527SRafael J. Wysocki .enter = &intel_idle, 35028ba086eSRafael J. Wysocki .enter_s2idle = intel_idle_s2idle, }, 351e022e7ebSLen Brown { 352de09cdd0SLen Brown .name = "C1E", 35332e95180SLen Brown .desc = "MWAIT 0x01", 354e6d4f08aSRafael J. Wysocki .flags = MWAIT2flg(0x01) | CPUIDLE_FLAG_ALWAYS_ENABLE, 35532e95180SLen Brown .exit_latency = 10, 35632e95180SLen Brown .target_residency = 20, 3575fe2e527SRafael J. Wysocki .enter = &intel_idle, 35828ba086eSRafael J. Wysocki .enter_s2idle = intel_idle_s2idle, }, 35932e95180SLen Brown { 360de09cdd0SLen Brown .name = "C3", 3616edab08cSLen Brown .desc = "MWAIT 0x10", 362b82b6ccaSDaniel Lezcano .flags = MWAIT2flg(0x10) | CPUIDLE_FLAG_TLB_FLUSHED, 3636edab08cSLen Brown .exit_latency = 59, 3646edab08cSLen Brown .target_residency = 156, 3655fe2e527SRafael J. Wysocki .enter = &intel_idle, 36628ba086eSRafael J. Wysocki .enter_s2idle = intel_idle_s2idle, }, 367e022e7ebSLen Brown { 368de09cdd0SLen Brown .name = "C6", 3696edab08cSLen Brown .desc = "MWAIT 0x20", 370b82b6ccaSDaniel Lezcano .flags = MWAIT2flg(0x20) | CPUIDLE_FLAG_TLB_FLUSHED, 3716edab08cSLen Brown .exit_latency = 80, 3726edab08cSLen Brown .target_residency = 300, 3735fe2e527SRafael J. Wysocki .enter = &intel_idle, 37428ba086eSRafael J. Wysocki .enter_s2idle = intel_idle_s2idle, }, 375e022e7ebSLen Brown { 376de09cdd0SLen Brown .name = "C7", 3776edab08cSLen Brown .desc = "MWAIT 0x30", 378b82b6ccaSDaniel Lezcano .flags = MWAIT2flg(0x30) | CPUIDLE_FLAG_TLB_FLUSHED, 3796edab08cSLen Brown .exit_latency = 87, 3806edab08cSLen Brown .target_residency = 300, 3815fe2e527SRafael J. Wysocki .enter = &intel_idle, 38228ba086eSRafael J. Wysocki .enter_s2idle = intel_idle_s2idle, }, 383e022e7ebSLen Brown { 384e022e7ebSLen Brown .enter = NULL } 3856edab08cSLen Brown }; 3866edab08cSLen Brown 387ab1a8522SRafael J. Wysocki static struct cpuidle_state ivt_cstates[] __initdata = { 3880138d8f0SLen Brown { 389de09cdd0SLen Brown .name = "C1", 3900138d8f0SLen Brown .desc = "MWAIT 0x00", 391b82b6ccaSDaniel Lezcano .flags = MWAIT2flg(0x00), 3920138d8f0SLen Brown .exit_latency = 1, 3930138d8f0SLen Brown .target_residency = 1, 3945fe2e527SRafael J. Wysocki .enter = &intel_idle, 39528ba086eSRafael J. Wysocki .enter_s2idle = intel_idle_s2idle, }, 3960138d8f0SLen Brown { 397de09cdd0SLen Brown .name = "C1E", 3980138d8f0SLen Brown .desc = "MWAIT 0x01", 399e6d4f08aSRafael J. Wysocki .flags = MWAIT2flg(0x01) | CPUIDLE_FLAG_ALWAYS_ENABLE, 4000138d8f0SLen Brown .exit_latency = 10, 4010138d8f0SLen Brown .target_residency = 80, 4025fe2e527SRafael J. Wysocki .enter = &intel_idle, 40328ba086eSRafael J. Wysocki .enter_s2idle = intel_idle_s2idle, }, 4040138d8f0SLen Brown { 405de09cdd0SLen Brown .name = "C3", 4060138d8f0SLen Brown .desc = "MWAIT 0x10", 407b82b6ccaSDaniel Lezcano .flags = MWAIT2flg(0x10) | CPUIDLE_FLAG_TLB_FLUSHED, 4080138d8f0SLen Brown .exit_latency = 59, 4090138d8f0SLen Brown .target_residency = 156, 4105fe2e527SRafael J. Wysocki .enter = &intel_idle, 41128ba086eSRafael J. Wysocki .enter_s2idle = intel_idle_s2idle, }, 4120138d8f0SLen Brown { 413de09cdd0SLen Brown .name = "C6", 4140138d8f0SLen Brown .desc = "MWAIT 0x20", 415b82b6ccaSDaniel Lezcano .flags = MWAIT2flg(0x20) | CPUIDLE_FLAG_TLB_FLUSHED, 4160138d8f0SLen Brown .exit_latency = 82, 4170138d8f0SLen Brown .target_residency = 300, 4185fe2e527SRafael J. Wysocki .enter = &intel_idle, 41928ba086eSRafael J. Wysocki .enter_s2idle = intel_idle_s2idle, }, 4200138d8f0SLen Brown { 4210138d8f0SLen Brown .enter = NULL } 4220138d8f0SLen Brown }; 4230138d8f0SLen Brown 424ab1a8522SRafael J. Wysocki static struct cpuidle_state ivt_cstates_4s[] __initdata = { 4250138d8f0SLen Brown { 426de09cdd0SLen Brown .name = "C1", 4270138d8f0SLen Brown .desc = "MWAIT 0x00", 428b82b6ccaSDaniel Lezcano .flags = MWAIT2flg(0x00), 4290138d8f0SLen Brown .exit_latency = 1, 4300138d8f0SLen Brown .target_residency = 1, 4315fe2e527SRafael J. Wysocki .enter = &intel_idle, 43228ba086eSRafael J. Wysocki .enter_s2idle = intel_idle_s2idle, }, 4330138d8f0SLen Brown { 434de09cdd0SLen Brown .name = "C1E", 4350138d8f0SLen Brown .desc = "MWAIT 0x01", 436e6d4f08aSRafael J. Wysocki .flags = MWAIT2flg(0x01) | CPUIDLE_FLAG_ALWAYS_ENABLE, 4370138d8f0SLen Brown .exit_latency = 10, 4380138d8f0SLen Brown .target_residency = 250, 4395fe2e527SRafael J. Wysocki .enter = &intel_idle, 44028ba086eSRafael J. Wysocki .enter_s2idle = intel_idle_s2idle, }, 4410138d8f0SLen Brown { 442de09cdd0SLen Brown .name = "C3", 4430138d8f0SLen Brown .desc = "MWAIT 0x10", 444b82b6ccaSDaniel Lezcano .flags = MWAIT2flg(0x10) | CPUIDLE_FLAG_TLB_FLUSHED, 4450138d8f0SLen Brown .exit_latency = 59, 4460138d8f0SLen Brown .target_residency = 300, 4475fe2e527SRafael J. Wysocki .enter = &intel_idle, 44828ba086eSRafael J. Wysocki .enter_s2idle = intel_idle_s2idle, }, 4490138d8f0SLen Brown { 450de09cdd0SLen Brown .name = "C6", 4510138d8f0SLen Brown .desc = "MWAIT 0x20", 452b82b6ccaSDaniel Lezcano .flags = MWAIT2flg(0x20) | CPUIDLE_FLAG_TLB_FLUSHED, 4530138d8f0SLen Brown .exit_latency = 84, 4540138d8f0SLen Brown .target_residency = 400, 4555fe2e527SRafael J. Wysocki .enter = &intel_idle, 45628ba086eSRafael J. Wysocki .enter_s2idle = intel_idle_s2idle, }, 4570138d8f0SLen Brown { 4580138d8f0SLen Brown .enter = NULL } 4590138d8f0SLen Brown }; 4600138d8f0SLen Brown 461ab1a8522SRafael J. Wysocki static struct cpuidle_state ivt_cstates_8s[] __initdata = { 4620138d8f0SLen Brown { 463de09cdd0SLen Brown .name = "C1", 4640138d8f0SLen Brown .desc = "MWAIT 0x00", 465b82b6ccaSDaniel Lezcano .flags = MWAIT2flg(0x00), 4660138d8f0SLen Brown .exit_latency = 1, 4670138d8f0SLen Brown .target_residency = 1, 4685fe2e527SRafael J. Wysocki .enter = &intel_idle, 46928ba086eSRafael J. Wysocki .enter_s2idle = intel_idle_s2idle, }, 4700138d8f0SLen Brown { 471de09cdd0SLen Brown .name = "C1E", 4720138d8f0SLen Brown .desc = "MWAIT 0x01", 473e6d4f08aSRafael J. Wysocki .flags = MWAIT2flg(0x01) | CPUIDLE_FLAG_ALWAYS_ENABLE, 4740138d8f0SLen Brown .exit_latency = 10, 4750138d8f0SLen Brown .target_residency = 500, 4765fe2e527SRafael J. Wysocki .enter = &intel_idle, 47728ba086eSRafael J. Wysocki .enter_s2idle = intel_idle_s2idle, }, 4780138d8f0SLen Brown { 479de09cdd0SLen Brown .name = "C3", 4800138d8f0SLen Brown .desc = "MWAIT 0x10", 481b82b6ccaSDaniel Lezcano .flags = MWAIT2flg(0x10) | CPUIDLE_FLAG_TLB_FLUSHED, 4820138d8f0SLen Brown .exit_latency = 59, 4830138d8f0SLen Brown .target_residency = 600, 4845fe2e527SRafael J. Wysocki .enter = &intel_idle, 48528ba086eSRafael J. Wysocki .enter_s2idle = intel_idle_s2idle, }, 4860138d8f0SLen Brown { 487de09cdd0SLen Brown .name = "C6", 4880138d8f0SLen Brown .desc = "MWAIT 0x20", 489b82b6ccaSDaniel Lezcano .flags = MWAIT2flg(0x20) | CPUIDLE_FLAG_TLB_FLUSHED, 4900138d8f0SLen Brown .exit_latency = 88, 4910138d8f0SLen Brown .target_residency = 700, 4925fe2e527SRafael J. Wysocki .enter = &intel_idle, 49328ba086eSRafael J. Wysocki .enter_s2idle = intel_idle_s2idle, }, 4940138d8f0SLen Brown { 4950138d8f0SLen Brown .enter = NULL } 4960138d8f0SLen Brown }; 4970138d8f0SLen Brown 498ab1a8522SRafael J. Wysocki static struct cpuidle_state hsw_cstates[] __initdata = { 499e022e7ebSLen Brown { 500de09cdd0SLen Brown .name = "C1", 50185a4d2d4SLen Brown .desc = "MWAIT 0x00", 502b82b6ccaSDaniel Lezcano .flags = MWAIT2flg(0x00), 50385a4d2d4SLen Brown .exit_latency = 2, 50485a4d2d4SLen Brown .target_residency = 2, 5055fe2e527SRafael J. Wysocki .enter = &intel_idle, 50628ba086eSRafael J. Wysocki .enter_s2idle = intel_idle_s2idle, }, 507e022e7ebSLen Brown { 508de09cdd0SLen Brown .name = "C1E", 50932e95180SLen Brown .desc = "MWAIT 0x01", 510e6d4f08aSRafael J. Wysocki .flags = MWAIT2flg(0x01) | CPUIDLE_FLAG_ALWAYS_ENABLE, 51132e95180SLen Brown .exit_latency = 10, 51232e95180SLen Brown .target_residency = 20, 5135fe2e527SRafael J. Wysocki .enter = &intel_idle, 51428ba086eSRafael J. Wysocki .enter_s2idle = intel_idle_s2idle, }, 51532e95180SLen Brown { 516de09cdd0SLen Brown .name = "C3", 51785a4d2d4SLen Brown .desc = "MWAIT 0x10", 518b82b6ccaSDaniel Lezcano .flags = MWAIT2flg(0x10) | CPUIDLE_FLAG_TLB_FLUSHED, 51985a4d2d4SLen Brown .exit_latency = 33, 52085a4d2d4SLen Brown .target_residency = 100, 5215fe2e527SRafael J. Wysocki .enter = &intel_idle, 52228ba086eSRafael J. Wysocki .enter_s2idle = intel_idle_s2idle, }, 523e022e7ebSLen Brown { 524de09cdd0SLen Brown .name = "C6", 52585a4d2d4SLen Brown .desc = "MWAIT 0x20", 526b82b6ccaSDaniel Lezcano .flags = MWAIT2flg(0x20) | CPUIDLE_FLAG_TLB_FLUSHED, 52785a4d2d4SLen Brown .exit_latency = 133, 52885a4d2d4SLen Brown .target_residency = 400, 5295fe2e527SRafael J. Wysocki .enter = &intel_idle, 53028ba086eSRafael J. Wysocki .enter_s2idle = intel_idle_s2idle, }, 531e022e7ebSLen Brown { 532de09cdd0SLen Brown .name = "C7s", 53385a4d2d4SLen Brown .desc = "MWAIT 0x32", 534b82b6ccaSDaniel Lezcano .flags = MWAIT2flg(0x32) | CPUIDLE_FLAG_TLB_FLUSHED, 53585a4d2d4SLen Brown .exit_latency = 166, 53685a4d2d4SLen Brown .target_residency = 500, 5375fe2e527SRafael J. Wysocki .enter = &intel_idle, 53828ba086eSRafael J. Wysocki .enter_s2idle = intel_idle_s2idle, }, 539e022e7ebSLen Brown { 540de09cdd0SLen Brown .name = "C8", 54186239cebSLen Brown .desc = "MWAIT 0x40", 542b82b6ccaSDaniel Lezcano .flags = MWAIT2flg(0x40) | CPUIDLE_FLAG_TLB_FLUSHED, 54386239cebSLen Brown .exit_latency = 300, 54486239cebSLen Brown .target_residency = 900, 5455fe2e527SRafael J. Wysocki .enter = &intel_idle, 54628ba086eSRafael J. Wysocki .enter_s2idle = intel_idle_s2idle, }, 54786239cebSLen Brown { 548de09cdd0SLen Brown .name = "C9", 54986239cebSLen Brown .desc = "MWAIT 0x50", 550b82b6ccaSDaniel Lezcano .flags = MWAIT2flg(0x50) | CPUIDLE_FLAG_TLB_FLUSHED, 55186239cebSLen Brown .exit_latency = 600, 55286239cebSLen Brown .target_residency = 1800, 5535fe2e527SRafael J. Wysocki .enter = &intel_idle, 55428ba086eSRafael J. Wysocki .enter_s2idle = intel_idle_s2idle, }, 55586239cebSLen Brown { 556de09cdd0SLen Brown .name = "C10", 55786239cebSLen Brown .desc = "MWAIT 0x60", 558b82b6ccaSDaniel Lezcano .flags = MWAIT2flg(0x60) | CPUIDLE_FLAG_TLB_FLUSHED, 55986239cebSLen Brown .exit_latency = 2600, 56086239cebSLen Brown .target_residency = 7700, 5615fe2e527SRafael J. Wysocki .enter = &intel_idle, 56228ba086eSRafael J. Wysocki .enter_s2idle = intel_idle_s2idle, }, 56386239cebSLen Brown { 564e022e7ebSLen Brown .enter = NULL } 56585a4d2d4SLen Brown }; 566ab1a8522SRafael J. Wysocki static struct cpuidle_state bdw_cstates[] __initdata = { 567a138b568SLen Brown { 568de09cdd0SLen Brown .name = "C1", 569a138b568SLen Brown .desc = "MWAIT 0x00", 570b82b6ccaSDaniel Lezcano .flags = MWAIT2flg(0x00), 571a138b568SLen Brown .exit_latency = 2, 572a138b568SLen Brown .target_residency = 2, 5735fe2e527SRafael J. Wysocki .enter = &intel_idle, 57428ba086eSRafael J. Wysocki .enter_s2idle = intel_idle_s2idle, }, 575a138b568SLen Brown { 576de09cdd0SLen Brown .name = "C1E", 577a138b568SLen Brown .desc = "MWAIT 0x01", 578e6d4f08aSRafael J. Wysocki .flags = MWAIT2flg(0x01) | CPUIDLE_FLAG_ALWAYS_ENABLE, 579a138b568SLen Brown .exit_latency = 10, 580a138b568SLen Brown .target_residency = 20, 5815fe2e527SRafael J. Wysocki .enter = &intel_idle, 58228ba086eSRafael J. Wysocki .enter_s2idle = intel_idle_s2idle, }, 583a138b568SLen Brown { 584de09cdd0SLen Brown .name = "C3", 585a138b568SLen Brown .desc = "MWAIT 0x10", 586b82b6ccaSDaniel Lezcano .flags = MWAIT2flg(0x10) | CPUIDLE_FLAG_TLB_FLUSHED, 587a138b568SLen Brown .exit_latency = 40, 588a138b568SLen Brown .target_residency = 100, 5895fe2e527SRafael J. Wysocki .enter = &intel_idle, 59028ba086eSRafael J. Wysocki .enter_s2idle = intel_idle_s2idle, }, 591a138b568SLen Brown { 592de09cdd0SLen Brown .name = "C6", 593a138b568SLen Brown .desc = "MWAIT 0x20", 594b82b6ccaSDaniel Lezcano .flags = MWAIT2flg(0x20) | CPUIDLE_FLAG_TLB_FLUSHED, 595a138b568SLen Brown .exit_latency = 133, 596a138b568SLen Brown .target_residency = 400, 5975fe2e527SRafael J. Wysocki .enter = &intel_idle, 59828ba086eSRafael J. Wysocki .enter_s2idle = intel_idle_s2idle, }, 599a138b568SLen Brown { 600de09cdd0SLen Brown .name = "C7s", 601a138b568SLen Brown .desc = "MWAIT 0x32", 602b82b6ccaSDaniel Lezcano .flags = MWAIT2flg(0x32) | CPUIDLE_FLAG_TLB_FLUSHED, 603a138b568SLen Brown .exit_latency = 166, 604a138b568SLen Brown .target_residency = 500, 6055fe2e527SRafael J. Wysocki .enter = &intel_idle, 60628ba086eSRafael J. Wysocki .enter_s2idle = intel_idle_s2idle, }, 607a138b568SLen Brown { 608de09cdd0SLen Brown .name = "C8", 609a138b568SLen Brown .desc = "MWAIT 0x40", 610b82b6ccaSDaniel Lezcano .flags = MWAIT2flg(0x40) | CPUIDLE_FLAG_TLB_FLUSHED, 611a138b568SLen Brown .exit_latency = 300, 612a138b568SLen Brown .target_residency = 900, 6135fe2e527SRafael J. Wysocki .enter = &intel_idle, 61428ba086eSRafael J. Wysocki .enter_s2idle = intel_idle_s2idle, }, 615a138b568SLen Brown { 616de09cdd0SLen Brown .name = "C9", 617a138b568SLen Brown .desc = "MWAIT 0x50", 618b82b6ccaSDaniel Lezcano .flags = MWAIT2flg(0x50) | CPUIDLE_FLAG_TLB_FLUSHED, 619a138b568SLen Brown .exit_latency = 600, 620a138b568SLen Brown .target_residency = 1800, 6215fe2e527SRafael J. Wysocki .enter = &intel_idle, 62228ba086eSRafael J. Wysocki .enter_s2idle = intel_idle_s2idle, }, 623a138b568SLen Brown { 624de09cdd0SLen Brown .name = "C10", 625a138b568SLen Brown .desc = "MWAIT 0x60", 626b82b6ccaSDaniel Lezcano .flags = MWAIT2flg(0x60) | CPUIDLE_FLAG_TLB_FLUSHED, 627a138b568SLen Brown .exit_latency = 2600, 628a138b568SLen Brown .target_residency = 7700, 6295fe2e527SRafael J. Wysocki .enter = &intel_idle, 63028ba086eSRafael J. Wysocki .enter_s2idle = intel_idle_s2idle, }, 631a138b568SLen Brown { 632a138b568SLen Brown .enter = NULL } 633a138b568SLen Brown }; 63485a4d2d4SLen Brown 635ab1a8522SRafael J. Wysocki static struct cpuidle_state skl_cstates[] __initdata = { 636493f133fSLen Brown { 637de09cdd0SLen Brown .name = "C1", 638493f133fSLen Brown .desc = "MWAIT 0x00", 639493f133fSLen Brown .flags = MWAIT2flg(0x00), 640493f133fSLen Brown .exit_latency = 2, 641493f133fSLen Brown .target_residency = 2, 642493f133fSLen Brown .enter = &intel_idle, 64328ba086eSRafael J. Wysocki .enter_s2idle = intel_idle_s2idle, }, 644493f133fSLen Brown { 645de09cdd0SLen Brown .name = "C1E", 646493f133fSLen Brown .desc = "MWAIT 0x01", 647e6d4f08aSRafael J. Wysocki .flags = MWAIT2flg(0x01) | CPUIDLE_FLAG_ALWAYS_ENABLE, 648493f133fSLen Brown .exit_latency = 10, 649493f133fSLen Brown .target_residency = 20, 650493f133fSLen Brown .enter = &intel_idle, 65128ba086eSRafael J. Wysocki .enter_s2idle = intel_idle_s2idle, }, 652493f133fSLen Brown { 653de09cdd0SLen Brown .name = "C3", 654493f133fSLen Brown .desc = "MWAIT 0x10", 655493f133fSLen Brown .flags = MWAIT2flg(0x10) | CPUIDLE_FLAG_TLB_FLUSHED, 656493f133fSLen Brown .exit_latency = 70, 657493f133fSLen Brown .target_residency = 100, 658493f133fSLen Brown .enter = &intel_idle, 65928ba086eSRafael J. Wysocki .enter_s2idle = intel_idle_s2idle, }, 660493f133fSLen Brown { 661de09cdd0SLen Brown .name = "C6", 662493f133fSLen Brown .desc = "MWAIT 0x20", 663493f133fSLen Brown .flags = MWAIT2flg(0x20) | CPUIDLE_FLAG_TLB_FLUSHED, 664135919a3SLen Brown .exit_latency = 85, 665493f133fSLen Brown .target_residency = 200, 666493f133fSLen Brown .enter = &intel_idle, 66728ba086eSRafael J. Wysocki .enter_s2idle = intel_idle_s2idle, }, 668493f133fSLen Brown { 669de09cdd0SLen Brown .name = "C7s", 670493f133fSLen Brown .desc = "MWAIT 0x33", 671493f133fSLen Brown .flags = MWAIT2flg(0x33) | CPUIDLE_FLAG_TLB_FLUSHED, 672493f133fSLen Brown .exit_latency = 124, 673493f133fSLen Brown .target_residency = 800, 674493f133fSLen Brown .enter = &intel_idle, 67528ba086eSRafael J. Wysocki .enter_s2idle = intel_idle_s2idle, }, 676493f133fSLen Brown { 677de09cdd0SLen Brown .name = "C8", 678493f133fSLen Brown .desc = "MWAIT 0x40", 679493f133fSLen Brown .flags = MWAIT2flg(0x40) | CPUIDLE_FLAG_TLB_FLUSHED, 680135919a3SLen Brown .exit_latency = 200, 681493f133fSLen Brown .target_residency = 800, 682493f133fSLen Brown .enter = &intel_idle, 68328ba086eSRafael J. Wysocki .enter_s2idle = intel_idle_s2idle, }, 684493f133fSLen Brown { 685de09cdd0SLen Brown .name = "C9", 686135919a3SLen Brown .desc = "MWAIT 0x50", 687135919a3SLen Brown .flags = MWAIT2flg(0x50) | CPUIDLE_FLAG_TLB_FLUSHED, 688135919a3SLen Brown .exit_latency = 480, 689135919a3SLen Brown .target_residency = 5000, 690135919a3SLen Brown .enter = &intel_idle, 69128ba086eSRafael J. Wysocki .enter_s2idle = intel_idle_s2idle, }, 692135919a3SLen Brown { 693de09cdd0SLen Brown .name = "C10", 694493f133fSLen Brown .desc = "MWAIT 0x60", 695493f133fSLen Brown .flags = MWAIT2flg(0x60) | CPUIDLE_FLAG_TLB_FLUSHED, 696493f133fSLen Brown .exit_latency = 890, 697493f133fSLen Brown .target_residency = 5000, 698493f133fSLen Brown .enter = &intel_idle, 69928ba086eSRafael J. Wysocki .enter_s2idle = intel_idle_s2idle, }, 700493f133fSLen Brown { 701493f133fSLen Brown .enter = NULL } 702493f133fSLen Brown }; 703493f133fSLen Brown 704ab1a8522SRafael J. Wysocki static struct cpuidle_state skx_cstates[] __initdata = { 705f9e71657SLen Brown { 706de09cdd0SLen Brown .name = "C1", 707f9e71657SLen Brown .desc = "MWAIT 0x00", 708c227233aSArtem Bityutskiy .flags = MWAIT2flg(0x00) | CPUIDLE_FLAG_IRQ_ENABLE, 709f9e71657SLen Brown .exit_latency = 2, 710f9e71657SLen Brown .target_residency = 2, 711f9e71657SLen Brown .enter = &intel_idle, 71228ba086eSRafael J. Wysocki .enter_s2idle = intel_idle_s2idle, }, 713f9e71657SLen Brown { 714de09cdd0SLen Brown .name = "C1E", 715f9e71657SLen Brown .desc = "MWAIT 0x01", 716e6d4f08aSRafael J. Wysocki .flags = MWAIT2flg(0x01) | CPUIDLE_FLAG_ALWAYS_ENABLE, 717f9e71657SLen Brown .exit_latency = 10, 718f9e71657SLen Brown .target_residency = 20, 719f9e71657SLen Brown .enter = &intel_idle, 72028ba086eSRafael J. Wysocki .enter_s2idle = intel_idle_s2idle, }, 721f9e71657SLen Brown { 722de09cdd0SLen Brown .name = "C6", 723f9e71657SLen Brown .desc = "MWAIT 0x20", 724f9e71657SLen Brown .flags = MWAIT2flg(0x20) | CPUIDLE_FLAG_TLB_FLUSHED, 725f9e71657SLen Brown .exit_latency = 133, 726f9e71657SLen Brown .target_residency = 600, 727f9e71657SLen Brown .enter = &intel_idle, 72828ba086eSRafael J. Wysocki .enter_s2idle = intel_idle_s2idle, }, 729f9e71657SLen Brown { 730f9e71657SLen Brown .enter = NULL } 731f9e71657SLen Brown }; 732f9e71657SLen Brown 733a472ad2bSChen Yu static struct cpuidle_state icx_cstates[] __initdata = { 734a472ad2bSChen Yu { 735a472ad2bSChen Yu .name = "C1", 736a472ad2bSChen Yu .desc = "MWAIT 0x00", 737c227233aSArtem Bityutskiy .flags = MWAIT2flg(0x00) | CPUIDLE_FLAG_IRQ_ENABLE, 738a472ad2bSChen Yu .exit_latency = 1, 739a472ad2bSChen Yu .target_residency = 1, 740a472ad2bSChen Yu .enter = &intel_idle, 741a472ad2bSChen Yu .enter_s2idle = intel_idle_s2idle, }, 742a472ad2bSChen Yu { 743a472ad2bSChen Yu .name = "C1E", 744a472ad2bSChen Yu .desc = "MWAIT 0x01", 745a472ad2bSChen Yu .flags = MWAIT2flg(0x01) | CPUIDLE_FLAG_ALWAYS_ENABLE, 746a472ad2bSChen Yu .exit_latency = 4, 747a472ad2bSChen Yu .target_residency = 4, 748a472ad2bSChen Yu .enter = &intel_idle, 749a472ad2bSChen Yu .enter_s2idle = intel_idle_s2idle, }, 750a472ad2bSChen Yu { 751a472ad2bSChen Yu .name = "C6", 752a472ad2bSChen Yu .desc = "MWAIT 0x20", 753a472ad2bSChen Yu .flags = MWAIT2flg(0x20) | CPUIDLE_FLAG_TLB_FLUSHED, 754d484b8bfSArtem Bityutskiy .exit_latency = 170, 755d484b8bfSArtem Bityutskiy .target_residency = 600, 756a472ad2bSChen Yu .enter = &intel_idle, 757a472ad2bSChen Yu .enter_s2idle = intel_idle_s2idle, }, 758a472ad2bSChen Yu { 759a472ad2bSChen Yu .enter = NULL } 760a472ad2bSChen Yu }; 761a472ad2bSChen Yu 7629edf3c0fSArtem Bityutskiy /* 7639edf3c0fSArtem Bityutskiy * On Sapphire Rapids Xeon C1 has to be disabled if C1E is enabled, and vice 7649edf3c0fSArtem Bityutskiy * versa. On SPR C1E is enabled only if "C1E promotion" bit is set in 7659edf3c0fSArtem Bityutskiy * MSR_IA32_POWER_CTL. But in this case there effectively no C1, because C1 7669edf3c0fSArtem Bityutskiy * requests are promoted to C1E. If the "C1E promotion" bit is cleared, then 7679edf3c0fSArtem Bityutskiy * both C1 and C1E requests end up with C1, so there is effectively no C1E. 7689edf3c0fSArtem Bityutskiy * 7699edf3c0fSArtem Bityutskiy * By default we enable C1 and disable C1E by marking it with 7709edf3c0fSArtem Bityutskiy * 'CPUIDLE_FLAG_UNUSABLE'. 7719edf3c0fSArtem Bityutskiy */ 7729edf3c0fSArtem Bityutskiy static struct cpuidle_state spr_cstates[] __initdata = { 7739edf3c0fSArtem Bityutskiy { 7749edf3c0fSArtem Bityutskiy .name = "C1", 7759edf3c0fSArtem Bityutskiy .desc = "MWAIT 0x00", 7769edf3c0fSArtem Bityutskiy .flags = MWAIT2flg(0x00), 7779edf3c0fSArtem Bityutskiy .exit_latency = 1, 7789edf3c0fSArtem Bityutskiy .target_residency = 1, 7799edf3c0fSArtem Bityutskiy .enter = &intel_idle, 7809edf3c0fSArtem Bityutskiy .enter_s2idle = intel_idle_s2idle, }, 7819edf3c0fSArtem Bityutskiy { 7829edf3c0fSArtem Bityutskiy .name = "C1E", 7839edf3c0fSArtem Bityutskiy .desc = "MWAIT 0x01", 784*03eb6522SRafael J. Wysocki .flags = MWAIT2flg(0x01) | CPUIDLE_FLAG_ALWAYS_ENABLE | 7859edf3c0fSArtem Bityutskiy CPUIDLE_FLAG_UNUSABLE, 7869edf3c0fSArtem Bityutskiy .exit_latency = 2, 7879edf3c0fSArtem Bityutskiy .target_residency = 4, 7889edf3c0fSArtem Bityutskiy .enter = &intel_idle, 7899edf3c0fSArtem Bityutskiy .enter_s2idle = intel_idle_s2idle, }, 7909edf3c0fSArtem Bityutskiy { 7919edf3c0fSArtem Bityutskiy .name = "C6", 7929edf3c0fSArtem Bityutskiy .desc = "MWAIT 0x20", 7939edf3c0fSArtem Bityutskiy .flags = MWAIT2flg(0x20) | CPUIDLE_FLAG_TLB_FLUSHED, 7949edf3c0fSArtem Bityutskiy .exit_latency = 290, 7959edf3c0fSArtem Bityutskiy .target_residency = 800, 7969edf3c0fSArtem Bityutskiy .enter = &intel_idle, 7979edf3c0fSArtem Bityutskiy .enter_s2idle = intel_idle_s2idle, }, 7989edf3c0fSArtem Bityutskiy { 7999edf3c0fSArtem Bityutskiy .enter = NULL } 8009edf3c0fSArtem Bityutskiy }; 8019edf3c0fSArtem Bityutskiy 802ab1a8522SRafael J. Wysocki static struct cpuidle_state atom_cstates[] __initdata = { 803e022e7ebSLen Brown { 804de09cdd0SLen Brown .name = "C1E", 80526717172SLen Brown .desc = "MWAIT 0x00", 806b82b6ccaSDaniel Lezcano .flags = MWAIT2flg(0x00), 80732e95180SLen Brown .exit_latency = 10, 80832e95180SLen Brown .target_residency = 20, 8095fe2e527SRafael J. Wysocki .enter = &intel_idle, 81028ba086eSRafael J. Wysocki .enter_s2idle = intel_idle_s2idle, }, 811e022e7ebSLen Brown { 812de09cdd0SLen Brown .name = "C2", 81326717172SLen Brown .desc = "MWAIT 0x10", 814b82b6ccaSDaniel Lezcano .flags = MWAIT2flg(0x10), 81526717172SLen Brown .exit_latency = 20, 81626717172SLen Brown .target_residency = 80, 8175fe2e527SRafael J. Wysocki .enter = &intel_idle, 81828ba086eSRafael J. Wysocki .enter_s2idle = intel_idle_s2idle, }, 819e022e7ebSLen Brown { 820de09cdd0SLen Brown .name = "C4", 82126717172SLen Brown .desc = "MWAIT 0x30", 822b82b6ccaSDaniel Lezcano .flags = MWAIT2flg(0x30) | CPUIDLE_FLAG_TLB_FLUSHED, 82326717172SLen Brown .exit_latency = 100, 82426717172SLen Brown .target_residency = 400, 8255fe2e527SRafael J. Wysocki .enter = &intel_idle, 82628ba086eSRafael J. Wysocki .enter_s2idle = intel_idle_s2idle, }, 827e022e7ebSLen Brown { 828de09cdd0SLen Brown .name = "C6", 8297fcca7d9SLen Brown .desc = "MWAIT 0x52", 830b82b6ccaSDaniel Lezcano .flags = MWAIT2flg(0x52) | CPUIDLE_FLAG_TLB_FLUSHED, 8317fcca7d9SLen Brown .exit_latency = 140, 8327fcca7d9SLen Brown .target_residency = 560, 8335fe2e527SRafael J. Wysocki .enter = &intel_idle, 83428ba086eSRafael J. Wysocki .enter_s2idle = intel_idle_s2idle, }, 835e022e7ebSLen Brown { 836e022e7ebSLen Brown .enter = NULL } 83726717172SLen Brown }; 838ab1a8522SRafael J. Wysocki static struct cpuidle_state tangier_cstates[] __initdata = { 8395e7ec268SAndy Shevchenko { 840de09cdd0SLen Brown .name = "C1", 8415e7ec268SAndy Shevchenko .desc = "MWAIT 0x00", 8425e7ec268SAndy Shevchenko .flags = MWAIT2flg(0x00), 8435e7ec268SAndy Shevchenko .exit_latency = 1, 8445e7ec268SAndy Shevchenko .target_residency = 4, 8455e7ec268SAndy Shevchenko .enter = &intel_idle, 84628ba086eSRafael J. Wysocki .enter_s2idle = intel_idle_s2idle, }, 8475e7ec268SAndy Shevchenko { 848de09cdd0SLen Brown .name = "C4", 8495e7ec268SAndy Shevchenko .desc = "MWAIT 0x30", 8505e7ec268SAndy Shevchenko .flags = MWAIT2flg(0x30) | CPUIDLE_FLAG_TLB_FLUSHED, 8515e7ec268SAndy Shevchenko .exit_latency = 100, 8525e7ec268SAndy Shevchenko .target_residency = 400, 8535e7ec268SAndy Shevchenko .enter = &intel_idle, 85428ba086eSRafael J. Wysocki .enter_s2idle = intel_idle_s2idle, }, 8555e7ec268SAndy Shevchenko { 856de09cdd0SLen Brown .name = "C6", 8575e7ec268SAndy Shevchenko .desc = "MWAIT 0x52", 8585e7ec268SAndy Shevchenko .flags = MWAIT2flg(0x52) | CPUIDLE_FLAG_TLB_FLUSHED, 8595e7ec268SAndy Shevchenko .exit_latency = 140, 8605e7ec268SAndy Shevchenko .target_residency = 560, 8615e7ec268SAndy Shevchenko .enter = &intel_idle, 86228ba086eSRafael J. Wysocki .enter_s2idle = intel_idle_s2idle, }, 8635e7ec268SAndy Shevchenko { 864de09cdd0SLen Brown .name = "C7", 8655e7ec268SAndy Shevchenko .desc = "MWAIT 0x60", 8665e7ec268SAndy Shevchenko .flags = MWAIT2flg(0x60) | CPUIDLE_FLAG_TLB_FLUSHED, 8675e7ec268SAndy Shevchenko .exit_latency = 1200, 8685e7ec268SAndy Shevchenko .target_residency = 4000, 8695e7ec268SAndy Shevchenko .enter = &intel_idle, 87028ba086eSRafael J. Wysocki .enter_s2idle = intel_idle_s2idle, }, 8715e7ec268SAndy Shevchenko { 872de09cdd0SLen Brown .name = "C9", 8735e7ec268SAndy Shevchenko .desc = "MWAIT 0x64", 8745e7ec268SAndy Shevchenko .flags = MWAIT2flg(0x64) | CPUIDLE_FLAG_TLB_FLUSHED, 8755e7ec268SAndy Shevchenko .exit_latency = 10000, 8765e7ec268SAndy Shevchenko .target_residency = 20000, 8775e7ec268SAndy Shevchenko .enter = &intel_idle, 87828ba086eSRafael J. Wysocki .enter_s2idle = intel_idle_s2idle, }, 8795e7ec268SAndy Shevchenko { 8805e7ec268SAndy Shevchenko .enter = NULL } 8815e7ec268SAndy Shevchenko }; 882ab1a8522SRafael J. Wysocki static struct cpuidle_state avn_cstates[] __initdata = { 883fab04b22SLen Brown { 884de09cdd0SLen Brown .name = "C1", 885fab04b22SLen Brown .desc = "MWAIT 0x00", 886b82b6ccaSDaniel Lezcano .flags = MWAIT2flg(0x00), 887fab04b22SLen Brown .exit_latency = 2, 888fab04b22SLen Brown .target_residency = 2, 8895fe2e527SRafael J. Wysocki .enter = &intel_idle, 89028ba086eSRafael J. Wysocki .enter_s2idle = intel_idle_s2idle, }, 891fab04b22SLen Brown { 892de09cdd0SLen Brown .name = "C6", 893fab04b22SLen Brown .desc = "MWAIT 0x51", 894b82b6ccaSDaniel Lezcano .flags = MWAIT2flg(0x51) | CPUIDLE_FLAG_TLB_FLUSHED, 895fab04b22SLen Brown .exit_latency = 15, 896fab04b22SLen Brown .target_residency = 45, 8975fe2e527SRafael J. Wysocki .enter = &intel_idle, 89828ba086eSRafael J. Wysocki .enter_s2idle = intel_idle_s2idle, }, 89988390996SJiang Liu { 90088390996SJiang Liu .enter = NULL } 901fab04b22SLen Brown }; 902ab1a8522SRafael J. Wysocki static struct cpuidle_state knl_cstates[] __initdata = { 903281baf7aSDasaratharaman Chandramouli { 904de09cdd0SLen Brown .name = "C1", 905281baf7aSDasaratharaman Chandramouli .desc = "MWAIT 0x00", 906281baf7aSDasaratharaman Chandramouli .flags = MWAIT2flg(0x00), 907281baf7aSDasaratharaman Chandramouli .exit_latency = 1, 908281baf7aSDasaratharaman Chandramouli .target_residency = 2, 909281baf7aSDasaratharaman Chandramouli .enter = &intel_idle, 91028ba086eSRafael J. Wysocki .enter_s2idle = intel_idle_s2idle }, 911281baf7aSDasaratharaman Chandramouli { 912de09cdd0SLen Brown .name = "C6", 913281baf7aSDasaratharaman Chandramouli .desc = "MWAIT 0x10", 914281baf7aSDasaratharaman Chandramouli .flags = MWAIT2flg(0x10) | CPUIDLE_FLAG_TLB_FLUSHED, 915281baf7aSDasaratharaman Chandramouli .exit_latency = 120, 916281baf7aSDasaratharaman Chandramouli .target_residency = 500, 917281baf7aSDasaratharaman Chandramouli .enter = &intel_idle, 91828ba086eSRafael J. Wysocki .enter_s2idle = intel_idle_s2idle }, 919281baf7aSDasaratharaman Chandramouli { 920281baf7aSDasaratharaman Chandramouli .enter = NULL } 921281baf7aSDasaratharaman Chandramouli }; 92226717172SLen Brown 923ab1a8522SRafael J. Wysocki static struct cpuidle_state bxt_cstates[] __initdata = { 9245dcef694SLen Brown { 925de09cdd0SLen Brown .name = "C1", 9265dcef694SLen Brown .desc = "MWAIT 0x00", 9275dcef694SLen Brown .flags = MWAIT2flg(0x00), 9285dcef694SLen Brown .exit_latency = 2, 9295dcef694SLen Brown .target_residency = 2, 9305dcef694SLen Brown .enter = &intel_idle, 93128ba086eSRafael J. Wysocki .enter_s2idle = intel_idle_s2idle, }, 9325dcef694SLen Brown { 933de09cdd0SLen Brown .name = "C1E", 9345dcef694SLen Brown .desc = "MWAIT 0x01", 935e6d4f08aSRafael J. Wysocki .flags = MWAIT2flg(0x01) | CPUIDLE_FLAG_ALWAYS_ENABLE, 9365dcef694SLen Brown .exit_latency = 10, 9375dcef694SLen Brown .target_residency = 20, 9385dcef694SLen Brown .enter = &intel_idle, 93928ba086eSRafael J. Wysocki .enter_s2idle = intel_idle_s2idle, }, 9405dcef694SLen Brown { 941de09cdd0SLen Brown .name = "C6", 9425dcef694SLen Brown .desc = "MWAIT 0x20", 9435dcef694SLen Brown .flags = MWAIT2flg(0x20) | CPUIDLE_FLAG_TLB_FLUSHED, 9445dcef694SLen Brown .exit_latency = 133, 9455dcef694SLen Brown .target_residency = 133, 9465dcef694SLen Brown .enter = &intel_idle, 94728ba086eSRafael J. Wysocki .enter_s2idle = intel_idle_s2idle, }, 9485dcef694SLen Brown { 949de09cdd0SLen Brown .name = "C7s", 9505dcef694SLen Brown .desc = "MWAIT 0x31", 9515dcef694SLen Brown .flags = MWAIT2flg(0x31) | CPUIDLE_FLAG_TLB_FLUSHED, 9525dcef694SLen Brown .exit_latency = 155, 9535dcef694SLen Brown .target_residency = 155, 9545dcef694SLen Brown .enter = &intel_idle, 95528ba086eSRafael J. Wysocki .enter_s2idle = intel_idle_s2idle, }, 9565dcef694SLen Brown { 957de09cdd0SLen Brown .name = "C8", 9585dcef694SLen Brown .desc = "MWAIT 0x40", 9595dcef694SLen Brown .flags = MWAIT2flg(0x40) | CPUIDLE_FLAG_TLB_FLUSHED, 9605dcef694SLen Brown .exit_latency = 1000, 9615dcef694SLen Brown .target_residency = 1000, 9625dcef694SLen Brown .enter = &intel_idle, 96328ba086eSRafael J. Wysocki .enter_s2idle = intel_idle_s2idle, }, 9645dcef694SLen Brown { 965de09cdd0SLen Brown .name = "C9", 9665dcef694SLen Brown .desc = "MWAIT 0x50", 9675dcef694SLen Brown .flags = MWAIT2flg(0x50) | CPUIDLE_FLAG_TLB_FLUSHED, 9685dcef694SLen Brown .exit_latency = 2000, 9695dcef694SLen Brown .target_residency = 2000, 9705dcef694SLen Brown .enter = &intel_idle, 97128ba086eSRafael J. Wysocki .enter_s2idle = intel_idle_s2idle, }, 9725dcef694SLen Brown { 973de09cdd0SLen Brown .name = "C10", 9745dcef694SLen Brown .desc = "MWAIT 0x60", 9755dcef694SLen Brown .flags = MWAIT2flg(0x60) | CPUIDLE_FLAG_TLB_FLUSHED, 9765dcef694SLen Brown .exit_latency = 10000, 9775dcef694SLen Brown .target_residency = 10000, 9785dcef694SLen Brown .enter = &intel_idle, 97928ba086eSRafael J. Wysocki .enter_s2idle = intel_idle_s2idle, }, 9805dcef694SLen Brown { 9815dcef694SLen Brown .enter = NULL } 9825dcef694SLen Brown }; 9835dcef694SLen Brown 984ab1a8522SRafael J. Wysocki static struct cpuidle_state dnv_cstates[] __initdata = { 9850080d65bSJacob Pan { 986de09cdd0SLen Brown .name = "C1", 9870080d65bSJacob Pan .desc = "MWAIT 0x00", 9880080d65bSJacob Pan .flags = MWAIT2flg(0x00), 9890080d65bSJacob Pan .exit_latency = 2, 9900080d65bSJacob Pan .target_residency = 2, 9910080d65bSJacob Pan .enter = &intel_idle, 99228ba086eSRafael J. Wysocki .enter_s2idle = intel_idle_s2idle, }, 9930080d65bSJacob Pan { 994de09cdd0SLen Brown .name = "C1E", 9950080d65bSJacob Pan .desc = "MWAIT 0x01", 996e6d4f08aSRafael J. Wysocki .flags = MWAIT2flg(0x01) | CPUIDLE_FLAG_ALWAYS_ENABLE, 9970080d65bSJacob Pan .exit_latency = 10, 9980080d65bSJacob Pan .target_residency = 20, 9990080d65bSJacob Pan .enter = &intel_idle, 100028ba086eSRafael J. Wysocki .enter_s2idle = intel_idle_s2idle, }, 10010080d65bSJacob Pan { 1002de09cdd0SLen Brown .name = "C6", 10030080d65bSJacob Pan .desc = "MWAIT 0x20", 10040080d65bSJacob Pan .flags = MWAIT2flg(0x20) | CPUIDLE_FLAG_TLB_FLUSHED, 10050080d65bSJacob Pan .exit_latency = 50, 10060080d65bSJacob Pan .target_residency = 500, 10070080d65bSJacob Pan .enter = &intel_idle, 100828ba086eSRafael J. Wysocki .enter_s2idle = intel_idle_s2idle, }, 10090080d65bSJacob Pan { 10100080d65bSJacob Pan .enter = NULL } 10110080d65bSJacob Pan }; 10120080d65bSJacob Pan 10139cf93f05SArtem Bityutskiy /* 10149cf93f05SArtem Bityutskiy * Note, depending on HW and FW revision, SnowRidge SoC may or may not support 10159cf93f05SArtem Bityutskiy * C6, and this is indicated in the CPUID mwait leaf. 10169cf93f05SArtem Bityutskiy */ 10179cf93f05SArtem Bityutskiy static struct cpuidle_state snr_cstates[] __initdata = { 10189cf93f05SArtem Bityutskiy { 10199cf93f05SArtem Bityutskiy .name = "C1", 10209cf93f05SArtem Bityutskiy .desc = "MWAIT 0x00", 10219cf93f05SArtem Bityutskiy .flags = MWAIT2flg(0x00), 10229cf93f05SArtem Bityutskiy .exit_latency = 2, 10239cf93f05SArtem Bityutskiy .target_residency = 2, 10249cf93f05SArtem Bityutskiy .enter = &intel_idle, 10259cf93f05SArtem Bityutskiy .enter_s2idle = intel_idle_s2idle, }, 10269cf93f05SArtem Bityutskiy { 10279cf93f05SArtem Bityutskiy .name = "C1E", 10289cf93f05SArtem Bityutskiy .desc = "MWAIT 0x01", 10299cf93f05SArtem Bityutskiy .flags = MWAIT2flg(0x01) | CPUIDLE_FLAG_ALWAYS_ENABLE, 10309cf93f05SArtem Bityutskiy .exit_latency = 15, 10319cf93f05SArtem Bityutskiy .target_residency = 25, 10329cf93f05SArtem Bityutskiy .enter = &intel_idle, 10339cf93f05SArtem Bityutskiy .enter_s2idle = intel_idle_s2idle, }, 10349cf93f05SArtem Bityutskiy { 10359cf93f05SArtem Bityutskiy .name = "C6", 10369cf93f05SArtem Bityutskiy .desc = "MWAIT 0x20", 10379cf93f05SArtem Bityutskiy .flags = MWAIT2flg(0x20) | CPUIDLE_FLAG_TLB_FLUSHED, 10389cf93f05SArtem Bityutskiy .exit_latency = 130, 10399cf93f05SArtem Bityutskiy .target_residency = 500, 10409cf93f05SArtem Bityutskiy .enter = &intel_idle, 10419cf93f05SArtem Bityutskiy .enter_s2idle = intel_idle_s2idle, }, 10429cf93f05SArtem Bityutskiy { 10439cf93f05SArtem Bityutskiy .enter = NULL } 10449cf93f05SArtem Bityutskiy }; 10459cf93f05SArtem Bityutskiy 1046ab1a8522SRafael J. Wysocki static const struct idle_cpu idle_cpu_nehalem __initconst = { 1047b66b8b9aSAndi Kleen .state_table = nehalem_cstates, 1048b66b8b9aSAndi Kleen .auto_demotion_disable_flags = NHM_C1_AUTO_DEMOTE | NHM_C3_AUTO_DEMOTE, 104932e95180SLen Brown .disable_promotion_to_c1e = true, 1050b66b8b9aSAndi Kleen }; 1051b66b8b9aSAndi Kleen 1052ab1a8522SRafael J. Wysocki static const struct idle_cpu idle_cpu_nhx __initconst = { 1053e6d4f08aSRafael J. Wysocki .state_table = nehalem_cstates, 1054e6d4f08aSRafael J. Wysocki .auto_demotion_disable_flags = NHM_C1_AUTO_DEMOTE | NHM_C3_AUTO_DEMOTE, 1055e6d4f08aSRafael J. Wysocki .disable_promotion_to_c1e = true, 1056e6d4f08aSRafael J. Wysocki .use_acpi = true, 1057e6d4f08aSRafael J. Wysocki }; 1058e6d4f08aSRafael J. Wysocki 1059ab1a8522SRafael J. Wysocki static const struct idle_cpu idle_cpu_atom __initconst = { 1060b66b8b9aSAndi Kleen .state_table = atom_cstates, 1061b66b8b9aSAndi Kleen }; 1062b66b8b9aSAndi Kleen 1063ab1a8522SRafael J. Wysocki static const struct idle_cpu idle_cpu_tangier __initconst = { 10645e7ec268SAndy Shevchenko .state_table = tangier_cstates, 10655e7ec268SAndy Shevchenko }; 10665e7ec268SAndy Shevchenko 1067ab1a8522SRafael J. Wysocki static const struct idle_cpu idle_cpu_lincroft __initconst = { 1068b66b8b9aSAndi Kleen .state_table = atom_cstates, 1069b66b8b9aSAndi Kleen .auto_demotion_disable_flags = ATM_LNC_C6_AUTO_DEMOTE, 1070b66b8b9aSAndi Kleen }; 1071b66b8b9aSAndi Kleen 1072ab1a8522SRafael J. Wysocki static const struct idle_cpu idle_cpu_snb __initconst = { 1073b66b8b9aSAndi Kleen .state_table = snb_cstates, 107432e95180SLen Brown .disable_promotion_to_c1e = true, 1075b66b8b9aSAndi Kleen }; 1076b66b8b9aSAndi Kleen 1077ab1a8522SRafael J. Wysocki static const struct idle_cpu idle_cpu_snx __initconst = { 1078e6d4f08aSRafael J. Wysocki .state_table = snb_cstates, 1079e6d4f08aSRafael J. Wysocki .disable_promotion_to_c1e = true, 1080e6d4f08aSRafael J. Wysocki .use_acpi = true, 1081e6d4f08aSRafael J. Wysocki }; 1082e6d4f08aSRafael J. Wysocki 1083ab1a8522SRafael J. Wysocki static const struct idle_cpu idle_cpu_byt __initconst = { 1084718987d6SLen Brown .state_table = byt_cstates, 1085718987d6SLen Brown .disable_promotion_to_c1e = true, 10868c058d53SLen Brown .byt_auto_demotion_disable_flag = true, 1087718987d6SLen Brown }; 1088718987d6SLen Brown 1089ab1a8522SRafael J. Wysocki static const struct idle_cpu idle_cpu_cht __initconst = { 1090cab07a56SLen Brown .state_table = cht_cstates, 1091cab07a56SLen Brown .disable_promotion_to_c1e = true, 1092cab07a56SLen Brown .byt_auto_demotion_disable_flag = true, 1093cab07a56SLen Brown }; 1094cab07a56SLen Brown 1095ab1a8522SRafael J. Wysocki static const struct idle_cpu idle_cpu_ivb __initconst = { 10966edab08cSLen Brown .state_table = ivb_cstates, 109732e95180SLen Brown .disable_promotion_to_c1e = true, 10986edab08cSLen Brown }; 10996edab08cSLen Brown 1100ab1a8522SRafael J. Wysocki static const struct idle_cpu idle_cpu_ivt __initconst = { 11010138d8f0SLen Brown .state_table = ivt_cstates, 11020138d8f0SLen Brown .disable_promotion_to_c1e = true, 1103e6d4f08aSRafael J. Wysocki .use_acpi = true, 11040138d8f0SLen Brown }; 11050138d8f0SLen Brown 1106ab1a8522SRafael J. Wysocki static const struct idle_cpu idle_cpu_hsw __initconst = { 110785a4d2d4SLen Brown .state_table = hsw_cstates, 110832e95180SLen Brown .disable_promotion_to_c1e = true, 110985a4d2d4SLen Brown }; 111085a4d2d4SLen Brown 1111ab1a8522SRafael J. Wysocki static const struct idle_cpu idle_cpu_hsx __initconst = { 1112e6d4f08aSRafael J. Wysocki .state_table = hsw_cstates, 1113e6d4f08aSRafael J. Wysocki .disable_promotion_to_c1e = true, 1114e6d4f08aSRafael J. Wysocki .use_acpi = true, 1115e6d4f08aSRafael J. Wysocki }; 1116e6d4f08aSRafael J. Wysocki 1117ab1a8522SRafael J. Wysocki static const struct idle_cpu idle_cpu_bdw __initconst = { 1118a138b568SLen Brown .state_table = bdw_cstates, 1119a138b568SLen Brown .disable_promotion_to_c1e = true, 1120a138b568SLen Brown }; 1121a138b568SLen Brown 1122ab1a8522SRafael J. Wysocki static const struct idle_cpu idle_cpu_bdx __initconst = { 1123e6d4f08aSRafael J. Wysocki .state_table = bdw_cstates, 1124e6d4f08aSRafael J. Wysocki .disable_promotion_to_c1e = true, 1125e6d4f08aSRafael J. Wysocki .use_acpi = true, 1126e6d4f08aSRafael J. Wysocki }; 1127e6d4f08aSRafael J. Wysocki 1128ab1a8522SRafael J. Wysocki static const struct idle_cpu idle_cpu_skl __initconst = { 1129493f133fSLen Brown .state_table = skl_cstates, 1130493f133fSLen Brown .disable_promotion_to_c1e = true, 1131493f133fSLen Brown }; 1132493f133fSLen Brown 1133ab1a8522SRafael J. Wysocki static const struct idle_cpu idle_cpu_skx __initconst = { 1134f9e71657SLen Brown .state_table = skx_cstates, 1135f9e71657SLen Brown .disable_promotion_to_c1e = true, 1136e6d4f08aSRafael J. Wysocki .use_acpi = true, 1137f9e71657SLen Brown }; 1138493f133fSLen Brown 1139a472ad2bSChen Yu static const struct idle_cpu idle_cpu_icx __initconst = { 1140a472ad2bSChen Yu .state_table = icx_cstates, 1141a472ad2bSChen Yu .disable_promotion_to_c1e = true, 1142a472ad2bSChen Yu .use_acpi = true, 1143a472ad2bSChen Yu }; 1144a472ad2bSChen Yu 11459edf3c0fSArtem Bityutskiy static const struct idle_cpu idle_cpu_spr __initconst = { 11469edf3c0fSArtem Bityutskiy .state_table = spr_cstates, 11479edf3c0fSArtem Bityutskiy .disable_promotion_to_c1e = true, 11489edf3c0fSArtem Bityutskiy .use_acpi = true, 11499edf3c0fSArtem Bityutskiy }; 11509edf3c0fSArtem Bityutskiy 1151ab1a8522SRafael J. Wysocki static const struct idle_cpu idle_cpu_avn __initconst = { 1152fab04b22SLen Brown .state_table = avn_cstates, 1153fab04b22SLen Brown .disable_promotion_to_c1e = true, 1154e6d4f08aSRafael J. Wysocki .use_acpi = true, 1155fab04b22SLen Brown }; 1156fab04b22SLen Brown 1157ab1a8522SRafael J. Wysocki static const struct idle_cpu idle_cpu_knl __initconst = { 1158281baf7aSDasaratharaman Chandramouli .state_table = knl_cstates, 1159e6d4f08aSRafael J. Wysocki .use_acpi = true, 1160281baf7aSDasaratharaman Chandramouli }; 1161281baf7aSDasaratharaman Chandramouli 1162ab1a8522SRafael J. Wysocki static const struct idle_cpu idle_cpu_bxt __initconst = { 11635dcef694SLen Brown .state_table = bxt_cstates, 11645dcef694SLen Brown .disable_promotion_to_c1e = true, 11655dcef694SLen Brown }; 11665dcef694SLen Brown 1167ab1a8522SRafael J. Wysocki static const struct idle_cpu idle_cpu_dnv __initconst = { 11680080d65bSJacob Pan .state_table = dnv_cstates, 11690080d65bSJacob Pan .disable_promotion_to_c1e = true, 1170e6d4f08aSRafael J. Wysocki .use_acpi = true, 11710080d65bSJacob Pan }; 11720080d65bSJacob Pan 11739cf93f05SArtem Bityutskiy static const struct idle_cpu idle_cpu_snr __initconst = { 11749cf93f05SArtem Bityutskiy .state_table = snr_cstates, 11759cf93f05SArtem Bityutskiy .disable_promotion_to_c1e = true, 11769cf93f05SArtem Bityutskiy .use_acpi = true, 11779cf93f05SArtem Bityutskiy }; 11789cf93f05SArtem Bityutskiy 1179d5cdc3c4SMathias Krause static const struct x86_cpu_id intel_idle_ids[] __initconst = { 11804a9f45a0SThomas Gleixner X86_MATCH_INTEL_FAM6_MODEL(NEHALEM_EP, &idle_cpu_nhx), 11814a9f45a0SThomas Gleixner X86_MATCH_INTEL_FAM6_MODEL(NEHALEM, &idle_cpu_nehalem), 11824a9f45a0SThomas Gleixner X86_MATCH_INTEL_FAM6_MODEL(NEHALEM_G, &idle_cpu_nehalem), 11834a9f45a0SThomas Gleixner X86_MATCH_INTEL_FAM6_MODEL(WESTMERE, &idle_cpu_nehalem), 11844a9f45a0SThomas Gleixner X86_MATCH_INTEL_FAM6_MODEL(WESTMERE_EP, &idle_cpu_nhx), 11854a9f45a0SThomas Gleixner X86_MATCH_INTEL_FAM6_MODEL(NEHALEM_EX, &idle_cpu_nhx), 11864a9f45a0SThomas Gleixner X86_MATCH_INTEL_FAM6_MODEL(ATOM_BONNELL, &idle_cpu_atom), 11874a9f45a0SThomas Gleixner X86_MATCH_INTEL_FAM6_MODEL(ATOM_BONNELL_MID, &idle_cpu_lincroft), 11884a9f45a0SThomas Gleixner X86_MATCH_INTEL_FAM6_MODEL(WESTMERE_EX, &idle_cpu_nhx), 11894a9f45a0SThomas Gleixner X86_MATCH_INTEL_FAM6_MODEL(SANDYBRIDGE, &idle_cpu_snb), 11904a9f45a0SThomas Gleixner X86_MATCH_INTEL_FAM6_MODEL(SANDYBRIDGE_X, &idle_cpu_snx), 11914a9f45a0SThomas Gleixner X86_MATCH_INTEL_FAM6_MODEL(ATOM_SALTWELL, &idle_cpu_atom), 11924a9f45a0SThomas Gleixner X86_MATCH_INTEL_FAM6_MODEL(ATOM_SILVERMONT, &idle_cpu_byt), 11934a9f45a0SThomas Gleixner X86_MATCH_INTEL_FAM6_MODEL(ATOM_SILVERMONT_MID, &idle_cpu_tangier), 11944a9f45a0SThomas Gleixner X86_MATCH_INTEL_FAM6_MODEL(ATOM_AIRMONT, &idle_cpu_cht), 11954a9f45a0SThomas Gleixner X86_MATCH_INTEL_FAM6_MODEL(IVYBRIDGE, &idle_cpu_ivb), 11964a9f45a0SThomas Gleixner X86_MATCH_INTEL_FAM6_MODEL(IVYBRIDGE_X, &idle_cpu_ivt), 11974a9f45a0SThomas Gleixner X86_MATCH_INTEL_FAM6_MODEL(HASWELL, &idle_cpu_hsw), 11984a9f45a0SThomas Gleixner X86_MATCH_INTEL_FAM6_MODEL(HASWELL_X, &idle_cpu_hsx), 11994a9f45a0SThomas Gleixner X86_MATCH_INTEL_FAM6_MODEL(HASWELL_L, &idle_cpu_hsw), 12004a9f45a0SThomas Gleixner X86_MATCH_INTEL_FAM6_MODEL(HASWELL_G, &idle_cpu_hsw), 12014a9f45a0SThomas Gleixner X86_MATCH_INTEL_FAM6_MODEL(ATOM_SILVERMONT_D, &idle_cpu_avn), 12024a9f45a0SThomas Gleixner X86_MATCH_INTEL_FAM6_MODEL(BROADWELL, &idle_cpu_bdw), 12034a9f45a0SThomas Gleixner X86_MATCH_INTEL_FAM6_MODEL(BROADWELL_G, &idle_cpu_bdw), 12044a9f45a0SThomas Gleixner X86_MATCH_INTEL_FAM6_MODEL(BROADWELL_X, &idle_cpu_bdx), 12054a9f45a0SThomas Gleixner X86_MATCH_INTEL_FAM6_MODEL(BROADWELL_D, &idle_cpu_bdx), 12064a9f45a0SThomas Gleixner X86_MATCH_INTEL_FAM6_MODEL(SKYLAKE_L, &idle_cpu_skl), 12074a9f45a0SThomas Gleixner X86_MATCH_INTEL_FAM6_MODEL(SKYLAKE, &idle_cpu_skl), 12084a9f45a0SThomas Gleixner X86_MATCH_INTEL_FAM6_MODEL(KABYLAKE_L, &idle_cpu_skl), 12094a9f45a0SThomas Gleixner X86_MATCH_INTEL_FAM6_MODEL(KABYLAKE, &idle_cpu_skl), 12104a9f45a0SThomas Gleixner X86_MATCH_INTEL_FAM6_MODEL(SKYLAKE_X, &idle_cpu_skx), 1211a472ad2bSChen Yu X86_MATCH_INTEL_FAM6_MODEL(ICELAKE_X, &idle_cpu_icx), 121222141d5fSArtem Bityutskiy X86_MATCH_INTEL_FAM6_MODEL(ICELAKE_D, &idle_cpu_icx), 12139edf3c0fSArtem Bityutskiy X86_MATCH_INTEL_FAM6_MODEL(SAPPHIRERAPIDS_X, &idle_cpu_spr), 12144a9f45a0SThomas Gleixner X86_MATCH_INTEL_FAM6_MODEL(XEON_PHI_KNL, &idle_cpu_knl), 12154a9f45a0SThomas Gleixner X86_MATCH_INTEL_FAM6_MODEL(XEON_PHI_KNM, &idle_cpu_knl), 12164a9f45a0SThomas Gleixner X86_MATCH_INTEL_FAM6_MODEL(ATOM_GOLDMONT, &idle_cpu_bxt), 12174a9f45a0SThomas Gleixner X86_MATCH_INTEL_FAM6_MODEL(ATOM_GOLDMONT_PLUS, &idle_cpu_bxt), 12184a9f45a0SThomas Gleixner X86_MATCH_INTEL_FAM6_MODEL(ATOM_GOLDMONT_D, &idle_cpu_dnv), 12199cf93f05SArtem Bityutskiy X86_MATCH_INTEL_FAM6_MODEL(ATOM_TREMONT_D, &idle_cpu_snr), 1220b66b8b9aSAndi Kleen {} 1221b66b8b9aSAndi Kleen }; 1222b66b8b9aSAndi Kleen 122318734958SRafael J. Wysocki static const struct x86_cpu_id intel_mwait_ids[] __initconst = { 12244a9f45a0SThomas Gleixner X86_MATCH_VENDOR_FAM_FEATURE(INTEL, 6, X86_FEATURE_MWAIT, NULL), 122518734958SRafael J. Wysocki {} 122618734958SRafael J. Wysocki }; 122718734958SRafael J. Wysocki 1228095928aeSRafael J. Wysocki static bool __init intel_idle_max_cstate_reached(int cstate) 122918734958SRafael J. Wysocki { 123018734958SRafael J. Wysocki if (cstate + 1 > max_cstate) { 123118734958SRafael J. Wysocki pr_info("max_cstate %d reached\n", max_cstate); 123218734958SRafael J. Wysocki return true; 123318734958SRafael J. Wysocki } 123418734958SRafael J. Wysocki return false; 123518734958SRafael J. Wysocki } 123618734958SRafael J. Wysocki 12374d916140SPeter Zijlstra static bool __init intel_idle_state_needs_timer_stop(struct cpuidle_state *state) 12384d916140SPeter Zijlstra { 12394d916140SPeter Zijlstra unsigned long eax = flg2MWAIT(state->flags); 12404d916140SPeter Zijlstra 12414d916140SPeter Zijlstra if (boot_cpu_has(X86_FEATURE_ARAT)) 12424d916140SPeter Zijlstra return false; 12434d916140SPeter Zijlstra 12444d916140SPeter Zijlstra /* 12454d916140SPeter Zijlstra * Switch over to one-shot tick broadcast if the target C-state 12464d916140SPeter Zijlstra * is deeper than C1. 12474d916140SPeter Zijlstra */ 12484d916140SPeter Zijlstra return !!((eax >> MWAIT_SUBSTATE_SIZE) & MWAIT_CSTATE_MASK); 12494d916140SPeter Zijlstra } 12504d916140SPeter Zijlstra 125118734958SRafael J. Wysocki #ifdef CONFIG_ACPI_PROCESSOR_CSTATE 125218734958SRafael J. Wysocki #include <acpi/processor.h> 125318734958SRafael J. Wysocki 12544ec32d9eSRafael J. Wysocki static bool no_acpi __read_mostly; 12554ec32d9eSRafael J. Wysocki module_param(no_acpi, bool, 0444); 12564ec32d9eSRafael J. Wysocki MODULE_PARM_DESC(no_acpi, "Do not use ACPI _CST for building the idle states list"); 12574ec32d9eSRafael J. Wysocki 12583a5be9b8SRafael J. Wysocki static bool force_use_acpi __read_mostly; /* No effect if no_acpi is set. */ 12593a5be9b8SRafael J. Wysocki module_param_named(use_acpi, force_use_acpi, bool, 0444); 12603a5be9b8SRafael J. Wysocki MODULE_PARM_DESC(use_acpi, "Use ACPI _CST for building the idle states list"); 12613a5be9b8SRafael J. Wysocki 1262095928aeSRafael J. Wysocki static struct acpi_processor_power acpi_state_table __initdata; 126318734958SRafael J. Wysocki 126418734958SRafael J. Wysocki /** 126518734958SRafael J. Wysocki * intel_idle_cst_usable - Check if the _CST information can be used. 126618734958SRafael J. Wysocki * 126718734958SRafael J. Wysocki * Check if all of the C-states listed by _CST in the max_cstate range are 126818734958SRafael J. Wysocki * ACPI_CSTATE_FFH, which means that they should be entered via MWAIT. 126918734958SRafael J. Wysocki */ 1270095928aeSRafael J. Wysocki static bool __init intel_idle_cst_usable(void) 127118734958SRafael J. Wysocki { 127218734958SRafael J. Wysocki int cstate, limit; 127318734958SRafael J. Wysocki 127418734958SRafael J. Wysocki limit = min_t(int, min_t(int, CPUIDLE_STATE_MAX, max_cstate + 1), 127518734958SRafael J. Wysocki acpi_state_table.count); 127618734958SRafael J. Wysocki 127718734958SRafael J. Wysocki for (cstate = 1; cstate < limit; cstate++) { 127818734958SRafael J. Wysocki struct acpi_processor_cx *cx = &acpi_state_table.states[cstate]; 127918734958SRafael J. Wysocki 128018734958SRafael J. Wysocki if (cx->entry_method != ACPI_CSTATE_FFH) 128118734958SRafael J. Wysocki return false; 128218734958SRafael J. Wysocki } 128318734958SRafael J. Wysocki 128418734958SRafael J. Wysocki return true; 128518734958SRafael J. Wysocki } 128618734958SRafael J. Wysocki 1287095928aeSRafael J. Wysocki static bool __init intel_idle_acpi_cst_extract(void) 128818734958SRafael J. Wysocki { 128918734958SRafael J. Wysocki unsigned int cpu; 129018734958SRafael J. Wysocki 12914ec32d9eSRafael J. Wysocki if (no_acpi) { 12924ec32d9eSRafael J. Wysocki pr_debug("Not allowed to use ACPI _CST\n"); 12934ec32d9eSRafael J. Wysocki return false; 12944ec32d9eSRafael J. Wysocki } 12954ec32d9eSRafael J. Wysocki 129618734958SRafael J. Wysocki for_each_possible_cpu(cpu) { 129718734958SRafael J. Wysocki struct acpi_processor *pr = per_cpu(processors, cpu); 129818734958SRafael J. Wysocki 129918734958SRafael J. Wysocki if (!pr) 130018734958SRafael J. Wysocki continue; 130118734958SRafael J. Wysocki 130218734958SRafael J. Wysocki if (acpi_processor_evaluate_cst(pr->handle, cpu, &acpi_state_table)) 130318734958SRafael J. Wysocki continue; 130418734958SRafael J. Wysocki 130518734958SRafael J. Wysocki acpi_state_table.count++; 130618734958SRafael J. Wysocki 130718734958SRafael J. Wysocki if (!intel_idle_cst_usable()) 130818734958SRafael J. Wysocki continue; 130918734958SRafael J. Wysocki 131075af76d0SMel Gorman if (!acpi_processor_claim_cst_control()) 131175af76d0SMel Gorman break; 131218734958SRafael J. Wysocki 131318734958SRafael J. Wysocki return true; 131418734958SRafael J. Wysocki } 131518734958SRafael J. Wysocki 131675af76d0SMel Gorman acpi_state_table.count = 0; 131718734958SRafael J. Wysocki pr_debug("ACPI _CST not found or not usable\n"); 131818734958SRafael J. Wysocki return false; 131918734958SRafael J. Wysocki } 132018734958SRafael J. Wysocki 1321095928aeSRafael J. Wysocki static void __init intel_idle_init_cstates_acpi(struct cpuidle_driver *drv) 132218734958SRafael J. Wysocki { 132318734958SRafael J. Wysocki int cstate, limit = min_t(int, CPUIDLE_STATE_MAX, acpi_state_table.count); 132418734958SRafael J. Wysocki 132518734958SRafael J. Wysocki /* 132618734958SRafael J. Wysocki * If limit > 0, intel_idle_cst_usable() has returned 'true', so all of 132718734958SRafael J. Wysocki * the interesting states are ACPI_CSTATE_FFH. 132818734958SRafael J. Wysocki */ 132918734958SRafael J. Wysocki for (cstate = 1; cstate < limit; cstate++) { 133018734958SRafael J. Wysocki struct acpi_processor_cx *cx; 133118734958SRafael J. Wysocki struct cpuidle_state *state; 133218734958SRafael J. Wysocki 13334e0ba557SChen Yu if (intel_idle_max_cstate_reached(cstate - 1)) 133418734958SRafael J. Wysocki break; 133518734958SRafael J. Wysocki 133618734958SRafael J. Wysocki cx = &acpi_state_table.states[cstate]; 133718734958SRafael J. Wysocki 133818734958SRafael J. Wysocki state = &drv->states[drv->state_count++]; 133918734958SRafael J. Wysocki 134018734958SRafael J. Wysocki snprintf(state->name, CPUIDLE_NAME_LEN, "C%d_ACPI", cstate); 134118734958SRafael J. Wysocki strlcpy(state->desc, cx->desc, CPUIDLE_DESC_LEN); 134218734958SRafael J. Wysocki state->exit_latency = cx->latency; 134318734958SRafael J. Wysocki /* 134418734958SRafael J. Wysocki * For C1-type C-states use the same number for both the exit 134518734958SRafael J. Wysocki * latency and target residency, because that is the case for 134618734958SRafael J. Wysocki * C1 in the majority of the static C-states tables above. 134718734958SRafael J. Wysocki * For the other types of C-states, however, set the target 134818734958SRafael J. Wysocki * residency to 3 times the exit latency which should lead to 134918734958SRafael J. Wysocki * a reasonable balance between energy-efficiency and 135018734958SRafael J. Wysocki * performance in the majority of interesting cases. 135118734958SRafael J. Wysocki */ 135218734958SRafael J. Wysocki state->target_residency = cx->latency; 135318734958SRafael J. Wysocki if (cx->type > ACPI_STATE_C1) 135418734958SRafael J. Wysocki state->target_residency *= 3; 135518734958SRafael J. Wysocki 135618734958SRafael J. Wysocki state->flags = MWAIT2flg(cx->address); 135718734958SRafael J. Wysocki if (cx->type > ACPI_STATE_C2) 135818734958SRafael J. Wysocki state->flags |= CPUIDLE_FLAG_TLB_FLUSHED; 135918734958SRafael J. Wysocki 13604dcb78eeSRafael J. Wysocki if (disabled_states_mask & BIT(cstate)) 13614dcb78eeSRafael J. Wysocki state->flags |= CPUIDLE_FLAG_OFF; 13624dcb78eeSRafael J. Wysocki 13636e1d2bc6SPeter Zijlstra if (intel_idle_state_needs_timer_stop(state)) 13646e1d2bc6SPeter Zijlstra state->flags |= CPUIDLE_FLAG_TIMER_STOP; 13656e1d2bc6SPeter Zijlstra 136618734958SRafael J. Wysocki state->enter = intel_idle; 136718734958SRafael J. Wysocki state->enter_s2idle = intel_idle_s2idle; 136818734958SRafael J. Wysocki } 136918734958SRafael J. Wysocki } 1370bff8e60aSRafael J. Wysocki 1371095928aeSRafael J. Wysocki static bool __init intel_idle_off_by_default(u32 mwait_hint) 1372bff8e60aSRafael J. Wysocki { 1373bff8e60aSRafael J. Wysocki int cstate, limit; 1374bff8e60aSRafael J. Wysocki 1375bff8e60aSRafael J. Wysocki /* 1376bff8e60aSRafael J. Wysocki * If there are no _CST C-states, do not disable any C-states by 1377bff8e60aSRafael J. Wysocki * default. 1378bff8e60aSRafael J. Wysocki */ 1379bff8e60aSRafael J. Wysocki if (!acpi_state_table.count) 1380bff8e60aSRafael J. Wysocki return false; 1381bff8e60aSRafael J. Wysocki 1382bff8e60aSRafael J. Wysocki limit = min_t(int, CPUIDLE_STATE_MAX, acpi_state_table.count); 1383bff8e60aSRafael J. Wysocki /* 1384bff8e60aSRafael J. Wysocki * If limit > 0, intel_idle_cst_usable() has returned 'true', so all of 1385bff8e60aSRafael J. Wysocki * the interesting states are ACPI_CSTATE_FFH. 1386bff8e60aSRafael J. Wysocki */ 1387bff8e60aSRafael J. Wysocki for (cstate = 1; cstate < limit; cstate++) { 1388bff8e60aSRafael J. Wysocki if (acpi_state_table.states[cstate].address == mwait_hint) 1389bff8e60aSRafael J. Wysocki return false; 1390bff8e60aSRafael J. Wysocki } 1391bff8e60aSRafael J. Wysocki return true; 1392bff8e60aSRafael J. Wysocki } 139318734958SRafael J. Wysocki #else /* !CONFIG_ACPI_PROCESSOR_CSTATE */ 13943a5be9b8SRafael J. Wysocki #define force_use_acpi (false) 13953a5be9b8SRafael J. Wysocki 139618734958SRafael J. Wysocki static inline bool intel_idle_acpi_cst_extract(void) { return false; } 139718734958SRafael J. Wysocki static inline void intel_idle_init_cstates_acpi(struct cpuidle_driver *drv) { } 1398bff8e60aSRafael J. Wysocki static inline bool intel_idle_off_by_default(u32 mwait_hint) { return false; } 139918734958SRafael J. Wysocki #endif /* !CONFIG_ACPI_PROCESSOR_CSTATE */ 140018734958SRafael J. Wysocki 1401da0e58c0SArtem Bityutskiy static void c1e_promotion_enable(void); 1402da0e58c0SArtem Bityutskiy 14036eacb15fSRafael J. Wysocki /** 14046eacb15fSRafael J. Wysocki * ivt_idle_state_table_update - Tune the idle states table for Ivy Town. 14050138d8f0SLen Brown * 14066eacb15fSRafael J. Wysocki * Tune IVT multi-socket targets. 14076eacb15fSRafael J. Wysocki * Assumption: num_sockets == (max_package_num + 1). 14080138d8f0SLen Brown */ 1409095928aeSRafael J. Wysocki static void __init ivt_idle_state_table_update(void) 14100138d8f0SLen Brown { 14110138d8f0SLen Brown /* IVT uses a different table for 1-2, 3-4, and > 4 sockets */ 14120138d8f0SLen Brown int cpu, package_num, num_sockets = 1; 14130138d8f0SLen Brown 14140138d8f0SLen Brown for_each_online_cpu(cpu) { 14150138d8f0SLen Brown package_num = topology_physical_package_id(cpu); 14160138d8f0SLen Brown if (package_num + 1 > num_sockets) { 14170138d8f0SLen Brown num_sockets = package_num + 1; 14180138d8f0SLen Brown 1419d27dca42SChristoph Jaeger if (num_sockets > 4) { 14200138d8f0SLen Brown cpuidle_state_table = ivt_cstates_8s; 14210138d8f0SLen Brown return; 14220138d8f0SLen Brown } 14230138d8f0SLen Brown } 1424d27dca42SChristoph Jaeger } 14250138d8f0SLen Brown 14260138d8f0SLen Brown if (num_sockets > 2) 14270138d8f0SLen Brown cpuidle_state_table = ivt_cstates_4s; 1428d70e28f5SLen Brown 14290138d8f0SLen Brown /* else, 1 and 2 socket systems use default ivt_cstates */ 14300138d8f0SLen Brown } 14315dcef694SLen Brown 143286e9466aSRafael J. Wysocki /** 143386e9466aSRafael J. Wysocki * irtl_2_usec - IRTL to microseconds conversion. 143486e9466aSRafael J. Wysocki * @irtl: IRTL MSR value. 143586e9466aSRafael J. Wysocki * 143686e9466aSRafael J. Wysocki * Translate the IRTL (Interrupt Response Time Limit) MSR value to microseconds. 14375dcef694SLen Brown */ 143886e9466aSRafael J. Wysocki static unsigned long long __init irtl_2_usec(unsigned long long irtl) 143986e9466aSRafael J. Wysocki { 1440095928aeSRafael J. Wysocki static const unsigned int irtl_ns_units[] __initconst = { 1441095928aeSRafael J. Wysocki 1, 32, 1024, 32768, 1048576, 33554432, 0, 0 1442095928aeSRafael J. Wysocki }; 14435dcef694SLen Brown unsigned long long ns; 14445dcef694SLen Brown 14453451ab3eSJan Beulich if (!irtl) 14463451ab3eSJan Beulich return 0; 14473451ab3eSJan Beulich 1448bef45096SJan Beulich ns = irtl_ns_units[(irtl >> 10) & 0x7]; 14495dcef694SLen Brown 145086e9466aSRafael J. Wysocki return div_u64((irtl & 0x3FF) * ns, NSEC_PER_USEC); 14515dcef694SLen Brown } 145286e9466aSRafael J. Wysocki 14536eacb15fSRafael J. Wysocki /** 14546eacb15fSRafael J. Wysocki * bxt_idle_state_table_update - Fix up the Broxton idle states table. 14555dcef694SLen Brown * 14566eacb15fSRafael J. Wysocki * On BXT, trust the IRTL (Interrupt Response Time Limit) MSR to show the 14576eacb15fSRafael J. Wysocki * definitive maximum latency and use the same value for target_residency. 14585dcef694SLen Brown */ 1459095928aeSRafael J. Wysocki static void __init bxt_idle_state_table_update(void) 14605dcef694SLen Brown { 14615dcef694SLen Brown unsigned long long msr; 14623451ab3eSJan Beulich unsigned int usec; 14635dcef694SLen Brown 14645dcef694SLen Brown rdmsrl(MSR_PKGC6_IRTL, msr); 14653451ab3eSJan Beulich usec = irtl_2_usec(msr); 14663451ab3eSJan Beulich if (usec) { 14675dcef694SLen Brown bxt_cstates[2].exit_latency = usec; 14685dcef694SLen Brown bxt_cstates[2].target_residency = usec; 14695dcef694SLen Brown } 14705dcef694SLen Brown 14715dcef694SLen Brown rdmsrl(MSR_PKGC7_IRTL, msr); 14723451ab3eSJan Beulich usec = irtl_2_usec(msr); 14733451ab3eSJan Beulich if (usec) { 14745dcef694SLen Brown bxt_cstates[3].exit_latency = usec; 14755dcef694SLen Brown bxt_cstates[3].target_residency = usec; 14765dcef694SLen Brown } 14775dcef694SLen Brown 14785dcef694SLen Brown rdmsrl(MSR_PKGC8_IRTL, msr); 14793451ab3eSJan Beulich usec = irtl_2_usec(msr); 14803451ab3eSJan Beulich if (usec) { 14815dcef694SLen Brown bxt_cstates[4].exit_latency = usec; 14825dcef694SLen Brown bxt_cstates[4].target_residency = usec; 14835dcef694SLen Brown } 14845dcef694SLen Brown 14855dcef694SLen Brown rdmsrl(MSR_PKGC9_IRTL, msr); 14863451ab3eSJan Beulich usec = irtl_2_usec(msr); 14873451ab3eSJan Beulich if (usec) { 14885dcef694SLen Brown bxt_cstates[5].exit_latency = usec; 14895dcef694SLen Brown bxt_cstates[5].target_residency = usec; 14905dcef694SLen Brown } 14915dcef694SLen Brown 14925dcef694SLen Brown rdmsrl(MSR_PKGC10_IRTL, msr); 14933451ab3eSJan Beulich usec = irtl_2_usec(msr); 14943451ab3eSJan Beulich if (usec) { 14955dcef694SLen Brown bxt_cstates[6].exit_latency = usec; 14965dcef694SLen Brown bxt_cstates[6].target_residency = usec; 14975dcef694SLen Brown } 14985dcef694SLen Brown 14995dcef694SLen Brown } 15006eacb15fSRafael J. Wysocki 15016eacb15fSRafael J. Wysocki /** 15026eacb15fSRafael J. Wysocki * sklh_idle_state_table_update - Fix up the Sky Lake idle states table. 1503d70e28f5SLen Brown * 15046eacb15fSRafael J. Wysocki * On SKL-H (model 0x5e) skip C8 and C9 if C10 is enabled and SGX disabled. 1505d70e28f5SLen Brown */ 1506095928aeSRafael J. Wysocki static void __init sklh_idle_state_table_update(void) 1507d70e28f5SLen Brown { 1508d70e28f5SLen Brown unsigned long long msr; 1509d70e28f5SLen Brown unsigned int eax, ebx, ecx, edx; 1510d70e28f5SLen Brown 1511d70e28f5SLen Brown 1512d70e28f5SLen Brown /* if PC10 disabled via cmdline intel_idle.max_cstate=7 or shallower */ 1513d70e28f5SLen Brown if (max_cstate <= 7) 15140138d8f0SLen Brown return; 1515d70e28f5SLen Brown 1516d70e28f5SLen Brown /* if PC10 not present in CPUID.MWAIT.EDX */ 1517d70e28f5SLen Brown if ((mwait_substates & (0xF << 28)) == 0) 1518d70e28f5SLen Brown return; 1519d70e28f5SLen Brown 15206cfb2374SLen Brown rdmsrl(MSR_PKG_CST_CONFIG_CONTROL, msr); 1521d70e28f5SLen Brown 1522d70e28f5SLen Brown /* PC10 is not enabled in PKG C-state limit */ 1523d70e28f5SLen Brown if ((msr & 0xF) != 8) 1524d70e28f5SLen Brown return; 1525d70e28f5SLen Brown 1526d70e28f5SLen Brown ecx = 0; 1527d70e28f5SLen Brown cpuid(7, &eax, &ebx, &ecx, &edx); 1528d70e28f5SLen Brown 1529d70e28f5SLen Brown /* if SGX is present */ 1530d70e28f5SLen Brown if (ebx & (1 << 2)) { 1531d70e28f5SLen Brown 153232ad73dbSSean Christopherson rdmsrl(MSR_IA32_FEAT_CTL, msr); 1533d70e28f5SLen Brown 1534d70e28f5SLen Brown /* if SGX is enabled */ 1535d70e28f5SLen Brown if (msr & (1 << 18)) 1536d70e28f5SLen Brown return; 1537d70e28f5SLen Brown } 1538d70e28f5SLen Brown 1539ba1e78a1SRafael J. Wysocki skl_cstates[5].flags |= CPUIDLE_FLAG_UNUSABLE; /* C8-SKL */ 1540ba1e78a1SRafael J. Wysocki skl_cstates[6].flags |= CPUIDLE_FLAG_UNUSABLE; /* C9-SKL */ 1541d70e28f5SLen Brown } 1542d70e28f5SLen Brown 154364233338SChen Yu /** 154464233338SChen Yu * skx_idle_state_table_update - Adjust the Sky Lake/Cascade Lake 154564233338SChen Yu * idle states table. 154664233338SChen Yu */ 154764233338SChen Yu static void __init skx_idle_state_table_update(void) 154864233338SChen Yu { 154964233338SChen Yu unsigned long long msr; 155064233338SChen Yu 155164233338SChen Yu rdmsrl(MSR_PKG_CST_CONFIG_CONTROL, msr); 155264233338SChen Yu 155364233338SChen Yu /* 155464233338SChen Yu * 000b: C0/C1 (no package C-state support) 155564233338SChen Yu * 001b: C2 155664233338SChen Yu * 010b: C6 (non-retention) 155764233338SChen Yu * 011b: C6 (retention) 155864233338SChen Yu * 111b: No Package C state limits. 155964233338SChen Yu */ 156064233338SChen Yu if ((msr & 0x7) < 2) { 156164233338SChen Yu /* 156264233338SChen Yu * Uses the CC6 + PC0 latency and 3 times of 156364233338SChen Yu * latency for target_residency if the PC6 156464233338SChen Yu * is disabled in BIOS. This is consistent 156564233338SChen Yu * with how intel_idle driver uses _CST 156664233338SChen Yu * to set the target_residency. 156764233338SChen Yu */ 156864233338SChen Yu skx_cstates[2].exit_latency = 92; 156964233338SChen Yu skx_cstates[2].target_residency = 276; 157064233338SChen Yu } 157164233338SChen Yu } 157264233338SChen Yu 1573da0e58c0SArtem Bityutskiy /** 1574da0e58c0SArtem Bityutskiy * spr_idle_state_table_update - Adjust Sapphire Rapids idle states table. 1575da0e58c0SArtem Bityutskiy */ 1576da0e58c0SArtem Bityutskiy static void __init spr_idle_state_table_update(void) 1577da0e58c0SArtem Bityutskiy { 15783a9cf77bSArtem Bityutskiy unsigned long long msr; 15793a9cf77bSArtem Bityutskiy 1580da0e58c0SArtem Bityutskiy /* Check if user prefers C1E over C1. */ 1581da0e58c0SArtem Bityutskiy if (preferred_states_mask & BIT(2)) { 1582da0e58c0SArtem Bityutskiy if (preferred_states_mask & BIT(1)) 1583da0e58c0SArtem Bityutskiy /* Both can't be enabled, stick to the defaults. */ 1584da0e58c0SArtem Bityutskiy return; 1585da0e58c0SArtem Bityutskiy 1586da0e58c0SArtem Bityutskiy spr_cstates[0].flags |= CPUIDLE_FLAG_UNUSABLE; 1587da0e58c0SArtem Bityutskiy spr_cstates[1].flags &= ~CPUIDLE_FLAG_UNUSABLE; 1588da0e58c0SArtem Bityutskiy 1589da0e58c0SArtem Bityutskiy /* Enable C1E using the "C1E promotion" bit. */ 1590da0e58c0SArtem Bityutskiy c1e_promotion_enable(); 1591da0e58c0SArtem Bityutskiy disable_promotion_to_c1e = false; 1592da0e58c0SArtem Bityutskiy } 15933a9cf77bSArtem Bityutskiy 15943a9cf77bSArtem Bityutskiy /* 15953a9cf77bSArtem Bityutskiy * By default, the C6 state assumes the worst-case scenario of package 15963a9cf77bSArtem Bityutskiy * C6. However, if PC6 is disabled, we update the numbers to match 15973a9cf77bSArtem Bityutskiy * core C6. 15983a9cf77bSArtem Bityutskiy */ 15993a9cf77bSArtem Bityutskiy rdmsrl(MSR_PKG_CST_CONFIG_CONTROL, msr); 16003a9cf77bSArtem Bityutskiy 16013a9cf77bSArtem Bityutskiy /* Limit value 2 and above allow for PC6. */ 16023a9cf77bSArtem Bityutskiy if ((msr & 0x7) < 2) { 16033a9cf77bSArtem Bityutskiy spr_cstates[2].exit_latency = 190; 16043a9cf77bSArtem Bityutskiy spr_cstates[2].target_residency = 600; 16053a9cf77bSArtem Bityutskiy } 1606da0e58c0SArtem Bityutskiy } 1607da0e58c0SArtem Bityutskiy 16081aefbd7aSRafael J. Wysocki static bool __init intel_idle_verify_cstate(unsigned int mwait_hint) 16091aefbd7aSRafael J. Wysocki { 16101aefbd7aSRafael J. Wysocki unsigned int mwait_cstate = MWAIT_HINT2CSTATE(mwait_hint) + 1; 16111aefbd7aSRafael J. Wysocki unsigned int num_substates = (mwait_substates >> mwait_cstate * 4) & 16121aefbd7aSRafael J. Wysocki MWAIT_SUBSTATE_MASK; 16131aefbd7aSRafael J. Wysocki 16141aefbd7aSRafael J. Wysocki /* Ignore the C-state if there are NO sub-states in CPUID for it. */ 16151aefbd7aSRafael J. Wysocki if (num_substates == 0) 16161aefbd7aSRafael J. Wysocki return false; 16171aefbd7aSRafael J. Wysocki 16181aefbd7aSRafael J. Wysocki if (mwait_cstate > 2 && !boot_cpu_has(X86_FEATURE_NONSTOP_TSC)) 16191aefbd7aSRafael J. Wysocki mark_tsc_unstable("TSC halts in idle states deeper than C2"); 16201aefbd7aSRafael J. Wysocki 16211aefbd7aSRafael J. Wysocki return true; 16221aefbd7aSRafael J. Wysocki } 16231aefbd7aSRafael J. Wysocki 1624095928aeSRafael J. Wysocki static void __init intel_idle_init_cstates_icpu(struct cpuidle_driver *drv) 1625d70e28f5SLen Brown { 16263d3a1ae9SRafael J. Wysocki int cstate; 1627d70e28f5SLen Brown 16283d3a1ae9SRafael J. Wysocki switch (boot_cpu_data.x86_model) { 1629db73c5a8SDave Hansen case INTEL_FAM6_IVYBRIDGE_X: 1630d70e28f5SLen Brown ivt_idle_state_table_update(); 1631d70e28f5SLen Brown break; 1632db73c5a8SDave Hansen case INTEL_FAM6_ATOM_GOLDMONT: 1633f2c4db1bSPeter Zijlstra case INTEL_FAM6_ATOM_GOLDMONT_PLUS: 16345dcef694SLen Brown bxt_idle_state_table_update(); 16355dcef694SLen Brown break; 1636c66f78a6SPeter Zijlstra case INTEL_FAM6_SKYLAKE: 1637d70e28f5SLen Brown sklh_idle_state_table_update(); 1638d70e28f5SLen Brown break; 163964233338SChen Yu case INTEL_FAM6_SKYLAKE_X: 164064233338SChen Yu skx_idle_state_table_update(); 164164233338SChen Yu break; 1642da0e58c0SArtem Bityutskiy case INTEL_FAM6_SAPPHIRERAPIDS_X: 1643da0e58c0SArtem Bityutskiy spr_idle_state_table_update(); 1644da0e58c0SArtem Bityutskiy break; 1645d70e28f5SLen Brown } 164646bcfad7SDeepthi Dharwar 1647e022e7ebSLen Brown for (cstate = 0; cstate < CPUIDLE_STATE_MAX; ++cstate) { 16489f3d6dafSRafael J. Wysocki unsigned int mwait_hint; 164946bcfad7SDeepthi Dharwar 165018734958SRafael J. Wysocki if (intel_idle_max_cstate_reached(cstate)) 165118734958SRafael J. Wysocki break; 165218734958SRafael J. Wysocki 16539f3d6dafSRafael J. Wysocki if (!cpuidle_state_table[cstate].enter && 16549f3d6dafSRafael J. Wysocki !cpuidle_state_table[cstate].enter_s2idle) 1655e022e7ebSLen Brown break; 1656e022e7ebSLen Brown 16579f3d6dafSRafael J. Wysocki /* If marked as unusable, skip this state. */ 1658ba1e78a1SRafael J. Wysocki if (cpuidle_state_table[cstate].flags & CPUIDLE_FLAG_UNUSABLE) { 1659654d08a4SJoe Perches pr_debug("state %s is disabled\n", 1660d70e28f5SLen Brown cpuidle_state_table[cstate].name); 1661d70e28f5SLen Brown continue; 1662d70e28f5SLen Brown } 1663d70e28f5SLen Brown 16649f3d6dafSRafael J. Wysocki mwait_hint = flg2MWAIT(cpuidle_state_table[cstate].flags); 16659f3d6dafSRafael J. Wysocki if (!intel_idle_verify_cstate(mwait_hint)) 16669f3d6dafSRafael J. Wysocki continue; 1667d70e28f5SLen Brown 16689f3d6dafSRafael J. Wysocki /* Structure copy. */ 1669bff8e60aSRafael J. Wysocki drv->states[drv->state_count] = cpuidle_state_table[cstate]; 1670bff8e60aSRafael J. Wysocki 16714dcb78eeSRafael J. Wysocki if ((disabled_states_mask & BIT(drv->state_count)) || 16724dcb78eeSRafael J. Wysocki ((icpu->use_acpi || force_use_acpi) && 16733a5be9b8SRafael J. Wysocki intel_idle_off_by_default(mwait_hint) && 16744dcb78eeSRafael J. Wysocki !(cpuidle_state_table[cstate].flags & CPUIDLE_FLAG_ALWAYS_ENABLE))) 1675bff8e60aSRafael J. Wysocki drv->states[drv->state_count].flags |= CPUIDLE_FLAG_OFF; 1676bff8e60aSRafael J. Wysocki 16776e1d2bc6SPeter Zijlstra if (intel_idle_state_needs_timer_stop(&drv->states[drv->state_count])) 16786e1d2bc6SPeter Zijlstra drv->states[drv->state_count].flags |= CPUIDLE_FLAG_TIMER_STOP; 16796e1d2bc6SPeter Zijlstra 1680bff8e60aSRafael J. Wysocki drv->state_count++; 168146bcfad7SDeepthi Dharwar } 168246bcfad7SDeepthi Dharwar 16838c058d53SLen Brown if (icpu->byt_auto_demotion_disable_flag) { 16848c058d53SLen Brown wrmsrl(MSR_CC6_DEMOTION_POLICY_CONFIG, 0); 16858c058d53SLen Brown wrmsrl(MSR_MC6_DEMOTION_POLICY_CONFIG, 0); 16868c058d53SLen Brown } 168746bcfad7SDeepthi Dharwar } 168846bcfad7SDeepthi Dharwar 16896eacb15fSRafael J. Wysocki /** 16906eacb15fSRafael J. Wysocki * intel_idle_cpuidle_driver_init - Create the list of available idle states. 16916eacb15fSRafael J. Wysocki * @drv: cpuidle driver structure to initialize. 169218734958SRafael J. Wysocki */ 16933d3a1ae9SRafael J. Wysocki static void __init intel_idle_cpuidle_driver_init(struct cpuidle_driver *drv) 169418734958SRafael J. Wysocki { 169518734958SRafael J. Wysocki cpuidle_poll_state_init(drv); 16964dcb78eeSRafael J. Wysocki 16974dcb78eeSRafael J. Wysocki if (disabled_states_mask & BIT(0)) 16984dcb78eeSRafael J. Wysocki drv->states[0].flags |= CPUIDLE_FLAG_OFF; 16994dcb78eeSRafael J. Wysocki 170018734958SRafael J. Wysocki drv->state_count = 1; 170118734958SRafael J. Wysocki 170218734958SRafael J. Wysocki if (icpu) 170318734958SRafael J. Wysocki intel_idle_init_cstates_icpu(drv); 170418734958SRafael J. Wysocki else 170518734958SRafael J. Wysocki intel_idle_init_cstates_acpi(drv); 170618734958SRafael J. Wysocki } 170746bcfad7SDeepthi Dharwar 17081aefbd7aSRafael J. Wysocki static void auto_demotion_disable(void) 17091aefbd7aSRafael J. Wysocki { 17101aefbd7aSRafael J. Wysocki unsigned long long msr_bits; 17111aefbd7aSRafael J. Wysocki 17121aefbd7aSRafael J. Wysocki rdmsrl(MSR_PKG_CST_CONFIG_CONTROL, msr_bits); 17137f843dd7SRafael J. Wysocki msr_bits &= ~auto_demotion_disable_flags; 17141aefbd7aSRafael J. Wysocki wrmsrl(MSR_PKG_CST_CONFIG_CONTROL, msr_bits); 17151aefbd7aSRafael J. Wysocki } 17161aefbd7aSRafael J. Wysocki 1717da0e58c0SArtem Bityutskiy static void c1e_promotion_enable(void) 1718da0e58c0SArtem Bityutskiy { 1719da0e58c0SArtem Bityutskiy unsigned long long msr_bits; 1720da0e58c0SArtem Bityutskiy 1721da0e58c0SArtem Bityutskiy rdmsrl(MSR_IA32_POWER_CTL, msr_bits); 1722da0e58c0SArtem Bityutskiy msr_bits |= 0x2; 1723da0e58c0SArtem Bityutskiy wrmsrl(MSR_IA32_POWER_CTL, msr_bits); 1724da0e58c0SArtem Bityutskiy } 1725da0e58c0SArtem Bityutskiy 17261aefbd7aSRafael J. Wysocki static void c1e_promotion_disable(void) 17271aefbd7aSRafael J. Wysocki { 17281aefbd7aSRafael J. Wysocki unsigned long long msr_bits; 17291aefbd7aSRafael J. Wysocki 17301aefbd7aSRafael J. Wysocki rdmsrl(MSR_IA32_POWER_CTL, msr_bits); 17311aefbd7aSRafael J. Wysocki msr_bits &= ~0x2; 17321aefbd7aSRafael J. Wysocki wrmsrl(MSR_IA32_POWER_CTL, msr_bits); 17331aefbd7aSRafael J. Wysocki } 17341aefbd7aSRafael J. Wysocki 17356eacb15fSRafael J. Wysocki /** 17366eacb15fSRafael J. Wysocki * intel_idle_cpu_init - Register the target CPU with the cpuidle core. 17376eacb15fSRafael J. Wysocki * @cpu: CPU to initialize. 17386eacb15fSRafael J. Wysocki * 17396eacb15fSRafael J. Wysocki * Register a cpuidle device object for @cpu and update its MSRs in accordance 17406eacb15fSRafael J. Wysocki * with the processor model flags. 174126717172SLen Brown */ 1742fb1013a0SSebastian Andrzej Siewior static int intel_idle_cpu_init(unsigned int cpu) 174326717172SLen Brown { 174426717172SLen Brown struct cpuidle_device *dev; 174526717172SLen Brown 174665b7f839SThomas Renninger dev = per_cpu_ptr(intel_idle_cpuidle_devices, cpu); 174765b7f839SThomas Renninger dev->cpu = cpu; 174826717172SLen Brown 174926717172SLen Brown if (cpuidle_register_device(dev)) { 1750654d08a4SJoe Perches pr_debug("cpuidle_register_device %d failed!\n", cpu); 175126717172SLen Brown return -EIO; 175226717172SLen Brown } 175365b7f839SThomas Renninger 17547f843dd7SRafael J. Wysocki if (auto_demotion_disable_flags) 1755fb1013a0SSebastian Andrzej Siewior auto_demotion_disable(); 175626717172SLen Brown 17577f843dd7SRafael J. Wysocki if (disable_promotion_to_c1e) 1758fb1013a0SSebastian Andrzej Siewior c1e_promotion_disable(); 1759fb1013a0SSebastian Andrzej Siewior 1760fb1013a0SSebastian Andrzej Siewior return 0; 1761fb1013a0SSebastian Andrzej Siewior } 1762fb1013a0SSebastian Andrzej Siewior 1763fb1013a0SSebastian Andrzej Siewior static int intel_idle_cpu_online(unsigned int cpu) 1764fb1013a0SSebastian Andrzej Siewior { 1765fb1013a0SSebastian Andrzej Siewior struct cpuidle_device *dev; 1766fb1013a0SSebastian Andrzej Siewior 1767dab20177SRafael J. Wysocki if (!boot_cpu_has(X86_FEATURE_ARAT)) 1768cbd2c4c2SRafael J. Wysocki tick_broadcast_enable(); 1769fb1013a0SSebastian Andrzej Siewior 1770fb1013a0SSebastian Andrzej Siewior /* 1771fb1013a0SSebastian Andrzej Siewior * Some systems can hotplug a cpu at runtime after 1772fb1013a0SSebastian Andrzej Siewior * the kernel has booted, we have to initialize the 1773fb1013a0SSebastian Andrzej Siewior * driver in this case 1774fb1013a0SSebastian Andrzej Siewior */ 1775fb1013a0SSebastian Andrzej Siewior dev = per_cpu_ptr(intel_idle_cpuidle_devices, cpu); 1776fb1013a0SSebastian Andrzej Siewior if (!dev->registered) 1777fb1013a0SSebastian Andrzej Siewior return intel_idle_cpu_init(cpu); 1778dbf87ab8SBartlomiej Zolnierkiewicz 177926717172SLen Brown return 0; 178026717172SLen Brown } 178126717172SLen Brown 17820755a9bdSRafael J. Wysocki /** 17830755a9bdSRafael J. Wysocki * intel_idle_cpuidle_devices_uninit - Unregister all cpuidle devices. 17840755a9bdSRafael J. Wysocki */ 17850755a9bdSRafael J. Wysocki static void __init intel_idle_cpuidle_devices_uninit(void) 17860755a9bdSRafael J. Wysocki { 17870755a9bdSRafael J. Wysocki int i; 17880755a9bdSRafael J. Wysocki 17890755a9bdSRafael J. Wysocki for_each_online_cpu(i) 17900755a9bdSRafael J. Wysocki cpuidle_unregister_device(per_cpu_ptr(intel_idle_cpuidle_devices, i)); 17910755a9bdSRafael J. Wysocki } 17920755a9bdSRafael J. Wysocki 179326717172SLen Brown static int __init intel_idle_init(void) 179426717172SLen Brown { 1795a6c86e33SRafael J. Wysocki const struct x86_cpu_id *id; 1796a6c86e33SRafael J. Wysocki unsigned int eax, ebx, ecx; 1797fb1013a0SSebastian Andrzej Siewior int retval; 179826717172SLen Brown 1799d1896049SThomas Renninger /* Do not load intel_idle at all for now if idle= is passed */ 1800d1896049SThomas Renninger if (boot_option_idle_override != IDLE_NO_OVERRIDE) 1801d1896049SThomas Renninger return -ENODEV; 1802d1896049SThomas Renninger 1803a6c86e33SRafael J. Wysocki if (max_cstate == 0) { 1804a6c86e33SRafael J. Wysocki pr_debug("disabled\n"); 1805a6c86e33SRafael J. Wysocki return -EPERM; 1806a6c86e33SRafael J. Wysocki } 1807a6c86e33SRafael J. Wysocki 1808a6c86e33SRafael J. Wysocki id = x86_match_cpu(intel_idle_ids); 1809a6c86e33SRafael J. Wysocki if (id) { 1810a6c86e33SRafael J. Wysocki if (!boot_cpu_has(X86_FEATURE_MWAIT)) { 1811a6c86e33SRafael J. Wysocki pr_debug("Please enable MWAIT in BIOS SETUP\n"); 1812a6c86e33SRafael J. Wysocki return -ENODEV; 1813a6c86e33SRafael J. Wysocki } 1814a6c86e33SRafael J. Wysocki } else { 1815a6c86e33SRafael J. Wysocki id = x86_match_cpu(intel_mwait_ids); 1816a6c86e33SRafael J. Wysocki if (!id) 1817a6c86e33SRafael J. Wysocki return -ENODEV; 1818a6c86e33SRafael J. Wysocki } 1819a6c86e33SRafael J. Wysocki 1820a6c86e33SRafael J. Wysocki if (boot_cpu_data.cpuid_level < CPUID_MWAIT_LEAF) 1821a6c86e33SRafael J. Wysocki return -ENODEV; 1822a6c86e33SRafael J. Wysocki 1823a6c86e33SRafael J. Wysocki cpuid(CPUID_MWAIT_LEAF, &eax, &ebx, &ecx, &mwait_substates); 1824a6c86e33SRafael J. Wysocki 1825a6c86e33SRafael J. Wysocki if (!(ecx & CPUID5_ECX_EXTENSIONS_SUPPORTED) || 1826a6c86e33SRafael J. Wysocki !(ecx & CPUID5_ECX_INTERRUPT_BREAK) || 1827a6c86e33SRafael J. Wysocki !mwait_substates) 1828a6c86e33SRafael J. Wysocki return -ENODEV; 1829a6c86e33SRafael J. Wysocki 1830a6c86e33SRafael J. Wysocki pr_debug("MWAIT substates: 0x%x\n", mwait_substates); 1831a6c86e33SRafael J. Wysocki 1832a6c86e33SRafael J. Wysocki icpu = (const struct idle_cpu *)id->driver_data; 1833a6c86e33SRafael J. Wysocki if (icpu) { 1834a6c86e33SRafael J. Wysocki cpuidle_state_table = icpu->state_table; 18357f843dd7SRafael J. Wysocki auto_demotion_disable_flags = icpu->auto_demotion_disable_flags; 18367f843dd7SRafael J. Wysocki disable_promotion_to_c1e = icpu->disable_promotion_to_c1e; 18373a5be9b8SRafael J. Wysocki if (icpu->use_acpi || force_use_acpi) 1838a6c86e33SRafael J. Wysocki intel_idle_acpi_cst_extract(); 1839a6c86e33SRafael J. Wysocki } else if (!intel_idle_acpi_cst_extract()) { 1840a6c86e33SRafael J. Wysocki return -ENODEV; 1841a6c86e33SRafael J. Wysocki } 1842a6c86e33SRafael J. Wysocki 1843a6c86e33SRafael J. Wysocki pr_debug("v" INTEL_IDLE_VERSION " model 0x%X\n", 1844a6c86e33SRafael J. Wysocki boot_cpu_data.x86_model); 184526717172SLen Brown 1846e9df69ccSRichard Cochran intel_idle_cpuidle_devices = alloc_percpu(struct cpuidle_device); 1847533da74aSRafael J. Wysocki if (!intel_idle_cpuidle_devices) 1848e9df69ccSRichard Cochran return -ENOMEM; 1849e9df69ccSRichard Cochran 18503d3a1ae9SRafael J. Wysocki intel_idle_cpuidle_driver_init(&intel_idle_driver); 18513d3a1ae9SRafael J. Wysocki 185226717172SLen Brown retval = cpuidle_register_driver(&intel_idle_driver); 185326717172SLen Brown if (retval) { 18543735d524SKonrad Rzeszutek Wilk struct cpuidle_driver *drv = cpuidle_get_driver(); 1855654d08a4SJoe Perches printk(KERN_DEBUG pr_fmt("intel_idle yielding to %s\n"), 18563735d524SKonrad Rzeszutek Wilk drv ? drv->name : "none"); 1857fb1013a0SSebastian Andrzej Siewior goto init_driver_fail; 185826717172SLen Brown } 185926717172SLen Brown 1860fb1013a0SSebastian Andrzej Siewior retval = cpuhp_setup_state(CPUHP_AP_ONLINE_DYN, "idle/intel:online", 1861fb1013a0SSebastian Andrzej Siewior intel_idle_cpu_online, NULL); 1862fb1013a0SSebastian Andrzej Siewior if (retval < 0) 1863fb1013a0SSebastian Andrzej Siewior goto hp_setup_fail; 186426717172SLen Brown 186540ab82e0SRafael J. Wysocki pr_debug("Local APIC timer is reliable in %s\n", 1866dab20177SRafael J. Wysocki boot_cpu_has(X86_FEATURE_ARAT) ? "all C-states" : "C1"); 18672259a819SRichard Cochran 186826717172SLen Brown return 0; 1869fb1013a0SSebastian Andrzej Siewior 1870fb1013a0SSebastian Andrzej Siewior hp_setup_fail: 1871fb1013a0SSebastian Andrzej Siewior intel_idle_cpuidle_devices_uninit(); 1872fb1013a0SSebastian Andrzej Siewior cpuidle_unregister_driver(&intel_idle_driver); 1873fb1013a0SSebastian Andrzej Siewior init_driver_fail: 1874fb1013a0SSebastian Andrzej Siewior free_percpu(intel_idle_cpuidle_devices); 1875fb1013a0SSebastian Andrzej Siewior return retval; 1876fb1013a0SSebastian Andrzej Siewior 187726717172SLen Brown } 187802c4fae9SPaul Gortmaker device_initcall(intel_idle_init); 187926717172SLen Brown 188002c4fae9SPaul Gortmaker /* 188102c4fae9SPaul Gortmaker * We are not really modular, but we used to support that. Meaning we also 188202c4fae9SPaul Gortmaker * support "intel_idle.max_cstate=..." at boot and also a read-only export of 188302c4fae9SPaul Gortmaker * it at /sys/module/intel_idle/parameters/max_cstate -- so using module_param 188402c4fae9SPaul Gortmaker * is the easiest way (currently) to continue doing that. 188502c4fae9SPaul Gortmaker */ 188626717172SLen Brown module_param(max_cstate, int, 0444); 18874dcb78eeSRafael J. Wysocki /* 18884dcb78eeSRafael J. Wysocki * The positions of the bits that are set in this number are the indices of the 18894dcb78eeSRafael J. Wysocki * idle states to be disabled by default (as reflected by the names of the 18904dcb78eeSRafael J. Wysocki * corresponding idle state directories in sysfs, "state0", "state1" ... 18914dcb78eeSRafael J. Wysocki * "state<i>" ..., where <i> is the index of the given state). 18924dcb78eeSRafael J. Wysocki */ 18934dcb78eeSRafael J. Wysocki module_param_named(states_off, disabled_states_mask, uint, 0444); 18944dcb78eeSRafael J. Wysocki MODULE_PARM_DESC(states_off, "Mask of disabled idle states"); 1895da0e58c0SArtem Bityutskiy /* 1896da0e58c0SArtem Bityutskiy * Some platforms come with mutually exclusive C-states, so that if one is 1897da0e58c0SArtem Bityutskiy * enabled, the other C-states must not be used. Example: C1 and C1E on 1898da0e58c0SArtem Bityutskiy * Sapphire Rapids platform. This parameter allows for selecting the 1899da0e58c0SArtem Bityutskiy * preferred C-states among the groups of mutually exclusive C-states - the 1900da0e58c0SArtem Bityutskiy * selected C-states will be registered, the other C-states from the mutually 1901da0e58c0SArtem Bityutskiy * exclusive group won't be registered. If the platform has no mutually 1902da0e58c0SArtem Bityutskiy * exclusive C-states, this parameter has no effect. 1903da0e58c0SArtem Bityutskiy */ 1904da0e58c0SArtem Bityutskiy module_param_named(preferred_cstates, preferred_states_mask, uint, 0444); 1905da0e58c0SArtem Bityutskiy MODULE_PARM_DESC(preferred_cstates, "Mask of preferred idle states"); 1906