xref: /openbmc/linux/drivers/idle/intel_idle.c (revision 0563bb7b) 
126717172SLen Brown /*
226717172SLen Brown  * intel_idle.c - native hardware idle loop for modern Intel processors
326717172SLen Brown  *
4fab04b22SLen Brown  * Copyright (c) 2013, Intel Corporation.
526717172SLen Brown  * Len Brown <len.brown@intel.com>
626717172SLen Brown  *
726717172SLen Brown  * This program is free software; you can redistribute it and/or modify it
826717172SLen Brown  * under the terms and conditions of the GNU General Public License,
926717172SLen Brown  * version 2, as published by the Free Software Foundation.
1026717172SLen Brown  *
1126717172SLen Brown  * This program is distributed in the hope it will be useful, but WITHOUT
1226717172SLen Brown  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
1326717172SLen Brown  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
1426717172SLen Brown  * more details.
1526717172SLen Brown  *
1626717172SLen Brown  * You should have received a copy of the GNU General Public License along with
1726717172SLen Brown  * this program; if not, write to the Free Software Foundation, Inc.,
1826717172SLen Brown  * 51 Franklin St - Fifth Floor, Boston, MA 02110-1301 USA.
1926717172SLen Brown  */
2026717172SLen Brown 
2126717172SLen Brown /*
2226717172SLen Brown  * intel_idle is a cpuidle driver that loads on specific Intel processors
2326717172SLen Brown  * in lieu of the legacy ACPI processor_idle driver.  The intent is to
2426717172SLen Brown  * make Linux more efficient on these processors, as intel_idle knows
2526717172SLen Brown  * more than ACPI, as well as make Linux more immune to ACPI BIOS bugs.
2626717172SLen Brown  */
2726717172SLen Brown 
2826717172SLen Brown /*
2926717172SLen Brown  * Design Assumptions
3026717172SLen Brown  *
3126717172SLen Brown  * All CPUs have same idle states as boot CPU
3226717172SLen Brown  *
3326717172SLen Brown  * Chipset BM_STS (bus master status) bit is a NOP
3426717172SLen Brown  *	for preventing entry into deep C-stats
3526717172SLen Brown  */
3626717172SLen Brown 
3726717172SLen Brown /*
3826717172SLen Brown  * Known limitations
3926717172SLen Brown  *
4026717172SLen Brown  * The driver currently initializes for_each_online_cpu() upon modprobe.
4126717172SLen Brown  * It it unaware of subsequent processors hot-added to the system.
4226717172SLen Brown  * This means that if you boot with maxcpus=n and later online
4326717172SLen Brown  * processors above n, those processors will use C1 only.
4426717172SLen Brown  *
4526717172SLen Brown  * ACPI has a .suspend hack to turn off deep c-statees during suspend
4626717172SLen Brown  * to avoid complications with the lapic timer workaround.
4726717172SLen Brown  * Have not seen issues with suspend, but may need same workaround here.
4826717172SLen Brown  *
4926717172SLen Brown  */
5026717172SLen Brown 
5126717172SLen Brown /* un-comment DEBUG to enable pr_debug() statements */
5226717172SLen Brown #define DEBUG
5326717172SLen Brown 
54654d08a4SJoe Perches #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
55654d08a4SJoe Perches 
5626717172SLen Brown #include <linux/kernel.h>
5726717172SLen Brown #include <linux/cpuidle.h>
5876962caaSThomas Gleixner #include <linux/tick.h>
5926717172SLen Brown #include <trace/events/power.h>
6026717172SLen Brown #include <linux/sched.h>
612a2d31c8SShaohua Li #include <linux/notifier.h>
622a2d31c8SShaohua Li #include <linux/cpu.h>
6302c4fae9SPaul Gortmaker #include <linux/moduleparam.h>
64b66b8b9aSAndi Kleen #include <asm/cpu_device_id.h>
65db73c5a8SDave Hansen #include <asm/intel-family.h>
66bc83ccccSH. Peter Anvin #include <asm/mwait.h>
6714796fcaSLen Brown #include <asm/msr.h>
6826717172SLen Brown 
69d70e28f5SLen Brown #define INTEL_IDLE_VERSION "0.4.1"
7026717172SLen Brown 
7126717172SLen Brown static struct cpuidle_driver intel_idle_driver = {
7226717172SLen Brown 	.name = "intel_idle",
7326717172SLen Brown 	.owner = THIS_MODULE,
7426717172SLen Brown };
7526717172SLen Brown /* intel_idle.max_cstate=0 disables driver */
76137ecc77SLen Brown static int max_cstate = CPUIDLE_STATE_MAX - 1;
7726717172SLen Brown 
78c4236282SLen Brown static unsigned int mwait_substates;
7926717172SLen Brown 
802a2d31c8SShaohua Li #define LAPIC_TIMER_ALWAYS_RELIABLE 0xFFFFFFFF
8126717172SLen Brown /* Reliable LAPIC Timer States, bit 1 for C1 etc.  */
82d13780d4SLen Brown static unsigned int lapic_timer_reliable_states = (1 << 1);	 /* Default to only C1 */
8326717172SLen Brown 
84b66b8b9aSAndi Kleen struct idle_cpu {
85b66b8b9aSAndi Kleen 	struct cpuidle_state *state_table;
8626717172SLen Brown 
8726717172SLen Brown 	/*
8814796fcaSLen Brown 	 * Hardware C-state auto-demotion may not always be optimal.
8914796fcaSLen Brown 	 * Indicate which enable bits to clear here.
9014796fcaSLen Brown 	 */
91b66b8b9aSAndi Kleen 	unsigned long auto_demotion_disable_flags;
928c058d53SLen Brown 	bool byt_auto_demotion_disable_flag;
9332e95180SLen Brown 	bool disable_promotion_to_c1e;
94b66b8b9aSAndi Kleen };
95b66b8b9aSAndi Kleen 
96b66b8b9aSAndi Kleen static const struct idle_cpu *icpu;
97b66b8b9aSAndi Kleen static struct cpuidle_device __percpu *intel_idle_cpuidle_devices;
98b66b8b9aSAndi Kleen static int intel_idle(struct cpuidle_device *dev,
99b66b8b9aSAndi Kleen 			struct cpuidle_driver *drv, int index);
10028ba086eSRafael J. Wysocki static void intel_idle_s2idle(struct cpuidle_device *dev,
1015fe2e527SRafael J. Wysocki 			      struct cpuidle_driver *drv, int index);
102b66b8b9aSAndi Kleen static struct cpuidle_state *cpuidle_state_table;
10314796fcaSLen Brown 
10414796fcaSLen Brown /*
105956d033fSLen Brown  * Set this flag for states where the HW flushes the TLB for us
106956d033fSLen Brown  * and so we don't need cross-calls to keep it consistent.
107956d033fSLen Brown  * If this flag is set, SW flushes the TLB, so even if the
108956d033fSLen Brown  * HW doesn't do the flushing, this flag is safe to use.
109956d033fSLen Brown  */
110956d033fSLen Brown #define CPUIDLE_FLAG_TLB_FLUSHED	0x10000
111956d033fSLen Brown 
112956d033fSLen Brown /*
113b1beab48SLen Brown  * MWAIT takes an 8-bit "hint" in EAX "suggesting"
114b1beab48SLen Brown  * the C-state (top nibble) and sub-state (bottom nibble)
115b1beab48SLen Brown  * 0x00 means "MWAIT(C1)", 0x10 means "MWAIT(C2)" etc.
116b1beab48SLen Brown  *
117b1beab48SLen Brown  * We store the hint at the top of our "flags" for each state.
118b1beab48SLen Brown  */
119b1beab48SLen Brown #define flg2MWAIT(flags) (((flags) >> 24) & 0xFF)
120b1beab48SLen Brown #define MWAIT2flg(eax) ((eax & 0xFF) << 24)
121b1beab48SLen Brown 
122b1beab48SLen Brown /*
12326717172SLen Brown  * States are indexed by the cstate number,
12426717172SLen Brown  * which is also the index into the MWAIT hint array.
12526717172SLen Brown  * Thus C0 is a dummy.
12626717172SLen Brown  */
127ba0dc81eSJiang Liu static struct cpuidle_state nehalem_cstates[] = {
128e022e7ebSLen Brown 	{
129de09cdd0SLen Brown 		.name = "C1",
13026717172SLen Brown 		.desc = "MWAIT 0x00",
131b82b6ccaSDaniel Lezcano 		.flags = MWAIT2flg(0x00),
13226717172SLen Brown 		.exit_latency = 3,
13326717172SLen Brown 		.target_residency = 6,
1345fe2e527SRafael J. Wysocki 		.enter = &intel_idle,
13528ba086eSRafael J. Wysocki 		.enter_s2idle = intel_idle_s2idle, },
136e022e7ebSLen Brown 	{
137de09cdd0SLen Brown 		.name = "C1E",
13832e95180SLen Brown 		.desc = "MWAIT 0x01",
139b82b6ccaSDaniel Lezcano 		.flags = MWAIT2flg(0x01),
14032e95180SLen Brown 		.exit_latency = 10,
14132e95180SLen Brown 		.target_residency = 20,
1425fe2e527SRafael J. Wysocki 		.enter = &intel_idle,
14328ba086eSRafael J. Wysocki 		.enter_s2idle = intel_idle_s2idle, },
14432e95180SLen Brown 	{
145de09cdd0SLen Brown 		.name = "C3",
14626717172SLen Brown 		.desc = "MWAIT 0x10",
147b82b6ccaSDaniel Lezcano 		.flags = MWAIT2flg(0x10) | CPUIDLE_FLAG_TLB_FLUSHED,
14826717172SLen Brown 		.exit_latency = 20,
14926717172SLen Brown 		.target_residency = 80,
1505fe2e527SRafael J. Wysocki 		.enter = &intel_idle,
15128ba086eSRafael J. Wysocki 		.enter_s2idle = intel_idle_s2idle, },
152e022e7ebSLen Brown 	{
153de09cdd0SLen Brown 		.name = "C6",
15426717172SLen Brown 		.desc = "MWAIT 0x20",
155b82b6ccaSDaniel Lezcano 		.flags = MWAIT2flg(0x20) | CPUIDLE_FLAG_TLB_FLUSHED,
15626717172SLen Brown 		.exit_latency = 200,
15726717172SLen Brown 		.target_residency = 800,
1585fe2e527SRafael J. Wysocki 		.enter = &intel_idle,
15928ba086eSRafael J. Wysocki 		.enter_s2idle = intel_idle_s2idle, },
160e022e7ebSLen Brown 	{
161e022e7ebSLen Brown 		.enter = NULL }
16226717172SLen Brown };
16326717172SLen Brown 
164ba0dc81eSJiang Liu static struct cpuidle_state snb_cstates[] = {
165e022e7ebSLen Brown 	{
166de09cdd0SLen Brown 		.name = "C1",
167d13780d4SLen Brown 		.desc = "MWAIT 0x00",
168b82b6ccaSDaniel Lezcano 		.flags = MWAIT2flg(0x00),
16932e95180SLen Brown 		.exit_latency = 2,
17032e95180SLen Brown 		.target_residency = 2,
1715fe2e527SRafael J. Wysocki 		.enter = &intel_idle,
17228ba086eSRafael J. Wysocki 		.enter_s2idle = intel_idle_s2idle, },
17332e95180SLen Brown 	{
174de09cdd0SLen Brown 		.name = "C1E",
17532e95180SLen Brown 		.desc = "MWAIT 0x01",
176b82b6ccaSDaniel Lezcano 		.flags = MWAIT2flg(0x01),
17732e95180SLen Brown 		.exit_latency = 10,
17832e95180SLen Brown 		.target_residency = 20,
1795fe2e527SRafael J. Wysocki 		.enter = &intel_idle,
18028ba086eSRafael J. Wysocki 		.enter_s2idle = intel_idle_s2idle, },
181e022e7ebSLen Brown 	{
182de09cdd0SLen Brown 		.name = "C3",
183d13780d4SLen Brown 		.desc = "MWAIT 0x10",
184b82b6ccaSDaniel Lezcano 		.flags = MWAIT2flg(0x10) | CPUIDLE_FLAG_TLB_FLUSHED,
185d13780d4SLen Brown 		.exit_latency = 80,
186ddbd550dSLen Brown 		.target_residency = 211,
1875fe2e527SRafael J. Wysocki 		.enter = &intel_idle,
18828ba086eSRafael J. Wysocki 		.enter_s2idle = intel_idle_s2idle, },
189e022e7ebSLen Brown 	{
190de09cdd0SLen Brown 		.name = "C6",
191d13780d4SLen Brown 		.desc = "MWAIT 0x20",
192b82b6ccaSDaniel Lezcano 		.flags = MWAIT2flg(0x20) | CPUIDLE_FLAG_TLB_FLUSHED,
193d13780d4SLen Brown 		.exit_latency = 104,
194ddbd550dSLen Brown 		.target_residency = 345,
1955fe2e527SRafael J. Wysocki 		.enter = &intel_idle,
19628ba086eSRafael J. Wysocki 		.enter_s2idle = intel_idle_s2idle, },
197e022e7ebSLen Brown 	{
198de09cdd0SLen Brown 		.name = "C7",
199d13780d4SLen Brown 		.desc = "MWAIT 0x30",
200b82b6ccaSDaniel Lezcano 		.flags = MWAIT2flg(0x30) | CPUIDLE_FLAG_TLB_FLUSHED,
201d13780d4SLen Brown 		.exit_latency = 109,
202ddbd550dSLen Brown 		.target_residency = 345,
2035fe2e527SRafael J. Wysocki 		.enter = &intel_idle,
20428ba086eSRafael J. Wysocki 		.enter_s2idle = intel_idle_s2idle, },
205e022e7ebSLen Brown 	{
206e022e7ebSLen Brown 		.enter = NULL }
207d13780d4SLen Brown };
208d13780d4SLen Brown 
209718987d6SLen Brown static struct cpuidle_state byt_cstates[] = {
210718987d6SLen Brown 	{
211de09cdd0SLen Brown 		.name = "C1",
212718987d6SLen Brown 		.desc = "MWAIT 0x00",
213b82b6ccaSDaniel Lezcano 		.flags = MWAIT2flg(0x00),
214718987d6SLen Brown 		.exit_latency = 1,
215718987d6SLen Brown 		.target_residency = 1,
2165fe2e527SRafael J. Wysocki 		.enter = &intel_idle,
21728ba086eSRafael J. Wysocki 		.enter_s2idle = intel_idle_s2idle, },
218718987d6SLen Brown 	{
219de09cdd0SLen Brown 		.name = "C6N",
220718987d6SLen Brown 		.desc = "MWAIT 0x58",
221b82b6ccaSDaniel Lezcano 		.flags = MWAIT2flg(0x58) | CPUIDLE_FLAG_TLB_FLUSHED,
222d7ef7671SLen Brown 		.exit_latency = 300,
223718987d6SLen Brown 		.target_residency = 275,
2245fe2e527SRafael J. Wysocki 		.enter = &intel_idle,
22528ba086eSRafael J. Wysocki 		.enter_s2idle = intel_idle_s2idle, },
226718987d6SLen Brown 	{
227de09cdd0SLen Brown 		.name = "C6S",
228718987d6SLen Brown 		.desc = "MWAIT 0x52",
229b82b6ccaSDaniel Lezcano 		.flags = MWAIT2flg(0x52) | CPUIDLE_FLAG_TLB_FLUSHED,
230d7ef7671SLen Brown 		.exit_latency = 500,
231718987d6SLen Brown 		.target_residency = 560,
2325fe2e527SRafael J. Wysocki 		.enter = &intel_idle,
23328ba086eSRafael J. Wysocki 		.enter_s2idle = intel_idle_s2idle, },
234718987d6SLen Brown 	{
235de09cdd0SLen Brown 		.name = "C7",
236718987d6SLen Brown 		.desc = "MWAIT 0x60",
237b82b6ccaSDaniel Lezcano 		.flags = MWAIT2flg(0x60) | CPUIDLE_FLAG_TLB_FLUSHED,
238718987d6SLen Brown 		.exit_latency = 1200,
239d7ef7671SLen Brown 		.target_residency = 4000,
2405fe2e527SRafael J. Wysocki 		.enter = &intel_idle,
24128ba086eSRafael J. Wysocki 		.enter_s2idle = intel_idle_s2idle, },
242718987d6SLen Brown 	{
243de09cdd0SLen Brown 		.name = "C7S",
244718987d6SLen Brown 		.desc = "MWAIT 0x64",
245b82b6ccaSDaniel Lezcano 		.flags = MWAIT2flg(0x64) | CPUIDLE_FLAG_TLB_FLUSHED,
246718987d6SLen Brown 		.exit_latency = 10000,
247718987d6SLen Brown 		.target_residency = 20000,
2485fe2e527SRafael J. Wysocki 		.enter = &intel_idle,
24928ba086eSRafael J. Wysocki 		.enter_s2idle = intel_idle_s2idle, },
250718987d6SLen Brown 	{
251718987d6SLen Brown 		.enter = NULL }
252718987d6SLen Brown };
253718987d6SLen Brown 
254cab07a56SLen Brown static struct cpuidle_state cht_cstates[] = {
255cab07a56SLen Brown 	{
256de09cdd0SLen Brown 		.name = "C1",
257cab07a56SLen Brown 		.desc = "MWAIT 0x00",
258cab07a56SLen Brown 		.flags = MWAIT2flg(0x00),
259cab07a56SLen Brown 		.exit_latency = 1,
260cab07a56SLen Brown 		.target_residency = 1,
261cab07a56SLen Brown 		.enter = &intel_idle,
26228ba086eSRafael J. Wysocki 		.enter_s2idle = intel_idle_s2idle, },
263cab07a56SLen Brown 	{
264de09cdd0SLen Brown 		.name = "C6N",
265cab07a56SLen Brown 		.desc = "MWAIT 0x58",
266cab07a56SLen Brown 		.flags = MWAIT2flg(0x58) | CPUIDLE_FLAG_TLB_FLUSHED,
267cab07a56SLen Brown 		.exit_latency = 80,
268cab07a56SLen Brown 		.target_residency = 275,
269cab07a56SLen Brown 		.enter = &intel_idle,
27028ba086eSRafael J. Wysocki 		.enter_s2idle = intel_idle_s2idle, },
271cab07a56SLen Brown 	{
272de09cdd0SLen Brown 		.name = "C6S",
273cab07a56SLen Brown 		.desc = "MWAIT 0x52",
274cab07a56SLen Brown 		.flags = MWAIT2flg(0x52) | CPUIDLE_FLAG_TLB_FLUSHED,
275cab07a56SLen Brown 		.exit_latency = 200,
276cab07a56SLen Brown 		.target_residency = 560,
277cab07a56SLen Brown 		.enter = &intel_idle,
27828ba086eSRafael J. Wysocki 		.enter_s2idle = intel_idle_s2idle, },
279cab07a56SLen Brown 	{
280de09cdd0SLen Brown 		.name = "C7",
281cab07a56SLen Brown 		.desc = "MWAIT 0x60",
282cab07a56SLen Brown 		.flags = MWAIT2flg(0x60) | CPUIDLE_FLAG_TLB_FLUSHED,
283cab07a56SLen Brown 		.exit_latency = 1200,
284cab07a56SLen Brown 		.target_residency = 4000,
285cab07a56SLen Brown 		.enter = &intel_idle,
28628ba086eSRafael J. Wysocki 		.enter_s2idle = intel_idle_s2idle, },
287cab07a56SLen Brown 	{
288de09cdd0SLen Brown 		.name = "C7S",
289cab07a56SLen Brown 		.desc = "MWAIT 0x64",
290cab07a56SLen Brown 		.flags = MWAIT2flg(0x64) | CPUIDLE_FLAG_TLB_FLUSHED,
291cab07a56SLen Brown 		.exit_latency = 10000,
292cab07a56SLen Brown 		.target_residency = 20000,
293cab07a56SLen Brown 		.enter = &intel_idle,
29428ba086eSRafael J. Wysocki 		.enter_s2idle = intel_idle_s2idle, },
295cab07a56SLen Brown 	{
296cab07a56SLen Brown 		.enter = NULL }
297cab07a56SLen Brown };
298cab07a56SLen Brown 
299ba0dc81eSJiang Liu static struct cpuidle_state ivb_cstates[] = {
300e022e7ebSLen Brown 	{
301de09cdd0SLen Brown 		.name = "C1",
3026edab08cSLen Brown 		.desc = "MWAIT 0x00",
303b82b6ccaSDaniel Lezcano 		.flags = MWAIT2flg(0x00),
3046edab08cSLen Brown 		.exit_latency = 1,
3056edab08cSLen Brown 		.target_residency = 1,
3065fe2e527SRafael J. Wysocki 		.enter = &intel_idle,
30728ba086eSRafael J. Wysocki 		.enter_s2idle = intel_idle_s2idle, },
308e022e7ebSLen Brown 	{
309de09cdd0SLen Brown 		.name = "C1E",
31032e95180SLen Brown 		.desc = "MWAIT 0x01",
311b82b6ccaSDaniel Lezcano 		.flags = MWAIT2flg(0x01),
31232e95180SLen Brown 		.exit_latency = 10,
31332e95180SLen Brown 		.target_residency = 20,
3145fe2e527SRafael J. Wysocki 		.enter = &intel_idle,
31528ba086eSRafael J. Wysocki 		.enter_s2idle = intel_idle_s2idle, },
31632e95180SLen Brown 	{
317de09cdd0SLen Brown 		.name = "C3",
3186edab08cSLen Brown 		.desc = "MWAIT 0x10",
319b82b6ccaSDaniel Lezcano 		.flags = MWAIT2flg(0x10) | CPUIDLE_FLAG_TLB_FLUSHED,
3206edab08cSLen Brown 		.exit_latency = 59,
3216edab08cSLen Brown 		.target_residency = 156,
3225fe2e527SRafael J. Wysocki 		.enter = &intel_idle,
32328ba086eSRafael J. Wysocki 		.enter_s2idle = intel_idle_s2idle, },
324e022e7ebSLen Brown 	{
325de09cdd0SLen Brown 		.name = "C6",
3266edab08cSLen Brown 		.desc = "MWAIT 0x20",
327b82b6ccaSDaniel Lezcano 		.flags = MWAIT2flg(0x20) | CPUIDLE_FLAG_TLB_FLUSHED,
3286edab08cSLen Brown 		.exit_latency = 80,
3296edab08cSLen Brown 		.target_residency = 300,
3305fe2e527SRafael J. Wysocki 		.enter = &intel_idle,
33128ba086eSRafael J. Wysocki 		.enter_s2idle = intel_idle_s2idle, },
332e022e7ebSLen Brown 	{
333de09cdd0SLen Brown 		.name = "C7",
3346edab08cSLen Brown 		.desc = "MWAIT 0x30",
335b82b6ccaSDaniel Lezcano 		.flags = MWAIT2flg(0x30) | CPUIDLE_FLAG_TLB_FLUSHED,
3366edab08cSLen Brown 		.exit_latency = 87,
3376edab08cSLen Brown 		.target_residency = 300,
3385fe2e527SRafael J. Wysocki 		.enter = &intel_idle,
33928ba086eSRafael J. Wysocki 		.enter_s2idle = intel_idle_s2idle, },
340e022e7ebSLen Brown 	{
341e022e7ebSLen Brown 		.enter = NULL }
3426edab08cSLen Brown };
3436edab08cSLen Brown 
3440138d8f0SLen Brown static struct cpuidle_state ivt_cstates[] = {
3450138d8f0SLen Brown 	{
346de09cdd0SLen Brown 		.name = "C1",
3470138d8f0SLen Brown 		.desc = "MWAIT 0x00",
348b82b6ccaSDaniel Lezcano 		.flags = MWAIT2flg(0x00),
3490138d8f0SLen Brown 		.exit_latency = 1,
3500138d8f0SLen Brown 		.target_residency = 1,
3515fe2e527SRafael J. Wysocki 		.enter = &intel_idle,
35228ba086eSRafael J. Wysocki 		.enter_s2idle = intel_idle_s2idle, },
3530138d8f0SLen Brown 	{
354de09cdd0SLen Brown 		.name = "C1E",
3550138d8f0SLen Brown 		.desc = "MWAIT 0x01",
356b82b6ccaSDaniel Lezcano 		.flags = MWAIT2flg(0x01),
3570138d8f0SLen Brown 		.exit_latency = 10,
3580138d8f0SLen Brown 		.target_residency = 80,
3595fe2e527SRafael J. Wysocki 		.enter = &intel_idle,
36028ba086eSRafael J. Wysocki 		.enter_s2idle = intel_idle_s2idle, },
3610138d8f0SLen Brown 	{
362de09cdd0SLen Brown 		.name = "C3",
3630138d8f0SLen Brown 		.desc = "MWAIT 0x10",
364b82b6ccaSDaniel Lezcano 		.flags = MWAIT2flg(0x10) | CPUIDLE_FLAG_TLB_FLUSHED,
3650138d8f0SLen Brown 		.exit_latency = 59,
3660138d8f0SLen Brown 		.target_residency = 156,
3675fe2e527SRafael J. Wysocki 		.enter = &intel_idle,
36828ba086eSRafael J. Wysocki 		.enter_s2idle = intel_idle_s2idle, },
3690138d8f0SLen Brown 	{
370de09cdd0SLen Brown 		.name = "C6",
3710138d8f0SLen Brown 		.desc = "MWAIT 0x20",
372b82b6ccaSDaniel Lezcano 		.flags = MWAIT2flg(0x20) | CPUIDLE_FLAG_TLB_FLUSHED,
3730138d8f0SLen Brown 		.exit_latency = 82,
3740138d8f0SLen Brown 		.target_residency = 300,
3755fe2e527SRafael J. Wysocki 		.enter = &intel_idle,
37628ba086eSRafael J. Wysocki 		.enter_s2idle = intel_idle_s2idle, },
3770138d8f0SLen Brown 	{
3780138d8f0SLen Brown 		.enter = NULL }
3790138d8f0SLen Brown };
3800138d8f0SLen Brown 
3810138d8f0SLen Brown static struct cpuidle_state ivt_cstates_4s[] = {
3820138d8f0SLen Brown 	{
383de09cdd0SLen Brown 		.name = "C1",
3840138d8f0SLen Brown 		.desc = "MWAIT 0x00",
385b82b6ccaSDaniel Lezcano 		.flags = MWAIT2flg(0x00),
3860138d8f0SLen Brown 		.exit_latency = 1,
3870138d8f0SLen Brown 		.target_residency = 1,
3885fe2e527SRafael J. Wysocki 		.enter = &intel_idle,
38928ba086eSRafael J. Wysocki 		.enter_s2idle = intel_idle_s2idle, },
3900138d8f0SLen Brown 	{
391de09cdd0SLen Brown 		.name = "C1E",
3920138d8f0SLen Brown 		.desc = "MWAIT 0x01",
393b82b6ccaSDaniel Lezcano 		.flags = MWAIT2flg(0x01),
3940138d8f0SLen Brown 		.exit_latency = 10,
3950138d8f0SLen Brown 		.target_residency = 250,
3965fe2e527SRafael J. Wysocki 		.enter = &intel_idle,
39728ba086eSRafael J. Wysocki 		.enter_s2idle = intel_idle_s2idle, },
3980138d8f0SLen Brown 	{
399de09cdd0SLen Brown 		.name = "C3",
4000138d8f0SLen Brown 		.desc = "MWAIT 0x10",
401b82b6ccaSDaniel Lezcano 		.flags = MWAIT2flg(0x10) | CPUIDLE_FLAG_TLB_FLUSHED,
4020138d8f0SLen Brown 		.exit_latency = 59,
4030138d8f0SLen Brown 		.target_residency = 300,
4045fe2e527SRafael J. Wysocki 		.enter = &intel_idle,
40528ba086eSRafael J. Wysocki 		.enter_s2idle = intel_idle_s2idle, },
4060138d8f0SLen Brown 	{
407de09cdd0SLen Brown 		.name = "C6",
4080138d8f0SLen Brown 		.desc = "MWAIT 0x20",
409b82b6ccaSDaniel Lezcano 		.flags = MWAIT2flg(0x20) | CPUIDLE_FLAG_TLB_FLUSHED,
4100138d8f0SLen Brown 		.exit_latency = 84,
4110138d8f0SLen Brown 		.target_residency = 400,
4125fe2e527SRafael J. Wysocki 		.enter = &intel_idle,
41328ba086eSRafael J. Wysocki 		.enter_s2idle = intel_idle_s2idle, },
4140138d8f0SLen Brown 	{
4150138d8f0SLen Brown 		.enter = NULL }
4160138d8f0SLen Brown };
4170138d8f0SLen Brown 
4180138d8f0SLen Brown static struct cpuidle_state ivt_cstates_8s[] = {
4190138d8f0SLen Brown 	{
420de09cdd0SLen Brown 		.name = "C1",
4210138d8f0SLen Brown 		.desc = "MWAIT 0x00",
422b82b6ccaSDaniel Lezcano 		.flags = MWAIT2flg(0x00),
4230138d8f0SLen Brown 		.exit_latency = 1,
4240138d8f0SLen Brown 		.target_residency = 1,
4255fe2e527SRafael J. Wysocki 		.enter = &intel_idle,
42628ba086eSRafael J. Wysocki 		.enter_s2idle = intel_idle_s2idle, },
4270138d8f0SLen Brown 	{
428de09cdd0SLen Brown 		.name = "C1E",
4290138d8f0SLen Brown 		.desc = "MWAIT 0x01",
430b82b6ccaSDaniel Lezcano 		.flags = MWAIT2flg(0x01),
4310138d8f0SLen Brown 		.exit_latency = 10,
4320138d8f0SLen Brown 		.target_residency = 500,
4335fe2e527SRafael J. Wysocki 		.enter = &intel_idle,
43428ba086eSRafael J. Wysocki 		.enter_s2idle = intel_idle_s2idle, },
4350138d8f0SLen Brown 	{
436de09cdd0SLen Brown 		.name = "C3",
4370138d8f0SLen Brown 		.desc = "MWAIT 0x10",
438b82b6ccaSDaniel Lezcano 		.flags = MWAIT2flg(0x10) | CPUIDLE_FLAG_TLB_FLUSHED,
4390138d8f0SLen Brown 		.exit_latency = 59,
4400138d8f0SLen Brown 		.target_residency = 600,
4415fe2e527SRafael J. Wysocki 		.enter = &intel_idle,
44228ba086eSRafael J. Wysocki 		.enter_s2idle = intel_idle_s2idle, },
4430138d8f0SLen Brown 	{
444de09cdd0SLen Brown 		.name = "C6",
4450138d8f0SLen Brown 		.desc = "MWAIT 0x20",
446b82b6ccaSDaniel Lezcano 		.flags = MWAIT2flg(0x20) | CPUIDLE_FLAG_TLB_FLUSHED,
4470138d8f0SLen Brown 		.exit_latency = 88,
4480138d8f0SLen Brown 		.target_residency = 700,
4495fe2e527SRafael J. Wysocki 		.enter = &intel_idle,
45028ba086eSRafael J. Wysocki 		.enter_s2idle = intel_idle_s2idle, },
4510138d8f0SLen Brown 	{
4520138d8f0SLen Brown 		.enter = NULL }
4530138d8f0SLen Brown };
4540138d8f0SLen Brown 
455ba0dc81eSJiang Liu static struct cpuidle_state hsw_cstates[] = {
456e022e7ebSLen Brown 	{
457de09cdd0SLen Brown 		.name = "C1",
45885a4d2d4SLen Brown 		.desc = "MWAIT 0x00",
459b82b6ccaSDaniel Lezcano 		.flags = MWAIT2flg(0x00),
46085a4d2d4SLen Brown 		.exit_latency = 2,
46185a4d2d4SLen Brown 		.target_residency = 2,
4625fe2e527SRafael J. Wysocki 		.enter = &intel_idle,
46328ba086eSRafael J. Wysocki 		.enter_s2idle = intel_idle_s2idle, },
464e022e7ebSLen Brown 	{
465de09cdd0SLen Brown 		.name = "C1E",
46632e95180SLen Brown 		.desc = "MWAIT 0x01",
467b82b6ccaSDaniel Lezcano 		.flags = MWAIT2flg(0x01),
46832e95180SLen Brown 		.exit_latency = 10,
46932e95180SLen Brown 		.target_residency = 20,
4705fe2e527SRafael J. Wysocki 		.enter = &intel_idle,
47128ba086eSRafael J. Wysocki 		.enter_s2idle = intel_idle_s2idle, },
47232e95180SLen Brown 	{
473de09cdd0SLen Brown 		.name = "C3",
47485a4d2d4SLen Brown 		.desc = "MWAIT 0x10",
475b82b6ccaSDaniel Lezcano 		.flags = MWAIT2flg(0x10) | CPUIDLE_FLAG_TLB_FLUSHED,
47685a4d2d4SLen Brown 		.exit_latency = 33,
47785a4d2d4SLen Brown 		.target_residency = 100,
4785fe2e527SRafael J. Wysocki 		.enter = &intel_idle,
47928ba086eSRafael J. Wysocki 		.enter_s2idle = intel_idle_s2idle, },
480e022e7ebSLen Brown 	{
481de09cdd0SLen Brown 		.name = "C6",
48285a4d2d4SLen Brown 		.desc = "MWAIT 0x20",
483b82b6ccaSDaniel Lezcano 		.flags = MWAIT2flg(0x20) | CPUIDLE_FLAG_TLB_FLUSHED,
48485a4d2d4SLen Brown 		.exit_latency = 133,
48585a4d2d4SLen Brown 		.target_residency = 400,
4865fe2e527SRafael J. Wysocki 		.enter = &intel_idle,
48728ba086eSRafael J. Wysocki 		.enter_s2idle = intel_idle_s2idle, },
488e022e7ebSLen Brown 	{
489de09cdd0SLen Brown 		.name = "C7s",
49085a4d2d4SLen Brown 		.desc = "MWAIT 0x32",
491b82b6ccaSDaniel Lezcano 		.flags = MWAIT2flg(0x32) | CPUIDLE_FLAG_TLB_FLUSHED,
49285a4d2d4SLen Brown 		.exit_latency = 166,
49385a4d2d4SLen Brown 		.target_residency = 500,
4945fe2e527SRafael J. Wysocki 		.enter = &intel_idle,
49528ba086eSRafael J. Wysocki 		.enter_s2idle = intel_idle_s2idle, },
496e022e7ebSLen Brown 	{
497de09cdd0SLen Brown 		.name = "C8",
49886239cebSLen Brown 		.desc = "MWAIT 0x40",
499b82b6ccaSDaniel Lezcano 		.flags = MWAIT2flg(0x40) | CPUIDLE_FLAG_TLB_FLUSHED,
50086239cebSLen Brown 		.exit_latency = 300,
50186239cebSLen Brown 		.target_residency = 900,
5025fe2e527SRafael J. Wysocki 		.enter = &intel_idle,
50328ba086eSRafael J. Wysocki 		.enter_s2idle = intel_idle_s2idle, },
50486239cebSLen Brown 	{
505de09cdd0SLen Brown 		.name = "C9",
50686239cebSLen Brown 		.desc = "MWAIT 0x50",
507b82b6ccaSDaniel Lezcano 		.flags = MWAIT2flg(0x50) | CPUIDLE_FLAG_TLB_FLUSHED,
50886239cebSLen Brown 		.exit_latency = 600,
50986239cebSLen Brown 		.target_residency = 1800,
5105fe2e527SRafael J. Wysocki 		.enter = &intel_idle,
51128ba086eSRafael J. Wysocki 		.enter_s2idle = intel_idle_s2idle, },
51286239cebSLen Brown 	{
513de09cdd0SLen Brown 		.name = "C10",
51486239cebSLen Brown 		.desc = "MWAIT 0x60",
515b82b6ccaSDaniel Lezcano 		.flags = MWAIT2flg(0x60) | CPUIDLE_FLAG_TLB_FLUSHED,
51686239cebSLen Brown 		.exit_latency = 2600,
51786239cebSLen Brown 		.target_residency = 7700,
5185fe2e527SRafael J. Wysocki 		.enter = &intel_idle,
51928ba086eSRafael J. Wysocki 		.enter_s2idle = intel_idle_s2idle, },
52086239cebSLen Brown 	{
521e022e7ebSLen Brown 		.enter = NULL }
52285a4d2d4SLen Brown };
523a138b568SLen Brown static struct cpuidle_state bdw_cstates[] = {
524a138b568SLen Brown 	{
525de09cdd0SLen Brown 		.name = "C1",
526a138b568SLen Brown 		.desc = "MWAIT 0x00",
527b82b6ccaSDaniel Lezcano 		.flags = MWAIT2flg(0x00),
528a138b568SLen Brown 		.exit_latency = 2,
529a138b568SLen Brown 		.target_residency = 2,
5305fe2e527SRafael J. Wysocki 		.enter = &intel_idle,
53128ba086eSRafael J. Wysocki 		.enter_s2idle = intel_idle_s2idle, },
532a138b568SLen Brown 	{
533de09cdd0SLen Brown 		.name = "C1E",
534a138b568SLen Brown 		.desc = "MWAIT 0x01",
535b82b6ccaSDaniel Lezcano 		.flags = MWAIT2flg(0x01),
536a138b568SLen Brown 		.exit_latency = 10,
537a138b568SLen Brown 		.target_residency = 20,
5385fe2e527SRafael J. Wysocki 		.enter = &intel_idle,
53928ba086eSRafael J. Wysocki 		.enter_s2idle = intel_idle_s2idle, },
540a138b568SLen Brown 	{
541de09cdd0SLen Brown 		.name = "C3",
542a138b568SLen Brown 		.desc = "MWAIT 0x10",
543b82b6ccaSDaniel Lezcano 		.flags = MWAIT2flg(0x10) | CPUIDLE_FLAG_TLB_FLUSHED,
544a138b568SLen Brown 		.exit_latency = 40,
545a138b568SLen Brown 		.target_residency = 100,
5465fe2e527SRafael J. Wysocki 		.enter = &intel_idle,
54728ba086eSRafael J. Wysocki 		.enter_s2idle = intel_idle_s2idle, },
548a138b568SLen Brown 	{
549de09cdd0SLen Brown 		.name = "C6",
550a138b568SLen Brown 		.desc = "MWAIT 0x20",
551b82b6ccaSDaniel Lezcano 		.flags = MWAIT2flg(0x20) | CPUIDLE_FLAG_TLB_FLUSHED,
552a138b568SLen Brown 		.exit_latency = 133,
553a138b568SLen Brown 		.target_residency = 400,
5545fe2e527SRafael J. Wysocki 		.enter = &intel_idle,
55528ba086eSRafael J. Wysocki 		.enter_s2idle = intel_idle_s2idle, },
556a138b568SLen Brown 	{
557de09cdd0SLen Brown 		.name = "C7s",
558a138b568SLen Brown 		.desc = "MWAIT 0x32",
559b82b6ccaSDaniel Lezcano 		.flags = MWAIT2flg(0x32) | CPUIDLE_FLAG_TLB_FLUSHED,
560a138b568SLen Brown 		.exit_latency = 166,
561a138b568SLen Brown 		.target_residency = 500,
5625fe2e527SRafael J. Wysocki 		.enter = &intel_idle,
56328ba086eSRafael J. Wysocki 		.enter_s2idle = intel_idle_s2idle, },
564a138b568SLen Brown 	{
565de09cdd0SLen Brown 		.name = "C8",
566a138b568SLen Brown 		.desc = "MWAIT 0x40",
567b82b6ccaSDaniel Lezcano 		.flags = MWAIT2flg(0x40) | CPUIDLE_FLAG_TLB_FLUSHED,
568a138b568SLen Brown 		.exit_latency = 300,
569a138b568SLen Brown 		.target_residency = 900,
5705fe2e527SRafael J. Wysocki 		.enter = &intel_idle,
57128ba086eSRafael J. Wysocki 		.enter_s2idle = intel_idle_s2idle, },
572a138b568SLen Brown 	{
573de09cdd0SLen Brown 		.name = "C9",
574a138b568SLen Brown 		.desc = "MWAIT 0x50",
575b82b6ccaSDaniel Lezcano 		.flags = MWAIT2flg(0x50) | CPUIDLE_FLAG_TLB_FLUSHED,
576a138b568SLen Brown 		.exit_latency = 600,
577a138b568SLen Brown 		.target_residency = 1800,
5785fe2e527SRafael J. Wysocki 		.enter = &intel_idle,
57928ba086eSRafael J. Wysocki 		.enter_s2idle = intel_idle_s2idle, },
580a138b568SLen Brown 	{
581de09cdd0SLen Brown 		.name = "C10",
582a138b568SLen Brown 		.desc = "MWAIT 0x60",
583b82b6ccaSDaniel Lezcano 		.flags = MWAIT2flg(0x60) | CPUIDLE_FLAG_TLB_FLUSHED,
584a138b568SLen Brown 		.exit_latency = 2600,
585a138b568SLen Brown 		.target_residency = 7700,
5865fe2e527SRafael J. Wysocki 		.enter = &intel_idle,
58728ba086eSRafael J. Wysocki 		.enter_s2idle = intel_idle_s2idle, },
588a138b568SLen Brown 	{
589a138b568SLen Brown 		.enter = NULL }
590a138b568SLen Brown };
59185a4d2d4SLen Brown 
592493f133fSLen Brown static struct cpuidle_state skl_cstates[] = {
593493f133fSLen Brown 	{
594de09cdd0SLen Brown 		.name = "C1",
595493f133fSLen Brown 		.desc = "MWAIT 0x00",
596493f133fSLen Brown 		.flags = MWAIT2flg(0x00),
597493f133fSLen Brown 		.exit_latency = 2,
598493f133fSLen Brown 		.target_residency = 2,
599493f133fSLen Brown 		.enter = &intel_idle,
60028ba086eSRafael J. Wysocki 		.enter_s2idle = intel_idle_s2idle, },
601493f133fSLen Brown 	{
602de09cdd0SLen Brown 		.name = "C1E",
603493f133fSLen Brown 		.desc = "MWAIT 0x01",
604493f133fSLen Brown 		.flags = MWAIT2flg(0x01),
605493f133fSLen Brown 		.exit_latency = 10,
606493f133fSLen Brown 		.target_residency = 20,
607493f133fSLen Brown 		.enter = &intel_idle,
60828ba086eSRafael J. Wysocki 		.enter_s2idle = intel_idle_s2idle, },
609493f133fSLen Brown 	{
610de09cdd0SLen Brown 		.name = "C3",
611493f133fSLen Brown 		.desc = "MWAIT 0x10",
612493f133fSLen Brown 		.flags = MWAIT2flg(0x10) | CPUIDLE_FLAG_TLB_FLUSHED,
613493f133fSLen Brown 		.exit_latency = 70,
614493f133fSLen Brown 		.target_residency = 100,
615493f133fSLen Brown 		.enter = &intel_idle,
61628ba086eSRafael J. Wysocki 		.enter_s2idle = intel_idle_s2idle, },
617493f133fSLen Brown 	{
618de09cdd0SLen Brown 		.name = "C6",
619493f133fSLen Brown 		.desc = "MWAIT 0x20",
620493f133fSLen Brown 		.flags = MWAIT2flg(0x20) | CPUIDLE_FLAG_TLB_FLUSHED,
621135919a3SLen Brown 		.exit_latency = 85,
622493f133fSLen Brown 		.target_residency = 200,
623493f133fSLen Brown 		.enter = &intel_idle,
62428ba086eSRafael J. Wysocki 		.enter_s2idle = intel_idle_s2idle, },
625493f133fSLen Brown 	{
626de09cdd0SLen Brown 		.name = "C7s",
627493f133fSLen Brown 		.desc = "MWAIT 0x33",
628493f133fSLen Brown 		.flags = MWAIT2flg(0x33) | CPUIDLE_FLAG_TLB_FLUSHED,
629493f133fSLen Brown 		.exit_latency = 124,
630493f133fSLen Brown 		.target_residency = 800,
631493f133fSLen Brown 		.enter = &intel_idle,
63228ba086eSRafael J. Wysocki 		.enter_s2idle = intel_idle_s2idle, },
633493f133fSLen Brown 	{
634de09cdd0SLen Brown 		.name = "C8",
635493f133fSLen Brown 		.desc = "MWAIT 0x40",
636493f133fSLen Brown 		.flags = MWAIT2flg(0x40) | CPUIDLE_FLAG_TLB_FLUSHED,
637135919a3SLen Brown 		.exit_latency = 200,
638493f133fSLen Brown 		.target_residency = 800,
639493f133fSLen Brown 		.enter = &intel_idle,
64028ba086eSRafael J. Wysocki 		.enter_s2idle = intel_idle_s2idle, },
641493f133fSLen Brown 	{
642de09cdd0SLen Brown 		.name = "C9",
643135919a3SLen Brown 		.desc = "MWAIT 0x50",
644135919a3SLen Brown 		.flags = MWAIT2flg(0x50) | CPUIDLE_FLAG_TLB_FLUSHED,
645135919a3SLen Brown 		.exit_latency = 480,
646135919a3SLen Brown 		.target_residency = 5000,
647135919a3SLen Brown 		.enter = &intel_idle,
64828ba086eSRafael J. Wysocki 		.enter_s2idle = intel_idle_s2idle, },
649135919a3SLen Brown 	{
650de09cdd0SLen Brown 		.name = "C10",
651493f133fSLen Brown 		.desc = "MWAIT 0x60",
652493f133fSLen Brown 		.flags = MWAIT2flg(0x60) | CPUIDLE_FLAG_TLB_FLUSHED,
653493f133fSLen Brown 		.exit_latency = 890,
654493f133fSLen Brown 		.target_residency = 5000,
655493f133fSLen Brown 		.enter = &intel_idle,
65628ba086eSRafael J. Wysocki 		.enter_s2idle = intel_idle_s2idle, },
657493f133fSLen Brown 	{
658493f133fSLen Brown 		.enter = NULL }
659493f133fSLen Brown };
660493f133fSLen Brown 
661f9e71657SLen Brown static struct cpuidle_state skx_cstates[] = {
662f9e71657SLen Brown 	{
663de09cdd0SLen Brown 		.name = "C1",
664f9e71657SLen Brown 		.desc = "MWAIT 0x00",
665f9e71657SLen Brown 		.flags = MWAIT2flg(0x00),
666f9e71657SLen Brown 		.exit_latency = 2,
667f9e71657SLen Brown 		.target_residency = 2,
668f9e71657SLen Brown 		.enter = &intel_idle,
66928ba086eSRafael J. Wysocki 		.enter_s2idle = intel_idle_s2idle, },
670f9e71657SLen Brown 	{
671de09cdd0SLen Brown 		.name = "C1E",
672f9e71657SLen Brown 		.desc = "MWAIT 0x01",
673f9e71657SLen Brown 		.flags = MWAIT2flg(0x01),
674f9e71657SLen Brown 		.exit_latency = 10,
675f9e71657SLen Brown 		.target_residency = 20,
676f9e71657SLen Brown 		.enter = &intel_idle,
67728ba086eSRafael J. Wysocki 		.enter_s2idle = intel_idle_s2idle, },
678f9e71657SLen Brown 	{
679de09cdd0SLen Brown 		.name = "C6",
680f9e71657SLen Brown 		.desc = "MWAIT 0x20",
681f9e71657SLen Brown 		.flags = MWAIT2flg(0x20) | CPUIDLE_FLAG_TLB_FLUSHED,
682f9e71657SLen Brown 		.exit_latency = 133,
683f9e71657SLen Brown 		.target_residency = 600,
684f9e71657SLen Brown 		.enter = &intel_idle,
68528ba086eSRafael J. Wysocki 		.enter_s2idle = intel_idle_s2idle, },
686f9e71657SLen Brown 	{
687f9e71657SLen Brown 		.enter = NULL }
688f9e71657SLen Brown };
689f9e71657SLen Brown 
690ba0dc81eSJiang Liu static struct cpuidle_state atom_cstates[] = {
691e022e7ebSLen Brown 	{
692de09cdd0SLen Brown 		.name = "C1E",
69326717172SLen Brown 		.desc = "MWAIT 0x00",
694b82b6ccaSDaniel Lezcano 		.flags = MWAIT2flg(0x00),
69532e95180SLen Brown 		.exit_latency = 10,
69632e95180SLen Brown 		.target_residency = 20,
6975fe2e527SRafael J. Wysocki 		.enter = &intel_idle,
69828ba086eSRafael J. Wysocki 		.enter_s2idle = intel_idle_s2idle, },
699e022e7ebSLen Brown 	{
700de09cdd0SLen Brown 		.name = "C2",
70126717172SLen Brown 		.desc = "MWAIT 0x10",
702b82b6ccaSDaniel Lezcano 		.flags = MWAIT2flg(0x10),
70326717172SLen Brown 		.exit_latency = 20,
70426717172SLen Brown 		.target_residency = 80,
7055fe2e527SRafael J. Wysocki 		.enter = &intel_idle,
70628ba086eSRafael J. Wysocki 		.enter_s2idle = intel_idle_s2idle, },
707e022e7ebSLen Brown 	{
708de09cdd0SLen Brown 		.name = "C4",
70926717172SLen Brown 		.desc = "MWAIT 0x30",
710b82b6ccaSDaniel Lezcano 		.flags = MWAIT2flg(0x30) | CPUIDLE_FLAG_TLB_FLUSHED,
71126717172SLen Brown 		.exit_latency = 100,
71226717172SLen Brown 		.target_residency = 400,
7135fe2e527SRafael J. Wysocki 		.enter = &intel_idle,
71428ba086eSRafael J. Wysocki 		.enter_s2idle = intel_idle_s2idle, },
715e022e7ebSLen Brown 	{
716de09cdd0SLen Brown 		.name = "C6",
7177fcca7d9SLen Brown 		.desc = "MWAIT 0x52",
718b82b6ccaSDaniel Lezcano 		.flags = MWAIT2flg(0x52) | CPUIDLE_FLAG_TLB_FLUSHED,
7197fcca7d9SLen Brown 		.exit_latency = 140,
7207fcca7d9SLen Brown 		.target_residency = 560,
7215fe2e527SRafael J. Wysocki 		.enter = &intel_idle,
72228ba086eSRafael J. Wysocki 		.enter_s2idle = intel_idle_s2idle, },
723e022e7ebSLen Brown 	{
724e022e7ebSLen Brown 		.enter = NULL }
72526717172SLen Brown };
7265e7ec268SAndy Shevchenko static struct cpuidle_state tangier_cstates[] = {
7275e7ec268SAndy Shevchenko 	{
728de09cdd0SLen Brown 		.name = "C1",
7295e7ec268SAndy Shevchenko 		.desc = "MWAIT 0x00",
7305e7ec268SAndy Shevchenko 		.flags = MWAIT2flg(0x00),
7315e7ec268SAndy Shevchenko 		.exit_latency = 1,
7325e7ec268SAndy Shevchenko 		.target_residency = 4,
7335e7ec268SAndy Shevchenko 		.enter = &intel_idle,
73428ba086eSRafael J. Wysocki 		.enter_s2idle = intel_idle_s2idle, },
7355e7ec268SAndy Shevchenko 	{
736de09cdd0SLen Brown 		.name = "C4",
7375e7ec268SAndy Shevchenko 		.desc = "MWAIT 0x30",
7385e7ec268SAndy Shevchenko 		.flags = MWAIT2flg(0x30) | CPUIDLE_FLAG_TLB_FLUSHED,
7395e7ec268SAndy Shevchenko 		.exit_latency = 100,
7405e7ec268SAndy Shevchenko 		.target_residency = 400,
7415e7ec268SAndy Shevchenko 		.enter = &intel_idle,
74228ba086eSRafael J. Wysocki 		.enter_s2idle = intel_idle_s2idle, },
7435e7ec268SAndy Shevchenko 	{
744de09cdd0SLen Brown 		.name = "C6",
7455e7ec268SAndy Shevchenko 		.desc = "MWAIT 0x52",
7465e7ec268SAndy Shevchenko 		.flags = MWAIT2flg(0x52) | CPUIDLE_FLAG_TLB_FLUSHED,
7475e7ec268SAndy Shevchenko 		.exit_latency = 140,
7485e7ec268SAndy Shevchenko 		.target_residency = 560,
7495e7ec268SAndy Shevchenko 		.enter = &intel_idle,
75028ba086eSRafael J. Wysocki 		.enter_s2idle = intel_idle_s2idle, },
7515e7ec268SAndy Shevchenko 	{
752de09cdd0SLen Brown 		.name = "C7",
7535e7ec268SAndy Shevchenko 		.desc = "MWAIT 0x60",
7545e7ec268SAndy Shevchenko 		.flags = MWAIT2flg(0x60) | CPUIDLE_FLAG_TLB_FLUSHED,
7555e7ec268SAndy Shevchenko 		.exit_latency = 1200,
7565e7ec268SAndy Shevchenko 		.target_residency = 4000,
7575e7ec268SAndy Shevchenko 		.enter = &intel_idle,
75828ba086eSRafael J. Wysocki 		.enter_s2idle = intel_idle_s2idle, },
7595e7ec268SAndy Shevchenko 	{
760de09cdd0SLen Brown 		.name = "C9",
7615e7ec268SAndy Shevchenko 		.desc = "MWAIT 0x64",
7625e7ec268SAndy Shevchenko 		.flags = MWAIT2flg(0x64) | CPUIDLE_FLAG_TLB_FLUSHED,
7635e7ec268SAndy Shevchenko 		.exit_latency = 10000,
7645e7ec268SAndy Shevchenko 		.target_residency = 20000,
7655e7ec268SAndy Shevchenko 		.enter = &intel_idle,
76628ba086eSRafael J. Wysocki 		.enter_s2idle = intel_idle_s2idle, },
7675e7ec268SAndy Shevchenko 	{
7685e7ec268SAndy Shevchenko 		.enter = NULL }
7695e7ec268SAndy Shevchenko };
77088390996SJiang Liu static struct cpuidle_state avn_cstates[] = {
771fab04b22SLen Brown 	{
772de09cdd0SLen Brown 		.name = "C1",
773fab04b22SLen Brown 		.desc = "MWAIT 0x00",
774b82b6ccaSDaniel Lezcano 		.flags = MWAIT2flg(0x00),
775fab04b22SLen Brown 		.exit_latency = 2,
776fab04b22SLen Brown 		.target_residency = 2,
7775fe2e527SRafael J. Wysocki 		.enter = &intel_idle,
77828ba086eSRafael J. Wysocki 		.enter_s2idle = intel_idle_s2idle, },
779fab04b22SLen Brown 	{
780de09cdd0SLen Brown 		.name = "C6",
781fab04b22SLen Brown 		.desc = "MWAIT 0x51",
782b82b6ccaSDaniel Lezcano 		.flags = MWAIT2flg(0x51) | CPUIDLE_FLAG_TLB_FLUSHED,
783fab04b22SLen Brown 		.exit_latency = 15,
784fab04b22SLen Brown 		.target_residency = 45,
7855fe2e527SRafael J. Wysocki 		.enter = &intel_idle,
78628ba086eSRafael J. Wysocki 		.enter_s2idle = intel_idle_s2idle, },
78788390996SJiang Liu 	{
78888390996SJiang Liu 		.enter = NULL }
789fab04b22SLen Brown };
790281baf7aSDasaratharaman Chandramouli static struct cpuidle_state knl_cstates[] = {
791281baf7aSDasaratharaman Chandramouli 	{
792de09cdd0SLen Brown 		.name = "C1",
793281baf7aSDasaratharaman Chandramouli 		.desc = "MWAIT 0x00",
794281baf7aSDasaratharaman Chandramouli 		.flags = MWAIT2flg(0x00),
795281baf7aSDasaratharaman Chandramouli 		.exit_latency = 1,
796281baf7aSDasaratharaman Chandramouli 		.target_residency = 2,
797281baf7aSDasaratharaman Chandramouli 		.enter = &intel_idle,
79828ba086eSRafael J. Wysocki 		.enter_s2idle = intel_idle_s2idle },
799281baf7aSDasaratharaman Chandramouli 	{
800de09cdd0SLen Brown 		.name = "C6",
801281baf7aSDasaratharaman Chandramouli 		.desc = "MWAIT 0x10",
802281baf7aSDasaratharaman Chandramouli 		.flags = MWAIT2flg(0x10) | CPUIDLE_FLAG_TLB_FLUSHED,
803281baf7aSDasaratharaman Chandramouli 		.exit_latency = 120,
804281baf7aSDasaratharaman Chandramouli 		.target_residency = 500,
805281baf7aSDasaratharaman Chandramouli 		.enter = &intel_idle,
80628ba086eSRafael J. Wysocki 		.enter_s2idle = intel_idle_s2idle },
807281baf7aSDasaratharaman Chandramouli 	{
808281baf7aSDasaratharaman Chandramouli 		.enter = NULL }
809281baf7aSDasaratharaman Chandramouli };
81026717172SLen Brown 
8115dcef694SLen Brown static struct cpuidle_state bxt_cstates[] = {
8125dcef694SLen Brown 	{
813de09cdd0SLen Brown 		.name = "C1",
8145dcef694SLen Brown 		.desc = "MWAIT 0x00",
8155dcef694SLen Brown 		.flags = MWAIT2flg(0x00),
8165dcef694SLen Brown 		.exit_latency = 2,
8175dcef694SLen Brown 		.target_residency = 2,
8185dcef694SLen Brown 		.enter = &intel_idle,
81928ba086eSRafael J. Wysocki 		.enter_s2idle = intel_idle_s2idle, },
8205dcef694SLen Brown 	{
821de09cdd0SLen Brown 		.name = "C1E",
8225dcef694SLen Brown 		.desc = "MWAIT 0x01",
8235dcef694SLen Brown 		.flags = MWAIT2flg(0x01),
8245dcef694SLen Brown 		.exit_latency = 10,
8255dcef694SLen Brown 		.target_residency = 20,
8265dcef694SLen Brown 		.enter = &intel_idle,
82728ba086eSRafael J. Wysocki 		.enter_s2idle = intel_idle_s2idle, },
8285dcef694SLen Brown 	{
829de09cdd0SLen Brown 		.name = "C6",
8305dcef694SLen Brown 		.desc = "MWAIT 0x20",
8315dcef694SLen Brown 		.flags = MWAIT2flg(0x20) | CPUIDLE_FLAG_TLB_FLUSHED,
8325dcef694SLen Brown 		.exit_latency = 133,
8335dcef694SLen Brown 		.target_residency = 133,
8345dcef694SLen Brown 		.enter = &intel_idle,
83528ba086eSRafael J. Wysocki 		.enter_s2idle = intel_idle_s2idle, },
8365dcef694SLen Brown 	{
837de09cdd0SLen Brown 		.name = "C7s",
8385dcef694SLen Brown 		.desc = "MWAIT 0x31",
8395dcef694SLen Brown 		.flags = MWAIT2flg(0x31) | CPUIDLE_FLAG_TLB_FLUSHED,
8405dcef694SLen Brown 		.exit_latency = 155,
8415dcef694SLen Brown 		.target_residency = 155,
8425dcef694SLen Brown 		.enter = &intel_idle,
84328ba086eSRafael J. Wysocki 		.enter_s2idle = intel_idle_s2idle, },
8445dcef694SLen Brown 	{
845de09cdd0SLen Brown 		.name = "C8",
8465dcef694SLen Brown 		.desc = "MWAIT 0x40",
8475dcef694SLen Brown 		.flags = MWAIT2flg(0x40) | CPUIDLE_FLAG_TLB_FLUSHED,
8485dcef694SLen Brown 		.exit_latency = 1000,
8495dcef694SLen Brown 		.target_residency = 1000,
8505dcef694SLen Brown 		.enter = &intel_idle,
85128ba086eSRafael J. Wysocki 		.enter_s2idle = intel_idle_s2idle, },
8525dcef694SLen Brown 	{
853de09cdd0SLen Brown 		.name = "C9",
8545dcef694SLen Brown 		.desc = "MWAIT 0x50",
8555dcef694SLen Brown 		.flags = MWAIT2flg(0x50) | CPUIDLE_FLAG_TLB_FLUSHED,
8565dcef694SLen Brown 		.exit_latency = 2000,
8575dcef694SLen Brown 		.target_residency = 2000,
8585dcef694SLen Brown 		.enter = &intel_idle,
85928ba086eSRafael J. Wysocki 		.enter_s2idle = intel_idle_s2idle, },
8605dcef694SLen Brown 	{
861de09cdd0SLen Brown 		.name = "C10",
8625dcef694SLen Brown 		.desc = "MWAIT 0x60",
8635dcef694SLen Brown 		.flags = MWAIT2flg(0x60) | CPUIDLE_FLAG_TLB_FLUSHED,
8645dcef694SLen Brown 		.exit_latency = 10000,
8655dcef694SLen Brown 		.target_residency = 10000,
8665dcef694SLen Brown 		.enter = &intel_idle,
86728ba086eSRafael J. Wysocki 		.enter_s2idle = intel_idle_s2idle, },
8685dcef694SLen Brown 	{
8695dcef694SLen Brown 		.enter = NULL }
8705dcef694SLen Brown };
8715dcef694SLen Brown 
8720080d65bSJacob Pan static struct cpuidle_state dnv_cstates[] = {
8730080d65bSJacob Pan 	{
874de09cdd0SLen Brown 		.name = "C1",
8750080d65bSJacob Pan 		.desc = "MWAIT 0x00",
8760080d65bSJacob Pan 		.flags = MWAIT2flg(0x00),
8770080d65bSJacob Pan 		.exit_latency = 2,
8780080d65bSJacob Pan 		.target_residency = 2,
8790080d65bSJacob Pan 		.enter = &intel_idle,
88028ba086eSRafael J. Wysocki 		.enter_s2idle = intel_idle_s2idle, },
8810080d65bSJacob Pan 	{
882de09cdd0SLen Brown 		.name = "C1E",
8830080d65bSJacob Pan 		.desc = "MWAIT 0x01",
8840080d65bSJacob Pan 		.flags = MWAIT2flg(0x01),
8850080d65bSJacob Pan 		.exit_latency = 10,
8860080d65bSJacob Pan 		.target_residency = 20,
8870080d65bSJacob Pan 		.enter = &intel_idle,
88828ba086eSRafael J. Wysocki 		.enter_s2idle = intel_idle_s2idle, },
8890080d65bSJacob Pan 	{
890de09cdd0SLen Brown 		.name = "C6",
8910080d65bSJacob Pan 		.desc = "MWAIT 0x20",
8920080d65bSJacob Pan 		.flags = MWAIT2flg(0x20) | CPUIDLE_FLAG_TLB_FLUSHED,
8930080d65bSJacob Pan 		.exit_latency = 50,
8940080d65bSJacob Pan 		.target_residency = 500,
8950080d65bSJacob Pan 		.enter = &intel_idle,
89628ba086eSRafael J. Wysocki 		.enter_s2idle = intel_idle_s2idle, },
8970080d65bSJacob Pan 	{
8980080d65bSJacob Pan 		.enter = NULL }
8990080d65bSJacob Pan };
9000080d65bSJacob Pan 
90126717172SLen Brown /**
90226717172SLen Brown  * intel_idle
90326717172SLen Brown  * @dev: cpuidle_device
90446bcfad7SDeepthi Dharwar  * @drv: cpuidle driver
905e978aa7dSDeepthi Dharwar  * @index: index of cpuidle state
90626717172SLen Brown  *
90763ff07beSYanmin Zhang  * Must be called under local_irq_disable().
90826717172SLen Brown  */
9096727ad9eSChris Metcalf static __cpuidle int intel_idle(struct cpuidle_device *dev,
91046bcfad7SDeepthi Dharwar 				struct cpuidle_driver *drv, int index)
91126717172SLen Brown {
91226717172SLen Brown 	unsigned long ecx = 1; /* break on interrupt flag */
91346bcfad7SDeepthi Dharwar 	struct cpuidle_state *state = &drv->states[index];
914b1beab48SLen Brown 	unsigned long eax = flg2MWAIT(state->flags);
91526717172SLen Brown 	unsigned int cstate;
9160563bb7bSJason Baron 	bool uninitialized_var(tick);
91726717172SLen Brown 
9186110a1f4SSuresh Siddha 	/*
91943858b4fSAndy Lutomirski 	 * NB: if CPUIDLE_FLAG_TLB_FLUSHED is set, this idle transition
92043858b4fSAndy Lutomirski 	 * will probably flush the TLB.  It's not guaranteed to flush
92143858b4fSAndy Lutomirski 	 * the TLB, though, so it's not clear that we can do anything
92243858b4fSAndy Lutomirski 	 * useful with this knowledge.
9236110a1f4SSuresh Siddha 	 */
9246110a1f4SSuresh Siddha 
9250563bb7bSJason Baron 	if (!static_cpu_has(X86_FEATURE_ARAT)) {
9260563bb7bSJason Baron 		cstate = (((eax) >> MWAIT_SUBSTATE_SIZE) &
9270563bb7bSJason Baron 				MWAIT_CSTATE_MASK) + 1;
9280563bb7bSJason Baron 		tick = false;
9290563bb7bSJason Baron 		if (!(lapic_timer_reliable_states & (1 << (cstate)))) {
9300563bb7bSJason Baron 			tick = true;
931f6cee191SThomas Gleixner 			tick_broadcast_enter();
9320563bb7bSJason Baron 		}
9330563bb7bSJason Baron 	}
93426717172SLen Brown 
93516824255SPeter Zijlstra 	mwait_idle_with_hints(eax, ecx);
93626717172SLen Brown 
9370563bb7bSJason Baron 	if (!static_cpu_has(X86_FEATURE_ARAT) && tick)
938f6cee191SThomas Gleixner 		tick_broadcast_exit();
93926717172SLen Brown 
940e978aa7dSDeepthi Dharwar 	return index;
94126717172SLen Brown }
94226717172SLen Brown 
9435fe2e527SRafael J. Wysocki /**
94428ba086eSRafael J. Wysocki  * intel_idle_s2idle - simplified "enter" callback routine for suspend-to-idle
9455fe2e527SRafael J. Wysocki  * @dev: cpuidle_device
9465fe2e527SRafael J. Wysocki  * @drv: cpuidle driver
9475fe2e527SRafael J. Wysocki  * @index: state index
9485fe2e527SRafael J. Wysocki  */
94928ba086eSRafael J. Wysocki static void intel_idle_s2idle(struct cpuidle_device *dev,
9505fe2e527SRafael J. Wysocki 			     struct cpuidle_driver *drv, int index)
9515fe2e527SRafael J. Wysocki {
9525fe2e527SRafael J. Wysocki 	unsigned long ecx = 1; /* break on interrupt flag */
9535fe2e527SRafael J. Wysocki 	unsigned long eax = flg2MWAIT(drv->states[index].flags);
9545fe2e527SRafael J. Wysocki 
9555fe2e527SRafael J. Wysocki 	mwait_idle_with_hints(eax, ecx);
9565fe2e527SRafael J. Wysocki }
9575fe2e527SRafael J. Wysocki 
958fb1013a0SSebastian Andrzej Siewior static void __setup_broadcast_timer(bool on)
9592a2d31c8SShaohua Li {
96076962caaSThomas Gleixner 	if (on)
96176962caaSThomas Gleixner 		tick_broadcast_enable();
96276962caaSThomas Gleixner 	else
96376962caaSThomas Gleixner 		tick_broadcast_disable();
9642a2d31c8SShaohua Li }
9652a2d31c8SShaohua Li 
966fb1013a0SSebastian Andrzej Siewior static void auto_demotion_disable(void)
96714796fcaSLen Brown {
96814796fcaSLen Brown 	unsigned long long msr_bits;
96914796fcaSLen Brown 
9706cfb2374SLen Brown 	rdmsrl(MSR_PKG_CST_CONFIG_CONTROL, msr_bits);
971b66b8b9aSAndi Kleen 	msr_bits &= ~(icpu->auto_demotion_disable_flags);
9726cfb2374SLen Brown 	wrmsrl(MSR_PKG_CST_CONFIG_CONTROL, msr_bits);
97314796fcaSLen Brown }
974fb1013a0SSebastian Andrzej Siewior static void c1e_promotion_disable(void)
97532e95180SLen Brown {
97632e95180SLen Brown 	unsigned long long msr_bits;
97732e95180SLen Brown 
97832e95180SLen Brown 	rdmsrl(MSR_IA32_POWER_CTL, msr_bits);
97932e95180SLen Brown 	msr_bits &= ~0x2;
98032e95180SLen Brown 	wrmsrl(MSR_IA32_POWER_CTL, msr_bits);
98132e95180SLen Brown }
98214796fcaSLen Brown 
983b66b8b9aSAndi Kleen static const struct idle_cpu idle_cpu_nehalem = {
984b66b8b9aSAndi Kleen 	.state_table = nehalem_cstates,
985b66b8b9aSAndi Kleen 	.auto_demotion_disable_flags = NHM_C1_AUTO_DEMOTE | NHM_C3_AUTO_DEMOTE,
98632e95180SLen Brown 	.disable_promotion_to_c1e = true,
987b66b8b9aSAndi Kleen };
988b66b8b9aSAndi Kleen 
989b66b8b9aSAndi Kleen static const struct idle_cpu idle_cpu_atom = {
990b66b8b9aSAndi Kleen 	.state_table = atom_cstates,
991b66b8b9aSAndi Kleen };
992b66b8b9aSAndi Kleen 
9935e7ec268SAndy Shevchenko static const struct idle_cpu idle_cpu_tangier = {
9945e7ec268SAndy Shevchenko 	.state_table = tangier_cstates,
9955e7ec268SAndy Shevchenko };
9965e7ec268SAndy Shevchenko 
997b66b8b9aSAndi Kleen static const struct idle_cpu idle_cpu_lincroft = {
998b66b8b9aSAndi Kleen 	.state_table = atom_cstates,
999b66b8b9aSAndi Kleen 	.auto_demotion_disable_flags = ATM_LNC_C6_AUTO_DEMOTE,
1000b66b8b9aSAndi Kleen };
1001b66b8b9aSAndi Kleen 
1002b66b8b9aSAndi Kleen static const struct idle_cpu idle_cpu_snb = {
1003b66b8b9aSAndi Kleen 	.state_table = snb_cstates,
100432e95180SLen Brown 	.disable_promotion_to_c1e = true,
1005b66b8b9aSAndi Kleen };
1006b66b8b9aSAndi Kleen 
1007718987d6SLen Brown static const struct idle_cpu idle_cpu_byt = {
1008718987d6SLen Brown 	.state_table = byt_cstates,
1009718987d6SLen Brown 	.disable_promotion_to_c1e = true,
10108c058d53SLen Brown 	.byt_auto_demotion_disable_flag = true,
1011718987d6SLen Brown };
1012718987d6SLen Brown 
1013cab07a56SLen Brown static const struct idle_cpu idle_cpu_cht = {
1014cab07a56SLen Brown 	.state_table = cht_cstates,
1015cab07a56SLen Brown 	.disable_promotion_to_c1e = true,
1016cab07a56SLen Brown 	.byt_auto_demotion_disable_flag = true,
1017cab07a56SLen Brown };
1018cab07a56SLen Brown 
10196edab08cSLen Brown static const struct idle_cpu idle_cpu_ivb = {
10206edab08cSLen Brown 	.state_table = ivb_cstates,
102132e95180SLen Brown 	.disable_promotion_to_c1e = true,
10226edab08cSLen Brown };
10236edab08cSLen Brown 
10240138d8f0SLen Brown static const struct idle_cpu idle_cpu_ivt = {
10250138d8f0SLen Brown 	.state_table = ivt_cstates,
10260138d8f0SLen Brown 	.disable_promotion_to_c1e = true,
10270138d8f0SLen Brown };
10280138d8f0SLen Brown 
102985a4d2d4SLen Brown static const struct idle_cpu idle_cpu_hsw = {
103085a4d2d4SLen Brown 	.state_table = hsw_cstates,
103132e95180SLen Brown 	.disable_promotion_to_c1e = true,
103285a4d2d4SLen Brown };
103385a4d2d4SLen Brown 
1034a138b568SLen Brown static const struct idle_cpu idle_cpu_bdw = {
1035a138b568SLen Brown 	.state_table = bdw_cstates,
1036a138b568SLen Brown 	.disable_promotion_to_c1e = true,
1037a138b568SLen Brown };
1038a138b568SLen Brown 
1039493f133fSLen Brown static const struct idle_cpu idle_cpu_skl = {
1040493f133fSLen Brown 	.state_table = skl_cstates,
1041493f133fSLen Brown 	.disable_promotion_to_c1e = true,
1042493f133fSLen Brown };
1043493f133fSLen Brown 
1044f9e71657SLen Brown static const struct idle_cpu idle_cpu_skx = {
1045f9e71657SLen Brown 	.state_table = skx_cstates,
1046f9e71657SLen Brown 	.disable_promotion_to_c1e = true,
1047f9e71657SLen Brown };
1048493f133fSLen Brown 
1049fab04b22SLen Brown static const struct idle_cpu idle_cpu_avn = {
1050fab04b22SLen Brown 	.state_table = avn_cstates,
1051fab04b22SLen Brown 	.disable_promotion_to_c1e = true,
1052fab04b22SLen Brown };
1053fab04b22SLen Brown 
1054281baf7aSDasaratharaman Chandramouli static const struct idle_cpu idle_cpu_knl = {
1055281baf7aSDasaratharaman Chandramouli 	.state_table = knl_cstates,
1056281baf7aSDasaratharaman Chandramouli };
1057281baf7aSDasaratharaman Chandramouli 
10585dcef694SLen Brown static const struct idle_cpu idle_cpu_bxt = {
10595dcef694SLen Brown 	.state_table = bxt_cstates,
10605dcef694SLen Brown 	.disable_promotion_to_c1e = true,
10615dcef694SLen Brown };
10625dcef694SLen Brown 
10630080d65bSJacob Pan static const struct idle_cpu idle_cpu_dnv = {
10640080d65bSJacob Pan 	.state_table = dnv_cstates,
10650080d65bSJacob Pan 	.disable_promotion_to_c1e = true,
10660080d65bSJacob Pan };
10670080d65bSJacob Pan 
1068b66b8b9aSAndi Kleen #define ICPU(model, cpu) \
1069b66b8b9aSAndi Kleen 	{ X86_VENDOR_INTEL, 6, model, X86_FEATURE_MWAIT, (unsigned long)&cpu }
1070b66b8b9aSAndi Kleen 
1071d5cdc3c4SMathias Krause static const struct x86_cpu_id intel_idle_ids[] __initconst = {
1072db73c5a8SDave Hansen 	ICPU(INTEL_FAM6_NEHALEM_EP,		idle_cpu_nehalem),
1073db73c5a8SDave Hansen 	ICPU(INTEL_FAM6_NEHALEM,		idle_cpu_nehalem),
10744b3b234fSDave Hansen 	ICPU(INTEL_FAM6_NEHALEM_G,		idle_cpu_nehalem),
1075db73c5a8SDave Hansen 	ICPU(INTEL_FAM6_WESTMERE,		idle_cpu_nehalem),
1076db73c5a8SDave Hansen 	ICPU(INTEL_FAM6_WESTMERE_EP,		idle_cpu_nehalem),
1077db73c5a8SDave Hansen 	ICPU(INTEL_FAM6_NEHALEM_EX,		idle_cpu_nehalem),
1078db73c5a8SDave Hansen 	ICPU(INTEL_FAM6_ATOM_PINEVIEW,		idle_cpu_atom),
1079db73c5a8SDave Hansen 	ICPU(INTEL_FAM6_ATOM_LINCROFT,		idle_cpu_lincroft),
1080db73c5a8SDave Hansen 	ICPU(INTEL_FAM6_WESTMERE_EX,		idle_cpu_nehalem),
1081db73c5a8SDave Hansen 	ICPU(INTEL_FAM6_SANDYBRIDGE,		idle_cpu_snb),
1082db73c5a8SDave Hansen 	ICPU(INTEL_FAM6_SANDYBRIDGE_X,		idle_cpu_snb),
1083db73c5a8SDave Hansen 	ICPU(INTEL_FAM6_ATOM_CEDARVIEW,		idle_cpu_atom),
1084db73c5a8SDave Hansen 	ICPU(INTEL_FAM6_ATOM_SILVERMONT1,	idle_cpu_byt),
10855e7ec268SAndy Shevchenko 	ICPU(INTEL_FAM6_ATOM_MERRIFIELD,	idle_cpu_tangier),
1086db73c5a8SDave Hansen 	ICPU(INTEL_FAM6_ATOM_AIRMONT,		idle_cpu_cht),
1087db73c5a8SDave Hansen 	ICPU(INTEL_FAM6_IVYBRIDGE,		idle_cpu_ivb),
1088db73c5a8SDave Hansen 	ICPU(INTEL_FAM6_IVYBRIDGE_X,		idle_cpu_ivt),
1089db73c5a8SDave Hansen 	ICPU(INTEL_FAM6_HASWELL_CORE,		idle_cpu_hsw),
1090db73c5a8SDave Hansen 	ICPU(INTEL_FAM6_HASWELL_X,		idle_cpu_hsw),
1091db73c5a8SDave Hansen 	ICPU(INTEL_FAM6_HASWELL_ULT,		idle_cpu_hsw),
1092db73c5a8SDave Hansen 	ICPU(INTEL_FAM6_HASWELL_GT3E,		idle_cpu_hsw),
1093db73c5a8SDave Hansen 	ICPU(INTEL_FAM6_ATOM_SILVERMONT2,	idle_cpu_avn),
1094db73c5a8SDave Hansen 	ICPU(INTEL_FAM6_BROADWELL_CORE,		idle_cpu_bdw),
1095db73c5a8SDave Hansen 	ICPU(INTEL_FAM6_BROADWELL_GT3E,		idle_cpu_bdw),
1096db73c5a8SDave Hansen 	ICPU(INTEL_FAM6_BROADWELL_X,		idle_cpu_bdw),
1097db73c5a8SDave Hansen 	ICPU(INTEL_FAM6_BROADWELL_XEON_D,	idle_cpu_bdw),
1098db73c5a8SDave Hansen 	ICPU(INTEL_FAM6_SKYLAKE_MOBILE,		idle_cpu_skl),
1099db73c5a8SDave Hansen 	ICPU(INTEL_FAM6_SKYLAKE_DESKTOP,	idle_cpu_skl),
1100db73c5a8SDave Hansen 	ICPU(INTEL_FAM6_KABYLAKE_MOBILE,	idle_cpu_skl),
1101db73c5a8SDave Hansen 	ICPU(INTEL_FAM6_KABYLAKE_DESKTOP,	idle_cpu_skl),
1102db73c5a8SDave Hansen 	ICPU(INTEL_FAM6_SKYLAKE_X,		idle_cpu_skx),
1103db73c5a8SDave Hansen 	ICPU(INTEL_FAM6_XEON_PHI_KNL,		idle_cpu_knl),
1104a2c1bc64SPiotr Luc 	ICPU(INTEL_FAM6_XEON_PHI_KNM,		idle_cpu_knl),
1105db73c5a8SDave Hansen 	ICPU(INTEL_FAM6_ATOM_GOLDMONT,		idle_cpu_bxt),
11061b2e8768SDavid E. Box 	ICPU(INTEL_FAM6_ATOM_GEMINI_LAKE,	idle_cpu_bxt),
11070080d65bSJacob Pan 	ICPU(INTEL_FAM6_ATOM_DENVERTON,		idle_cpu_dnv),
1108b66b8b9aSAndi Kleen 	{}
1109b66b8b9aSAndi Kleen };
1110b66b8b9aSAndi Kleen 
111126717172SLen Brown /*
111226717172SLen Brown  * intel_idle_probe()
111326717172SLen Brown  */
111400f3e755SBartlomiej Zolnierkiewicz static int __init intel_idle_probe(void)
111526717172SLen Brown {
1116c4236282SLen Brown 	unsigned int eax, ebx, ecx;
1117b66b8b9aSAndi Kleen 	const struct x86_cpu_id *id;
111826717172SLen Brown 
111926717172SLen Brown 	if (max_cstate == 0) {
1120654d08a4SJoe Perches 		pr_debug("disabled\n");
112126717172SLen Brown 		return -EPERM;
112226717172SLen Brown 	}
112326717172SLen Brown 
1124b66b8b9aSAndi Kleen 	id = x86_match_cpu(intel_idle_ids);
1125b66b8b9aSAndi Kleen 	if (!id) {
1126b66b8b9aSAndi Kleen 		if (boot_cpu_data.x86_vendor == X86_VENDOR_INTEL &&
1127b66b8b9aSAndi Kleen 		    boot_cpu_data.x86 == 6)
1128654d08a4SJoe Perches 			pr_debug("does not run on family %d model %d\n",
1129b66b8b9aSAndi Kleen 				 boot_cpu_data.x86, boot_cpu_data.x86_model);
113026717172SLen Brown 		return -ENODEV;
1131b66b8b9aSAndi Kleen 	}
113226717172SLen Brown 
113326717172SLen Brown 	if (boot_cpu_data.cpuid_level < CPUID_MWAIT_LEAF)
113426717172SLen Brown 		return -ENODEV;
113526717172SLen Brown 
1136c4236282SLen Brown 	cpuid(CPUID_MWAIT_LEAF, &eax, &ebx, &ecx, &mwait_substates);
113726717172SLen Brown 
113826717172SLen Brown 	if (!(ecx & CPUID5_ECX_EXTENSIONS_SUPPORTED) ||
11395c2a9f06SThomas Renninger 	    !(ecx & CPUID5_ECX_INTERRUPT_BREAK) ||
11405c2a9f06SThomas Renninger 	    !mwait_substates)
114126717172SLen Brown 			return -ENODEV;
114226717172SLen Brown 
1143654d08a4SJoe Perches 	pr_debug("MWAIT substates: 0x%x\n", mwait_substates);
114426717172SLen Brown 
1145b66b8b9aSAndi Kleen 	icpu = (const struct idle_cpu *)id->driver_data;
1146b66b8b9aSAndi Kleen 	cpuidle_state_table = icpu->state_table;
114726717172SLen Brown 
1148654d08a4SJoe Perches 	pr_debug("v" INTEL_IDLE_VERSION " model 0x%X\n",
1149654d08a4SJoe Perches 		 boot_cpu_data.x86_model);
115026717172SLen Brown 
115126717172SLen Brown 	return 0;
115226717172SLen Brown }
115326717172SLen Brown 
115426717172SLen Brown /*
115526717172SLen Brown  * intel_idle_cpuidle_devices_uninit()
1156ca42489dSRichard Cochran  * Unregisters the cpuidle devices.
115726717172SLen Brown  */
115826717172SLen Brown static void intel_idle_cpuidle_devices_uninit(void)
115926717172SLen Brown {
116026717172SLen Brown 	int i;
116126717172SLen Brown 	struct cpuidle_device *dev;
116226717172SLen Brown 
116326717172SLen Brown 	for_each_online_cpu(i) {
116426717172SLen Brown 		dev = per_cpu_ptr(intel_idle_cpuidle_devices, i);
116526717172SLen Brown 		cpuidle_unregister_device(dev);
116626717172SLen Brown 	}
116726717172SLen Brown }
11680138d8f0SLen Brown 
11690138d8f0SLen Brown /*
1170d70e28f5SLen Brown  * ivt_idle_state_table_update(void)
11710138d8f0SLen Brown  *
1172d70e28f5SLen Brown  * Tune IVT multi-socket targets
11730138d8f0SLen Brown  * Assumption: num_sockets == (max_package_num + 1)
11740138d8f0SLen Brown  */
1175d70e28f5SLen Brown static void ivt_idle_state_table_update(void)
11760138d8f0SLen Brown {
11770138d8f0SLen Brown 	/* IVT uses a different table for 1-2, 3-4, and > 4 sockets */
11780138d8f0SLen Brown 	int cpu, package_num, num_sockets = 1;
11790138d8f0SLen Brown 
11800138d8f0SLen Brown 	for_each_online_cpu(cpu) {
11810138d8f0SLen Brown 		package_num = topology_physical_package_id(cpu);
11820138d8f0SLen Brown 		if (package_num + 1 > num_sockets) {
11830138d8f0SLen Brown 			num_sockets = package_num + 1;
11840138d8f0SLen Brown 
1185d27dca42SChristoph Jaeger 			if (num_sockets > 4) {
11860138d8f0SLen Brown 				cpuidle_state_table = ivt_cstates_8s;
11870138d8f0SLen Brown 				return;
11880138d8f0SLen Brown 			}
11890138d8f0SLen Brown 		}
1190d27dca42SChristoph Jaeger 	}
11910138d8f0SLen Brown 
11920138d8f0SLen Brown 	if (num_sockets > 2)
11930138d8f0SLen Brown 		cpuidle_state_table = ivt_cstates_4s;
1194d70e28f5SLen Brown 
11950138d8f0SLen Brown 	/* else, 1 and 2 socket systems use default ivt_cstates */
11960138d8f0SLen Brown }
11975dcef694SLen Brown 
11985dcef694SLen Brown /*
11995dcef694SLen Brown  * Translate IRTL (Interrupt Response Time Limit) MSR to usec
12005dcef694SLen Brown  */
12015dcef694SLen Brown 
12025dcef694SLen Brown static unsigned int irtl_ns_units[] = {
12035dcef694SLen Brown 	1, 32, 1024, 32768, 1048576, 33554432, 0, 0 };
12045dcef694SLen Brown 
12055dcef694SLen Brown static unsigned long long irtl_2_usec(unsigned long long irtl)
12065dcef694SLen Brown {
12075dcef694SLen Brown 	unsigned long long ns;
12085dcef694SLen Brown 
12093451ab3eSJan Beulich 	if (!irtl)
12103451ab3eSJan Beulich 		return 0;
12113451ab3eSJan Beulich 
1212bef45096SJan Beulich 	ns = irtl_ns_units[(irtl >> 10) & 0x7];
12135dcef694SLen Brown 
12145dcef694SLen Brown 	return div64_u64((irtl & 0x3FF) * ns, 1000);
12155dcef694SLen Brown }
12165dcef694SLen Brown /*
12175dcef694SLen Brown  * bxt_idle_state_table_update(void)
12185dcef694SLen Brown  *
12195dcef694SLen Brown  * On BXT, we trust the IRTL to show the definitive maximum latency
12205dcef694SLen Brown  * We use the same value for target_residency.
12215dcef694SLen Brown  */
12225dcef694SLen Brown static void bxt_idle_state_table_update(void)
12235dcef694SLen Brown {
12245dcef694SLen Brown 	unsigned long long msr;
12253451ab3eSJan Beulich 	unsigned int usec;
12265dcef694SLen Brown 
12275dcef694SLen Brown 	rdmsrl(MSR_PKGC6_IRTL, msr);
12283451ab3eSJan Beulich 	usec = irtl_2_usec(msr);
12293451ab3eSJan Beulich 	if (usec) {
12305dcef694SLen Brown 		bxt_cstates[2].exit_latency = usec;
12315dcef694SLen Brown 		bxt_cstates[2].target_residency = usec;
12325dcef694SLen Brown 	}
12335dcef694SLen Brown 
12345dcef694SLen Brown 	rdmsrl(MSR_PKGC7_IRTL, msr);
12353451ab3eSJan Beulich 	usec = irtl_2_usec(msr);
12363451ab3eSJan Beulich 	if (usec) {
12375dcef694SLen Brown 		bxt_cstates[3].exit_latency = usec;
12385dcef694SLen Brown 		bxt_cstates[3].target_residency = usec;
12395dcef694SLen Brown 	}
12405dcef694SLen Brown 
12415dcef694SLen Brown 	rdmsrl(MSR_PKGC8_IRTL, msr);
12423451ab3eSJan Beulich 	usec = irtl_2_usec(msr);
12433451ab3eSJan Beulich 	if (usec) {
12445dcef694SLen Brown 		bxt_cstates[4].exit_latency = usec;
12455dcef694SLen Brown 		bxt_cstates[4].target_residency = usec;
12465dcef694SLen Brown 	}
12475dcef694SLen Brown 
12485dcef694SLen Brown 	rdmsrl(MSR_PKGC9_IRTL, msr);
12493451ab3eSJan Beulich 	usec = irtl_2_usec(msr);
12503451ab3eSJan Beulich 	if (usec) {
12515dcef694SLen Brown 		bxt_cstates[5].exit_latency = usec;
12525dcef694SLen Brown 		bxt_cstates[5].target_residency = usec;
12535dcef694SLen Brown 	}
12545dcef694SLen Brown 
12555dcef694SLen Brown 	rdmsrl(MSR_PKGC10_IRTL, msr);
12563451ab3eSJan Beulich 	usec = irtl_2_usec(msr);
12573451ab3eSJan Beulich 	if (usec) {
12585dcef694SLen Brown 		bxt_cstates[6].exit_latency = usec;
12595dcef694SLen Brown 		bxt_cstates[6].target_residency = usec;
12605dcef694SLen Brown 	}
12615dcef694SLen Brown 
12625dcef694SLen Brown }
1263d70e28f5SLen Brown /*
1264d70e28f5SLen Brown  * sklh_idle_state_table_update(void)
1265d70e28f5SLen Brown  *
1266d70e28f5SLen Brown  * On SKL-H (model 0x5e) disable C8 and C9 if:
1267d70e28f5SLen Brown  * C10 is enabled and SGX disabled
1268d70e28f5SLen Brown  */
1269d70e28f5SLen Brown static void sklh_idle_state_table_update(void)
1270d70e28f5SLen Brown {
1271d70e28f5SLen Brown 	unsigned long long msr;
1272d70e28f5SLen Brown 	unsigned int eax, ebx, ecx, edx;
1273d70e28f5SLen Brown 
1274d70e28f5SLen Brown 
1275d70e28f5SLen Brown 	/* if PC10 disabled via cmdline intel_idle.max_cstate=7 or shallower */
1276d70e28f5SLen Brown 	if (max_cstate <= 7)
12770138d8f0SLen Brown 		return;
1278d70e28f5SLen Brown 
1279d70e28f5SLen Brown 	/* if PC10 not present in CPUID.MWAIT.EDX */
1280d70e28f5SLen Brown 	if ((mwait_substates & (0xF << 28)) == 0)
1281d70e28f5SLen Brown 		return;
1282d70e28f5SLen Brown 
12836cfb2374SLen Brown 	rdmsrl(MSR_PKG_CST_CONFIG_CONTROL, msr);
1284d70e28f5SLen Brown 
1285d70e28f5SLen Brown 	/* PC10 is not enabled in PKG C-state limit */
1286d70e28f5SLen Brown 	if ((msr & 0xF) != 8)
1287d70e28f5SLen Brown 		return;
1288d70e28f5SLen Brown 
1289d70e28f5SLen Brown 	ecx = 0;
1290d70e28f5SLen Brown 	cpuid(7, &eax, &ebx, &ecx, &edx);
1291d70e28f5SLen Brown 
1292d70e28f5SLen Brown 	/* if SGX is present */
1293d70e28f5SLen Brown 	if (ebx & (1 << 2)) {
1294d70e28f5SLen Brown 
1295d70e28f5SLen Brown 		rdmsrl(MSR_IA32_FEATURE_CONTROL, msr);
1296d70e28f5SLen Brown 
1297d70e28f5SLen Brown 		/* if SGX is enabled */
1298d70e28f5SLen Brown 		if (msr & (1 << 18))
1299d70e28f5SLen Brown 			return;
1300d70e28f5SLen Brown 	}
1301d70e28f5SLen Brown 
1302d70e28f5SLen Brown 	skl_cstates[5].disabled = 1;	/* C8-SKL */
1303d70e28f5SLen Brown 	skl_cstates[6].disabled = 1;	/* C9-SKL */
1304d70e28f5SLen Brown }
1305d70e28f5SLen Brown /*
1306d70e28f5SLen Brown  * intel_idle_state_table_update()
1307d70e28f5SLen Brown  *
1308d70e28f5SLen Brown  * Update the default state_table for this CPU-id
1309d70e28f5SLen Brown  */
1310d70e28f5SLen Brown 
1311d70e28f5SLen Brown static void intel_idle_state_table_update(void)
1312d70e28f5SLen Brown {
1313d70e28f5SLen Brown 	switch (boot_cpu_data.x86_model) {
1314d70e28f5SLen Brown 
1315db73c5a8SDave Hansen 	case INTEL_FAM6_IVYBRIDGE_X:
1316d70e28f5SLen Brown 		ivt_idle_state_table_update();
1317d70e28f5SLen Brown 		break;
1318db73c5a8SDave Hansen 	case INTEL_FAM6_ATOM_GOLDMONT:
13191b2e8768SDavid E. Box 	case INTEL_FAM6_ATOM_GEMINI_LAKE:
13205dcef694SLen Brown 		bxt_idle_state_table_update();
13215dcef694SLen Brown 		break;
1322db73c5a8SDave Hansen 	case INTEL_FAM6_SKYLAKE_DESKTOP:
1323d70e28f5SLen Brown 		sklh_idle_state_table_update();
1324d70e28f5SLen Brown 		break;
1325d70e28f5SLen Brown 	}
13260138d8f0SLen Brown }
13270138d8f0SLen Brown 
132826717172SLen Brown /*
132946bcfad7SDeepthi Dharwar  * intel_idle_cpuidle_driver_init()
133046bcfad7SDeepthi Dharwar  * allocate, initialize cpuidle_states
133146bcfad7SDeepthi Dharwar  */
13325469c827SRichard Cochran static void __init intel_idle_cpuidle_driver_init(void)
133346bcfad7SDeepthi Dharwar {
133446bcfad7SDeepthi Dharwar 	int cstate;
133546bcfad7SDeepthi Dharwar 	struct cpuidle_driver *drv = &intel_idle_driver;
133646bcfad7SDeepthi Dharwar 
13370138d8f0SLen Brown 	intel_idle_state_table_update();
13380138d8f0SLen Brown 
13391b39e3f8SRafael J. Wysocki 	cpuidle_poll_state_init(drv);
134046bcfad7SDeepthi Dharwar 	drv->state_count = 1;
134146bcfad7SDeepthi Dharwar 
1342e022e7ebSLen Brown 	for (cstate = 0; cstate < CPUIDLE_STATE_MAX; ++cstate) {
134324bfa950SLen Brown 		int num_substates, mwait_hint, mwait_cstate;
134446bcfad7SDeepthi Dharwar 
13457dd0e0afSLen Brown 		if ((cpuidle_state_table[cstate].enter == NULL) &&
134628ba086eSRafael J. Wysocki 		    (cpuidle_state_table[cstate].enter_s2idle == NULL))
1347e022e7ebSLen Brown 			break;
1348e022e7ebSLen Brown 
1349e022e7ebSLen Brown 		if (cstate + 1 > max_cstate) {
1350654d08a4SJoe Perches 			pr_info("max_cstate %d reached\n", max_cstate);
135146bcfad7SDeepthi Dharwar 			break;
135246bcfad7SDeepthi Dharwar 		}
135346bcfad7SDeepthi Dharwar 
1354e022e7ebSLen Brown 		mwait_hint = flg2MWAIT(cpuidle_state_table[cstate].flags);
1355e022e7ebSLen Brown 		mwait_cstate = MWAIT_HINT2CSTATE(mwait_hint);
135646bcfad7SDeepthi Dharwar 
135724bfa950SLen Brown 		/* number of sub-states for this state in CPUID.MWAIT */
1358e022e7ebSLen Brown 		num_substates = (mwait_substates >> ((mwait_cstate + 1) * 4))
1359e022e7ebSLen Brown 					& MWAIT_SUBSTATE_MASK;
1360e022e7ebSLen Brown 
136124bfa950SLen Brown 		/* if NO sub-states for this state in CPUID, skip it */
136224bfa950SLen Brown 		if (num_substates == 0)
1363e022e7ebSLen Brown 			continue;
1364e022e7ebSLen Brown 
1365d70e28f5SLen Brown 		/* if state marked as disabled, skip it */
1366d70e28f5SLen Brown 		if (cpuidle_state_table[cstate].disabled != 0) {
1367654d08a4SJoe Perches 			pr_debug("state %s is disabled\n",
1368d70e28f5SLen Brown 				 cpuidle_state_table[cstate].name);
1369d70e28f5SLen Brown 			continue;
1370d70e28f5SLen Brown 		}
1371d70e28f5SLen Brown 
1372d70e28f5SLen Brown 
1373e022e7ebSLen Brown 		if (((mwait_cstate + 1) > 2) &&
137446bcfad7SDeepthi Dharwar 			!boot_cpu_has(X86_FEATURE_NONSTOP_TSC))
137546bcfad7SDeepthi Dharwar 			mark_tsc_unstable("TSC halts in idle"
137646bcfad7SDeepthi Dharwar 					" states deeper than C2");
137746bcfad7SDeepthi Dharwar 
137846bcfad7SDeepthi Dharwar 		drv->states[drv->state_count] =	/* structure copy */
137946bcfad7SDeepthi Dharwar 			cpuidle_state_table[cstate];
138046bcfad7SDeepthi Dharwar 
138146bcfad7SDeepthi Dharwar 		drv->state_count += 1;
138246bcfad7SDeepthi Dharwar 	}
138346bcfad7SDeepthi Dharwar 
13848c058d53SLen Brown 	if (icpu->byt_auto_demotion_disable_flag) {
13858c058d53SLen Brown 		wrmsrl(MSR_CC6_DEMOTION_POLICY_CONFIG, 0);
13868c058d53SLen Brown 		wrmsrl(MSR_MC6_DEMOTION_POLICY_CONFIG, 0);
13878c058d53SLen Brown 	}
138846bcfad7SDeepthi Dharwar }
138946bcfad7SDeepthi Dharwar 
139046bcfad7SDeepthi Dharwar 
139146bcfad7SDeepthi Dharwar /*
139265b7f839SThomas Renninger  * intel_idle_cpu_init()
139326717172SLen Brown  * allocate, initialize, register cpuidle_devices
139465b7f839SThomas Renninger  * @cpu: cpu/core to initialize
139526717172SLen Brown  */
1396fb1013a0SSebastian Andrzej Siewior static int intel_idle_cpu_init(unsigned int cpu)
139726717172SLen Brown {
139826717172SLen Brown 	struct cpuidle_device *dev;
139926717172SLen Brown 
140065b7f839SThomas Renninger 	dev = per_cpu_ptr(intel_idle_cpuidle_devices, cpu);
140165b7f839SThomas Renninger 	dev->cpu = cpu;
140226717172SLen Brown 
140326717172SLen Brown 	if (cpuidle_register_device(dev)) {
1404654d08a4SJoe Perches 		pr_debug("cpuidle_register_device %d failed!\n", cpu);
140526717172SLen Brown 		return -EIO;
140626717172SLen Brown 	}
140765b7f839SThomas Renninger 
1408b66b8b9aSAndi Kleen 	if (icpu->auto_demotion_disable_flags)
1409fb1013a0SSebastian Andrzej Siewior 		auto_demotion_disable();
141026717172SLen Brown 
1411dbf87ab8SBartlomiej Zolnierkiewicz 	if (icpu->disable_promotion_to_c1e)
1412fb1013a0SSebastian Andrzej Siewior 		c1e_promotion_disable();
1413fb1013a0SSebastian Andrzej Siewior 
1414fb1013a0SSebastian Andrzej Siewior 	return 0;
1415fb1013a0SSebastian Andrzej Siewior }
1416fb1013a0SSebastian Andrzej Siewior 
1417fb1013a0SSebastian Andrzej Siewior static int intel_idle_cpu_online(unsigned int cpu)
1418fb1013a0SSebastian Andrzej Siewior {
1419fb1013a0SSebastian Andrzej Siewior 	struct cpuidle_device *dev;
1420fb1013a0SSebastian Andrzej Siewior 
1421fb1013a0SSebastian Andrzej Siewior 	if (lapic_timer_reliable_states != LAPIC_TIMER_ALWAYS_RELIABLE)
1422fb1013a0SSebastian Andrzej Siewior 		__setup_broadcast_timer(true);
1423fb1013a0SSebastian Andrzej Siewior 
1424fb1013a0SSebastian Andrzej Siewior 	/*
1425fb1013a0SSebastian Andrzej Siewior 	 * Some systems can hotplug a cpu at runtime after
1426fb1013a0SSebastian Andrzej Siewior 	 * the kernel has booted, we have to initialize the
1427fb1013a0SSebastian Andrzej Siewior 	 * driver in this case
1428fb1013a0SSebastian Andrzej Siewior 	 */
1429fb1013a0SSebastian Andrzej Siewior 	dev = per_cpu_ptr(intel_idle_cpuidle_devices, cpu);
1430fb1013a0SSebastian Andrzej Siewior 	if (!dev->registered)
1431fb1013a0SSebastian Andrzej Siewior 		return intel_idle_cpu_init(cpu);
1432dbf87ab8SBartlomiej Zolnierkiewicz 
143326717172SLen Brown 	return 0;
143426717172SLen Brown }
143526717172SLen Brown 
143626717172SLen Brown static int __init intel_idle_init(void)
143726717172SLen Brown {
1438fb1013a0SSebastian Andrzej Siewior 	int retval;
143926717172SLen Brown 
1440d1896049SThomas Renninger 	/* Do not load intel_idle at all for now if idle= is passed */
1441d1896049SThomas Renninger 	if (boot_option_idle_override != IDLE_NO_OVERRIDE)
1442d1896049SThomas Renninger 		return -ENODEV;
1443d1896049SThomas Renninger 
144426717172SLen Brown 	retval = intel_idle_probe();
144526717172SLen Brown 	if (retval)
144626717172SLen Brown 		return retval;
144726717172SLen Brown 
1448e9df69ccSRichard Cochran 	intel_idle_cpuidle_devices = alloc_percpu(struct cpuidle_device);
1449e9df69ccSRichard Cochran 	if (intel_idle_cpuidle_devices == NULL)
1450e9df69ccSRichard Cochran 		return -ENOMEM;
1451e9df69ccSRichard Cochran 
145246bcfad7SDeepthi Dharwar 	intel_idle_cpuidle_driver_init();
145326717172SLen Brown 	retval = cpuidle_register_driver(&intel_idle_driver);
145426717172SLen Brown 	if (retval) {
14553735d524SKonrad Rzeszutek Wilk 		struct cpuidle_driver *drv = cpuidle_get_driver();
1456654d08a4SJoe Perches 		printk(KERN_DEBUG pr_fmt("intel_idle yielding to %s\n"),
14573735d524SKonrad Rzeszutek Wilk 		       drv ? drv->name : "none");
1458fb1013a0SSebastian Andrzej Siewior 		goto init_driver_fail;
145926717172SLen Brown 	}
146026717172SLen Brown 
14612259a819SRichard Cochran 	if (boot_cpu_has(X86_FEATURE_ARAT))	/* Always Reliable APIC Timer */
14622259a819SRichard Cochran 		lapic_timer_reliable_states = LAPIC_TIMER_ALWAYS_RELIABLE;
14632259a819SRichard Cochran 
1464fb1013a0SSebastian Andrzej Siewior 	retval = cpuhp_setup_state(CPUHP_AP_ONLINE_DYN, "idle/intel:online",
1465fb1013a0SSebastian Andrzej Siewior 				   intel_idle_cpu_online, NULL);
1466fb1013a0SSebastian Andrzej Siewior 	if (retval < 0)
1467fb1013a0SSebastian Andrzej Siewior 		goto hp_setup_fail;
146826717172SLen Brown 
1469654d08a4SJoe Perches 	pr_debug("lapic_timer_reliable_states 0x%x\n",
14702259a819SRichard Cochran 		 lapic_timer_reliable_states);
14712259a819SRichard Cochran 
147226717172SLen Brown 	return 0;
1473fb1013a0SSebastian Andrzej Siewior 
1474fb1013a0SSebastian Andrzej Siewior hp_setup_fail:
1475fb1013a0SSebastian Andrzej Siewior 	intel_idle_cpuidle_devices_uninit();
1476fb1013a0SSebastian Andrzej Siewior 	cpuidle_unregister_driver(&intel_idle_driver);
1477fb1013a0SSebastian Andrzej Siewior init_driver_fail:
1478fb1013a0SSebastian Andrzej Siewior 	free_percpu(intel_idle_cpuidle_devices);
1479fb1013a0SSebastian Andrzej Siewior 	return retval;
1480fb1013a0SSebastian Andrzej Siewior 
148126717172SLen Brown }
148202c4fae9SPaul Gortmaker device_initcall(intel_idle_init);
148326717172SLen Brown 
148402c4fae9SPaul Gortmaker /*
148502c4fae9SPaul Gortmaker  * We are not really modular, but we used to support that.  Meaning we also
148602c4fae9SPaul Gortmaker  * support "intel_idle.max_cstate=..." at boot and also a read-only export of
148702c4fae9SPaul Gortmaker  * it at /sys/module/intel_idle/parameters/max_cstate -- so using module_param
148802c4fae9SPaul Gortmaker  * is the easiest way (currently) to continue doing that.
148902c4fae9SPaul Gortmaker  */
149026717172SLen Brown module_param(max_cstate, int, 0444);
1491