1 // SPDX-License-Identifier: GPL-2.0-only
2 /*
3 * intel_idle.c - native hardware idle loop for modern Intel processors
4 *
5 * Copyright (c) 2013 - 2020, Intel Corporation.
6 * Len Brown <len.brown@intel.com>
7 * Rafael J. Wysocki <rafael.j.wysocki@intel.com>
8 */
9
10 /*
11 * intel_idle is a cpuidle driver that loads on all Intel CPUs with MWAIT
12 * in lieu of the legacy ACPI processor_idle driver. The intent is to
13 * make Linux more efficient on these processors, as intel_idle knows
14 * more than ACPI, as well as make Linux more immune to ACPI BIOS bugs.
15 */
16
17 /*
18 * Design Assumptions
19 *
20 * All CPUs have same idle states as boot CPU
21 *
22 * Chipset BM_STS (bus master status) bit is a NOP
23 * for preventing entry into deep C-states
24 *
25 * CPU will flush caches as needed when entering a C-state via MWAIT
26 * (in contrast to entering ACPI C3, in which case the WBINVD
27 * instruction needs to be executed to flush the caches)
28 */
29
30 /*
31 * Known limitations
32 *
33 * ACPI has a .suspend hack to turn off deep c-statees during suspend
34 * to avoid complications with the lapic timer workaround.
35 * Have not seen issues with suspend, but may need same workaround here.
36 *
37 */
38
39 /* un-comment DEBUG to enable pr_debug() statements */
40 /* #define DEBUG */
41
42 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
43
44 #include <linux/acpi.h>
45 #include <linux/kernel.h>
46 #include <linux/cpuidle.h>
47 #include <linux/tick.h>
48 #include <trace/events/power.h>
49 #include <linux/sched.h>
50 #include <linux/sched/smt.h>
51 #include <linux/notifier.h>
52 #include <linux/cpu.h>
53 #include <linux/moduleparam.h>
54 #include <asm/cpu_device_id.h>
55 #include <asm/intel-family.h>
56 #include <asm/nospec-branch.h>
57 #include <asm/mwait.h>
58 #include <asm/msr.h>
59 #include <asm/fpu/api.h>
60
61 #define INTEL_IDLE_VERSION "0.5.1"
62
63 static struct cpuidle_driver intel_idle_driver = {
64 .name = "intel_idle",
65 .owner = THIS_MODULE,
66 };
67 /* intel_idle.max_cstate=0 disables driver */
68 static int max_cstate = CPUIDLE_STATE_MAX - 1;
69 static unsigned int disabled_states_mask __read_mostly;
70 static unsigned int preferred_states_mask __read_mostly;
71 static bool force_irq_on __read_mostly;
72
73 static struct cpuidle_device __percpu *intel_idle_cpuidle_devices;
74
75 static unsigned long auto_demotion_disable_flags;
76
77 static enum {
78 C1E_PROMOTION_PRESERVE,
79 C1E_PROMOTION_ENABLE,
80 C1E_PROMOTION_DISABLE
81 } c1e_promotion = C1E_PROMOTION_PRESERVE;
82
83 struct idle_cpu {
84 struct cpuidle_state *state_table;
85
86 /*
87 * Hardware C-state auto-demotion may not always be optimal.
88 * Indicate which enable bits to clear here.
89 */
90 unsigned long auto_demotion_disable_flags;
91 bool byt_auto_demotion_disable_flag;
92 bool disable_promotion_to_c1e;
93 bool use_acpi;
94 };
95
96 static const struct idle_cpu *icpu __initdata;
97 static struct cpuidle_state *cpuidle_state_table __initdata;
98
99 static unsigned int mwait_substates __initdata;
100
101 /*
102 * Enable interrupts before entering the C-state. On some platforms and for
103 * some C-states, this may measurably decrease interrupt latency.
104 */
105 #define CPUIDLE_FLAG_IRQ_ENABLE BIT(14)
106
107 /*
108 * Enable this state by default even if the ACPI _CST does not list it.
109 */
110 #define CPUIDLE_FLAG_ALWAYS_ENABLE BIT(15)
111
112 /*
113 * Disable IBRS across idle (when KERNEL_IBRS), is exclusive vs IRQ_ENABLE
114 * above.
115 */
116 #define CPUIDLE_FLAG_IBRS BIT(16)
117
118 /*
119 * Initialize large xstate for the C6-state entrance.
120 */
121 #define CPUIDLE_FLAG_INIT_XSTATE BIT(17)
122
123 /*
124 * MWAIT takes an 8-bit "hint" in EAX "suggesting"
125 * the C-state (top nibble) and sub-state (bottom nibble)
126 * 0x00 means "MWAIT(C1)", 0x10 means "MWAIT(C2)" etc.
127 *
128 * We store the hint at the top of our "flags" for each state.
129 */
130 #define flg2MWAIT(flags) (((flags) >> 24) & 0xFF)
131 #define MWAIT2flg(eax) ((eax & 0xFF) << 24)
132
__intel_idle(struct cpuidle_device * dev,struct cpuidle_driver * drv,int index,bool irqoff)133 static __always_inline int __intel_idle(struct cpuidle_device *dev,
134 struct cpuidle_driver *drv,
135 int index, bool irqoff)
136 {
137 struct cpuidle_state *state = &drv->states[index];
138 unsigned long eax = flg2MWAIT(state->flags);
139 unsigned long ecx = 1*irqoff; /* break on interrupt flag */
140
141 mwait_idle_with_hints(eax, ecx);
142
143 return index;
144 }
145
146 /**
147 * intel_idle - Ask the processor to enter the given idle state.
148 * @dev: cpuidle device of the target CPU.
149 * @drv: cpuidle driver (assumed to point to intel_idle_driver).
150 * @index: Target idle state index.
151 *
152 * Use the MWAIT instruction to notify the processor that the CPU represented by
153 * @dev is idle and it can try to enter the idle state corresponding to @index.
154 *
155 * If the local APIC timer is not known to be reliable in the target idle state,
156 * enable one-shot tick broadcasting for the target CPU before executing MWAIT.
157 *
158 * Must be called under local_irq_disable().
159 */
intel_idle(struct cpuidle_device * dev,struct cpuidle_driver * drv,int index)160 static __cpuidle int intel_idle(struct cpuidle_device *dev,
161 struct cpuidle_driver *drv, int index)
162 {
163 return __intel_idle(dev, drv, index, true);
164 }
165
intel_idle_irq(struct cpuidle_device * dev,struct cpuidle_driver * drv,int index)166 static __cpuidle int intel_idle_irq(struct cpuidle_device *dev,
167 struct cpuidle_driver *drv, int index)
168 {
169 return __intel_idle(dev, drv, index, false);
170 }
171
intel_idle_ibrs(struct cpuidle_device * dev,struct cpuidle_driver * drv,int index)172 static __cpuidle int intel_idle_ibrs(struct cpuidle_device *dev,
173 struct cpuidle_driver *drv, int index)
174 {
175 bool smt_active = sched_smt_active();
176 u64 spec_ctrl = spec_ctrl_current();
177 int ret;
178
179 if (smt_active)
180 native_wrmsrl(MSR_IA32_SPEC_CTRL, 0);
181
182 ret = __intel_idle(dev, drv, index, true);
183
184 if (smt_active)
185 native_wrmsrl(MSR_IA32_SPEC_CTRL, spec_ctrl);
186
187 return ret;
188 }
189
intel_idle_xstate(struct cpuidle_device * dev,struct cpuidle_driver * drv,int index)190 static __cpuidle int intel_idle_xstate(struct cpuidle_device *dev,
191 struct cpuidle_driver *drv, int index)
192 {
193 fpu_idle_fpregs();
194 return __intel_idle(dev, drv, index, true);
195 }
196
197 /**
198 * intel_idle_s2idle - Ask the processor to enter the given idle state.
199 * @dev: cpuidle device of the target CPU.
200 * @drv: cpuidle driver (assumed to point to intel_idle_driver).
201 * @index: Target idle state index.
202 *
203 * Use the MWAIT instruction to notify the processor that the CPU represented by
204 * @dev is idle and it can try to enter the idle state corresponding to @index.
205 *
206 * Invoked as a suspend-to-idle callback routine with frozen user space, frozen
207 * scheduler tick and suspended scheduler clock on the target CPU.
208 */
intel_idle_s2idle(struct cpuidle_device * dev,struct cpuidle_driver * drv,int index)209 static __cpuidle int intel_idle_s2idle(struct cpuidle_device *dev,
210 struct cpuidle_driver *drv, int index)
211 {
212 unsigned long ecx = 1; /* break on interrupt flag */
213 struct cpuidle_state *state = &drv->states[index];
214 unsigned long eax = flg2MWAIT(state->flags);
215
216 if (state->flags & CPUIDLE_FLAG_INIT_XSTATE)
217 fpu_idle_fpregs();
218
219 mwait_idle_with_hints(eax, ecx);
220
221 return 0;
222 }
223
224 /*
225 * States are indexed by the cstate number,
226 * which is also the index into the MWAIT hint array.
227 * Thus C0 is a dummy.
228 */
229 static struct cpuidle_state nehalem_cstates[] __initdata = {
230 {
231 .name = "C1",
232 .desc = "MWAIT 0x00",
233 .flags = MWAIT2flg(0x00),
234 .exit_latency = 3,
235 .target_residency = 6,
236 .enter = &intel_idle,
237 .enter_s2idle = intel_idle_s2idle, },
238 {
239 .name = "C1E",
240 .desc = "MWAIT 0x01",
241 .flags = MWAIT2flg(0x01) | CPUIDLE_FLAG_ALWAYS_ENABLE,
242 .exit_latency = 10,
243 .target_residency = 20,
244 .enter = &intel_idle,
245 .enter_s2idle = intel_idle_s2idle, },
246 {
247 .name = "C3",
248 .desc = "MWAIT 0x10",
249 .flags = MWAIT2flg(0x10) | CPUIDLE_FLAG_TLB_FLUSHED,
250 .exit_latency = 20,
251 .target_residency = 80,
252 .enter = &intel_idle,
253 .enter_s2idle = intel_idle_s2idle, },
254 {
255 .name = "C6",
256 .desc = "MWAIT 0x20",
257 .flags = MWAIT2flg(0x20) | CPUIDLE_FLAG_TLB_FLUSHED,
258 .exit_latency = 200,
259 .target_residency = 800,
260 .enter = &intel_idle,
261 .enter_s2idle = intel_idle_s2idle, },
262 {
263 .enter = NULL }
264 };
265
266 static struct cpuidle_state snb_cstates[] __initdata = {
267 {
268 .name = "C1",
269 .desc = "MWAIT 0x00",
270 .flags = MWAIT2flg(0x00),
271 .exit_latency = 2,
272 .target_residency = 2,
273 .enter = &intel_idle,
274 .enter_s2idle = intel_idle_s2idle, },
275 {
276 .name = "C1E",
277 .desc = "MWAIT 0x01",
278 .flags = MWAIT2flg(0x01) | CPUIDLE_FLAG_ALWAYS_ENABLE,
279 .exit_latency = 10,
280 .target_residency = 20,
281 .enter = &intel_idle,
282 .enter_s2idle = intel_idle_s2idle, },
283 {
284 .name = "C3",
285 .desc = "MWAIT 0x10",
286 .flags = MWAIT2flg(0x10) | CPUIDLE_FLAG_TLB_FLUSHED,
287 .exit_latency = 80,
288 .target_residency = 211,
289 .enter = &intel_idle,
290 .enter_s2idle = intel_idle_s2idle, },
291 {
292 .name = "C6",
293 .desc = "MWAIT 0x20",
294 .flags = MWAIT2flg(0x20) | CPUIDLE_FLAG_TLB_FLUSHED,
295 .exit_latency = 104,
296 .target_residency = 345,
297 .enter = &intel_idle,
298 .enter_s2idle = intel_idle_s2idle, },
299 {
300 .name = "C7",
301 .desc = "MWAIT 0x30",
302 .flags = MWAIT2flg(0x30) | CPUIDLE_FLAG_TLB_FLUSHED,
303 .exit_latency = 109,
304 .target_residency = 345,
305 .enter = &intel_idle,
306 .enter_s2idle = intel_idle_s2idle, },
307 {
308 .enter = NULL }
309 };
310
311 static struct cpuidle_state byt_cstates[] __initdata = {
312 {
313 .name = "C1",
314 .desc = "MWAIT 0x00",
315 .flags = MWAIT2flg(0x00),
316 .exit_latency = 1,
317 .target_residency = 1,
318 .enter = &intel_idle,
319 .enter_s2idle = intel_idle_s2idle, },
320 {
321 .name = "C6N",
322 .desc = "MWAIT 0x58",
323 .flags = MWAIT2flg(0x58) | CPUIDLE_FLAG_TLB_FLUSHED,
324 .exit_latency = 300,
325 .target_residency = 275,
326 .enter = &intel_idle,
327 .enter_s2idle = intel_idle_s2idle, },
328 {
329 .name = "C6S",
330 .desc = "MWAIT 0x52",
331 .flags = MWAIT2flg(0x52) | CPUIDLE_FLAG_TLB_FLUSHED,
332 .exit_latency = 500,
333 .target_residency = 560,
334 .enter = &intel_idle,
335 .enter_s2idle = intel_idle_s2idle, },
336 {
337 .name = "C7",
338 .desc = "MWAIT 0x60",
339 .flags = MWAIT2flg(0x60) | CPUIDLE_FLAG_TLB_FLUSHED,
340 .exit_latency = 1200,
341 .target_residency = 4000,
342 .enter = &intel_idle,
343 .enter_s2idle = intel_idle_s2idle, },
344 {
345 .name = "C7S",
346 .desc = "MWAIT 0x64",
347 .flags = MWAIT2flg(0x64) | CPUIDLE_FLAG_TLB_FLUSHED,
348 .exit_latency = 10000,
349 .target_residency = 20000,
350 .enter = &intel_idle,
351 .enter_s2idle = intel_idle_s2idle, },
352 {
353 .enter = NULL }
354 };
355
356 static struct cpuidle_state cht_cstates[] __initdata = {
357 {
358 .name = "C1",
359 .desc = "MWAIT 0x00",
360 .flags = MWAIT2flg(0x00),
361 .exit_latency = 1,
362 .target_residency = 1,
363 .enter = &intel_idle,
364 .enter_s2idle = intel_idle_s2idle, },
365 {
366 .name = "C6N",
367 .desc = "MWAIT 0x58",
368 .flags = MWAIT2flg(0x58) | CPUIDLE_FLAG_TLB_FLUSHED,
369 .exit_latency = 80,
370 .target_residency = 275,
371 .enter = &intel_idle,
372 .enter_s2idle = intel_idle_s2idle, },
373 {
374 .name = "C6S",
375 .desc = "MWAIT 0x52",
376 .flags = MWAIT2flg(0x52) | CPUIDLE_FLAG_TLB_FLUSHED,
377 .exit_latency = 200,
378 .target_residency = 560,
379 .enter = &intel_idle,
380 .enter_s2idle = intel_idle_s2idle, },
381 {
382 .name = "C7",
383 .desc = "MWAIT 0x60",
384 .flags = MWAIT2flg(0x60) | CPUIDLE_FLAG_TLB_FLUSHED,
385 .exit_latency = 1200,
386 .target_residency = 4000,
387 .enter = &intel_idle,
388 .enter_s2idle = intel_idle_s2idle, },
389 {
390 .name = "C7S",
391 .desc = "MWAIT 0x64",
392 .flags = MWAIT2flg(0x64) | CPUIDLE_FLAG_TLB_FLUSHED,
393 .exit_latency = 10000,
394 .target_residency = 20000,
395 .enter = &intel_idle,
396 .enter_s2idle = intel_idle_s2idle, },
397 {
398 .enter = NULL }
399 };
400
401 static struct cpuidle_state ivb_cstates[] __initdata = {
402 {
403 .name = "C1",
404 .desc = "MWAIT 0x00",
405 .flags = MWAIT2flg(0x00),
406 .exit_latency = 1,
407 .target_residency = 1,
408 .enter = &intel_idle,
409 .enter_s2idle = intel_idle_s2idle, },
410 {
411 .name = "C1E",
412 .desc = "MWAIT 0x01",
413 .flags = MWAIT2flg(0x01) | CPUIDLE_FLAG_ALWAYS_ENABLE,
414 .exit_latency = 10,
415 .target_residency = 20,
416 .enter = &intel_idle,
417 .enter_s2idle = intel_idle_s2idle, },
418 {
419 .name = "C3",
420 .desc = "MWAIT 0x10",
421 .flags = MWAIT2flg(0x10) | CPUIDLE_FLAG_TLB_FLUSHED,
422 .exit_latency = 59,
423 .target_residency = 156,
424 .enter = &intel_idle,
425 .enter_s2idle = intel_idle_s2idle, },
426 {
427 .name = "C6",
428 .desc = "MWAIT 0x20",
429 .flags = MWAIT2flg(0x20) | CPUIDLE_FLAG_TLB_FLUSHED,
430 .exit_latency = 80,
431 .target_residency = 300,
432 .enter = &intel_idle,
433 .enter_s2idle = intel_idle_s2idle, },
434 {
435 .name = "C7",
436 .desc = "MWAIT 0x30",
437 .flags = MWAIT2flg(0x30) | CPUIDLE_FLAG_TLB_FLUSHED,
438 .exit_latency = 87,
439 .target_residency = 300,
440 .enter = &intel_idle,
441 .enter_s2idle = intel_idle_s2idle, },
442 {
443 .enter = NULL }
444 };
445
446 static struct cpuidle_state ivt_cstates[] __initdata = {
447 {
448 .name = "C1",
449 .desc = "MWAIT 0x00",
450 .flags = MWAIT2flg(0x00),
451 .exit_latency = 1,
452 .target_residency = 1,
453 .enter = &intel_idle,
454 .enter_s2idle = intel_idle_s2idle, },
455 {
456 .name = "C1E",
457 .desc = "MWAIT 0x01",
458 .flags = MWAIT2flg(0x01) | CPUIDLE_FLAG_ALWAYS_ENABLE,
459 .exit_latency = 10,
460 .target_residency = 80,
461 .enter = &intel_idle,
462 .enter_s2idle = intel_idle_s2idle, },
463 {
464 .name = "C3",
465 .desc = "MWAIT 0x10",
466 .flags = MWAIT2flg(0x10) | CPUIDLE_FLAG_TLB_FLUSHED,
467 .exit_latency = 59,
468 .target_residency = 156,
469 .enter = &intel_idle,
470 .enter_s2idle = intel_idle_s2idle, },
471 {
472 .name = "C6",
473 .desc = "MWAIT 0x20",
474 .flags = MWAIT2flg(0x20) | CPUIDLE_FLAG_TLB_FLUSHED,
475 .exit_latency = 82,
476 .target_residency = 300,
477 .enter = &intel_idle,
478 .enter_s2idle = intel_idle_s2idle, },
479 {
480 .enter = NULL }
481 };
482
483 static struct cpuidle_state ivt_cstates_4s[] __initdata = {
484 {
485 .name = "C1",
486 .desc = "MWAIT 0x00",
487 .flags = MWAIT2flg(0x00),
488 .exit_latency = 1,
489 .target_residency = 1,
490 .enter = &intel_idle,
491 .enter_s2idle = intel_idle_s2idle, },
492 {
493 .name = "C1E",
494 .desc = "MWAIT 0x01",
495 .flags = MWAIT2flg(0x01) | CPUIDLE_FLAG_ALWAYS_ENABLE,
496 .exit_latency = 10,
497 .target_residency = 250,
498 .enter = &intel_idle,
499 .enter_s2idle = intel_idle_s2idle, },
500 {
501 .name = "C3",
502 .desc = "MWAIT 0x10",
503 .flags = MWAIT2flg(0x10) | CPUIDLE_FLAG_TLB_FLUSHED,
504 .exit_latency = 59,
505 .target_residency = 300,
506 .enter = &intel_idle,
507 .enter_s2idle = intel_idle_s2idle, },
508 {
509 .name = "C6",
510 .desc = "MWAIT 0x20",
511 .flags = MWAIT2flg(0x20) | CPUIDLE_FLAG_TLB_FLUSHED,
512 .exit_latency = 84,
513 .target_residency = 400,
514 .enter = &intel_idle,
515 .enter_s2idle = intel_idle_s2idle, },
516 {
517 .enter = NULL }
518 };
519
520 static struct cpuidle_state ivt_cstates_8s[] __initdata = {
521 {
522 .name = "C1",
523 .desc = "MWAIT 0x00",
524 .flags = MWAIT2flg(0x00),
525 .exit_latency = 1,
526 .target_residency = 1,
527 .enter = &intel_idle,
528 .enter_s2idle = intel_idle_s2idle, },
529 {
530 .name = "C1E",
531 .desc = "MWAIT 0x01",
532 .flags = MWAIT2flg(0x01) | CPUIDLE_FLAG_ALWAYS_ENABLE,
533 .exit_latency = 10,
534 .target_residency = 500,
535 .enter = &intel_idle,
536 .enter_s2idle = intel_idle_s2idle, },
537 {
538 .name = "C3",
539 .desc = "MWAIT 0x10",
540 .flags = MWAIT2flg(0x10) | CPUIDLE_FLAG_TLB_FLUSHED,
541 .exit_latency = 59,
542 .target_residency = 600,
543 .enter = &intel_idle,
544 .enter_s2idle = intel_idle_s2idle, },
545 {
546 .name = "C6",
547 .desc = "MWAIT 0x20",
548 .flags = MWAIT2flg(0x20) | CPUIDLE_FLAG_TLB_FLUSHED,
549 .exit_latency = 88,
550 .target_residency = 700,
551 .enter = &intel_idle,
552 .enter_s2idle = intel_idle_s2idle, },
553 {
554 .enter = NULL }
555 };
556
557 static struct cpuidle_state hsw_cstates[] __initdata = {
558 {
559 .name = "C1",
560 .desc = "MWAIT 0x00",
561 .flags = MWAIT2flg(0x00),
562 .exit_latency = 2,
563 .target_residency = 2,
564 .enter = &intel_idle,
565 .enter_s2idle = intel_idle_s2idle, },
566 {
567 .name = "C1E",
568 .desc = "MWAIT 0x01",
569 .flags = MWAIT2flg(0x01) | CPUIDLE_FLAG_ALWAYS_ENABLE,
570 .exit_latency = 10,
571 .target_residency = 20,
572 .enter = &intel_idle,
573 .enter_s2idle = intel_idle_s2idle, },
574 {
575 .name = "C3",
576 .desc = "MWAIT 0x10",
577 .flags = MWAIT2flg(0x10) | CPUIDLE_FLAG_TLB_FLUSHED,
578 .exit_latency = 33,
579 .target_residency = 100,
580 .enter = &intel_idle,
581 .enter_s2idle = intel_idle_s2idle, },
582 {
583 .name = "C6",
584 .desc = "MWAIT 0x20",
585 .flags = MWAIT2flg(0x20) | CPUIDLE_FLAG_TLB_FLUSHED,
586 .exit_latency = 133,
587 .target_residency = 400,
588 .enter = &intel_idle,
589 .enter_s2idle = intel_idle_s2idle, },
590 {
591 .name = "C7s",
592 .desc = "MWAIT 0x32",
593 .flags = MWAIT2flg(0x32) | CPUIDLE_FLAG_TLB_FLUSHED,
594 .exit_latency = 166,
595 .target_residency = 500,
596 .enter = &intel_idle,
597 .enter_s2idle = intel_idle_s2idle, },
598 {
599 .name = "C8",
600 .desc = "MWAIT 0x40",
601 .flags = MWAIT2flg(0x40) | CPUIDLE_FLAG_TLB_FLUSHED,
602 .exit_latency = 300,
603 .target_residency = 900,
604 .enter = &intel_idle,
605 .enter_s2idle = intel_idle_s2idle, },
606 {
607 .name = "C9",
608 .desc = "MWAIT 0x50",
609 .flags = MWAIT2flg(0x50) | CPUIDLE_FLAG_TLB_FLUSHED,
610 .exit_latency = 600,
611 .target_residency = 1800,
612 .enter = &intel_idle,
613 .enter_s2idle = intel_idle_s2idle, },
614 {
615 .name = "C10",
616 .desc = "MWAIT 0x60",
617 .flags = MWAIT2flg(0x60) | CPUIDLE_FLAG_TLB_FLUSHED,
618 .exit_latency = 2600,
619 .target_residency = 7700,
620 .enter = &intel_idle,
621 .enter_s2idle = intel_idle_s2idle, },
622 {
623 .enter = NULL }
624 };
625 static struct cpuidle_state bdw_cstates[] __initdata = {
626 {
627 .name = "C1",
628 .desc = "MWAIT 0x00",
629 .flags = MWAIT2flg(0x00),
630 .exit_latency = 2,
631 .target_residency = 2,
632 .enter = &intel_idle,
633 .enter_s2idle = intel_idle_s2idle, },
634 {
635 .name = "C1E",
636 .desc = "MWAIT 0x01",
637 .flags = MWAIT2flg(0x01) | CPUIDLE_FLAG_ALWAYS_ENABLE,
638 .exit_latency = 10,
639 .target_residency = 20,
640 .enter = &intel_idle,
641 .enter_s2idle = intel_idle_s2idle, },
642 {
643 .name = "C3",
644 .desc = "MWAIT 0x10",
645 .flags = MWAIT2flg(0x10) | CPUIDLE_FLAG_TLB_FLUSHED,
646 .exit_latency = 40,
647 .target_residency = 100,
648 .enter = &intel_idle,
649 .enter_s2idle = intel_idle_s2idle, },
650 {
651 .name = "C6",
652 .desc = "MWAIT 0x20",
653 .flags = MWAIT2flg(0x20) | CPUIDLE_FLAG_TLB_FLUSHED,
654 .exit_latency = 133,
655 .target_residency = 400,
656 .enter = &intel_idle,
657 .enter_s2idle = intel_idle_s2idle, },
658 {
659 .name = "C7s",
660 .desc = "MWAIT 0x32",
661 .flags = MWAIT2flg(0x32) | CPUIDLE_FLAG_TLB_FLUSHED,
662 .exit_latency = 166,
663 .target_residency = 500,
664 .enter = &intel_idle,
665 .enter_s2idle = intel_idle_s2idle, },
666 {
667 .name = "C8",
668 .desc = "MWAIT 0x40",
669 .flags = MWAIT2flg(0x40) | CPUIDLE_FLAG_TLB_FLUSHED,
670 .exit_latency = 300,
671 .target_residency = 900,
672 .enter = &intel_idle,
673 .enter_s2idle = intel_idle_s2idle, },
674 {
675 .name = "C9",
676 .desc = "MWAIT 0x50",
677 .flags = MWAIT2flg(0x50) | CPUIDLE_FLAG_TLB_FLUSHED,
678 .exit_latency = 600,
679 .target_residency = 1800,
680 .enter = &intel_idle,
681 .enter_s2idle = intel_idle_s2idle, },
682 {
683 .name = "C10",
684 .desc = "MWAIT 0x60",
685 .flags = MWAIT2flg(0x60) | CPUIDLE_FLAG_TLB_FLUSHED,
686 .exit_latency = 2600,
687 .target_residency = 7700,
688 .enter = &intel_idle,
689 .enter_s2idle = intel_idle_s2idle, },
690 {
691 .enter = NULL }
692 };
693
694 static struct cpuidle_state skl_cstates[] __initdata = {
695 {
696 .name = "C1",
697 .desc = "MWAIT 0x00",
698 .flags = MWAIT2flg(0x00),
699 .exit_latency = 2,
700 .target_residency = 2,
701 .enter = &intel_idle,
702 .enter_s2idle = intel_idle_s2idle, },
703 {
704 .name = "C1E",
705 .desc = "MWAIT 0x01",
706 .flags = MWAIT2flg(0x01) | CPUIDLE_FLAG_ALWAYS_ENABLE,
707 .exit_latency = 10,
708 .target_residency = 20,
709 .enter = &intel_idle,
710 .enter_s2idle = intel_idle_s2idle, },
711 {
712 .name = "C3",
713 .desc = "MWAIT 0x10",
714 .flags = MWAIT2flg(0x10) | CPUIDLE_FLAG_TLB_FLUSHED,
715 .exit_latency = 70,
716 .target_residency = 100,
717 .enter = &intel_idle,
718 .enter_s2idle = intel_idle_s2idle, },
719 {
720 .name = "C6",
721 .desc = "MWAIT 0x20",
722 .flags = MWAIT2flg(0x20) | CPUIDLE_FLAG_TLB_FLUSHED | CPUIDLE_FLAG_IBRS,
723 .exit_latency = 85,
724 .target_residency = 200,
725 .enter = &intel_idle,
726 .enter_s2idle = intel_idle_s2idle, },
727 {
728 .name = "C7s",
729 .desc = "MWAIT 0x33",
730 .flags = MWAIT2flg(0x33) | CPUIDLE_FLAG_TLB_FLUSHED | CPUIDLE_FLAG_IBRS,
731 .exit_latency = 124,
732 .target_residency = 800,
733 .enter = &intel_idle,
734 .enter_s2idle = intel_idle_s2idle, },
735 {
736 .name = "C8",
737 .desc = "MWAIT 0x40",
738 .flags = MWAIT2flg(0x40) | CPUIDLE_FLAG_TLB_FLUSHED | CPUIDLE_FLAG_IBRS,
739 .exit_latency = 200,
740 .target_residency = 800,
741 .enter = &intel_idle,
742 .enter_s2idle = intel_idle_s2idle, },
743 {
744 .name = "C9",
745 .desc = "MWAIT 0x50",
746 .flags = MWAIT2flg(0x50) | CPUIDLE_FLAG_TLB_FLUSHED | CPUIDLE_FLAG_IBRS,
747 .exit_latency = 480,
748 .target_residency = 5000,
749 .enter = &intel_idle,
750 .enter_s2idle = intel_idle_s2idle, },
751 {
752 .name = "C10",
753 .desc = "MWAIT 0x60",
754 .flags = MWAIT2flg(0x60) | CPUIDLE_FLAG_TLB_FLUSHED | CPUIDLE_FLAG_IBRS,
755 .exit_latency = 890,
756 .target_residency = 5000,
757 .enter = &intel_idle,
758 .enter_s2idle = intel_idle_s2idle, },
759 {
760 .enter = NULL }
761 };
762
763 static struct cpuidle_state skx_cstates[] __initdata = {
764 {
765 .name = "C1",
766 .desc = "MWAIT 0x00",
767 .flags = MWAIT2flg(0x00) | CPUIDLE_FLAG_IRQ_ENABLE,
768 .exit_latency = 2,
769 .target_residency = 2,
770 .enter = &intel_idle,
771 .enter_s2idle = intel_idle_s2idle, },
772 {
773 .name = "C1E",
774 .desc = "MWAIT 0x01",
775 .flags = MWAIT2flg(0x01) | CPUIDLE_FLAG_ALWAYS_ENABLE,
776 .exit_latency = 10,
777 .target_residency = 20,
778 .enter = &intel_idle,
779 .enter_s2idle = intel_idle_s2idle, },
780 {
781 .name = "C6",
782 .desc = "MWAIT 0x20",
783 .flags = MWAIT2flg(0x20) | CPUIDLE_FLAG_TLB_FLUSHED | CPUIDLE_FLAG_IBRS,
784 .exit_latency = 133,
785 .target_residency = 600,
786 .enter = &intel_idle,
787 .enter_s2idle = intel_idle_s2idle, },
788 {
789 .enter = NULL }
790 };
791
792 static struct cpuidle_state icx_cstates[] __initdata = {
793 {
794 .name = "C1",
795 .desc = "MWAIT 0x00",
796 .flags = MWAIT2flg(0x00) | CPUIDLE_FLAG_IRQ_ENABLE,
797 .exit_latency = 1,
798 .target_residency = 1,
799 .enter = &intel_idle,
800 .enter_s2idle = intel_idle_s2idle, },
801 {
802 .name = "C1E",
803 .desc = "MWAIT 0x01",
804 .flags = MWAIT2flg(0x01) | CPUIDLE_FLAG_ALWAYS_ENABLE,
805 .exit_latency = 4,
806 .target_residency = 4,
807 .enter = &intel_idle,
808 .enter_s2idle = intel_idle_s2idle, },
809 {
810 .name = "C6",
811 .desc = "MWAIT 0x20",
812 .flags = MWAIT2flg(0x20) | CPUIDLE_FLAG_TLB_FLUSHED,
813 .exit_latency = 170,
814 .target_residency = 600,
815 .enter = &intel_idle,
816 .enter_s2idle = intel_idle_s2idle, },
817 {
818 .enter = NULL }
819 };
820
821 /*
822 * On AlderLake C1 has to be disabled if C1E is enabled, and vice versa.
823 * C1E is enabled only if "C1E promotion" bit is set in MSR_IA32_POWER_CTL.
824 * But in this case there is effectively no C1, because C1 requests are
825 * promoted to C1E. If the "C1E promotion" bit is cleared, then both C1
826 * and C1E requests end up with C1, so there is effectively no C1E.
827 *
828 * By default we enable C1E and disable C1 by marking it with
829 * 'CPUIDLE_FLAG_UNUSABLE'.
830 */
831 static struct cpuidle_state adl_cstates[] __initdata = {
832 {
833 .name = "C1",
834 .desc = "MWAIT 0x00",
835 .flags = MWAIT2flg(0x00) | CPUIDLE_FLAG_UNUSABLE,
836 .exit_latency = 1,
837 .target_residency = 1,
838 .enter = &intel_idle,
839 .enter_s2idle = intel_idle_s2idle, },
840 {
841 .name = "C1E",
842 .desc = "MWAIT 0x01",
843 .flags = MWAIT2flg(0x01) | CPUIDLE_FLAG_ALWAYS_ENABLE,
844 .exit_latency = 2,
845 .target_residency = 4,
846 .enter = &intel_idle,
847 .enter_s2idle = intel_idle_s2idle, },
848 {
849 .name = "C6",
850 .desc = "MWAIT 0x20",
851 .flags = MWAIT2flg(0x20) | CPUIDLE_FLAG_TLB_FLUSHED,
852 .exit_latency = 220,
853 .target_residency = 600,
854 .enter = &intel_idle,
855 .enter_s2idle = intel_idle_s2idle, },
856 {
857 .name = "C8",
858 .desc = "MWAIT 0x40",
859 .flags = MWAIT2flg(0x40) | CPUIDLE_FLAG_TLB_FLUSHED,
860 .exit_latency = 280,
861 .target_residency = 800,
862 .enter = &intel_idle,
863 .enter_s2idle = intel_idle_s2idle, },
864 {
865 .name = "C10",
866 .desc = "MWAIT 0x60",
867 .flags = MWAIT2flg(0x60) | CPUIDLE_FLAG_TLB_FLUSHED,
868 .exit_latency = 680,
869 .target_residency = 2000,
870 .enter = &intel_idle,
871 .enter_s2idle = intel_idle_s2idle, },
872 {
873 .enter = NULL }
874 };
875
876 static struct cpuidle_state adl_l_cstates[] __initdata = {
877 {
878 .name = "C1",
879 .desc = "MWAIT 0x00",
880 .flags = MWAIT2flg(0x00) | CPUIDLE_FLAG_UNUSABLE,
881 .exit_latency = 1,
882 .target_residency = 1,
883 .enter = &intel_idle,
884 .enter_s2idle = intel_idle_s2idle, },
885 {
886 .name = "C1E",
887 .desc = "MWAIT 0x01",
888 .flags = MWAIT2flg(0x01) | CPUIDLE_FLAG_ALWAYS_ENABLE,
889 .exit_latency = 2,
890 .target_residency = 4,
891 .enter = &intel_idle,
892 .enter_s2idle = intel_idle_s2idle, },
893 {
894 .name = "C6",
895 .desc = "MWAIT 0x20",
896 .flags = MWAIT2flg(0x20) | CPUIDLE_FLAG_TLB_FLUSHED,
897 .exit_latency = 170,
898 .target_residency = 500,
899 .enter = &intel_idle,
900 .enter_s2idle = intel_idle_s2idle, },
901 {
902 .name = "C8",
903 .desc = "MWAIT 0x40",
904 .flags = MWAIT2flg(0x40) | CPUIDLE_FLAG_TLB_FLUSHED,
905 .exit_latency = 200,
906 .target_residency = 600,
907 .enter = &intel_idle,
908 .enter_s2idle = intel_idle_s2idle, },
909 {
910 .name = "C10",
911 .desc = "MWAIT 0x60",
912 .flags = MWAIT2flg(0x60) | CPUIDLE_FLAG_TLB_FLUSHED,
913 .exit_latency = 230,
914 .target_residency = 700,
915 .enter = &intel_idle,
916 .enter_s2idle = intel_idle_s2idle, },
917 {
918 .enter = NULL }
919 };
920
921 static struct cpuidle_state gmt_cstates[] __initdata = {
922 {
923 .name = "C1",
924 .desc = "MWAIT 0x00",
925 .flags = MWAIT2flg(0x00) | CPUIDLE_FLAG_UNUSABLE,
926 .exit_latency = 1,
927 .target_residency = 1,
928 .enter = &intel_idle,
929 .enter_s2idle = intel_idle_s2idle, },
930 {
931 .name = "C1E",
932 .desc = "MWAIT 0x01",
933 .flags = MWAIT2flg(0x01) | CPUIDLE_FLAG_ALWAYS_ENABLE,
934 .exit_latency = 2,
935 .target_residency = 4,
936 .enter = &intel_idle,
937 .enter_s2idle = intel_idle_s2idle, },
938 {
939 .name = "C6",
940 .desc = "MWAIT 0x20",
941 .flags = MWAIT2flg(0x20) | CPUIDLE_FLAG_TLB_FLUSHED,
942 .exit_latency = 195,
943 .target_residency = 585,
944 .enter = &intel_idle,
945 .enter_s2idle = intel_idle_s2idle, },
946 {
947 .name = "C8",
948 .desc = "MWAIT 0x40",
949 .flags = MWAIT2flg(0x40) | CPUIDLE_FLAG_TLB_FLUSHED,
950 .exit_latency = 260,
951 .target_residency = 1040,
952 .enter = &intel_idle,
953 .enter_s2idle = intel_idle_s2idle, },
954 {
955 .name = "C10",
956 .desc = "MWAIT 0x60",
957 .flags = MWAIT2flg(0x60) | CPUIDLE_FLAG_TLB_FLUSHED,
958 .exit_latency = 660,
959 .target_residency = 1980,
960 .enter = &intel_idle,
961 .enter_s2idle = intel_idle_s2idle, },
962 {
963 .enter = NULL }
964 };
965
966 static struct cpuidle_state spr_cstates[] __initdata = {
967 {
968 .name = "C1",
969 .desc = "MWAIT 0x00",
970 .flags = MWAIT2flg(0x00),
971 .exit_latency = 1,
972 .target_residency = 1,
973 .enter = &intel_idle,
974 .enter_s2idle = intel_idle_s2idle, },
975 {
976 .name = "C1E",
977 .desc = "MWAIT 0x01",
978 .flags = MWAIT2flg(0x01) | CPUIDLE_FLAG_ALWAYS_ENABLE,
979 .exit_latency = 2,
980 .target_residency = 4,
981 .enter = &intel_idle,
982 .enter_s2idle = intel_idle_s2idle, },
983 {
984 .name = "C6",
985 .desc = "MWAIT 0x20",
986 .flags = MWAIT2flg(0x20) | CPUIDLE_FLAG_TLB_FLUSHED |
987 CPUIDLE_FLAG_INIT_XSTATE,
988 .exit_latency = 290,
989 .target_residency = 800,
990 .enter = &intel_idle,
991 .enter_s2idle = intel_idle_s2idle, },
992 {
993 .enter = NULL }
994 };
995
996 static struct cpuidle_state atom_cstates[] __initdata = {
997 {
998 .name = "C1E",
999 .desc = "MWAIT 0x00",
1000 .flags = MWAIT2flg(0x00),
1001 .exit_latency = 10,
1002 .target_residency = 20,
1003 .enter = &intel_idle,
1004 .enter_s2idle = intel_idle_s2idle, },
1005 {
1006 .name = "C2",
1007 .desc = "MWAIT 0x10",
1008 .flags = MWAIT2flg(0x10),
1009 .exit_latency = 20,
1010 .target_residency = 80,
1011 .enter = &intel_idle,
1012 .enter_s2idle = intel_idle_s2idle, },
1013 {
1014 .name = "C4",
1015 .desc = "MWAIT 0x30",
1016 .flags = MWAIT2flg(0x30) | CPUIDLE_FLAG_TLB_FLUSHED,
1017 .exit_latency = 100,
1018 .target_residency = 400,
1019 .enter = &intel_idle,
1020 .enter_s2idle = intel_idle_s2idle, },
1021 {
1022 .name = "C6",
1023 .desc = "MWAIT 0x52",
1024 .flags = MWAIT2flg(0x52) | CPUIDLE_FLAG_TLB_FLUSHED,
1025 .exit_latency = 140,
1026 .target_residency = 560,
1027 .enter = &intel_idle,
1028 .enter_s2idle = intel_idle_s2idle, },
1029 {
1030 .enter = NULL }
1031 };
1032 static struct cpuidle_state tangier_cstates[] __initdata = {
1033 {
1034 .name = "C1",
1035 .desc = "MWAIT 0x00",
1036 .flags = MWAIT2flg(0x00),
1037 .exit_latency = 1,
1038 .target_residency = 4,
1039 .enter = &intel_idle,
1040 .enter_s2idle = intel_idle_s2idle, },
1041 {
1042 .name = "C4",
1043 .desc = "MWAIT 0x30",
1044 .flags = MWAIT2flg(0x30) | CPUIDLE_FLAG_TLB_FLUSHED,
1045 .exit_latency = 100,
1046 .target_residency = 400,
1047 .enter = &intel_idle,
1048 .enter_s2idle = intel_idle_s2idle, },
1049 {
1050 .name = "C6",
1051 .desc = "MWAIT 0x52",
1052 .flags = MWAIT2flg(0x52) | CPUIDLE_FLAG_TLB_FLUSHED,
1053 .exit_latency = 140,
1054 .target_residency = 560,
1055 .enter = &intel_idle,
1056 .enter_s2idle = intel_idle_s2idle, },
1057 {
1058 .name = "C7",
1059 .desc = "MWAIT 0x60",
1060 .flags = MWAIT2flg(0x60) | CPUIDLE_FLAG_TLB_FLUSHED,
1061 .exit_latency = 1200,
1062 .target_residency = 4000,
1063 .enter = &intel_idle,
1064 .enter_s2idle = intel_idle_s2idle, },
1065 {
1066 .name = "C9",
1067 .desc = "MWAIT 0x64",
1068 .flags = MWAIT2flg(0x64) | CPUIDLE_FLAG_TLB_FLUSHED,
1069 .exit_latency = 10000,
1070 .target_residency = 20000,
1071 .enter = &intel_idle,
1072 .enter_s2idle = intel_idle_s2idle, },
1073 {
1074 .enter = NULL }
1075 };
1076 static struct cpuidle_state avn_cstates[] __initdata = {
1077 {
1078 .name = "C1",
1079 .desc = "MWAIT 0x00",
1080 .flags = MWAIT2flg(0x00),
1081 .exit_latency = 2,
1082 .target_residency = 2,
1083 .enter = &intel_idle,
1084 .enter_s2idle = intel_idle_s2idle, },
1085 {
1086 .name = "C6",
1087 .desc = "MWAIT 0x51",
1088 .flags = MWAIT2flg(0x51) | CPUIDLE_FLAG_TLB_FLUSHED,
1089 .exit_latency = 15,
1090 .target_residency = 45,
1091 .enter = &intel_idle,
1092 .enter_s2idle = intel_idle_s2idle, },
1093 {
1094 .enter = NULL }
1095 };
1096 static struct cpuidle_state knl_cstates[] __initdata = {
1097 {
1098 .name = "C1",
1099 .desc = "MWAIT 0x00",
1100 .flags = MWAIT2flg(0x00),
1101 .exit_latency = 1,
1102 .target_residency = 2,
1103 .enter = &intel_idle,
1104 .enter_s2idle = intel_idle_s2idle },
1105 {
1106 .name = "C6",
1107 .desc = "MWAIT 0x10",
1108 .flags = MWAIT2flg(0x10) | CPUIDLE_FLAG_TLB_FLUSHED,
1109 .exit_latency = 120,
1110 .target_residency = 500,
1111 .enter = &intel_idle,
1112 .enter_s2idle = intel_idle_s2idle },
1113 {
1114 .enter = NULL }
1115 };
1116
1117 static struct cpuidle_state bxt_cstates[] __initdata = {
1118 {
1119 .name = "C1",
1120 .desc = "MWAIT 0x00",
1121 .flags = MWAIT2flg(0x00),
1122 .exit_latency = 2,
1123 .target_residency = 2,
1124 .enter = &intel_idle,
1125 .enter_s2idle = intel_idle_s2idle, },
1126 {
1127 .name = "C1E",
1128 .desc = "MWAIT 0x01",
1129 .flags = MWAIT2flg(0x01) | CPUIDLE_FLAG_ALWAYS_ENABLE,
1130 .exit_latency = 10,
1131 .target_residency = 20,
1132 .enter = &intel_idle,
1133 .enter_s2idle = intel_idle_s2idle, },
1134 {
1135 .name = "C6",
1136 .desc = "MWAIT 0x20",
1137 .flags = MWAIT2flg(0x20) | CPUIDLE_FLAG_TLB_FLUSHED,
1138 .exit_latency = 133,
1139 .target_residency = 133,
1140 .enter = &intel_idle,
1141 .enter_s2idle = intel_idle_s2idle, },
1142 {
1143 .name = "C7s",
1144 .desc = "MWAIT 0x31",
1145 .flags = MWAIT2flg(0x31) | CPUIDLE_FLAG_TLB_FLUSHED,
1146 .exit_latency = 155,
1147 .target_residency = 155,
1148 .enter = &intel_idle,
1149 .enter_s2idle = intel_idle_s2idle, },
1150 {
1151 .name = "C8",
1152 .desc = "MWAIT 0x40",
1153 .flags = MWAIT2flg(0x40) | CPUIDLE_FLAG_TLB_FLUSHED,
1154 .exit_latency = 1000,
1155 .target_residency = 1000,
1156 .enter = &intel_idle,
1157 .enter_s2idle = intel_idle_s2idle, },
1158 {
1159 .name = "C9",
1160 .desc = "MWAIT 0x50",
1161 .flags = MWAIT2flg(0x50) | CPUIDLE_FLAG_TLB_FLUSHED,
1162 .exit_latency = 2000,
1163 .target_residency = 2000,
1164 .enter = &intel_idle,
1165 .enter_s2idle = intel_idle_s2idle, },
1166 {
1167 .name = "C10",
1168 .desc = "MWAIT 0x60",
1169 .flags = MWAIT2flg(0x60) | CPUIDLE_FLAG_TLB_FLUSHED,
1170 .exit_latency = 10000,
1171 .target_residency = 10000,
1172 .enter = &intel_idle,
1173 .enter_s2idle = intel_idle_s2idle, },
1174 {
1175 .enter = NULL }
1176 };
1177
1178 static struct cpuidle_state dnv_cstates[] __initdata = {
1179 {
1180 .name = "C1",
1181 .desc = "MWAIT 0x00",
1182 .flags = MWAIT2flg(0x00),
1183 .exit_latency = 2,
1184 .target_residency = 2,
1185 .enter = &intel_idle,
1186 .enter_s2idle = intel_idle_s2idle, },
1187 {
1188 .name = "C1E",
1189 .desc = "MWAIT 0x01",
1190 .flags = MWAIT2flg(0x01) | CPUIDLE_FLAG_ALWAYS_ENABLE,
1191 .exit_latency = 10,
1192 .target_residency = 20,
1193 .enter = &intel_idle,
1194 .enter_s2idle = intel_idle_s2idle, },
1195 {
1196 .name = "C6",
1197 .desc = "MWAIT 0x20",
1198 .flags = MWAIT2flg(0x20) | CPUIDLE_FLAG_TLB_FLUSHED,
1199 .exit_latency = 50,
1200 .target_residency = 500,
1201 .enter = &intel_idle,
1202 .enter_s2idle = intel_idle_s2idle, },
1203 {
1204 .enter = NULL }
1205 };
1206
1207 /*
1208 * Note, depending on HW and FW revision, SnowRidge SoC may or may not support
1209 * C6, and this is indicated in the CPUID mwait leaf.
1210 */
1211 static struct cpuidle_state snr_cstates[] __initdata = {
1212 {
1213 .name = "C1",
1214 .desc = "MWAIT 0x00",
1215 .flags = MWAIT2flg(0x00),
1216 .exit_latency = 2,
1217 .target_residency = 2,
1218 .enter = &intel_idle,
1219 .enter_s2idle = intel_idle_s2idle, },
1220 {
1221 .name = "C1E",
1222 .desc = "MWAIT 0x01",
1223 .flags = MWAIT2flg(0x01) | CPUIDLE_FLAG_ALWAYS_ENABLE,
1224 .exit_latency = 15,
1225 .target_residency = 25,
1226 .enter = &intel_idle,
1227 .enter_s2idle = intel_idle_s2idle, },
1228 {
1229 .name = "C6",
1230 .desc = "MWAIT 0x20",
1231 .flags = MWAIT2flg(0x20) | CPUIDLE_FLAG_TLB_FLUSHED,
1232 .exit_latency = 130,
1233 .target_residency = 500,
1234 .enter = &intel_idle,
1235 .enter_s2idle = intel_idle_s2idle, },
1236 {
1237 .enter = NULL }
1238 };
1239
1240 static const struct idle_cpu idle_cpu_nehalem __initconst = {
1241 .state_table = nehalem_cstates,
1242 .auto_demotion_disable_flags = NHM_C1_AUTO_DEMOTE | NHM_C3_AUTO_DEMOTE,
1243 .disable_promotion_to_c1e = true,
1244 };
1245
1246 static const struct idle_cpu idle_cpu_nhx __initconst = {
1247 .state_table = nehalem_cstates,
1248 .auto_demotion_disable_flags = NHM_C1_AUTO_DEMOTE | NHM_C3_AUTO_DEMOTE,
1249 .disable_promotion_to_c1e = true,
1250 .use_acpi = true,
1251 };
1252
1253 static const struct idle_cpu idle_cpu_atom __initconst = {
1254 .state_table = atom_cstates,
1255 };
1256
1257 static const struct idle_cpu idle_cpu_tangier __initconst = {
1258 .state_table = tangier_cstates,
1259 };
1260
1261 static const struct idle_cpu idle_cpu_lincroft __initconst = {
1262 .state_table = atom_cstates,
1263 .auto_demotion_disable_flags = ATM_LNC_C6_AUTO_DEMOTE,
1264 };
1265
1266 static const struct idle_cpu idle_cpu_snb __initconst = {
1267 .state_table = snb_cstates,
1268 .disable_promotion_to_c1e = true,
1269 };
1270
1271 static const struct idle_cpu idle_cpu_snx __initconst = {
1272 .state_table = snb_cstates,
1273 .disable_promotion_to_c1e = true,
1274 .use_acpi = true,
1275 };
1276
1277 static const struct idle_cpu idle_cpu_byt __initconst = {
1278 .state_table = byt_cstates,
1279 .disable_promotion_to_c1e = true,
1280 .byt_auto_demotion_disable_flag = true,
1281 };
1282
1283 static const struct idle_cpu idle_cpu_cht __initconst = {
1284 .state_table = cht_cstates,
1285 .disable_promotion_to_c1e = true,
1286 .byt_auto_demotion_disable_flag = true,
1287 };
1288
1289 static const struct idle_cpu idle_cpu_ivb __initconst = {
1290 .state_table = ivb_cstates,
1291 .disable_promotion_to_c1e = true,
1292 };
1293
1294 static const struct idle_cpu idle_cpu_ivt __initconst = {
1295 .state_table = ivt_cstates,
1296 .disable_promotion_to_c1e = true,
1297 .use_acpi = true,
1298 };
1299
1300 static const struct idle_cpu idle_cpu_hsw __initconst = {
1301 .state_table = hsw_cstates,
1302 .disable_promotion_to_c1e = true,
1303 };
1304
1305 static const struct idle_cpu idle_cpu_hsx __initconst = {
1306 .state_table = hsw_cstates,
1307 .disable_promotion_to_c1e = true,
1308 .use_acpi = true,
1309 };
1310
1311 static const struct idle_cpu idle_cpu_bdw __initconst = {
1312 .state_table = bdw_cstates,
1313 .disable_promotion_to_c1e = true,
1314 };
1315
1316 static const struct idle_cpu idle_cpu_bdx __initconst = {
1317 .state_table = bdw_cstates,
1318 .disable_promotion_to_c1e = true,
1319 .use_acpi = true,
1320 };
1321
1322 static const struct idle_cpu idle_cpu_skl __initconst = {
1323 .state_table = skl_cstates,
1324 .disable_promotion_to_c1e = true,
1325 };
1326
1327 static const struct idle_cpu idle_cpu_skx __initconst = {
1328 .state_table = skx_cstates,
1329 .disable_promotion_to_c1e = true,
1330 .use_acpi = true,
1331 };
1332
1333 static const struct idle_cpu idle_cpu_icx __initconst = {
1334 .state_table = icx_cstates,
1335 .disable_promotion_to_c1e = true,
1336 .use_acpi = true,
1337 };
1338
1339 static const struct idle_cpu idle_cpu_adl __initconst = {
1340 .state_table = adl_cstates,
1341 };
1342
1343 static const struct idle_cpu idle_cpu_adl_l __initconst = {
1344 .state_table = adl_l_cstates,
1345 };
1346
1347 static const struct idle_cpu idle_cpu_gmt __initconst = {
1348 .state_table = gmt_cstates,
1349 };
1350
1351 static const struct idle_cpu idle_cpu_spr __initconst = {
1352 .state_table = spr_cstates,
1353 .disable_promotion_to_c1e = true,
1354 .use_acpi = true,
1355 };
1356
1357 static const struct idle_cpu idle_cpu_avn __initconst = {
1358 .state_table = avn_cstates,
1359 .disable_promotion_to_c1e = true,
1360 .use_acpi = true,
1361 };
1362
1363 static const struct idle_cpu idle_cpu_knl __initconst = {
1364 .state_table = knl_cstates,
1365 .use_acpi = true,
1366 };
1367
1368 static const struct idle_cpu idle_cpu_bxt __initconst = {
1369 .state_table = bxt_cstates,
1370 .disable_promotion_to_c1e = true,
1371 };
1372
1373 static const struct idle_cpu idle_cpu_dnv __initconst = {
1374 .state_table = dnv_cstates,
1375 .disable_promotion_to_c1e = true,
1376 .use_acpi = true,
1377 };
1378
1379 static const struct idle_cpu idle_cpu_snr __initconst = {
1380 .state_table = snr_cstates,
1381 .disable_promotion_to_c1e = true,
1382 .use_acpi = true,
1383 };
1384
1385 static const struct x86_cpu_id intel_idle_ids[] __initconst = {
1386 X86_MATCH_INTEL_FAM6_MODEL(NEHALEM_EP, &idle_cpu_nhx),
1387 X86_MATCH_INTEL_FAM6_MODEL(NEHALEM, &idle_cpu_nehalem),
1388 X86_MATCH_INTEL_FAM6_MODEL(NEHALEM_G, &idle_cpu_nehalem),
1389 X86_MATCH_INTEL_FAM6_MODEL(WESTMERE, &idle_cpu_nehalem),
1390 X86_MATCH_INTEL_FAM6_MODEL(WESTMERE_EP, &idle_cpu_nhx),
1391 X86_MATCH_INTEL_FAM6_MODEL(NEHALEM_EX, &idle_cpu_nhx),
1392 X86_MATCH_INTEL_FAM6_MODEL(ATOM_BONNELL, &idle_cpu_atom),
1393 X86_MATCH_INTEL_FAM6_MODEL(ATOM_BONNELL_MID, &idle_cpu_lincroft),
1394 X86_MATCH_INTEL_FAM6_MODEL(WESTMERE_EX, &idle_cpu_nhx),
1395 X86_MATCH_INTEL_FAM6_MODEL(SANDYBRIDGE, &idle_cpu_snb),
1396 X86_MATCH_INTEL_FAM6_MODEL(SANDYBRIDGE_X, &idle_cpu_snx),
1397 X86_MATCH_INTEL_FAM6_MODEL(ATOM_SALTWELL, &idle_cpu_atom),
1398 X86_MATCH_INTEL_FAM6_MODEL(ATOM_SILVERMONT, &idle_cpu_byt),
1399 X86_MATCH_INTEL_FAM6_MODEL(ATOM_SILVERMONT_MID, &idle_cpu_tangier),
1400 X86_MATCH_INTEL_FAM6_MODEL(ATOM_AIRMONT, &idle_cpu_cht),
1401 X86_MATCH_INTEL_FAM6_MODEL(IVYBRIDGE, &idle_cpu_ivb),
1402 X86_MATCH_INTEL_FAM6_MODEL(IVYBRIDGE_X, &idle_cpu_ivt),
1403 X86_MATCH_INTEL_FAM6_MODEL(HASWELL, &idle_cpu_hsw),
1404 X86_MATCH_INTEL_FAM6_MODEL(HASWELL_X, &idle_cpu_hsx),
1405 X86_MATCH_INTEL_FAM6_MODEL(HASWELL_L, &idle_cpu_hsw),
1406 X86_MATCH_INTEL_FAM6_MODEL(HASWELL_G, &idle_cpu_hsw),
1407 X86_MATCH_INTEL_FAM6_MODEL(ATOM_SILVERMONT_D, &idle_cpu_avn),
1408 X86_MATCH_INTEL_FAM6_MODEL(BROADWELL, &idle_cpu_bdw),
1409 X86_MATCH_INTEL_FAM6_MODEL(BROADWELL_G, &idle_cpu_bdw),
1410 X86_MATCH_INTEL_FAM6_MODEL(BROADWELL_X, &idle_cpu_bdx),
1411 X86_MATCH_INTEL_FAM6_MODEL(BROADWELL_D, &idle_cpu_bdx),
1412 X86_MATCH_INTEL_FAM6_MODEL(SKYLAKE_L, &idle_cpu_skl),
1413 X86_MATCH_INTEL_FAM6_MODEL(SKYLAKE, &idle_cpu_skl),
1414 X86_MATCH_INTEL_FAM6_MODEL(KABYLAKE_L, &idle_cpu_skl),
1415 X86_MATCH_INTEL_FAM6_MODEL(KABYLAKE, &idle_cpu_skl),
1416 X86_MATCH_INTEL_FAM6_MODEL(SKYLAKE_X, &idle_cpu_skx),
1417 X86_MATCH_INTEL_FAM6_MODEL(ICELAKE_X, &idle_cpu_icx),
1418 X86_MATCH_INTEL_FAM6_MODEL(ICELAKE_D, &idle_cpu_icx),
1419 X86_MATCH_INTEL_FAM6_MODEL(ALDERLAKE, &idle_cpu_adl),
1420 X86_MATCH_INTEL_FAM6_MODEL(ALDERLAKE_L, &idle_cpu_adl_l),
1421 X86_MATCH_INTEL_FAM6_MODEL(ATOM_GRACEMONT, &idle_cpu_gmt),
1422 X86_MATCH_INTEL_FAM6_MODEL(SAPPHIRERAPIDS_X, &idle_cpu_spr),
1423 X86_MATCH_INTEL_FAM6_MODEL(EMERALDRAPIDS_X, &idle_cpu_spr),
1424 X86_MATCH_INTEL_FAM6_MODEL(XEON_PHI_KNL, &idle_cpu_knl),
1425 X86_MATCH_INTEL_FAM6_MODEL(XEON_PHI_KNM, &idle_cpu_knl),
1426 X86_MATCH_INTEL_FAM6_MODEL(ATOM_GOLDMONT, &idle_cpu_bxt),
1427 X86_MATCH_INTEL_FAM6_MODEL(ATOM_GOLDMONT_PLUS, &idle_cpu_bxt),
1428 X86_MATCH_INTEL_FAM6_MODEL(ATOM_GOLDMONT_D, &idle_cpu_dnv),
1429 X86_MATCH_INTEL_FAM6_MODEL(ATOM_TREMONT_D, &idle_cpu_snr),
1430 {}
1431 };
1432
1433 static const struct x86_cpu_id intel_mwait_ids[] __initconst = {
1434 X86_MATCH_VENDOR_FAM_FEATURE(INTEL, 6, X86_FEATURE_MWAIT, NULL),
1435 {}
1436 };
1437
intel_idle_max_cstate_reached(int cstate)1438 static bool __init intel_idle_max_cstate_reached(int cstate)
1439 {
1440 if (cstate + 1 > max_cstate) {
1441 pr_info("max_cstate %d reached\n", max_cstate);
1442 return true;
1443 }
1444 return false;
1445 }
1446
intel_idle_state_needs_timer_stop(struct cpuidle_state * state)1447 static bool __init intel_idle_state_needs_timer_stop(struct cpuidle_state *state)
1448 {
1449 unsigned long eax = flg2MWAIT(state->flags);
1450
1451 if (boot_cpu_has(X86_FEATURE_ARAT))
1452 return false;
1453
1454 /*
1455 * Switch over to one-shot tick broadcast if the target C-state
1456 * is deeper than C1.
1457 */
1458 return !!((eax >> MWAIT_SUBSTATE_SIZE) & MWAIT_CSTATE_MASK);
1459 }
1460
1461 #ifdef CONFIG_ACPI_PROCESSOR_CSTATE
1462 #include <acpi/processor.h>
1463
1464 static bool no_acpi __read_mostly;
1465 module_param(no_acpi, bool, 0444);
1466 MODULE_PARM_DESC(no_acpi, "Do not use ACPI _CST for building the idle states list");
1467
1468 static bool force_use_acpi __read_mostly; /* No effect if no_acpi is set. */
1469 module_param_named(use_acpi, force_use_acpi, bool, 0444);
1470 MODULE_PARM_DESC(use_acpi, "Use ACPI _CST for building the idle states list");
1471
1472 static struct acpi_processor_power acpi_state_table __initdata;
1473
1474 /**
1475 * intel_idle_cst_usable - Check if the _CST information can be used.
1476 *
1477 * Check if all of the C-states listed by _CST in the max_cstate range are
1478 * ACPI_CSTATE_FFH, which means that they should be entered via MWAIT.
1479 */
intel_idle_cst_usable(void)1480 static bool __init intel_idle_cst_usable(void)
1481 {
1482 int cstate, limit;
1483
1484 limit = min_t(int, min_t(int, CPUIDLE_STATE_MAX, max_cstate + 1),
1485 acpi_state_table.count);
1486
1487 for (cstate = 1; cstate < limit; cstate++) {
1488 struct acpi_processor_cx *cx = &acpi_state_table.states[cstate];
1489
1490 if (cx->entry_method != ACPI_CSTATE_FFH)
1491 return false;
1492 }
1493
1494 return true;
1495 }
1496
intel_idle_acpi_cst_extract(void)1497 static bool __init intel_idle_acpi_cst_extract(void)
1498 {
1499 unsigned int cpu;
1500
1501 if (no_acpi) {
1502 pr_debug("Not allowed to use ACPI _CST\n");
1503 return false;
1504 }
1505
1506 for_each_possible_cpu(cpu) {
1507 struct acpi_processor *pr = per_cpu(processors, cpu);
1508
1509 if (!pr)
1510 continue;
1511
1512 if (acpi_processor_evaluate_cst(pr->handle, cpu, &acpi_state_table))
1513 continue;
1514
1515 acpi_state_table.count++;
1516
1517 if (!intel_idle_cst_usable())
1518 continue;
1519
1520 if (!acpi_processor_claim_cst_control())
1521 break;
1522
1523 return true;
1524 }
1525
1526 acpi_state_table.count = 0;
1527 pr_debug("ACPI _CST not found or not usable\n");
1528 return false;
1529 }
1530
intel_idle_init_cstates_acpi(struct cpuidle_driver * drv)1531 static void __init intel_idle_init_cstates_acpi(struct cpuidle_driver *drv)
1532 {
1533 int cstate, limit = min_t(int, CPUIDLE_STATE_MAX, acpi_state_table.count);
1534
1535 /*
1536 * If limit > 0, intel_idle_cst_usable() has returned 'true', so all of
1537 * the interesting states are ACPI_CSTATE_FFH.
1538 */
1539 for (cstate = 1; cstate < limit; cstate++) {
1540 struct acpi_processor_cx *cx;
1541 struct cpuidle_state *state;
1542
1543 if (intel_idle_max_cstate_reached(cstate - 1))
1544 break;
1545
1546 cx = &acpi_state_table.states[cstate];
1547
1548 state = &drv->states[drv->state_count++];
1549
1550 snprintf(state->name, CPUIDLE_NAME_LEN, "C%d_ACPI", cstate);
1551 strscpy(state->desc, cx->desc, CPUIDLE_DESC_LEN);
1552 state->exit_latency = cx->latency;
1553 /*
1554 * For C1-type C-states use the same number for both the exit
1555 * latency and target residency, because that is the case for
1556 * C1 in the majority of the static C-states tables above.
1557 * For the other types of C-states, however, set the target
1558 * residency to 3 times the exit latency which should lead to
1559 * a reasonable balance between energy-efficiency and
1560 * performance in the majority of interesting cases.
1561 */
1562 state->target_residency = cx->latency;
1563 if (cx->type > ACPI_STATE_C1)
1564 state->target_residency *= 3;
1565
1566 state->flags = MWAIT2flg(cx->address);
1567 if (cx->type > ACPI_STATE_C2)
1568 state->flags |= CPUIDLE_FLAG_TLB_FLUSHED;
1569
1570 if (disabled_states_mask & BIT(cstate))
1571 state->flags |= CPUIDLE_FLAG_OFF;
1572
1573 if (intel_idle_state_needs_timer_stop(state))
1574 state->flags |= CPUIDLE_FLAG_TIMER_STOP;
1575
1576 state->enter = intel_idle;
1577 state->enter_s2idle = intel_idle_s2idle;
1578 }
1579 }
1580
intel_idle_off_by_default(u32 mwait_hint)1581 static bool __init intel_idle_off_by_default(u32 mwait_hint)
1582 {
1583 int cstate, limit;
1584
1585 /*
1586 * If there are no _CST C-states, do not disable any C-states by
1587 * default.
1588 */
1589 if (!acpi_state_table.count)
1590 return false;
1591
1592 limit = min_t(int, CPUIDLE_STATE_MAX, acpi_state_table.count);
1593 /*
1594 * If limit > 0, intel_idle_cst_usable() has returned 'true', so all of
1595 * the interesting states are ACPI_CSTATE_FFH.
1596 */
1597 for (cstate = 1; cstate < limit; cstate++) {
1598 if (acpi_state_table.states[cstate].address == mwait_hint)
1599 return false;
1600 }
1601 return true;
1602 }
1603 #else /* !CONFIG_ACPI_PROCESSOR_CSTATE */
1604 #define force_use_acpi (false)
1605
intel_idle_acpi_cst_extract(void)1606 static inline bool intel_idle_acpi_cst_extract(void) { return false; }
intel_idle_init_cstates_acpi(struct cpuidle_driver * drv)1607 static inline void intel_idle_init_cstates_acpi(struct cpuidle_driver *drv) { }
intel_idle_off_by_default(u32 mwait_hint)1608 static inline bool intel_idle_off_by_default(u32 mwait_hint) { return false; }
1609 #endif /* !CONFIG_ACPI_PROCESSOR_CSTATE */
1610
1611 /**
1612 * ivt_idle_state_table_update - Tune the idle states table for Ivy Town.
1613 *
1614 * Tune IVT multi-socket targets.
1615 * Assumption: num_sockets == (max_package_num + 1).
1616 */
ivt_idle_state_table_update(void)1617 static void __init ivt_idle_state_table_update(void)
1618 {
1619 /* IVT uses a different table for 1-2, 3-4, and > 4 sockets */
1620 int cpu, package_num, num_sockets = 1;
1621
1622 for_each_online_cpu(cpu) {
1623 package_num = topology_physical_package_id(cpu);
1624 if (package_num + 1 > num_sockets) {
1625 num_sockets = package_num + 1;
1626
1627 if (num_sockets > 4) {
1628 cpuidle_state_table = ivt_cstates_8s;
1629 return;
1630 }
1631 }
1632 }
1633
1634 if (num_sockets > 2)
1635 cpuidle_state_table = ivt_cstates_4s;
1636
1637 /* else, 1 and 2 socket systems use default ivt_cstates */
1638 }
1639
1640 /**
1641 * irtl_2_usec - IRTL to microseconds conversion.
1642 * @irtl: IRTL MSR value.
1643 *
1644 * Translate the IRTL (Interrupt Response Time Limit) MSR value to microseconds.
1645 */
irtl_2_usec(unsigned long long irtl)1646 static unsigned long long __init irtl_2_usec(unsigned long long irtl)
1647 {
1648 static const unsigned int irtl_ns_units[] __initconst = {
1649 1, 32, 1024, 32768, 1048576, 33554432, 0, 0
1650 };
1651 unsigned long long ns;
1652
1653 if (!irtl)
1654 return 0;
1655
1656 ns = irtl_ns_units[(irtl >> 10) & 0x7];
1657
1658 return div_u64((irtl & 0x3FF) * ns, NSEC_PER_USEC);
1659 }
1660
1661 /**
1662 * bxt_idle_state_table_update - Fix up the Broxton idle states table.
1663 *
1664 * On BXT, trust the IRTL (Interrupt Response Time Limit) MSR to show the
1665 * definitive maximum latency and use the same value for target_residency.
1666 */
bxt_idle_state_table_update(void)1667 static void __init bxt_idle_state_table_update(void)
1668 {
1669 unsigned long long msr;
1670 unsigned int usec;
1671
1672 rdmsrl(MSR_PKGC6_IRTL, msr);
1673 usec = irtl_2_usec(msr);
1674 if (usec) {
1675 bxt_cstates[2].exit_latency = usec;
1676 bxt_cstates[2].target_residency = usec;
1677 }
1678
1679 rdmsrl(MSR_PKGC7_IRTL, msr);
1680 usec = irtl_2_usec(msr);
1681 if (usec) {
1682 bxt_cstates[3].exit_latency = usec;
1683 bxt_cstates[3].target_residency = usec;
1684 }
1685
1686 rdmsrl(MSR_PKGC8_IRTL, msr);
1687 usec = irtl_2_usec(msr);
1688 if (usec) {
1689 bxt_cstates[4].exit_latency = usec;
1690 bxt_cstates[4].target_residency = usec;
1691 }
1692
1693 rdmsrl(MSR_PKGC9_IRTL, msr);
1694 usec = irtl_2_usec(msr);
1695 if (usec) {
1696 bxt_cstates[5].exit_latency = usec;
1697 bxt_cstates[5].target_residency = usec;
1698 }
1699
1700 rdmsrl(MSR_PKGC10_IRTL, msr);
1701 usec = irtl_2_usec(msr);
1702 if (usec) {
1703 bxt_cstates[6].exit_latency = usec;
1704 bxt_cstates[6].target_residency = usec;
1705 }
1706
1707 }
1708
1709 /**
1710 * sklh_idle_state_table_update - Fix up the Sky Lake idle states table.
1711 *
1712 * On SKL-H (model 0x5e) skip C8 and C9 if C10 is enabled and SGX disabled.
1713 */
sklh_idle_state_table_update(void)1714 static void __init sklh_idle_state_table_update(void)
1715 {
1716 unsigned long long msr;
1717 unsigned int eax, ebx, ecx, edx;
1718
1719
1720 /* if PC10 disabled via cmdline intel_idle.max_cstate=7 or shallower */
1721 if (max_cstate <= 7)
1722 return;
1723
1724 /* if PC10 not present in CPUID.MWAIT.EDX */
1725 if ((mwait_substates & (0xF << 28)) == 0)
1726 return;
1727
1728 rdmsrl(MSR_PKG_CST_CONFIG_CONTROL, msr);
1729
1730 /* PC10 is not enabled in PKG C-state limit */
1731 if ((msr & 0xF) != 8)
1732 return;
1733
1734 ecx = 0;
1735 cpuid(7, &eax, &ebx, &ecx, &edx);
1736
1737 /* if SGX is present */
1738 if (ebx & (1 << 2)) {
1739
1740 rdmsrl(MSR_IA32_FEAT_CTL, msr);
1741
1742 /* if SGX is enabled */
1743 if (msr & (1 << 18))
1744 return;
1745 }
1746
1747 skl_cstates[5].flags |= CPUIDLE_FLAG_UNUSABLE; /* C8-SKL */
1748 skl_cstates[6].flags |= CPUIDLE_FLAG_UNUSABLE; /* C9-SKL */
1749 }
1750
1751 /**
1752 * skx_idle_state_table_update - Adjust the Sky Lake/Cascade Lake
1753 * idle states table.
1754 */
skx_idle_state_table_update(void)1755 static void __init skx_idle_state_table_update(void)
1756 {
1757 unsigned long long msr;
1758
1759 rdmsrl(MSR_PKG_CST_CONFIG_CONTROL, msr);
1760
1761 /*
1762 * 000b: C0/C1 (no package C-state support)
1763 * 001b: C2
1764 * 010b: C6 (non-retention)
1765 * 011b: C6 (retention)
1766 * 111b: No Package C state limits.
1767 */
1768 if ((msr & 0x7) < 2) {
1769 /*
1770 * Uses the CC6 + PC0 latency and 3 times of
1771 * latency for target_residency if the PC6
1772 * is disabled in BIOS. This is consistent
1773 * with how intel_idle driver uses _CST
1774 * to set the target_residency.
1775 */
1776 skx_cstates[2].exit_latency = 92;
1777 skx_cstates[2].target_residency = 276;
1778 }
1779 }
1780
1781 /**
1782 * adl_idle_state_table_update - Adjust AlderLake idle states table.
1783 */
adl_idle_state_table_update(void)1784 static void __init adl_idle_state_table_update(void)
1785 {
1786 /* Check if user prefers C1 over C1E. */
1787 if (preferred_states_mask & BIT(1) && !(preferred_states_mask & BIT(2))) {
1788 cpuidle_state_table[0].flags &= ~CPUIDLE_FLAG_UNUSABLE;
1789 cpuidle_state_table[1].flags |= CPUIDLE_FLAG_UNUSABLE;
1790
1791 /* Disable C1E by clearing the "C1E promotion" bit. */
1792 c1e_promotion = C1E_PROMOTION_DISABLE;
1793 return;
1794 }
1795
1796 /* Make sure C1E is enabled by default */
1797 c1e_promotion = C1E_PROMOTION_ENABLE;
1798 }
1799
1800 /**
1801 * spr_idle_state_table_update - Adjust Sapphire Rapids idle states table.
1802 */
spr_idle_state_table_update(void)1803 static void __init spr_idle_state_table_update(void)
1804 {
1805 unsigned long long msr;
1806
1807 /*
1808 * By default, the C6 state assumes the worst-case scenario of package
1809 * C6. However, if PC6 is disabled, we update the numbers to match
1810 * core C6.
1811 */
1812 rdmsrl(MSR_PKG_CST_CONFIG_CONTROL, msr);
1813
1814 /* Limit value 2 and above allow for PC6. */
1815 if ((msr & 0x7) < 2) {
1816 spr_cstates[2].exit_latency = 190;
1817 spr_cstates[2].target_residency = 600;
1818 }
1819 }
1820
intel_idle_verify_cstate(unsigned int mwait_hint)1821 static bool __init intel_idle_verify_cstate(unsigned int mwait_hint)
1822 {
1823 unsigned int mwait_cstate = MWAIT_HINT2CSTATE(mwait_hint) + 1;
1824 unsigned int num_substates = (mwait_substates >> mwait_cstate * 4) &
1825 MWAIT_SUBSTATE_MASK;
1826
1827 /* Ignore the C-state if there are NO sub-states in CPUID for it. */
1828 if (num_substates == 0)
1829 return false;
1830
1831 if (mwait_cstate > 2 && !boot_cpu_has(X86_FEATURE_NONSTOP_TSC))
1832 mark_tsc_unstable("TSC halts in idle states deeper than C2");
1833
1834 return true;
1835 }
1836
state_update_enter_method(struct cpuidle_state * state,int cstate)1837 static void state_update_enter_method(struct cpuidle_state *state, int cstate)
1838 {
1839 if (state->flags & CPUIDLE_FLAG_INIT_XSTATE) {
1840 /*
1841 * Combining with XSTATE with IBRS or IRQ_ENABLE flags
1842 * is not currently supported but this driver.
1843 */
1844 WARN_ON_ONCE(state->flags & CPUIDLE_FLAG_IBRS);
1845 WARN_ON_ONCE(state->flags & CPUIDLE_FLAG_IRQ_ENABLE);
1846 state->enter = intel_idle_xstate;
1847 return;
1848 }
1849
1850 if (cpu_feature_enabled(X86_FEATURE_KERNEL_IBRS) &&
1851 state->flags & CPUIDLE_FLAG_IBRS) {
1852 /*
1853 * IBRS mitigation requires that C-states are entered
1854 * with interrupts disabled.
1855 */
1856 WARN_ON_ONCE(state->flags & CPUIDLE_FLAG_IRQ_ENABLE);
1857 state->enter = intel_idle_ibrs;
1858 return;
1859 }
1860
1861 if (state->flags & CPUIDLE_FLAG_IRQ_ENABLE) {
1862 state->enter = intel_idle_irq;
1863 return;
1864 }
1865
1866 if (force_irq_on) {
1867 pr_info("forced intel_idle_irq for state %d\n", cstate);
1868 state->enter = intel_idle_irq;
1869 }
1870 }
1871
intel_idle_init_cstates_icpu(struct cpuidle_driver * drv)1872 static void __init intel_idle_init_cstates_icpu(struct cpuidle_driver *drv)
1873 {
1874 int cstate;
1875
1876 switch (boot_cpu_data.x86_model) {
1877 case INTEL_FAM6_IVYBRIDGE_X:
1878 ivt_idle_state_table_update();
1879 break;
1880 case INTEL_FAM6_ATOM_GOLDMONT:
1881 case INTEL_FAM6_ATOM_GOLDMONT_PLUS:
1882 bxt_idle_state_table_update();
1883 break;
1884 case INTEL_FAM6_SKYLAKE:
1885 sklh_idle_state_table_update();
1886 break;
1887 case INTEL_FAM6_SKYLAKE_X:
1888 skx_idle_state_table_update();
1889 break;
1890 case INTEL_FAM6_SAPPHIRERAPIDS_X:
1891 case INTEL_FAM6_EMERALDRAPIDS_X:
1892 spr_idle_state_table_update();
1893 break;
1894 case INTEL_FAM6_ALDERLAKE:
1895 case INTEL_FAM6_ALDERLAKE_L:
1896 case INTEL_FAM6_ATOM_GRACEMONT:
1897 adl_idle_state_table_update();
1898 break;
1899 }
1900
1901 for (cstate = 0; cstate < CPUIDLE_STATE_MAX; ++cstate) {
1902 struct cpuidle_state *state;
1903 unsigned int mwait_hint;
1904
1905 if (intel_idle_max_cstate_reached(cstate))
1906 break;
1907
1908 if (!cpuidle_state_table[cstate].enter &&
1909 !cpuidle_state_table[cstate].enter_s2idle)
1910 break;
1911
1912 /* If marked as unusable, skip this state. */
1913 if (cpuidle_state_table[cstate].flags & CPUIDLE_FLAG_UNUSABLE) {
1914 pr_debug("state %s is disabled\n",
1915 cpuidle_state_table[cstate].name);
1916 continue;
1917 }
1918
1919 mwait_hint = flg2MWAIT(cpuidle_state_table[cstate].flags);
1920 if (!intel_idle_verify_cstate(mwait_hint))
1921 continue;
1922
1923 /* Structure copy. */
1924 drv->states[drv->state_count] = cpuidle_state_table[cstate];
1925 state = &drv->states[drv->state_count];
1926
1927 state_update_enter_method(state, cstate);
1928
1929
1930 if ((disabled_states_mask & BIT(drv->state_count)) ||
1931 ((icpu->use_acpi || force_use_acpi) &&
1932 intel_idle_off_by_default(mwait_hint) &&
1933 !(state->flags & CPUIDLE_FLAG_ALWAYS_ENABLE)))
1934 state->flags |= CPUIDLE_FLAG_OFF;
1935
1936 if (intel_idle_state_needs_timer_stop(state))
1937 state->flags |= CPUIDLE_FLAG_TIMER_STOP;
1938
1939 drv->state_count++;
1940 }
1941
1942 if (icpu->byt_auto_demotion_disable_flag) {
1943 wrmsrl(MSR_CC6_DEMOTION_POLICY_CONFIG, 0);
1944 wrmsrl(MSR_MC6_DEMOTION_POLICY_CONFIG, 0);
1945 }
1946 }
1947
1948 /**
1949 * intel_idle_cpuidle_driver_init - Create the list of available idle states.
1950 * @drv: cpuidle driver structure to initialize.
1951 */
intel_idle_cpuidle_driver_init(struct cpuidle_driver * drv)1952 static void __init intel_idle_cpuidle_driver_init(struct cpuidle_driver *drv)
1953 {
1954 cpuidle_poll_state_init(drv);
1955
1956 if (disabled_states_mask & BIT(0))
1957 drv->states[0].flags |= CPUIDLE_FLAG_OFF;
1958
1959 drv->state_count = 1;
1960
1961 if (icpu)
1962 intel_idle_init_cstates_icpu(drv);
1963 else
1964 intel_idle_init_cstates_acpi(drv);
1965 }
1966
auto_demotion_disable(void)1967 static void auto_demotion_disable(void)
1968 {
1969 unsigned long long msr_bits;
1970
1971 rdmsrl(MSR_PKG_CST_CONFIG_CONTROL, msr_bits);
1972 msr_bits &= ~auto_demotion_disable_flags;
1973 wrmsrl(MSR_PKG_CST_CONFIG_CONTROL, msr_bits);
1974 }
1975
c1e_promotion_enable(void)1976 static void c1e_promotion_enable(void)
1977 {
1978 unsigned long long msr_bits;
1979
1980 rdmsrl(MSR_IA32_POWER_CTL, msr_bits);
1981 msr_bits |= 0x2;
1982 wrmsrl(MSR_IA32_POWER_CTL, msr_bits);
1983 }
1984
c1e_promotion_disable(void)1985 static void c1e_promotion_disable(void)
1986 {
1987 unsigned long long msr_bits;
1988
1989 rdmsrl(MSR_IA32_POWER_CTL, msr_bits);
1990 msr_bits &= ~0x2;
1991 wrmsrl(MSR_IA32_POWER_CTL, msr_bits);
1992 }
1993
1994 /**
1995 * intel_idle_cpu_init - Register the target CPU with the cpuidle core.
1996 * @cpu: CPU to initialize.
1997 *
1998 * Register a cpuidle device object for @cpu and update its MSRs in accordance
1999 * with the processor model flags.
2000 */
intel_idle_cpu_init(unsigned int cpu)2001 static int intel_idle_cpu_init(unsigned int cpu)
2002 {
2003 struct cpuidle_device *dev;
2004
2005 dev = per_cpu_ptr(intel_idle_cpuidle_devices, cpu);
2006 dev->cpu = cpu;
2007
2008 if (cpuidle_register_device(dev)) {
2009 pr_debug("cpuidle_register_device %d failed!\n", cpu);
2010 return -EIO;
2011 }
2012
2013 if (auto_demotion_disable_flags)
2014 auto_demotion_disable();
2015
2016 if (c1e_promotion == C1E_PROMOTION_ENABLE)
2017 c1e_promotion_enable();
2018 else if (c1e_promotion == C1E_PROMOTION_DISABLE)
2019 c1e_promotion_disable();
2020
2021 return 0;
2022 }
2023
intel_idle_cpu_online(unsigned int cpu)2024 static int intel_idle_cpu_online(unsigned int cpu)
2025 {
2026 struct cpuidle_device *dev;
2027
2028 if (!boot_cpu_has(X86_FEATURE_ARAT))
2029 tick_broadcast_enable();
2030
2031 /*
2032 * Some systems can hotplug a cpu at runtime after
2033 * the kernel has booted, we have to initialize the
2034 * driver in this case
2035 */
2036 dev = per_cpu_ptr(intel_idle_cpuidle_devices, cpu);
2037 if (!dev->registered)
2038 return intel_idle_cpu_init(cpu);
2039
2040 return 0;
2041 }
2042
2043 /**
2044 * intel_idle_cpuidle_devices_uninit - Unregister all cpuidle devices.
2045 */
intel_idle_cpuidle_devices_uninit(void)2046 static void __init intel_idle_cpuidle_devices_uninit(void)
2047 {
2048 int i;
2049
2050 for_each_online_cpu(i)
2051 cpuidle_unregister_device(per_cpu_ptr(intel_idle_cpuidle_devices, i));
2052 }
2053
intel_idle_init(void)2054 static int __init intel_idle_init(void)
2055 {
2056 const struct x86_cpu_id *id;
2057 unsigned int eax, ebx, ecx;
2058 int retval;
2059
2060 /* Do not load intel_idle at all for now if idle= is passed */
2061 if (boot_option_idle_override != IDLE_NO_OVERRIDE)
2062 return -ENODEV;
2063
2064 if (max_cstate == 0) {
2065 pr_debug("disabled\n");
2066 return -EPERM;
2067 }
2068
2069 id = x86_match_cpu(intel_idle_ids);
2070 if (id) {
2071 if (!boot_cpu_has(X86_FEATURE_MWAIT)) {
2072 pr_debug("Please enable MWAIT in BIOS SETUP\n");
2073 return -ENODEV;
2074 }
2075 } else {
2076 id = x86_match_cpu(intel_mwait_ids);
2077 if (!id)
2078 return -ENODEV;
2079 }
2080
2081 if (boot_cpu_data.cpuid_level < CPUID_MWAIT_LEAF)
2082 return -ENODEV;
2083
2084 cpuid(CPUID_MWAIT_LEAF, &eax, &ebx, &ecx, &mwait_substates);
2085
2086 if (!(ecx & CPUID5_ECX_EXTENSIONS_SUPPORTED) ||
2087 !(ecx & CPUID5_ECX_INTERRUPT_BREAK) ||
2088 !mwait_substates)
2089 return -ENODEV;
2090
2091 pr_debug("MWAIT substates: 0x%x\n", mwait_substates);
2092
2093 icpu = (const struct idle_cpu *)id->driver_data;
2094 if (icpu) {
2095 cpuidle_state_table = icpu->state_table;
2096 auto_demotion_disable_flags = icpu->auto_demotion_disable_flags;
2097 if (icpu->disable_promotion_to_c1e)
2098 c1e_promotion = C1E_PROMOTION_DISABLE;
2099 if (icpu->use_acpi || force_use_acpi)
2100 intel_idle_acpi_cst_extract();
2101 } else if (!intel_idle_acpi_cst_extract()) {
2102 return -ENODEV;
2103 }
2104
2105 pr_debug("v" INTEL_IDLE_VERSION " model 0x%X\n",
2106 boot_cpu_data.x86_model);
2107
2108 intel_idle_cpuidle_devices = alloc_percpu(struct cpuidle_device);
2109 if (!intel_idle_cpuidle_devices)
2110 return -ENOMEM;
2111
2112 intel_idle_cpuidle_driver_init(&intel_idle_driver);
2113
2114 retval = cpuidle_register_driver(&intel_idle_driver);
2115 if (retval) {
2116 struct cpuidle_driver *drv = cpuidle_get_driver();
2117 printk(KERN_DEBUG pr_fmt("intel_idle yielding to %s\n"),
2118 drv ? drv->name : "none");
2119 goto init_driver_fail;
2120 }
2121
2122 retval = cpuhp_setup_state(CPUHP_AP_ONLINE_DYN, "idle/intel:online",
2123 intel_idle_cpu_online, NULL);
2124 if (retval < 0)
2125 goto hp_setup_fail;
2126
2127 pr_debug("Local APIC timer is reliable in %s\n",
2128 boot_cpu_has(X86_FEATURE_ARAT) ? "all C-states" : "C1");
2129
2130 return 0;
2131
2132 hp_setup_fail:
2133 intel_idle_cpuidle_devices_uninit();
2134 cpuidle_unregister_driver(&intel_idle_driver);
2135 init_driver_fail:
2136 free_percpu(intel_idle_cpuidle_devices);
2137 return retval;
2138
2139 }
2140 device_initcall(intel_idle_init);
2141
2142 /*
2143 * We are not really modular, but we used to support that. Meaning we also
2144 * support "intel_idle.max_cstate=..." at boot and also a read-only export of
2145 * it at /sys/module/intel_idle/parameters/max_cstate -- so using module_param
2146 * is the easiest way (currently) to continue doing that.
2147 */
2148 module_param(max_cstate, int, 0444);
2149 /*
2150 * The positions of the bits that are set in this number are the indices of the
2151 * idle states to be disabled by default (as reflected by the names of the
2152 * corresponding idle state directories in sysfs, "state0", "state1" ...
2153 * "state<i>" ..., where <i> is the index of the given state).
2154 */
2155 module_param_named(states_off, disabled_states_mask, uint, 0444);
2156 MODULE_PARM_DESC(states_off, "Mask of disabled idle states");
2157 /*
2158 * Some platforms come with mutually exclusive C-states, so that if one is
2159 * enabled, the other C-states must not be used. Example: C1 and C1E on
2160 * Sapphire Rapids platform. This parameter allows for selecting the
2161 * preferred C-states among the groups of mutually exclusive C-states - the
2162 * selected C-states will be registered, the other C-states from the mutually
2163 * exclusive group won't be registered. If the platform has no mutually
2164 * exclusive C-states, this parameter has no effect.
2165 */
2166 module_param_named(preferred_cstates, preferred_states_mask, uint, 0444);
2167 MODULE_PARM_DESC(preferred_cstates, "Mask of preferred idle states");
2168 /*
2169 * Debugging option that forces the driver to enter all C-states with
2170 * interrupts enabled. Does not apply to C-states with
2171 * 'CPUIDLE_FLAG_INIT_XSTATE' and 'CPUIDLE_FLAG_IBRS' flags.
2172 */
2173 module_param(force_irq_on, bool, 0444);
2174