1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * intel_idle.c - native hardware idle loop for modern Intel processors 4 * 5 * Copyright (c) 2013 - 2020, Intel Corporation. 6 * Len Brown <len.brown@intel.com> 7 * Rafael J. Wysocki <rafael.j.wysocki@intel.com> 8 */ 9 10 /* 11 * intel_idle is a cpuidle driver that loads on all Intel CPUs with MWAIT 12 * in lieu of the legacy ACPI processor_idle driver. The intent is to 13 * make Linux more efficient on these processors, as intel_idle knows 14 * more than ACPI, as well as make Linux more immune to ACPI BIOS bugs. 15 */ 16 17 /* 18 * Design Assumptions 19 * 20 * All CPUs have same idle states as boot CPU 21 * 22 * Chipset BM_STS (bus master status) bit is a NOP 23 * for preventing entry into deep C-states 24 * 25 * CPU will flush caches as needed when entering a C-state via MWAIT 26 * (in contrast to entering ACPI C3, in which case the WBINVD 27 * instruction needs to be executed to flush the caches) 28 */ 29 30 /* 31 * Known limitations 32 * 33 * ACPI has a .suspend hack to turn off deep c-statees during suspend 34 * to avoid complications with the lapic timer workaround. 35 * Have not seen issues with suspend, but may need same workaround here. 36 * 37 */ 38 39 /* un-comment DEBUG to enable pr_debug() statements */ 40 /* #define DEBUG */ 41 42 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt 43 44 #include <linux/acpi.h> 45 #include <linux/kernel.h> 46 #include <linux/cpuidle.h> 47 #include <linux/tick.h> 48 #include <trace/events/power.h> 49 #include <linux/sched.h> 50 #include <linux/notifier.h> 51 #include <linux/cpu.h> 52 #include <linux/moduleparam.h> 53 #include <asm/cpu_device_id.h> 54 #include <asm/intel-family.h> 55 #include <asm/mwait.h> 56 #include <asm/msr.h> 57 58 #define INTEL_IDLE_VERSION "0.5.1" 59 60 static struct cpuidle_driver intel_idle_driver = { 61 .name = "intel_idle", 62 .owner = THIS_MODULE, 63 }; 64 /* intel_idle.max_cstate=0 disables driver */ 65 static int max_cstate = CPUIDLE_STATE_MAX - 1; 66 static unsigned int disabled_states_mask; 67 static unsigned int preferred_states_mask; 68 69 static struct cpuidle_device __percpu *intel_idle_cpuidle_devices; 70 71 static unsigned long auto_demotion_disable_flags; 72 static bool disable_promotion_to_c1e; 73 74 struct idle_cpu { 75 struct cpuidle_state *state_table; 76 77 /* 78 * Hardware C-state auto-demotion may not always be optimal. 79 * Indicate which enable bits to clear here. 80 */ 81 unsigned long auto_demotion_disable_flags; 82 bool byt_auto_demotion_disable_flag; 83 bool disable_promotion_to_c1e; 84 bool use_acpi; 85 }; 86 87 static const struct idle_cpu *icpu __initdata; 88 static struct cpuidle_state *cpuidle_state_table __initdata; 89 90 static unsigned int mwait_substates __initdata; 91 92 /* 93 * Enable interrupts before entering the C-state. On some platforms and for 94 * some C-states, this may measurably decrease interrupt latency. 95 */ 96 #define CPUIDLE_FLAG_IRQ_ENABLE BIT(14) 97 98 /* 99 * Enable this state by default even if the ACPI _CST does not list it. 100 */ 101 #define CPUIDLE_FLAG_ALWAYS_ENABLE BIT(15) 102 103 /* 104 * MWAIT takes an 8-bit "hint" in EAX "suggesting" 105 * the C-state (top nibble) and sub-state (bottom nibble) 106 * 0x00 means "MWAIT(C1)", 0x10 means "MWAIT(C2)" etc. 107 * 108 * We store the hint at the top of our "flags" for each state. 109 */ 110 #define flg2MWAIT(flags) (((flags) >> 24) & 0xFF) 111 #define MWAIT2flg(eax) ((eax & 0xFF) << 24) 112 113 /** 114 * intel_idle - Ask the processor to enter the given idle state. 115 * @dev: cpuidle device of the target CPU. 116 * @drv: cpuidle driver (assumed to point to intel_idle_driver). 117 * @index: Target idle state index. 118 * 119 * Use the MWAIT instruction to notify the processor that the CPU represented by 120 * @dev is idle and it can try to enter the idle state corresponding to @index. 121 * 122 * If the local APIC timer is not known to be reliable in the target idle state, 123 * enable one-shot tick broadcasting for the target CPU before executing MWAIT. 124 * 125 * Must be called under local_irq_disable(). 126 */ 127 static __cpuidle int intel_idle(struct cpuidle_device *dev, 128 struct cpuidle_driver *drv, int index) 129 { 130 struct cpuidle_state *state = &drv->states[index]; 131 unsigned long eax = flg2MWAIT(state->flags); 132 unsigned long ecx = 1; /* break on interrupt flag */ 133 134 if (state->flags & CPUIDLE_FLAG_IRQ_ENABLE) 135 local_irq_enable(); 136 137 mwait_idle_with_hints(eax, ecx); 138 139 return index; 140 } 141 142 /** 143 * intel_idle_s2idle - Ask the processor to enter the given idle state. 144 * @dev: cpuidle device of the target CPU. 145 * @drv: cpuidle driver (assumed to point to intel_idle_driver). 146 * @index: Target idle state index. 147 * 148 * Use the MWAIT instruction to notify the processor that the CPU represented by 149 * @dev is idle and it can try to enter the idle state corresponding to @index. 150 * 151 * Invoked as a suspend-to-idle callback routine with frozen user space, frozen 152 * scheduler tick and suspended scheduler clock on the target CPU. 153 */ 154 static __cpuidle int intel_idle_s2idle(struct cpuidle_device *dev, 155 struct cpuidle_driver *drv, int index) 156 { 157 unsigned long eax = flg2MWAIT(drv->states[index].flags); 158 unsigned long ecx = 1; /* break on interrupt flag */ 159 160 mwait_idle_with_hints(eax, ecx); 161 162 return 0; 163 } 164 165 /* 166 * States are indexed by the cstate number, 167 * which is also the index into the MWAIT hint array. 168 * Thus C0 is a dummy. 169 */ 170 static struct cpuidle_state nehalem_cstates[] __initdata = { 171 { 172 .name = "C1", 173 .desc = "MWAIT 0x00", 174 .flags = MWAIT2flg(0x00), 175 .exit_latency = 3, 176 .target_residency = 6, 177 .enter = &intel_idle, 178 .enter_s2idle = intel_idle_s2idle, }, 179 { 180 .name = "C1E", 181 .desc = "MWAIT 0x01", 182 .flags = MWAIT2flg(0x01) | CPUIDLE_FLAG_ALWAYS_ENABLE, 183 .exit_latency = 10, 184 .target_residency = 20, 185 .enter = &intel_idle, 186 .enter_s2idle = intel_idle_s2idle, }, 187 { 188 .name = "C3", 189 .desc = "MWAIT 0x10", 190 .flags = MWAIT2flg(0x10) | CPUIDLE_FLAG_TLB_FLUSHED, 191 .exit_latency = 20, 192 .target_residency = 80, 193 .enter = &intel_idle, 194 .enter_s2idle = intel_idle_s2idle, }, 195 { 196 .name = "C6", 197 .desc = "MWAIT 0x20", 198 .flags = MWAIT2flg(0x20) | CPUIDLE_FLAG_TLB_FLUSHED, 199 .exit_latency = 200, 200 .target_residency = 800, 201 .enter = &intel_idle, 202 .enter_s2idle = intel_idle_s2idle, }, 203 { 204 .enter = NULL } 205 }; 206 207 static struct cpuidle_state snb_cstates[] __initdata = { 208 { 209 .name = "C1", 210 .desc = "MWAIT 0x00", 211 .flags = MWAIT2flg(0x00), 212 .exit_latency = 2, 213 .target_residency = 2, 214 .enter = &intel_idle, 215 .enter_s2idle = intel_idle_s2idle, }, 216 { 217 .name = "C1E", 218 .desc = "MWAIT 0x01", 219 .flags = MWAIT2flg(0x01) | CPUIDLE_FLAG_ALWAYS_ENABLE, 220 .exit_latency = 10, 221 .target_residency = 20, 222 .enter = &intel_idle, 223 .enter_s2idle = intel_idle_s2idle, }, 224 { 225 .name = "C3", 226 .desc = "MWAIT 0x10", 227 .flags = MWAIT2flg(0x10) | CPUIDLE_FLAG_TLB_FLUSHED, 228 .exit_latency = 80, 229 .target_residency = 211, 230 .enter = &intel_idle, 231 .enter_s2idle = intel_idle_s2idle, }, 232 { 233 .name = "C6", 234 .desc = "MWAIT 0x20", 235 .flags = MWAIT2flg(0x20) | CPUIDLE_FLAG_TLB_FLUSHED, 236 .exit_latency = 104, 237 .target_residency = 345, 238 .enter = &intel_idle, 239 .enter_s2idle = intel_idle_s2idle, }, 240 { 241 .name = "C7", 242 .desc = "MWAIT 0x30", 243 .flags = MWAIT2flg(0x30) | CPUIDLE_FLAG_TLB_FLUSHED, 244 .exit_latency = 109, 245 .target_residency = 345, 246 .enter = &intel_idle, 247 .enter_s2idle = intel_idle_s2idle, }, 248 { 249 .enter = NULL } 250 }; 251 252 static struct cpuidle_state byt_cstates[] __initdata = { 253 { 254 .name = "C1", 255 .desc = "MWAIT 0x00", 256 .flags = MWAIT2flg(0x00), 257 .exit_latency = 1, 258 .target_residency = 1, 259 .enter = &intel_idle, 260 .enter_s2idle = intel_idle_s2idle, }, 261 { 262 .name = "C6N", 263 .desc = "MWAIT 0x58", 264 .flags = MWAIT2flg(0x58) | CPUIDLE_FLAG_TLB_FLUSHED, 265 .exit_latency = 300, 266 .target_residency = 275, 267 .enter = &intel_idle, 268 .enter_s2idle = intel_idle_s2idle, }, 269 { 270 .name = "C6S", 271 .desc = "MWAIT 0x52", 272 .flags = MWAIT2flg(0x52) | CPUIDLE_FLAG_TLB_FLUSHED, 273 .exit_latency = 500, 274 .target_residency = 560, 275 .enter = &intel_idle, 276 .enter_s2idle = intel_idle_s2idle, }, 277 { 278 .name = "C7", 279 .desc = "MWAIT 0x60", 280 .flags = MWAIT2flg(0x60) | CPUIDLE_FLAG_TLB_FLUSHED, 281 .exit_latency = 1200, 282 .target_residency = 4000, 283 .enter = &intel_idle, 284 .enter_s2idle = intel_idle_s2idle, }, 285 { 286 .name = "C7S", 287 .desc = "MWAIT 0x64", 288 .flags = MWAIT2flg(0x64) | CPUIDLE_FLAG_TLB_FLUSHED, 289 .exit_latency = 10000, 290 .target_residency = 20000, 291 .enter = &intel_idle, 292 .enter_s2idle = intel_idle_s2idle, }, 293 { 294 .enter = NULL } 295 }; 296 297 static struct cpuidle_state cht_cstates[] __initdata = { 298 { 299 .name = "C1", 300 .desc = "MWAIT 0x00", 301 .flags = MWAIT2flg(0x00), 302 .exit_latency = 1, 303 .target_residency = 1, 304 .enter = &intel_idle, 305 .enter_s2idle = intel_idle_s2idle, }, 306 { 307 .name = "C6N", 308 .desc = "MWAIT 0x58", 309 .flags = MWAIT2flg(0x58) | CPUIDLE_FLAG_TLB_FLUSHED, 310 .exit_latency = 80, 311 .target_residency = 275, 312 .enter = &intel_idle, 313 .enter_s2idle = intel_idle_s2idle, }, 314 { 315 .name = "C6S", 316 .desc = "MWAIT 0x52", 317 .flags = MWAIT2flg(0x52) | CPUIDLE_FLAG_TLB_FLUSHED, 318 .exit_latency = 200, 319 .target_residency = 560, 320 .enter = &intel_idle, 321 .enter_s2idle = intel_idle_s2idle, }, 322 { 323 .name = "C7", 324 .desc = "MWAIT 0x60", 325 .flags = MWAIT2flg(0x60) | CPUIDLE_FLAG_TLB_FLUSHED, 326 .exit_latency = 1200, 327 .target_residency = 4000, 328 .enter = &intel_idle, 329 .enter_s2idle = intel_idle_s2idle, }, 330 { 331 .name = "C7S", 332 .desc = "MWAIT 0x64", 333 .flags = MWAIT2flg(0x64) | CPUIDLE_FLAG_TLB_FLUSHED, 334 .exit_latency = 10000, 335 .target_residency = 20000, 336 .enter = &intel_idle, 337 .enter_s2idle = intel_idle_s2idle, }, 338 { 339 .enter = NULL } 340 }; 341 342 static struct cpuidle_state ivb_cstates[] __initdata = { 343 { 344 .name = "C1", 345 .desc = "MWAIT 0x00", 346 .flags = MWAIT2flg(0x00), 347 .exit_latency = 1, 348 .target_residency = 1, 349 .enter = &intel_idle, 350 .enter_s2idle = intel_idle_s2idle, }, 351 { 352 .name = "C1E", 353 .desc = "MWAIT 0x01", 354 .flags = MWAIT2flg(0x01) | CPUIDLE_FLAG_ALWAYS_ENABLE, 355 .exit_latency = 10, 356 .target_residency = 20, 357 .enter = &intel_idle, 358 .enter_s2idle = intel_idle_s2idle, }, 359 { 360 .name = "C3", 361 .desc = "MWAIT 0x10", 362 .flags = MWAIT2flg(0x10) | CPUIDLE_FLAG_TLB_FLUSHED, 363 .exit_latency = 59, 364 .target_residency = 156, 365 .enter = &intel_idle, 366 .enter_s2idle = intel_idle_s2idle, }, 367 { 368 .name = "C6", 369 .desc = "MWAIT 0x20", 370 .flags = MWAIT2flg(0x20) | CPUIDLE_FLAG_TLB_FLUSHED, 371 .exit_latency = 80, 372 .target_residency = 300, 373 .enter = &intel_idle, 374 .enter_s2idle = intel_idle_s2idle, }, 375 { 376 .name = "C7", 377 .desc = "MWAIT 0x30", 378 .flags = MWAIT2flg(0x30) | CPUIDLE_FLAG_TLB_FLUSHED, 379 .exit_latency = 87, 380 .target_residency = 300, 381 .enter = &intel_idle, 382 .enter_s2idle = intel_idle_s2idle, }, 383 { 384 .enter = NULL } 385 }; 386 387 static struct cpuidle_state ivt_cstates[] __initdata = { 388 { 389 .name = "C1", 390 .desc = "MWAIT 0x00", 391 .flags = MWAIT2flg(0x00), 392 .exit_latency = 1, 393 .target_residency = 1, 394 .enter = &intel_idle, 395 .enter_s2idle = intel_idle_s2idle, }, 396 { 397 .name = "C1E", 398 .desc = "MWAIT 0x01", 399 .flags = MWAIT2flg(0x01) | CPUIDLE_FLAG_ALWAYS_ENABLE, 400 .exit_latency = 10, 401 .target_residency = 80, 402 .enter = &intel_idle, 403 .enter_s2idle = intel_idle_s2idle, }, 404 { 405 .name = "C3", 406 .desc = "MWAIT 0x10", 407 .flags = MWAIT2flg(0x10) | CPUIDLE_FLAG_TLB_FLUSHED, 408 .exit_latency = 59, 409 .target_residency = 156, 410 .enter = &intel_idle, 411 .enter_s2idle = intel_idle_s2idle, }, 412 { 413 .name = "C6", 414 .desc = "MWAIT 0x20", 415 .flags = MWAIT2flg(0x20) | CPUIDLE_FLAG_TLB_FLUSHED, 416 .exit_latency = 82, 417 .target_residency = 300, 418 .enter = &intel_idle, 419 .enter_s2idle = intel_idle_s2idle, }, 420 { 421 .enter = NULL } 422 }; 423 424 static struct cpuidle_state ivt_cstates_4s[] __initdata = { 425 { 426 .name = "C1", 427 .desc = "MWAIT 0x00", 428 .flags = MWAIT2flg(0x00), 429 .exit_latency = 1, 430 .target_residency = 1, 431 .enter = &intel_idle, 432 .enter_s2idle = intel_idle_s2idle, }, 433 { 434 .name = "C1E", 435 .desc = "MWAIT 0x01", 436 .flags = MWAIT2flg(0x01) | CPUIDLE_FLAG_ALWAYS_ENABLE, 437 .exit_latency = 10, 438 .target_residency = 250, 439 .enter = &intel_idle, 440 .enter_s2idle = intel_idle_s2idle, }, 441 { 442 .name = "C3", 443 .desc = "MWAIT 0x10", 444 .flags = MWAIT2flg(0x10) | CPUIDLE_FLAG_TLB_FLUSHED, 445 .exit_latency = 59, 446 .target_residency = 300, 447 .enter = &intel_idle, 448 .enter_s2idle = intel_idle_s2idle, }, 449 { 450 .name = "C6", 451 .desc = "MWAIT 0x20", 452 .flags = MWAIT2flg(0x20) | CPUIDLE_FLAG_TLB_FLUSHED, 453 .exit_latency = 84, 454 .target_residency = 400, 455 .enter = &intel_idle, 456 .enter_s2idle = intel_idle_s2idle, }, 457 { 458 .enter = NULL } 459 }; 460 461 static struct cpuidle_state ivt_cstates_8s[] __initdata = { 462 { 463 .name = "C1", 464 .desc = "MWAIT 0x00", 465 .flags = MWAIT2flg(0x00), 466 .exit_latency = 1, 467 .target_residency = 1, 468 .enter = &intel_idle, 469 .enter_s2idle = intel_idle_s2idle, }, 470 { 471 .name = "C1E", 472 .desc = "MWAIT 0x01", 473 .flags = MWAIT2flg(0x01) | CPUIDLE_FLAG_ALWAYS_ENABLE, 474 .exit_latency = 10, 475 .target_residency = 500, 476 .enter = &intel_idle, 477 .enter_s2idle = intel_idle_s2idle, }, 478 { 479 .name = "C3", 480 .desc = "MWAIT 0x10", 481 .flags = MWAIT2flg(0x10) | CPUIDLE_FLAG_TLB_FLUSHED, 482 .exit_latency = 59, 483 .target_residency = 600, 484 .enter = &intel_idle, 485 .enter_s2idle = intel_idle_s2idle, }, 486 { 487 .name = "C6", 488 .desc = "MWAIT 0x20", 489 .flags = MWAIT2flg(0x20) | CPUIDLE_FLAG_TLB_FLUSHED, 490 .exit_latency = 88, 491 .target_residency = 700, 492 .enter = &intel_idle, 493 .enter_s2idle = intel_idle_s2idle, }, 494 { 495 .enter = NULL } 496 }; 497 498 static struct cpuidle_state hsw_cstates[] __initdata = { 499 { 500 .name = "C1", 501 .desc = "MWAIT 0x00", 502 .flags = MWAIT2flg(0x00), 503 .exit_latency = 2, 504 .target_residency = 2, 505 .enter = &intel_idle, 506 .enter_s2idle = intel_idle_s2idle, }, 507 { 508 .name = "C1E", 509 .desc = "MWAIT 0x01", 510 .flags = MWAIT2flg(0x01) | CPUIDLE_FLAG_ALWAYS_ENABLE, 511 .exit_latency = 10, 512 .target_residency = 20, 513 .enter = &intel_idle, 514 .enter_s2idle = intel_idle_s2idle, }, 515 { 516 .name = "C3", 517 .desc = "MWAIT 0x10", 518 .flags = MWAIT2flg(0x10) | CPUIDLE_FLAG_TLB_FLUSHED, 519 .exit_latency = 33, 520 .target_residency = 100, 521 .enter = &intel_idle, 522 .enter_s2idle = intel_idle_s2idle, }, 523 { 524 .name = "C6", 525 .desc = "MWAIT 0x20", 526 .flags = MWAIT2flg(0x20) | CPUIDLE_FLAG_TLB_FLUSHED, 527 .exit_latency = 133, 528 .target_residency = 400, 529 .enter = &intel_idle, 530 .enter_s2idle = intel_idle_s2idle, }, 531 { 532 .name = "C7s", 533 .desc = "MWAIT 0x32", 534 .flags = MWAIT2flg(0x32) | CPUIDLE_FLAG_TLB_FLUSHED, 535 .exit_latency = 166, 536 .target_residency = 500, 537 .enter = &intel_idle, 538 .enter_s2idle = intel_idle_s2idle, }, 539 { 540 .name = "C8", 541 .desc = "MWAIT 0x40", 542 .flags = MWAIT2flg(0x40) | CPUIDLE_FLAG_TLB_FLUSHED, 543 .exit_latency = 300, 544 .target_residency = 900, 545 .enter = &intel_idle, 546 .enter_s2idle = intel_idle_s2idle, }, 547 { 548 .name = "C9", 549 .desc = "MWAIT 0x50", 550 .flags = MWAIT2flg(0x50) | CPUIDLE_FLAG_TLB_FLUSHED, 551 .exit_latency = 600, 552 .target_residency = 1800, 553 .enter = &intel_idle, 554 .enter_s2idle = intel_idle_s2idle, }, 555 { 556 .name = "C10", 557 .desc = "MWAIT 0x60", 558 .flags = MWAIT2flg(0x60) | CPUIDLE_FLAG_TLB_FLUSHED, 559 .exit_latency = 2600, 560 .target_residency = 7700, 561 .enter = &intel_idle, 562 .enter_s2idle = intel_idle_s2idle, }, 563 { 564 .enter = NULL } 565 }; 566 static struct cpuidle_state bdw_cstates[] __initdata = { 567 { 568 .name = "C1", 569 .desc = "MWAIT 0x00", 570 .flags = MWAIT2flg(0x00), 571 .exit_latency = 2, 572 .target_residency = 2, 573 .enter = &intel_idle, 574 .enter_s2idle = intel_idle_s2idle, }, 575 { 576 .name = "C1E", 577 .desc = "MWAIT 0x01", 578 .flags = MWAIT2flg(0x01) | CPUIDLE_FLAG_ALWAYS_ENABLE, 579 .exit_latency = 10, 580 .target_residency = 20, 581 .enter = &intel_idle, 582 .enter_s2idle = intel_idle_s2idle, }, 583 { 584 .name = "C3", 585 .desc = "MWAIT 0x10", 586 .flags = MWAIT2flg(0x10) | CPUIDLE_FLAG_TLB_FLUSHED, 587 .exit_latency = 40, 588 .target_residency = 100, 589 .enter = &intel_idle, 590 .enter_s2idle = intel_idle_s2idle, }, 591 { 592 .name = "C6", 593 .desc = "MWAIT 0x20", 594 .flags = MWAIT2flg(0x20) | CPUIDLE_FLAG_TLB_FLUSHED, 595 .exit_latency = 133, 596 .target_residency = 400, 597 .enter = &intel_idle, 598 .enter_s2idle = intel_idle_s2idle, }, 599 { 600 .name = "C7s", 601 .desc = "MWAIT 0x32", 602 .flags = MWAIT2flg(0x32) | CPUIDLE_FLAG_TLB_FLUSHED, 603 .exit_latency = 166, 604 .target_residency = 500, 605 .enter = &intel_idle, 606 .enter_s2idle = intel_idle_s2idle, }, 607 { 608 .name = "C8", 609 .desc = "MWAIT 0x40", 610 .flags = MWAIT2flg(0x40) | CPUIDLE_FLAG_TLB_FLUSHED, 611 .exit_latency = 300, 612 .target_residency = 900, 613 .enter = &intel_idle, 614 .enter_s2idle = intel_idle_s2idle, }, 615 { 616 .name = "C9", 617 .desc = "MWAIT 0x50", 618 .flags = MWAIT2flg(0x50) | CPUIDLE_FLAG_TLB_FLUSHED, 619 .exit_latency = 600, 620 .target_residency = 1800, 621 .enter = &intel_idle, 622 .enter_s2idle = intel_idle_s2idle, }, 623 { 624 .name = "C10", 625 .desc = "MWAIT 0x60", 626 .flags = MWAIT2flg(0x60) | CPUIDLE_FLAG_TLB_FLUSHED, 627 .exit_latency = 2600, 628 .target_residency = 7700, 629 .enter = &intel_idle, 630 .enter_s2idle = intel_idle_s2idle, }, 631 { 632 .enter = NULL } 633 }; 634 635 static struct cpuidle_state skl_cstates[] __initdata = { 636 { 637 .name = "C1", 638 .desc = "MWAIT 0x00", 639 .flags = MWAIT2flg(0x00), 640 .exit_latency = 2, 641 .target_residency = 2, 642 .enter = &intel_idle, 643 .enter_s2idle = intel_idle_s2idle, }, 644 { 645 .name = "C1E", 646 .desc = "MWAIT 0x01", 647 .flags = MWAIT2flg(0x01) | CPUIDLE_FLAG_ALWAYS_ENABLE, 648 .exit_latency = 10, 649 .target_residency = 20, 650 .enter = &intel_idle, 651 .enter_s2idle = intel_idle_s2idle, }, 652 { 653 .name = "C3", 654 .desc = "MWAIT 0x10", 655 .flags = MWAIT2flg(0x10) | CPUIDLE_FLAG_TLB_FLUSHED, 656 .exit_latency = 70, 657 .target_residency = 100, 658 .enter = &intel_idle, 659 .enter_s2idle = intel_idle_s2idle, }, 660 { 661 .name = "C6", 662 .desc = "MWAIT 0x20", 663 .flags = MWAIT2flg(0x20) | CPUIDLE_FLAG_TLB_FLUSHED, 664 .exit_latency = 85, 665 .target_residency = 200, 666 .enter = &intel_idle, 667 .enter_s2idle = intel_idle_s2idle, }, 668 { 669 .name = "C7s", 670 .desc = "MWAIT 0x33", 671 .flags = MWAIT2flg(0x33) | CPUIDLE_FLAG_TLB_FLUSHED, 672 .exit_latency = 124, 673 .target_residency = 800, 674 .enter = &intel_idle, 675 .enter_s2idle = intel_idle_s2idle, }, 676 { 677 .name = "C8", 678 .desc = "MWAIT 0x40", 679 .flags = MWAIT2flg(0x40) | CPUIDLE_FLAG_TLB_FLUSHED, 680 .exit_latency = 200, 681 .target_residency = 800, 682 .enter = &intel_idle, 683 .enter_s2idle = intel_idle_s2idle, }, 684 { 685 .name = "C9", 686 .desc = "MWAIT 0x50", 687 .flags = MWAIT2flg(0x50) | CPUIDLE_FLAG_TLB_FLUSHED, 688 .exit_latency = 480, 689 .target_residency = 5000, 690 .enter = &intel_idle, 691 .enter_s2idle = intel_idle_s2idle, }, 692 { 693 .name = "C10", 694 .desc = "MWAIT 0x60", 695 .flags = MWAIT2flg(0x60) | CPUIDLE_FLAG_TLB_FLUSHED, 696 .exit_latency = 890, 697 .target_residency = 5000, 698 .enter = &intel_idle, 699 .enter_s2idle = intel_idle_s2idle, }, 700 { 701 .enter = NULL } 702 }; 703 704 static struct cpuidle_state skx_cstates[] __initdata = { 705 { 706 .name = "C1", 707 .desc = "MWAIT 0x00", 708 .flags = MWAIT2flg(0x00) | CPUIDLE_FLAG_IRQ_ENABLE, 709 .exit_latency = 2, 710 .target_residency = 2, 711 .enter = &intel_idle, 712 .enter_s2idle = intel_idle_s2idle, }, 713 { 714 .name = "C1E", 715 .desc = "MWAIT 0x01", 716 .flags = MWAIT2flg(0x01) | CPUIDLE_FLAG_ALWAYS_ENABLE, 717 .exit_latency = 10, 718 .target_residency = 20, 719 .enter = &intel_idle, 720 .enter_s2idle = intel_idle_s2idle, }, 721 { 722 .name = "C6", 723 .desc = "MWAIT 0x20", 724 .flags = MWAIT2flg(0x20) | CPUIDLE_FLAG_TLB_FLUSHED, 725 .exit_latency = 133, 726 .target_residency = 600, 727 .enter = &intel_idle, 728 .enter_s2idle = intel_idle_s2idle, }, 729 { 730 .enter = NULL } 731 }; 732 733 static struct cpuidle_state icx_cstates[] __initdata = { 734 { 735 .name = "C1", 736 .desc = "MWAIT 0x00", 737 .flags = MWAIT2flg(0x00) | CPUIDLE_FLAG_IRQ_ENABLE, 738 .exit_latency = 1, 739 .target_residency = 1, 740 .enter = &intel_idle, 741 .enter_s2idle = intel_idle_s2idle, }, 742 { 743 .name = "C1E", 744 .desc = "MWAIT 0x01", 745 .flags = MWAIT2flg(0x01) | CPUIDLE_FLAG_ALWAYS_ENABLE, 746 .exit_latency = 4, 747 .target_residency = 4, 748 .enter = &intel_idle, 749 .enter_s2idle = intel_idle_s2idle, }, 750 { 751 .name = "C6", 752 .desc = "MWAIT 0x20", 753 .flags = MWAIT2flg(0x20) | CPUIDLE_FLAG_TLB_FLUSHED, 754 .exit_latency = 170, 755 .target_residency = 600, 756 .enter = &intel_idle, 757 .enter_s2idle = intel_idle_s2idle, }, 758 { 759 .enter = NULL } 760 }; 761 762 /* 763 * On Sapphire Rapids Xeon C1 has to be disabled if C1E is enabled, and vice 764 * versa. On SPR C1E is enabled only if "C1E promotion" bit is set in 765 * MSR_IA32_POWER_CTL. But in this case there effectively no C1, because C1 766 * requests are promoted to C1E. If the "C1E promotion" bit is cleared, then 767 * both C1 and C1E requests end up with C1, so there is effectively no C1E. 768 * 769 * By default we enable C1 and disable C1E by marking it with 770 * 'CPUIDLE_FLAG_UNUSABLE'. 771 */ 772 static struct cpuidle_state spr_cstates[] __initdata = { 773 { 774 .name = "C1", 775 .desc = "MWAIT 0x00", 776 .flags = MWAIT2flg(0x00), 777 .exit_latency = 1, 778 .target_residency = 1, 779 .enter = &intel_idle, 780 .enter_s2idle = intel_idle_s2idle, }, 781 { 782 .name = "C1E", 783 .desc = "MWAIT 0x01", 784 .flags = MWAIT2flg(0x01) | CPUIDLE_FLAG_ALWAYS_ENABLE | 785 CPUIDLE_FLAG_UNUSABLE, 786 .exit_latency = 2, 787 .target_residency = 4, 788 .enter = &intel_idle, 789 .enter_s2idle = intel_idle_s2idle, }, 790 { 791 .name = "C6", 792 .desc = "MWAIT 0x20", 793 .flags = MWAIT2flg(0x20) | CPUIDLE_FLAG_TLB_FLUSHED, 794 .exit_latency = 290, 795 .target_residency = 800, 796 .enter = &intel_idle, 797 .enter_s2idle = intel_idle_s2idle, }, 798 { 799 .enter = NULL } 800 }; 801 802 static struct cpuidle_state atom_cstates[] __initdata = { 803 { 804 .name = "C1E", 805 .desc = "MWAIT 0x00", 806 .flags = MWAIT2flg(0x00), 807 .exit_latency = 10, 808 .target_residency = 20, 809 .enter = &intel_idle, 810 .enter_s2idle = intel_idle_s2idle, }, 811 { 812 .name = "C2", 813 .desc = "MWAIT 0x10", 814 .flags = MWAIT2flg(0x10), 815 .exit_latency = 20, 816 .target_residency = 80, 817 .enter = &intel_idle, 818 .enter_s2idle = intel_idle_s2idle, }, 819 { 820 .name = "C4", 821 .desc = "MWAIT 0x30", 822 .flags = MWAIT2flg(0x30) | CPUIDLE_FLAG_TLB_FLUSHED, 823 .exit_latency = 100, 824 .target_residency = 400, 825 .enter = &intel_idle, 826 .enter_s2idle = intel_idle_s2idle, }, 827 { 828 .name = "C6", 829 .desc = "MWAIT 0x52", 830 .flags = MWAIT2flg(0x52) | CPUIDLE_FLAG_TLB_FLUSHED, 831 .exit_latency = 140, 832 .target_residency = 560, 833 .enter = &intel_idle, 834 .enter_s2idle = intel_idle_s2idle, }, 835 { 836 .enter = NULL } 837 }; 838 static struct cpuidle_state tangier_cstates[] __initdata = { 839 { 840 .name = "C1", 841 .desc = "MWAIT 0x00", 842 .flags = MWAIT2flg(0x00), 843 .exit_latency = 1, 844 .target_residency = 4, 845 .enter = &intel_idle, 846 .enter_s2idle = intel_idle_s2idle, }, 847 { 848 .name = "C4", 849 .desc = "MWAIT 0x30", 850 .flags = MWAIT2flg(0x30) | CPUIDLE_FLAG_TLB_FLUSHED, 851 .exit_latency = 100, 852 .target_residency = 400, 853 .enter = &intel_idle, 854 .enter_s2idle = intel_idle_s2idle, }, 855 { 856 .name = "C6", 857 .desc = "MWAIT 0x52", 858 .flags = MWAIT2flg(0x52) | CPUIDLE_FLAG_TLB_FLUSHED, 859 .exit_latency = 140, 860 .target_residency = 560, 861 .enter = &intel_idle, 862 .enter_s2idle = intel_idle_s2idle, }, 863 { 864 .name = "C7", 865 .desc = "MWAIT 0x60", 866 .flags = MWAIT2flg(0x60) | CPUIDLE_FLAG_TLB_FLUSHED, 867 .exit_latency = 1200, 868 .target_residency = 4000, 869 .enter = &intel_idle, 870 .enter_s2idle = intel_idle_s2idle, }, 871 { 872 .name = "C9", 873 .desc = "MWAIT 0x64", 874 .flags = MWAIT2flg(0x64) | CPUIDLE_FLAG_TLB_FLUSHED, 875 .exit_latency = 10000, 876 .target_residency = 20000, 877 .enter = &intel_idle, 878 .enter_s2idle = intel_idle_s2idle, }, 879 { 880 .enter = NULL } 881 }; 882 static struct cpuidle_state avn_cstates[] __initdata = { 883 { 884 .name = "C1", 885 .desc = "MWAIT 0x00", 886 .flags = MWAIT2flg(0x00), 887 .exit_latency = 2, 888 .target_residency = 2, 889 .enter = &intel_idle, 890 .enter_s2idle = intel_idle_s2idle, }, 891 { 892 .name = "C6", 893 .desc = "MWAIT 0x51", 894 .flags = MWAIT2flg(0x51) | CPUIDLE_FLAG_TLB_FLUSHED, 895 .exit_latency = 15, 896 .target_residency = 45, 897 .enter = &intel_idle, 898 .enter_s2idle = intel_idle_s2idle, }, 899 { 900 .enter = NULL } 901 }; 902 static struct cpuidle_state knl_cstates[] __initdata = { 903 { 904 .name = "C1", 905 .desc = "MWAIT 0x00", 906 .flags = MWAIT2flg(0x00), 907 .exit_latency = 1, 908 .target_residency = 2, 909 .enter = &intel_idle, 910 .enter_s2idle = intel_idle_s2idle }, 911 { 912 .name = "C6", 913 .desc = "MWAIT 0x10", 914 .flags = MWAIT2flg(0x10) | CPUIDLE_FLAG_TLB_FLUSHED, 915 .exit_latency = 120, 916 .target_residency = 500, 917 .enter = &intel_idle, 918 .enter_s2idle = intel_idle_s2idle }, 919 { 920 .enter = NULL } 921 }; 922 923 static struct cpuidle_state bxt_cstates[] __initdata = { 924 { 925 .name = "C1", 926 .desc = "MWAIT 0x00", 927 .flags = MWAIT2flg(0x00), 928 .exit_latency = 2, 929 .target_residency = 2, 930 .enter = &intel_idle, 931 .enter_s2idle = intel_idle_s2idle, }, 932 { 933 .name = "C1E", 934 .desc = "MWAIT 0x01", 935 .flags = MWAIT2flg(0x01) | CPUIDLE_FLAG_ALWAYS_ENABLE, 936 .exit_latency = 10, 937 .target_residency = 20, 938 .enter = &intel_idle, 939 .enter_s2idle = intel_idle_s2idle, }, 940 { 941 .name = "C6", 942 .desc = "MWAIT 0x20", 943 .flags = MWAIT2flg(0x20) | CPUIDLE_FLAG_TLB_FLUSHED, 944 .exit_latency = 133, 945 .target_residency = 133, 946 .enter = &intel_idle, 947 .enter_s2idle = intel_idle_s2idle, }, 948 { 949 .name = "C7s", 950 .desc = "MWAIT 0x31", 951 .flags = MWAIT2flg(0x31) | CPUIDLE_FLAG_TLB_FLUSHED, 952 .exit_latency = 155, 953 .target_residency = 155, 954 .enter = &intel_idle, 955 .enter_s2idle = intel_idle_s2idle, }, 956 { 957 .name = "C8", 958 .desc = "MWAIT 0x40", 959 .flags = MWAIT2flg(0x40) | CPUIDLE_FLAG_TLB_FLUSHED, 960 .exit_latency = 1000, 961 .target_residency = 1000, 962 .enter = &intel_idle, 963 .enter_s2idle = intel_idle_s2idle, }, 964 { 965 .name = "C9", 966 .desc = "MWAIT 0x50", 967 .flags = MWAIT2flg(0x50) | CPUIDLE_FLAG_TLB_FLUSHED, 968 .exit_latency = 2000, 969 .target_residency = 2000, 970 .enter = &intel_idle, 971 .enter_s2idle = intel_idle_s2idle, }, 972 { 973 .name = "C10", 974 .desc = "MWAIT 0x60", 975 .flags = MWAIT2flg(0x60) | CPUIDLE_FLAG_TLB_FLUSHED, 976 .exit_latency = 10000, 977 .target_residency = 10000, 978 .enter = &intel_idle, 979 .enter_s2idle = intel_idle_s2idle, }, 980 { 981 .enter = NULL } 982 }; 983 984 static struct cpuidle_state dnv_cstates[] __initdata = { 985 { 986 .name = "C1", 987 .desc = "MWAIT 0x00", 988 .flags = MWAIT2flg(0x00), 989 .exit_latency = 2, 990 .target_residency = 2, 991 .enter = &intel_idle, 992 .enter_s2idle = intel_idle_s2idle, }, 993 { 994 .name = "C1E", 995 .desc = "MWAIT 0x01", 996 .flags = MWAIT2flg(0x01) | CPUIDLE_FLAG_ALWAYS_ENABLE, 997 .exit_latency = 10, 998 .target_residency = 20, 999 .enter = &intel_idle, 1000 .enter_s2idle = intel_idle_s2idle, }, 1001 { 1002 .name = "C6", 1003 .desc = "MWAIT 0x20", 1004 .flags = MWAIT2flg(0x20) | CPUIDLE_FLAG_TLB_FLUSHED, 1005 .exit_latency = 50, 1006 .target_residency = 500, 1007 .enter = &intel_idle, 1008 .enter_s2idle = intel_idle_s2idle, }, 1009 { 1010 .enter = NULL } 1011 }; 1012 1013 /* 1014 * Note, depending on HW and FW revision, SnowRidge SoC may or may not support 1015 * C6, and this is indicated in the CPUID mwait leaf. 1016 */ 1017 static struct cpuidle_state snr_cstates[] __initdata = { 1018 { 1019 .name = "C1", 1020 .desc = "MWAIT 0x00", 1021 .flags = MWAIT2flg(0x00), 1022 .exit_latency = 2, 1023 .target_residency = 2, 1024 .enter = &intel_idle, 1025 .enter_s2idle = intel_idle_s2idle, }, 1026 { 1027 .name = "C1E", 1028 .desc = "MWAIT 0x01", 1029 .flags = MWAIT2flg(0x01) | CPUIDLE_FLAG_ALWAYS_ENABLE, 1030 .exit_latency = 15, 1031 .target_residency = 25, 1032 .enter = &intel_idle, 1033 .enter_s2idle = intel_idle_s2idle, }, 1034 { 1035 .name = "C6", 1036 .desc = "MWAIT 0x20", 1037 .flags = MWAIT2flg(0x20) | CPUIDLE_FLAG_TLB_FLUSHED, 1038 .exit_latency = 130, 1039 .target_residency = 500, 1040 .enter = &intel_idle, 1041 .enter_s2idle = intel_idle_s2idle, }, 1042 { 1043 .enter = NULL } 1044 }; 1045 1046 static const struct idle_cpu idle_cpu_nehalem __initconst = { 1047 .state_table = nehalem_cstates, 1048 .auto_demotion_disable_flags = NHM_C1_AUTO_DEMOTE | NHM_C3_AUTO_DEMOTE, 1049 .disable_promotion_to_c1e = true, 1050 }; 1051 1052 static const struct idle_cpu idle_cpu_nhx __initconst = { 1053 .state_table = nehalem_cstates, 1054 .auto_demotion_disable_flags = NHM_C1_AUTO_DEMOTE | NHM_C3_AUTO_DEMOTE, 1055 .disable_promotion_to_c1e = true, 1056 .use_acpi = true, 1057 }; 1058 1059 static const struct idle_cpu idle_cpu_atom __initconst = { 1060 .state_table = atom_cstates, 1061 }; 1062 1063 static const struct idle_cpu idle_cpu_tangier __initconst = { 1064 .state_table = tangier_cstates, 1065 }; 1066 1067 static const struct idle_cpu idle_cpu_lincroft __initconst = { 1068 .state_table = atom_cstates, 1069 .auto_demotion_disable_flags = ATM_LNC_C6_AUTO_DEMOTE, 1070 }; 1071 1072 static const struct idle_cpu idle_cpu_snb __initconst = { 1073 .state_table = snb_cstates, 1074 .disable_promotion_to_c1e = true, 1075 }; 1076 1077 static const struct idle_cpu idle_cpu_snx __initconst = { 1078 .state_table = snb_cstates, 1079 .disable_promotion_to_c1e = true, 1080 .use_acpi = true, 1081 }; 1082 1083 static const struct idle_cpu idle_cpu_byt __initconst = { 1084 .state_table = byt_cstates, 1085 .disable_promotion_to_c1e = true, 1086 .byt_auto_demotion_disable_flag = true, 1087 }; 1088 1089 static const struct idle_cpu idle_cpu_cht __initconst = { 1090 .state_table = cht_cstates, 1091 .disable_promotion_to_c1e = true, 1092 .byt_auto_demotion_disable_flag = true, 1093 }; 1094 1095 static const struct idle_cpu idle_cpu_ivb __initconst = { 1096 .state_table = ivb_cstates, 1097 .disable_promotion_to_c1e = true, 1098 }; 1099 1100 static const struct idle_cpu idle_cpu_ivt __initconst = { 1101 .state_table = ivt_cstates, 1102 .disable_promotion_to_c1e = true, 1103 .use_acpi = true, 1104 }; 1105 1106 static const struct idle_cpu idle_cpu_hsw __initconst = { 1107 .state_table = hsw_cstates, 1108 .disable_promotion_to_c1e = true, 1109 }; 1110 1111 static const struct idle_cpu idle_cpu_hsx __initconst = { 1112 .state_table = hsw_cstates, 1113 .disable_promotion_to_c1e = true, 1114 .use_acpi = true, 1115 }; 1116 1117 static const struct idle_cpu idle_cpu_bdw __initconst = { 1118 .state_table = bdw_cstates, 1119 .disable_promotion_to_c1e = true, 1120 }; 1121 1122 static const struct idle_cpu idle_cpu_bdx __initconst = { 1123 .state_table = bdw_cstates, 1124 .disable_promotion_to_c1e = true, 1125 .use_acpi = true, 1126 }; 1127 1128 static const struct idle_cpu idle_cpu_skl __initconst = { 1129 .state_table = skl_cstates, 1130 .disable_promotion_to_c1e = true, 1131 }; 1132 1133 static const struct idle_cpu idle_cpu_skx __initconst = { 1134 .state_table = skx_cstates, 1135 .disable_promotion_to_c1e = true, 1136 .use_acpi = true, 1137 }; 1138 1139 static const struct idle_cpu idle_cpu_icx __initconst = { 1140 .state_table = icx_cstates, 1141 .disable_promotion_to_c1e = true, 1142 .use_acpi = true, 1143 }; 1144 1145 static const struct idle_cpu idle_cpu_spr __initconst = { 1146 .state_table = spr_cstates, 1147 .disable_promotion_to_c1e = true, 1148 .use_acpi = true, 1149 }; 1150 1151 static const struct idle_cpu idle_cpu_avn __initconst = { 1152 .state_table = avn_cstates, 1153 .disable_promotion_to_c1e = true, 1154 .use_acpi = true, 1155 }; 1156 1157 static const struct idle_cpu idle_cpu_knl __initconst = { 1158 .state_table = knl_cstates, 1159 .use_acpi = true, 1160 }; 1161 1162 static const struct idle_cpu idle_cpu_bxt __initconst = { 1163 .state_table = bxt_cstates, 1164 .disable_promotion_to_c1e = true, 1165 }; 1166 1167 static const struct idle_cpu idle_cpu_dnv __initconst = { 1168 .state_table = dnv_cstates, 1169 .disable_promotion_to_c1e = true, 1170 .use_acpi = true, 1171 }; 1172 1173 static const struct idle_cpu idle_cpu_snr __initconst = { 1174 .state_table = snr_cstates, 1175 .disable_promotion_to_c1e = true, 1176 .use_acpi = true, 1177 }; 1178 1179 static const struct x86_cpu_id intel_idle_ids[] __initconst = { 1180 X86_MATCH_INTEL_FAM6_MODEL(NEHALEM_EP, &idle_cpu_nhx), 1181 X86_MATCH_INTEL_FAM6_MODEL(NEHALEM, &idle_cpu_nehalem), 1182 X86_MATCH_INTEL_FAM6_MODEL(NEHALEM_G, &idle_cpu_nehalem), 1183 X86_MATCH_INTEL_FAM6_MODEL(WESTMERE, &idle_cpu_nehalem), 1184 X86_MATCH_INTEL_FAM6_MODEL(WESTMERE_EP, &idle_cpu_nhx), 1185 X86_MATCH_INTEL_FAM6_MODEL(NEHALEM_EX, &idle_cpu_nhx), 1186 X86_MATCH_INTEL_FAM6_MODEL(ATOM_BONNELL, &idle_cpu_atom), 1187 X86_MATCH_INTEL_FAM6_MODEL(ATOM_BONNELL_MID, &idle_cpu_lincroft), 1188 X86_MATCH_INTEL_FAM6_MODEL(WESTMERE_EX, &idle_cpu_nhx), 1189 X86_MATCH_INTEL_FAM6_MODEL(SANDYBRIDGE, &idle_cpu_snb), 1190 X86_MATCH_INTEL_FAM6_MODEL(SANDYBRIDGE_X, &idle_cpu_snx), 1191 X86_MATCH_INTEL_FAM6_MODEL(ATOM_SALTWELL, &idle_cpu_atom), 1192 X86_MATCH_INTEL_FAM6_MODEL(ATOM_SILVERMONT, &idle_cpu_byt), 1193 X86_MATCH_INTEL_FAM6_MODEL(ATOM_SILVERMONT_MID, &idle_cpu_tangier), 1194 X86_MATCH_INTEL_FAM6_MODEL(ATOM_AIRMONT, &idle_cpu_cht), 1195 X86_MATCH_INTEL_FAM6_MODEL(IVYBRIDGE, &idle_cpu_ivb), 1196 X86_MATCH_INTEL_FAM6_MODEL(IVYBRIDGE_X, &idle_cpu_ivt), 1197 X86_MATCH_INTEL_FAM6_MODEL(HASWELL, &idle_cpu_hsw), 1198 X86_MATCH_INTEL_FAM6_MODEL(HASWELL_X, &idle_cpu_hsx), 1199 X86_MATCH_INTEL_FAM6_MODEL(HASWELL_L, &idle_cpu_hsw), 1200 X86_MATCH_INTEL_FAM6_MODEL(HASWELL_G, &idle_cpu_hsw), 1201 X86_MATCH_INTEL_FAM6_MODEL(ATOM_SILVERMONT_D, &idle_cpu_avn), 1202 X86_MATCH_INTEL_FAM6_MODEL(BROADWELL, &idle_cpu_bdw), 1203 X86_MATCH_INTEL_FAM6_MODEL(BROADWELL_G, &idle_cpu_bdw), 1204 X86_MATCH_INTEL_FAM6_MODEL(BROADWELL_X, &idle_cpu_bdx), 1205 X86_MATCH_INTEL_FAM6_MODEL(BROADWELL_D, &idle_cpu_bdx), 1206 X86_MATCH_INTEL_FAM6_MODEL(SKYLAKE_L, &idle_cpu_skl), 1207 X86_MATCH_INTEL_FAM6_MODEL(SKYLAKE, &idle_cpu_skl), 1208 X86_MATCH_INTEL_FAM6_MODEL(KABYLAKE_L, &idle_cpu_skl), 1209 X86_MATCH_INTEL_FAM6_MODEL(KABYLAKE, &idle_cpu_skl), 1210 X86_MATCH_INTEL_FAM6_MODEL(SKYLAKE_X, &idle_cpu_skx), 1211 X86_MATCH_INTEL_FAM6_MODEL(ICELAKE_X, &idle_cpu_icx), 1212 X86_MATCH_INTEL_FAM6_MODEL(ICELAKE_D, &idle_cpu_icx), 1213 X86_MATCH_INTEL_FAM6_MODEL(SAPPHIRERAPIDS_X, &idle_cpu_spr), 1214 X86_MATCH_INTEL_FAM6_MODEL(XEON_PHI_KNL, &idle_cpu_knl), 1215 X86_MATCH_INTEL_FAM6_MODEL(XEON_PHI_KNM, &idle_cpu_knl), 1216 X86_MATCH_INTEL_FAM6_MODEL(ATOM_GOLDMONT, &idle_cpu_bxt), 1217 X86_MATCH_INTEL_FAM6_MODEL(ATOM_GOLDMONT_PLUS, &idle_cpu_bxt), 1218 X86_MATCH_INTEL_FAM6_MODEL(ATOM_GOLDMONT_D, &idle_cpu_dnv), 1219 X86_MATCH_INTEL_FAM6_MODEL(ATOM_TREMONT_D, &idle_cpu_snr), 1220 {} 1221 }; 1222 1223 static const struct x86_cpu_id intel_mwait_ids[] __initconst = { 1224 X86_MATCH_VENDOR_FAM_FEATURE(INTEL, 6, X86_FEATURE_MWAIT, NULL), 1225 {} 1226 }; 1227 1228 static bool __init intel_idle_max_cstate_reached(int cstate) 1229 { 1230 if (cstate + 1 > max_cstate) { 1231 pr_info("max_cstate %d reached\n", max_cstate); 1232 return true; 1233 } 1234 return false; 1235 } 1236 1237 static bool __init intel_idle_state_needs_timer_stop(struct cpuidle_state *state) 1238 { 1239 unsigned long eax = flg2MWAIT(state->flags); 1240 1241 if (boot_cpu_has(X86_FEATURE_ARAT)) 1242 return false; 1243 1244 /* 1245 * Switch over to one-shot tick broadcast if the target C-state 1246 * is deeper than C1. 1247 */ 1248 return !!((eax >> MWAIT_SUBSTATE_SIZE) & MWAIT_CSTATE_MASK); 1249 } 1250 1251 #ifdef CONFIG_ACPI_PROCESSOR_CSTATE 1252 #include <acpi/processor.h> 1253 1254 static bool no_acpi __read_mostly; 1255 module_param(no_acpi, bool, 0444); 1256 MODULE_PARM_DESC(no_acpi, "Do not use ACPI _CST for building the idle states list"); 1257 1258 static bool force_use_acpi __read_mostly; /* No effect if no_acpi is set. */ 1259 module_param_named(use_acpi, force_use_acpi, bool, 0444); 1260 MODULE_PARM_DESC(use_acpi, "Use ACPI _CST for building the idle states list"); 1261 1262 static struct acpi_processor_power acpi_state_table __initdata; 1263 1264 /** 1265 * intel_idle_cst_usable - Check if the _CST information can be used. 1266 * 1267 * Check if all of the C-states listed by _CST in the max_cstate range are 1268 * ACPI_CSTATE_FFH, which means that they should be entered via MWAIT. 1269 */ 1270 static bool __init intel_idle_cst_usable(void) 1271 { 1272 int cstate, limit; 1273 1274 limit = min_t(int, min_t(int, CPUIDLE_STATE_MAX, max_cstate + 1), 1275 acpi_state_table.count); 1276 1277 for (cstate = 1; cstate < limit; cstate++) { 1278 struct acpi_processor_cx *cx = &acpi_state_table.states[cstate]; 1279 1280 if (cx->entry_method != ACPI_CSTATE_FFH) 1281 return false; 1282 } 1283 1284 return true; 1285 } 1286 1287 static bool __init intel_idle_acpi_cst_extract(void) 1288 { 1289 unsigned int cpu; 1290 1291 if (no_acpi) { 1292 pr_debug("Not allowed to use ACPI _CST\n"); 1293 return false; 1294 } 1295 1296 for_each_possible_cpu(cpu) { 1297 struct acpi_processor *pr = per_cpu(processors, cpu); 1298 1299 if (!pr) 1300 continue; 1301 1302 if (acpi_processor_evaluate_cst(pr->handle, cpu, &acpi_state_table)) 1303 continue; 1304 1305 acpi_state_table.count++; 1306 1307 if (!intel_idle_cst_usable()) 1308 continue; 1309 1310 if (!acpi_processor_claim_cst_control()) 1311 break; 1312 1313 return true; 1314 } 1315 1316 acpi_state_table.count = 0; 1317 pr_debug("ACPI _CST not found or not usable\n"); 1318 return false; 1319 } 1320 1321 static void __init intel_idle_init_cstates_acpi(struct cpuidle_driver *drv) 1322 { 1323 int cstate, limit = min_t(int, CPUIDLE_STATE_MAX, acpi_state_table.count); 1324 1325 /* 1326 * If limit > 0, intel_idle_cst_usable() has returned 'true', so all of 1327 * the interesting states are ACPI_CSTATE_FFH. 1328 */ 1329 for (cstate = 1; cstate < limit; cstate++) { 1330 struct acpi_processor_cx *cx; 1331 struct cpuidle_state *state; 1332 1333 if (intel_idle_max_cstate_reached(cstate - 1)) 1334 break; 1335 1336 cx = &acpi_state_table.states[cstate]; 1337 1338 state = &drv->states[drv->state_count++]; 1339 1340 snprintf(state->name, CPUIDLE_NAME_LEN, "C%d_ACPI", cstate); 1341 strlcpy(state->desc, cx->desc, CPUIDLE_DESC_LEN); 1342 state->exit_latency = cx->latency; 1343 /* 1344 * For C1-type C-states use the same number for both the exit 1345 * latency and target residency, because that is the case for 1346 * C1 in the majority of the static C-states tables above. 1347 * For the other types of C-states, however, set the target 1348 * residency to 3 times the exit latency which should lead to 1349 * a reasonable balance between energy-efficiency and 1350 * performance in the majority of interesting cases. 1351 */ 1352 state->target_residency = cx->latency; 1353 if (cx->type > ACPI_STATE_C1) 1354 state->target_residency *= 3; 1355 1356 state->flags = MWAIT2flg(cx->address); 1357 if (cx->type > ACPI_STATE_C2) 1358 state->flags |= CPUIDLE_FLAG_TLB_FLUSHED; 1359 1360 if (disabled_states_mask & BIT(cstate)) 1361 state->flags |= CPUIDLE_FLAG_OFF; 1362 1363 if (intel_idle_state_needs_timer_stop(state)) 1364 state->flags |= CPUIDLE_FLAG_TIMER_STOP; 1365 1366 state->enter = intel_idle; 1367 state->enter_s2idle = intel_idle_s2idle; 1368 } 1369 } 1370 1371 static bool __init intel_idle_off_by_default(u32 mwait_hint) 1372 { 1373 int cstate, limit; 1374 1375 /* 1376 * If there are no _CST C-states, do not disable any C-states by 1377 * default. 1378 */ 1379 if (!acpi_state_table.count) 1380 return false; 1381 1382 limit = min_t(int, CPUIDLE_STATE_MAX, acpi_state_table.count); 1383 /* 1384 * If limit > 0, intel_idle_cst_usable() has returned 'true', so all of 1385 * the interesting states are ACPI_CSTATE_FFH. 1386 */ 1387 for (cstate = 1; cstate < limit; cstate++) { 1388 if (acpi_state_table.states[cstate].address == mwait_hint) 1389 return false; 1390 } 1391 return true; 1392 } 1393 #else /* !CONFIG_ACPI_PROCESSOR_CSTATE */ 1394 #define force_use_acpi (false) 1395 1396 static inline bool intel_idle_acpi_cst_extract(void) { return false; } 1397 static inline void intel_idle_init_cstates_acpi(struct cpuidle_driver *drv) { } 1398 static inline bool intel_idle_off_by_default(u32 mwait_hint) { return false; } 1399 #endif /* !CONFIG_ACPI_PROCESSOR_CSTATE */ 1400 1401 static void c1e_promotion_enable(void); 1402 1403 /** 1404 * ivt_idle_state_table_update - Tune the idle states table for Ivy Town. 1405 * 1406 * Tune IVT multi-socket targets. 1407 * Assumption: num_sockets == (max_package_num + 1). 1408 */ 1409 static void __init ivt_idle_state_table_update(void) 1410 { 1411 /* IVT uses a different table for 1-2, 3-4, and > 4 sockets */ 1412 int cpu, package_num, num_sockets = 1; 1413 1414 for_each_online_cpu(cpu) { 1415 package_num = topology_physical_package_id(cpu); 1416 if (package_num + 1 > num_sockets) { 1417 num_sockets = package_num + 1; 1418 1419 if (num_sockets > 4) { 1420 cpuidle_state_table = ivt_cstates_8s; 1421 return; 1422 } 1423 } 1424 } 1425 1426 if (num_sockets > 2) 1427 cpuidle_state_table = ivt_cstates_4s; 1428 1429 /* else, 1 and 2 socket systems use default ivt_cstates */ 1430 } 1431 1432 /** 1433 * irtl_2_usec - IRTL to microseconds conversion. 1434 * @irtl: IRTL MSR value. 1435 * 1436 * Translate the IRTL (Interrupt Response Time Limit) MSR value to microseconds. 1437 */ 1438 static unsigned long long __init irtl_2_usec(unsigned long long irtl) 1439 { 1440 static const unsigned int irtl_ns_units[] __initconst = { 1441 1, 32, 1024, 32768, 1048576, 33554432, 0, 0 1442 }; 1443 unsigned long long ns; 1444 1445 if (!irtl) 1446 return 0; 1447 1448 ns = irtl_ns_units[(irtl >> 10) & 0x7]; 1449 1450 return div_u64((irtl & 0x3FF) * ns, NSEC_PER_USEC); 1451 } 1452 1453 /** 1454 * bxt_idle_state_table_update - Fix up the Broxton idle states table. 1455 * 1456 * On BXT, trust the IRTL (Interrupt Response Time Limit) MSR to show the 1457 * definitive maximum latency and use the same value for target_residency. 1458 */ 1459 static void __init bxt_idle_state_table_update(void) 1460 { 1461 unsigned long long msr; 1462 unsigned int usec; 1463 1464 rdmsrl(MSR_PKGC6_IRTL, msr); 1465 usec = irtl_2_usec(msr); 1466 if (usec) { 1467 bxt_cstates[2].exit_latency = usec; 1468 bxt_cstates[2].target_residency = usec; 1469 } 1470 1471 rdmsrl(MSR_PKGC7_IRTL, msr); 1472 usec = irtl_2_usec(msr); 1473 if (usec) { 1474 bxt_cstates[3].exit_latency = usec; 1475 bxt_cstates[3].target_residency = usec; 1476 } 1477 1478 rdmsrl(MSR_PKGC8_IRTL, msr); 1479 usec = irtl_2_usec(msr); 1480 if (usec) { 1481 bxt_cstates[4].exit_latency = usec; 1482 bxt_cstates[4].target_residency = usec; 1483 } 1484 1485 rdmsrl(MSR_PKGC9_IRTL, msr); 1486 usec = irtl_2_usec(msr); 1487 if (usec) { 1488 bxt_cstates[5].exit_latency = usec; 1489 bxt_cstates[5].target_residency = usec; 1490 } 1491 1492 rdmsrl(MSR_PKGC10_IRTL, msr); 1493 usec = irtl_2_usec(msr); 1494 if (usec) { 1495 bxt_cstates[6].exit_latency = usec; 1496 bxt_cstates[6].target_residency = usec; 1497 } 1498 1499 } 1500 1501 /** 1502 * sklh_idle_state_table_update - Fix up the Sky Lake idle states table. 1503 * 1504 * On SKL-H (model 0x5e) skip C8 and C9 if C10 is enabled and SGX disabled. 1505 */ 1506 static void __init sklh_idle_state_table_update(void) 1507 { 1508 unsigned long long msr; 1509 unsigned int eax, ebx, ecx, edx; 1510 1511 1512 /* if PC10 disabled via cmdline intel_idle.max_cstate=7 or shallower */ 1513 if (max_cstate <= 7) 1514 return; 1515 1516 /* if PC10 not present in CPUID.MWAIT.EDX */ 1517 if ((mwait_substates & (0xF << 28)) == 0) 1518 return; 1519 1520 rdmsrl(MSR_PKG_CST_CONFIG_CONTROL, msr); 1521 1522 /* PC10 is not enabled in PKG C-state limit */ 1523 if ((msr & 0xF) != 8) 1524 return; 1525 1526 ecx = 0; 1527 cpuid(7, &eax, &ebx, &ecx, &edx); 1528 1529 /* if SGX is present */ 1530 if (ebx & (1 << 2)) { 1531 1532 rdmsrl(MSR_IA32_FEAT_CTL, msr); 1533 1534 /* if SGX is enabled */ 1535 if (msr & (1 << 18)) 1536 return; 1537 } 1538 1539 skl_cstates[5].flags |= CPUIDLE_FLAG_UNUSABLE; /* C8-SKL */ 1540 skl_cstates[6].flags |= CPUIDLE_FLAG_UNUSABLE; /* C9-SKL */ 1541 } 1542 1543 /** 1544 * skx_idle_state_table_update - Adjust the Sky Lake/Cascade Lake 1545 * idle states table. 1546 */ 1547 static void __init skx_idle_state_table_update(void) 1548 { 1549 unsigned long long msr; 1550 1551 rdmsrl(MSR_PKG_CST_CONFIG_CONTROL, msr); 1552 1553 /* 1554 * 000b: C0/C1 (no package C-state support) 1555 * 001b: C2 1556 * 010b: C6 (non-retention) 1557 * 011b: C6 (retention) 1558 * 111b: No Package C state limits. 1559 */ 1560 if ((msr & 0x7) < 2) { 1561 /* 1562 * Uses the CC6 + PC0 latency and 3 times of 1563 * latency for target_residency if the PC6 1564 * is disabled in BIOS. This is consistent 1565 * with how intel_idle driver uses _CST 1566 * to set the target_residency. 1567 */ 1568 skx_cstates[2].exit_latency = 92; 1569 skx_cstates[2].target_residency = 276; 1570 } 1571 } 1572 1573 /** 1574 * spr_idle_state_table_update - Adjust Sapphire Rapids idle states table. 1575 */ 1576 static void __init spr_idle_state_table_update(void) 1577 { 1578 unsigned long long msr; 1579 1580 /* Check if user prefers C1E over C1. */ 1581 if (preferred_states_mask & BIT(2)) { 1582 if (preferred_states_mask & BIT(1)) 1583 /* Both can't be enabled, stick to the defaults. */ 1584 return; 1585 1586 spr_cstates[0].flags |= CPUIDLE_FLAG_UNUSABLE; 1587 spr_cstates[1].flags &= ~CPUIDLE_FLAG_UNUSABLE; 1588 1589 /* Enable C1E using the "C1E promotion" bit. */ 1590 c1e_promotion_enable(); 1591 disable_promotion_to_c1e = false; 1592 } 1593 1594 /* 1595 * By default, the C6 state assumes the worst-case scenario of package 1596 * C6. However, if PC6 is disabled, we update the numbers to match 1597 * core C6. 1598 */ 1599 rdmsrl(MSR_PKG_CST_CONFIG_CONTROL, msr); 1600 1601 /* Limit value 2 and above allow for PC6. */ 1602 if ((msr & 0x7) < 2) { 1603 spr_cstates[2].exit_latency = 190; 1604 spr_cstates[2].target_residency = 600; 1605 } 1606 } 1607 1608 static bool __init intel_idle_verify_cstate(unsigned int mwait_hint) 1609 { 1610 unsigned int mwait_cstate = MWAIT_HINT2CSTATE(mwait_hint) + 1; 1611 unsigned int num_substates = (mwait_substates >> mwait_cstate * 4) & 1612 MWAIT_SUBSTATE_MASK; 1613 1614 /* Ignore the C-state if there are NO sub-states in CPUID for it. */ 1615 if (num_substates == 0) 1616 return false; 1617 1618 if (mwait_cstate > 2 && !boot_cpu_has(X86_FEATURE_NONSTOP_TSC)) 1619 mark_tsc_unstable("TSC halts in idle states deeper than C2"); 1620 1621 return true; 1622 } 1623 1624 static void __init intel_idle_init_cstates_icpu(struct cpuidle_driver *drv) 1625 { 1626 int cstate; 1627 1628 switch (boot_cpu_data.x86_model) { 1629 case INTEL_FAM6_IVYBRIDGE_X: 1630 ivt_idle_state_table_update(); 1631 break; 1632 case INTEL_FAM6_ATOM_GOLDMONT: 1633 case INTEL_FAM6_ATOM_GOLDMONT_PLUS: 1634 bxt_idle_state_table_update(); 1635 break; 1636 case INTEL_FAM6_SKYLAKE: 1637 sklh_idle_state_table_update(); 1638 break; 1639 case INTEL_FAM6_SKYLAKE_X: 1640 skx_idle_state_table_update(); 1641 break; 1642 case INTEL_FAM6_SAPPHIRERAPIDS_X: 1643 spr_idle_state_table_update(); 1644 break; 1645 } 1646 1647 for (cstate = 0; cstate < CPUIDLE_STATE_MAX; ++cstate) { 1648 unsigned int mwait_hint; 1649 1650 if (intel_idle_max_cstate_reached(cstate)) 1651 break; 1652 1653 if (!cpuidle_state_table[cstate].enter && 1654 !cpuidle_state_table[cstate].enter_s2idle) 1655 break; 1656 1657 /* If marked as unusable, skip this state. */ 1658 if (cpuidle_state_table[cstate].flags & CPUIDLE_FLAG_UNUSABLE) { 1659 pr_debug("state %s is disabled\n", 1660 cpuidle_state_table[cstate].name); 1661 continue; 1662 } 1663 1664 mwait_hint = flg2MWAIT(cpuidle_state_table[cstate].flags); 1665 if (!intel_idle_verify_cstate(mwait_hint)) 1666 continue; 1667 1668 /* Structure copy. */ 1669 drv->states[drv->state_count] = cpuidle_state_table[cstate]; 1670 1671 if ((disabled_states_mask & BIT(drv->state_count)) || 1672 ((icpu->use_acpi || force_use_acpi) && 1673 intel_idle_off_by_default(mwait_hint) && 1674 !(cpuidle_state_table[cstate].flags & CPUIDLE_FLAG_ALWAYS_ENABLE))) 1675 drv->states[drv->state_count].flags |= CPUIDLE_FLAG_OFF; 1676 1677 if (intel_idle_state_needs_timer_stop(&drv->states[drv->state_count])) 1678 drv->states[drv->state_count].flags |= CPUIDLE_FLAG_TIMER_STOP; 1679 1680 drv->state_count++; 1681 } 1682 1683 if (icpu->byt_auto_demotion_disable_flag) { 1684 wrmsrl(MSR_CC6_DEMOTION_POLICY_CONFIG, 0); 1685 wrmsrl(MSR_MC6_DEMOTION_POLICY_CONFIG, 0); 1686 } 1687 } 1688 1689 /** 1690 * intel_idle_cpuidle_driver_init - Create the list of available idle states. 1691 * @drv: cpuidle driver structure to initialize. 1692 */ 1693 static void __init intel_idle_cpuidle_driver_init(struct cpuidle_driver *drv) 1694 { 1695 cpuidle_poll_state_init(drv); 1696 1697 if (disabled_states_mask & BIT(0)) 1698 drv->states[0].flags |= CPUIDLE_FLAG_OFF; 1699 1700 drv->state_count = 1; 1701 1702 if (icpu) 1703 intel_idle_init_cstates_icpu(drv); 1704 else 1705 intel_idle_init_cstates_acpi(drv); 1706 } 1707 1708 static void auto_demotion_disable(void) 1709 { 1710 unsigned long long msr_bits; 1711 1712 rdmsrl(MSR_PKG_CST_CONFIG_CONTROL, msr_bits); 1713 msr_bits &= ~auto_demotion_disable_flags; 1714 wrmsrl(MSR_PKG_CST_CONFIG_CONTROL, msr_bits); 1715 } 1716 1717 static void c1e_promotion_enable(void) 1718 { 1719 unsigned long long msr_bits; 1720 1721 rdmsrl(MSR_IA32_POWER_CTL, msr_bits); 1722 msr_bits |= 0x2; 1723 wrmsrl(MSR_IA32_POWER_CTL, msr_bits); 1724 } 1725 1726 static void c1e_promotion_disable(void) 1727 { 1728 unsigned long long msr_bits; 1729 1730 rdmsrl(MSR_IA32_POWER_CTL, msr_bits); 1731 msr_bits &= ~0x2; 1732 wrmsrl(MSR_IA32_POWER_CTL, msr_bits); 1733 } 1734 1735 /** 1736 * intel_idle_cpu_init - Register the target CPU with the cpuidle core. 1737 * @cpu: CPU to initialize. 1738 * 1739 * Register a cpuidle device object for @cpu and update its MSRs in accordance 1740 * with the processor model flags. 1741 */ 1742 static int intel_idle_cpu_init(unsigned int cpu) 1743 { 1744 struct cpuidle_device *dev; 1745 1746 dev = per_cpu_ptr(intel_idle_cpuidle_devices, cpu); 1747 dev->cpu = cpu; 1748 1749 if (cpuidle_register_device(dev)) { 1750 pr_debug("cpuidle_register_device %d failed!\n", cpu); 1751 return -EIO; 1752 } 1753 1754 if (auto_demotion_disable_flags) 1755 auto_demotion_disable(); 1756 1757 if (disable_promotion_to_c1e) 1758 c1e_promotion_disable(); 1759 1760 return 0; 1761 } 1762 1763 static int intel_idle_cpu_online(unsigned int cpu) 1764 { 1765 struct cpuidle_device *dev; 1766 1767 if (!boot_cpu_has(X86_FEATURE_ARAT)) 1768 tick_broadcast_enable(); 1769 1770 /* 1771 * Some systems can hotplug a cpu at runtime after 1772 * the kernel has booted, we have to initialize the 1773 * driver in this case 1774 */ 1775 dev = per_cpu_ptr(intel_idle_cpuidle_devices, cpu); 1776 if (!dev->registered) 1777 return intel_idle_cpu_init(cpu); 1778 1779 return 0; 1780 } 1781 1782 /** 1783 * intel_idle_cpuidle_devices_uninit - Unregister all cpuidle devices. 1784 */ 1785 static void __init intel_idle_cpuidle_devices_uninit(void) 1786 { 1787 int i; 1788 1789 for_each_online_cpu(i) 1790 cpuidle_unregister_device(per_cpu_ptr(intel_idle_cpuidle_devices, i)); 1791 } 1792 1793 static int __init intel_idle_init(void) 1794 { 1795 const struct x86_cpu_id *id; 1796 unsigned int eax, ebx, ecx; 1797 int retval; 1798 1799 /* Do not load intel_idle at all for now if idle= is passed */ 1800 if (boot_option_idle_override != IDLE_NO_OVERRIDE) 1801 return -ENODEV; 1802 1803 if (max_cstate == 0) { 1804 pr_debug("disabled\n"); 1805 return -EPERM; 1806 } 1807 1808 id = x86_match_cpu(intel_idle_ids); 1809 if (id) { 1810 if (!boot_cpu_has(X86_FEATURE_MWAIT)) { 1811 pr_debug("Please enable MWAIT in BIOS SETUP\n"); 1812 return -ENODEV; 1813 } 1814 } else { 1815 id = x86_match_cpu(intel_mwait_ids); 1816 if (!id) 1817 return -ENODEV; 1818 } 1819 1820 if (boot_cpu_data.cpuid_level < CPUID_MWAIT_LEAF) 1821 return -ENODEV; 1822 1823 cpuid(CPUID_MWAIT_LEAF, &eax, &ebx, &ecx, &mwait_substates); 1824 1825 if (!(ecx & CPUID5_ECX_EXTENSIONS_SUPPORTED) || 1826 !(ecx & CPUID5_ECX_INTERRUPT_BREAK) || 1827 !mwait_substates) 1828 return -ENODEV; 1829 1830 pr_debug("MWAIT substates: 0x%x\n", mwait_substates); 1831 1832 icpu = (const struct idle_cpu *)id->driver_data; 1833 if (icpu) { 1834 cpuidle_state_table = icpu->state_table; 1835 auto_demotion_disable_flags = icpu->auto_demotion_disable_flags; 1836 disable_promotion_to_c1e = icpu->disable_promotion_to_c1e; 1837 if (icpu->use_acpi || force_use_acpi) 1838 intel_idle_acpi_cst_extract(); 1839 } else if (!intel_idle_acpi_cst_extract()) { 1840 return -ENODEV; 1841 } 1842 1843 pr_debug("v" INTEL_IDLE_VERSION " model 0x%X\n", 1844 boot_cpu_data.x86_model); 1845 1846 intel_idle_cpuidle_devices = alloc_percpu(struct cpuidle_device); 1847 if (!intel_idle_cpuidle_devices) 1848 return -ENOMEM; 1849 1850 intel_idle_cpuidle_driver_init(&intel_idle_driver); 1851 1852 retval = cpuidle_register_driver(&intel_idle_driver); 1853 if (retval) { 1854 struct cpuidle_driver *drv = cpuidle_get_driver(); 1855 printk(KERN_DEBUG pr_fmt("intel_idle yielding to %s\n"), 1856 drv ? drv->name : "none"); 1857 goto init_driver_fail; 1858 } 1859 1860 retval = cpuhp_setup_state(CPUHP_AP_ONLINE_DYN, "idle/intel:online", 1861 intel_idle_cpu_online, NULL); 1862 if (retval < 0) 1863 goto hp_setup_fail; 1864 1865 pr_debug("Local APIC timer is reliable in %s\n", 1866 boot_cpu_has(X86_FEATURE_ARAT) ? "all C-states" : "C1"); 1867 1868 return 0; 1869 1870 hp_setup_fail: 1871 intel_idle_cpuidle_devices_uninit(); 1872 cpuidle_unregister_driver(&intel_idle_driver); 1873 init_driver_fail: 1874 free_percpu(intel_idle_cpuidle_devices); 1875 return retval; 1876 1877 } 1878 device_initcall(intel_idle_init); 1879 1880 /* 1881 * We are not really modular, but we used to support that. Meaning we also 1882 * support "intel_idle.max_cstate=..." at boot and also a read-only export of 1883 * it at /sys/module/intel_idle/parameters/max_cstate -- so using module_param 1884 * is the easiest way (currently) to continue doing that. 1885 */ 1886 module_param(max_cstate, int, 0444); 1887 /* 1888 * The positions of the bits that are set in this number are the indices of the 1889 * idle states to be disabled by default (as reflected by the names of the 1890 * corresponding idle state directories in sysfs, "state0", "state1" ... 1891 * "state<i>" ..., where <i> is the index of the given state). 1892 */ 1893 module_param_named(states_off, disabled_states_mask, uint, 0444); 1894 MODULE_PARM_DESC(states_off, "Mask of disabled idle states"); 1895 /* 1896 * Some platforms come with mutually exclusive C-states, so that if one is 1897 * enabled, the other C-states must not be used. Example: C1 and C1E on 1898 * Sapphire Rapids platform. This parameter allows for selecting the 1899 * preferred C-states among the groups of mutually exclusive C-states - the 1900 * selected C-states will be registered, the other C-states from the mutually 1901 * exclusive group won't be registered. If the platform has no mutually 1902 * exclusive C-states, this parameter has no effect. 1903 */ 1904 module_param_named(preferred_cstates, preferred_states_mask, uint, 0444); 1905 MODULE_PARM_DESC(preferred_cstates, "Mask of preferred idle states"); 1906