1 /* 2 * PowerNV cpuidle code 3 * 4 * Copyright 2015 IBM Corp. 5 * 6 * This program is free software; you can redistribute it and/or 7 * modify it under the terms of the GNU General Public License 8 * as published by the Free Software Foundation; either version 9 * 2 of the License, or (at your option) any later version. 10 */ 11 12 #include <linux/types.h> 13 #include <linux/mm.h> 14 #include <linux/slab.h> 15 #include <linux/of.h> 16 #include <linux/device.h> 17 #include <linux/cpu.h> 18 19 #include <asm/firmware.h> 20 #include <asm/machdep.h> 21 #include <asm/opal.h> 22 #include <asm/cputhreads.h> 23 #include <asm/cpuidle.h> 24 #include <asm/code-patching.h> 25 #include <asm/smp.h> 26 #include <asm/runlatch.h> 27 28 #include "powernv.h" 29 #include "subcore.h" 30 31 /* Power ISA 3.0 allows for stop states 0x0 - 0xF */ 32 #define MAX_STOP_STATE 0xF 33 34 #define P9_STOP_SPR_MSR 2000 35 #define P9_STOP_SPR_PSSCR 855 36 37 static u32 supported_cpuidle_states; 38 39 /* 40 * The default stop state that will be used by ppc_md.power_save 41 * function on platforms that support stop instruction. 42 */ 43 static u64 pnv_default_stop_val; 44 static u64 pnv_default_stop_mask; 45 static bool default_stop_found; 46 47 /* 48 * First deep stop state. Used to figure out when to save/restore 49 * hypervisor context. 50 */ 51 u64 pnv_first_deep_stop_state = MAX_STOP_STATE; 52 53 /* 54 * psscr value and mask of the deepest stop idle state. 55 * Used when a cpu is offlined. 56 */ 57 static u64 pnv_deepest_stop_psscr_val; 58 static u64 pnv_deepest_stop_psscr_mask; 59 static u64 pnv_deepest_stop_flag; 60 static bool deepest_stop_found; 61 62 static int pnv_save_sprs_for_deep_states(void) 63 { 64 int cpu; 65 int rc; 66 67 /* 68 * hid0, hid1, hid4, hid5, hmeer and lpcr values are symmetric across 69 * all cpus at boot. Get these reg values of current cpu and use the 70 * same across all cpus. 71 */ 72 uint64_t lpcr_val = mfspr(SPRN_LPCR); 73 uint64_t hid0_val = mfspr(SPRN_HID0); 74 uint64_t hid1_val = mfspr(SPRN_HID1); 75 uint64_t hid4_val = mfspr(SPRN_HID4); 76 uint64_t hid5_val = mfspr(SPRN_HID5); 77 uint64_t hmeer_val = mfspr(SPRN_HMEER); 78 uint64_t msr_val = MSR_IDLE; 79 uint64_t psscr_val = pnv_deepest_stop_psscr_val; 80 81 for_each_possible_cpu(cpu) { 82 uint64_t pir = get_hard_smp_processor_id(cpu); 83 uint64_t hsprg0_val = (uint64_t)&paca[cpu]; 84 85 rc = opal_slw_set_reg(pir, SPRN_HSPRG0, hsprg0_val); 86 if (rc != 0) 87 return rc; 88 89 rc = opal_slw_set_reg(pir, SPRN_LPCR, lpcr_val); 90 if (rc != 0) 91 return rc; 92 93 if (cpu_has_feature(CPU_FTR_ARCH_300)) { 94 rc = opal_slw_set_reg(pir, P9_STOP_SPR_MSR, msr_val); 95 if (rc) 96 return rc; 97 98 rc = opal_slw_set_reg(pir, 99 P9_STOP_SPR_PSSCR, psscr_val); 100 101 if (rc) 102 return rc; 103 } 104 105 /* HIDs are per core registers */ 106 if (cpu_thread_in_core(cpu) == 0) { 107 108 rc = opal_slw_set_reg(pir, SPRN_HMEER, hmeer_val); 109 if (rc != 0) 110 return rc; 111 112 rc = opal_slw_set_reg(pir, SPRN_HID0, hid0_val); 113 if (rc != 0) 114 return rc; 115 116 /* Only p8 needs to set extra HID regiters */ 117 if (!cpu_has_feature(CPU_FTR_ARCH_300)) { 118 119 rc = opal_slw_set_reg(pir, SPRN_HID1, hid1_val); 120 if (rc != 0) 121 return rc; 122 123 rc = opal_slw_set_reg(pir, SPRN_HID4, hid4_val); 124 if (rc != 0) 125 return rc; 126 127 rc = opal_slw_set_reg(pir, SPRN_HID5, hid5_val); 128 if (rc != 0) 129 return rc; 130 } 131 } 132 } 133 134 return 0; 135 } 136 137 static void pnv_alloc_idle_core_states(void) 138 { 139 int i, j; 140 int nr_cores = cpu_nr_cores(); 141 u32 *core_idle_state; 142 143 /* 144 * core_idle_state - The lower 8 bits track the idle state of 145 * each thread of the core. 146 * 147 * The most significant bit is the lock bit. 148 * 149 * Initially all the bits corresponding to threads_per_core 150 * are set. They are cleared when the thread enters deep idle 151 * state like sleep and winkle/stop. 152 * 153 * Initially the lock bit is cleared. The lock bit has 2 154 * purposes: 155 * a. While the first thread in the core waking up from 156 * idle is restoring core state, it prevents other 157 * threads in the core from switching to process 158 * context. 159 * b. While the last thread in the core is saving the 160 * core state, it prevents a different thread from 161 * waking up. 162 */ 163 for (i = 0; i < nr_cores; i++) { 164 int first_cpu = i * threads_per_core; 165 int node = cpu_to_node(first_cpu); 166 size_t paca_ptr_array_size; 167 168 core_idle_state = kmalloc_node(sizeof(u32), GFP_KERNEL, node); 169 *core_idle_state = (1 << threads_per_core) - 1; 170 paca_ptr_array_size = (threads_per_core * 171 sizeof(struct paca_struct *)); 172 173 for (j = 0; j < threads_per_core; j++) { 174 int cpu = first_cpu + j; 175 176 paca[cpu].core_idle_state_ptr = core_idle_state; 177 paca[cpu].thread_idle_state = PNV_THREAD_RUNNING; 178 paca[cpu].thread_mask = 1 << j; 179 if (!cpu_has_feature(CPU_FTR_POWER9_DD1)) 180 continue; 181 paca[cpu].thread_sibling_pacas = 182 kmalloc_node(paca_ptr_array_size, 183 GFP_KERNEL, node); 184 } 185 } 186 187 update_subcore_sibling_mask(); 188 189 if (supported_cpuidle_states & OPAL_PM_LOSE_FULL_CONTEXT) { 190 int rc = pnv_save_sprs_for_deep_states(); 191 192 if (likely(!rc)) 193 return; 194 195 /* 196 * The stop-api is unable to restore hypervisor 197 * resources on wakeup from platform idle states which 198 * lose full context. So disable such states. 199 */ 200 supported_cpuidle_states &= ~OPAL_PM_LOSE_FULL_CONTEXT; 201 pr_warn("cpuidle-powernv: Disabling idle states that lose full context\n"); 202 pr_warn("cpuidle-powernv: Idle power-savings, CPU-Hotplug affected\n"); 203 204 if (cpu_has_feature(CPU_FTR_ARCH_300) && 205 (pnv_deepest_stop_flag & OPAL_PM_LOSE_FULL_CONTEXT)) { 206 /* 207 * Use the default stop state for CPU-Hotplug 208 * if available. 209 */ 210 if (default_stop_found) { 211 pnv_deepest_stop_psscr_val = 212 pnv_default_stop_val; 213 pnv_deepest_stop_psscr_mask = 214 pnv_default_stop_mask; 215 pr_warn("cpuidle-powernv: Offlined CPUs will stop with psscr = 0x%016llx\n", 216 pnv_deepest_stop_psscr_val); 217 } else { /* Fallback to snooze loop for CPU-Hotplug */ 218 deepest_stop_found = false; 219 pr_warn("cpuidle-powernv: Offlined CPUs will busy wait\n"); 220 } 221 } 222 } 223 } 224 225 u32 pnv_get_supported_cpuidle_states(void) 226 { 227 return supported_cpuidle_states; 228 } 229 EXPORT_SYMBOL_GPL(pnv_get_supported_cpuidle_states); 230 231 static void pnv_fastsleep_workaround_apply(void *info) 232 233 { 234 int rc; 235 int *err = info; 236 237 rc = opal_config_cpu_idle_state(OPAL_CONFIG_IDLE_FASTSLEEP, 238 OPAL_CONFIG_IDLE_APPLY); 239 if (rc) 240 *err = 1; 241 } 242 243 /* 244 * Used to store fastsleep workaround state 245 * 0 - Workaround applied/undone at fastsleep entry/exit path (Default) 246 * 1 - Workaround applied once, never undone. 247 */ 248 static u8 fastsleep_workaround_applyonce; 249 250 static ssize_t show_fastsleep_workaround_applyonce(struct device *dev, 251 struct device_attribute *attr, char *buf) 252 { 253 return sprintf(buf, "%u\n", fastsleep_workaround_applyonce); 254 } 255 256 static ssize_t store_fastsleep_workaround_applyonce(struct device *dev, 257 struct device_attribute *attr, const char *buf, 258 size_t count) 259 { 260 cpumask_t primary_thread_mask; 261 int err; 262 u8 val; 263 264 if (kstrtou8(buf, 0, &val) || val != 1) 265 return -EINVAL; 266 267 if (fastsleep_workaround_applyonce == 1) 268 return count; 269 270 /* 271 * fastsleep_workaround_applyonce = 1 implies 272 * fastsleep workaround needs to be left in 'applied' state on all 273 * the cores. Do this by- 274 * 1. Patching out the call to 'undo' workaround in fastsleep exit path 275 * 2. Sending ipi to all the cores which have at least one online thread 276 * 3. Patching out the call to 'apply' workaround in fastsleep entry 277 * path 278 * There is no need to send ipi to cores which have all threads 279 * offlined, as last thread of the core entering fastsleep or deeper 280 * state would have applied workaround. 281 */ 282 err = patch_instruction( 283 (unsigned int *)pnv_fastsleep_workaround_at_exit, 284 PPC_INST_NOP); 285 if (err) { 286 pr_err("fastsleep_workaround_applyonce change failed while patching pnv_fastsleep_workaround_at_exit"); 287 goto fail; 288 } 289 290 get_online_cpus(); 291 primary_thread_mask = cpu_online_cores_map(); 292 on_each_cpu_mask(&primary_thread_mask, 293 pnv_fastsleep_workaround_apply, 294 &err, 1); 295 put_online_cpus(); 296 if (err) { 297 pr_err("fastsleep_workaround_applyonce change failed while running pnv_fastsleep_workaround_apply"); 298 goto fail; 299 } 300 301 err = patch_instruction( 302 (unsigned int *)pnv_fastsleep_workaround_at_entry, 303 PPC_INST_NOP); 304 if (err) { 305 pr_err("fastsleep_workaround_applyonce change failed while patching pnv_fastsleep_workaround_at_entry"); 306 goto fail; 307 } 308 309 fastsleep_workaround_applyonce = 1; 310 311 return count; 312 fail: 313 return -EIO; 314 } 315 316 static DEVICE_ATTR(fastsleep_workaround_applyonce, 0600, 317 show_fastsleep_workaround_applyonce, 318 store_fastsleep_workaround_applyonce); 319 320 static unsigned long __power7_idle_type(unsigned long type) 321 { 322 unsigned long srr1; 323 324 if (!prep_irq_for_idle_irqsoff()) 325 return 0; 326 327 __ppc64_runlatch_off(); 328 srr1 = power7_idle_insn(type); 329 __ppc64_runlatch_on(); 330 331 fini_irq_for_idle_irqsoff(); 332 333 return srr1; 334 } 335 336 void power7_idle_type(unsigned long type) 337 { 338 unsigned long srr1; 339 340 srr1 = __power7_idle_type(type); 341 irq_set_pending_from_srr1(srr1); 342 } 343 344 void power7_idle(void) 345 { 346 if (!powersave_nap) 347 return; 348 349 power7_idle_type(PNV_THREAD_NAP); 350 } 351 352 static unsigned long __power9_idle_type(unsigned long stop_psscr_val, 353 unsigned long stop_psscr_mask) 354 { 355 unsigned long psscr; 356 unsigned long srr1; 357 358 if (!prep_irq_for_idle_irqsoff()) 359 return 0; 360 361 psscr = mfspr(SPRN_PSSCR); 362 psscr = (psscr & ~stop_psscr_mask) | stop_psscr_val; 363 364 __ppc64_runlatch_off(); 365 srr1 = power9_idle_stop(psscr); 366 __ppc64_runlatch_on(); 367 368 fini_irq_for_idle_irqsoff(); 369 370 return srr1; 371 } 372 373 void power9_idle_type(unsigned long stop_psscr_val, 374 unsigned long stop_psscr_mask) 375 { 376 unsigned long srr1; 377 378 srr1 = __power9_idle_type(stop_psscr_val, stop_psscr_mask); 379 irq_set_pending_from_srr1(srr1); 380 } 381 382 /* 383 * Used for ppc_md.power_save which needs a function with no parameters 384 */ 385 void power9_idle(void) 386 { 387 power9_idle_type(pnv_default_stop_val, pnv_default_stop_mask); 388 } 389 390 #ifdef CONFIG_HOTPLUG_CPU 391 static void pnv_program_cpu_hotplug_lpcr(unsigned int cpu, u64 lpcr_val) 392 { 393 u64 pir = get_hard_smp_processor_id(cpu); 394 395 mtspr(SPRN_LPCR, lpcr_val); 396 397 /* 398 * Program the LPCR via stop-api only if the deepest stop state 399 * can lose hypervisor context. 400 */ 401 if (supported_cpuidle_states & OPAL_PM_LOSE_FULL_CONTEXT) 402 opal_slw_set_reg(pir, SPRN_LPCR, lpcr_val); 403 } 404 405 /* 406 * pnv_cpu_offline: A function that puts the CPU into the deepest 407 * available platform idle state on a CPU-Offline. 408 * interrupts hard disabled and no lazy irq pending. 409 */ 410 unsigned long pnv_cpu_offline(unsigned int cpu) 411 { 412 unsigned long srr1; 413 u32 idle_states = pnv_get_supported_cpuidle_states(); 414 u64 lpcr_val; 415 416 /* 417 * We don't want to take decrementer interrupts while we are 418 * offline, so clear LPCR:PECE1. We keep PECE2 (and 419 * LPCR_PECE_HVEE on P9) enabled as to let IPIs in. 420 * 421 * If the CPU gets woken up by a special wakeup, ensure that 422 * the SLW engine sets LPCR with decrementer bit cleared, else 423 * the CPU will come back to the kernel due to a spurious 424 * wakeup. 425 */ 426 lpcr_val = mfspr(SPRN_LPCR) & ~(u64)LPCR_PECE1; 427 pnv_program_cpu_hotplug_lpcr(cpu, lpcr_val); 428 429 __ppc64_runlatch_off(); 430 431 if (cpu_has_feature(CPU_FTR_ARCH_300) && deepest_stop_found) { 432 unsigned long psscr; 433 434 psscr = mfspr(SPRN_PSSCR); 435 psscr = (psscr & ~pnv_deepest_stop_psscr_mask) | 436 pnv_deepest_stop_psscr_val; 437 srr1 = power9_idle_stop(psscr); 438 439 } else if ((idle_states & OPAL_PM_WINKLE_ENABLED) && 440 (idle_states & OPAL_PM_LOSE_FULL_CONTEXT)) { 441 srr1 = power7_idle_insn(PNV_THREAD_WINKLE); 442 } else if ((idle_states & OPAL_PM_SLEEP_ENABLED) || 443 (idle_states & OPAL_PM_SLEEP_ENABLED_ER1)) { 444 srr1 = power7_idle_insn(PNV_THREAD_SLEEP); 445 } else if (idle_states & OPAL_PM_NAP_ENABLED) { 446 srr1 = power7_idle_insn(PNV_THREAD_NAP); 447 } else { 448 /* This is the fallback method. We emulate snooze */ 449 while (!generic_check_cpu_restart(cpu)) { 450 HMT_low(); 451 HMT_very_low(); 452 } 453 srr1 = 0; 454 HMT_medium(); 455 } 456 457 __ppc64_runlatch_on(); 458 459 /* 460 * Re-enable decrementer interrupts in LPCR. 461 * 462 * Further, we want stop states to be woken up by decrementer 463 * for non-hotplug cases. So program the LPCR via stop api as 464 * well. 465 */ 466 lpcr_val = mfspr(SPRN_LPCR) | (u64)LPCR_PECE1; 467 pnv_program_cpu_hotplug_lpcr(cpu, lpcr_val); 468 469 return srr1; 470 } 471 #endif 472 473 /* 474 * Power ISA 3.0 idle initialization. 475 * 476 * POWER ISA 3.0 defines a new SPR Processor stop Status and Control 477 * Register (PSSCR) to control idle behavior. 478 * 479 * PSSCR layout: 480 * ---------------------------------------------------------- 481 * | PLS | /// | SD | ESL | EC | PSLL | /// | TR | MTL | RL | 482 * ---------------------------------------------------------- 483 * 0 4 41 42 43 44 48 54 56 60 484 * 485 * PSSCR key fields: 486 * Bits 0:3 - Power-Saving Level Status (PLS). This field indicates the 487 * lowest power-saving state the thread entered since stop instruction was 488 * last executed. 489 * 490 * Bit 41 - Status Disable(SD) 491 * 0 - Shows PLS entries 492 * 1 - PLS entries are all 0 493 * 494 * Bit 42 - Enable State Loss 495 * 0 - No state is lost irrespective of other fields 496 * 1 - Allows state loss 497 * 498 * Bit 43 - Exit Criterion 499 * 0 - Exit from power-save mode on any interrupt 500 * 1 - Exit from power-save mode controlled by LPCR's PECE bits 501 * 502 * Bits 44:47 - Power-Saving Level Limit 503 * This limits the power-saving level that can be entered into. 504 * 505 * Bits 60:63 - Requested Level 506 * Used to specify which power-saving level must be entered on executing 507 * stop instruction 508 */ 509 510 int validate_psscr_val_mask(u64 *psscr_val, u64 *psscr_mask, u32 flags) 511 { 512 int err = 0; 513 514 /* 515 * psscr_mask == 0xf indicates an older firmware. 516 * Set remaining fields of psscr to the default values. 517 * See NOTE above definition of PSSCR_HV_DEFAULT_VAL 518 */ 519 if (*psscr_mask == 0xf) { 520 *psscr_val = *psscr_val | PSSCR_HV_DEFAULT_VAL; 521 *psscr_mask = PSSCR_HV_DEFAULT_MASK; 522 return err; 523 } 524 525 /* 526 * New firmware is expected to set the psscr_val bits correctly. 527 * Validate that the following invariants are correctly maintained by 528 * the new firmware. 529 * - ESL bit value matches the EC bit value. 530 * - ESL bit is set for all the deep stop states. 531 */ 532 if (GET_PSSCR_ESL(*psscr_val) != GET_PSSCR_EC(*psscr_val)) { 533 err = ERR_EC_ESL_MISMATCH; 534 } else if ((flags & OPAL_PM_LOSE_FULL_CONTEXT) && 535 GET_PSSCR_ESL(*psscr_val) == 0) { 536 err = ERR_DEEP_STATE_ESL_MISMATCH; 537 } 538 539 return err; 540 } 541 542 /* 543 * pnv_arch300_idle_init: Initializes the default idle state, first 544 * deep idle state and deepest idle state on 545 * ISA 3.0 CPUs. 546 * 547 * @np: /ibm,opal/power-mgt device node 548 * @flags: cpu-idle-state-flags array 549 * @dt_idle_states: Number of idle state entries 550 * Returns 0 on success 551 */ 552 static int __init pnv_power9_idle_init(struct device_node *np, u32 *flags, 553 int dt_idle_states) 554 { 555 u64 *psscr_val = NULL; 556 u64 *psscr_mask = NULL; 557 u32 *residency_ns = NULL; 558 u64 max_residency_ns = 0; 559 int rc = 0, i; 560 561 psscr_val = kcalloc(dt_idle_states, sizeof(*psscr_val), GFP_KERNEL); 562 psscr_mask = kcalloc(dt_idle_states, sizeof(*psscr_mask), GFP_KERNEL); 563 residency_ns = kcalloc(dt_idle_states, sizeof(*residency_ns), 564 GFP_KERNEL); 565 566 if (!psscr_val || !psscr_mask || !residency_ns) { 567 rc = -1; 568 goto out; 569 } 570 571 if (of_property_read_u64_array(np, 572 "ibm,cpu-idle-state-psscr", 573 psscr_val, dt_idle_states)) { 574 pr_warn("cpuidle-powernv: missing ibm,cpu-idle-state-psscr in DT\n"); 575 rc = -1; 576 goto out; 577 } 578 579 if (of_property_read_u64_array(np, 580 "ibm,cpu-idle-state-psscr-mask", 581 psscr_mask, dt_idle_states)) { 582 pr_warn("cpuidle-powernv: missing ibm,cpu-idle-state-psscr-mask in DT\n"); 583 rc = -1; 584 goto out; 585 } 586 587 if (of_property_read_u32_array(np, 588 "ibm,cpu-idle-state-residency-ns", 589 residency_ns, dt_idle_states)) { 590 pr_warn("cpuidle-powernv: missing ibm,cpu-idle-state-residency-ns in DT\n"); 591 rc = -1; 592 goto out; 593 } 594 595 /* 596 * Set pnv_first_deep_stop_state, pnv_deepest_stop_psscr_{val,mask}, 597 * and the pnv_default_stop_{val,mask}. 598 * 599 * pnv_first_deep_stop_state should be set to the first stop 600 * level to cause hypervisor state loss. 601 * 602 * pnv_deepest_stop_{val,mask} should be set to values corresponding to 603 * the deepest stop state. 604 * 605 * pnv_default_stop_{val,mask} should be set to values corresponding to 606 * the shallowest (OPAL_PM_STOP_INST_FAST) loss-less stop state. 607 */ 608 pnv_first_deep_stop_state = MAX_STOP_STATE; 609 for (i = 0; i < dt_idle_states; i++) { 610 int err; 611 u64 psscr_rl = psscr_val[i] & PSSCR_RL_MASK; 612 613 if ((flags[i] & OPAL_PM_LOSE_FULL_CONTEXT) && 614 (pnv_first_deep_stop_state > psscr_rl)) 615 pnv_first_deep_stop_state = psscr_rl; 616 617 err = validate_psscr_val_mask(&psscr_val[i], &psscr_mask[i], 618 flags[i]); 619 if (err) { 620 report_invalid_psscr_val(psscr_val[i], err); 621 continue; 622 } 623 624 if (max_residency_ns < residency_ns[i]) { 625 max_residency_ns = residency_ns[i]; 626 pnv_deepest_stop_psscr_val = psscr_val[i]; 627 pnv_deepest_stop_psscr_mask = psscr_mask[i]; 628 pnv_deepest_stop_flag = flags[i]; 629 deepest_stop_found = true; 630 } 631 632 if (!default_stop_found && 633 (flags[i] & OPAL_PM_STOP_INST_FAST)) { 634 pnv_default_stop_val = psscr_val[i]; 635 pnv_default_stop_mask = psscr_mask[i]; 636 default_stop_found = true; 637 } 638 } 639 640 if (unlikely(!default_stop_found)) { 641 pr_warn("cpuidle-powernv: No suitable default stop state found. Disabling platform idle.\n"); 642 } else { 643 ppc_md.power_save = power9_idle; 644 pr_info("cpuidle-powernv: Default stop: psscr = 0x%016llx,mask=0x%016llx\n", 645 pnv_default_stop_val, pnv_default_stop_mask); 646 } 647 648 if (unlikely(!deepest_stop_found)) { 649 pr_warn("cpuidle-powernv: No suitable stop state for CPU-Hotplug. Offlined CPUs will busy wait"); 650 } else { 651 pr_info("cpuidle-powernv: Deepest stop: psscr = 0x%016llx,mask=0x%016llx\n", 652 pnv_deepest_stop_psscr_val, 653 pnv_deepest_stop_psscr_mask); 654 } 655 656 pr_info("cpuidle-powernv: Requested Level (RL) value of first deep stop = 0x%llx\n", 657 pnv_first_deep_stop_state); 658 out: 659 kfree(psscr_val); 660 kfree(psscr_mask); 661 kfree(residency_ns); 662 return rc; 663 } 664 665 /* 666 * Probe device tree for supported idle states 667 */ 668 static void __init pnv_probe_idle_states(void) 669 { 670 struct device_node *np; 671 int dt_idle_states; 672 u32 *flags = NULL; 673 int i; 674 675 np = of_find_node_by_path("/ibm,opal/power-mgt"); 676 if (!np) { 677 pr_warn("opal: PowerMgmt Node not found\n"); 678 goto out; 679 } 680 dt_idle_states = of_property_count_u32_elems(np, 681 "ibm,cpu-idle-state-flags"); 682 if (dt_idle_states < 0) { 683 pr_warn("cpuidle-powernv: no idle states found in the DT\n"); 684 goto out; 685 } 686 687 flags = kcalloc(dt_idle_states, sizeof(*flags), GFP_KERNEL); 688 689 if (of_property_read_u32_array(np, 690 "ibm,cpu-idle-state-flags", flags, dt_idle_states)) { 691 pr_warn("cpuidle-powernv: missing ibm,cpu-idle-state-flags in DT\n"); 692 goto out; 693 } 694 695 if (cpu_has_feature(CPU_FTR_ARCH_300)) { 696 if (pnv_power9_idle_init(np, flags, dt_idle_states)) 697 goto out; 698 } 699 700 for (i = 0; i < dt_idle_states; i++) 701 supported_cpuidle_states |= flags[i]; 702 703 out: 704 kfree(flags); 705 } 706 static int __init pnv_init_idle_states(void) 707 { 708 709 supported_cpuidle_states = 0; 710 711 if (cpuidle_disable != IDLE_NO_OVERRIDE) 712 goto out; 713 714 pnv_probe_idle_states(); 715 716 if (!(supported_cpuidle_states & OPAL_PM_SLEEP_ENABLED_ER1)) { 717 patch_instruction( 718 (unsigned int *)pnv_fastsleep_workaround_at_entry, 719 PPC_INST_NOP); 720 patch_instruction( 721 (unsigned int *)pnv_fastsleep_workaround_at_exit, 722 PPC_INST_NOP); 723 } else { 724 /* 725 * OPAL_PM_SLEEP_ENABLED_ER1 is set. It indicates that 726 * workaround is needed to use fastsleep. Provide sysfs 727 * control to choose how this workaround has to be applied. 728 */ 729 device_create_file(cpu_subsys.dev_root, 730 &dev_attr_fastsleep_workaround_applyonce); 731 } 732 733 pnv_alloc_idle_core_states(); 734 735 /* 736 * For each CPU, record its PACA address in each of it's 737 * sibling thread's PACA at the slot corresponding to this 738 * CPU's index in the core. 739 */ 740 if (cpu_has_feature(CPU_FTR_POWER9_DD1)) { 741 int cpu; 742 743 pr_info("powernv: idle: Saving PACA pointers of all CPUs in their thread sibling PACA\n"); 744 for_each_possible_cpu(cpu) { 745 int base_cpu = cpu_first_thread_sibling(cpu); 746 int idx = cpu_thread_in_core(cpu); 747 int i; 748 749 for (i = 0; i < threads_per_core; i++) { 750 int j = base_cpu + i; 751 752 paca[j].thread_sibling_pacas[idx] = &paca[cpu]; 753 } 754 } 755 } 756 757 if (supported_cpuidle_states & OPAL_PM_NAP_ENABLED) 758 ppc_md.power_save = power7_idle; 759 760 out: 761 return 0; 762 } 763 machine_subsys_initcall(powernv, pnv_init_idle_states); 764