1 // SPDX-License-Identifier: GPL-2.0-or-later 2 /* 3 * PowerNV cpuidle code 4 * 5 * Copyright 2015 IBM Corp. 6 */ 7 8 #include <linux/types.h> 9 #include <linux/mm.h> 10 #include <linux/slab.h> 11 #include <linux/of.h> 12 #include <linux/device.h> 13 #include <linux/cpu.h> 14 15 #include <asm/asm-prototypes.h> 16 #include <asm/firmware.h> 17 #include <asm/machdep.h> 18 #include <asm/opal.h> 19 #include <asm/cputhreads.h> 20 #include <asm/cpuidle.h> 21 #include <asm/code-patching.h> 22 #include <asm/smp.h> 23 #include <asm/runlatch.h> 24 #include <asm/dbell.h> 25 26 #include "powernv.h" 27 #include "subcore.h" 28 29 /* Power ISA 3.0 allows for stop states 0x0 - 0xF */ 30 #define MAX_STOP_STATE 0xF 31 32 #define P9_STOP_SPR_MSR 2000 33 #define P9_STOP_SPR_PSSCR 855 34 35 static u32 supported_cpuidle_states; 36 struct pnv_idle_states_t *pnv_idle_states; 37 int nr_pnv_idle_states; 38 39 /* 40 * The default stop state that will be used by ppc_md.power_save 41 * function on platforms that support stop instruction. 42 */ 43 static u64 pnv_default_stop_val; 44 static u64 pnv_default_stop_mask; 45 static bool default_stop_found; 46 47 /* 48 * First stop state levels when SPR and TB loss can occur. 49 */ 50 static u64 pnv_first_tb_loss_level = MAX_STOP_STATE + 1; 51 static u64 pnv_first_spr_loss_level = MAX_STOP_STATE + 1; 52 53 /* 54 * psscr value and mask of the deepest stop idle state. 55 * Used when a cpu is offlined. 56 */ 57 static u64 pnv_deepest_stop_psscr_val; 58 static u64 pnv_deepest_stop_psscr_mask; 59 static u64 pnv_deepest_stop_flag; 60 static bool deepest_stop_found; 61 62 static unsigned long power7_offline_type; 63 64 static int pnv_save_sprs_for_deep_states(void) 65 { 66 int cpu; 67 int rc; 68 69 /* 70 * hid0, hid1, hid4, hid5, hmeer and lpcr values are symmetric across 71 * all cpus at boot. Get these reg values of current cpu and use the 72 * same across all cpus. 73 */ 74 uint64_t lpcr_val = mfspr(SPRN_LPCR); 75 uint64_t hid0_val = mfspr(SPRN_HID0); 76 uint64_t hid1_val = mfspr(SPRN_HID1); 77 uint64_t hid4_val = mfspr(SPRN_HID4); 78 uint64_t hid5_val = mfspr(SPRN_HID5); 79 uint64_t hmeer_val = mfspr(SPRN_HMEER); 80 uint64_t msr_val = MSR_IDLE; 81 uint64_t psscr_val = pnv_deepest_stop_psscr_val; 82 83 for_each_present_cpu(cpu) { 84 uint64_t pir = get_hard_smp_processor_id(cpu); 85 uint64_t hsprg0_val = (uint64_t)paca_ptrs[cpu]; 86 87 rc = opal_slw_set_reg(pir, SPRN_HSPRG0, hsprg0_val); 88 if (rc != 0) 89 return rc; 90 91 rc = opal_slw_set_reg(pir, SPRN_LPCR, lpcr_val); 92 if (rc != 0) 93 return rc; 94 95 if (cpu_has_feature(CPU_FTR_ARCH_300)) { 96 rc = opal_slw_set_reg(pir, P9_STOP_SPR_MSR, msr_val); 97 if (rc) 98 return rc; 99 100 rc = opal_slw_set_reg(pir, 101 P9_STOP_SPR_PSSCR, psscr_val); 102 103 if (rc) 104 return rc; 105 } 106 107 /* HIDs are per core registers */ 108 if (cpu_thread_in_core(cpu) == 0) { 109 110 rc = opal_slw_set_reg(pir, SPRN_HMEER, hmeer_val); 111 if (rc != 0) 112 return rc; 113 114 rc = opal_slw_set_reg(pir, SPRN_HID0, hid0_val); 115 if (rc != 0) 116 return rc; 117 118 /* Only p8 needs to set extra HID regiters */ 119 if (!cpu_has_feature(CPU_FTR_ARCH_300)) { 120 121 rc = opal_slw_set_reg(pir, SPRN_HID1, hid1_val); 122 if (rc != 0) 123 return rc; 124 125 rc = opal_slw_set_reg(pir, SPRN_HID4, hid4_val); 126 if (rc != 0) 127 return rc; 128 129 rc = opal_slw_set_reg(pir, SPRN_HID5, hid5_val); 130 if (rc != 0) 131 return rc; 132 } 133 } 134 } 135 136 return 0; 137 } 138 139 u32 pnv_get_supported_cpuidle_states(void) 140 { 141 return supported_cpuidle_states; 142 } 143 EXPORT_SYMBOL_GPL(pnv_get_supported_cpuidle_states); 144 145 static void pnv_fastsleep_workaround_apply(void *info) 146 147 { 148 int rc; 149 int *err = info; 150 151 rc = opal_config_cpu_idle_state(OPAL_CONFIG_IDLE_FASTSLEEP, 152 OPAL_CONFIG_IDLE_APPLY); 153 if (rc) 154 *err = 1; 155 } 156 157 static bool power7_fastsleep_workaround_entry = true; 158 static bool power7_fastsleep_workaround_exit = true; 159 160 /* 161 * Used to store fastsleep workaround state 162 * 0 - Workaround applied/undone at fastsleep entry/exit path (Default) 163 * 1 - Workaround applied once, never undone. 164 */ 165 static u8 fastsleep_workaround_applyonce; 166 167 static ssize_t show_fastsleep_workaround_applyonce(struct device *dev, 168 struct device_attribute *attr, char *buf) 169 { 170 return sprintf(buf, "%u\n", fastsleep_workaround_applyonce); 171 } 172 173 static ssize_t store_fastsleep_workaround_applyonce(struct device *dev, 174 struct device_attribute *attr, const char *buf, 175 size_t count) 176 { 177 cpumask_t primary_thread_mask; 178 int err; 179 u8 val; 180 181 if (kstrtou8(buf, 0, &val) || val != 1) 182 return -EINVAL; 183 184 if (fastsleep_workaround_applyonce == 1) 185 return count; 186 187 /* 188 * fastsleep_workaround_applyonce = 1 implies 189 * fastsleep workaround needs to be left in 'applied' state on all 190 * the cores. Do this by- 191 * 1. Disable the 'undo' workaround in fastsleep exit path 192 * 2. Sendi IPIs to all the cores which have at least one online thread 193 * 3. Disable the 'apply' workaround in fastsleep entry path 194 * 195 * There is no need to send ipi to cores which have all threads 196 * offlined, as last thread of the core entering fastsleep or deeper 197 * state would have applied workaround. 198 */ 199 power7_fastsleep_workaround_exit = false; 200 201 get_online_cpus(); 202 primary_thread_mask = cpu_online_cores_map(); 203 on_each_cpu_mask(&primary_thread_mask, 204 pnv_fastsleep_workaround_apply, 205 &err, 1); 206 put_online_cpus(); 207 if (err) { 208 pr_err("fastsleep_workaround_applyonce change failed while running pnv_fastsleep_workaround_apply"); 209 goto fail; 210 } 211 212 power7_fastsleep_workaround_entry = false; 213 214 fastsleep_workaround_applyonce = 1; 215 216 return count; 217 fail: 218 return -EIO; 219 } 220 221 static DEVICE_ATTR(fastsleep_workaround_applyonce, 0600, 222 show_fastsleep_workaround_applyonce, 223 store_fastsleep_workaround_applyonce); 224 225 static inline void atomic_start_thread_idle(void) 226 { 227 int cpu = raw_smp_processor_id(); 228 int first = cpu_first_thread_sibling(cpu); 229 int thread_nr = cpu_thread_in_core(cpu); 230 unsigned long *state = &paca_ptrs[first]->idle_state; 231 232 clear_bit(thread_nr, state); 233 } 234 235 static inline void atomic_stop_thread_idle(void) 236 { 237 int cpu = raw_smp_processor_id(); 238 int first = cpu_first_thread_sibling(cpu); 239 int thread_nr = cpu_thread_in_core(cpu); 240 unsigned long *state = &paca_ptrs[first]->idle_state; 241 242 set_bit(thread_nr, state); 243 } 244 245 static inline void atomic_lock_thread_idle(void) 246 { 247 int cpu = raw_smp_processor_id(); 248 int first = cpu_first_thread_sibling(cpu); 249 unsigned long *state = &paca_ptrs[first]->idle_state; 250 251 while (unlikely(test_and_set_bit_lock(NR_PNV_CORE_IDLE_LOCK_BIT, state))) 252 barrier(); 253 } 254 255 static inline void atomic_unlock_and_stop_thread_idle(void) 256 { 257 int cpu = raw_smp_processor_id(); 258 int first = cpu_first_thread_sibling(cpu); 259 unsigned long thread = 1UL << cpu_thread_in_core(cpu); 260 unsigned long *state = &paca_ptrs[first]->idle_state; 261 u64 s = READ_ONCE(*state); 262 u64 new, tmp; 263 264 BUG_ON(!(s & PNV_CORE_IDLE_LOCK_BIT)); 265 BUG_ON(s & thread); 266 267 again: 268 new = (s | thread) & ~PNV_CORE_IDLE_LOCK_BIT; 269 tmp = cmpxchg(state, s, new); 270 if (unlikely(tmp != s)) { 271 s = tmp; 272 goto again; 273 } 274 } 275 276 static inline void atomic_unlock_thread_idle(void) 277 { 278 int cpu = raw_smp_processor_id(); 279 int first = cpu_first_thread_sibling(cpu); 280 unsigned long *state = &paca_ptrs[first]->idle_state; 281 282 BUG_ON(!test_bit(NR_PNV_CORE_IDLE_LOCK_BIT, state)); 283 clear_bit_unlock(NR_PNV_CORE_IDLE_LOCK_BIT, state); 284 } 285 286 /* P7 and P8 */ 287 struct p7_sprs { 288 /* per core */ 289 u64 tscr; 290 u64 worc; 291 292 /* per subcore */ 293 u64 sdr1; 294 u64 rpr; 295 296 /* per thread */ 297 u64 lpcr; 298 u64 hfscr; 299 u64 fscr; 300 u64 purr; 301 u64 spurr; 302 u64 dscr; 303 u64 wort; 304 305 /* per thread SPRs that get lost in shallow states */ 306 u64 amr; 307 u64 iamr; 308 u64 amor; 309 u64 uamor; 310 }; 311 312 static unsigned long power7_idle_insn(unsigned long type) 313 { 314 int cpu = raw_smp_processor_id(); 315 int first = cpu_first_thread_sibling(cpu); 316 unsigned long *state = &paca_ptrs[first]->idle_state; 317 unsigned long thread = 1UL << cpu_thread_in_core(cpu); 318 unsigned long core_thread_mask = (1UL << threads_per_core) - 1; 319 unsigned long srr1; 320 bool full_winkle; 321 struct p7_sprs sprs = {}; /* avoid false use-uninitialised */ 322 bool sprs_saved = false; 323 int rc; 324 325 if (unlikely(type != PNV_THREAD_NAP)) { 326 atomic_lock_thread_idle(); 327 328 BUG_ON(!(*state & thread)); 329 *state &= ~thread; 330 331 if (power7_fastsleep_workaround_entry) { 332 if ((*state & core_thread_mask) == 0) { 333 rc = opal_config_cpu_idle_state( 334 OPAL_CONFIG_IDLE_FASTSLEEP, 335 OPAL_CONFIG_IDLE_APPLY); 336 BUG_ON(rc); 337 } 338 } 339 340 if (type == PNV_THREAD_WINKLE) { 341 sprs.tscr = mfspr(SPRN_TSCR); 342 sprs.worc = mfspr(SPRN_WORC); 343 344 sprs.sdr1 = mfspr(SPRN_SDR1); 345 sprs.rpr = mfspr(SPRN_RPR); 346 347 sprs.lpcr = mfspr(SPRN_LPCR); 348 if (cpu_has_feature(CPU_FTR_ARCH_207S)) { 349 sprs.hfscr = mfspr(SPRN_HFSCR); 350 sprs.fscr = mfspr(SPRN_FSCR); 351 } 352 sprs.purr = mfspr(SPRN_PURR); 353 sprs.spurr = mfspr(SPRN_SPURR); 354 sprs.dscr = mfspr(SPRN_DSCR); 355 sprs.wort = mfspr(SPRN_WORT); 356 357 sprs_saved = true; 358 359 /* 360 * Increment winkle counter and set all winkle bits if 361 * all threads are winkling. This allows wakeup side to 362 * distinguish between fast sleep and winkle state 363 * loss. Fast sleep still has to resync the timebase so 364 * this may not be a really big win. 365 */ 366 *state += 1 << PNV_CORE_IDLE_WINKLE_COUNT_SHIFT; 367 if ((*state & PNV_CORE_IDLE_WINKLE_COUNT_BITS) 368 >> PNV_CORE_IDLE_WINKLE_COUNT_SHIFT 369 == threads_per_core) 370 *state |= PNV_CORE_IDLE_THREAD_WINKLE_BITS; 371 WARN_ON((*state & PNV_CORE_IDLE_WINKLE_COUNT_BITS) == 0); 372 } 373 374 atomic_unlock_thread_idle(); 375 } 376 377 if (cpu_has_feature(CPU_FTR_ARCH_207S)) { 378 sprs.amr = mfspr(SPRN_AMR); 379 sprs.iamr = mfspr(SPRN_IAMR); 380 sprs.amor = mfspr(SPRN_AMOR); 381 sprs.uamor = mfspr(SPRN_UAMOR); 382 } 383 384 local_paca->thread_idle_state = type; 385 srr1 = isa206_idle_insn_mayloss(type); /* go idle */ 386 local_paca->thread_idle_state = PNV_THREAD_RUNNING; 387 388 WARN_ON_ONCE(!srr1); 389 WARN_ON_ONCE(mfmsr() & (MSR_IR|MSR_DR)); 390 391 if (cpu_has_feature(CPU_FTR_ARCH_207S)) { 392 if ((srr1 & SRR1_WAKESTATE) != SRR1_WS_NOLOSS) { 393 /* 394 * We don't need an isync after the mtsprs here because 395 * the upcoming mtmsrd is execution synchronizing. 396 */ 397 mtspr(SPRN_AMR, sprs.amr); 398 mtspr(SPRN_IAMR, sprs.iamr); 399 mtspr(SPRN_AMOR, sprs.amor); 400 mtspr(SPRN_UAMOR, sprs.uamor); 401 } 402 } 403 404 if (unlikely((srr1 & SRR1_WAKEMASK_P8) == SRR1_WAKEHMI)) 405 hmi_exception_realmode(NULL); 406 407 if (likely((srr1 & SRR1_WAKESTATE) != SRR1_WS_HVLOSS)) { 408 if (unlikely(type != PNV_THREAD_NAP)) { 409 atomic_lock_thread_idle(); 410 if (type == PNV_THREAD_WINKLE) { 411 WARN_ON((*state & PNV_CORE_IDLE_WINKLE_COUNT_BITS) == 0); 412 *state -= 1 << PNV_CORE_IDLE_WINKLE_COUNT_SHIFT; 413 *state &= ~(thread << PNV_CORE_IDLE_THREAD_WINKLE_BITS_SHIFT); 414 } 415 atomic_unlock_and_stop_thread_idle(); 416 } 417 return srr1; 418 } 419 420 /* HV state loss */ 421 BUG_ON(type == PNV_THREAD_NAP); 422 423 atomic_lock_thread_idle(); 424 425 full_winkle = false; 426 if (type == PNV_THREAD_WINKLE) { 427 WARN_ON((*state & PNV_CORE_IDLE_WINKLE_COUNT_BITS) == 0); 428 *state -= 1 << PNV_CORE_IDLE_WINKLE_COUNT_SHIFT; 429 if (*state & (thread << PNV_CORE_IDLE_THREAD_WINKLE_BITS_SHIFT)) { 430 *state &= ~(thread << PNV_CORE_IDLE_THREAD_WINKLE_BITS_SHIFT); 431 full_winkle = true; 432 BUG_ON(!sprs_saved); 433 } 434 } 435 436 WARN_ON(*state & thread); 437 438 if ((*state & core_thread_mask) != 0) 439 goto core_woken; 440 441 /* Per-core SPRs */ 442 if (full_winkle) { 443 mtspr(SPRN_TSCR, sprs.tscr); 444 mtspr(SPRN_WORC, sprs.worc); 445 } 446 447 if (power7_fastsleep_workaround_exit) { 448 rc = opal_config_cpu_idle_state(OPAL_CONFIG_IDLE_FASTSLEEP, 449 OPAL_CONFIG_IDLE_UNDO); 450 BUG_ON(rc); 451 } 452 453 /* TB */ 454 if (opal_resync_timebase() != OPAL_SUCCESS) 455 BUG(); 456 457 core_woken: 458 if (!full_winkle) 459 goto subcore_woken; 460 461 if ((*state & local_paca->subcore_sibling_mask) != 0) 462 goto subcore_woken; 463 464 /* Per-subcore SPRs */ 465 mtspr(SPRN_SDR1, sprs.sdr1); 466 mtspr(SPRN_RPR, sprs.rpr); 467 468 subcore_woken: 469 /* 470 * isync after restoring shared SPRs and before unlocking. Unlock 471 * only contains hwsync which does not necessarily do the right 472 * thing for SPRs. 473 */ 474 isync(); 475 atomic_unlock_and_stop_thread_idle(); 476 477 /* Fast sleep does not lose SPRs */ 478 if (!full_winkle) 479 return srr1; 480 481 /* Per-thread SPRs */ 482 mtspr(SPRN_LPCR, sprs.lpcr); 483 if (cpu_has_feature(CPU_FTR_ARCH_207S)) { 484 mtspr(SPRN_HFSCR, sprs.hfscr); 485 mtspr(SPRN_FSCR, sprs.fscr); 486 } 487 mtspr(SPRN_PURR, sprs.purr); 488 mtspr(SPRN_SPURR, sprs.spurr); 489 mtspr(SPRN_DSCR, sprs.dscr); 490 mtspr(SPRN_WORT, sprs.wort); 491 492 mtspr(SPRN_SPRG3, local_paca->sprg_vdso); 493 494 /* 495 * The SLB has to be restored here, but it sometimes still 496 * contains entries, so the __ variant must be used to prevent 497 * multi hits. 498 */ 499 __slb_restore_bolted_realmode(); 500 501 return srr1; 502 } 503 504 extern unsigned long idle_kvm_start_guest(unsigned long srr1); 505 506 #ifdef CONFIG_HOTPLUG_CPU 507 static unsigned long power7_offline(void) 508 { 509 unsigned long srr1; 510 511 mtmsr(MSR_IDLE); 512 513 #ifdef CONFIG_KVM_BOOK3S_HV_POSSIBLE 514 /* Tell KVM we're entering idle. */ 515 /******************************************************/ 516 /* N O T E W E L L ! ! ! N O T E W E L L */ 517 /* The following store to HSTATE_HWTHREAD_STATE(r13) */ 518 /* MUST occur in real mode, i.e. with the MMU off, */ 519 /* and the MMU must stay off until we clear this flag */ 520 /* and test HSTATE_HWTHREAD_REQ(r13) in */ 521 /* pnv_powersave_wakeup in this file. */ 522 /* The reason is that another thread can switch the */ 523 /* MMU to a guest context whenever this flag is set */ 524 /* to KVM_HWTHREAD_IN_IDLE, and if the MMU was on, */ 525 /* that would potentially cause this thread to start */ 526 /* executing instructions from guest memory in */ 527 /* hypervisor mode, leading to a host crash or data */ 528 /* corruption, or worse. */ 529 /******************************************************/ 530 local_paca->kvm_hstate.hwthread_state = KVM_HWTHREAD_IN_IDLE; 531 #endif 532 533 __ppc64_runlatch_off(); 534 srr1 = power7_idle_insn(power7_offline_type); 535 __ppc64_runlatch_on(); 536 537 #ifdef CONFIG_KVM_BOOK3S_HV_POSSIBLE 538 local_paca->kvm_hstate.hwthread_state = KVM_HWTHREAD_IN_KERNEL; 539 /* Order setting hwthread_state vs. testing hwthread_req */ 540 smp_mb(); 541 if (local_paca->kvm_hstate.hwthread_req) 542 srr1 = idle_kvm_start_guest(srr1); 543 #endif 544 545 mtmsr(MSR_KERNEL); 546 547 return srr1; 548 } 549 #endif 550 551 void power7_idle_type(unsigned long type) 552 { 553 unsigned long srr1; 554 555 if (!prep_irq_for_idle_irqsoff()) 556 return; 557 558 mtmsr(MSR_IDLE); 559 __ppc64_runlatch_off(); 560 srr1 = power7_idle_insn(type); 561 __ppc64_runlatch_on(); 562 mtmsr(MSR_KERNEL); 563 564 fini_irq_for_idle_irqsoff(); 565 irq_set_pending_from_srr1(srr1); 566 } 567 568 void power7_idle(void) 569 { 570 if (!powersave_nap) 571 return; 572 573 power7_idle_type(PNV_THREAD_NAP); 574 } 575 576 struct p9_sprs { 577 /* per core */ 578 u64 ptcr; 579 u64 rpr; 580 u64 tscr; 581 u64 ldbar; 582 583 /* per thread */ 584 u64 lpcr; 585 u64 hfscr; 586 u64 fscr; 587 u64 pid; 588 u64 purr; 589 u64 spurr; 590 u64 dscr; 591 u64 wort; 592 593 u64 mmcra; 594 u32 mmcr0; 595 u32 mmcr1; 596 u64 mmcr2; 597 598 /* per thread SPRs that get lost in shallow states */ 599 u64 amr; 600 u64 iamr; 601 u64 amor; 602 u64 uamor; 603 }; 604 605 static unsigned long power9_idle_stop(unsigned long psscr, bool mmu_on) 606 { 607 int cpu = raw_smp_processor_id(); 608 int first = cpu_first_thread_sibling(cpu); 609 unsigned long *state = &paca_ptrs[first]->idle_state; 610 unsigned long core_thread_mask = (1UL << threads_per_core) - 1; 611 unsigned long srr1; 612 unsigned long pls; 613 unsigned long mmcr0 = 0; 614 unsigned long mmcra = 0; 615 struct p9_sprs sprs = {}; /* avoid false used-uninitialised */ 616 bool sprs_saved = false; 617 618 if (!(psscr & (PSSCR_EC|PSSCR_ESL))) { 619 /* EC=ESL=0 case */ 620 621 BUG_ON(!mmu_on); 622 623 /* 624 * Wake synchronously. SRESET via xscom may still cause 625 * a 0x100 powersave wakeup with SRR1 reason! 626 */ 627 srr1 = isa300_idle_stop_noloss(psscr); /* go idle */ 628 if (likely(!srr1)) 629 return 0; 630 631 /* 632 * Registers not saved, can't recover! 633 * This would be a hardware bug 634 */ 635 BUG_ON((srr1 & SRR1_WAKESTATE) != SRR1_WS_NOLOSS); 636 637 goto out; 638 } 639 640 /* EC=ESL=1 case */ 641 #ifdef CONFIG_KVM_BOOK3S_HV_POSSIBLE 642 if (cpu_has_feature(CPU_FTR_P9_TM_XER_SO_BUG)) { 643 local_paca->requested_psscr = psscr; 644 /* order setting requested_psscr vs testing dont_stop */ 645 smp_mb(); 646 if (atomic_read(&local_paca->dont_stop)) { 647 local_paca->requested_psscr = 0; 648 return 0; 649 } 650 } 651 #endif 652 653 if (!cpu_has_feature(CPU_FTR_POWER9_DD2_1)) { 654 /* 655 * POWER9 DD2 can incorrectly set PMAO when waking up 656 * after a state-loss idle. Saving and restoring MMCR0 657 * over idle is a workaround. 658 */ 659 mmcr0 = mfspr(SPRN_MMCR0); 660 } 661 662 if (cpu_has_feature(CPU_FTR_ARCH_31)) { 663 /* 664 * POWER10 uses MMCRA (BHRBRD) as BHRB disable bit. 665 * If the user hasn't asked for the BHRB to be 666 * written, the value of MMCRA[BHRBRD] is 1. 667 * On wakeup from stop, MMCRA[BHRBD] will be 0, 668 * since it is previleged resource and will be lost. 669 * Thus, if we do not save and restore the MMCRA[BHRBD], 670 * hardware will be needlessly writing to the BHRB 671 * in problem mode. 672 */ 673 mmcra = mfspr(SPRN_MMCRA); 674 } 675 676 if ((psscr & PSSCR_RL_MASK) >= pnv_first_spr_loss_level) { 677 sprs.lpcr = mfspr(SPRN_LPCR); 678 sprs.hfscr = mfspr(SPRN_HFSCR); 679 sprs.fscr = mfspr(SPRN_FSCR); 680 sprs.pid = mfspr(SPRN_PID); 681 sprs.purr = mfspr(SPRN_PURR); 682 sprs.spurr = mfspr(SPRN_SPURR); 683 sprs.dscr = mfspr(SPRN_DSCR); 684 sprs.wort = mfspr(SPRN_WORT); 685 686 sprs.mmcra = mfspr(SPRN_MMCRA); 687 sprs.mmcr0 = mfspr(SPRN_MMCR0); 688 sprs.mmcr1 = mfspr(SPRN_MMCR1); 689 sprs.mmcr2 = mfspr(SPRN_MMCR2); 690 691 sprs.ptcr = mfspr(SPRN_PTCR); 692 sprs.rpr = mfspr(SPRN_RPR); 693 sprs.tscr = mfspr(SPRN_TSCR); 694 if (!firmware_has_feature(FW_FEATURE_ULTRAVISOR)) 695 sprs.ldbar = mfspr(SPRN_LDBAR); 696 697 sprs_saved = true; 698 699 atomic_start_thread_idle(); 700 } 701 702 sprs.amr = mfspr(SPRN_AMR); 703 sprs.iamr = mfspr(SPRN_IAMR); 704 sprs.amor = mfspr(SPRN_AMOR); 705 sprs.uamor = mfspr(SPRN_UAMOR); 706 707 srr1 = isa300_idle_stop_mayloss(psscr); /* go idle */ 708 709 #ifdef CONFIG_KVM_BOOK3S_HV_POSSIBLE 710 local_paca->requested_psscr = 0; 711 #endif 712 713 psscr = mfspr(SPRN_PSSCR); 714 715 WARN_ON_ONCE(!srr1); 716 WARN_ON_ONCE(mfmsr() & (MSR_IR|MSR_DR)); 717 718 if ((srr1 & SRR1_WAKESTATE) != SRR1_WS_NOLOSS) { 719 /* 720 * We don't need an isync after the mtsprs here because the 721 * upcoming mtmsrd is execution synchronizing. 722 */ 723 mtspr(SPRN_AMR, sprs.amr); 724 mtspr(SPRN_IAMR, sprs.iamr); 725 mtspr(SPRN_AMOR, sprs.amor); 726 mtspr(SPRN_UAMOR, sprs.uamor); 727 728 /* 729 * Workaround for POWER9 DD2.0, if we lost resources, the ERAT 730 * might have been corrupted and needs flushing. We also need 731 * to reload MMCR0 (see mmcr0 comment above). 732 */ 733 if (!cpu_has_feature(CPU_FTR_POWER9_DD2_1)) { 734 asm volatile(PPC_ISA_3_0_INVALIDATE_ERAT); 735 mtspr(SPRN_MMCR0, mmcr0); 736 } 737 738 /* Reload MMCRA to restore BHRB disable bit for POWER10 */ 739 if (cpu_has_feature(CPU_FTR_ARCH_31)) 740 mtspr(SPRN_MMCRA, mmcra); 741 742 /* 743 * DD2.2 and earlier need to set then clear bit 60 in MMCRA 744 * to ensure the PMU starts running. 745 */ 746 mmcra = mfspr(SPRN_MMCRA); 747 mmcra |= PPC_BIT(60); 748 mtspr(SPRN_MMCRA, mmcra); 749 mmcra &= ~PPC_BIT(60); 750 mtspr(SPRN_MMCRA, mmcra); 751 } 752 753 if (unlikely((srr1 & SRR1_WAKEMASK_P8) == SRR1_WAKEHMI)) 754 hmi_exception_realmode(NULL); 755 756 /* 757 * On POWER9, SRR1 bits do not match exactly as expected. 758 * SRR1_WS_GPRLOSS (10b) can also result in SPR loss, so 759 * just always test PSSCR for SPR/TB state loss. 760 */ 761 pls = (psscr & PSSCR_PLS) >> PSSCR_PLS_SHIFT; 762 if (likely(pls < pnv_first_spr_loss_level)) { 763 if (sprs_saved) 764 atomic_stop_thread_idle(); 765 goto out; 766 } 767 768 /* HV state loss */ 769 BUG_ON(!sprs_saved); 770 771 atomic_lock_thread_idle(); 772 773 if ((*state & core_thread_mask) != 0) 774 goto core_woken; 775 776 /* Per-core SPRs */ 777 mtspr(SPRN_PTCR, sprs.ptcr); 778 mtspr(SPRN_RPR, sprs.rpr); 779 mtspr(SPRN_TSCR, sprs.tscr); 780 781 if (pls >= pnv_first_tb_loss_level) { 782 /* TB loss */ 783 if (opal_resync_timebase() != OPAL_SUCCESS) 784 BUG(); 785 } 786 787 /* 788 * isync after restoring shared SPRs and before unlocking. Unlock 789 * only contains hwsync which does not necessarily do the right 790 * thing for SPRs. 791 */ 792 isync(); 793 794 core_woken: 795 atomic_unlock_and_stop_thread_idle(); 796 797 /* Per-thread SPRs */ 798 mtspr(SPRN_LPCR, sprs.lpcr); 799 mtspr(SPRN_HFSCR, sprs.hfscr); 800 mtspr(SPRN_FSCR, sprs.fscr); 801 mtspr(SPRN_PID, sprs.pid); 802 mtspr(SPRN_PURR, sprs.purr); 803 mtspr(SPRN_SPURR, sprs.spurr); 804 mtspr(SPRN_DSCR, sprs.dscr); 805 mtspr(SPRN_WORT, sprs.wort); 806 807 mtspr(SPRN_MMCRA, sprs.mmcra); 808 mtspr(SPRN_MMCR0, sprs.mmcr0); 809 mtspr(SPRN_MMCR1, sprs.mmcr1); 810 mtspr(SPRN_MMCR2, sprs.mmcr2); 811 if (!firmware_has_feature(FW_FEATURE_ULTRAVISOR)) 812 mtspr(SPRN_LDBAR, sprs.ldbar); 813 814 mtspr(SPRN_SPRG3, local_paca->sprg_vdso); 815 816 if (!radix_enabled()) 817 __slb_restore_bolted_realmode(); 818 819 out: 820 if (mmu_on) 821 mtmsr(MSR_KERNEL); 822 823 return srr1; 824 } 825 826 #ifdef CONFIG_HOTPLUG_CPU 827 static unsigned long power9_offline_stop(unsigned long psscr) 828 { 829 unsigned long srr1; 830 831 #ifndef CONFIG_KVM_BOOK3S_HV_POSSIBLE 832 __ppc64_runlatch_off(); 833 srr1 = power9_idle_stop(psscr, true); 834 __ppc64_runlatch_on(); 835 #else 836 /* 837 * Tell KVM we're entering idle. 838 * This does not have to be done in real mode because the P9 MMU 839 * is independent per-thread. Some steppings share radix/hash mode 840 * between threads, but in that case KVM has a barrier sync in real 841 * mode before and after switching between radix and hash. 842 * 843 * kvm_start_guest must still be called in real mode though, hence 844 * the false argument. 845 */ 846 local_paca->kvm_hstate.hwthread_state = KVM_HWTHREAD_IN_IDLE; 847 848 __ppc64_runlatch_off(); 849 srr1 = power9_idle_stop(psscr, false); 850 __ppc64_runlatch_on(); 851 852 local_paca->kvm_hstate.hwthread_state = KVM_HWTHREAD_IN_KERNEL; 853 /* Order setting hwthread_state vs. testing hwthread_req */ 854 smp_mb(); 855 if (local_paca->kvm_hstate.hwthread_req) 856 srr1 = idle_kvm_start_guest(srr1); 857 mtmsr(MSR_KERNEL); 858 #endif 859 860 return srr1; 861 } 862 #endif 863 864 void power9_idle_type(unsigned long stop_psscr_val, 865 unsigned long stop_psscr_mask) 866 { 867 unsigned long psscr; 868 unsigned long srr1; 869 870 if (!prep_irq_for_idle_irqsoff()) 871 return; 872 873 psscr = mfspr(SPRN_PSSCR); 874 psscr = (psscr & ~stop_psscr_mask) | stop_psscr_val; 875 876 __ppc64_runlatch_off(); 877 srr1 = power9_idle_stop(psscr, true); 878 __ppc64_runlatch_on(); 879 880 fini_irq_for_idle_irqsoff(); 881 882 irq_set_pending_from_srr1(srr1); 883 } 884 885 /* 886 * Used for ppc_md.power_save which needs a function with no parameters 887 */ 888 void power9_idle(void) 889 { 890 power9_idle_type(pnv_default_stop_val, pnv_default_stop_mask); 891 } 892 893 #ifdef CONFIG_KVM_BOOK3S_HV_POSSIBLE 894 /* 895 * This is used in working around bugs in thread reconfiguration 896 * on POWER9 (at least up to Nimbus DD2.2) relating to transactional 897 * memory and the way that XER[SO] is checkpointed. 898 * This function forces the core into SMT4 in order by asking 899 * all other threads not to stop, and sending a message to any 900 * that are in a stop state. 901 * Must be called with preemption disabled. 902 */ 903 void pnv_power9_force_smt4_catch(void) 904 { 905 int cpu, cpu0, thr; 906 int awake_threads = 1; /* this thread is awake */ 907 int poke_threads = 0; 908 int need_awake = threads_per_core; 909 910 cpu = smp_processor_id(); 911 cpu0 = cpu & ~(threads_per_core - 1); 912 for (thr = 0; thr < threads_per_core; ++thr) { 913 if (cpu != cpu0 + thr) 914 atomic_inc(&paca_ptrs[cpu0+thr]->dont_stop); 915 } 916 /* order setting dont_stop vs testing requested_psscr */ 917 smp_mb(); 918 for (thr = 0; thr < threads_per_core; ++thr) { 919 if (!paca_ptrs[cpu0+thr]->requested_psscr) 920 ++awake_threads; 921 else 922 poke_threads |= (1 << thr); 923 } 924 925 /* If at least 3 threads are awake, the core is in SMT4 already */ 926 if (awake_threads < need_awake) { 927 /* We have to wake some threads; we'll use msgsnd */ 928 for (thr = 0; thr < threads_per_core; ++thr) { 929 if (poke_threads & (1 << thr)) { 930 ppc_msgsnd_sync(); 931 ppc_msgsnd(PPC_DBELL_MSGTYPE, 0, 932 paca_ptrs[cpu0+thr]->hw_cpu_id); 933 } 934 } 935 /* now spin until at least 3 threads are awake */ 936 do { 937 for (thr = 0; thr < threads_per_core; ++thr) { 938 if ((poke_threads & (1 << thr)) && 939 !paca_ptrs[cpu0+thr]->requested_psscr) { 940 ++awake_threads; 941 poke_threads &= ~(1 << thr); 942 } 943 } 944 } while (awake_threads < need_awake); 945 } 946 } 947 EXPORT_SYMBOL_GPL(pnv_power9_force_smt4_catch); 948 949 void pnv_power9_force_smt4_release(void) 950 { 951 int cpu, cpu0, thr; 952 953 cpu = smp_processor_id(); 954 cpu0 = cpu & ~(threads_per_core - 1); 955 956 /* clear all the dont_stop flags */ 957 for (thr = 0; thr < threads_per_core; ++thr) { 958 if (cpu != cpu0 + thr) 959 atomic_dec(&paca_ptrs[cpu0+thr]->dont_stop); 960 } 961 } 962 EXPORT_SYMBOL_GPL(pnv_power9_force_smt4_release); 963 #endif /* CONFIG_KVM_BOOK3S_HV_POSSIBLE */ 964 965 #ifdef CONFIG_HOTPLUG_CPU 966 967 void pnv_program_cpu_hotplug_lpcr(unsigned int cpu, u64 lpcr_val) 968 { 969 u64 pir = get_hard_smp_processor_id(cpu); 970 971 mtspr(SPRN_LPCR, lpcr_val); 972 973 /* 974 * Program the LPCR via stop-api only if the deepest stop state 975 * can lose hypervisor context. 976 */ 977 if (supported_cpuidle_states & OPAL_PM_LOSE_FULL_CONTEXT) 978 opal_slw_set_reg(pir, SPRN_LPCR, lpcr_val); 979 } 980 981 /* 982 * pnv_cpu_offline: A function that puts the CPU into the deepest 983 * available platform idle state on a CPU-Offline. 984 * interrupts hard disabled and no lazy irq pending. 985 */ 986 unsigned long pnv_cpu_offline(unsigned int cpu) 987 { 988 unsigned long srr1; 989 990 __ppc64_runlatch_off(); 991 992 if (cpu_has_feature(CPU_FTR_ARCH_300) && deepest_stop_found) { 993 unsigned long psscr; 994 995 psscr = mfspr(SPRN_PSSCR); 996 psscr = (psscr & ~pnv_deepest_stop_psscr_mask) | 997 pnv_deepest_stop_psscr_val; 998 srr1 = power9_offline_stop(psscr); 999 } else if (cpu_has_feature(CPU_FTR_ARCH_206) && power7_offline_type) { 1000 srr1 = power7_offline(); 1001 } else { 1002 /* This is the fallback method. We emulate snooze */ 1003 while (!generic_check_cpu_restart(cpu)) { 1004 HMT_low(); 1005 HMT_very_low(); 1006 } 1007 srr1 = 0; 1008 HMT_medium(); 1009 } 1010 1011 __ppc64_runlatch_on(); 1012 1013 return srr1; 1014 } 1015 #endif 1016 1017 /* 1018 * Power ISA 3.0 idle initialization. 1019 * 1020 * POWER ISA 3.0 defines a new SPR Processor stop Status and Control 1021 * Register (PSSCR) to control idle behavior. 1022 * 1023 * PSSCR layout: 1024 * ---------------------------------------------------------- 1025 * | PLS | /// | SD | ESL | EC | PSLL | /// | TR | MTL | RL | 1026 * ---------------------------------------------------------- 1027 * 0 4 41 42 43 44 48 54 56 60 1028 * 1029 * PSSCR key fields: 1030 * Bits 0:3 - Power-Saving Level Status (PLS). This field indicates the 1031 * lowest power-saving state the thread entered since stop instruction was 1032 * last executed. 1033 * 1034 * Bit 41 - Status Disable(SD) 1035 * 0 - Shows PLS entries 1036 * 1 - PLS entries are all 0 1037 * 1038 * Bit 42 - Enable State Loss 1039 * 0 - No state is lost irrespective of other fields 1040 * 1 - Allows state loss 1041 * 1042 * Bit 43 - Exit Criterion 1043 * 0 - Exit from power-save mode on any interrupt 1044 * 1 - Exit from power-save mode controlled by LPCR's PECE bits 1045 * 1046 * Bits 44:47 - Power-Saving Level Limit 1047 * This limits the power-saving level that can be entered into. 1048 * 1049 * Bits 60:63 - Requested Level 1050 * Used to specify which power-saving level must be entered on executing 1051 * stop instruction 1052 */ 1053 1054 int validate_psscr_val_mask(u64 *psscr_val, u64 *psscr_mask, u32 flags) 1055 { 1056 int err = 0; 1057 1058 /* 1059 * psscr_mask == 0xf indicates an older firmware. 1060 * Set remaining fields of psscr to the default values. 1061 * See NOTE above definition of PSSCR_HV_DEFAULT_VAL 1062 */ 1063 if (*psscr_mask == 0xf) { 1064 *psscr_val = *psscr_val | PSSCR_HV_DEFAULT_VAL; 1065 *psscr_mask = PSSCR_HV_DEFAULT_MASK; 1066 return err; 1067 } 1068 1069 /* 1070 * New firmware is expected to set the psscr_val bits correctly. 1071 * Validate that the following invariants are correctly maintained by 1072 * the new firmware. 1073 * - ESL bit value matches the EC bit value. 1074 * - ESL bit is set for all the deep stop states. 1075 */ 1076 if (GET_PSSCR_ESL(*psscr_val) != GET_PSSCR_EC(*psscr_val)) { 1077 err = ERR_EC_ESL_MISMATCH; 1078 } else if ((flags & OPAL_PM_LOSE_FULL_CONTEXT) && 1079 GET_PSSCR_ESL(*psscr_val) == 0) { 1080 err = ERR_DEEP_STATE_ESL_MISMATCH; 1081 } 1082 1083 return err; 1084 } 1085 1086 /* 1087 * pnv_arch300_idle_init: Initializes the default idle state, first 1088 * deep idle state and deepest idle state on 1089 * ISA 3.0 CPUs. 1090 * 1091 * @np: /ibm,opal/power-mgt device node 1092 * @flags: cpu-idle-state-flags array 1093 * @dt_idle_states: Number of idle state entries 1094 * Returns 0 on success 1095 */ 1096 static void __init pnv_power9_idle_init(void) 1097 { 1098 u64 max_residency_ns = 0; 1099 int i; 1100 1101 /* 1102 * pnv_deepest_stop_{val,mask} should be set to values corresponding to 1103 * the deepest stop state. 1104 * 1105 * pnv_default_stop_{val,mask} should be set to values corresponding to 1106 * the deepest loss-less (OPAL_PM_STOP_INST_FAST) stop state. 1107 */ 1108 pnv_first_tb_loss_level = MAX_STOP_STATE + 1; 1109 pnv_first_spr_loss_level = MAX_STOP_STATE + 1; 1110 for (i = 0; i < nr_pnv_idle_states; i++) { 1111 int err; 1112 struct pnv_idle_states_t *state = &pnv_idle_states[i]; 1113 u64 psscr_rl = state->psscr_val & PSSCR_RL_MASK; 1114 1115 if ((state->flags & OPAL_PM_TIMEBASE_STOP) && 1116 (pnv_first_tb_loss_level > psscr_rl)) 1117 pnv_first_tb_loss_level = psscr_rl; 1118 1119 if ((state->flags & OPAL_PM_LOSE_FULL_CONTEXT) && 1120 (pnv_first_spr_loss_level > psscr_rl)) 1121 pnv_first_spr_loss_level = psscr_rl; 1122 1123 /* 1124 * The idle code does not deal with TB loss occurring 1125 * in a shallower state than SPR loss, so force it to 1126 * behave like SPRs are lost if TB is lost. POWER9 would 1127 * never encouter this, but a POWER8 core would if it 1128 * implemented the stop instruction. So this is for forward 1129 * compatibility. 1130 */ 1131 if ((state->flags & OPAL_PM_TIMEBASE_STOP) && 1132 (pnv_first_spr_loss_level > psscr_rl)) 1133 pnv_first_spr_loss_level = psscr_rl; 1134 1135 err = validate_psscr_val_mask(&state->psscr_val, 1136 &state->psscr_mask, 1137 state->flags); 1138 if (err) { 1139 report_invalid_psscr_val(state->psscr_val, err); 1140 continue; 1141 } 1142 1143 state->valid = true; 1144 1145 if (max_residency_ns < state->residency_ns) { 1146 max_residency_ns = state->residency_ns; 1147 pnv_deepest_stop_psscr_val = state->psscr_val; 1148 pnv_deepest_stop_psscr_mask = state->psscr_mask; 1149 pnv_deepest_stop_flag = state->flags; 1150 deepest_stop_found = true; 1151 } 1152 1153 if (!default_stop_found && 1154 (state->flags & OPAL_PM_STOP_INST_FAST)) { 1155 pnv_default_stop_val = state->psscr_val; 1156 pnv_default_stop_mask = state->psscr_mask; 1157 default_stop_found = true; 1158 WARN_ON(state->flags & OPAL_PM_LOSE_FULL_CONTEXT); 1159 } 1160 } 1161 1162 if (unlikely(!default_stop_found)) { 1163 pr_warn("cpuidle-powernv: No suitable default stop state found. Disabling platform idle.\n"); 1164 } else { 1165 ppc_md.power_save = power9_idle; 1166 pr_info("cpuidle-powernv: Default stop: psscr = 0x%016llx,mask=0x%016llx\n", 1167 pnv_default_stop_val, pnv_default_stop_mask); 1168 } 1169 1170 if (unlikely(!deepest_stop_found)) { 1171 pr_warn("cpuidle-powernv: No suitable stop state for CPU-Hotplug. Offlined CPUs will busy wait"); 1172 } else { 1173 pr_info("cpuidle-powernv: Deepest stop: psscr = 0x%016llx,mask=0x%016llx\n", 1174 pnv_deepest_stop_psscr_val, 1175 pnv_deepest_stop_psscr_mask); 1176 } 1177 1178 pr_info("cpuidle-powernv: First stop level that may lose SPRs = 0x%llx\n", 1179 pnv_first_spr_loss_level); 1180 1181 pr_info("cpuidle-powernv: First stop level that may lose timebase = 0x%llx\n", 1182 pnv_first_tb_loss_level); 1183 } 1184 1185 static void __init pnv_disable_deep_states(void) 1186 { 1187 /* 1188 * The stop-api is unable to restore hypervisor 1189 * resources on wakeup from platform idle states which 1190 * lose full context. So disable such states. 1191 */ 1192 supported_cpuidle_states &= ~OPAL_PM_LOSE_FULL_CONTEXT; 1193 pr_warn("cpuidle-powernv: Disabling idle states that lose full context\n"); 1194 pr_warn("cpuidle-powernv: Idle power-savings, CPU-Hotplug affected\n"); 1195 1196 if (cpu_has_feature(CPU_FTR_ARCH_300) && 1197 (pnv_deepest_stop_flag & OPAL_PM_LOSE_FULL_CONTEXT)) { 1198 /* 1199 * Use the default stop state for CPU-Hotplug 1200 * if available. 1201 */ 1202 if (default_stop_found) { 1203 pnv_deepest_stop_psscr_val = pnv_default_stop_val; 1204 pnv_deepest_stop_psscr_mask = pnv_default_stop_mask; 1205 pr_warn("cpuidle-powernv: Offlined CPUs will stop with psscr = 0x%016llx\n", 1206 pnv_deepest_stop_psscr_val); 1207 } else { /* Fallback to snooze loop for CPU-Hotplug */ 1208 deepest_stop_found = false; 1209 pr_warn("cpuidle-powernv: Offlined CPUs will busy wait\n"); 1210 } 1211 } 1212 } 1213 1214 /* 1215 * Probe device tree for supported idle states 1216 */ 1217 static void __init pnv_probe_idle_states(void) 1218 { 1219 int i; 1220 1221 if (nr_pnv_idle_states < 0) { 1222 pr_warn("cpuidle-powernv: no idle states found in the DT\n"); 1223 return; 1224 } 1225 1226 if (pvr_version_is(PVR_POWER9)) 1227 pnv_power9_idle_init(); 1228 1229 for (i = 0; i < nr_pnv_idle_states; i++) 1230 supported_cpuidle_states |= pnv_idle_states[i].flags; 1231 } 1232 1233 /* 1234 * This function parses device-tree and populates all the information 1235 * into pnv_idle_states structure. It also sets up nr_pnv_idle_states 1236 * which is the number of cpuidle states discovered through device-tree. 1237 */ 1238 1239 static int pnv_parse_cpuidle_dt(void) 1240 { 1241 struct device_node *np; 1242 int nr_idle_states, i; 1243 int rc = 0; 1244 u32 *temp_u32; 1245 u64 *temp_u64; 1246 const char **temp_string; 1247 1248 np = of_find_node_by_path("/ibm,opal/power-mgt"); 1249 if (!np) { 1250 pr_warn("opal: PowerMgmt Node not found\n"); 1251 return -ENODEV; 1252 } 1253 nr_idle_states = of_property_count_u32_elems(np, 1254 "ibm,cpu-idle-state-flags"); 1255 1256 pnv_idle_states = kcalloc(nr_idle_states, sizeof(*pnv_idle_states), 1257 GFP_KERNEL); 1258 temp_u32 = kcalloc(nr_idle_states, sizeof(u32), GFP_KERNEL); 1259 temp_u64 = kcalloc(nr_idle_states, sizeof(u64), GFP_KERNEL); 1260 temp_string = kcalloc(nr_idle_states, sizeof(char *), GFP_KERNEL); 1261 1262 if (!(pnv_idle_states && temp_u32 && temp_u64 && temp_string)) { 1263 pr_err("Could not allocate memory for dt parsing\n"); 1264 rc = -ENOMEM; 1265 goto out; 1266 } 1267 1268 /* Read flags */ 1269 if (of_property_read_u32_array(np, "ibm,cpu-idle-state-flags", 1270 temp_u32, nr_idle_states)) { 1271 pr_warn("cpuidle-powernv: missing ibm,cpu-idle-state-flags in DT\n"); 1272 rc = -EINVAL; 1273 goto out; 1274 } 1275 for (i = 0; i < nr_idle_states; i++) 1276 pnv_idle_states[i].flags = temp_u32[i]; 1277 1278 /* Read latencies */ 1279 if (of_property_read_u32_array(np, "ibm,cpu-idle-state-latencies-ns", 1280 temp_u32, nr_idle_states)) { 1281 pr_warn("cpuidle-powernv: missing ibm,cpu-idle-state-latencies-ns in DT\n"); 1282 rc = -EINVAL; 1283 goto out; 1284 } 1285 for (i = 0; i < nr_idle_states; i++) 1286 pnv_idle_states[i].latency_ns = temp_u32[i]; 1287 1288 /* Read residencies */ 1289 if (of_property_read_u32_array(np, "ibm,cpu-idle-state-residency-ns", 1290 temp_u32, nr_idle_states)) { 1291 pr_warn("cpuidle-powernv: missing ibm,cpu-idle-state-residency-ns in DT\n"); 1292 rc = -EINVAL; 1293 goto out; 1294 } 1295 for (i = 0; i < nr_idle_states; i++) 1296 pnv_idle_states[i].residency_ns = temp_u32[i]; 1297 1298 /* For power9 */ 1299 if (cpu_has_feature(CPU_FTR_ARCH_300)) { 1300 /* Read pm_crtl_val */ 1301 if (of_property_read_u64_array(np, "ibm,cpu-idle-state-psscr", 1302 temp_u64, nr_idle_states)) { 1303 pr_warn("cpuidle-powernv: missing ibm,cpu-idle-state-psscr in DT\n"); 1304 rc = -EINVAL; 1305 goto out; 1306 } 1307 for (i = 0; i < nr_idle_states; i++) 1308 pnv_idle_states[i].psscr_val = temp_u64[i]; 1309 1310 /* Read pm_crtl_mask */ 1311 if (of_property_read_u64_array(np, "ibm,cpu-idle-state-psscr-mask", 1312 temp_u64, nr_idle_states)) { 1313 pr_warn("cpuidle-powernv: missing ibm,cpu-idle-state-psscr-mask in DT\n"); 1314 rc = -EINVAL; 1315 goto out; 1316 } 1317 for (i = 0; i < nr_idle_states; i++) 1318 pnv_idle_states[i].psscr_mask = temp_u64[i]; 1319 } 1320 1321 /* 1322 * power8 specific properties ibm,cpu-idle-state-pmicr-mask and 1323 * ibm,cpu-idle-state-pmicr-val were never used and there is no 1324 * plan to use it in near future. Hence, not parsing these properties 1325 */ 1326 1327 if (of_property_read_string_array(np, "ibm,cpu-idle-state-names", 1328 temp_string, nr_idle_states) < 0) { 1329 pr_warn("cpuidle-powernv: missing ibm,cpu-idle-state-names in DT\n"); 1330 rc = -EINVAL; 1331 goto out; 1332 } 1333 for (i = 0; i < nr_idle_states; i++) 1334 strlcpy(pnv_idle_states[i].name, temp_string[i], 1335 PNV_IDLE_NAME_LEN); 1336 nr_pnv_idle_states = nr_idle_states; 1337 rc = 0; 1338 out: 1339 kfree(temp_u32); 1340 kfree(temp_u64); 1341 kfree(temp_string); 1342 return rc; 1343 } 1344 1345 static int __init pnv_init_idle_states(void) 1346 { 1347 int cpu; 1348 int rc = 0; 1349 1350 /* Set up PACA fields */ 1351 for_each_present_cpu(cpu) { 1352 struct paca_struct *p = paca_ptrs[cpu]; 1353 1354 p->idle_state = 0; 1355 if (cpu == cpu_first_thread_sibling(cpu)) 1356 p->idle_state = (1 << threads_per_core) - 1; 1357 1358 if (!cpu_has_feature(CPU_FTR_ARCH_300)) { 1359 /* P7/P8 nap */ 1360 p->thread_idle_state = PNV_THREAD_RUNNING; 1361 } else { 1362 /* P9 stop */ 1363 #ifdef CONFIG_KVM_BOOK3S_HV_POSSIBLE 1364 p->requested_psscr = 0; 1365 atomic_set(&p->dont_stop, 0); 1366 #endif 1367 } 1368 } 1369 1370 /* In case we error out nr_pnv_idle_states will be zero */ 1371 nr_pnv_idle_states = 0; 1372 supported_cpuidle_states = 0; 1373 1374 if (cpuidle_disable != IDLE_NO_OVERRIDE) 1375 goto out; 1376 rc = pnv_parse_cpuidle_dt(); 1377 if (rc) 1378 return rc; 1379 pnv_probe_idle_states(); 1380 1381 if (!cpu_has_feature(CPU_FTR_ARCH_300)) { 1382 if (!(supported_cpuidle_states & OPAL_PM_SLEEP_ENABLED_ER1)) { 1383 power7_fastsleep_workaround_entry = false; 1384 power7_fastsleep_workaround_exit = false; 1385 } else { 1386 /* 1387 * OPAL_PM_SLEEP_ENABLED_ER1 is set. It indicates that 1388 * workaround is needed to use fastsleep. Provide sysfs 1389 * control to choose how this workaround has to be 1390 * applied. 1391 */ 1392 device_create_file(cpu_subsys.dev_root, 1393 &dev_attr_fastsleep_workaround_applyonce); 1394 } 1395 1396 update_subcore_sibling_mask(); 1397 1398 if (supported_cpuidle_states & OPAL_PM_NAP_ENABLED) { 1399 ppc_md.power_save = power7_idle; 1400 power7_offline_type = PNV_THREAD_NAP; 1401 } 1402 1403 if ((supported_cpuidle_states & OPAL_PM_WINKLE_ENABLED) && 1404 (supported_cpuidle_states & OPAL_PM_LOSE_FULL_CONTEXT)) 1405 power7_offline_type = PNV_THREAD_WINKLE; 1406 else if ((supported_cpuidle_states & OPAL_PM_SLEEP_ENABLED) || 1407 (supported_cpuidle_states & OPAL_PM_SLEEP_ENABLED_ER1)) 1408 power7_offline_type = PNV_THREAD_SLEEP; 1409 } 1410 1411 if (supported_cpuidle_states & OPAL_PM_LOSE_FULL_CONTEXT) { 1412 if (pnv_save_sprs_for_deep_states()) 1413 pnv_disable_deep_states(); 1414 } 1415 1416 out: 1417 return 0; 1418 } 1419 machine_subsys_initcall(powernv, pnv_init_idle_states); 1420