1/* 2 * This file contains idle entry/exit functions for POWER7, 3 * POWER8 and POWER9 CPUs. 4 * 5 * This program is free software; you can redistribute it and/or 6 * modify it under the terms of the GNU General Public License 7 * as published by the Free Software Foundation; either version 8 * 2 of the License, or (at your option) any later version. 9 */ 10 11#include <linux/threads.h> 12#include <asm/processor.h> 13#include <asm/page.h> 14#include <asm/cputable.h> 15#include <asm/thread_info.h> 16#include <asm/ppc_asm.h> 17#include <asm/asm-offsets.h> 18#include <asm/ppc-opcode.h> 19#include <asm/hw_irq.h> 20#include <asm/kvm_book3s_asm.h> 21#include <asm/opal.h> 22#include <asm/cpuidle.h> 23#include <asm/exception-64s.h> 24#include <asm/book3s/64/mmu-hash.h> 25#include <asm/mmu.h> 26#include <asm/asm-compat.h> 27#include <asm/feature-fixups.h> 28 29#undef DEBUG 30 31/* 32 * Use unused space in the interrupt stack to save and restore 33 * registers for winkle support. 34 */ 35#define _MMCR0 GPR0 36#define _SDR1 GPR3 37#define _PTCR GPR3 38#define _RPR GPR4 39#define _SPURR GPR5 40#define _PURR GPR6 41#define _TSCR GPR7 42#define _DSCR GPR8 43#define _AMOR GPR9 44#define _WORT GPR10 45#define _WORC GPR11 46#define _LPCR GPR12 47 48#define PSSCR_EC_ESL_MASK_SHIFTED (PSSCR_EC | PSSCR_ESL) >> 16 49 50 .text 51 52/* 53 * Used by threads before entering deep idle states. Saves SPRs 54 * in interrupt stack frame 55 */ 56save_sprs_to_stack: 57 /* 58 * Note all register i.e per-core, per-subcore or per-thread is saved 59 * here since any thread in the core might wake up first 60 */ 61BEGIN_FTR_SECTION 62 /* 63 * Note - SDR1 is dropped in Power ISA v3. Hence not restoring 64 * SDR1 here 65 */ 66 mfspr r3,SPRN_PTCR 67 std r3,_PTCR(r1) 68 mfspr r3,SPRN_LPCR 69 std r3,_LPCR(r1) 70FTR_SECTION_ELSE 71 mfspr r3,SPRN_SDR1 72 std r3,_SDR1(r1) 73ALT_FTR_SECTION_END_IFSET(CPU_FTR_ARCH_300) 74 mfspr r3,SPRN_RPR 75 std r3,_RPR(r1) 76 mfspr r3,SPRN_SPURR 77 std r3,_SPURR(r1) 78 mfspr r3,SPRN_PURR 79 std r3,_PURR(r1) 80 mfspr r3,SPRN_TSCR 81 std r3,_TSCR(r1) 82 mfspr r3,SPRN_DSCR 83 std r3,_DSCR(r1) 84 mfspr r3,SPRN_AMOR 85 std r3,_AMOR(r1) 86 mfspr r3,SPRN_WORT 87 std r3,_WORT(r1) 88 mfspr r3,SPRN_WORC 89 std r3,_WORC(r1) 90/* 91 * On POWER9, there are idle states such as stop4, invoked via cpuidle, 92 * that lose hypervisor resources. In such cases, we need to save 93 * additional SPRs before entering those idle states so that they can 94 * be restored to their older values on wakeup from the idle state. 95 * 96 * On POWER8, the only such deep idle state is winkle which is used 97 * only in the context of CPU-Hotplug, where these additional SPRs are 98 * reinitiazed to a sane value. Hence there is no need to save/restore 99 * these SPRs. 100 */ 101BEGIN_FTR_SECTION 102 blr 103END_FTR_SECTION_IFCLR(CPU_FTR_ARCH_300) 104 105power9_save_additional_sprs: 106 mfspr r3, SPRN_PID 107 mfspr r4, SPRN_LDBAR 108 std r3, STOP_PID(r13) 109 std r4, STOP_LDBAR(r13) 110 111 mfspr r3, SPRN_FSCR 112 mfspr r4, SPRN_HFSCR 113 std r3, STOP_FSCR(r13) 114 std r4, STOP_HFSCR(r13) 115 116 mfspr r3, SPRN_MMCRA 117 mfspr r4, SPRN_MMCR0 118 std r3, STOP_MMCRA(r13) 119 std r4, _MMCR0(r1) 120 121 mfspr r3, SPRN_MMCR1 122 mfspr r4, SPRN_MMCR2 123 std r3, STOP_MMCR1(r13) 124 std r4, STOP_MMCR2(r13) 125 blr 126 127power9_restore_additional_sprs: 128 ld r3,_LPCR(r1) 129 ld r4, STOP_PID(r13) 130 mtspr SPRN_LPCR,r3 131 mtspr SPRN_PID, r4 132 133 ld r3, STOP_LDBAR(r13) 134 ld r4, STOP_FSCR(r13) 135 mtspr SPRN_LDBAR, r3 136 mtspr SPRN_FSCR, r4 137 138 ld r3, STOP_HFSCR(r13) 139 ld r4, STOP_MMCRA(r13) 140 mtspr SPRN_HFSCR, r3 141 mtspr SPRN_MMCRA, r4 142 143 ld r3, _MMCR0(r1) 144 ld r4, STOP_MMCR1(r13) 145 mtspr SPRN_MMCR0, r3 146 mtspr SPRN_MMCR1, r4 147 148 ld r3, STOP_MMCR2(r13) 149 ld r4, PACA_SPRG_VDSO(r13) 150 mtspr SPRN_MMCR2, r3 151 mtspr SPRN_SPRG3, r4 152 blr 153 154/* 155 * Used by threads when the lock bit of core_idle_state is set. 156 * Threads will spin in HMT_LOW until the lock bit is cleared. 157 * r14 - pointer to core_idle_state 158 * r15 - used to load contents of core_idle_state 159 * r9 - used as a temporary variable 160 */ 161 162core_idle_lock_held: 163 HMT_LOW 1643: lwz r15,0(r14) 165 andis. r15,r15,PNV_CORE_IDLE_LOCK_BIT@h 166 bne 3b 167 HMT_MEDIUM 168 lwarx r15,0,r14 169 andis. r9,r15,PNV_CORE_IDLE_LOCK_BIT@h 170 bne- core_idle_lock_held 171 blr 172 173/* 174 * Pass requested state in r3: 175 * r3 - PNV_THREAD_NAP/SLEEP/WINKLE in POWER8 176 * - Requested PSSCR value in POWER9 177 * 178 * Address of idle handler to branch to in realmode in r4 179 */ 180pnv_powersave_common: 181 /* Use r3 to pass state nap/sleep/winkle */ 182 /* NAP is a state loss, we create a regs frame on the 183 * stack, fill it up with the state we care about and 184 * stick a pointer to it in PACAR1. We really only 185 * need to save PC, some CR bits and the NV GPRs, 186 * but for now an interrupt frame will do. 187 */ 188 mtctr r4 189 190 mflr r0 191 std r0,16(r1) 192 stdu r1,-INT_FRAME_SIZE(r1) 193 std r0,_LINK(r1) 194 std r0,_NIP(r1) 195 196 /* We haven't lost state ... yet */ 197 li r0,0 198 stb r0,PACA_NAPSTATELOST(r13) 199 200 /* Continue saving state */ 201 SAVE_GPR(2, r1) 202 SAVE_NVGPRS(r1) 203 mfcr r5 204 std r5,_CCR(r1) 205 std r1,PACAR1(r13) 206 207BEGIN_FTR_SECTION 208 /* 209 * POWER9 does not require real mode to stop, and presently does not 210 * set hwthread_state for KVM (threads don't share MMU context), so 211 * we can remain in virtual mode for this. 212 */ 213 bctr 214END_FTR_SECTION_IFSET(CPU_FTR_ARCH_300) 215 /* 216 * POWER8 217 * Go to real mode to do the nap, as required by the architecture. 218 * Also, we need to be in real mode before setting hwthread_state, 219 * because as soon as we do that, another thread can switch 220 * the MMU context to the guest. 221 */ 222 LOAD_REG_IMMEDIATE(r7, MSR_IDLE) 223 mtmsrd r7,0 224 bctr 225 226/* 227 * This is the sequence required to execute idle instructions, as 228 * specified in ISA v2.07 (and earlier). MSR[IR] and MSR[DR] must be 0. 229 */ 230#define IDLE_STATE_ENTER_SEQ_NORET(IDLE_INST) \ 231 /* Magic NAP/SLEEP/WINKLE mode enter sequence */ \ 232 std r0,0(r1); \ 233 ptesync; \ 234 ld r0,0(r1); \ 235236: cmpd cr0,r0,r0; \ 236 bne 236b; \ 237 IDLE_INST; 238 239 240 .globl pnv_enter_arch207_idle_mode 241pnv_enter_arch207_idle_mode: 242#ifdef CONFIG_KVM_BOOK3S_HV_POSSIBLE 243 /* Tell KVM we're entering idle */ 244 li r4,KVM_HWTHREAD_IN_IDLE 245 /******************************************************/ 246 /* N O T E W E L L ! ! ! N O T E W E L L */ 247 /* The following store to HSTATE_HWTHREAD_STATE(r13) */ 248 /* MUST occur in real mode, i.e. with the MMU off, */ 249 /* and the MMU must stay off until we clear this flag */ 250 /* and test HSTATE_HWTHREAD_REQ(r13) in */ 251 /* pnv_powersave_wakeup in this file. */ 252 /* The reason is that another thread can switch the */ 253 /* MMU to a guest context whenever this flag is set */ 254 /* to KVM_HWTHREAD_IN_IDLE, and if the MMU was on, */ 255 /* that would potentially cause this thread to start */ 256 /* executing instructions from guest memory in */ 257 /* hypervisor mode, leading to a host crash or data */ 258 /* corruption, or worse. */ 259 /******************************************************/ 260 stb r4,HSTATE_HWTHREAD_STATE(r13) 261#endif 262 stb r3,PACA_THREAD_IDLE_STATE(r13) 263 cmpwi cr3,r3,PNV_THREAD_SLEEP 264 bge cr3,2f 265 IDLE_STATE_ENTER_SEQ_NORET(PPC_NAP) 266 /* No return */ 2672: 268 /* Sleep or winkle */ 269 lbz r7,PACA_THREAD_MASK(r13) 270 ld r14,PACA_CORE_IDLE_STATE_PTR(r13) 271 li r5,0 272 beq cr3,3f 273 lis r5,PNV_CORE_IDLE_WINKLE_COUNT@h 2743: 275lwarx_loop1: 276 lwarx r15,0,r14 277 278 andis. r9,r15,PNV_CORE_IDLE_LOCK_BIT@h 279 bnel- core_idle_lock_held 280 281 add r15,r15,r5 /* Add if winkle */ 282 andc r15,r15,r7 /* Clear thread bit */ 283 284 andi. r9,r15,PNV_CORE_IDLE_THREAD_BITS 285 286/* 287 * If cr0 = 0, then current thread is the last thread of the core entering 288 * sleep. Last thread needs to execute the hardware bug workaround code if 289 * required by the platform. 290 * Make the workaround call unconditionally here. The below branch call is 291 * patched out when the idle states are discovered if the platform does not 292 * require it. 293 */ 294.global pnv_fastsleep_workaround_at_entry 295pnv_fastsleep_workaround_at_entry: 296 beq fastsleep_workaround_at_entry 297 298 stwcx. r15,0,r14 299 bne- lwarx_loop1 300 isync 301 302common_enter: /* common code for all the threads entering sleep or winkle */ 303 bgt cr3,enter_winkle 304 IDLE_STATE_ENTER_SEQ_NORET(PPC_SLEEP) 305 306fastsleep_workaround_at_entry: 307 oris r15,r15,PNV_CORE_IDLE_LOCK_BIT@h 308 stwcx. r15,0,r14 309 bne- lwarx_loop1 310 isync 311 312 /* Fast sleep workaround */ 313 li r3,1 314 li r4,1 315 bl opal_config_cpu_idle_state 316 317 /* Unlock */ 318 xoris r15,r15,PNV_CORE_IDLE_LOCK_BIT@h 319 lwsync 320 stw r15,0(r14) 321 b common_enter 322 323enter_winkle: 324 bl save_sprs_to_stack 325 326 IDLE_STATE_ENTER_SEQ_NORET(PPC_WINKLE) 327 328/* 329 * r3 - PSSCR value corresponding to the requested stop state. 330 */ 331power_enter_stop: 332/* 333 * Check if we are executing the lite variant with ESL=EC=0 334 */ 335 andis. r4,r3,PSSCR_EC_ESL_MASK_SHIFTED 336 clrldi r3,r3,60 /* r3 = Bits[60:63] = Requested Level (RL) */ 337 bne .Lhandle_esl_ec_set 338 PPC_STOP 339 li r3,0 /* Since we didn't lose state, return 0 */ 340 std r3, PACA_REQ_PSSCR(r13) 341 342 /* 343 * pnv_wakeup_noloss() expects r12 to contain the SRR1 value so 344 * it can determine if the wakeup reason is an HMI in 345 * CHECK_HMI_INTERRUPT. 346 * 347 * However, when we wakeup with ESL=0, SRR1 will not contain the wakeup 348 * reason, so there is no point setting r12 to SRR1. 349 * 350 * Further, we clear r12 here, so that we don't accidentally enter the 351 * HMI in pnv_wakeup_noloss() if the value of r12[42:45] == WAKE_HMI. 352 */ 353 li r12, 0 354 b pnv_wakeup_noloss 355 356.Lhandle_esl_ec_set: 357BEGIN_FTR_SECTION 358 /* 359 * POWER9 DD2.0 or earlier can incorrectly set PMAO when waking up after 360 * a state-loss idle. Saving and restoring MMCR0 over idle is a 361 * workaround. 362 */ 363 mfspr r4,SPRN_MMCR0 364 std r4,_MMCR0(r1) 365END_FTR_SECTION_IFCLR(CPU_FTR_POWER9_DD2_1) 366 367/* 368 * Check if the requested state is a deep idle state. 369 */ 370 LOAD_REG_ADDRBASE(r5,pnv_first_deep_stop_state) 371 ld r4,ADDROFF(pnv_first_deep_stop_state)(r5) 372 cmpd r3,r4 373 bge .Lhandle_deep_stop 374 PPC_STOP /* Does not return (system reset interrupt) */ 375 376.Lhandle_deep_stop: 377/* 378 * Entering deep idle state. 379 * Clear thread bit in PACA_CORE_IDLE_STATE, save SPRs to 380 * stack and enter stop 381 */ 382 lbz r7,PACA_THREAD_MASK(r13) 383 ld r14,PACA_CORE_IDLE_STATE_PTR(r13) 384 385lwarx_loop_stop: 386 lwarx r15,0,r14 387 andis. r9,r15,PNV_CORE_IDLE_LOCK_BIT@h 388 bnel- core_idle_lock_held 389 andc r15,r15,r7 /* Clear thread bit */ 390 391 stwcx. r15,0,r14 392 bne- lwarx_loop_stop 393 isync 394 395 bl save_sprs_to_stack 396 397 PPC_STOP /* Does not return (system reset interrupt) */ 398 399/* 400 * Entered with MSR[EE]=0 and no soft-masked interrupts pending. 401 * r3 contains desired idle state (PNV_THREAD_NAP/SLEEP/WINKLE). 402 */ 403_GLOBAL(power7_idle_insn) 404 /* Now check if user or arch enabled NAP mode */ 405 LOAD_REG_ADDR(r4, pnv_enter_arch207_idle_mode) 406 b pnv_powersave_common 407 408#define CHECK_HMI_INTERRUPT \ 409BEGIN_FTR_SECTION_NESTED(66); \ 410 rlwinm r0,r12,45-31,0xf; /* extract wake reason field (P8) */ \ 411FTR_SECTION_ELSE_NESTED(66); \ 412 rlwinm r0,r12,45-31,0xe; /* P7 wake reason field is 3 bits */ \ 413ALT_FTR_SECTION_END_NESTED_IFSET(CPU_FTR_ARCH_207S, 66); \ 414 cmpwi r0,0xa; /* Hypervisor maintenance ? */ \ 415 bne+ 20f; \ 416 /* Invoke opal call to handle hmi */ \ 417 ld r2,PACATOC(r13); \ 418 ld r1,PACAR1(r13); \ 419 std r3,ORIG_GPR3(r1); /* Save original r3 */ \ 420 li r3,0; /* NULL argument */ \ 421 bl hmi_exception_realmode; \ 422 nop; \ 423 ld r3,ORIG_GPR3(r1); /* Restore original r3 */ \ 42420: nop; 425 426/* 427 * Entered with MSR[EE]=0 and no soft-masked interrupts pending. 428 * r3 contains desired PSSCR register value. 429 * 430 * Offline (CPU unplug) case also must notify KVM that the CPU is 431 * idle. 432 */ 433_GLOBAL(power9_offline_stop) 434#ifdef CONFIG_KVM_BOOK3S_HV_POSSIBLE 435 /* 436 * Tell KVM we're entering idle. 437 * This does not have to be done in real mode because the P9 MMU 438 * is independent per-thread. Some steppings share radix/hash mode 439 * between threads, but in that case KVM has a barrier sync in real 440 * mode before and after switching between radix and hash. 441 */ 442 li r4,KVM_HWTHREAD_IN_IDLE 443 stb r4,HSTATE_HWTHREAD_STATE(r13) 444#endif 445 /* fall through */ 446 447_GLOBAL(power9_idle_stop) 448 std r3, PACA_REQ_PSSCR(r13) 449#ifdef CONFIG_KVM_BOOK3S_HV_POSSIBLE 450BEGIN_FTR_SECTION 451 sync 452 lwz r5, PACA_DONT_STOP(r13) 453 cmpwi r5, 0 454 bne 1f 455END_FTR_SECTION_IFSET(CPU_FTR_P9_TM_XER_SO_BUG) 456#endif 457 mtspr SPRN_PSSCR,r3 458 LOAD_REG_ADDR(r4,power_enter_stop) 459 b pnv_powersave_common 460 /* No return */ 461#ifdef CONFIG_KVM_BOOK3S_HV_POSSIBLE 4621: 463 /* 464 * We get here when TM / thread reconfiguration bug workaround 465 * code wants to get the CPU into SMT4 mode, and therefore 466 * we are being asked not to stop. 467 */ 468 li r3, 0 469 std r3, PACA_REQ_PSSCR(r13) 470 blr /* return 0 for wakeup cause / SRR1 value */ 471#endif 472 473/* 474 * Called from machine check handler for powersave wakeups. 475 * Low level machine check processing has already been done. Now just 476 * go through the wake up path to get everything in order. 477 * 478 * r3 - The original SRR1 value. 479 * Original SRR[01] have been clobbered. 480 * MSR_RI is clear. 481 */ 482.global pnv_powersave_wakeup_mce 483pnv_powersave_wakeup_mce: 484 /* Set cr3 for pnv_powersave_wakeup */ 485 rlwinm r11,r3,47-31,30,31 486 cmpwi cr3,r11,2 487 488 /* 489 * Now put the original SRR1 with SRR1_WAKEMCE_RESVD as the wake 490 * reason into r12, which allows reuse of the system reset wakeup 491 * code without being mistaken for another type of wakeup. 492 */ 493 oris r12,r3,SRR1_WAKEMCE_RESVD@h 494 495 b pnv_powersave_wakeup 496 497/* 498 * Called from reset vector for powersave wakeups. 499 * cr3 - set to gt if waking up with partial/complete hypervisor state loss 500 * r12 - SRR1 501 */ 502.global pnv_powersave_wakeup 503pnv_powersave_wakeup: 504 ld r2, PACATOC(r13) 505 506BEGIN_FTR_SECTION 507 bl pnv_restore_hyp_resource_arch300 508FTR_SECTION_ELSE 509 bl pnv_restore_hyp_resource_arch207 510ALT_FTR_SECTION_END_IFSET(CPU_FTR_ARCH_300) 511 512 li r0,PNV_THREAD_RUNNING 513 stb r0,PACA_THREAD_IDLE_STATE(r13) /* Clear thread state */ 514 515 mr r3,r12 516 517#ifdef CONFIG_KVM_BOOK3S_HV_POSSIBLE 518 lbz r0,HSTATE_HWTHREAD_STATE(r13) 519 cmpwi r0,KVM_HWTHREAD_IN_KERNEL 520 beq 0f 521 li r0,KVM_HWTHREAD_IN_KERNEL 522 stb r0,HSTATE_HWTHREAD_STATE(r13) 523 /* Order setting hwthread_state vs. testing hwthread_req */ 524 sync 5250: lbz r0,HSTATE_HWTHREAD_REQ(r13) 526 cmpwi r0,0 527 beq 1f 528 b kvm_start_guest 5291: 530#endif 531 532 /* Return SRR1 from power7_nap() */ 533 blt cr3,pnv_wakeup_noloss 534 b pnv_wakeup_loss 535 536/* 537 * Check whether we have woken up with hypervisor state loss. 538 * If yes, restore hypervisor state and return back to link. 539 * 540 * cr3 - set to gt if waking up with partial/complete hypervisor state loss 541 */ 542pnv_restore_hyp_resource_arch300: 543 /* 544 * Workaround for POWER9, if we lost resources, the ERAT 545 * might have been mixed up and needs flushing. We also need 546 * to reload MMCR0 (see comment above). We also need to set 547 * then clear bit 60 in MMCRA to ensure the PMU starts running. 548 */ 549 blt cr3,1f 550BEGIN_FTR_SECTION 551 PPC_INVALIDATE_ERAT 552 ld r1,PACAR1(r13) 553 ld r4,_MMCR0(r1) 554 mtspr SPRN_MMCR0,r4 555END_FTR_SECTION_IFCLR(CPU_FTR_POWER9_DD2_1) 556 mfspr r4,SPRN_MMCRA 557 ori r4,r4,(1 << (63-60)) 558 mtspr SPRN_MMCRA,r4 559 xori r4,r4,(1 << (63-60)) 560 mtspr SPRN_MMCRA,r4 5611: 562 /* 563 * POWER ISA 3. Use PSSCR to determine if we 564 * are waking up from deep idle state 565 */ 566 LOAD_REG_ADDRBASE(r5,pnv_first_deep_stop_state) 567 ld r4,ADDROFF(pnv_first_deep_stop_state)(r5) 568 569 /* 570 * 0-3 bits correspond to Power-Saving Level Status 571 * which indicates the idle state we are waking up from 572 */ 573 mfspr r5, SPRN_PSSCR 574 rldicl r5,r5,4,60 575 li r0, 0 /* clear requested_psscr to say we're awake */ 576 std r0, PACA_REQ_PSSCR(r13) 577 cmpd cr4,r5,r4 578 bge cr4,pnv_wakeup_tb_loss /* returns to caller */ 579 580 blr /* Waking up without hypervisor state loss. */ 581 582/* Same calling convention as arch300 */ 583pnv_restore_hyp_resource_arch207: 584 /* 585 * POWER ISA 2.07 or less. 586 * Check if we slept with sleep or winkle. 587 */ 588 lbz r4,PACA_THREAD_IDLE_STATE(r13) 589 cmpwi cr2,r4,PNV_THREAD_NAP 590 bgt cr2,pnv_wakeup_tb_loss /* Either sleep or Winkle */ 591 592 /* 593 * We fall through here if PACA_THREAD_IDLE_STATE shows we are waking 594 * up from nap. At this stage CR3 shouldn't contains 'gt' since that 595 * indicates we are waking with hypervisor state loss from nap. 596 */ 597 bgt cr3,. 598 599 blr /* Waking up without hypervisor state loss */ 600 601/* 602 * Called if waking up from idle state which can cause either partial or 603 * complete hyp state loss. 604 * In POWER8, called if waking up from fastsleep or winkle 605 * In POWER9, called if waking up from stop state >= pnv_first_deep_stop_state 606 * 607 * r13 - PACA 608 * cr3 - gt if waking up with partial/complete hypervisor state loss 609 * 610 * If ISA300: 611 * cr4 - gt or eq if waking up from complete hypervisor state loss. 612 * 613 * If ISA207: 614 * r4 - PACA_THREAD_IDLE_STATE 615 */ 616pnv_wakeup_tb_loss: 617 ld r1,PACAR1(r13) 618 /* 619 * Before entering any idle state, the NVGPRs are saved in the stack. 620 * If there was a state loss, or PACA_NAPSTATELOST was set, then the 621 * NVGPRs are restored. If we are here, it is likely that state is lost, 622 * but not guaranteed -- neither ISA207 nor ISA300 tests to reach 623 * here are the same as the test to restore NVGPRS: 624 * PACA_THREAD_IDLE_STATE test for ISA207, PSSCR test for ISA300, 625 * and SRR1 test for restoring NVGPRs. 626 * 627 * We are about to clobber NVGPRs now, so set NAPSTATELOST to 628 * guarantee they will always be restored. This might be tightened 629 * with careful reading of specs (particularly for ISA300) but this 630 * is already a slow wakeup path and it's simpler to be safe. 631 */ 632 li r0,1 633 stb r0,PACA_NAPSTATELOST(r13) 634 635 /* 636 * 637 * Save SRR1 and LR in NVGPRs as they might be clobbered in 638 * opal_call() (called in CHECK_HMI_INTERRUPT). SRR1 is required 639 * to determine the wakeup reason if we branch to kvm_start_guest. LR 640 * is required to return back to reset vector after hypervisor state 641 * restore is complete. 642 */ 643 mr r19,r12 644 mr r18,r4 645 mflr r17 646BEGIN_FTR_SECTION 647 CHECK_HMI_INTERRUPT 648END_FTR_SECTION_IFSET(CPU_FTR_HVMODE) 649 650 ld r14,PACA_CORE_IDLE_STATE_PTR(r13) 651 lbz r7,PACA_THREAD_MASK(r13) 652 653 /* 654 * Take the core lock to synchronize against other threads. 655 * 656 * Lock bit is set in one of the 2 cases- 657 * a. In the sleep/winkle enter path, the last thread is executing 658 * fastsleep workaround code. 659 * b. In the wake up path, another thread is executing fastsleep 660 * workaround undo code or resyncing timebase or restoring context 661 * In either case loop until the lock bit is cleared. 662 */ 6631: 664 lwarx r15,0,r14 665 andis. r9,r15,PNV_CORE_IDLE_LOCK_BIT@h 666 bnel- core_idle_lock_held 667 oris r15,r15,PNV_CORE_IDLE_LOCK_BIT@h 668 stwcx. r15,0,r14 669 bne- 1b 670 isync 671 672 andi. r9,r15,PNV_CORE_IDLE_THREAD_BITS 673 cmpwi cr2,r9,0 674 675 /* 676 * At this stage 677 * cr2 - eq if first thread to wakeup in core 678 * cr3- gt if waking up with partial/complete hypervisor state loss 679 * ISA300: 680 * cr4 - gt or eq if waking up from complete hypervisor state loss. 681 */ 682 683BEGIN_FTR_SECTION 684 /* 685 * Were we in winkle? 686 * If yes, check if all threads were in winkle, decrement our 687 * winkle count, set all thread winkle bits if all were in winkle. 688 * Check if our thread has a winkle bit set, and set cr4 accordingly 689 * (to match ISA300, above). Pseudo-code for core idle state 690 * transitions for ISA207 is as follows (everything happens atomically 691 * due to store conditional and/or lock bit): 692 * 693 * nap_idle() { } 694 * nap_wake() { } 695 * 696 * sleep_idle() 697 * { 698 * core_idle_state &= ~thread_in_core 699 * } 700 * 701 * sleep_wake() 702 * { 703 * bool first_in_core, first_in_subcore; 704 * 705 * first_in_core = (core_idle_state & IDLE_THREAD_BITS) == 0; 706 * first_in_subcore = (core_idle_state & SUBCORE_SIBLING_MASK) == 0; 707 * 708 * core_idle_state |= thread_in_core; 709 * } 710 * 711 * winkle_idle() 712 * { 713 * core_idle_state &= ~thread_in_core; 714 * core_idle_state += 1 << WINKLE_COUNT_SHIFT; 715 * } 716 * 717 * winkle_wake() 718 * { 719 * bool first_in_core, first_in_subcore, winkle_state_lost; 720 * 721 * first_in_core = (core_idle_state & IDLE_THREAD_BITS) == 0; 722 * first_in_subcore = (core_idle_state & SUBCORE_SIBLING_MASK) == 0; 723 * 724 * core_idle_state |= thread_in_core; 725 * 726 * if ((core_idle_state & WINKLE_MASK) == (8 << WINKLE_COUNT_SIHFT)) 727 * core_idle_state |= THREAD_WINKLE_BITS; 728 * core_idle_state -= 1 << WINKLE_COUNT_SHIFT; 729 * 730 * winkle_state_lost = core_idle_state & 731 * (thread_in_core << WINKLE_THREAD_SHIFT); 732 * core_idle_state &= ~(thread_in_core << WINKLE_THREAD_SHIFT); 733 * } 734 * 735 */ 736 cmpwi r18,PNV_THREAD_WINKLE 737 bne 2f 738 andis. r9,r15,PNV_CORE_IDLE_WINKLE_COUNT_ALL_BIT@h 739 subis r15,r15,PNV_CORE_IDLE_WINKLE_COUNT@h 740 beq 2f 741 ori r15,r15,PNV_CORE_IDLE_THREAD_WINKLE_BITS /* all were winkle */ 7422: 743 /* Shift thread bit to winkle mask, then test if this thread is set, 744 * and remove it from the winkle bits */ 745 slwi r8,r7,8 746 and r8,r8,r15 747 andc r15,r15,r8 748 cmpwi cr4,r8,1 /* cr4 will be gt if our bit is set, lt if not */ 749 750 lbz r4,PACA_SUBCORE_SIBLING_MASK(r13) 751 and r4,r4,r15 752 cmpwi r4,0 /* Check if first in subcore */ 753 754 or r15,r15,r7 /* Set thread bit */ 755 beq first_thread_in_subcore 756END_FTR_SECTION_IFCLR(CPU_FTR_ARCH_300) 757 758 or r15,r15,r7 /* Set thread bit */ 759 beq cr2,first_thread_in_core 760 761 /* Not first thread in core or subcore to wake up */ 762 b clear_lock 763 764first_thread_in_subcore: 765 /* 766 * If waking up from sleep, subcore state is not lost. Hence 767 * skip subcore state restore 768 */ 769 blt cr4,subcore_state_restored 770 771 /* Restore per-subcore state */ 772 ld r4,_SDR1(r1) 773 mtspr SPRN_SDR1,r4 774 775 ld r4,_RPR(r1) 776 mtspr SPRN_RPR,r4 777 ld r4,_AMOR(r1) 778 mtspr SPRN_AMOR,r4 779 780subcore_state_restored: 781 /* 782 * Check if the thread is also the first thread in the core. If not, 783 * skip to clear_lock. 784 */ 785 bne cr2,clear_lock 786 787first_thread_in_core: 788 789 /* 790 * First thread in the core waking up from any state which can cause 791 * partial or complete hypervisor state loss. It needs to 792 * call the fastsleep workaround code if the platform requires it. 793 * Call it unconditionally here. The below branch instruction will 794 * be patched out if the platform does not have fastsleep or does not 795 * require the workaround. Patching will be performed during the 796 * discovery of idle-states. 797 */ 798.global pnv_fastsleep_workaround_at_exit 799pnv_fastsleep_workaround_at_exit: 800 b fastsleep_workaround_at_exit 801 802timebase_resync: 803 /* 804 * Use cr3 which indicates that we are waking up with atleast partial 805 * hypervisor state loss to determine if TIMEBASE RESYNC is needed. 806 */ 807 ble cr3,.Ltb_resynced 808 /* Time base re-sync */ 809 bl opal_resync_timebase; 810 /* 811 * If waking up from sleep (POWER8), per core state 812 * is not lost, skip to clear_lock. 813 */ 814.Ltb_resynced: 815 blt cr4,clear_lock 816 817 /* 818 * First thread in the core to wake up and its waking up with 819 * complete hypervisor state loss. Restore per core hypervisor 820 * state. 821 */ 822BEGIN_FTR_SECTION 823 ld r4,_PTCR(r1) 824 mtspr SPRN_PTCR,r4 825 ld r4,_RPR(r1) 826 mtspr SPRN_RPR,r4 827 ld r4,_AMOR(r1) 828 mtspr SPRN_AMOR,r4 829END_FTR_SECTION_IFSET(CPU_FTR_ARCH_300) 830 831 ld r4,_TSCR(r1) 832 mtspr SPRN_TSCR,r4 833 ld r4,_WORC(r1) 834 mtspr SPRN_WORC,r4 835 836clear_lock: 837 xoris r15,r15,PNV_CORE_IDLE_LOCK_BIT@h 838 lwsync 839 stw r15,0(r14) 840 841common_exit: 842 /* 843 * Common to all threads. 844 * 845 * If waking up from sleep, hypervisor state is not lost. Hence 846 * skip hypervisor state restore. 847 */ 848 blt cr4,hypervisor_state_restored 849 850 /* Waking up from winkle */ 851 852BEGIN_MMU_FTR_SECTION 853 b no_segments 854END_MMU_FTR_SECTION_IFSET(MMU_FTR_TYPE_RADIX) 855 /* Restore SLB from PACA */ 856 ld r8,PACA_SLBSHADOWPTR(r13) 857 858 .rept SLB_NUM_BOLTED 859 li r3, SLBSHADOW_SAVEAREA 860 LDX_BE r5, r8, r3 861 addi r3, r3, 8 862 LDX_BE r6, r8, r3 863 andis. r7,r5,SLB_ESID_V@h 864 beq 1f 865 slbmte r6,r5 8661: addi r8,r8,16 867 .endr 868no_segments: 869 870 /* Restore per thread state */ 871 872 ld r4,_SPURR(r1) 873 mtspr SPRN_SPURR,r4 874 ld r4,_PURR(r1) 875 mtspr SPRN_PURR,r4 876 ld r4,_DSCR(r1) 877 mtspr SPRN_DSCR,r4 878 ld r4,_WORT(r1) 879 mtspr SPRN_WORT,r4 880 881 /* Call cur_cpu_spec->cpu_restore() */ 882 LOAD_REG_ADDR(r4, cur_cpu_spec) 883 ld r4,0(r4) 884 ld r12,CPU_SPEC_RESTORE(r4) 885#ifdef PPC64_ELF_ABI_v1 886 ld r12,0(r12) 887#endif 888 mtctr r12 889 bctrl 890 891/* 892 * On POWER9, we can come here on wakeup from a cpuidle stop state. 893 * Hence restore the additional SPRs to the saved value. 894 * 895 * On POWER8, we come here only on winkle. Since winkle is used 896 * only in the case of CPU-Hotplug, we don't need to restore 897 * the additional SPRs. 898 */ 899BEGIN_FTR_SECTION 900 bl power9_restore_additional_sprs 901END_FTR_SECTION_IFSET(CPU_FTR_ARCH_300) 902hypervisor_state_restored: 903 904 mr r12,r19 905 mtlr r17 906 blr /* return to pnv_powersave_wakeup */ 907 908fastsleep_workaround_at_exit: 909 li r3,1 910 li r4,0 911 bl opal_config_cpu_idle_state 912 b timebase_resync 913 914/* 915 * R3 here contains the value that will be returned to the caller 916 * of power7_nap. 917 * R12 contains SRR1 for CHECK_HMI_INTERRUPT. 918 */ 919.global pnv_wakeup_loss 920pnv_wakeup_loss: 921 ld r1,PACAR1(r13) 922BEGIN_FTR_SECTION 923 CHECK_HMI_INTERRUPT 924END_FTR_SECTION_IFSET(CPU_FTR_HVMODE) 925 REST_NVGPRS(r1) 926 REST_GPR(2, r1) 927 ld r4,PACAKMSR(r13) 928 ld r5,_LINK(r1) 929 ld r6,_CCR(r1) 930 addi r1,r1,INT_FRAME_SIZE 931 mtlr r5 932 mtcr r6 933 mtmsrd r4 934 blr 935 936/* 937 * R3 here contains the value that will be returned to the caller 938 * of power7_nap. 939 * R12 contains SRR1 for CHECK_HMI_INTERRUPT. 940 */ 941pnv_wakeup_noloss: 942 lbz r0,PACA_NAPSTATELOST(r13) 943 cmpwi r0,0 944 bne pnv_wakeup_loss 945 ld r1,PACAR1(r13) 946BEGIN_FTR_SECTION 947 CHECK_HMI_INTERRUPT 948END_FTR_SECTION_IFSET(CPU_FTR_HVMODE) 949 ld r4,PACAKMSR(r13) 950 ld r5,_NIP(r1) 951 ld r6,_CCR(r1) 952 addi r1,r1,INT_FRAME_SIZE 953 mtlr r5 954 mtcr r6 955 mtmsrd r4 956 blr 957