1/* SPDX-License-Identifier: GPL-2.0-only */ 2/* 3 * 4 * Copyright 2011 Paul Mackerras, IBM Corp. <paulus@au1.ibm.com> 5 * 6 * Derived from book3s_rmhandlers.S and other files, which are: 7 * 8 * Copyright SUSE Linux Products GmbH 2009 9 * 10 * Authors: Alexander Graf <agraf@suse.de> 11 */ 12 13#include <asm/ppc_asm.h> 14#include <asm/kvm_asm.h> 15#include <asm/reg.h> 16#include <asm/mmu.h> 17#include <asm/page.h> 18#include <asm/ptrace.h> 19#include <asm/hvcall.h> 20#include <asm/asm-offsets.h> 21#include <asm/exception-64s.h> 22#include <asm/kvm_book3s_asm.h> 23#include <asm/book3s/64/mmu-hash.h> 24#include <asm/export.h> 25#include <asm/tm.h> 26#include <asm/opal.h> 27#include <asm/xive-regs.h> 28#include <asm/thread_info.h> 29#include <asm/asm-compat.h> 30#include <asm/feature-fixups.h> 31#include <asm/cpuidle.h> 32 33/* Sign-extend HDEC if not on POWER9 */ 34#define EXTEND_HDEC(reg) \ 35BEGIN_FTR_SECTION; \ 36 extsw reg, reg; \ 37END_FTR_SECTION_IFCLR(CPU_FTR_ARCH_300) 38 39/* Values in HSTATE_NAPPING(r13) */ 40#define NAPPING_CEDE 1 41#define NAPPING_NOVCPU 2 42#define NAPPING_UNSPLIT 3 43 44/* Stack frame offsets for kvmppc_hv_entry */ 45#define SFS 208 46#define STACK_SLOT_TRAP (SFS-4) 47#define STACK_SLOT_SHORT_PATH (SFS-8) 48#define STACK_SLOT_TID (SFS-16) 49#define STACK_SLOT_PSSCR (SFS-24) 50#define STACK_SLOT_PID (SFS-32) 51#define STACK_SLOT_IAMR (SFS-40) 52#define STACK_SLOT_CIABR (SFS-48) 53#define STACK_SLOT_DAWR (SFS-56) 54#define STACK_SLOT_DAWRX (SFS-64) 55#define STACK_SLOT_HFSCR (SFS-72) 56#define STACK_SLOT_AMR (SFS-80) 57#define STACK_SLOT_UAMOR (SFS-88) 58/* the following is used by the P9 short path */ 59#define STACK_SLOT_NVGPRS (SFS-152) /* 18 gprs */ 60 61/* 62 * Call kvmppc_hv_entry in real mode. 63 * Must be called with interrupts hard-disabled. 64 * 65 * Input Registers: 66 * 67 * LR = return address to continue at after eventually re-enabling MMU 68 */ 69_GLOBAL_TOC(kvmppc_hv_entry_trampoline) 70 mflr r0 71 std r0, PPC_LR_STKOFF(r1) 72 stdu r1, -112(r1) 73 mfmsr r10 74 std r10, HSTATE_HOST_MSR(r13) 75 LOAD_REG_ADDR(r5, kvmppc_call_hv_entry) 76 li r0,MSR_RI 77 andc r0,r10,r0 78 li r6,MSR_IR | MSR_DR 79 andc r6,r10,r6 80 mtmsrd r0,1 /* clear RI in MSR */ 81 mtsrr0 r5 82 mtsrr1 r6 83 RFI_TO_KERNEL 84 85kvmppc_call_hv_entry: 86BEGIN_FTR_SECTION 87 /* On P9, do LPCR setting, if necessary */ 88 ld r3, HSTATE_SPLIT_MODE(r13) 89 cmpdi r3, 0 90 beq 46f 91 lwz r4, KVM_SPLIT_DO_SET(r3) 92 cmpwi r4, 0 93 beq 46f 94 bl kvmhv_p9_set_lpcr 95 nop 9646: 97END_FTR_SECTION_IFSET(CPU_FTR_ARCH_300) 98 99 ld r4, HSTATE_KVM_VCPU(r13) 100 bl kvmppc_hv_entry 101 102 /* Back from guest - restore host state and return to caller */ 103 104BEGIN_FTR_SECTION 105 /* Restore host DABR and DABRX */ 106 ld r5,HSTATE_DABR(r13) 107 li r6,7 108 mtspr SPRN_DABR,r5 109 mtspr SPRN_DABRX,r6 110END_FTR_SECTION_IFCLR(CPU_FTR_ARCH_207S) 111 112 /* Restore SPRG3 */ 113 ld r3,PACA_SPRG_VDSO(r13) 114 mtspr SPRN_SPRG_VDSO_WRITE,r3 115 116 /* Reload the host's PMU registers */ 117 bl kvmhv_load_host_pmu 118 119 /* 120 * Reload DEC. HDEC interrupts were disabled when 121 * we reloaded the host's LPCR value. 122 */ 123 ld r3, HSTATE_DECEXP(r13) 124 mftb r4 125 subf r4, r4, r3 126 mtspr SPRN_DEC, r4 127 128 /* hwthread_req may have got set by cede or no vcpu, so clear it */ 129 li r0, 0 130 stb r0, HSTATE_HWTHREAD_REQ(r13) 131 132 /* 133 * For external interrupts we need to call the Linux 134 * handler to process the interrupt. We do that by jumping 135 * to absolute address 0x500 for external interrupts. 136 * The [h]rfid at the end of the handler will return to 137 * the book3s_hv_interrupts.S code. For other interrupts 138 * we do the rfid to get back to the book3s_hv_interrupts.S 139 * code here. 140 */ 141 ld r8, 112+PPC_LR_STKOFF(r1) 142 addi r1, r1, 112 143 ld r7, HSTATE_HOST_MSR(r13) 144 145 /* Return the trap number on this thread as the return value */ 146 mr r3, r12 147 148 /* 149 * If we came back from the guest via a relocation-on interrupt, 150 * we will be in virtual mode at this point, which makes it a 151 * little easier to get back to the caller. 152 */ 153 mfmsr r0 154 andi. r0, r0, MSR_IR /* in real mode? */ 155 bne .Lvirt_return 156 157 /* RFI into the highmem handler */ 158 mfmsr r6 159 li r0, MSR_RI 160 andc r6, r6, r0 161 mtmsrd r6, 1 /* Clear RI in MSR */ 162 mtsrr0 r8 163 mtsrr1 r7 164 RFI_TO_KERNEL 165 166 /* Virtual-mode return */ 167.Lvirt_return: 168 mtlr r8 169 blr 170 171kvmppc_primary_no_guest: 172 /* We handle this much like a ceded vcpu */ 173 /* put the HDEC into the DEC, since HDEC interrupts don't wake us */ 174 /* HDEC may be larger than DEC for arch >= v3.00, but since the */ 175 /* HDEC value came from DEC in the first place, it will fit */ 176 mfspr r3, SPRN_HDEC 177 mtspr SPRN_DEC, r3 178 /* 179 * Make sure the primary has finished the MMU switch. 180 * We should never get here on a secondary thread, but 181 * check it for robustness' sake. 182 */ 183 ld r5, HSTATE_KVM_VCORE(r13) 18465: lbz r0, VCORE_IN_GUEST(r5) 185 cmpwi r0, 0 186 beq 65b 187 /* Set LPCR. */ 188 ld r8,VCORE_LPCR(r5) 189 mtspr SPRN_LPCR,r8 190 isync 191 /* set our bit in napping_threads */ 192 ld r5, HSTATE_KVM_VCORE(r13) 193 lbz r7, HSTATE_PTID(r13) 194 li r0, 1 195 sld r0, r0, r7 196 addi r6, r5, VCORE_NAPPING_THREADS 1971: lwarx r3, 0, r6 198 or r3, r3, r0 199 stwcx. r3, 0, r6 200 bne 1b 201 /* order napping_threads update vs testing entry_exit_map */ 202 isync 203 li r12, 0 204 lwz r7, VCORE_ENTRY_EXIT(r5) 205 cmpwi r7, 0x100 206 bge kvm_novcpu_exit /* another thread already exiting */ 207 li r3, NAPPING_NOVCPU 208 stb r3, HSTATE_NAPPING(r13) 209 210 li r3, 0 /* Don't wake on privileged (OS) doorbell */ 211 b kvm_do_nap 212 213/* 214 * kvm_novcpu_wakeup 215 * Entered from kvm_start_guest if kvm_hstate.napping is set 216 * to NAPPING_NOVCPU 217 * r2 = kernel TOC 218 * r13 = paca 219 */ 220kvm_novcpu_wakeup: 221 ld r1, HSTATE_HOST_R1(r13) 222 ld r5, HSTATE_KVM_VCORE(r13) 223 li r0, 0 224 stb r0, HSTATE_NAPPING(r13) 225 226 /* check the wake reason */ 227 bl kvmppc_check_wake_reason 228 229 /* 230 * Restore volatile registers since we could have called 231 * a C routine in kvmppc_check_wake_reason. 232 * r5 = VCORE 233 */ 234 ld r5, HSTATE_KVM_VCORE(r13) 235 236 /* see if any other thread is already exiting */ 237 lwz r0, VCORE_ENTRY_EXIT(r5) 238 cmpwi r0, 0x100 239 bge kvm_novcpu_exit 240 241 /* clear our bit in napping_threads */ 242 lbz r7, HSTATE_PTID(r13) 243 li r0, 1 244 sld r0, r0, r7 245 addi r6, r5, VCORE_NAPPING_THREADS 2464: lwarx r7, 0, r6 247 andc r7, r7, r0 248 stwcx. r7, 0, r6 249 bne 4b 250 251 /* See if the wake reason means we need to exit */ 252 cmpdi r3, 0 253 bge kvm_novcpu_exit 254 255 /* See if our timeslice has expired (HDEC is negative) */ 256 mfspr r0, SPRN_HDEC 257 EXTEND_HDEC(r0) 258 li r12, BOOK3S_INTERRUPT_HV_DECREMENTER 259 cmpdi r0, 0 260 blt kvm_novcpu_exit 261 262 /* Got an IPI but other vcpus aren't yet exiting, must be a latecomer */ 263 ld r4, HSTATE_KVM_VCPU(r13) 264 cmpdi r4, 0 265 beq kvmppc_primary_no_guest 266 267#ifdef CONFIG_KVM_BOOK3S_HV_EXIT_TIMING 268 addi r3, r4, VCPU_TB_RMENTRY 269 bl kvmhv_start_timing 270#endif 271 b kvmppc_got_guest 272 273kvm_novcpu_exit: 274#ifdef CONFIG_KVM_BOOK3S_HV_EXIT_TIMING 275 ld r4, HSTATE_KVM_VCPU(r13) 276 cmpdi r4, 0 277 beq 13f 278 addi r3, r4, VCPU_TB_RMEXIT 279 bl kvmhv_accumulate_time 280#endif 28113: mr r3, r12 282 stw r12, STACK_SLOT_TRAP(r1) 283 bl kvmhv_commence_exit 284 nop 285 b kvmhv_switch_to_host 286 287/* 288 * We come in here when wakened from Linux offline idle code. 289 * Relocation is off 290 * r3 contains the SRR1 wakeup value, SRR1 is trashed. 291 */ 292_GLOBAL(idle_kvm_start_guest) 293 ld r4,PACAEMERGSP(r13) 294 mfcr r5 295 mflr r0 296 std r1,0(r4) 297 std r5,8(r4) 298 std r0,16(r4) 299 subi r1,r4,STACK_FRAME_OVERHEAD 300 SAVE_NVGPRS(r1) 301 302 /* 303 * Could avoid this and pass it through in r3. For now, 304 * code expects it to be in SRR1. 305 */ 306 mtspr SPRN_SRR1,r3 307 308 li r0,0 309 stb r0,PACA_FTRACE_ENABLED(r13) 310 311 li r0,KVM_HWTHREAD_IN_KVM 312 stb r0,HSTATE_HWTHREAD_STATE(r13) 313 314 /* kvm cede / napping does not come through here */ 315 lbz r0,HSTATE_NAPPING(r13) 316 twnei r0,0 317 318 b 1f 319 320kvm_unsplit_wakeup: 321 li r0, 0 322 stb r0, HSTATE_NAPPING(r13) 323 3241: 325 326 /* 327 * We weren't napping due to cede, so this must be a secondary 328 * thread being woken up to run a guest, or being woken up due 329 * to a stray IPI. (Or due to some machine check or hypervisor 330 * maintenance interrupt while the core is in KVM.) 331 */ 332 333 /* Check the wake reason in SRR1 to see why we got here */ 334 bl kvmppc_check_wake_reason 335 /* 336 * kvmppc_check_wake_reason could invoke a C routine, but we 337 * have no volatile registers to restore when we return. 338 */ 339 340 cmpdi r3, 0 341 bge kvm_no_guest 342 343 /* get vcore pointer, NULL if we have nothing to run */ 344 ld r5,HSTATE_KVM_VCORE(r13) 345 cmpdi r5,0 346 /* if we have no vcore to run, go back to sleep */ 347 beq kvm_no_guest 348 349kvm_secondary_got_guest: 350 351 /* Set HSTATE_DSCR(r13) to something sensible */ 352 ld r6, PACA_DSCR_DEFAULT(r13) 353 std r6, HSTATE_DSCR(r13) 354 355 /* On thread 0 of a subcore, set HDEC to max */ 356 lbz r4, HSTATE_PTID(r13) 357 cmpwi r4, 0 358 bne 63f 359 LOAD_REG_ADDR(r6, decrementer_max) 360 ld r6, 0(r6) 361 mtspr SPRN_HDEC, r6 362 /* and set per-LPAR registers, if doing dynamic micro-threading */ 363 ld r6, HSTATE_SPLIT_MODE(r13) 364 cmpdi r6, 0 365 beq 63f 366BEGIN_FTR_SECTION 367 ld r0, KVM_SPLIT_RPR(r6) 368 mtspr SPRN_RPR, r0 369 ld r0, KVM_SPLIT_PMMAR(r6) 370 mtspr SPRN_PMMAR, r0 371 ld r0, KVM_SPLIT_LDBAR(r6) 372 mtspr SPRN_LDBAR, r0 373 isync 374FTR_SECTION_ELSE 375 /* On P9 we use the split_info for coordinating LPCR changes */ 376 lwz r4, KVM_SPLIT_DO_SET(r6) 377 cmpwi r4, 0 378 beq 1f 379 mr r3, r6 380 bl kvmhv_p9_set_lpcr 381 nop 3821: 383ALT_FTR_SECTION_END_IFCLR(CPU_FTR_ARCH_300) 38463: 385 /* Order load of vcpu after load of vcore */ 386 lwsync 387 ld r4, HSTATE_KVM_VCPU(r13) 388 bl kvmppc_hv_entry 389 390 /* Back from the guest, go back to nap */ 391 /* Clear our vcpu and vcore pointers so we don't come back in early */ 392 li r0, 0 393 std r0, HSTATE_KVM_VCPU(r13) 394 /* 395 * Once we clear HSTATE_KVM_VCORE(r13), the code in 396 * kvmppc_run_core() is going to assume that all our vcpu 397 * state is visible in memory. This lwsync makes sure 398 * that that is true. 399 */ 400 lwsync 401 std r0, HSTATE_KVM_VCORE(r13) 402 403 /* 404 * All secondaries exiting guest will fall through this path. 405 * Before proceeding, just check for HMI interrupt and 406 * invoke opal hmi handler. By now we are sure that the 407 * primary thread on this core/subcore has already made partition 408 * switch/TB resync and we are good to call opal hmi handler. 409 */ 410 cmpwi r12, BOOK3S_INTERRUPT_HMI 411 bne kvm_no_guest 412 413 li r3,0 /* NULL argument */ 414 bl hmi_exception_realmode 415/* 416 * At this point we have finished executing in the guest. 417 * We need to wait for hwthread_req to become zero, since 418 * we may not turn on the MMU while hwthread_req is non-zero. 419 * While waiting we also need to check if we get given a vcpu to run. 420 */ 421kvm_no_guest: 422 lbz r3, HSTATE_HWTHREAD_REQ(r13) 423 cmpwi r3, 0 424 bne 53f 425 HMT_MEDIUM 426 li r0, KVM_HWTHREAD_IN_KERNEL 427 stb r0, HSTATE_HWTHREAD_STATE(r13) 428 /* need to recheck hwthread_req after a barrier, to avoid race */ 429 sync 430 lbz r3, HSTATE_HWTHREAD_REQ(r13) 431 cmpwi r3, 0 432 bne 54f 433 434 /* 435 * Jump to idle_return_gpr_loss, which returns to the 436 * idle_kvm_start_guest caller. 437 */ 438 li r3, LPCR_PECE0 439 mfspr r4, SPRN_LPCR 440 rlwimi r4, r3, 0, LPCR_PECE0 | LPCR_PECE1 441 mtspr SPRN_LPCR, r4 442 /* set up r3 for return */ 443 mfspr r3,SPRN_SRR1 444 REST_NVGPRS(r1) 445 addi r1, r1, STACK_FRAME_OVERHEAD 446 ld r0, 16(r1) 447 ld r5, 8(r1) 448 ld r1, 0(r1) 449 mtlr r0 450 mtcr r5 451 blr 452 45353: HMT_LOW 454 ld r5, HSTATE_KVM_VCORE(r13) 455 cmpdi r5, 0 456 bne 60f 457 ld r3, HSTATE_SPLIT_MODE(r13) 458 cmpdi r3, 0 459 beq kvm_no_guest 460 lwz r0, KVM_SPLIT_DO_SET(r3) 461 cmpwi r0, 0 462 bne kvmhv_do_set 463 lwz r0, KVM_SPLIT_DO_RESTORE(r3) 464 cmpwi r0, 0 465 bne kvmhv_do_restore 466 lbz r0, KVM_SPLIT_DO_NAP(r3) 467 cmpwi r0, 0 468 beq kvm_no_guest 469 HMT_MEDIUM 470 b kvm_unsplit_nap 47160: HMT_MEDIUM 472 b kvm_secondary_got_guest 473 47454: li r0, KVM_HWTHREAD_IN_KVM 475 stb r0, HSTATE_HWTHREAD_STATE(r13) 476 b kvm_no_guest 477 478kvmhv_do_set: 479 /* Set LPCR, LPIDR etc. on P9 */ 480 HMT_MEDIUM 481 bl kvmhv_p9_set_lpcr 482 nop 483 b kvm_no_guest 484 485kvmhv_do_restore: 486 HMT_MEDIUM 487 bl kvmhv_p9_restore_lpcr 488 nop 489 b kvm_no_guest 490 491/* 492 * Here the primary thread is trying to return the core to 493 * whole-core mode, so we need to nap. 494 */ 495kvm_unsplit_nap: 496 /* 497 * When secondaries are napping in kvm_unsplit_nap() with 498 * hwthread_req = 1, HMI goes ignored even though subcores are 499 * already exited the guest. Hence HMI keeps waking up secondaries 500 * from nap in a loop and secondaries always go back to nap since 501 * no vcore is assigned to them. This makes impossible for primary 502 * thread to get hold of secondary threads resulting into a soft 503 * lockup in KVM path. 504 * 505 * Let us check if HMI is pending and handle it before we go to nap. 506 */ 507 cmpwi r12, BOOK3S_INTERRUPT_HMI 508 bne 55f 509 li r3, 0 /* NULL argument */ 510 bl hmi_exception_realmode 51155: 512 /* 513 * Ensure that secondary doesn't nap when it has 514 * its vcore pointer set. 515 */ 516 sync /* matches smp_mb() before setting split_info.do_nap */ 517 ld r0, HSTATE_KVM_VCORE(r13) 518 cmpdi r0, 0 519 bne kvm_no_guest 520 /* clear any pending message */ 521BEGIN_FTR_SECTION 522 lis r6, (PPC_DBELL_SERVER << (63-36))@h 523 PPC_MSGCLR(6) 524END_FTR_SECTION_IFSET(CPU_FTR_ARCH_207S) 525 /* Set kvm_split_mode.napped[tid] = 1 */ 526 ld r3, HSTATE_SPLIT_MODE(r13) 527 li r0, 1 528 lbz r4, HSTATE_TID(r13) 529 addi r4, r4, KVM_SPLIT_NAPPED 530 stbx r0, r3, r4 531 /* Check the do_nap flag again after setting napped[] */ 532 sync 533 lbz r0, KVM_SPLIT_DO_NAP(r3) 534 cmpwi r0, 0 535 beq 57f 536 li r3, NAPPING_UNSPLIT 537 stb r3, HSTATE_NAPPING(r13) 538 li r3, (LPCR_PECEDH | LPCR_PECE0) >> 4 539 mfspr r5, SPRN_LPCR 540 rlwimi r5, r3, 4, (LPCR_PECEDP | LPCR_PECEDH | LPCR_PECE0 | LPCR_PECE1) 541 b kvm_nap_sequence 542 54357: li r0, 0 544 stbx r0, r3, r4 545 b kvm_no_guest 546 547/****************************************************************************** 548 * * 549 * Entry code * 550 * * 551 *****************************************************************************/ 552 553.global kvmppc_hv_entry 554kvmppc_hv_entry: 555 556 /* Required state: 557 * 558 * R4 = vcpu pointer (or NULL) 559 * MSR = ~IR|DR 560 * R13 = PACA 561 * R1 = host R1 562 * R2 = TOC 563 * all other volatile GPRS = free 564 * Does not preserve non-volatile GPRs or CR fields 565 */ 566 mflr r0 567 std r0, PPC_LR_STKOFF(r1) 568 stdu r1, -SFS(r1) 569 570 /* Save R1 in the PACA */ 571 std r1, HSTATE_HOST_R1(r13) 572 573 li r6, KVM_GUEST_MODE_HOST_HV 574 stb r6, HSTATE_IN_GUEST(r13) 575 576#ifdef CONFIG_KVM_BOOK3S_HV_EXIT_TIMING 577 /* Store initial timestamp */ 578 cmpdi r4, 0 579 beq 1f 580 addi r3, r4, VCPU_TB_RMENTRY 581 bl kvmhv_start_timing 5821: 583#endif 584 585 ld r5, HSTATE_KVM_VCORE(r13) 586 ld r9, VCORE_KVM(r5) /* pointer to struct kvm */ 587 588 /* 589 * POWER7/POWER8 host -> guest partition switch code. 590 * We don't have to lock against concurrent tlbies, 591 * but we do have to coordinate across hardware threads. 592 */ 593 /* Set bit in entry map iff exit map is zero. */ 594 li r7, 1 595 lbz r6, HSTATE_PTID(r13) 596 sld r7, r7, r6 597 addi r8, r5, VCORE_ENTRY_EXIT 59821: lwarx r3, 0, r8 599 cmpwi r3, 0x100 /* any threads starting to exit? */ 600 bge secondary_too_late /* if so we're too late to the party */ 601 or r3, r3, r7 602 stwcx. r3, 0, r8 603 bne 21b 604 605 /* Primary thread switches to guest partition. */ 606 cmpwi r6,0 607 bne 10f 608 609 lwz r7,KVM_LPID(r9) 610BEGIN_FTR_SECTION 611 ld r6,KVM_SDR1(r9) 612 li r0,LPID_RSVD /* switch to reserved LPID */ 613 mtspr SPRN_LPID,r0 614 ptesync 615 mtspr SPRN_SDR1,r6 /* switch to partition page table */ 616END_FTR_SECTION_IFCLR(CPU_FTR_ARCH_300) 617 mtspr SPRN_LPID,r7 618 isync 619 620 /* See if we need to flush the TLB. */ 621 mr r3, r9 /* kvm pointer */ 622 lhz r4, PACAPACAINDEX(r13) /* physical cpu number */ 623 li r5, 0 /* nested vcpu pointer */ 624 bl kvmppc_check_need_tlb_flush 625 nop 626 ld r5, HSTATE_KVM_VCORE(r13) 627 628 /* Add timebase offset onto timebase */ 62922: ld r8,VCORE_TB_OFFSET(r5) 630 cmpdi r8,0 631 beq 37f 632 std r8, VCORE_TB_OFFSET_APPL(r5) 633 mftb r6 /* current host timebase */ 634 add r8,r8,r6 635 mtspr SPRN_TBU40,r8 /* update upper 40 bits */ 636 mftb r7 /* check if lower 24 bits overflowed */ 637 clrldi r6,r6,40 638 clrldi r7,r7,40 639 cmpld r7,r6 640 bge 37f 641 addis r8,r8,0x100 /* if so, increment upper 40 bits */ 642 mtspr SPRN_TBU40,r8 643 644 /* Load guest PCR value to select appropriate compat mode */ 64537: ld r7, VCORE_PCR(r5) 646 cmpdi r7, 0 647 beq 38f 648 mtspr SPRN_PCR, r7 64938: 650 651BEGIN_FTR_SECTION 652 /* DPDES and VTB are shared between threads */ 653 ld r8, VCORE_DPDES(r5) 654 ld r7, VCORE_VTB(r5) 655 mtspr SPRN_DPDES, r8 656 mtspr SPRN_VTB, r7 657END_FTR_SECTION_IFSET(CPU_FTR_ARCH_207S) 658 659 /* Mark the subcore state as inside guest */ 660 bl kvmppc_subcore_enter_guest 661 nop 662 ld r5, HSTATE_KVM_VCORE(r13) 663 ld r4, HSTATE_KVM_VCPU(r13) 664 li r0,1 665 stb r0,VCORE_IN_GUEST(r5) /* signal secondaries to continue */ 666 667 /* Do we have a guest vcpu to run? */ 66810: cmpdi r4, 0 669 beq kvmppc_primary_no_guest 670kvmppc_got_guest: 671 /* Increment yield count if they have a VPA */ 672 ld r3, VCPU_VPA(r4) 673 cmpdi r3, 0 674 beq 25f 675 li r6, LPPACA_YIELDCOUNT 676 LWZX_BE r5, r3, r6 677 addi r5, r5, 1 678 STWX_BE r5, r3, r6 679 li r6, 1 680 stb r6, VCPU_VPA_DIRTY(r4) 68125: 682 683 /* Save purr/spurr */ 684 mfspr r5,SPRN_PURR 685 mfspr r6,SPRN_SPURR 686 std r5,HSTATE_PURR(r13) 687 std r6,HSTATE_SPURR(r13) 688 ld r7,VCPU_PURR(r4) 689 ld r8,VCPU_SPURR(r4) 690 mtspr SPRN_PURR,r7 691 mtspr SPRN_SPURR,r8 692 693 /* Save host values of some registers */ 694BEGIN_FTR_SECTION 695 mfspr r5, SPRN_TIDR 696 mfspr r6, SPRN_PSSCR 697 mfspr r7, SPRN_PID 698 std r5, STACK_SLOT_TID(r1) 699 std r6, STACK_SLOT_PSSCR(r1) 700 std r7, STACK_SLOT_PID(r1) 701 mfspr r5, SPRN_HFSCR 702 std r5, STACK_SLOT_HFSCR(r1) 703END_FTR_SECTION_IFSET(CPU_FTR_ARCH_300) 704BEGIN_FTR_SECTION 705 mfspr r5, SPRN_CIABR 706 mfspr r6, SPRN_DAWR 707 mfspr r7, SPRN_DAWRX 708 mfspr r8, SPRN_IAMR 709 std r5, STACK_SLOT_CIABR(r1) 710 std r6, STACK_SLOT_DAWR(r1) 711 std r7, STACK_SLOT_DAWRX(r1) 712 std r8, STACK_SLOT_IAMR(r1) 713END_FTR_SECTION_IFSET(CPU_FTR_ARCH_207S) 714 715 mfspr r5, SPRN_AMR 716 std r5, STACK_SLOT_AMR(r1) 717 mfspr r6, SPRN_UAMOR 718 std r6, STACK_SLOT_UAMOR(r1) 719 720BEGIN_FTR_SECTION 721 /* Set partition DABR */ 722 /* Do this before re-enabling PMU to avoid P7 DABR corruption bug */ 723 lwz r5,VCPU_DABRX(r4) 724 ld r6,VCPU_DABR(r4) 725 mtspr SPRN_DABRX,r5 726 mtspr SPRN_DABR,r6 727 isync 728END_FTR_SECTION_IFCLR(CPU_FTR_ARCH_207S) 729 730#ifdef CONFIG_PPC_TRANSACTIONAL_MEM 731/* 732 * Branch around the call if both CPU_FTR_TM and 733 * CPU_FTR_P9_TM_HV_ASSIST are off. 734 */ 735BEGIN_FTR_SECTION 736 b 91f 737END_FTR_SECTION(CPU_FTR_TM | CPU_FTR_P9_TM_HV_ASSIST, 0) 738 /* 739 * NOTE THAT THIS TRASHES ALL NON-VOLATILE REGISTERS (but not CR) 740 */ 741 mr r3, r4 742 ld r4, VCPU_MSR(r3) 743 li r5, 0 /* don't preserve non-vol regs */ 744 bl kvmppc_restore_tm_hv 745 nop 746 ld r4, HSTATE_KVM_VCPU(r13) 74791: 748#endif 749 750 /* Load guest PMU registers; r4 = vcpu pointer here */ 751 mr r3, r4 752 bl kvmhv_load_guest_pmu 753 754 /* Load up FP, VMX and VSX registers */ 755 ld r4, HSTATE_KVM_VCPU(r13) 756 bl kvmppc_load_fp 757 758 ld r14, VCPU_GPR(R14)(r4) 759 ld r15, VCPU_GPR(R15)(r4) 760 ld r16, VCPU_GPR(R16)(r4) 761 ld r17, VCPU_GPR(R17)(r4) 762 ld r18, VCPU_GPR(R18)(r4) 763 ld r19, VCPU_GPR(R19)(r4) 764 ld r20, VCPU_GPR(R20)(r4) 765 ld r21, VCPU_GPR(R21)(r4) 766 ld r22, VCPU_GPR(R22)(r4) 767 ld r23, VCPU_GPR(R23)(r4) 768 ld r24, VCPU_GPR(R24)(r4) 769 ld r25, VCPU_GPR(R25)(r4) 770 ld r26, VCPU_GPR(R26)(r4) 771 ld r27, VCPU_GPR(R27)(r4) 772 ld r28, VCPU_GPR(R28)(r4) 773 ld r29, VCPU_GPR(R29)(r4) 774 ld r30, VCPU_GPR(R30)(r4) 775 ld r31, VCPU_GPR(R31)(r4) 776 777 /* Switch DSCR to guest value */ 778 ld r5, VCPU_DSCR(r4) 779 mtspr SPRN_DSCR, r5 780 781BEGIN_FTR_SECTION 782 /* Skip next section on POWER7 */ 783 b 8f 784END_FTR_SECTION_IFCLR(CPU_FTR_ARCH_207S) 785 /* Load up POWER8-specific registers */ 786 ld r5, VCPU_IAMR(r4) 787 lwz r6, VCPU_PSPB(r4) 788 ld r7, VCPU_FSCR(r4) 789 mtspr SPRN_IAMR, r5 790 mtspr SPRN_PSPB, r6 791 mtspr SPRN_FSCR, r7 792 /* 793 * Handle broken DAWR case by not writing it. This means we 794 * can still store the DAWR register for migration. 795 */ 796 LOAD_REG_ADDR(r5, dawr_force_enable) 797 lbz r5, 0(r5) 798 cmpdi r5, 0 799 beq 1f 800 ld r5, VCPU_DAWR(r4) 801 ld r6, VCPU_DAWRX(r4) 802 mtspr SPRN_DAWR, r5 803 mtspr SPRN_DAWRX, r6 8041: 805 ld r7, VCPU_CIABR(r4) 806 ld r8, VCPU_TAR(r4) 807 mtspr SPRN_CIABR, r7 808 mtspr SPRN_TAR, r8 809 ld r5, VCPU_IC(r4) 810 ld r8, VCPU_EBBHR(r4) 811 mtspr SPRN_IC, r5 812 mtspr SPRN_EBBHR, r8 813 ld r5, VCPU_EBBRR(r4) 814 ld r6, VCPU_BESCR(r4) 815 lwz r7, VCPU_GUEST_PID(r4) 816 ld r8, VCPU_WORT(r4) 817 mtspr SPRN_EBBRR, r5 818 mtspr SPRN_BESCR, r6 819 mtspr SPRN_PID, r7 820 mtspr SPRN_WORT, r8 821BEGIN_FTR_SECTION 822 /* POWER8-only registers */ 823 ld r5, VCPU_TCSCR(r4) 824 ld r6, VCPU_ACOP(r4) 825 ld r7, VCPU_CSIGR(r4) 826 ld r8, VCPU_TACR(r4) 827 mtspr SPRN_TCSCR, r5 828 mtspr SPRN_ACOP, r6 829 mtspr SPRN_CSIGR, r7 830 mtspr SPRN_TACR, r8 831 nop 832FTR_SECTION_ELSE 833 /* POWER9-only registers */ 834 ld r5, VCPU_TID(r4) 835 ld r6, VCPU_PSSCR(r4) 836 lbz r8, HSTATE_FAKE_SUSPEND(r13) 837 oris r6, r6, PSSCR_EC@h /* This makes stop trap to HV */ 838 rldimi r6, r8, PSSCR_FAKE_SUSPEND_LG, 63 - PSSCR_FAKE_SUSPEND_LG 839 ld r7, VCPU_HFSCR(r4) 840 mtspr SPRN_TIDR, r5 841 mtspr SPRN_PSSCR, r6 842 mtspr SPRN_HFSCR, r7 843ALT_FTR_SECTION_END_IFCLR(CPU_FTR_ARCH_300) 8448: 845 846 ld r5, VCPU_SPRG0(r4) 847 ld r6, VCPU_SPRG1(r4) 848 ld r7, VCPU_SPRG2(r4) 849 ld r8, VCPU_SPRG3(r4) 850 mtspr SPRN_SPRG0, r5 851 mtspr SPRN_SPRG1, r6 852 mtspr SPRN_SPRG2, r7 853 mtspr SPRN_SPRG3, r8 854 855 /* Load up DAR and DSISR */ 856 ld r5, VCPU_DAR(r4) 857 lwz r6, VCPU_DSISR(r4) 858 mtspr SPRN_DAR, r5 859 mtspr SPRN_DSISR, r6 860 861 /* Restore AMR and UAMOR, set AMOR to all 1s */ 862 ld r5,VCPU_AMR(r4) 863 ld r6,VCPU_UAMOR(r4) 864 li r7,-1 865 mtspr SPRN_AMR,r5 866 mtspr SPRN_UAMOR,r6 867 mtspr SPRN_AMOR,r7 868 869 /* Restore state of CTRL run bit; assume 1 on entry */ 870 lwz r5,VCPU_CTRL(r4) 871 andi. r5,r5,1 872 bne 4f 873 mfspr r6,SPRN_CTRLF 874 clrrdi r6,r6,1 875 mtspr SPRN_CTRLT,r6 8764: 877 /* Secondary threads wait for primary to have done partition switch */ 878 ld r5, HSTATE_KVM_VCORE(r13) 879 lbz r6, HSTATE_PTID(r13) 880 cmpwi r6, 0 881 beq 21f 882 lbz r0, VCORE_IN_GUEST(r5) 883 cmpwi r0, 0 884 bne 21f 885 HMT_LOW 88620: lwz r3, VCORE_ENTRY_EXIT(r5) 887 cmpwi r3, 0x100 888 bge no_switch_exit 889 lbz r0, VCORE_IN_GUEST(r5) 890 cmpwi r0, 0 891 beq 20b 892 HMT_MEDIUM 89321: 894 /* Set LPCR. */ 895 ld r8,VCORE_LPCR(r5) 896 mtspr SPRN_LPCR,r8 897 isync 898 899 /* 900 * Set the decrementer to the guest decrementer. 901 */ 902 ld r8,VCPU_DEC_EXPIRES(r4) 903 /* r8 is a host timebase value here, convert to guest TB */ 904 ld r5,HSTATE_KVM_VCORE(r13) 905 ld r6,VCORE_TB_OFFSET_APPL(r5) 906 add r8,r8,r6 907 mftb r7 908 subf r3,r7,r8 909 mtspr SPRN_DEC,r3 910 911 /* Check if HDEC expires soon */ 912 mfspr r3, SPRN_HDEC 913 EXTEND_HDEC(r3) 914 cmpdi r3, 512 /* 1 microsecond */ 915 blt hdec_soon 916 917 /* For hash guest, clear out and reload the SLB */ 918 ld r6, VCPU_KVM(r4) 919 lbz r0, KVM_RADIX(r6) 920 cmpwi r0, 0 921 bne 9f 922 li r6, 0 923 slbmte r6, r6 924 slbia 925 ptesync 926 927 /* Load up guest SLB entries (N.B. slb_max will be 0 for radix) */ 928 lwz r5,VCPU_SLB_MAX(r4) 929 cmpwi r5,0 930 beq 9f 931 mtctr r5 932 addi r6,r4,VCPU_SLB 9331: ld r8,VCPU_SLB_E(r6) 934 ld r9,VCPU_SLB_V(r6) 935 slbmte r9,r8 936 addi r6,r6,VCPU_SLB_SIZE 937 bdnz 1b 9389: 939 940#ifdef CONFIG_KVM_XICS 941 /* We are entering the guest on that thread, push VCPU to XIVE */ 942 ld r11, VCPU_XIVE_SAVED_STATE(r4) 943 li r9, TM_QW1_OS 944 lwz r8, VCPU_XIVE_CAM_WORD(r4) 945 cmpwi r8, 0 946 beq no_xive 947 li r7, TM_QW1_OS + TM_WORD2 948 mfmsr r0 949 andi. r0, r0, MSR_DR /* in real mode? */ 950 beq 2f 951 ld r10, HSTATE_XIVE_TIMA_VIRT(r13) 952 cmpldi cr1, r10, 0 953 beq cr1, no_xive 954 eieio 955 stdx r11,r9,r10 956 stwx r8,r7,r10 957 b 3f 9582: ld r10, HSTATE_XIVE_TIMA_PHYS(r13) 959 cmpldi cr1, r10, 0 960 beq cr1, no_xive 961 eieio 962 stdcix r11,r9,r10 963 stwcix r8,r7,r10 9643: li r9, 1 965 stb r9, VCPU_XIVE_PUSHED(r4) 966 eieio 967 968 /* 969 * We clear the irq_pending flag. There is a small chance of a 970 * race vs. the escalation interrupt happening on another 971 * processor setting it again, but the only consequence is to 972 * cause a spurrious wakeup on the next H_CEDE which is not an 973 * issue. 974 */ 975 li r0,0 976 stb r0, VCPU_IRQ_PENDING(r4) 977 978 /* 979 * In single escalation mode, if the escalation interrupt is 980 * on, we mask it. 981 */ 982 lbz r0, VCPU_XIVE_ESC_ON(r4) 983 cmpwi cr1, r0,0 984 beq cr1, 1f 985 li r9, XIVE_ESB_SET_PQ_01 986 beq 4f /* in real mode? */ 987 ld r10, VCPU_XIVE_ESC_VADDR(r4) 988 ldx r0, r10, r9 989 b 5f 9904: ld r10, VCPU_XIVE_ESC_RADDR(r4) 991 ldcix r0, r10, r9 9925: sync 993 994 /* We have a possible subtle race here: The escalation interrupt might 995 * have fired and be on its way to the host queue while we mask it, 996 * and if we unmask it early enough (re-cede right away), there is 997 * a theorical possibility that it fires again, thus landing in the 998 * target queue more than once which is a big no-no. 999 * 1000 * Fortunately, solving this is rather easy. If the above load setting 1001 * PQ to 01 returns a previous value where P is set, then we know the 1002 * escalation interrupt is somewhere on its way to the host. In that 1003 * case we simply don't clear the xive_esc_on flag below. It will be 1004 * eventually cleared by the handler for the escalation interrupt. 1005 * 1006 * Then, when doing a cede, we check that flag again before re-enabling 1007 * the escalation interrupt, and if set, we abort the cede. 1008 */ 1009 andi. r0, r0, XIVE_ESB_VAL_P 1010 bne- 1f 1011 1012 /* Now P is 0, we can clear the flag */ 1013 li r0, 0 1014 stb r0, VCPU_XIVE_ESC_ON(r4) 10151: 1016no_xive: 1017#endif /* CONFIG_KVM_XICS */ 1018 1019 li r0, 0 1020 stw r0, STACK_SLOT_SHORT_PATH(r1) 1021 1022deliver_guest_interrupt: /* r4 = vcpu, r13 = paca */ 1023 /* Check if we can deliver an external or decrementer interrupt now */ 1024 ld r0, VCPU_PENDING_EXC(r4) 1025BEGIN_FTR_SECTION 1026 /* On POWER9, also check for emulated doorbell interrupt */ 1027 lbz r3, VCPU_DBELL_REQ(r4) 1028 or r0, r0, r3 1029END_FTR_SECTION_IFSET(CPU_FTR_ARCH_300) 1030 cmpdi r0, 0 1031 beq 71f 1032 mr r3, r4 1033 bl kvmppc_guest_entry_inject_int 1034 ld r4, HSTATE_KVM_VCPU(r13) 103571: 1036 ld r6, VCPU_SRR0(r4) 1037 ld r7, VCPU_SRR1(r4) 1038 mtspr SPRN_SRR0, r6 1039 mtspr SPRN_SRR1, r7 1040 1041fast_guest_entry_c: 1042 ld r10, VCPU_PC(r4) 1043 ld r11, VCPU_MSR(r4) 1044 /* r11 = vcpu->arch.msr & ~MSR_HV */ 1045 rldicl r11, r11, 63 - MSR_HV_LG, 1 1046 rotldi r11, r11, 1 + MSR_HV_LG 1047 ori r11, r11, MSR_ME 1048 1049 ld r6, VCPU_CTR(r4) 1050 ld r7, VCPU_XER(r4) 1051 mtctr r6 1052 mtxer r7 1053 1054/* 1055 * Required state: 1056 * R4 = vcpu 1057 * R10: value for HSRR0 1058 * R11: value for HSRR1 1059 * R13 = PACA 1060 */ 1061fast_guest_return: 1062 li r0,0 1063 stb r0,VCPU_CEDED(r4) /* cancel cede */ 1064 mtspr SPRN_HSRR0,r10 1065 mtspr SPRN_HSRR1,r11 1066 1067 /* Activate guest mode, so faults get handled by KVM */ 1068 li r9, KVM_GUEST_MODE_GUEST_HV 1069 stb r9, HSTATE_IN_GUEST(r13) 1070 1071#ifdef CONFIG_KVM_BOOK3S_HV_EXIT_TIMING 1072 /* Accumulate timing */ 1073 addi r3, r4, VCPU_TB_GUEST 1074 bl kvmhv_accumulate_time 1075#endif 1076 1077 /* Enter guest */ 1078 1079BEGIN_FTR_SECTION 1080 ld r5, VCPU_CFAR(r4) 1081 mtspr SPRN_CFAR, r5 1082END_FTR_SECTION_IFSET(CPU_FTR_CFAR) 1083BEGIN_FTR_SECTION 1084 ld r0, VCPU_PPR(r4) 1085END_FTR_SECTION_IFSET(CPU_FTR_HAS_PPR) 1086 1087 ld r5, VCPU_LR(r4) 1088 ld r6, VCPU_CR(r4) 1089 mtlr r5 1090 mtcr r6 1091 1092 ld r1, VCPU_GPR(R1)(r4) 1093 ld r2, VCPU_GPR(R2)(r4) 1094 ld r3, VCPU_GPR(R3)(r4) 1095 ld r5, VCPU_GPR(R5)(r4) 1096 ld r6, VCPU_GPR(R6)(r4) 1097 ld r7, VCPU_GPR(R7)(r4) 1098 ld r8, VCPU_GPR(R8)(r4) 1099 ld r9, VCPU_GPR(R9)(r4) 1100 ld r10, VCPU_GPR(R10)(r4) 1101 ld r11, VCPU_GPR(R11)(r4) 1102 ld r12, VCPU_GPR(R12)(r4) 1103 ld r13, VCPU_GPR(R13)(r4) 1104 1105BEGIN_FTR_SECTION 1106 mtspr SPRN_PPR, r0 1107END_FTR_SECTION_IFSET(CPU_FTR_HAS_PPR) 1108 1109/* Move canary into DSISR to check for later */ 1110BEGIN_FTR_SECTION 1111 li r0, 0x7fff 1112 mtspr SPRN_HDSISR, r0 1113END_FTR_SECTION_IFSET(CPU_FTR_ARCH_300) 1114 1115 ld r0, VCPU_GPR(R0)(r4) 1116 ld r4, VCPU_GPR(R4)(r4) 1117 HRFI_TO_GUEST 1118 b . 1119 1120/* 1121 * Enter the guest on a P9 or later system where we have exactly 1122 * one vcpu per vcore and we don't need to go to real mode 1123 * (which implies that host and guest are both using radix MMU mode). 1124 * r3 = vcpu pointer 1125 * Most SPRs and all the VSRs have been loaded already. 1126 */ 1127_GLOBAL(__kvmhv_vcpu_entry_p9) 1128EXPORT_SYMBOL_GPL(__kvmhv_vcpu_entry_p9) 1129 mflr r0 1130 std r0, PPC_LR_STKOFF(r1) 1131 stdu r1, -SFS(r1) 1132 1133 li r0, 1 1134 stw r0, STACK_SLOT_SHORT_PATH(r1) 1135 1136 std r3, HSTATE_KVM_VCPU(r13) 1137 mfcr r4 1138 stw r4, SFS+8(r1) 1139 1140 std r1, HSTATE_HOST_R1(r13) 1141 1142 reg = 14 1143 .rept 18 1144 std reg, STACK_SLOT_NVGPRS + ((reg - 14) * 8)(r1) 1145 reg = reg + 1 1146 .endr 1147 1148 reg = 14 1149 .rept 18 1150 ld reg, __VCPU_GPR(reg)(r3) 1151 reg = reg + 1 1152 .endr 1153 1154 mfmsr r10 1155 std r10, HSTATE_HOST_MSR(r13) 1156 1157 mr r4, r3 1158 b fast_guest_entry_c 1159guest_exit_short_path: 1160 1161 li r0, KVM_GUEST_MODE_NONE 1162 stb r0, HSTATE_IN_GUEST(r13) 1163 1164 reg = 14 1165 .rept 18 1166 std reg, __VCPU_GPR(reg)(r9) 1167 reg = reg + 1 1168 .endr 1169 1170 reg = 14 1171 .rept 18 1172 ld reg, STACK_SLOT_NVGPRS + ((reg - 14) * 8)(r1) 1173 reg = reg + 1 1174 .endr 1175 1176 lwz r4, SFS+8(r1) 1177 mtcr r4 1178 1179 mr r3, r12 /* trap number */ 1180 1181 addi r1, r1, SFS 1182 ld r0, PPC_LR_STKOFF(r1) 1183 mtlr r0 1184 1185 /* If we are in real mode, do a rfid to get back to the caller */ 1186 mfmsr r4 1187 andi. r5, r4, MSR_IR 1188 bnelr 1189 rldicl r5, r4, 64 - MSR_TS_S_LG, 62 /* extract TS field */ 1190 mtspr SPRN_SRR0, r0 1191 ld r10, HSTATE_HOST_MSR(r13) 1192 rldimi r10, r5, MSR_TS_S_LG, 63 - MSR_TS_T_LG 1193 mtspr SPRN_SRR1, r10 1194 RFI_TO_KERNEL 1195 b . 1196 1197secondary_too_late: 1198 li r12, 0 1199 stw r12, STACK_SLOT_TRAP(r1) 1200 cmpdi r4, 0 1201 beq 11f 1202 stw r12, VCPU_TRAP(r4) 1203#ifdef CONFIG_KVM_BOOK3S_HV_EXIT_TIMING 1204 addi r3, r4, VCPU_TB_RMEXIT 1205 bl kvmhv_accumulate_time 1206#endif 120711: b kvmhv_switch_to_host 1208 1209no_switch_exit: 1210 HMT_MEDIUM 1211 li r12, 0 1212 b 12f 1213hdec_soon: 1214 li r12, BOOK3S_INTERRUPT_HV_DECREMENTER 121512: stw r12, VCPU_TRAP(r4) 1216 mr r9, r4 1217#ifdef CONFIG_KVM_BOOK3S_HV_EXIT_TIMING 1218 addi r3, r4, VCPU_TB_RMEXIT 1219 bl kvmhv_accumulate_time 1220#endif 1221 b guest_bypass 1222 1223/****************************************************************************** 1224 * * 1225 * Exit code * 1226 * * 1227 *****************************************************************************/ 1228 1229/* 1230 * We come here from the first-level interrupt handlers. 1231 */ 1232 .globl kvmppc_interrupt_hv 1233kvmppc_interrupt_hv: 1234 /* 1235 * Register contents: 1236 * R12 = (guest CR << 32) | interrupt vector 1237 * R13 = PACA 1238 * guest R12 saved in shadow VCPU SCRATCH0 1239 * guest CTR saved in shadow VCPU SCRATCH1 if RELOCATABLE 1240 * guest R13 saved in SPRN_SCRATCH0 1241 */ 1242 std r9, HSTATE_SCRATCH2(r13) 1243 lbz r9, HSTATE_IN_GUEST(r13) 1244 cmpwi r9, KVM_GUEST_MODE_HOST_HV 1245 beq kvmppc_bad_host_intr 1246#ifdef CONFIG_KVM_BOOK3S_PR_POSSIBLE 1247 cmpwi r9, KVM_GUEST_MODE_GUEST 1248 ld r9, HSTATE_SCRATCH2(r13) 1249 beq kvmppc_interrupt_pr 1250#endif 1251 /* We're now back in the host but in guest MMU context */ 1252 li r9, KVM_GUEST_MODE_HOST_HV 1253 stb r9, HSTATE_IN_GUEST(r13) 1254 1255 ld r9, HSTATE_KVM_VCPU(r13) 1256 1257 /* Save registers */ 1258 1259 std r0, VCPU_GPR(R0)(r9) 1260 std r1, VCPU_GPR(R1)(r9) 1261 std r2, VCPU_GPR(R2)(r9) 1262 std r3, VCPU_GPR(R3)(r9) 1263 std r4, VCPU_GPR(R4)(r9) 1264 std r5, VCPU_GPR(R5)(r9) 1265 std r6, VCPU_GPR(R6)(r9) 1266 std r7, VCPU_GPR(R7)(r9) 1267 std r8, VCPU_GPR(R8)(r9) 1268 ld r0, HSTATE_SCRATCH2(r13) 1269 std r0, VCPU_GPR(R9)(r9) 1270 std r10, VCPU_GPR(R10)(r9) 1271 std r11, VCPU_GPR(R11)(r9) 1272 ld r3, HSTATE_SCRATCH0(r13) 1273 std r3, VCPU_GPR(R12)(r9) 1274 /* CR is in the high half of r12 */ 1275 srdi r4, r12, 32 1276 std r4, VCPU_CR(r9) 1277BEGIN_FTR_SECTION 1278 ld r3, HSTATE_CFAR(r13) 1279 std r3, VCPU_CFAR(r9) 1280END_FTR_SECTION_IFSET(CPU_FTR_CFAR) 1281BEGIN_FTR_SECTION 1282 ld r4, HSTATE_PPR(r13) 1283 std r4, VCPU_PPR(r9) 1284END_FTR_SECTION_IFSET(CPU_FTR_HAS_PPR) 1285 1286 /* Restore R1/R2 so we can handle faults */ 1287 ld r1, HSTATE_HOST_R1(r13) 1288 ld r2, PACATOC(r13) 1289 1290 mfspr r10, SPRN_SRR0 1291 mfspr r11, SPRN_SRR1 1292 std r10, VCPU_SRR0(r9) 1293 std r11, VCPU_SRR1(r9) 1294 /* trap is in the low half of r12, clear CR from the high half */ 1295 clrldi r12, r12, 32 1296 andi. r0, r12, 2 /* need to read HSRR0/1? */ 1297 beq 1f 1298 mfspr r10, SPRN_HSRR0 1299 mfspr r11, SPRN_HSRR1 1300 clrrdi r12, r12, 2 13011: std r10, VCPU_PC(r9) 1302 std r11, VCPU_MSR(r9) 1303 1304 GET_SCRATCH0(r3) 1305 mflr r4 1306 std r3, VCPU_GPR(R13)(r9) 1307 std r4, VCPU_LR(r9) 1308 1309 stw r12,VCPU_TRAP(r9) 1310 1311 /* 1312 * Now that we have saved away SRR0/1 and HSRR0/1, 1313 * interrupts are recoverable in principle, so set MSR_RI. 1314 * This becomes important for relocation-on interrupts from 1315 * the guest, which we can get in radix mode on POWER9. 1316 */ 1317 li r0, MSR_RI 1318 mtmsrd r0, 1 1319 1320#ifdef CONFIG_KVM_BOOK3S_HV_EXIT_TIMING 1321 addi r3, r9, VCPU_TB_RMINTR 1322 mr r4, r9 1323 bl kvmhv_accumulate_time 1324 ld r5, VCPU_GPR(R5)(r9) 1325 ld r6, VCPU_GPR(R6)(r9) 1326 ld r7, VCPU_GPR(R7)(r9) 1327 ld r8, VCPU_GPR(R8)(r9) 1328#endif 1329 1330 /* Save HEIR (HV emulation assist reg) in emul_inst 1331 if this is an HEI (HV emulation interrupt, e40) */ 1332 li r3,KVM_INST_FETCH_FAILED 1333 stw r3,VCPU_LAST_INST(r9) 1334 cmpwi r12,BOOK3S_INTERRUPT_H_EMUL_ASSIST 1335 bne 11f 1336 mfspr r3,SPRN_HEIR 133711: stw r3,VCPU_HEIR(r9) 1338 1339 /* these are volatile across C function calls */ 1340#ifdef CONFIG_RELOCATABLE 1341 ld r3, HSTATE_SCRATCH1(r13) 1342 mtctr r3 1343#else 1344 mfctr r3 1345#endif 1346 mfxer r4 1347 std r3, VCPU_CTR(r9) 1348 std r4, VCPU_XER(r9) 1349 1350 /* Save more register state */ 1351 mfdar r3 1352 mfdsisr r4 1353 std r3, VCPU_DAR(r9) 1354 stw r4, VCPU_DSISR(r9) 1355 1356 /* If this is a page table miss then see if it's theirs or ours */ 1357 cmpwi r12, BOOK3S_INTERRUPT_H_DATA_STORAGE 1358 beq kvmppc_hdsi 1359 std r3, VCPU_FAULT_DAR(r9) 1360 stw r4, VCPU_FAULT_DSISR(r9) 1361 cmpwi r12, BOOK3S_INTERRUPT_H_INST_STORAGE 1362 beq kvmppc_hisi 1363 1364#ifdef CONFIG_PPC_TRANSACTIONAL_MEM 1365 /* For softpatch interrupt, go off and do TM instruction emulation */ 1366 cmpwi r12, BOOK3S_INTERRUPT_HV_SOFTPATCH 1367 beq kvmppc_tm_emul 1368#endif 1369 1370 /* See if this is a leftover HDEC interrupt */ 1371 cmpwi r12,BOOK3S_INTERRUPT_HV_DECREMENTER 1372 bne 2f 1373 mfspr r3,SPRN_HDEC 1374 EXTEND_HDEC(r3) 1375 cmpdi r3,0 1376 mr r4,r9 1377 bge fast_guest_return 13782: 1379 /* See if this is an hcall we can handle in real mode */ 1380 cmpwi r12,BOOK3S_INTERRUPT_SYSCALL 1381 beq hcall_try_real_mode 1382 1383 /* Hypervisor doorbell - exit only if host IPI flag set */ 1384 cmpwi r12, BOOK3S_INTERRUPT_H_DOORBELL 1385 bne 3f 1386BEGIN_FTR_SECTION 1387 PPC_MSGSYNC 1388 lwsync 1389 /* always exit if we're running a nested guest */ 1390 ld r0, VCPU_NESTED(r9) 1391 cmpdi r0, 0 1392 bne guest_exit_cont 1393END_FTR_SECTION_IFSET(CPU_FTR_ARCH_300) 1394 lbz r0, HSTATE_HOST_IPI(r13) 1395 cmpwi r0, 0 1396 beq maybe_reenter_guest 1397 b guest_exit_cont 13983: 1399 /* If it's a hypervisor facility unavailable interrupt, save HFSCR */ 1400 cmpwi r12, BOOK3S_INTERRUPT_H_FAC_UNAVAIL 1401 bne 14f 1402 mfspr r3, SPRN_HFSCR 1403 std r3, VCPU_HFSCR(r9) 1404 b guest_exit_cont 140514: 1406 /* External interrupt ? */ 1407 cmpwi r12, BOOK3S_INTERRUPT_EXTERNAL 1408 beq kvmppc_guest_external 1409 /* See if it is a machine check */ 1410 cmpwi r12, BOOK3S_INTERRUPT_MACHINE_CHECK 1411 beq machine_check_realmode 1412 /* Or a hypervisor maintenance interrupt */ 1413 cmpwi r12, BOOK3S_INTERRUPT_HMI 1414 beq hmi_realmode 1415 1416guest_exit_cont: /* r9 = vcpu, r12 = trap, r13 = paca */ 1417 1418#ifdef CONFIG_KVM_BOOK3S_HV_EXIT_TIMING 1419 addi r3, r9, VCPU_TB_RMEXIT 1420 mr r4, r9 1421 bl kvmhv_accumulate_time 1422#endif 1423#ifdef CONFIG_KVM_XICS 1424 /* We are exiting, pull the VP from the XIVE */ 1425 lbz r0, VCPU_XIVE_PUSHED(r9) 1426 cmpwi cr0, r0, 0 1427 beq 1f 1428 li r7, TM_SPC_PULL_OS_CTX 1429 li r6, TM_QW1_OS 1430 mfmsr r0 1431 andi. r0, r0, MSR_DR /* in real mode? */ 1432 beq 2f 1433 ld r10, HSTATE_XIVE_TIMA_VIRT(r13) 1434 cmpldi cr0, r10, 0 1435 beq 1f 1436 /* First load to pull the context, we ignore the value */ 1437 eieio 1438 lwzx r11, r7, r10 1439 /* Second load to recover the context state (Words 0 and 1) */ 1440 ldx r11, r6, r10 1441 b 3f 14422: ld r10, HSTATE_XIVE_TIMA_PHYS(r13) 1443 cmpldi cr0, r10, 0 1444 beq 1f 1445 /* First load to pull the context, we ignore the value */ 1446 eieio 1447 lwzcix r11, r7, r10 1448 /* Second load to recover the context state (Words 0 and 1) */ 1449 ldcix r11, r6, r10 14503: std r11, VCPU_XIVE_SAVED_STATE(r9) 1451 /* Fixup some of the state for the next load */ 1452 li r10, 0 1453 li r0, 0xff 1454 stb r10, VCPU_XIVE_PUSHED(r9) 1455 stb r10, (VCPU_XIVE_SAVED_STATE+3)(r9) 1456 stb r0, (VCPU_XIVE_SAVED_STATE+4)(r9) 1457 eieio 14581: 1459#endif /* CONFIG_KVM_XICS */ 1460 1461 /* If we came in through the P9 short path, go back out to C now */ 1462 lwz r0, STACK_SLOT_SHORT_PATH(r1) 1463 cmpwi r0, 0 1464 bne guest_exit_short_path 1465 1466 /* For hash guest, read the guest SLB and save it away */ 1467 ld r5, VCPU_KVM(r9) 1468 lbz r0, KVM_RADIX(r5) 1469 li r5, 0 1470 cmpwi r0, 0 1471 bne 3f /* for radix, save 0 entries */ 1472 lwz r0,VCPU_SLB_NR(r9) /* number of entries in SLB */ 1473 mtctr r0 1474 li r6,0 1475 addi r7,r9,VCPU_SLB 14761: slbmfee r8,r6 1477 andis. r0,r8,SLB_ESID_V@h 1478 beq 2f 1479 add r8,r8,r6 /* put index in */ 1480 slbmfev r3,r6 1481 std r8,VCPU_SLB_E(r7) 1482 std r3,VCPU_SLB_V(r7) 1483 addi r7,r7,VCPU_SLB_SIZE 1484 addi r5,r5,1 14852: addi r6,r6,1 1486 bdnz 1b 1487 /* Finally clear out the SLB */ 1488 li r0,0 1489 slbmte r0,r0 1490 slbia 1491 ptesync 14923: stw r5,VCPU_SLB_MAX(r9) 1493 1494 /* load host SLB entries */ 1495BEGIN_MMU_FTR_SECTION 1496 b 0f 1497END_MMU_FTR_SECTION_IFSET(MMU_FTR_TYPE_RADIX) 1498 ld r8,PACA_SLBSHADOWPTR(r13) 1499 1500 .rept SLB_NUM_BOLTED 1501 li r3, SLBSHADOW_SAVEAREA 1502 LDX_BE r5, r8, r3 1503 addi r3, r3, 8 1504 LDX_BE r6, r8, r3 1505 andis. r7,r5,SLB_ESID_V@h 1506 beq 1f 1507 slbmte r6,r5 15081: addi r8,r8,16 1509 .endr 15100: 1511 1512guest_bypass: 1513 stw r12, STACK_SLOT_TRAP(r1) 1514 1515 /* Save DEC */ 1516 /* Do this before kvmhv_commence_exit so we know TB is guest TB */ 1517 ld r3, HSTATE_KVM_VCORE(r13) 1518 mfspr r5,SPRN_DEC 1519 mftb r6 1520 /* On P9, if the guest has large decr enabled, don't sign extend */ 1521BEGIN_FTR_SECTION 1522 ld r4, VCORE_LPCR(r3) 1523 andis. r4, r4, LPCR_LD@h 1524 bne 16f 1525END_FTR_SECTION_IFSET(CPU_FTR_ARCH_300) 1526 extsw r5,r5 152716: add r5,r5,r6 1528 /* r5 is a guest timebase value here, convert to host TB */ 1529 ld r4,VCORE_TB_OFFSET_APPL(r3) 1530 subf r5,r4,r5 1531 std r5,VCPU_DEC_EXPIRES(r9) 1532 1533 /* Increment exit count, poke other threads to exit */ 1534 mr r3, r12 1535 bl kvmhv_commence_exit 1536 nop 1537 ld r9, HSTATE_KVM_VCPU(r13) 1538 1539 /* Stop others sending VCPU interrupts to this physical CPU */ 1540 li r0, -1 1541 stw r0, VCPU_CPU(r9) 1542 stw r0, VCPU_THREAD_CPU(r9) 1543 1544 /* Save guest CTRL register, set runlatch to 1 */ 1545 mfspr r6,SPRN_CTRLF 1546 stw r6,VCPU_CTRL(r9) 1547 andi. r0,r6,1 1548 bne 4f 1549 ori r6,r6,1 1550 mtspr SPRN_CTRLT,r6 15514: 1552 /* 1553 * Save the guest PURR/SPURR 1554 */ 1555 mfspr r5,SPRN_PURR 1556 mfspr r6,SPRN_SPURR 1557 ld r7,VCPU_PURR(r9) 1558 ld r8,VCPU_SPURR(r9) 1559 std r5,VCPU_PURR(r9) 1560 std r6,VCPU_SPURR(r9) 1561 subf r5,r7,r5 1562 subf r6,r8,r6 1563 1564 /* 1565 * Restore host PURR/SPURR and add guest times 1566 * so that the time in the guest gets accounted. 1567 */ 1568 ld r3,HSTATE_PURR(r13) 1569 ld r4,HSTATE_SPURR(r13) 1570 add r3,r3,r5 1571 add r4,r4,r6 1572 mtspr SPRN_PURR,r3 1573 mtspr SPRN_SPURR,r4 1574 1575BEGIN_FTR_SECTION 1576 b 8f 1577END_FTR_SECTION_IFCLR(CPU_FTR_ARCH_207S) 1578 /* Save POWER8-specific registers */ 1579 mfspr r5, SPRN_IAMR 1580 mfspr r6, SPRN_PSPB 1581 mfspr r7, SPRN_FSCR 1582 std r5, VCPU_IAMR(r9) 1583 stw r6, VCPU_PSPB(r9) 1584 std r7, VCPU_FSCR(r9) 1585 mfspr r5, SPRN_IC 1586 mfspr r7, SPRN_TAR 1587 std r5, VCPU_IC(r9) 1588 std r7, VCPU_TAR(r9) 1589 mfspr r8, SPRN_EBBHR 1590 std r8, VCPU_EBBHR(r9) 1591 mfspr r5, SPRN_EBBRR 1592 mfspr r6, SPRN_BESCR 1593 mfspr r7, SPRN_PID 1594 mfspr r8, SPRN_WORT 1595 std r5, VCPU_EBBRR(r9) 1596 std r6, VCPU_BESCR(r9) 1597 stw r7, VCPU_GUEST_PID(r9) 1598 std r8, VCPU_WORT(r9) 1599BEGIN_FTR_SECTION 1600 mfspr r5, SPRN_TCSCR 1601 mfspr r6, SPRN_ACOP 1602 mfspr r7, SPRN_CSIGR 1603 mfspr r8, SPRN_TACR 1604 std r5, VCPU_TCSCR(r9) 1605 std r6, VCPU_ACOP(r9) 1606 std r7, VCPU_CSIGR(r9) 1607 std r8, VCPU_TACR(r9) 1608FTR_SECTION_ELSE 1609 mfspr r5, SPRN_TIDR 1610 mfspr r6, SPRN_PSSCR 1611 std r5, VCPU_TID(r9) 1612 rldicl r6, r6, 4, 50 /* r6 &= PSSCR_GUEST_VIS */ 1613 rotldi r6, r6, 60 1614 std r6, VCPU_PSSCR(r9) 1615 /* Restore host HFSCR value */ 1616 ld r7, STACK_SLOT_HFSCR(r1) 1617 mtspr SPRN_HFSCR, r7 1618ALT_FTR_SECTION_END_IFCLR(CPU_FTR_ARCH_300) 1619 /* 1620 * Restore various registers to 0, where non-zero values 1621 * set by the guest could disrupt the host. 1622 */ 1623 li r0, 0 1624 mtspr SPRN_PSPB, r0 1625 mtspr SPRN_WORT, r0 1626BEGIN_FTR_SECTION 1627 mtspr SPRN_TCSCR, r0 1628 /* Set MMCRS to 1<<31 to freeze and disable the SPMC counters */ 1629 li r0, 1 1630 sldi r0, r0, 31 1631 mtspr SPRN_MMCRS, r0 1632END_FTR_SECTION_IFCLR(CPU_FTR_ARCH_300) 1633 1634 /* Save and restore AMR, IAMR and UAMOR before turning on the MMU */ 1635 ld r8, STACK_SLOT_IAMR(r1) 1636 mtspr SPRN_IAMR, r8 1637 16388: /* Power7 jumps back in here */ 1639 mfspr r5,SPRN_AMR 1640 mfspr r6,SPRN_UAMOR 1641 std r5,VCPU_AMR(r9) 1642 std r6,VCPU_UAMOR(r9) 1643 ld r5,STACK_SLOT_AMR(r1) 1644 ld r6,STACK_SLOT_UAMOR(r1) 1645 mtspr SPRN_AMR, r5 1646 mtspr SPRN_UAMOR, r6 1647 1648 /* Switch DSCR back to host value */ 1649 mfspr r8, SPRN_DSCR 1650 ld r7, HSTATE_DSCR(r13) 1651 std r8, VCPU_DSCR(r9) 1652 mtspr SPRN_DSCR, r7 1653 1654 /* Save non-volatile GPRs */ 1655 std r14, VCPU_GPR(R14)(r9) 1656 std r15, VCPU_GPR(R15)(r9) 1657 std r16, VCPU_GPR(R16)(r9) 1658 std r17, VCPU_GPR(R17)(r9) 1659 std r18, VCPU_GPR(R18)(r9) 1660 std r19, VCPU_GPR(R19)(r9) 1661 std r20, VCPU_GPR(R20)(r9) 1662 std r21, VCPU_GPR(R21)(r9) 1663 std r22, VCPU_GPR(R22)(r9) 1664 std r23, VCPU_GPR(R23)(r9) 1665 std r24, VCPU_GPR(R24)(r9) 1666 std r25, VCPU_GPR(R25)(r9) 1667 std r26, VCPU_GPR(R26)(r9) 1668 std r27, VCPU_GPR(R27)(r9) 1669 std r28, VCPU_GPR(R28)(r9) 1670 std r29, VCPU_GPR(R29)(r9) 1671 std r30, VCPU_GPR(R30)(r9) 1672 std r31, VCPU_GPR(R31)(r9) 1673 1674 /* Save SPRGs */ 1675 mfspr r3, SPRN_SPRG0 1676 mfspr r4, SPRN_SPRG1 1677 mfspr r5, SPRN_SPRG2 1678 mfspr r6, SPRN_SPRG3 1679 std r3, VCPU_SPRG0(r9) 1680 std r4, VCPU_SPRG1(r9) 1681 std r5, VCPU_SPRG2(r9) 1682 std r6, VCPU_SPRG3(r9) 1683 1684 /* save FP state */ 1685 mr r3, r9 1686 bl kvmppc_save_fp 1687 1688#ifdef CONFIG_PPC_TRANSACTIONAL_MEM 1689/* 1690 * Branch around the call if both CPU_FTR_TM and 1691 * CPU_FTR_P9_TM_HV_ASSIST are off. 1692 */ 1693BEGIN_FTR_SECTION 1694 b 91f 1695END_FTR_SECTION(CPU_FTR_TM | CPU_FTR_P9_TM_HV_ASSIST, 0) 1696 /* 1697 * NOTE THAT THIS TRASHES ALL NON-VOLATILE REGISTERS (but not CR) 1698 */ 1699 mr r3, r9 1700 ld r4, VCPU_MSR(r3) 1701 li r5, 0 /* don't preserve non-vol regs */ 1702 bl kvmppc_save_tm_hv 1703 nop 1704 ld r9, HSTATE_KVM_VCPU(r13) 170591: 1706#endif 1707 1708 /* Increment yield count if they have a VPA */ 1709 ld r8, VCPU_VPA(r9) /* do they have a VPA? */ 1710 cmpdi r8, 0 1711 beq 25f 1712 li r4, LPPACA_YIELDCOUNT 1713 LWZX_BE r3, r8, r4 1714 addi r3, r3, 1 1715 STWX_BE r3, r8, r4 1716 li r3, 1 1717 stb r3, VCPU_VPA_DIRTY(r9) 171825: 1719 /* Save PMU registers if requested */ 1720 /* r8 and cr0.eq are live here */ 1721 mr r3, r9 1722 li r4, 1 1723 beq 21f /* if no VPA, save PMU stuff anyway */ 1724 lbz r4, LPPACA_PMCINUSE(r8) 172521: bl kvmhv_save_guest_pmu 1726 ld r9, HSTATE_KVM_VCPU(r13) 1727 1728 /* Restore host values of some registers */ 1729BEGIN_FTR_SECTION 1730 ld r5, STACK_SLOT_CIABR(r1) 1731 ld r6, STACK_SLOT_DAWR(r1) 1732 ld r7, STACK_SLOT_DAWRX(r1) 1733 mtspr SPRN_CIABR, r5 1734 /* 1735 * If the DAWR doesn't work, it's ok to write these here as 1736 * this value should always be zero 1737 */ 1738 mtspr SPRN_DAWR, r6 1739 mtspr SPRN_DAWRX, r7 1740END_FTR_SECTION_IFSET(CPU_FTR_ARCH_207S) 1741BEGIN_FTR_SECTION 1742 ld r5, STACK_SLOT_TID(r1) 1743 ld r6, STACK_SLOT_PSSCR(r1) 1744 ld r7, STACK_SLOT_PID(r1) 1745 mtspr SPRN_TIDR, r5 1746 mtspr SPRN_PSSCR, r6 1747 mtspr SPRN_PID, r7 1748END_FTR_SECTION_IFSET(CPU_FTR_ARCH_300) 1749 1750#ifdef CONFIG_PPC_RADIX_MMU 1751 /* 1752 * Are we running hash or radix ? 1753 */ 1754 ld r5, VCPU_KVM(r9) 1755 lbz r0, KVM_RADIX(r5) 1756 cmpwi cr2, r0, 0 1757 beq cr2, 2f 1758 1759 /* 1760 * Radix: do eieio; tlbsync; ptesync sequence in case we 1761 * interrupted the guest between a tlbie and a ptesync. 1762 */ 1763 eieio 1764 tlbsync 1765 ptesync 1766 1767 /* Radix: Handle the case where the guest used an illegal PID */ 1768 LOAD_REG_ADDR(r4, mmu_base_pid) 1769 lwz r3, VCPU_GUEST_PID(r9) 1770 lwz r5, 0(r4) 1771 cmpw cr0,r3,r5 1772 blt 2f 1773 1774 /* 1775 * Illegal PID, the HW might have prefetched and cached in the TLB 1776 * some translations for the LPID 0 / guest PID combination which 1777 * Linux doesn't know about, so we need to flush that PID out of 1778 * the TLB. First we need to set LPIDR to 0 so tlbiel applies to 1779 * the right context. 1780 */ 1781 li r0,0 1782 mtspr SPRN_LPID,r0 1783 isync 1784 1785 /* Then do a congruence class local flush */ 1786 ld r6,VCPU_KVM(r9) 1787 lwz r0,KVM_TLB_SETS(r6) 1788 mtctr r0 1789 li r7,0x400 /* IS field = 0b01 */ 1790 ptesync 1791 sldi r0,r3,32 /* RS has PID */ 17921: PPC_TLBIEL(7,0,2,1,1) /* RIC=2, PRS=1, R=1 */ 1793 addi r7,r7,0x1000 1794 bdnz 1b 1795 ptesync 1796 17972: 1798#endif /* CONFIG_PPC_RADIX_MMU */ 1799 1800 /* 1801 * POWER7/POWER8 guest -> host partition switch code. 1802 * We don't have to lock against tlbies but we do 1803 * have to coordinate the hardware threads. 1804 * Here STACK_SLOT_TRAP(r1) contains the trap number. 1805 */ 1806kvmhv_switch_to_host: 1807 /* Secondary threads wait for primary to do partition switch */ 1808 ld r5,HSTATE_KVM_VCORE(r13) 1809 ld r4,VCORE_KVM(r5) /* pointer to struct kvm */ 1810 lbz r3,HSTATE_PTID(r13) 1811 cmpwi r3,0 1812 beq 15f 1813 HMT_LOW 181413: lbz r3,VCORE_IN_GUEST(r5) 1815 cmpwi r3,0 1816 bne 13b 1817 HMT_MEDIUM 1818 b 16f 1819 1820 /* Primary thread waits for all the secondaries to exit guest */ 182115: lwz r3,VCORE_ENTRY_EXIT(r5) 1822 rlwinm r0,r3,32-8,0xff 1823 clrldi r3,r3,56 1824 cmpw r3,r0 1825 bne 15b 1826 isync 1827 1828 /* Did we actually switch to the guest at all? */ 1829 lbz r6, VCORE_IN_GUEST(r5) 1830 cmpwi r6, 0 1831 beq 19f 1832 1833 /* Primary thread switches back to host partition */ 1834 lwz r7,KVM_HOST_LPID(r4) 1835BEGIN_FTR_SECTION 1836 ld r6,KVM_HOST_SDR1(r4) 1837 li r8,LPID_RSVD /* switch to reserved LPID */ 1838 mtspr SPRN_LPID,r8 1839 ptesync 1840 mtspr SPRN_SDR1,r6 /* switch to host page table */ 1841END_FTR_SECTION_IFCLR(CPU_FTR_ARCH_300) 1842 mtspr SPRN_LPID,r7 1843 isync 1844 1845BEGIN_FTR_SECTION 1846 /* DPDES and VTB are shared between threads */ 1847 mfspr r7, SPRN_DPDES 1848 mfspr r8, SPRN_VTB 1849 std r7, VCORE_DPDES(r5) 1850 std r8, VCORE_VTB(r5) 1851 /* clear DPDES so we don't get guest doorbells in the host */ 1852 li r8, 0 1853 mtspr SPRN_DPDES, r8 1854END_FTR_SECTION_IFSET(CPU_FTR_ARCH_207S) 1855 1856 /* Subtract timebase offset from timebase */ 1857 ld r8, VCORE_TB_OFFSET_APPL(r5) 1858 cmpdi r8,0 1859 beq 17f 1860 li r0, 0 1861 std r0, VCORE_TB_OFFSET_APPL(r5) 1862 mftb r6 /* current guest timebase */ 1863 subf r8,r8,r6 1864 mtspr SPRN_TBU40,r8 /* update upper 40 bits */ 1865 mftb r7 /* check if lower 24 bits overflowed */ 1866 clrldi r6,r6,40 1867 clrldi r7,r7,40 1868 cmpld r7,r6 1869 bge 17f 1870 addis r8,r8,0x100 /* if so, increment upper 40 bits */ 1871 mtspr SPRN_TBU40,r8 1872 187317: 1874 /* 1875 * If this is an HMI, we called kvmppc_realmode_hmi_handler 1876 * above, which may or may not have already called 1877 * kvmppc_subcore_exit_guest. Fortunately, all that 1878 * kvmppc_subcore_exit_guest does is clear a flag, so calling 1879 * it again here is benign even if kvmppc_realmode_hmi_handler 1880 * has already called it. 1881 */ 1882 bl kvmppc_subcore_exit_guest 1883 nop 188430: ld r5,HSTATE_KVM_VCORE(r13) 1885 ld r4,VCORE_KVM(r5) /* pointer to struct kvm */ 1886 1887 /* Reset PCR */ 1888 ld r0, VCORE_PCR(r5) 1889 cmpdi r0, 0 1890 beq 18f 1891 li r0, 0 1892 mtspr SPRN_PCR, r0 189318: 1894 /* Signal secondary CPUs to continue */ 1895 stb r0,VCORE_IN_GUEST(r5) 189619: lis r8,0x7fff /* MAX_INT@h */ 1897 mtspr SPRN_HDEC,r8 1898 189916: 1900BEGIN_FTR_SECTION 1901 /* On POWER9 with HPT-on-radix we need to wait for all other threads */ 1902 ld r3, HSTATE_SPLIT_MODE(r13) 1903 cmpdi r3, 0 1904 beq 47f 1905 lwz r8, KVM_SPLIT_DO_RESTORE(r3) 1906 cmpwi r8, 0 1907 beq 47f 1908 bl kvmhv_p9_restore_lpcr 1909 nop 1910 b 48f 191147: 1912END_FTR_SECTION_IFSET(CPU_FTR_ARCH_300) 1913 ld r8,KVM_HOST_LPCR(r4) 1914 mtspr SPRN_LPCR,r8 1915 isync 191648: 1917#ifdef CONFIG_KVM_BOOK3S_HV_EXIT_TIMING 1918 /* Finish timing, if we have a vcpu */ 1919 ld r4, HSTATE_KVM_VCPU(r13) 1920 cmpdi r4, 0 1921 li r3, 0 1922 beq 2f 1923 bl kvmhv_accumulate_time 19242: 1925#endif 1926 /* Unset guest mode */ 1927 li r0, KVM_GUEST_MODE_NONE 1928 stb r0, HSTATE_IN_GUEST(r13) 1929 1930 lwz r12, STACK_SLOT_TRAP(r1) /* return trap # in r12 */ 1931 ld r0, SFS+PPC_LR_STKOFF(r1) 1932 addi r1, r1, SFS 1933 mtlr r0 1934 blr 1935 1936kvmppc_guest_external: 1937 /* External interrupt, first check for host_ipi. If this is 1938 * set, we know the host wants us out so let's do it now 1939 */ 1940 bl kvmppc_read_intr 1941 1942 /* 1943 * Restore the active volatile registers after returning from 1944 * a C function. 1945 */ 1946 ld r9, HSTATE_KVM_VCPU(r13) 1947 li r12, BOOK3S_INTERRUPT_EXTERNAL 1948 1949 /* 1950 * kvmppc_read_intr return codes: 1951 * 1952 * Exit to host (r3 > 0) 1953 * 1 An interrupt is pending that needs to be handled by the host 1954 * Exit guest and return to host by branching to guest_exit_cont 1955 * 1956 * 2 Passthrough that needs completion in the host 1957 * Exit guest and return to host by branching to guest_exit_cont 1958 * However, we also set r12 to BOOK3S_INTERRUPT_HV_RM_HARD 1959 * to indicate to the host to complete handling the interrupt 1960 * 1961 * Before returning to guest, we check if any CPU is heading out 1962 * to the host and if so, we head out also. If no CPUs are heading 1963 * check return values <= 0. 1964 * 1965 * Return to guest (r3 <= 0) 1966 * 0 No external interrupt is pending 1967 * -1 A guest wakeup IPI (which has now been cleared) 1968 * In either case, we return to guest to deliver any pending 1969 * guest interrupts. 1970 * 1971 * -2 A PCI passthrough external interrupt was handled 1972 * (interrupt was delivered directly to guest) 1973 * Return to guest to deliver any pending guest interrupts. 1974 */ 1975 1976 cmpdi r3, 1 1977 ble 1f 1978 1979 /* Return code = 2 */ 1980 li r12, BOOK3S_INTERRUPT_HV_RM_HARD 1981 stw r12, VCPU_TRAP(r9) 1982 b guest_exit_cont 1983 19841: /* Return code <= 1 */ 1985 cmpdi r3, 0 1986 bgt guest_exit_cont 1987 1988 /* Return code <= 0 */ 1989maybe_reenter_guest: 1990 ld r5, HSTATE_KVM_VCORE(r13) 1991 lwz r0, VCORE_ENTRY_EXIT(r5) 1992 cmpwi r0, 0x100 1993 mr r4, r9 1994 blt deliver_guest_interrupt 1995 b guest_exit_cont 1996 1997#ifdef CONFIG_PPC_TRANSACTIONAL_MEM 1998/* 1999 * Softpatch interrupt for transactional memory emulation cases 2000 * on POWER9 DD2.2. This is early in the guest exit path - we 2001 * haven't saved registers or done a treclaim yet. 2002 */ 2003kvmppc_tm_emul: 2004 /* Save instruction image in HEIR */ 2005 mfspr r3, SPRN_HEIR 2006 stw r3, VCPU_HEIR(r9) 2007 2008 /* 2009 * The cases we want to handle here are those where the guest 2010 * is in real suspend mode and is trying to transition to 2011 * transactional mode. 2012 */ 2013 lbz r0, HSTATE_FAKE_SUSPEND(r13) 2014 cmpwi r0, 0 /* keep exiting guest if in fake suspend */ 2015 bne guest_exit_cont 2016 rldicl r3, r11, 64 - MSR_TS_S_LG, 62 2017 cmpwi r3, 1 /* or if not in suspend state */ 2018 bne guest_exit_cont 2019 2020 /* Call C code to do the emulation */ 2021 mr r3, r9 2022 bl kvmhv_p9_tm_emulation_early 2023 nop 2024 ld r9, HSTATE_KVM_VCPU(r13) 2025 li r12, BOOK3S_INTERRUPT_HV_SOFTPATCH 2026 cmpwi r3, 0 2027 beq guest_exit_cont /* continue exiting if not handled */ 2028 ld r10, VCPU_PC(r9) 2029 ld r11, VCPU_MSR(r9) 2030 b fast_interrupt_c_return /* go back to guest if handled */ 2031#endif /* CONFIG_PPC_TRANSACTIONAL_MEM */ 2032 2033/* 2034 * Check whether an HDSI is an HPTE not found fault or something else. 2035 * If it is an HPTE not found fault that is due to the guest accessing 2036 * a page that they have mapped but which we have paged out, then 2037 * we continue on with the guest exit path. In all other cases, 2038 * reflect the HDSI to the guest as a DSI. 2039 */ 2040kvmppc_hdsi: 2041 ld r3, VCPU_KVM(r9) 2042 lbz r0, KVM_RADIX(r3) 2043 mfspr r4, SPRN_HDAR 2044 mfspr r6, SPRN_HDSISR 2045BEGIN_FTR_SECTION 2046 /* Look for DSISR canary. If we find it, retry instruction */ 2047 cmpdi r6, 0x7fff 2048 beq 6f 2049END_FTR_SECTION_IFSET(CPU_FTR_ARCH_300) 2050 cmpwi r0, 0 2051 bne .Lradix_hdsi /* on radix, just save DAR/DSISR/ASDR */ 2052 /* HPTE not found fault or protection fault? */ 2053 andis. r0, r6, (DSISR_NOHPTE | DSISR_PROTFAULT)@h 2054 beq 1f /* if not, send it to the guest */ 2055 andi. r0, r11, MSR_DR /* data relocation enabled? */ 2056 beq 3f 2057BEGIN_FTR_SECTION 2058 mfspr r5, SPRN_ASDR /* on POWER9, use ASDR to get VSID */ 2059 b 4f 2060END_FTR_SECTION_IFSET(CPU_FTR_ARCH_300) 2061 clrrdi r0, r4, 28 2062 PPC_SLBFEE_DOT(R5, R0) /* if so, look up SLB */ 2063 li r0, BOOK3S_INTERRUPT_DATA_SEGMENT 2064 bne 7f /* if no SLB entry found */ 20654: std r4, VCPU_FAULT_DAR(r9) 2066 stw r6, VCPU_FAULT_DSISR(r9) 2067 2068 /* Search the hash table. */ 2069 mr r3, r9 /* vcpu pointer */ 2070 li r7, 1 /* data fault */ 2071 bl kvmppc_hpte_hv_fault 2072 ld r9, HSTATE_KVM_VCPU(r13) 2073 ld r10, VCPU_PC(r9) 2074 ld r11, VCPU_MSR(r9) 2075 li r12, BOOK3S_INTERRUPT_H_DATA_STORAGE 2076 cmpdi r3, 0 /* retry the instruction */ 2077 beq 6f 2078 cmpdi r3, -1 /* handle in kernel mode */ 2079 beq guest_exit_cont 2080 cmpdi r3, -2 /* MMIO emulation; need instr word */ 2081 beq 2f 2082 2083 /* Synthesize a DSI (or DSegI) for the guest */ 2084 ld r4, VCPU_FAULT_DAR(r9) 2085 mr r6, r3 20861: li r0, BOOK3S_INTERRUPT_DATA_STORAGE 2087 mtspr SPRN_DSISR, r6 20887: mtspr SPRN_DAR, r4 2089 mtspr SPRN_SRR0, r10 2090 mtspr SPRN_SRR1, r11 2091 mr r10, r0 2092 bl kvmppc_msr_interrupt 2093fast_interrupt_c_return: 20946: ld r7, VCPU_CTR(r9) 2095 ld r8, VCPU_XER(r9) 2096 mtctr r7 2097 mtxer r8 2098 mr r4, r9 2099 b fast_guest_return 2100 21013: ld r5, VCPU_KVM(r9) /* not relocated, use VRMA */ 2102 ld r5, KVM_VRMA_SLB_V(r5) 2103 b 4b 2104 2105 /* If this is for emulated MMIO, load the instruction word */ 21062: li r8, KVM_INST_FETCH_FAILED /* In case lwz faults */ 2107 2108 /* Set guest mode to 'jump over instruction' so if lwz faults 2109 * we'll just continue at the next IP. */ 2110 li r0, KVM_GUEST_MODE_SKIP 2111 stb r0, HSTATE_IN_GUEST(r13) 2112 2113 /* Do the access with MSR:DR enabled */ 2114 mfmsr r3 2115 ori r4, r3, MSR_DR /* Enable paging for data */ 2116 mtmsrd r4 2117 lwz r8, 0(r10) 2118 mtmsrd r3 2119 2120 /* Store the result */ 2121 stw r8, VCPU_LAST_INST(r9) 2122 2123 /* Unset guest mode. */ 2124 li r0, KVM_GUEST_MODE_HOST_HV 2125 stb r0, HSTATE_IN_GUEST(r13) 2126 b guest_exit_cont 2127 2128.Lradix_hdsi: 2129 std r4, VCPU_FAULT_DAR(r9) 2130 stw r6, VCPU_FAULT_DSISR(r9) 2131.Lradix_hisi: 2132 mfspr r5, SPRN_ASDR 2133 std r5, VCPU_FAULT_GPA(r9) 2134 b guest_exit_cont 2135 2136/* 2137 * Similarly for an HISI, reflect it to the guest as an ISI unless 2138 * it is an HPTE not found fault for a page that we have paged out. 2139 */ 2140kvmppc_hisi: 2141 ld r3, VCPU_KVM(r9) 2142 lbz r0, KVM_RADIX(r3) 2143 cmpwi r0, 0 2144 bne .Lradix_hisi /* for radix, just save ASDR */ 2145 andis. r0, r11, SRR1_ISI_NOPT@h 2146 beq 1f 2147 andi. r0, r11, MSR_IR /* instruction relocation enabled? */ 2148 beq 3f 2149BEGIN_FTR_SECTION 2150 mfspr r5, SPRN_ASDR /* on POWER9, use ASDR to get VSID */ 2151 b 4f 2152END_FTR_SECTION_IFSET(CPU_FTR_ARCH_300) 2153 clrrdi r0, r10, 28 2154 PPC_SLBFEE_DOT(R5, R0) /* if so, look up SLB */ 2155 li r0, BOOK3S_INTERRUPT_INST_SEGMENT 2156 bne 7f /* if no SLB entry found */ 21574: 2158 /* Search the hash table. */ 2159 mr r3, r9 /* vcpu pointer */ 2160 mr r4, r10 2161 mr r6, r11 2162 li r7, 0 /* instruction fault */ 2163 bl kvmppc_hpte_hv_fault 2164 ld r9, HSTATE_KVM_VCPU(r13) 2165 ld r10, VCPU_PC(r9) 2166 ld r11, VCPU_MSR(r9) 2167 li r12, BOOK3S_INTERRUPT_H_INST_STORAGE 2168 cmpdi r3, 0 /* retry the instruction */ 2169 beq fast_interrupt_c_return 2170 cmpdi r3, -1 /* handle in kernel mode */ 2171 beq guest_exit_cont 2172 2173 /* Synthesize an ISI (or ISegI) for the guest */ 2174 mr r11, r3 21751: li r0, BOOK3S_INTERRUPT_INST_STORAGE 21767: mtspr SPRN_SRR0, r10 2177 mtspr SPRN_SRR1, r11 2178 mr r10, r0 2179 bl kvmppc_msr_interrupt 2180 b fast_interrupt_c_return 2181 21823: ld r6, VCPU_KVM(r9) /* not relocated, use VRMA */ 2183 ld r5, KVM_VRMA_SLB_V(r6) 2184 b 4b 2185 2186/* 2187 * Try to handle an hcall in real mode. 2188 * Returns to the guest if we handle it, or continues on up to 2189 * the kernel if we can't (i.e. if we don't have a handler for 2190 * it, or if the handler returns H_TOO_HARD). 2191 * 2192 * r5 - r8 contain hcall args, 2193 * r9 = vcpu, r10 = pc, r11 = msr, r12 = trap, r13 = paca 2194 */ 2195hcall_try_real_mode: 2196 ld r3,VCPU_GPR(R3)(r9) 2197 andi. r0,r11,MSR_PR 2198 /* sc 1 from userspace - reflect to guest syscall */ 2199 bne sc_1_fast_return 2200 /* sc 1 from nested guest - give it to L1 to handle */ 2201 ld r0, VCPU_NESTED(r9) 2202 cmpdi r0, 0 2203 bne guest_exit_cont 2204 clrrdi r3,r3,2 2205 cmpldi r3,hcall_real_table_end - hcall_real_table 2206 bge guest_exit_cont 2207 /* See if this hcall is enabled for in-kernel handling */ 2208 ld r4, VCPU_KVM(r9) 2209 srdi r0, r3, 8 /* r0 = (r3 / 4) >> 6 */ 2210 sldi r0, r0, 3 /* index into kvm->arch.enabled_hcalls[] */ 2211 add r4, r4, r0 2212 ld r0, KVM_ENABLED_HCALLS(r4) 2213 rlwinm r4, r3, 32-2, 0x3f /* r4 = (r3 / 4) & 0x3f */ 2214 srd r0, r0, r4 2215 andi. r0, r0, 1 2216 beq guest_exit_cont 2217 /* Get pointer to handler, if any, and call it */ 2218 LOAD_REG_ADDR(r4, hcall_real_table) 2219 lwax r3,r3,r4 2220 cmpwi r3,0 2221 beq guest_exit_cont 2222 add r12,r3,r4 2223 mtctr r12 2224 mr r3,r9 /* get vcpu pointer */ 2225 ld r4,VCPU_GPR(R4)(r9) 2226 bctrl 2227 cmpdi r3,H_TOO_HARD 2228 beq hcall_real_fallback 2229 ld r4,HSTATE_KVM_VCPU(r13) 2230 std r3,VCPU_GPR(R3)(r4) 2231 ld r10,VCPU_PC(r4) 2232 ld r11,VCPU_MSR(r4) 2233 b fast_guest_return 2234 2235sc_1_fast_return: 2236 mtspr SPRN_SRR0,r10 2237 mtspr SPRN_SRR1,r11 2238 li r10, BOOK3S_INTERRUPT_SYSCALL 2239 bl kvmppc_msr_interrupt 2240 mr r4,r9 2241 b fast_guest_return 2242 2243 /* We've attempted a real mode hcall, but it's punted it back 2244 * to userspace. We need to restore some clobbered volatiles 2245 * before resuming the pass-it-to-qemu path */ 2246hcall_real_fallback: 2247 li r12,BOOK3S_INTERRUPT_SYSCALL 2248 ld r9, HSTATE_KVM_VCPU(r13) 2249 2250 b guest_exit_cont 2251 2252 .globl hcall_real_table 2253hcall_real_table: 2254 .long 0 /* 0 - unused */ 2255 .long DOTSYM(kvmppc_h_remove) - hcall_real_table 2256 .long DOTSYM(kvmppc_h_enter) - hcall_real_table 2257 .long DOTSYM(kvmppc_h_read) - hcall_real_table 2258 .long DOTSYM(kvmppc_h_clear_mod) - hcall_real_table 2259 .long DOTSYM(kvmppc_h_clear_ref) - hcall_real_table 2260 .long DOTSYM(kvmppc_h_protect) - hcall_real_table 2261#ifdef CONFIG_SPAPR_TCE_IOMMU 2262 .long DOTSYM(kvmppc_h_get_tce) - hcall_real_table 2263 .long DOTSYM(kvmppc_rm_h_put_tce) - hcall_real_table 2264#else 2265 .long 0 /* 0x1c */ 2266 .long 0 /* 0x20 */ 2267#endif 2268 .long 0 /* 0x24 - H_SET_SPRG0 */ 2269 .long DOTSYM(kvmppc_h_set_dabr) - hcall_real_table 2270 .long DOTSYM(kvmppc_rm_h_page_init) - hcall_real_table 2271 .long 0 /* 0x30 */ 2272 .long 0 /* 0x34 */ 2273 .long 0 /* 0x38 */ 2274 .long 0 /* 0x3c */ 2275 .long 0 /* 0x40 */ 2276 .long 0 /* 0x44 */ 2277 .long 0 /* 0x48 */ 2278 .long 0 /* 0x4c */ 2279 .long 0 /* 0x50 */ 2280 .long 0 /* 0x54 */ 2281 .long 0 /* 0x58 */ 2282 .long 0 /* 0x5c */ 2283 .long 0 /* 0x60 */ 2284#ifdef CONFIG_KVM_XICS 2285 .long DOTSYM(kvmppc_rm_h_eoi) - hcall_real_table 2286 .long DOTSYM(kvmppc_rm_h_cppr) - hcall_real_table 2287 .long DOTSYM(kvmppc_rm_h_ipi) - hcall_real_table 2288 .long DOTSYM(kvmppc_rm_h_ipoll) - hcall_real_table 2289 .long DOTSYM(kvmppc_rm_h_xirr) - hcall_real_table 2290#else 2291 .long 0 /* 0x64 - H_EOI */ 2292 .long 0 /* 0x68 - H_CPPR */ 2293 .long 0 /* 0x6c - H_IPI */ 2294 .long 0 /* 0x70 - H_IPOLL */ 2295 .long 0 /* 0x74 - H_XIRR */ 2296#endif 2297 .long 0 /* 0x78 */ 2298 .long 0 /* 0x7c */ 2299 .long 0 /* 0x80 */ 2300 .long 0 /* 0x84 */ 2301 .long 0 /* 0x88 */ 2302 .long 0 /* 0x8c */ 2303 .long 0 /* 0x90 */ 2304 .long 0 /* 0x94 */ 2305 .long 0 /* 0x98 */ 2306 .long 0 /* 0x9c */ 2307 .long 0 /* 0xa0 */ 2308 .long 0 /* 0xa4 */ 2309 .long 0 /* 0xa8 */ 2310 .long 0 /* 0xac */ 2311 .long 0 /* 0xb0 */ 2312 .long 0 /* 0xb4 */ 2313 .long 0 /* 0xb8 */ 2314 .long 0 /* 0xbc */ 2315 .long 0 /* 0xc0 */ 2316 .long 0 /* 0xc4 */ 2317 .long 0 /* 0xc8 */ 2318 .long 0 /* 0xcc */ 2319 .long 0 /* 0xd0 */ 2320 .long 0 /* 0xd4 */ 2321 .long 0 /* 0xd8 */ 2322 .long 0 /* 0xdc */ 2323 .long DOTSYM(kvmppc_h_cede) - hcall_real_table 2324 .long DOTSYM(kvmppc_rm_h_confer) - hcall_real_table 2325 .long 0 /* 0xe8 */ 2326 .long 0 /* 0xec */ 2327 .long 0 /* 0xf0 */ 2328 .long 0 /* 0xf4 */ 2329 .long 0 /* 0xf8 */ 2330 .long 0 /* 0xfc */ 2331 .long 0 /* 0x100 */ 2332 .long 0 /* 0x104 */ 2333 .long 0 /* 0x108 */ 2334 .long 0 /* 0x10c */ 2335 .long 0 /* 0x110 */ 2336 .long 0 /* 0x114 */ 2337 .long 0 /* 0x118 */ 2338 .long 0 /* 0x11c */ 2339 .long 0 /* 0x120 */ 2340 .long DOTSYM(kvmppc_h_bulk_remove) - hcall_real_table 2341 .long 0 /* 0x128 */ 2342 .long 0 /* 0x12c */ 2343 .long 0 /* 0x130 */ 2344 .long DOTSYM(kvmppc_h_set_xdabr) - hcall_real_table 2345#ifdef CONFIG_SPAPR_TCE_IOMMU 2346 .long DOTSYM(kvmppc_rm_h_stuff_tce) - hcall_real_table 2347 .long DOTSYM(kvmppc_rm_h_put_tce_indirect) - hcall_real_table 2348#else 2349 .long 0 /* 0x138 */ 2350 .long 0 /* 0x13c */ 2351#endif 2352 .long 0 /* 0x140 */ 2353 .long 0 /* 0x144 */ 2354 .long 0 /* 0x148 */ 2355 .long 0 /* 0x14c */ 2356 .long 0 /* 0x150 */ 2357 .long 0 /* 0x154 */ 2358 .long 0 /* 0x158 */ 2359 .long 0 /* 0x15c */ 2360 .long 0 /* 0x160 */ 2361 .long 0 /* 0x164 */ 2362 .long 0 /* 0x168 */ 2363 .long 0 /* 0x16c */ 2364 .long 0 /* 0x170 */ 2365 .long 0 /* 0x174 */ 2366 .long 0 /* 0x178 */ 2367 .long 0 /* 0x17c */ 2368 .long 0 /* 0x180 */ 2369 .long 0 /* 0x184 */ 2370 .long 0 /* 0x188 */ 2371 .long 0 /* 0x18c */ 2372 .long 0 /* 0x190 */ 2373 .long 0 /* 0x194 */ 2374 .long 0 /* 0x198 */ 2375 .long 0 /* 0x19c */ 2376 .long 0 /* 0x1a0 */ 2377 .long 0 /* 0x1a4 */ 2378 .long 0 /* 0x1a8 */ 2379 .long 0 /* 0x1ac */ 2380 .long 0 /* 0x1b0 */ 2381 .long 0 /* 0x1b4 */ 2382 .long 0 /* 0x1b8 */ 2383 .long 0 /* 0x1bc */ 2384 .long 0 /* 0x1c0 */ 2385 .long 0 /* 0x1c4 */ 2386 .long 0 /* 0x1c8 */ 2387 .long 0 /* 0x1cc */ 2388 .long 0 /* 0x1d0 */ 2389 .long 0 /* 0x1d4 */ 2390 .long 0 /* 0x1d8 */ 2391 .long 0 /* 0x1dc */ 2392 .long 0 /* 0x1e0 */ 2393 .long 0 /* 0x1e4 */ 2394 .long 0 /* 0x1e8 */ 2395 .long 0 /* 0x1ec */ 2396 .long 0 /* 0x1f0 */ 2397 .long 0 /* 0x1f4 */ 2398 .long 0 /* 0x1f8 */ 2399 .long 0 /* 0x1fc */ 2400 .long 0 /* 0x200 */ 2401 .long 0 /* 0x204 */ 2402 .long 0 /* 0x208 */ 2403 .long 0 /* 0x20c */ 2404 .long 0 /* 0x210 */ 2405 .long 0 /* 0x214 */ 2406 .long 0 /* 0x218 */ 2407 .long 0 /* 0x21c */ 2408 .long 0 /* 0x220 */ 2409 .long 0 /* 0x224 */ 2410 .long 0 /* 0x228 */ 2411 .long 0 /* 0x22c */ 2412 .long 0 /* 0x230 */ 2413 .long 0 /* 0x234 */ 2414 .long 0 /* 0x238 */ 2415 .long 0 /* 0x23c */ 2416 .long 0 /* 0x240 */ 2417 .long 0 /* 0x244 */ 2418 .long 0 /* 0x248 */ 2419 .long 0 /* 0x24c */ 2420 .long 0 /* 0x250 */ 2421 .long 0 /* 0x254 */ 2422 .long 0 /* 0x258 */ 2423 .long 0 /* 0x25c */ 2424 .long 0 /* 0x260 */ 2425 .long 0 /* 0x264 */ 2426 .long 0 /* 0x268 */ 2427 .long 0 /* 0x26c */ 2428 .long 0 /* 0x270 */ 2429 .long 0 /* 0x274 */ 2430 .long 0 /* 0x278 */ 2431 .long 0 /* 0x27c */ 2432 .long 0 /* 0x280 */ 2433 .long 0 /* 0x284 */ 2434 .long 0 /* 0x288 */ 2435 .long 0 /* 0x28c */ 2436 .long 0 /* 0x290 */ 2437 .long 0 /* 0x294 */ 2438 .long 0 /* 0x298 */ 2439 .long 0 /* 0x29c */ 2440 .long 0 /* 0x2a0 */ 2441 .long 0 /* 0x2a4 */ 2442 .long 0 /* 0x2a8 */ 2443 .long 0 /* 0x2ac */ 2444 .long 0 /* 0x2b0 */ 2445 .long 0 /* 0x2b4 */ 2446 .long 0 /* 0x2b8 */ 2447 .long 0 /* 0x2bc */ 2448 .long 0 /* 0x2c0 */ 2449 .long 0 /* 0x2c4 */ 2450 .long 0 /* 0x2c8 */ 2451 .long 0 /* 0x2cc */ 2452 .long 0 /* 0x2d0 */ 2453 .long 0 /* 0x2d4 */ 2454 .long 0 /* 0x2d8 */ 2455 .long 0 /* 0x2dc */ 2456 .long 0 /* 0x2e0 */ 2457 .long 0 /* 0x2e4 */ 2458 .long 0 /* 0x2e8 */ 2459 .long 0 /* 0x2ec */ 2460 .long 0 /* 0x2f0 */ 2461 .long 0 /* 0x2f4 */ 2462 .long 0 /* 0x2f8 */ 2463#ifdef CONFIG_KVM_XICS 2464 .long DOTSYM(kvmppc_rm_h_xirr_x) - hcall_real_table 2465#else 2466 .long 0 /* 0x2fc - H_XIRR_X*/ 2467#endif 2468 .long DOTSYM(kvmppc_h_random) - hcall_real_table 2469 .globl hcall_real_table_end 2470hcall_real_table_end: 2471 2472_GLOBAL(kvmppc_h_set_xdabr) 2473EXPORT_SYMBOL_GPL(kvmppc_h_set_xdabr) 2474 andi. r0, r5, DABRX_USER | DABRX_KERNEL 2475 beq 6f 2476 li r0, DABRX_USER | DABRX_KERNEL | DABRX_BTI 2477 andc. r0, r5, r0 2478 beq 3f 24796: li r3, H_PARAMETER 2480 blr 2481 2482_GLOBAL(kvmppc_h_set_dabr) 2483EXPORT_SYMBOL_GPL(kvmppc_h_set_dabr) 2484 li r5, DABRX_USER | DABRX_KERNEL 24853: 2486BEGIN_FTR_SECTION 2487 b 2f 2488END_FTR_SECTION_IFSET(CPU_FTR_ARCH_207S) 2489 std r4,VCPU_DABR(r3) 2490 stw r5, VCPU_DABRX(r3) 2491 mtspr SPRN_DABRX, r5 2492 /* Work around P7 bug where DABR can get corrupted on mtspr */ 24931: mtspr SPRN_DABR,r4 2494 mfspr r5, SPRN_DABR 2495 cmpd r4, r5 2496 bne 1b 2497 isync 2498 li r3,0 2499 blr 2500 25012: 2502 LOAD_REG_ADDR(r11, dawr_force_enable) 2503 lbz r11, 0(r11) 2504 cmpdi r11, 0 2505 bne 3f 2506 li r3, H_HARDWARE 2507 blr 25083: 2509 /* Emulate H_SET_DABR/X on P8 for the sake of compat mode guests */ 2510 rlwimi r5, r4, 5, DAWRX_DR | DAWRX_DW 2511 rlwimi r5, r4, 2, DAWRX_WT 2512 clrrdi r4, r4, 3 2513 std r4, VCPU_DAWR(r3) 2514 std r5, VCPU_DAWRX(r3) 2515 /* 2516 * If came in through the real mode hcall handler then it is necessary 2517 * to write the registers since the return path won't. Otherwise it is 2518 * sufficient to store then in the vcpu struct as they will be loaded 2519 * next time the vcpu is run. 2520 */ 2521 mfmsr r6 2522 andi. r6, r6, MSR_DR /* in real mode? */ 2523 bne 4f 2524 mtspr SPRN_DAWR, r4 2525 mtspr SPRN_DAWRX, r5 25264: li r3, 0 2527 blr 2528 2529_GLOBAL(kvmppc_h_cede) /* r3 = vcpu pointer, r11 = msr, r13 = paca */ 2530 ori r11,r11,MSR_EE 2531 std r11,VCPU_MSR(r3) 2532 li r0,1 2533 stb r0,VCPU_CEDED(r3) 2534 sync /* order setting ceded vs. testing prodded */ 2535 lbz r5,VCPU_PRODDED(r3) 2536 cmpwi r5,0 2537 bne kvm_cede_prodded 2538 li r12,0 /* set trap to 0 to say hcall is handled */ 2539 stw r12,VCPU_TRAP(r3) 2540 li r0,H_SUCCESS 2541 std r0,VCPU_GPR(R3)(r3) 2542 2543 /* 2544 * Set our bit in the bitmask of napping threads unless all the 2545 * other threads are already napping, in which case we send this 2546 * up to the host. 2547 */ 2548 ld r5,HSTATE_KVM_VCORE(r13) 2549 lbz r6,HSTATE_PTID(r13) 2550 lwz r8,VCORE_ENTRY_EXIT(r5) 2551 clrldi r8,r8,56 2552 li r0,1 2553 sld r0,r0,r6 2554 addi r6,r5,VCORE_NAPPING_THREADS 255531: lwarx r4,0,r6 2556 or r4,r4,r0 2557 cmpw r4,r8 2558 beq kvm_cede_exit 2559 stwcx. r4,0,r6 2560 bne 31b 2561 /* order napping_threads update vs testing entry_exit_map */ 2562 isync 2563 li r0,NAPPING_CEDE 2564 stb r0,HSTATE_NAPPING(r13) 2565 lwz r7,VCORE_ENTRY_EXIT(r5) 2566 cmpwi r7,0x100 2567 bge 33f /* another thread already exiting */ 2568 2569/* 2570 * Although not specifically required by the architecture, POWER7 2571 * preserves the following registers in nap mode, even if an SMT mode 2572 * switch occurs: SLB entries, PURR, SPURR, AMOR, UAMOR, AMR, SPRG0-3, 2573 * DAR, DSISR, DABR, DABRX, DSCR, PMCx, MMCRx, SIAR, SDAR. 2574 */ 2575 /* Save non-volatile GPRs */ 2576 std r14, VCPU_GPR(R14)(r3) 2577 std r15, VCPU_GPR(R15)(r3) 2578 std r16, VCPU_GPR(R16)(r3) 2579 std r17, VCPU_GPR(R17)(r3) 2580 std r18, VCPU_GPR(R18)(r3) 2581 std r19, VCPU_GPR(R19)(r3) 2582 std r20, VCPU_GPR(R20)(r3) 2583 std r21, VCPU_GPR(R21)(r3) 2584 std r22, VCPU_GPR(R22)(r3) 2585 std r23, VCPU_GPR(R23)(r3) 2586 std r24, VCPU_GPR(R24)(r3) 2587 std r25, VCPU_GPR(R25)(r3) 2588 std r26, VCPU_GPR(R26)(r3) 2589 std r27, VCPU_GPR(R27)(r3) 2590 std r28, VCPU_GPR(R28)(r3) 2591 std r29, VCPU_GPR(R29)(r3) 2592 std r30, VCPU_GPR(R30)(r3) 2593 std r31, VCPU_GPR(R31)(r3) 2594 2595 /* save FP state */ 2596 bl kvmppc_save_fp 2597 2598#ifdef CONFIG_PPC_TRANSACTIONAL_MEM 2599/* 2600 * Branch around the call if both CPU_FTR_TM and 2601 * CPU_FTR_P9_TM_HV_ASSIST are off. 2602 */ 2603BEGIN_FTR_SECTION 2604 b 91f 2605END_FTR_SECTION(CPU_FTR_TM | CPU_FTR_P9_TM_HV_ASSIST, 0) 2606 /* 2607 * NOTE THAT THIS TRASHES ALL NON-VOLATILE REGISTERS (but not CR) 2608 */ 2609 ld r3, HSTATE_KVM_VCPU(r13) 2610 ld r4, VCPU_MSR(r3) 2611 li r5, 0 /* don't preserve non-vol regs */ 2612 bl kvmppc_save_tm_hv 2613 nop 261491: 2615#endif 2616 2617 /* 2618 * Set DEC to the smaller of DEC and HDEC, so that we wake 2619 * no later than the end of our timeslice (HDEC interrupts 2620 * don't wake us from nap). 2621 */ 2622 mfspr r3, SPRN_DEC 2623 mfspr r4, SPRN_HDEC 2624 mftb r5 2625BEGIN_FTR_SECTION 2626 /* On P9 check whether the guest has large decrementer mode enabled */ 2627 ld r6, HSTATE_KVM_VCORE(r13) 2628 ld r6, VCORE_LPCR(r6) 2629 andis. r6, r6, LPCR_LD@h 2630 bne 68f 2631END_FTR_SECTION_IFSET(CPU_FTR_ARCH_300) 2632 extsw r3, r3 263368: EXTEND_HDEC(r4) 2634 cmpd r3, r4 2635 ble 67f 2636 mtspr SPRN_DEC, r4 263767: 2638 /* save expiry time of guest decrementer */ 2639 add r3, r3, r5 2640 ld r4, HSTATE_KVM_VCPU(r13) 2641 ld r5, HSTATE_KVM_VCORE(r13) 2642 ld r6, VCORE_TB_OFFSET_APPL(r5) 2643 subf r3, r6, r3 /* convert to host TB value */ 2644 std r3, VCPU_DEC_EXPIRES(r4) 2645 2646#ifdef CONFIG_KVM_BOOK3S_HV_EXIT_TIMING 2647 ld r4, HSTATE_KVM_VCPU(r13) 2648 addi r3, r4, VCPU_TB_CEDE 2649 bl kvmhv_accumulate_time 2650#endif 2651 2652 lis r3, LPCR_PECEDP@h /* Do wake on privileged doorbell */ 2653 2654 /* Go back to host stack */ 2655 ld r1, HSTATE_HOST_R1(r13) 2656 2657 /* 2658 * Take a nap until a decrementer or external or doobell interrupt 2659 * occurs, with PECE1 and PECE0 set in LPCR. 2660 * On POWER8, set PECEDH, and if we are ceding, also set PECEDP. 2661 * Also clear the runlatch bit before napping. 2662 */ 2663kvm_do_nap: 2664 mfspr r0, SPRN_CTRLF 2665 clrrdi r0, r0, 1 2666 mtspr SPRN_CTRLT, r0 2667 2668 li r0,1 2669 stb r0,HSTATE_HWTHREAD_REQ(r13) 2670 mfspr r5,SPRN_LPCR 2671 ori r5,r5,LPCR_PECE0 | LPCR_PECE1 2672BEGIN_FTR_SECTION 2673 ori r5, r5, LPCR_PECEDH 2674 rlwimi r5, r3, 0, LPCR_PECEDP 2675END_FTR_SECTION_IFSET(CPU_FTR_ARCH_207S) 2676 2677kvm_nap_sequence: /* desired LPCR value in r5 */ 2678BEGIN_FTR_SECTION 2679 /* 2680 * PSSCR bits: exit criterion = 1 (wakeup based on LPCR at sreset) 2681 * enable state loss = 1 (allow SMT mode switch) 2682 * requested level = 0 (just stop dispatching) 2683 */ 2684 lis r3, (PSSCR_EC | PSSCR_ESL)@h 2685 /* Set LPCR_PECE_HVEE bit to enable wakeup by HV interrupts */ 2686 li r4, LPCR_PECE_HVEE@higher 2687 sldi r4, r4, 32 2688 or r5, r5, r4 2689FTR_SECTION_ELSE 2690 li r3, PNV_THREAD_NAP 2691ALT_FTR_SECTION_END_IFSET(CPU_FTR_ARCH_300) 2692 mtspr SPRN_LPCR,r5 2693 isync 2694 2695BEGIN_FTR_SECTION 2696 bl isa300_idle_stop_mayloss 2697FTR_SECTION_ELSE 2698 bl isa206_idle_insn_mayloss 2699ALT_FTR_SECTION_END_IFSET(CPU_FTR_ARCH_300) 2700 2701 mfspr r0, SPRN_CTRLF 2702 ori r0, r0, 1 2703 mtspr SPRN_CTRLT, r0 2704 2705 mtspr SPRN_SRR1, r3 2706 2707 li r0, 0 2708 stb r0, PACA_FTRACE_ENABLED(r13) 2709 2710 li r0, KVM_HWTHREAD_IN_KVM 2711 stb r0, HSTATE_HWTHREAD_STATE(r13) 2712 2713 lbz r0, HSTATE_NAPPING(r13) 2714 cmpwi r0, NAPPING_CEDE 2715 beq kvm_end_cede 2716 cmpwi r0, NAPPING_NOVCPU 2717 beq kvm_novcpu_wakeup 2718 cmpwi r0, NAPPING_UNSPLIT 2719 beq kvm_unsplit_wakeup 2720 twi 31,0,0 /* Nap state must not be zero */ 2721 272233: mr r4, r3 2723 li r3, 0 2724 li r12, 0 2725 b 34f 2726 2727kvm_end_cede: 2728 /* Woken by external or decrementer interrupt */ 2729 2730 /* get vcpu pointer */ 2731 ld r4, HSTATE_KVM_VCPU(r13) 2732 2733#ifdef CONFIG_KVM_BOOK3S_HV_EXIT_TIMING 2734 addi r3, r4, VCPU_TB_RMINTR 2735 bl kvmhv_accumulate_time 2736#endif 2737 2738#ifdef CONFIG_PPC_TRANSACTIONAL_MEM 2739/* 2740 * Branch around the call if both CPU_FTR_TM and 2741 * CPU_FTR_P9_TM_HV_ASSIST are off. 2742 */ 2743BEGIN_FTR_SECTION 2744 b 91f 2745END_FTR_SECTION(CPU_FTR_TM | CPU_FTR_P9_TM_HV_ASSIST, 0) 2746 /* 2747 * NOTE THAT THIS TRASHES ALL NON-VOLATILE REGISTERS (but not CR) 2748 */ 2749 mr r3, r4 2750 ld r4, VCPU_MSR(r3) 2751 li r5, 0 /* don't preserve non-vol regs */ 2752 bl kvmppc_restore_tm_hv 2753 nop 2754 ld r4, HSTATE_KVM_VCPU(r13) 275591: 2756#endif 2757 2758 /* load up FP state */ 2759 bl kvmppc_load_fp 2760 2761 /* Restore guest decrementer */ 2762 ld r3, VCPU_DEC_EXPIRES(r4) 2763 ld r5, HSTATE_KVM_VCORE(r13) 2764 ld r6, VCORE_TB_OFFSET_APPL(r5) 2765 add r3, r3, r6 /* convert host TB to guest TB value */ 2766 mftb r7 2767 subf r3, r7, r3 2768 mtspr SPRN_DEC, r3 2769 2770 /* Load NV GPRS */ 2771 ld r14, VCPU_GPR(R14)(r4) 2772 ld r15, VCPU_GPR(R15)(r4) 2773 ld r16, VCPU_GPR(R16)(r4) 2774 ld r17, VCPU_GPR(R17)(r4) 2775 ld r18, VCPU_GPR(R18)(r4) 2776 ld r19, VCPU_GPR(R19)(r4) 2777 ld r20, VCPU_GPR(R20)(r4) 2778 ld r21, VCPU_GPR(R21)(r4) 2779 ld r22, VCPU_GPR(R22)(r4) 2780 ld r23, VCPU_GPR(R23)(r4) 2781 ld r24, VCPU_GPR(R24)(r4) 2782 ld r25, VCPU_GPR(R25)(r4) 2783 ld r26, VCPU_GPR(R26)(r4) 2784 ld r27, VCPU_GPR(R27)(r4) 2785 ld r28, VCPU_GPR(R28)(r4) 2786 ld r29, VCPU_GPR(R29)(r4) 2787 ld r30, VCPU_GPR(R30)(r4) 2788 ld r31, VCPU_GPR(R31)(r4) 2789 2790 /* Check the wake reason in SRR1 to see why we got here */ 2791 bl kvmppc_check_wake_reason 2792 2793 /* 2794 * Restore volatile registers since we could have called a 2795 * C routine in kvmppc_check_wake_reason 2796 * r4 = VCPU 2797 * r3 tells us whether we need to return to host or not 2798 * WARNING: it gets checked further down: 2799 * should not modify r3 until this check is done. 2800 */ 2801 ld r4, HSTATE_KVM_VCPU(r13) 2802 2803 /* clear our bit in vcore->napping_threads */ 280434: ld r5,HSTATE_KVM_VCORE(r13) 2805 lbz r7,HSTATE_PTID(r13) 2806 li r0,1 2807 sld r0,r0,r7 2808 addi r6,r5,VCORE_NAPPING_THREADS 280932: lwarx r7,0,r6 2810 andc r7,r7,r0 2811 stwcx. r7,0,r6 2812 bne 32b 2813 li r0,0 2814 stb r0,HSTATE_NAPPING(r13) 2815 2816 /* See if the wake reason saved in r3 means we need to exit */ 2817 stw r12, VCPU_TRAP(r4) 2818 mr r9, r4 2819 cmpdi r3, 0 2820 bgt guest_exit_cont 2821 b maybe_reenter_guest 2822 2823 /* cede when already previously prodded case */ 2824kvm_cede_prodded: 2825 li r0,0 2826 stb r0,VCPU_PRODDED(r3) 2827 sync /* order testing prodded vs. clearing ceded */ 2828 stb r0,VCPU_CEDED(r3) 2829 li r3,H_SUCCESS 2830 blr 2831 2832 /* we've ceded but we want to give control to the host */ 2833kvm_cede_exit: 2834 ld r9, HSTATE_KVM_VCPU(r13) 2835#ifdef CONFIG_KVM_XICS 2836 /* are we using XIVE with single escalation? */ 2837 ld r10, VCPU_XIVE_ESC_VADDR(r9) 2838 cmpdi r10, 0 2839 beq 3f 2840 li r6, XIVE_ESB_SET_PQ_00 2841 /* 2842 * If we still have a pending escalation, abort the cede, 2843 * and we must set PQ to 10 rather than 00 so that we don't 2844 * potentially end up with two entries for the escalation 2845 * interrupt in the XIVE interrupt queue. In that case 2846 * we also don't want to set xive_esc_on to 1 here in 2847 * case we race with xive_esc_irq(). 2848 */ 2849 lbz r5, VCPU_XIVE_ESC_ON(r9) 2850 cmpwi r5, 0 2851 beq 4f 2852 li r0, 0 2853 stb r0, VCPU_CEDED(r9) 2854 li r6, XIVE_ESB_SET_PQ_10 2855 b 5f 28564: li r0, 1 2857 stb r0, VCPU_XIVE_ESC_ON(r9) 2858 /* make sure store to xive_esc_on is seen before xive_esc_irq runs */ 2859 sync 28605: /* Enable XIVE escalation */ 2861 mfmsr r0 2862 andi. r0, r0, MSR_DR /* in real mode? */ 2863 beq 1f 2864 ldx r0, r10, r6 2865 b 2f 28661: ld r10, VCPU_XIVE_ESC_RADDR(r9) 2867 ldcix r0, r10, r6 28682: sync 2869#endif /* CONFIG_KVM_XICS */ 28703: b guest_exit_cont 2871 2872 /* Try to do machine check recovery in real mode */ 2873machine_check_realmode: 2874 mr r3, r9 /* get vcpu pointer */ 2875 bl kvmppc_realmode_machine_check 2876 nop 2877 /* all machine checks go to virtual mode for further handling */ 2878 ld r9, HSTATE_KVM_VCPU(r13) 2879 li r12, BOOK3S_INTERRUPT_MACHINE_CHECK 2880 b guest_exit_cont 2881 2882/* 2883 * Call C code to handle a HMI in real mode. 2884 * Only the primary thread does the call, secondary threads are handled 2885 * by calling hmi_exception_realmode() after kvmppc_hv_entry returns. 2886 * r9 points to the vcpu on entry 2887 */ 2888hmi_realmode: 2889 lbz r0, HSTATE_PTID(r13) 2890 cmpwi r0, 0 2891 bne guest_exit_cont 2892 bl kvmppc_realmode_hmi_handler 2893 ld r9, HSTATE_KVM_VCPU(r13) 2894 li r12, BOOK3S_INTERRUPT_HMI 2895 b guest_exit_cont 2896 2897/* 2898 * Check the reason we woke from nap, and take appropriate action. 2899 * Returns (in r3): 2900 * 0 if nothing needs to be done 2901 * 1 if something happened that needs to be handled by the host 2902 * -1 if there was a guest wakeup (IPI or msgsnd) 2903 * -2 if we handled a PCI passthrough interrupt (returned by 2904 * kvmppc_read_intr only) 2905 * 2906 * Also sets r12 to the interrupt vector for any interrupt that needs 2907 * to be handled now by the host (0x500 for external interrupt), or zero. 2908 * Modifies all volatile registers (since it may call a C function). 2909 * This routine calls kvmppc_read_intr, a C function, if an external 2910 * interrupt is pending. 2911 */ 2912kvmppc_check_wake_reason: 2913 mfspr r6, SPRN_SRR1 2914BEGIN_FTR_SECTION 2915 rlwinm r6, r6, 45-31, 0xf /* extract wake reason field (P8) */ 2916FTR_SECTION_ELSE 2917 rlwinm r6, r6, 45-31, 0xe /* P7 wake reason field is 3 bits */ 2918ALT_FTR_SECTION_END_IFSET(CPU_FTR_ARCH_207S) 2919 cmpwi r6, 8 /* was it an external interrupt? */ 2920 beq 7f /* if so, see what it was */ 2921 li r3, 0 2922 li r12, 0 2923 cmpwi r6, 6 /* was it the decrementer? */ 2924 beq 0f 2925BEGIN_FTR_SECTION 2926 cmpwi r6, 5 /* privileged doorbell? */ 2927 beq 0f 2928 cmpwi r6, 3 /* hypervisor doorbell? */ 2929 beq 3f 2930END_FTR_SECTION_IFSET(CPU_FTR_ARCH_207S) 2931 cmpwi r6, 0xa /* Hypervisor maintenance ? */ 2932 beq 4f 2933 li r3, 1 /* anything else, return 1 */ 29340: blr 2935 2936 /* hypervisor doorbell */ 29373: li r12, BOOK3S_INTERRUPT_H_DOORBELL 2938 2939 /* 2940 * Clear the doorbell as we will invoke the handler 2941 * explicitly in the guest exit path. 2942 */ 2943 lis r6, (PPC_DBELL_SERVER << (63-36))@h 2944 PPC_MSGCLR(6) 2945 /* see if it's a host IPI */ 2946 li r3, 1 2947BEGIN_FTR_SECTION 2948 PPC_MSGSYNC 2949 lwsync 2950END_FTR_SECTION_IFSET(CPU_FTR_ARCH_300) 2951 lbz r0, HSTATE_HOST_IPI(r13) 2952 cmpwi r0, 0 2953 bnelr 2954 /* if not, return -1 */ 2955 li r3, -1 2956 blr 2957 2958 /* Woken up due to Hypervisor maintenance interrupt */ 29594: li r12, BOOK3S_INTERRUPT_HMI 2960 li r3, 1 2961 blr 2962 2963 /* external interrupt - create a stack frame so we can call C */ 29647: mflr r0 2965 std r0, PPC_LR_STKOFF(r1) 2966 stdu r1, -PPC_MIN_STKFRM(r1) 2967 bl kvmppc_read_intr 2968 nop 2969 li r12, BOOK3S_INTERRUPT_EXTERNAL 2970 cmpdi r3, 1 2971 ble 1f 2972 2973 /* 2974 * Return code of 2 means PCI passthrough interrupt, but 2975 * we need to return back to host to complete handling the 2976 * interrupt. Trap reason is expected in r12 by guest 2977 * exit code. 2978 */ 2979 li r12, BOOK3S_INTERRUPT_HV_RM_HARD 29801: 2981 ld r0, PPC_MIN_STKFRM+PPC_LR_STKOFF(r1) 2982 addi r1, r1, PPC_MIN_STKFRM 2983 mtlr r0 2984 blr 2985 2986/* 2987 * Save away FP, VMX and VSX registers. 2988 * r3 = vcpu pointer 2989 * N.B. r30 and r31 are volatile across this function, 2990 * thus it is not callable from C. 2991 */ 2992kvmppc_save_fp: 2993 mflr r30 2994 mr r31,r3 2995 mfmsr r5 2996 ori r8,r5,MSR_FP 2997#ifdef CONFIG_ALTIVEC 2998BEGIN_FTR_SECTION 2999 oris r8,r8,MSR_VEC@h 3000END_FTR_SECTION_IFSET(CPU_FTR_ALTIVEC) 3001#endif 3002#ifdef CONFIG_VSX 3003BEGIN_FTR_SECTION 3004 oris r8,r8,MSR_VSX@h 3005END_FTR_SECTION_IFSET(CPU_FTR_VSX) 3006#endif 3007 mtmsrd r8 3008 addi r3,r3,VCPU_FPRS 3009 bl store_fp_state 3010#ifdef CONFIG_ALTIVEC 3011BEGIN_FTR_SECTION 3012 addi r3,r31,VCPU_VRS 3013 bl store_vr_state 3014END_FTR_SECTION_IFSET(CPU_FTR_ALTIVEC) 3015#endif 3016 mfspr r6,SPRN_VRSAVE 3017 stw r6,VCPU_VRSAVE(r31) 3018 mtlr r30 3019 blr 3020 3021/* 3022 * Load up FP, VMX and VSX registers 3023 * r4 = vcpu pointer 3024 * N.B. r30 and r31 are volatile across this function, 3025 * thus it is not callable from C. 3026 */ 3027kvmppc_load_fp: 3028 mflr r30 3029 mr r31,r4 3030 mfmsr r9 3031 ori r8,r9,MSR_FP 3032#ifdef CONFIG_ALTIVEC 3033BEGIN_FTR_SECTION 3034 oris r8,r8,MSR_VEC@h 3035END_FTR_SECTION_IFSET(CPU_FTR_ALTIVEC) 3036#endif 3037#ifdef CONFIG_VSX 3038BEGIN_FTR_SECTION 3039 oris r8,r8,MSR_VSX@h 3040END_FTR_SECTION_IFSET(CPU_FTR_VSX) 3041#endif 3042 mtmsrd r8 3043 addi r3,r4,VCPU_FPRS 3044 bl load_fp_state 3045#ifdef CONFIG_ALTIVEC 3046BEGIN_FTR_SECTION 3047 addi r3,r31,VCPU_VRS 3048 bl load_vr_state 3049END_FTR_SECTION_IFSET(CPU_FTR_ALTIVEC) 3050#endif 3051 lwz r7,VCPU_VRSAVE(r31) 3052 mtspr SPRN_VRSAVE,r7 3053 mtlr r30 3054 mr r4,r31 3055 blr 3056 3057#ifdef CONFIG_PPC_TRANSACTIONAL_MEM 3058/* 3059 * Save transactional state and TM-related registers. 3060 * Called with r3 pointing to the vcpu struct and r4 containing 3061 * the guest MSR value. 3062 * r5 is non-zero iff non-volatile register state needs to be maintained. 3063 * If r5 == 0, this can modify all checkpointed registers, but 3064 * restores r1 and r2 before exit. 3065 */ 3066_GLOBAL_TOC(kvmppc_save_tm_hv) 3067EXPORT_SYMBOL_GPL(kvmppc_save_tm_hv) 3068 /* See if we need to handle fake suspend mode */ 3069BEGIN_FTR_SECTION 3070 b __kvmppc_save_tm 3071END_FTR_SECTION_IFCLR(CPU_FTR_P9_TM_HV_ASSIST) 3072 3073 lbz r0, HSTATE_FAKE_SUSPEND(r13) /* Were we fake suspended? */ 3074 cmpwi r0, 0 3075 beq __kvmppc_save_tm 3076 3077 /* The following code handles the fake_suspend = 1 case */ 3078 mflr r0 3079 std r0, PPC_LR_STKOFF(r1) 3080 stdu r1, -PPC_MIN_STKFRM(r1) 3081 3082 /* Turn on TM. */ 3083 mfmsr r8 3084 li r0, 1 3085 rldimi r8, r0, MSR_TM_LG, 63-MSR_TM_LG 3086 mtmsrd r8 3087 3088 rldicl. r8, r8, 64 - MSR_TS_S_LG, 62 /* Did we actually hrfid? */ 3089 beq 4f 3090BEGIN_FTR_SECTION 3091 bl pnv_power9_force_smt4_catch 3092END_FTR_SECTION_IFSET(CPU_FTR_P9_TM_XER_SO_BUG) 3093 nop 3094 3095 /* We have to treclaim here because that's the only way to do S->N */ 3096 li r3, TM_CAUSE_KVM_RESCHED 3097 TRECLAIM(R3) 3098 3099 /* 3100 * We were in fake suspend, so we are not going to save the 3101 * register state as the guest checkpointed state (since 3102 * we already have it), therefore we can now use any volatile GPR. 3103 * In fact treclaim in fake suspend state doesn't modify 3104 * any registers. 3105 */ 3106 3107BEGIN_FTR_SECTION 3108 bl pnv_power9_force_smt4_release 3109END_FTR_SECTION_IFSET(CPU_FTR_P9_TM_XER_SO_BUG) 3110 nop 3111 31124: 3113 mfspr r3, SPRN_PSSCR 3114 /* PSSCR_FAKE_SUSPEND is a write-only bit, but clear it anyway */ 3115 li r0, PSSCR_FAKE_SUSPEND 3116 andc r3, r3, r0 3117 mtspr SPRN_PSSCR, r3 3118 3119 /* Don't save TEXASR, use value from last exit in real suspend state */ 3120 ld r9, HSTATE_KVM_VCPU(r13) 3121 mfspr r5, SPRN_TFHAR 3122 mfspr r6, SPRN_TFIAR 3123 std r5, VCPU_TFHAR(r9) 3124 std r6, VCPU_TFIAR(r9) 3125 3126 addi r1, r1, PPC_MIN_STKFRM 3127 ld r0, PPC_LR_STKOFF(r1) 3128 mtlr r0 3129 blr 3130 3131/* 3132 * Restore transactional state and TM-related registers. 3133 * Called with r3 pointing to the vcpu struct 3134 * and r4 containing the guest MSR value. 3135 * r5 is non-zero iff non-volatile register state needs to be maintained. 3136 * This potentially modifies all checkpointed registers. 3137 * It restores r1 and r2 from the PACA. 3138 */ 3139_GLOBAL_TOC(kvmppc_restore_tm_hv) 3140EXPORT_SYMBOL_GPL(kvmppc_restore_tm_hv) 3141 /* 3142 * If we are doing TM emulation for the guest on a POWER9 DD2, 3143 * then we don't actually do a trechkpt -- we either set up 3144 * fake-suspend mode, or emulate a TM rollback. 3145 */ 3146BEGIN_FTR_SECTION 3147 b __kvmppc_restore_tm 3148END_FTR_SECTION_IFCLR(CPU_FTR_P9_TM_HV_ASSIST) 3149 mflr r0 3150 std r0, PPC_LR_STKOFF(r1) 3151 3152 li r0, 0 3153 stb r0, HSTATE_FAKE_SUSPEND(r13) 3154 3155 /* Turn on TM so we can restore TM SPRs */ 3156 mfmsr r5 3157 li r0, 1 3158 rldimi r5, r0, MSR_TM_LG, 63-MSR_TM_LG 3159 mtmsrd r5 3160 3161 /* 3162 * The user may change these outside of a transaction, so they must 3163 * always be context switched. 3164 */ 3165 ld r5, VCPU_TFHAR(r3) 3166 ld r6, VCPU_TFIAR(r3) 3167 ld r7, VCPU_TEXASR(r3) 3168 mtspr SPRN_TFHAR, r5 3169 mtspr SPRN_TFIAR, r6 3170 mtspr SPRN_TEXASR, r7 3171 3172 rldicl. r5, r4, 64 - MSR_TS_S_LG, 62 3173 beqlr /* TM not active in guest */ 3174 3175 /* Make sure the failure summary is set */ 3176 oris r7, r7, (TEXASR_FS)@h 3177 mtspr SPRN_TEXASR, r7 3178 3179 cmpwi r5, 1 /* check for suspended state */ 3180 bgt 10f 3181 stb r5, HSTATE_FAKE_SUSPEND(r13) 3182 b 9f /* and return */ 318310: stdu r1, -PPC_MIN_STKFRM(r1) 3184 /* guest is in transactional state, so simulate rollback */ 3185 bl kvmhv_emulate_tm_rollback 3186 nop 3187 addi r1, r1, PPC_MIN_STKFRM 31889: ld r0, PPC_LR_STKOFF(r1) 3189 mtlr r0 3190 blr 3191#endif /* CONFIG_PPC_TRANSACTIONAL_MEM */ 3192 3193/* 3194 * We come here if we get any exception or interrupt while we are 3195 * executing host real mode code while in guest MMU context. 3196 * r12 is (CR << 32) | vector 3197 * r13 points to our PACA 3198 * r12 is saved in HSTATE_SCRATCH0(r13) 3199 * ctr is saved in HSTATE_SCRATCH1(r13) if RELOCATABLE 3200 * r9 is saved in HSTATE_SCRATCH2(r13) 3201 * r13 is saved in HSPRG1 3202 * cfar is saved in HSTATE_CFAR(r13) 3203 * ppr is saved in HSTATE_PPR(r13) 3204 */ 3205kvmppc_bad_host_intr: 3206 /* 3207 * Switch to the emergency stack, but start half-way down in 3208 * case we were already on it. 3209 */ 3210 mr r9, r1 3211 std r1, PACAR1(r13) 3212 ld r1, PACAEMERGSP(r13) 3213 subi r1, r1, THREAD_SIZE/2 + INT_FRAME_SIZE 3214 std r9, 0(r1) 3215 std r0, GPR0(r1) 3216 std r9, GPR1(r1) 3217 std r2, GPR2(r1) 3218 SAVE_4GPRS(3, r1) 3219 SAVE_2GPRS(7, r1) 3220 srdi r0, r12, 32 3221 clrldi r12, r12, 32 3222 std r0, _CCR(r1) 3223 std r12, _TRAP(r1) 3224 andi. r0, r12, 2 3225 beq 1f 3226 mfspr r3, SPRN_HSRR0 3227 mfspr r4, SPRN_HSRR1 3228 mfspr r5, SPRN_HDAR 3229 mfspr r6, SPRN_HDSISR 3230 b 2f 32311: mfspr r3, SPRN_SRR0 3232 mfspr r4, SPRN_SRR1 3233 mfspr r5, SPRN_DAR 3234 mfspr r6, SPRN_DSISR 32352: std r3, _NIP(r1) 3236 std r4, _MSR(r1) 3237 std r5, _DAR(r1) 3238 std r6, _DSISR(r1) 3239 ld r9, HSTATE_SCRATCH2(r13) 3240 ld r12, HSTATE_SCRATCH0(r13) 3241 GET_SCRATCH0(r0) 3242 SAVE_4GPRS(9, r1) 3243 std r0, GPR13(r1) 3244 SAVE_NVGPRS(r1) 3245 ld r5, HSTATE_CFAR(r13) 3246 std r5, ORIG_GPR3(r1) 3247 mflr r3 3248#ifdef CONFIG_RELOCATABLE 3249 ld r4, HSTATE_SCRATCH1(r13) 3250#else 3251 mfctr r4 3252#endif 3253 mfxer r5 3254 lbz r6, PACAIRQSOFTMASK(r13) 3255 std r3, _LINK(r1) 3256 std r4, _CTR(r1) 3257 std r5, _XER(r1) 3258 std r6, SOFTE(r1) 3259 ld r2, PACATOC(r13) 3260 LOAD_REG_IMMEDIATE(3, 0x7265677368657265) 3261 std r3, STACK_FRAME_OVERHEAD-16(r1) 3262 3263 /* 3264 * On POWER9 do a minimal restore of the MMU and call C code, 3265 * which will print a message and panic. 3266 * XXX On POWER7 and POWER8, we just spin here since we don't 3267 * know what the other threads are doing (and we don't want to 3268 * coordinate with them) - but at least we now have register state 3269 * in memory that we might be able to look at from another CPU. 3270 */ 3271BEGIN_FTR_SECTION 3272 b . 3273END_FTR_SECTION_IFCLR(CPU_FTR_ARCH_300) 3274 ld r9, HSTATE_KVM_VCPU(r13) 3275 ld r10, VCPU_KVM(r9) 3276 3277 li r0, 0 3278 mtspr SPRN_AMR, r0 3279 mtspr SPRN_IAMR, r0 3280 mtspr SPRN_CIABR, r0 3281 mtspr SPRN_DAWRX, r0 3282 3283BEGIN_MMU_FTR_SECTION 3284 b 4f 3285END_MMU_FTR_SECTION_IFSET(MMU_FTR_TYPE_RADIX) 3286 3287 slbmte r0, r0 3288 slbia 3289 ptesync 3290 ld r8, PACA_SLBSHADOWPTR(r13) 3291 .rept SLB_NUM_BOLTED 3292 li r3, SLBSHADOW_SAVEAREA 3293 LDX_BE r5, r8, r3 3294 addi r3, r3, 8 3295 LDX_BE r6, r8, r3 3296 andis. r7, r5, SLB_ESID_V@h 3297 beq 3f 3298 slbmte r6, r5 32993: addi r8, r8, 16 3300 .endr 3301 33024: lwz r7, KVM_HOST_LPID(r10) 3303 mtspr SPRN_LPID, r7 3304 mtspr SPRN_PID, r0 3305 ld r8, KVM_HOST_LPCR(r10) 3306 mtspr SPRN_LPCR, r8 3307 isync 3308 li r0, KVM_GUEST_MODE_NONE 3309 stb r0, HSTATE_IN_GUEST(r13) 3310 3311 /* 3312 * Turn on the MMU and jump to C code 3313 */ 3314 bcl 20, 31, .+4 33155: mflr r3 3316 addi r3, r3, 9f - 5b 3317 li r4, -1 3318 rldimi r3, r4, 62, 0 /* ensure 0xc000000000000000 bits are set */ 3319 ld r4, PACAKMSR(r13) 3320 mtspr SPRN_SRR0, r3 3321 mtspr SPRN_SRR1, r4 3322 RFI_TO_KERNEL 33239: addi r3, r1, STACK_FRAME_OVERHEAD 3324 bl kvmppc_bad_interrupt 3325 b 9b 3326 3327/* 3328 * This mimics the MSR transition on IRQ delivery. The new guest MSR is taken 3329 * from VCPU_INTR_MSR and is modified based on the required TM state changes. 3330 * r11 has the guest MSR value (in/out) 3331 * r9 has a vcpu pointer (in) 3332 * r0 is used as a scratch register 3333 */ 3334kvmppc_msr_interrupt: 3335 rldicl r0, r11, 64 - MSR_TS_S_LG, 62 3336 cmpwi r0, 2 /* Check if we are in transactional state.. */ 3337 ld r11, VCPU_INTR_MSR(r9) 3338 bne 1f 3339 /* ... if transactional, change to suspended */ 3340 li r0, 1 33411: rldimi r11, r0, MSR_TS_S_LG, 63 - MSR_TS_T_LG 3342 blr 3343 3344/* 3345 * Load up guest PMU state. R3 points to the vcpu struct. 3346 */ 3347_GLOBAL(kvmhv_load_guest_pmu) 3348EXPORT_SYMBOL_GPL(kvmhv_load_guest_pmu) 3349 mr r4, r3 3350 mflr r0 3351 li r3, 1 3352 sldi r3, r3, 31 /* MMCR0_FC (freeze counters) bit */ 3353 mtspr SPRN_MMCR0, r3 /* freeze all counters, disable ints */ 3354 isync 3355BEGIN_FTR_SECTION 3356 ld r3, VCPU_MMCR(r4) 3357 andi. r5, r3, MMCR0_PMAO_SYNC | MMCR0_PMAO 3358 cmpwi r5, MMCR0_PMAO 3359 beql kvmppc_fix_pmao 3360END_FTR_SECTION_IFSET(CPU_FTR_PMAO_BUG) 3361 lwz r3, VCPU_PMC(r4) /* always load up guest PMU registers */ 3362 lwz r5, VCPU_PMC + 4(r4) /* to prevent information leak */ 3363 lwz r6, VCPU_PMC + 8(r4) 3364 lwz r7, VCPU_PMC + 12(r4) 3365 lwz r8, VCPU_PMC + 16(r4) 3366 lwz r9, VCPU_PMC + 20(r4) 3367 mtspr SPRN_PMC1, r3 3368 mtspr SPRN_PMC2, r5 3369 mtspr SPRN_PMC3, r6 3370 mtspr SPRN_PMC4, r7 3371 mtspr SPRN_PMC5, r8 3372 mtspr SPRN_PMC6, r9 3373 ld r3, VCPU_MMCR(r4) 3374 ld r5, VCPU_MMCR + 8(r4) 3375 ld r6, VCPU_MMCR + 16(r4) 3376 ld r7, VCPU_SIAR(r4) 3377 ld r8, VCPU_SDAR(r4) 3378 mtspr SPRN_MMCR1, r5 3379 mtspr SPRN_MMCRA, r6 3380 mtspr SPRN_SIAR, r7 3381 mtspr SPRN_SDAR, r8 3382BEGIN_FTR_SECTION 3383 ld r5, VCPU_MMCR + 24(r4) 3384 ld r6, VCPU_SIER(r4) 3385 mtspr SPRN_MMCR2, r5 3386 mtspr SPRN_SIER, r6 3387BEGIN_FTR_SECTION_NESTED(96) 3388 lwz r7, VCPU_PMC + 24(r4) 3389 lwz r8, VCPU_PMC + 28(r4) 3390 ld r9, VCPU_MMCR + 32(r4) 3391 mtspr SPRN_SPMC1, r7 3392 mtspr SPRN_SPMC2, r8 3393 mtspr SPRN_MMCRS, r9 3394END_FTR_SECTION_NESTED(CPU_FTR_ARCH_300, 0, 96) 3395END_FTR_SECTION_IFSET(CPU_FTR_ARCH_207S) 3396 mtspr SPRN_MMCR0, r3 3397 isync 3398 mtlr r0 3399 blr 3400 3401/* 3402 * Reload host PMU state saved in the PACA by kvmhv_save_host_pmu. 3403 */ 3404_GLOBAL(kvmhv_load_host_pmu) 3405EXPORT_SYMBOL_GPL(kvmhv_load_host_pmu) 3406 mflr r0 3407 lbz r4, PACA_PMCINUSE(r13) /* is the host using the PMU? */ 3408 cmpwi r4, 0 3409 beq 23f /* skip if not */ 3410BEGIN_FTR_SECTION 3411 ld r3, HSTATE_MMCR0(r13) 3412 andi. r4, r3, MMCR0_PMAO_SYNC | MMCR0_PMAO 3413 cmpwi r4, MMCR0_PMAO 3414 beql kvmppc_fix_pmao 3415END_FTR_SECTION_IFSET(CPU_FTR_PMAO_BUG) 3416 lwz r3, HSTATE_PMC1(r13) 3417 lwz r4, HSTATE_PMC2(r13) 3418 lwz r5, HSTATE_PMC3(r13) 3419 lwz r6, HSTATE_PMC4(r13) 3420 lwz r8, HSTATE_PMC5(r13) 3421 lwz r9, HSTATE_PMC6(r13) 3422 mtspr SPRN_PMC1, r3 3423 mtspr SPRN_PMC2, r4 3424 mtspr SPRN_PMC3, r5 3425 mtspr SPRN_PMC4, r6 3426 mtspr SPRN_PMC5, r8 3427 mtspr SPRN_PMC6, r9 3428 ld r3, HSTATE_MMCR0(r13) 3429 ld r4, HSTATE_MMCR1(r13) 3430 ld r5, HSTATE_MMCRA(r13) 3431 ld r6, HSTATE_SIAR(r13) 3432 ld r7, HSTATE_SDAR(r13) 3433 mtspr SPRN_MMCR1, r4 3434 mtspr SPRN_MMCRA, r5 3435 mtspr SPRN_SIAR, r6 3436 mtspr SPRN_SDAR, r7 3437BEGIN_FTR_SECTION 3438 ld r8, HSTATE_MMCR2(r13) 3439 ld r9, HSTATE_SIER(r13) 3440 mtspr SPRN_MMCR2, r8 3441 mtspr SPRN_SIER, r9 3442END_FTR_SECTION_IFSET(CPU_FTR_ARCH_207S) 3443 mtspr SPRN_MMCR0, r3 3444 isync 3445 mtlr r0 344623: blr 3447 3448/* 3449 * Save guest PMU state into the vcpu struct. 3450 * r3 = vcpu, r4 = full save flag (PMU in use flag set in VPA) 3451 */ 3452_GLOBAL(kvmhv_save_guest_pmu) 3453EXPORT_SYMBOL_GPL(kvmhv_save_guest_pmu) 3454 mr r9, r3 3455 mr r8, r4 3456BEGIN_FTR_SECTION 3457 /* 3458 * POWER8 seems to have a hardware bug where setting 3459 * MMCR0[PMAE] along with MMCR0[PMC1CE] and/or MMCR0[PMCjCE] 3460 * when some counters are already negative doesn't seem 3461 * to cause a performance monitor alert (and hence interrupt). 3462 * The effect of this is that when saving the PMU state, 3463 * if there is no PMU alert pending when we read MMCR0 3464 * before freezing the counters, but one becomes pending 3465 * before we read the counters, we lose it. 3466 * To work around this, we need a way to freeze the counters 3467 * before reading MMCR0. Normally, freezing the counters 3468 * is done by writing MMCR0 (to set MMCR0[FC]) which 3469 * unavoidably writes MMCR0[PMA0] as well. On POWER8, 3470 * we can also freeze the counters using MMCR2, by writing 3471 * 1s to all the counter freeze condition bits (there are 3472 * 9 bits each for 6 counters). 3473 */ 3474 li r3, -1 /* set all freeze bits */ 3475 clrrdi r3, r3, 10 3476 mfspr r10, SPRN_MMCR2 3477 mtspr SPRN_MMCR2, r3 3478 isync 3479END_FTR_SECTION_IFSET(CPU_FTR_ARCH_207S) 3480 li r3, 1 3481 sldi r3, r3, 31 /* MMCR0_FC (freeze counters) bit */ 3482 mfspr r4, SPRN_MMCR0 /* save MMCR0 */ 3483 mtspr SPRN_MMCR0, r3 /* freeze all counters, disable ints */ 3484 mfspr r6, SPRN_MMCRA 3485 /* Clear MMCRA in order to disable SDAR updates */ 3486 li r7, 0 3487 mtspr SPRN_MMCRA, r7 3488 isync 3489 cmpwi r8, 0 /* did they ask for PMU stuff to be saved? */ 3490 bne 21f 3491 std r3, VCPU_MMCR(r9) /* if not, set saved MMCR0 to FC */ 3492 b 22f 349321: mfspr r5, SPRN_MMCR1 3494 mfspr r7, SPRN_SIAR 3495 mfspr r8, SPRN_SDAR 3496 std r4, VCPU_MMCR(r9) 3497 std r5, VCPU_MMCR + 8(r9) 3498 std r6, VCPU_MMCR + 16(r9) 3499BEGIN_FTR_SECTION 3500 std r10, VCPU_MMCR + 24(r9) 3501END_FTR_SECTION_IFSET(CPU_FTR_ARCH_207S) 3502 std r7, VCPU_SIAR(r9) 3503 std r8, VCPU_SDAR(r9) 3504 mfspr r3, SPRN_PMC1 3505 mfspr r4, SPRN_PMC2 3506 mfspr r5, SPRN_PMC3 3507 mfspr r6, SPRN_PMC4 3508 mfspr r7, SPRN_PMC5 3509 mfspr r8, SPRN_PMC6 3510 stw r3, VCPU_PMC(r9) 3511 stw r4, VCPU_PMC + 4(r9) 3512 stw r5, VCPU_PMC + 8(r9) 3513 stw r6, VCPU_PMC + 12(r9) 3514 stw r7, VCPU_PMC + 16(r9) 3515 stw r8, VCPU_PMC + 20(r9) 3516BEGIN_FTR_SECTION 3517 mfspr r5, SPRN_SIER 3518 std r5, VCPU_SIER(r9) 3519BEGIN_FTR_SECTION_NESTED(96) 3520 mfspr r6, SPRN_SPMC1 3521 mfspr r7, SPRN_SPMC2 3522 mfspr r8, SPRN_MMCRS 3523 stw r6, VCPU_PMC + 24(r9) 3524 stw r7, VCPU_PMC + 28(r9) 3525 std r8, VCPU_MMCR + 32(r9) 3526 lis r4, 0x8000 3527 mtspr SPRN_MMCRS, r4 3528END_FTR_SECTION_NESTED(CPU_FTR_ARCH_300, 0, 96) 3529END_FTR_SECTION_IFSET(CPU_FTR_ARCH_207S) 353022: blr 3531 3532/* 3533 * This works around a hardware bug on POWER8E processors, where 3534 * writing a 1 to the MMCR0[PMAO] bit doesn't generate a 3535 * performance monitor interrupt. Instead, when we need to have 3536 * an interrupt pending, we have to arrange for a counter to overflow. 3537 */ 3538kvmppc_fix_pmao: 3539 li r3, 0 3540 mtspr SPRN_MMCR2, r3 3541 lis r3, (MMCR0_PMXE | MMCR0_FCECE)@h 3542 ori r3, r3, MMCR0_PMCjCE | MMCR0_C56RUN 3543 mtspr SPRN_MMCR0, r3 3544 lis r3, 0x7fff 3545 ori r3, r3, 0xffff 3546 mtspr SPRN_PMC6, r3 3547 isync 3548 blr 3549 3550#ifdef CONFIG_KVM_BOOK3S_HV_EXIT_TIMING 3551/* 3552 * Start timing an activity 3553 * r3 = pointer to time accumulation struct, r4 = vcpu 3554 */ 3555kvmhv_start_timing: 3556 ld r5, HSTATE_KVM_VCORE(r13) 3557 ld r6, VCORE_TB_OFFSET_APPL(r5) 3558 mftb r5 3559 subf r5, r6, r5 /* subtract current timebase offset */ 3560 std r3, VCPU_CUR_ACTIVITY(r4) 3561 std r5, VCPU_ACTIVITY_START(r4) 3562 blr 3563 3564/* 3565 * Accumulate time to one activity and start another. 3566 * r3 = pointer to new time accumulation struct, r4 = vcpu 3567 */ 3568kvmhv_accumulate_time: 3569 ld r5, HSTATE_KVM_VCORE(r13) 3570 ld r8, VCORE_TB_OFFSET_APPL(r5) 3571 ld r5, VCPU_CUR_ACTIVITY(r4) 3572 ld r6, VCPU_ACTIVITY_START(r4) 3573 std r3, VCPU_CUR_ACTIVITY(r4) 3574 mftb r7 3575 subf r7, r8, r7 /* subtract current timebase offset */ 3576 std r7, VCPU_ACTIVITY_START(r4) 3577 cmpdi r5, 0 3578 beqlr 3579 subf r3, r6, r7 3580 ld r8, TAS_SEQCOUNT(r5) 3581 cmpdi r8, 0 3582 addi r8, r8, 1 3583 std r8, TAS_SEQCOUNT(r5) 3584 lwsync 3585 ld r7, TAS_TOTAL(r5) 3586 add r7, r7, r3 3587 std r7, TAS_TOTAL(r5) 3588 ld r6, TAS_MIN(r5) 3589 ld r7, TAS_MAX(r5) 3590 beq 3f 3591 cmpd r3, r6 3592 bge 1f 35933: std r3, TAS_MIN(r5) 35941: cmpd r3, r7 3595 ble 2f 3596 std r3, TAS_MAX(r5) 35972: lwsync 3598 addi r8, r8, 1 3599 std r8, TAS_SEQCOUNT(r5) 3600 blr 3601#endif 3602