1/* SPDX-License-Identifier: GPL-2.0-only */ 2/* 3 * 4 * Copyright 2011 Paul Mackerras, IBM Corp. <paulus@au1.ibm.com> 5 * 6 * Derived from book3s_rmhandlers.S and other files, which are: 7 * 8 * Copyright SUSE Linux Products GmbH 2009 9 * 10 * Authors: Alexander Graf <agraf@suse.de> 11 */ 12 13#include <asm/ppc_asm.h> 14#include <asm/code-patching-asm.h> 15#include <asm/kvm_asm.h> 16#include <asm/reg.h> 17#include <asm/mmu.h> 18#include <asm/page.h> 19#include <asm/ptrace.h> 20#include <asm/hvcall.h> 21#include <asm/asm-offsets.h> 22#include <asm/exception-64s.h> 23#include <asm/kvm_book3s_asm.h> 24#include <asm/book3s/64/mmu-hash.h> 25#include <asm/export.h> 26#include <asm/tm.h> 27#include <asm/opal.h> 28#include <asm/thread_info.h> 29#include <asm/asm-compat.h> 30#include <asm/feature-fixups.h> 31#include <asm/cpuidle.h> 32 33/* Values in HSTATE_NAPPING(r13) */ 34#define NAPPING_CEDE 1 35#define NAPPING_NOVCPU 2 36#define NAPPING_UNSPLIT 3 37 38/* Stack frame offsets for kvmppc_hv_entry */ 39#define SFS 160 40#define STACK_SLOT_TRAP (SFS-4) 41#define STACK_SLOT_TID (SFS-16) 42#define STACK_SLOT_PSSCR (SFS-24) 43#define STACK_SLOT_PID (SFS-32) 44#define STACK_SLOT_IAMR (SFS-40) 45#define STACK_SLOT_CIABR (SFS-48) 46#define STACK_SLOT_DAWR0 (SFS-56) 47#define STACK_SLOT_DAWRX0 (SFS-64) 48#define STACK_SLOT_HFSCR (SFS-72) 49#define STACK_SLOT_AMR (SFS-80) 50#define STACK_SLOT_UAMOR (SFS-88) 51#define STACK_SLOT_FSCR (SFS-96) 52 53/* 54 * Call kvmppc_hv_entry in real mode. 55 * Must be called with interrupts hard-disabled. 56 * 57 * Input Registers: 58 * 59 * LR = return address to continue at after eventually re-enabling MMU 60 */ 61_GLOBAL_TOC(kvmppc_hv_entry_trampoline) 62 mflr r0 63 std r0, PPC_LR_STKOFF(r1) 64 stdu r1, -112(r1) 65 mfmsr r10 66 std r10, HSTATE_HOST_MSR(r13) 67 LOAD_REG_ADDR(r5, kvmppc_call_hv_entry) 68 li r0,MSR_RI 69 andc r0,r10,r0 70 li r6,MSR_IR | MSR_DR 71 andc r6,r10,r6 72 mtmsrd r0,1 /* clear RI in MSR */ 73 mtsrr0 r5 74 mtsrr1 r6 75 RFI_TO_KERNEL 76 77kvmppc_call_hv_entry: 78 ld r4, HSTATE_KVM_VCPU(r13) 79 bl kvmppc_hv_entry 80 81 /* Back from guest - restore host state and return to caller */ 82 83BEGIN_FTR_SECTION 84 /* Restore host DABR and DABRX */ 85 ld r5,HSTATE_DABR(r13) 86 li r6,7 87 mtspr SPRN_DABR,r5 88 mtspr SPRN_DABRX,r6 89END_FTR_SECTION_IFCLR(CPU_FTR_ARCH_207S) 90 91 /* Restore SPRG3 */ 92 ld r3,PACA_SPRG_VDSO(r13) 93 mtspr SPRN_SPRG_VDSO_WRITE,r3 94 95 /* Reload the host's PMU registers */ 96 bl kvmhv_load_host_pmu 97 98 /* 99 * Reload DEC. HDEC interrupts were disabled when 100 * we reloaded the host's LPCR value. 101 */ 102 ld r3, HSTATE_DECEXP(r13) 103 mftb r4 104 subf r4, r4, r3 105 mtspr SPRN_DEC, r4 106 107 /* hwthread_req may have got set by cede or no vcpu, so clear it */ 108 li r0, 0 109 stb r0, HSTATE_HWTHREAD_REQ(r13) 110 111 /* 112 * For external interrupts we need to call the Linux 113 * handler to process the interrupt. We do that by jumping 114 * to absolute address 0x500 for external interrupts. 115 * The [h]rfid at the end of the handler will return to 116 * the book3s_hv_interrupts.S code. For other interrupts 117 * we do the rfid to get back to the book3s_hv_interrupts.S 118 * code here. 119 */ 120 ld r8, 112+PPC_LR_STKOFF(r1) 121 addi r1, r1, 112 122 ld r7, HSTATE_HOST_MSR(r13) 123 124 /* Return the trap number on this thread as the return value */ 125 mr r3, r12 126 127 /* RFI into the highmem handler */ 128 mfmsr r6 129 li r0, MSR_RI 130 andc r6, r6, r0 131 mtmsrd r6, 1 /* Clear RI in MSR */ 132 mtsrr0 r8 133 mtsrr1 r7 134 RFI_TO_KERNEL 135 136kvmppc_primary_no_guest: 137 /* We handle this much like a ceded vcpu */ 138 /* put the HDEC into the DEC, since HDEC interrupts don't wake us */ 139 /* HDEC may be larger than DEC for arch >= v3.00, but since the */ 140 /* HDEC value came from DEC in the first place, it will fit */ 141 mfspr r3, SPRN_HDEC 142 mtspr SPRN_DEC, r3 143 /* 144 * Make sure the primary has finished the MMU switch. 145 * We should never get here on a secondary thread, but 146 * check it for robustness' sake. 147 */ 148 ld r5, HSTATE_KVM_VCORE(r13) 14965: lbz r0, VCORE_IN_GUEST(r5) 150 cmpwi r0, 0 151 beq 65b 152 /* Set LPCR. */ 153 ld r8,VCORE_LPCR(r5) 154 mtspr SPRN_LPCR,r8 155 isync 156 /* set our bit in napping_threads */ 157 ld r5, HSTATE_KVM_VCORE(r13) 158 lbz r7, HSTATE_PTID(r13) 159 li r0, 1 160 sld r0, r0, r7 161 addi r6, r5, VCORE_NAPPING_THREADS 1621: lwarx r3, 0, r6 163 or r3, r3, r0 164 stwcx. r3, 0, r6 165 bne 1b 166 /* order napping_threads update vs testing entry_exit_map */ 167 isync 168 li r12, 0 169 lwz r7, VCORE_ENTRY_EXIT(r5) 170 cmpwi r7, 0x100 171 bge kvm_novcpu_exit /* another thread already exiting */ 172 li r3, NAPPING_NOVCPU 173 stb r3, HSTATE_NAPPING(r13) 174 175 li r3, 0 /* Don't wake on privileged (OS) doorbell */ 176 b kvm_do_nap 177 178/* 179 * kvm_novcpu_wakeup 180 * Entered from kvm_start_guest if kvm_hstate.napping is set 181 * to NAPPING_NOVCPU 182 * r2 = kernel TOC 183 * r13 = paca 184 */ 185kvm_novcpu_wakeup: 186 ld r1, HSTATE_HOST_R1(r13) 187 ld r5, HSTATE_KVM_VCORE(r13) 188 li r0, 0 189 stb r0, HSTATE_NAPPING(r13) 190 191 /* check the wake reason */ 192 bl kvmppc_check_wake_reason 193 194 /* 195 * Restore volatile registers since we could have called 196 * a C routine in kvmppc_check_wake_reason. 197 * r5 = VCORE 198 */ 199 ld r5, HSTATE_KVM_VCORE(r13) 200 201 /* see if any other thread is already exiting */ 202 lwz r0, VCORE_ENTRY_EXIT(r5) 203 cmpwi r0, 0x100 204 bge kvm_novcpu_exit 205 206 /* clear our bit in napping_threads */ 207 lbz r7, HSTATE_PTID(r13) 208 li r0, 1 209 sld r0, r0, r7 210 addi r6, r5, VCORE_NAPPING_THREADS 2114: lwarx r7, 0, r6 212 andc r7, r7, r0 213 stwcx. r7, 0, r6 214 bne 4b 215 216 /* See if the wake reason means we need to exit */ 217 cmpdi r3, 0 218 bge kvm_novcpu_exit 219 220 /* See if our timeslice has expired (HDEC is negative) */ 221 mfspr r0, SPRN_HDEC 222 extsw r0, r0 223 li r12, BOOK3S_INTERRUPT_HV_DECREMENTER 224 cmpdi r0, 0 225 blt kvm_novcpu_exit 226 227 /* Got an IPI but other vcpus aren't yet exiting, must be a latecomer */ 228 ld r4, HSTATE_KVM_VCPU(r13) 229 cmpdi r4, 0 230 beq kvmppc_primary_no_guest 231 232#ifdef CONFIG_KVM_BOOK3S_HV_EXIT_TIMING 233 addi r3, r4, VCPU_TB_RMENTRY 234 bl kvmhv_start_timing 235#endif 236 b kvmppc_got_guest 237 238kvm_novcpu_exit: 239#ifdef CONFIG_KVM_BOOK3S_HV_EXIT_TIMING 240 ld r4, HSTATE_KVM_VCPU(r13) 241 cmpdi r4, 0 242 beq 13f 243 addi r3, r4, VCPU_TB_RMEXIT 244 bl kvmhv_accumulate_time 245#endif 24613: mr r3, r12 247 stw r12, STACK_SLOT_TRAP(r1) 248 bl kvmhv_commence_exit 249 nop 250 b kvmhv_switch_to_host 251 252/* 253 * We come in here when wakened from Linux offline idle code. 254 * Relocation is off 255 * r3 contains the SRR1 wakeup value, SRR1 is trashed. 256 */ 257_GLOBAL(idle_kvm_start_guest) 258 ld r4,PACAEMERGSP(r13) 259 mfcr r5 260 mflr r0 261 std r1,0(r4) 262 std r5,8(r4) 263 std r0,16(r4) 264 subi r1,r4,STACK_FRAME_OVERHEAD 265 SAVE_NVGPRS(r1) 266 267 /* 268 * Could avoid this and pass it through in r3. For now, 269 * code expects it to be in SRR1. 270 */ 271 mtspr SPRN_SRR1,r3 272 273 li r0,0 274 stb r0,PACA_FTRACE_ENABLED(r13) 275 276 li r0,KVM_HWTHREAD_IN_KVM 277 stb r0,HSTATE_HWTHREAD_STATE(r13) 278 279 /* kvm cede / napping does not come through here */ 280 lbz r0,HSTATE_NAPPING(r13) 281 twnei r0,0 282 283 b 1f 284 285kvm_unsplit_wakeup: 286 li r0, 0 287 stb r0, HSTATE_NAPPING(r13) 288 2891: 290 291 /* 292 * We weren't napping due to cede, so this must be a secondary 293 * thread being woken up to run a guest, or being woken up due 294 * to a stray IPI. (Or due to some machine check or hypervisor 295 * maintenance interrupt while the core is in KVM.) 296 */ 297 298 /* Check the wake reason in SRR1 to see why we got here */ 299 bl kvmppc_check_wake_reason 300 /* 301 * kvmppc_check_wake_reason could invoke a C routine, but we 302 * have no volatile registers to restore when we return. 303 */ 304 305 cmpdi r3, 0 306 bge kvm_no_guest 307 308 /* get vcore pointer, NULL if we have nothing to run */ 309 ld r5,HSTATE_KVM_VCORE(r13) 310 cmpdi r5,0 311 /* if we have no vcore to run, go back to sleep */ 312 beq kvm_no_guest 313 314kvm_secondary_got_guest: 315 316 /* Set HSTATE_DSCR(r13) to something sensible */ 317 ld r6, PACA_DSCR_DEFAULT(r13) 318 std r6, HSTATE_DSCR(r13) 319 320 /* On thread 0 of a subcore, set HDEC to max */ 321 lbz r4, HSTATE_PTID(r13) 322 cmpwi r4, 0 323 bne 63f 324 lis r6,0x7fff /* MAX_INT@h */ 325 mtspr SPRN_HDEC, r6 326 /* and set per-LPAR registers, if doing dynamic micro-threading */ 327 ld r6, HSTATE_SPLIT_MODE(r13) 328 cmpdi r6, 0 329 beq 63f 330 ld r0, KVM_SPLIT_RPR(r6) 331 mtspr SPRN_RPR, r0 332 ld r0, KVM_SPLIT_PMMAR(r6) 333 mtspr SPRN_PMMAR, r0 334 ld r0, KVM_SPLIT_LDBAR(r6) 335 mtspr SPRN_LDBAR, r0 336 isync 33763: 338 /* Order load of vcpu after load of vcore */ 339 lwsync 340 ld r4, HSTATE_KVM_VCPU(r13) 341 bl kvmppc_hv_entry 342 343 /* Back from the guest, go back to nap */ 344 /* Clear our vcpu and vcore pointers so we don't come back in early */ 345 li r0, 0 346 std r0, HSTATE_KVM_VCPU(r13) 347 /* 348 * Once we clear HSTATE_KVM_VCORE(r13), the code in 349 * kvmppc_run_core() is going to assume that all our vcpu 350 * state is visible in memory. This lwsync makes sure 351 * that that is true. 352 */ 353 lwsync 354 std r0, HSTATE_KVM_VCORE(r13) 355 356 /* 357 * All secondaries exiting guest will fall through this path. 358 * Before proceeding, just check for HMI interrupt and 359 * invoke opal hmi handler. By now we are sure that the 360 * primary thread on this core/subcore has already made partition 361 * switch/TB resync and we are good to call opal hmi handler. 362 */ 363 cmpwi r12, BOOK3S_INTERRUPT_HMI 364 bne kvm_no_guest 365 366 li r3,0 /* NULL argument */ 367 bl hmi_exception_realmode 368/* 369 * At this point we have finished executing in the guest. 370 * We need to wait for hwthread_req to become zero, since 371 * we may not turn on the MMU while hwthread_req is non-zero. 372 * While waiting we also need to check if we get given a vcpu to run. 373 */ 374kvm_no_guest: 375 lbz r3, HSTATE_HWTHREAD_REQ(r13) 376 cmpwi r3, 0 377 bne 53f 378 HMT_MEDIUM 379 li r0, KVM_HWTHREAD_IN_KERNEL 380 stb r0, HSTATE_HWTHREAD_STATE(r13) 381 /* need to recheck hwthread_req after a barrier, to avoid race */ 382 sync 383 lbz r3, HSTATE_HWTHREAD_REQ(r13) 384 cmpwi r3, 0 385 bne 54f 386 387 /* 388 * Jump to idle_return_gpr_loss, which returns to the 389 * idle_kvm_start_guest caller. 390 */ 391 li r3, LPCR_PECE0 392 mfspr r4, SPRN_LPCR 393 rlwimi r4, r3, 0, LPCR_PECE0 | LPCR_PECE1 394 mtspr SPRN_LPCR, r4 395 /* set up r3 for return */ 396 mfspr r3,SPRN_SRR1 397 REST_NVGPRS(r1) 398 addi r1, r1, STACK_FRAME_OVERHEAD 399 ld r0, 16(r1) 400 ld r5, 8(r1) 401 ld r1, 0(r1) 402 mtlr r0 403 mtcr r5 404 blr 405 40653: 407 HMT_LOW 408 ld r5, HSTATE_KVM_VCORE(r13) 409 cmpdi r5, 0 410 bne 60f 411 ld r3, HSTATE_SPLIT_MODE(r13) 412 cmpdi r3, 0 413 beq kvm_no_guest 414 lbz r0, KVM_SPLIT_DO_NAP(r3) 415 cmpwi r0, 0 416 beq kvm_no_guest 417 HMT_MEDIUM 418 b kvm_unsplit_nap 41960: HMT_MEDIUM 420 b kvm_secondary_got_guest 421 42254: li r0, KVM_HWTHREAD_IN_KVM 423 stb r0, HSTATE_HWTHREAD_STATE(r13) 424 b kvm_no_guest 425 426/* 427 * Here the primary thread is trying to return the core to 428 * whole-core mode, so we need to nap. 429 */ 430kvm_unsplit_nap: 431 /* 432 * When secondaries are napping in kvm_unsplit_nap() with 433 * hwthread_req = 1, HMI goes ignored even though subcores are 434 * already exited the guest. Hence HMI keeps waking up secondaries 435 * from nap in a loop and secondaries always go back to nap since 436 * no vcore is assigned to them. This makes impossible for primary 437 * thread to get hold of secondary threads resulting into a soft 438 * lockup in KVM path. 439 * 440 * Let us check if HMI is pending and handle it before we go to nap. 441 */ 442 cmpwi r12, BOOK3S_INTERRUPT_HMI 443 bne 55f 444 li r3, 0 /* NULL argument */ 445 bl hmi_exception_realmode 44655: 447 /* 448 * Ensure that secondary doesn't nap when it has 449 * its vcore pointer set. 450 */ 451 sync /* matches smp_mb() before setting split_info.do_nap */ 452 ld r0, HSTATE_KVM_VCORE(r13) 453 cmpdi r0, 0 454 bne kvm_no_guest 455 /* clear any pending message */ 456BEGIN_FTR_SECTION 457 lis r6, (PPC_DBELL_SERVER << (63-36))@h 458 PPC_MSGCLR(6) 459END_FTR_SECTION_IFSET(CPU_FTR_ARCH_207S) 460 /* Set kvm_split_mode.napped[tid] = 1 */ 461 ld r3, HSTATE_SPLIT_MODE(r13) 462 li r0, 1 463 lhz r4, PACAPACAINDEX(r13) 464 clrldi r4, r4, 61 /* micro-threading => P8 => 8 threads/core */ 465 addi r4, r4, KVM_SPLIT_NAPPED 466 stbx r0, r3, r4 467 /* Check the do_nap flag again after setting napped[] */ 468 sync 469 lbz r0, KVM_SPLIT_DO_NAP(r3) 470 cmpwi r0, 0 471 beq 57f 472 li r3, NAPPING_UNSPLIT 473 stb r3, HSTATE_NAPPING(r13) 474 li r3, (LPCR_PECEDH | LPCR_PECE0) >> 4 475 mfspr r5, SPRN_LPCR 476 rlwimi r5, r3, 4, (LPCR_PECEDP | LPCR_PECEDH | LPCR_PECE0 | LPCR_PECE1) 477 b kvm_nap_sequence 478 47957: li r0, 0 480 stbx r0, r3, r4 481 b kvm_no_guest 482 483/****************************************************************************** 484 * * 485 * Entry code * 486 * * 487 *****************************************************************************/ 488 489.global kvmppc_hv_entry 490kvmppc_hv_entry: 491 492 /* Required state: 493 * 494 * R4 = vcpu pointer (or NULL) 495 * MSR = ~IR|DR 496 * R13 = PACA 497 * R1 = host R1 498 * R2 = TOC 499 * all other volatile GPRS = free 500 * Does not preserve non-volatile GPRs or CR fields 501 */ 502 mflr r0 503 std r0, PPC_LR_STKOFF(r1) 504 stdu r1, -SFS(r1) 505 506 /* Save R1 in the PACA */ 507 std r1, HSTATE_HOST_R1(r13) 508 509 li r6, KVM_GUEST_MODE_HOST_HV 510 stb r6, HSTATE_IN_GUEST(r13) 511 512#ifdef CONFIG_KVM_BOOK3S_HV_EXIT_TIMING 513 /* Store initial timestamp */ 514 cmpdi r4, 0 515 beq 1f 516 addi r3, r4, VCPU_TB_RMENTRY 517 bl kvmhv_start_timing 5181: 519#endif 520 521 ld r5, HSTATE_KVM_VCORE(r13) 522 ld r9, VCORE_KVM(r5) /* pointer to struct kvm */ 523 524 /* 525 * POWER7/POWER8 host -> guest partition switch code. 526 * We don't have to lock against concurrent tlbies, 527 * but we do have to coordinate across hardware threads. 528 */ 529 /* Set bit in entry map iff exit map is zero. */ 530 li r7, 1 531 lbz r6, HSTATE_PTID(r13) 532 sld r7, r7, r6 533 addi r8, r5, VCORE_ENTRY_EXIT 53421: lwarx r3, 0, r8 535 cmpwi r3, 0x100 /* any threads starting to exit? */ 536 bge secondary_too_late /* if so we're too late to the party */ 537 or r3, r3, r7 538 stwcx. r3, 0, r8 539 bne 21b 540 541 /* Primary thread switches to guest partition. */ 542 cmpwi r6,0 543 bne 10f 544 545 lwz r7,KVM_LPID(r9) 546 ld r6,KVM_SDR1(r9) 547 li r0,LPID_RSVD /* switch to reserved LPID */ 548 mtspr SPRN_LPID,r0 549 ptesync 550 mtspr SPRN_SDR1,r6 /* switch to partition page table */ 551 mtspr SPRN_LPID,r7 552 isync 553 554 /* See if we need to flush the TLB. */ 555 mr r3, r9 /* kvm pointer */ 556 lhz r4, PACAPACAINDEX(r13) /* physical cpu number */ 557 li r5, 0 /* nested vcpu pointer */ 558 bl kvmppc_check_need_tlb_flush 559 nop 560 ld r5, HSTATE_KVM_VCORE(r13) 561 562 /* Add timebase offset onto timebase */ 56322: ld r8,VCORE_TB_OFFSET(r5) 564 cmpdi r8,0 565 beq 37f 566 std r8, VCORE_TB_OFFSET_APPL(r5) 567 mftb r6 /* current host timebase */ 568 add r8,r8,r6 569 mtspr SPRN_TBU40,r8 /* update upper 40 bits */ 570 mftb r7 /* check if lower 24 bits overflowed */ 571 clrldi r6,r6,40 572 clrldi r7,r7,40 573 cmpld r7,r6 574 bge 37f 575 addis r8,r8,0x100 /* if so, increment upper 40 bits */ 576 mtspr SPRN_TBU40,r8 577 578 /* Load guest PCR value to select appropriate compat mode */ 57937: ld r7, VCORE_PCR(r5) 580 LOAD_REG_IMMEDIATE(r6, PCR_MASK) 581 cmpld r7, r6 582 beq 38f 583 or r7, r7, r6 584 mtspr SPRN_PCR, r7 58538: 586 587BEGIN_FTR_SECTION 588 /* DPDES and VTB are shared between threads */ 589 ld r8, VCORE_DPDES(r5) 590 ld r7, VCORE_VTB(r5) 591 mtspr SPRN_DPDES, r8 592 mtspr SPRN_VTB, r7 593END_FTR_SECTION_IFSET(CPU_FTR_ARCH_207S) 594 595 /* Mark the subcore state as inside guest */ 596 bl kvmppc_subcore_enter_guest 597 nop 598 ld r5, HSTATE_KVM_VCORE(r13) 599 ld r4, HSTATE_KVM_VCPU(r13) 600 li r0,1 601 stb r0,VCORE_IN_GUEST(r5) /* signal secondaries to continue */ 602 603 /* Do we have a guest vcpu to run? */ 60410: cmpdi r4, 0 605 beq kvmppc_primary_no_guest 606kvmppc_got_guest: 607 /* Increment yield count if they have a VPA */ 608 ld r3, VCPU_VPA(r4) 609 cmpdi r3, 0 610 beq 25f 611 li r6, LPPACA_YIELDCOUNT 612 LWZX_BE r5, r3, r6 613 addi r5, r5, 1 614 STWX_BE r5, r3, r6 615 li r6, 1 616 stb r6, VCPU_VPA_DIRTY(r4) 61725: 618 619 /* Save purr/spurr */ 620 mfspr r5,SPRN_PURR 621 mfspr r6,SPRN_SPURR 622 std r5,HSTATE_PURR(r13) 623 std r6,HSTATE_SPURR(r13) 624 ld r7,VCPU_PURR(r4) 625 ld r8,VCPU_SPURR(r4) 626 mtspr SPRN_PURR,r7 627 mtspr SPRN_SPURR,r8 628 629 /* Save host values of some registers */ 630BEGIN_FTR_SECTION 631 mfspr r5, SPRN_CIABR 632 mfspr r6, SPRN_DAWR0 633 mfspr r7, SPRN_DAWRX0 634 mfspr r8, SPRN_IAMR 635 std r5, STACK_SLOT_CIABR(r1) 636 std r6, STACK_SLOT_DAWR0(r1) 637 std r7, STACK_SLOT_DAWRX0(r1) 638 std r8, STACK_SLOT_IAMR(r1) 639 mfspr r5, SPRN_FSCR 640 std r5, STACK_SLOT_FSCR(r1) 641END_FTR_SECTION_IFSET(CPU_FTR_ARCH_207S) 642 643 mfspr r5, SPRN_AMR 644 std r5, STACK_SLOT_AMR(r1) 645 mfspr r6, SPRN_UAMOR 646 std r6, STACK_SLOT_UAMOR(r1) 647 648BEGIN_FTR_SECTION 649 /* Set partition DABR */ 650 /* Do this before re-enabling PMU to avoid P7 DABR corruption bug */ 651 lwz r5,VCPU_DABRX(r4) 652 ld r6,VCPU_DABR(r4) 653 mtspr SPRN_DABRX,r5 654 mtspr SPRN_DABR,r6 655 isync 656END_FTR_SECTION_IFCLR(CPU_FTR_ARCH_207S) 657 658#ifdef CONFIG_PPC_TRANSACTIONAL_MEM 659BEGIN_FTR_SECTION 660 b 91f 661END_FTR_SECTION_IFCLR(CPU_FTR_TM) 662 /* 663 * NOTE THAT THIS TRASHES ALL NON-VOLATILE REGISTERS (but not CR) 664 */ 665 mr r3, r4 666 ld r4, VCPU_MSR(r3) 667 li r5, 0 /* don't preserve non-vol regs */ 668 bl kvmppc_restore_tm_hv 669 nop 670 ld r4, HSTATE_KVM_VCPU(r13) 67191: 672#endif 673 674 /* Load guest PMU registers; r4 = vcpu pointer here */ 675 mr r3, r4 676 bl kvmhv_load_guest_pmu 677 678 /* Load up FP, VMX and VSX registers */ 679 ld r4, HSTATE_KVM_VCPU(r13) 680 bl kvmppc_load_fp 681 682 ld r14, VCPU_GPR(R14)(r4) 683 ld r15, VCPU_GPR(R15)(r4) 684 ld r16, VCPU_GPR(R16)(r4) 685 ld r17, VCPU_GPR(R17)(r4) 686 ld r18, VCPU_GPR(R18)(r4) 687 ld r19, VCPU_GPR(R19)(r4) 688 ld r20, VCPU_GPR(R20)(r4) 689 ld r21, VCPU_GPR(R21)(r4) 690 ld r22, VCPU_GPR(R22)(r4) 691 ld r23, VCPU_GPR(R23)(r4) 692 ld r24, VCPU_GPR(R24)(r4) 693 ld r25, VCPU_GPR(R25)(r4) 694 ld r26, VCPU_GPR(R26)(r4) 695 ld r27, VCPU_GPR(R27)(r4) 696 ld r28, VCPU_GPR(R28)(r4) 697 ld r29, VCPU_GPR(R29)(r4) 698 ld r30, VCPU_GPR(R30)(r4) 699 ld r31, VCPU_GPR(R31)(r4) 700 701 /* Switch DSCR to guest value */ 702 ld r5, VCPU_DSCR(r4) 703 mtspr SPRN_DSCR, r5 704 705BEGIN_FTR_SECTION 706 /* Skip next section on POWER7 */ 707 b 8f 708END_FTR_SECTION_IFCLR(CPU_FTR_ARCH_207S) 709 /* Load up POWER8-specific registers */ 710 ld r5, VCPU_IAMR(r4) 711 lwz r6, VCPU_PSPB(r4) 712 ld r7, VCPU_FSCR(r4) 713 mtspr SPRN_IAMR, r5 714 mtspr SPRN_PSPB, r6 715 mtspr SPRN_FSCR, r7 716 /* 717 * Handle broken DAWR case by not writing it. This means we 718 * can still store the DAWR register for migration. 719 */ 720 LOAD_REG_ADDR(r5, dawr_force_enable) 721 lbz r5, 0(r5) 722 cmpdi r5, 0 723 beq 1f 724 ld r5, VCPU_DAWR0(r4) 725 ld r6, VCPU_DAWRX0(r4) 726 mtspr SPRN_DAWR0, r5 727 mtspr SPRN_DAWRX0, r6 7281: 729 ld r7, VCPU_CIABR(r4) 730 ld r8, VCPU_TAR(r4) 731 mtspr SPRN_CIABR, r7 732 mtspr SPRN_TAR, r8 733 ld r5, VCPU_IC(r4) 734 ld r8, VCPU_EBBHR(r4) 735 mtspr SPRN_IC, r5 736 mtspr SPRN_EBBHR, r8 737 ld r5, VCPU_EBBRR(r4) 738 ld r6, VCPU_BESCR(r4) 739 lwz r7, VCPU_GUEST_PID(r4) 740 ld r8, VCPU_WORT(r4) 741 mtspr SPRN_EBBRR, r5 742 mtspr SPRN_BESCR, r6 743 mtspr SPRN_PID, r7 744 mtspr SPRN_WORT, r8 745 /* POWER8-only registers */ 746 ld r5, VCPU_TCSCR(r4) 747 ld r6, VCPU_ACOP(r4) 748 ld r7, VCPU_CSIGR(r4) 749 ld r8, VCPU_TACR(r4) 750 mtspr SPRN_TCSCR, r5 751 mtspr SPRN_ACOP, r6 752 mtspr SPRN_CSIGR, r7 753 mtspr SPRN_TACR, r8 754 nop 7558: 756 757 ld r5, VCPU_SPRG0(r4) 758 ld r6, VCPU_SPRG1(r4) 759 ld r7, VCPU_SPRG2(r4) 760 ld r8, VCPU_SPRG3(r4) 761 mtspr SPRN_SPRG0, r5 762 mtspr SPRN_SPRG1, r6 763 mtspr SPRN_SPRG2, r7 764 mtspr SPRN_SPRG3, r8 765 766 /* Load up DAR and DSISR */ 767 ld r5, VCPU_DAR(r4) 768 lwz r6, VCPU_DSISR(r4) 769 mtspr SPRN_DAR, r5 770 mtspr SPRN_DSISR, r6 771 772 /* Restore AMR and UAMOR, set AMOR to all 1s */ 773 ld r5,VCPU_AMR(r4) 774 ld r6,VCPU_UAMOR(r4) 775 li r7,-1 776 mtspr SPRN_AMR,r5 777 mtspr SPRN_UAMOR,r6 778 mtspr SPRN_AMOR,r7 779 780 /* Restore state of CTRL run bit; assume 1 on entry */ 781 lwz r5,VCPU_CTRL(r4) 782 andi. r5,r5,1 783 bne 4f 784 mfspr r6,SPRN_CTRLF 785 clrrdi r6,r6,1 786 mtspr SPRN_CTRLT,r6 7874: 788 /* Secondary threads wait for primary to have done partition switch */ 789 ld r5, HSTATE_KVM_VCORE(r13) 790 lbz r6, HSTATE_PTID(r13) 791 cmpwi r6, 0 792 beq 21f 793 lbz r0, VCORE_IN_GUEST(r5) 794 cmpwi r0, 0 795 bne 21f 796 HMT_LOW 79720: lwz r3, VCORE_ENTRY_EXIT(r5) 798 cmpwi r3, 0x100 799 bge no_switch_exit 800 lbz r0, VCORE_IN_GUEST(r5) 801 cmpwi r0, 0 802 beq 20b 803 HMT_MEDIUM 80421: 805 /* Set LPCR. */ 806 ld r8,VCORE_LPCR(r5) 807 mtspr SPRN_LPCR,r8 808 isync 809 810 /* 811 * Set the decrementer to the guest decrementer. 812 */ 813 ld r8,VCPU_DEC_EXPIRES(r4) 814 /* r8 is a host timebase value here, convert to guest TB */ 815 ld r5,HSTATE_KVM_VCORE(r13) 816 ld r6,VCORE_TB_OFFSET_APPL(r5) 817 add r8,r8,r6 818 mftb r7 819 subf r3,r7,r8 820 mtspr SPRN_DEC,r3 821 822 /* Check if HDEC expires soon */ 823 mfspr r3, SPRN_HDEC 824 extsw r3, r3 825 cmpdi r3, 512 /* 1 microsecond */ 826 blt hdec_soon 827 828 /* Clear out and reload the SLB */ 829 li r6, 0 830 slbmte r6, r6 831 PPC_SLBIA(6) 832 ptesync 833 834 /* Load up guest SLB entries (N.B. slb_max will be 0 for radix) */ 835 lwz r5,VCPU_SLB_MAX(r4) 836 cmpwi r5,0 837 beq 9f 838 mtctr r5 839 addi r6,r4,VCPU_SLB 8401: ld r8,VCPU_SLB_E(r6) 841 ld r9,VCPU_SLB_V(r6) 842 slbmte r9,r8 843 addi r6,r6,VCPU_SLB_SIZE 844 bdnz 1b 8459: 846 847deliver_guest_interrupt: /* r4 = vcpu, r13 = paca */ 848 /* Check if we can deliver an external or decrementer interrupt now */ 849 ld r0, VCPU_PENDING_EXC(r4) 850 cmpdi r0, 0 851 beq 71f 852 mr r3, r4 853 bl kvmppc_guest_entry_inject_int 854 ld r4, HSTATE_KVM_VCPU(r13) 85571: 856 ld r6, VCPU_SRR0(r4) 857 ld r7, VCPU_SRR1(r4) 858 mtspr SPRN_SRR0, r6 859 mtspr SPRN_SRR1, r7 860 861 ld r10, VCPU_PC(r4) 862 ld r11, VCPU_MSR(r4) 863 /* r11 = vcpu->arch.msr & ~MSR_HV */ 864 rldicl r11, r11, 63 - MSR_HV_LG, 1 865 rotldi r11, r11, 1 + MSR_HV_LG 866 ori r11, r11, MSR_ME 867 868 ld r6, VCPU_CTR(r4) 869 ld r7, VCPU_XER(r4) 870 mtctr r6 871 mtxer r7 872 873/* 874 * Required state: 875 * R4 = vcpu 876 * R10: value for HSRR0 877 * R11: value for HSRR1 878 * R13 = PACA 879 */ 880fast_guest_return: 881 li r0,0 882 stb r0,VCPU_CEDED(r4) /* cancel cede */ 883 mtspr SPRN_HSRR0,r10 884 mtspr SPRN_HSRR1,r11 885 886 /* Activate guest mode, so faults get handled by KVM */ 887 li r9, KVM_GUEST_MODE_GUEST_HV 888 stb r9, HSTATE_IN_GUEST(r13) 889 890#ifdef CONFIG_KVM_BOOK3S_HV_EXIT_TIMING 891 /* Accumulate timing */ 892 addi r3, r4, VCPU_TB_GUEST 893 bl kvmhv_accumulate_time 894#endif 895 896 /* Enter guest */ 897 898BEGIN_FTR_SECTION 899 ld r5, VCPU_CFAR(r4) 900 mtspr SPRN_CFAR, r5 901END_FTR_SECTION_IFSET(CPU_FTR_CFAR) 902BEGIN_FTR_SECTION 903 ld r0, VCPU_PPR(r4) 904END_FTR_SECTION_IFSET(CPU_FTR_HAS_PPR) 905 906 ld r5, VCPU_LR(r4) 907 mtlr r5 908 909 ld r1, VCPU_GPR(R1)(r4) 910 ld r5, VCPU_GPR(R5)(r4) 911 ld r8, VCPU_GPR(R8)(r4) 912 ld r9, VCPU_GPR(R9)(r4) 913 ld r10, VCPU_GPR(R10)(r4) 914 ld r11, VCPU_GPR(R11)(r4) 915 ld r12, VCPU_GPR(R12)(r4) 916 ld r13, VCPU_GPR(R13)(r4) 917 918BEGIN_FTR_SECTION 919 mtspr SPRN_PPR, r0 920END_FTR_SECTION_IFSET(CPU_FTR_HAS_PPR) 921 922 ld r6, VCPU_GPR(R6)(r4) 923 ld r7, VCPU_GPR(R7)(r4) 924 925 ld r0, VCPU_CR(r4) 926 mtcr r0 927 928 ld r0, VCPU_GPR(R0)(r4) 929 ld r2, VCPU_GPR(R2)(r4) 930 ld r3, VCPU_GPR(R3)(r4) 931 ld r4, VCPU_GPR(R4)(r4) 932 HRFI_TO_GUEST 933 b . 934 935secondary_too_late: 936 li r12, 0 937 stw r12, STACK_SLOT_TRAP(r1) 938 cmpdi r4, 0 939 beq 11f 940 stw r12, VCPU_TRAP(r4) 941#ifdef CONFIG_KVM_BOOK3S_HV_EXIT_TIMING 942 addi r3, r4, VCPU_TB_RMEXIT 943 bl kvmhv_accumulate_time 944#endif 94511: b kvmhv_switch_to_host 946 947no_switch_exit: 948 HMT_MEDIUM 949 li r12, 0 950 b 12f 951hdec_soon: 952 li r12, BOOK3S_INTERRUPT_HV_DECREMENTER 95312: stw r12, VCPU_TRAP(r4) 954 mr r9, r4 955#ifdef CONFIG_KVM_BOOK3S_HV_EXIT_TIMING 956 addi r3, r4, VCPU_TB_RMEXIT 957 bl kvmhv_accumulate_time 958#endif 959 b guest_bypass 960 961/****************************************************************************** 962 * * 963 * Exit code * 964 * * 965 *****************************************************************************/ 966 967/* 968 * We come here from the first-level interrupt handlers. 969 */ 970 .globl kvmppc_interrupt_hv 971kvmppc_interrupt_hv: 972 /* 973 * Register contents: 974 * R9 = HSTATE_IN_GUEST 975 * R12 = (guest CR << 32) | interrupt vector 976 * R13 = PACA 977 * guest R12 saved in shadow VCPU SCRATCH0 978 * guest R13 saved in SPRN_SCRATCH0 979 * guest R9 saved in HSTATE_SCRATCH2 980 */ 981 /* We're now back in the host but in guest MMU context */ 982 cmpwi r9,KVM_GUEST_MODE_HOST_HV 983 beq kvmppc_bad_host_intr 984 li r9, KVM_GUEST_MODE_HOST_HV 985 stb r9, HSTATE_IN_GUEST(r13) 986 987 ld r9, HSTATE_KVM_VCPU(r13) 988 989 /* Save registers */ 990 991 std r0, VCPU_GPR(R0)(r9) 992 std r1, VCPU_GPR(R1)(r9) 993 std r2, VCPU_GPR(R2)(r9) 994 std r3, VCPU_GPR(R3)(r9) 995 std r4, VCPU_GPR(R4)(r9) 996 std r5, VCPU_GPR(R5)(r9) 997 std r6, VCPU_GPR(R6)(r9) 998 std r7, VCPU_GPR(R7)(r9) 999 std r8, VCPU_GPR(R8)(r9) 1000 ld r0, HSTATE_SCRATCH2(r13) 1001 std r0, VCPU_GPR(R9)(r9) 1002 std r10, VCPU_GPR(R10)(r9) 1003 std r11, VCPU_GPR(R11)(r9) 1004 ld r3, HSTATE_SCRATCH0(r13) 1005 std r3, VCPU_GPR(R12)(r9) 1006 /* CR is in the high half of r12 */ 1007 srdi r4, r12, 32 1008 std r4, VCPU_CR(r9) 1009BEGIN_FTR_SECTION 1010 ld r3, HSTATE_CFAR(r13) 1011 std r3, VCPU_CFAR(r9) 1012END_FTR_SECTION_IFSET(CPU_FTR_CFAR) 1013BEGIN_FTR_SECTION 1014 ld r4, HSTATE_PPR(r13) 1015 std r4, VCPU_PPR(r9) 1016END_FTR_SECTION_IFSET(CPU_FTR_HAS_PPR) 1017 1018 /* Restore R1/R2 so we can handle faults */ 1019 ld r1, HSTATE_HOST_R1(r13) 1020 ld r2, PACATOC(r13) 1021 1022 mfspr r10, SPRN_SRR0 1023 mfspr r11, SPRN_SRR1 1024 std r10, VCPU_SRR0(r9) 1025 std r11, VCPU_SRR1(r9) 1026 /* trap is in the low half of r12, clear CR from the high half */ 1027 clrldi r12, r12, 32 1028 andi. r0, r12, 2 /* need to read HSRR0/1? */ 1029 beq 1f 1030 mfspr r10, SPRN_HSRR0 1031 mfspr r11, SPRN_HSRR1 1032 clrrdi r12, r12, 2 10331: std r10, VCPU_PC(r9) 1034 std r11, VCPU_MSR(r9) 1035 1036 GET_SCRATCH0(r3) 1037 mflr r4 1038 std r3, VCPU_GPR(R13)(r9) 1039 std r4, VCPU_LR(r9) 1040 1041 stw r12,VCPU_TRAP(r9) 1042 1043 /* 1044 * Now that we have saved away SRR0/1 and HSRR0/1, 1045 * interrupts are recoverable in principle, so set MSR_RI. 1046 * This becomes important for relocation-on interrupts from 1047 * the guest, which we can get in radix mode on POWER9. 1048 */ 1049 li r0, MSR_RI 1050 mtmsrd r0, 1 1051 1052#ifdef CONFIG_KVM_BOOK3S_HV_EXIT_TIMING 1053 addi r3, r9, VCPU_TB_RMINTR 1054 mr r4, r9 1055 bl kvmhv_accumulate_time 1056 ld r5, VCPU_GPR(R5)(r9) 1057 ld r6, VCPU_GPR(R6)(r9) 1058 ld r7, VCPU_GPR(R7)(r9) 1059 ld r8, VCPU_GPR(R8)(r9) 1060#endif 1061 1062 /* Save HEIR (HV emulation assist reg) in emul_inst 1063 if this is an HEI (HV emulation interrupt, e40) */ 1064 li r3,KVM_INST_FETCH_FAILED 1065 stw r3,VCPU_LAST_INST(r9) 1066 cmpwi r12,BOOK3S_INTERRUPT_H_EMUL_ASSIST 1067 bne 11f 1068 mfspr r3,SPRN_HEIR 106911: stw r3,VCPU_HEIR(r9) 1070 1071 /* these are volatile across C function calls */ 1072 mfctr r3 1073 mfxer r4 1074 std r3, VCPU_CTR(r9) 1075 std r4, VCPU_XER(r9) 1076 1077 /* Save more register state */ 1078 mfdar r3 1079 mfdsisr r4 1080 std r3, VCPU_DAR(r9) 1081 stw r4, VCPU_DSISR(r9) 1082 1083 /* If this is a page table miss then see if it's theirs or ours */ 1084 cmpwi r12, BOOK3S_INTERRUPT_H_DATA_STORAGE 1085 beq kvmppc_hdsi 1086 std r3, VCPU_FAULT_DAR(r9) 1087 stw r4, VCPU_FAULT_DSISR(r9) 1088 cmpwi r12, BOOK3S_INTERRUPT_H_INST_STORAGE 1089 beq kvmppc_hisi 1090 1091#ifdef CONFIG_PPC_TRANSACTIONAL_MEM 1092 /* For softpatch interrupt, go off and do TM instruction emulation */ 1093 cmpwi r12, BOOK3S_INTERRUPT_HV_SOFTPATCH 1094 beq kvmppc_tm_emul 1095#endif 1096 1097 /* See if this is a leftover HDEC interrupt */ 1098 cmpwi r12,BOOK3S_INTERRUPT_HV_DECREMENTER 1099 bne 2f 1100 mfspr r3,SPRN_HDEC 1101 extsw r3, r3 1102 cmpdi r3,0 1103 mr r4,r9 1104 bge fast_guest_return 11052: 1106 /* See if this is an hcall we can handle in real mode */ 1107 cmpwi r12,BOOK3S_INTERRUPT_SYSCALL 1108 beq hcall_try_real_mode 1109 1110 /* Hypervisor doorbell - exit only if host IPI flag set */ 1111 cmpwi r12, BOOK3S_INTERRUPT_H_DOORBELL 1112 bne 3f 1113 lbz r0, HSTATE_HOST_IPI(r13) 1114 cmpwi r0, 0 1115 beq maybe_reenter_guest 1116 b guest_exit_cont 11173: 1118 /* If it's a hypervisor facility unavailable interrupt, save HFSCR */ 1119 cmpwi r12, BOOK3S_INTERRUPT_H_FAC_UNAVAIL 1120 bne 14f 1121 mfspr r3, SPRN_HFSCR 1122 std r3, VCPU_HFSCR(r9) 1123 b guest_exit_cont 112414: 1125 /* External interrupt ? */ 1126 cmpwi r12, BOOK3S_INTERRUPT_EXTERNAL 1127 beq kvmppc_guest_external 1128 /* See if it is a machine check */ 1129 cmpwi r12, BOOK3S_INTERRUPT_MACHINE_CHECK 1130 beq machine_check_realmode 1131 /* Or a hypervisor maintenance interrupt */ 1132 cmpwi r12, BOOK3S_INTERRUPT_HMI 1133 beq hmi_realmode 1134 1135guest_exit_cont: /* r9 = vcpu, r12 = trap, r13 = paca */ 1136 1137#ifdef CONFIG_KVM_BOOK3S_HV_EXIT_TIMING 1138 addi r3, r9, VCPU_TB_RMEXIT 1139 mr r4, r9 1140 bl kvmhv_accumulate_time 1141#endif 1142 1143 /* 1144 * Possibly flush the link stack here, before we do a blr in 1145 * kvmhv_switch_to_host. 1146 */ 11471: nop 1148 patch_site 1b patch__call_kvm_flush_link_stack 1149 1150 /* For hash guest, read the guest SLB and save it away */ 1151 li r5, 0 1152 lwz r0,VCPU_SLB_NR(r9) /* number of entries in SLB */ 1153 mtctr r0 1154 li r6,0 1155 addi r7,r9,VCPU_SLB 11561: slbmfee r8,r6 1157 andis. r0,r8,SLB_ESID_V@h 1158 beq 2f 1159 add r8,r8,r6 /* put index in */ 1160 slbmfev r3,r6 1161 std r8,VCPU_SLB_E(r7) 1162 std r3,VCPU_SLB_V(r7) 1163 addi r7,r7,VCPU_SLB_SIZE 1164 addi r5,r5,1 11652: addi r6,r6,1 1166 bdnz 1b 1167 /* Finally clear out the SLB */ 1168 li r0,0 1169 slbmte r0,r0 1170 PPC_SLBIA(6) 1171 ptesync 1172 stw r5,VCPU_SLB_MAX(r9) 1173 1174 /* load host SLB entries */ 1175 ld r8,PACA_SLBSHADOWPTR(r13) 1176 1177 .rept SLB_NUM_BOLTED 1178 li r3, SLBSHADOW_SAVEAREA 1179 LDX_BE r5, r8, r3 1180 addi r3, r3, 8 1181 LDX_BE r6, r8, r3 1182 andis. r7,r5,SLB_ESID_V@h 1183 beq 1f 1184 slbmte r6,r5 11851: addi r8,r8,16 1186 .endr 1187 1188guest_bypass: 1189 stw r12, STACK_SLOT_TRAP(r1) 1190 1191 /* Save DEC */ 1192 /* Do this before kvmhv_commence_exit so we know TB is guest TB */ 1193 ld r3, HSTATE_KVM_VCORE(r13) 1194 mfspr r5,SPRN_DEC 1195 mftb r6 1196 extsw r5,r5 119716: add r5,r5,r6 1198 /* r5 is a guest timebase value here, convert to host TB */ 1199 ld r4,VCORE_TB_OFFSET_APPL(r3) 1200 subf r5,r4,r5 1201 std r5,VCPU_DEC_EXPIRES(r9) 1202 1203 /* Increment exit count, poke other threads to exit */ 1204 mr r3, r12 1205 bl kvmhv_commence_exit 1206 nop 1207 ld r9, HSTATE_KVM_VCPU(r13) 1208 1209 /* Stop others sending VCPU interrupts to this physical CPU */ 1210 li r0, -1 1211 stw r0, VCPU_CPU(r9) 1212 stw r0, VCPU_THREAD_CPU(r9) 1213 1214 /* Save guest CTRL register, set runlatch to 1 */ 1215 mfspr r6,SPRN_CTRLF 1216 stw r6,VCPU_CTRL(r9) 1217 andi. r0,r6,1 1218 bne 4f 1219 ori r6,r6,1 1220 mtspr SPRN_CTRLT,r6 12214: 1222 /* 1223 * Save the guest PURR/SPURR 1224 */ 1225 mfspr r5,SPRN_PURR 1226 mfspr r6,SPRN_SPURR 1227 ld r7,VCPU_PURR(r9) 1228 ld r8,VCPU_SPURR(r9) 1229 std r5,VCPU_PURR(r9) 1230 std r6,VCPU_SPURR(r9) 1231 subf r5,r7,r5 1232 subf r6,r8,r6 1233 1234 /* 1235 * Restore host PURR/SPURR and add guest times 1236 * so that the time in the guest gets accounted. 1237 */ 1238 ld r3,HSTATE_PURR(r13) 1239 ld r4,HSTATE_SPURR(r13) 1240 add r3,r3,r5 1241 add r4,r4,r6 1242 mtspr SPRN_PURR,r3 1243 mtspr SPRN_SPURR,r4 1244 1245BEGIN_FTR_SECTION 1246 b 8f 1247END_FTR_SECTION_IFCLR(CPU_FTR_ARCH_207S) 1248 /* Save POWER8-specific registers */ 1249 mfspr r5, SPRN_IAMR 1250 mfspr r6, SPRN_PSPB 1251 mfspr r7, SPRN_FSCR 1252 std r5, VCPU_IAMR(r9) 1253 stw r6, VCPU_PSPB(r9) 1254 std r7, VCPU_FSCR(r9) 1255 mfspr r5, SPRN_IC 1256 mfspr r7, SPRN_TAR 1257 std r5, VCPU_IC(r9) 1258 std r7, VCPU_TAR(r9) 1259 mfspr r8, SPRN_EBBHR 1260 std r8, VCPU_EBBHR(r9) 1261 mfspr r5, SPRN_EBBRR 1262 mfspr r6, SPRN_BESCR 1263 mfspr r7, SPRN_PID 1264 mfspr r8, SPRN_WORT 1265 std r5, VCPU_EBBRR(r9) 1266 std r6, VCPU_BESCR(r9) 1267 stw r7, VCPU_GUEST_PID(r9) 1268 std r8, VCPU_WORT(r9) 1269 mfspr r5, SPRN_TCSCR 1270 mfspr r6, SPRN_ACOP 1271 mfspr r7, SPRN_CSIGR 1272 mfspr r8, SPRN_TACR 1273 std r5, VCPU_TCSCR(r9) 1274 std r6, VCPU_ACOP(r9) 1275 std r7, VCPU_CSIGR(r9) 1276 std r8, VCPU_TACR(r9) 1277BEGIN_FTR_SECTION 1278 ld r5, STACK_SLOT_FSCR(r1) 1279 mtspr SPRN_FSCR, r5 1280END_FTR_SECTION_IFSET(CPU_FTR_ARCH_207S) 1281 /* 1282 * Restore various registers to 0, where non-zero values 1283 * set by the guest could disrupt the host. 1284 */ 1285 li r0, 0 1286 mtspr SPRN_PSPB, r0 1287 mtspr SPRN_WORT, r0 1288 mtspr SPRN_TCSCR, r0 1289 /* Set MMCRS to 1<<31 to freeze and disable the SPMC counters */ 1290 li r0, 1 1291 sldi r0, r0, 31 1292 mtspr SPRN_MMCRS, r0 1293 1294 /* Save and restore AMR, IAMR and UAMOR before turning on the MMU */ 1295 ld r8, STACK_SLOT_IAMR(r1) 1296 mtspr SPRN_IAMR, r8 1297 12988: /* Power7 jumps back in here */ 1299 mfspr r5,SPRN_AMR 1300 mfspr r6,SPRN_UAMOR 1301 std r5,VCPU_AMR(r9) 1302 std r6,VCPU_UAMOR(r9) 1303 ld r5,STACK_SLOT_AMR(r1) 1304 ld r6,STACK_SLOT_UAMOR(r1) 1305 mtspr SPRN_AMR, r5 1306 mtspr SPRN_UAMOR, r6 1307 1308 /* Switch DSCR back to host value */ 1309 mfspr r8, SPRN_DSCR 1310 ld r7, HSTATE_DSCR(r13) 1311 std r8, VCPU_DSCR(r9) 1312 mtspr SPRN_DSCR, r7 1313 1314 /* Save non-volatile GPRs */ 1315 std r14, VCPU_GPR(R14)(r9) 1316 std r15, VCPU_GPR(R15)(r9) 1317 std r16, VCPU_GPR(R16)(r9) 1318 std r17, VCPU_GPR(R17)(r9) 1319 std r18, VCPU_GPR(R18)(r9) 1320 std r19, VCPU_GPR(R19)(r9) 1321 std r20, VCPU_GPR(R20)(r9) 1322 std r21, VCPU_GPR(R21)(r9) 1323 std r22, VCPU_GPR(R22)(r9) 1324 std r23, VCPU_GPR(R23)(r9) 1325 std r24, VCPU_GPR(R24)(r9) 1326 std r25, VCPU_GPR(R25)(r9) 1327 std r26, VCPU_GPR(R26)(r9) 1328 std r27, VCPU_GPR(R27)(r9) 1329 std r28, VCPU_GPR(R28)(r9) 1330 std r29, VCPU_GPR(R29)(r9) 1331 std r30, VCPU_GPR(R30)(r9) 1332 std r31, VCPU_GPR(R31)(r9) 1333 1334 /* Save SPRGs */ 1335 mfspr r3, SPRN_SPRG0 1336 mfspr r4, SPRN_SPRG1 1337 mfspr r5, SPRN_SPRG2 1338 mfspr r6, SPRN_SPRG3 1339 std r3, VCPU_SPRG0(r9) 1340 std r4, VCPU_SPRG1(r9) 1341 std r5, VCPU_SPRG2(r9) 1342 std r6, VCPU_SPRG3(r9) 1343 1344 /* save FP state */ 1345 mr r3, r9 1346 bl kvmppc_save_fp 1347 1348#ifdef CONFIG_PPC_TRANSACTIONAL_MEM 1349BEGIN_FTR_SECTION 1350 b 91f 1351END_FTR_SECTION_IFCLR(CPU_FTR_TM) 1352 /* 1353 * NOTE THAT THIS TRASHES ALL NON-VOLATILE REGISTERS (but not CR) 1354 */ 1355 mr r3, r9 1356 ld r4, VCPU_MSR(r3) 1357 li r5, 0 /* don't preserve non-vol regs */ 1358 bl kvmppc_save_tm_hv 1359 nop 1360 ld r9, HSTATE_KVM_VCPU(r13) 136191: 1362#endif 1363 1364 /* Increment yield count if they have a VPA */ 1365 ld r8, VCPU_VPA(r9) /* do they have a VPA? */ 1366 cmpdi r8, 0 1367 beq 25f 1368 li r4, LPPACA_YIELDCOUNT 1369 LWZX_BE r3, r8, r4 1370 addi r3, r3, 1 1371 STWX_BE r3, r8, r4 1372 li r3, 1 1373 stb r3, VCPU_VPA_DIRTY(r9) 137425: 1375 /* Save PMU registers if requested */ 1376 /* r8 and cr0.eq are live here */ 1377 mr r3, r9 1378 li r4, 1 1379 beq 21f /* if no VPA, save PMU stuff anyway */ 1380 lbz r4, LPPACA_PMCINUSE(r8) 138121: bl kvmhv_save_guest_pmu 1382 ld r9, HSTATE_KVM_VCPU(r13) 1383 1384 /* Restore host values of some registers */ 1385BEGIN_FTR_SECTION 1386 ld r5, STACK_SLOT_CIABR(r1) 1387 ld r6, STACK_SLOT_DAWR0(r1) 1388 ld r7, STACK_SLOT_DAWRX0(r1) 1389 mtspr SPRN_CIABR, r5 1390 /* 1391 * If the DAWR doesn't work, it's ok to write these here as 1392 * this value should always be zero 1393 */ 1394 mtspr SPRN_DAWR0, r6 1395 mtspr SPRN_DAWRX0, r7 1396END_FTR_SECTION_IFSET(CPU_FTR_ARCH_207S) 1397 1398 /* 1399 * POWER7/POWER8 guest -> host partition switch code. 1400 * We don't have to lock against tlbies but we do 1401 * have to coordinate the hardware threads. 1402 * Here STACK_SLOT_TRAP(r1) contains the trap number. 1403 */ 1404kvmhv_switch_to_host: 1405 /* Secondary threads wait for primary to do partition switch */ 1406 ld r5,HSTATE_KVM_VCORE(r13) 1407 ld r4,VCORE_KVM(r5) /* pointer to struct kvm */ 1408 lbz r3,HSTATE_PTID(r13) 1409 cmpwi r3,0 1410 beq 15f 1411 HMT_LOW 141213: lbz r3,VCORE_IN_GUEST(r5) 1413 cmpwi r3,0 1414 bne 13b 1415 HMT_MEDIUM 1416 b 16f 1417 1418 /* Primary thread waits for all the secondaries to exit guest */ 141915: lwz r3,VCORE_ENTRY_EXIT(r5) 1420 rlwinm r0,r3,32-8,0xff 1421 clrldi r3,r3,56 1422 cmpw r3,r0 1423 bne 15b 1424 isync 1425 1426 /* Did we actually switch to the guest at all? */ 1427 lbz r6, VCORE_IN_GUEST(r5) 1428 cmpwi r6, 0 1429 beq 19f 1430 1431 /* Primary thread switches back to host partition */ 1432 lwz r7,KVM_HOST_LPID(r4) 1433 ld r6,KVM_HOST_SDR1(r4) 1434 li r8,LPID_RSVD /* switch to reserved LPID */ 1435 mtspr SPRN_LPID,r8 1436 ptesync 1437 mtspr SPRN_SDR1,r6 /* switch to host page table */ 1438 mtspr SPRN_LPID,r7 1439 isync 1440 1441BEGIN_FTR_SECTION 1442 /* DPDES and VTB are shared between threads */ 1443 mfspr r7, SPRN_DPDES 1444 mfspr r8, SPRN_VTB 1445 std r7, VCORE_DPDES(r5) 1446 std r8, VCORE_VTB(r5) 1447 /* clear DPDES so we don't get guest doorbells in the host */ 1448 li r8, 0 1449 mtspr SPRN_DPDES, r8 1450END_FTR_SECTION_IFSET(CPU_FTR_ARCH_207S) 1451 1452 /* Subtract timebase offset from timebase */ 1453 ld r8, VCORE_TB_OFFSET_APPL(r5) 1454 cmpdi r8,0 1455 beq 17f 1456 li r0, 0 1457 std r0, VCORE_TB_OFFSET_APPL(r5) 1458 mftb r6 /* current guest timebase */ 1459 subf r8,r8,r6 1460 mtspr SPRN_TBU40,r8 /* update upper 40 bits */ 1461 mftb r7 /* check if lower 24 bits overflowed */ 1462 clrldi r6,r6,40 1463 clrldi r7,r7,40 1464 cmpld r7,r6 1465 bge 17f 1466 addis r8,r8,0x100 /* if so, increment upper 40 bits */ 1467 mtspr SPRN_TBU40,r8 1468 146917: 1470 /* 1471 * If this is an HMI, we called kvmppc_realmode_hmi_handler 1472 * above, which may or may not have already called 1473 * kvmppc_subcore_exit_guest. Fortunately, all that 1474 * kvmppc_subcore_exit_guest does is clear a flag, so calling 1475 * it again here is benign even if kvmppc_realmode_hmi_handler 1476 * has already called it. 1477 */ 1478 bl kvmppc_subcore_exit_guest 1479 nop 148030: ld r5,HSTATE_KVM_VCORE(r13) 1481 ld r4,VCORE_KVM(r5) /* pointer to struct kvm */ 1482 1483 /* Reset PCR */ 1484 ld r0, VCORE_PCR(r5) 1485 LOAD_REG_IMMEDIATE(r6, PCR_MASK) 1486 cmpld r0, r6 1487 beq 18f 1488 mtspr SPRN_PCR, r6 148918: 1490 /* Signal secondary CPUs to continue */ 1491 li r0, 0 1492 stb r0,VCORE_IN_GUEST(r5) 149319: lis r8,0x7fff /* MAX_INT@h */ 1494 mtspr SPRN_HDEC,r8 1495 149616: ld r8,KVM_HOST_LPCR(r4) 1497 mtspr SPRN_LPCR,r8 1498 isync 1499 1500#ifdef CONFIG_KVM_BOOK3S_HV_EXIT_TIMING 1501 /* Finish timing, if we have a vcpu */ 1502 ld r4, HSTATE_KVM_VCPU(r13) 1503 cmpdi r4, 0 1504 li r3, 0 1505 beq 2f 1506 bl kvmhv_accumulate_time 15072: 1508#endif 1509 /* Unset guest mode */ 1510 li r0, KVM_GUEST_MODE_NONE 1511 stb r0, HSTATE_IN_GUEST(r13) 1512 1513 lwz r12, STACK_SLOT_TRAP(r1) /* return trap # in r12 */ 1514 ld r0, SFS+PPC_LR_STKOFF(r1) 1515 addi r1, r1, SFS 1516 mtlr r0 1517 blr 1518 1519.balign 32 1520.global kvm_flush_link_stack 1521kvm_flush_link_stack: 1522 /* Save LR into r0 */ 1523 mflr r0 1524 1525 /* Flush the link stack. On Power8 it's up to 32 entries in size. */ 1526 .rept 32 1527 bl .+4 1528 .endr 1529 1530 /* And on Power9 it's up to 64. */ 1531BEGIN_FTR_SECTION 1532 .rept 32 1533 bl .+4 1534 .endr 1535END_FTR_SECTION_IFSET(CPU_FTR_ARCH_300) 1536 1537 /* Restore LR */ 1538 mtlr r0 1539 blr 1540 1541kvmppc_guest_external: 1542 /* External interrupt, first check for host_ipi. If this is 1543 * set, we know the host wants us out so let's do it now 1544 */ 1545 bl kvmppc_read_intr 1546 1547 /* 1548 * Restore the active volatile registers after returning from 1549 * a C function. 1550 */ 1551 ld r9, HSTATE_KVM_VCPU(r13) 1552 li r12, BOOK3S_INTERRUPT_EXTERNAL 1553 1554 /* 1555 * kvmppc_read_intr return codes: 1556 * 1557 * Exit to host (r3 > 0) 1558 * 1 An interrupt is pending that needs to be handled by the host 1559 * Exit guest and return to host by branching to guest_exit_cont 1560 * 1561 * 2 Passthrough that needs completion in the host 1562 * Exit guest and return to host by branching to guest_exit_cont 1563 * However, we also set r12 to BOOK3S_INTERRUPT_HV_RM_HARD 1564 * to indicate to the host to complete handling the interrupt 1565 * 1566 * Before returning to guest, we check if any CPU is heading out 1567 * to the host and if so, we head out also. If no CPUs are heading 1568 * check return values <= 0. 1569 * 1570 * Return to guest (r3 <= 0) 1571 * 0 No external interrupt is pending 1572 * -1 A guest wakeup IPI (which has now been cleared) 1573 * In either case, we return to guest to deliver any pending 1574 * guest interrupts. 1575 * 1576 * -2 A PCI passthrough external interrupt was handled 1577 * (interrupt was delivered directly to guest) 1578 * Return to guest to deliver any pending guest interrupts. 1579 */ 1580 1581 cmpdi r3, 1 1582 ble 1f 1583 1584 /* Return code = 2 */ 1585 li r12, BOOK3S_INTERRUPT_HV_RM_HARD 1586 stw r12, VCPU_TRAP(r9) 1587 b guest_exit_cont 1588 15891: /* Return code <= 1 */ 1590 cmpdi r3, 0 1591 bgt guest_exit_cont 1592 1593 /* Return code <= 0 */ 1594maybe_reenter_guest: 1595 ld r5, HSTATE_KVM_VCORE(r13) 1596 lwz r0, VCORE_ENTRY_EXIT(r5) 1597 cmpwi r0, 0x100 1598 mr r4, r9 1599 blt deliver_guest_interrupt 1600 b guest_exit_cont 1601 1602#ifdef CONFIG_PPC_TRANSACTIONAL_MEM 1603/* 1604 * Softpatch interrupt for transactional memory emulation cases 1605 * on POWER9 DD2.2. This is early in the guest exit path - we 1606 * haven't saved registers or done a treclaim yet. 1607 */ 1608kvmppc_tm_emul: 1609 /* Save instruction image in HEIR */ 1610 mfspr r3, SPRN_HEIR 1611 stw r3, VCPU_HEIR(r9) 1612 1613 /* 1614 * The cases we want to handle here are those where the guest 1615 * is in real suspend mode and is trying to transition to 1616 * transactional mode. 1617 */ 1618 lbz r0, HSTATE_FAKE_SUSPEND(r13) 1619 cmpwi r0, 0 /* keep exiting guest if in fake suspend */ 1620 bne guest_exit_cont 1621 rldicl r3, r11, 64 - MSR_TS_S_LG, 62 1622 cmpwi r3, 1 /* or if not in suspend state */ 1623 bne guest_exit_cont 1624 1625 /* Call C code to do the emulation */ 1626 mr r3, r9 1627 bl kvmhv_p9_tm_emulation_early 1628 nop 1629 ld r9, HSTATE_KVM_VCPU(r13) 1630 li r12, BOOK3S_INTERRUPT_HV_SOFTPATCH 1631 cmpwi r3, 0 1632 beq guest_exit_cont /* continue exiting if not handled */ 1633 ld r10, VCPU_PC(r9) 1634 ld r11, VCPU_MSR(r9) 1635 b fast_interrupt_c_return /* go back to guest if handled */ 1636#endif /* CONFIG_PPC_TRANSACTIONAL_MEM */ 1637 1638/* 1639 * Check whether an HDSI is an HPTE not found fault or something else. 1640 * If it is an HPTE not found fault that is due to the guest accessing 1641 * a page that they have mapped but which we have paged out, then 1642 * we continue on with the guest exit path. In all other cases, 1643 * reflect the HDSI to the guest as a DSI. 1644 */ 1645kvmppc_hdsi: 1646 mfspr r4, SPRN_HDAR 1647 mfspr r6, SPRN_HDSISR 1648 /* HPTE not found fault or protection fault? */ 1649 andis. r0, r6, (DSISR_NOHPTE | DSISR_PROTFAULT)@h 1650 beq 1f /* if not, send it to the guest */ 1651 andi. r0, r11, MSR_DR /* data relocation enabled? */ 1652 beq 3f 1653 clrrdi r0, r4, 28 1654 PPC_SLBFEE_DOT(R5, R0) /* if so, look up SLB */ 1655 li r0, BOOK3S_INTERRUPT_DATA_SEGMENT 1656 bne 7f /* if no SLB entry found */ 16574: std r4, VCPU_FAULT_DAR(r9) 1658 stw r6, VCPU_FAULT_DSISR(r9) 1659 1660 /* Search the hash table. */ 1661 mr r3, r9 /* vcpu pointer */ 1662 li r7, 1 /* data fault */ 1663 bl kvmppc_hpte_hv_fault 1664 ld r9, HSTATE_KVM_VCPU(r13) 1665 ld r10, VCPU_PC(r9) 1666 ld r11, VCPU_MSR(r9) 1667 li r12, BOOK3S_INTERRUPT_H_DATA_STORAGE 1668 cmpdi r3, 0 /* retry the instruction */ 1669 beq 6f 1670 cmpdi r3, -1 /* handle in kernel mode */ 1671 beq guest_exit_cont 1672 cmpdi r3, -2 /* MMIO emulation; need instr word */ 1673 beq 2f 1674 1675 /* Synthesize a DSI (or DSegI) for the guest */ 1676 ld r4, VCPU_FAULT_DAR(r9) 1677 mr r6, r3 16781: li r0, BOOK3S_INTERRUPT_DATA_STORAGE 1679 mtspr SPRN_DSISR, r6 16807: mtspr SPRN_DAR, r4 1681 mtspr SPRN_SRR0, r10 1682 mtspr SPRN_SRR1, r11 1683 mr r10, r0 1684 bl kvmppc_msr_interrupt 1685fast_interrupt_c_return: 16866: ld r7, VCPU_CTR(r9) 1687 ld r8, VCPU_XER(r9) 1688 mtctr r7 1689 mtxer r8 1690 mr r4, r9 1691 b fast_guest_return 1692 16933: ld r5, VCPU_KVM(r9) /* not relocated, use VRMA */ 1694 ld r5, KVM_VRMA_SLB_V(r5) 1695 b 4b 1696 1697 /* If this is for emulated MMIO, load the instruction word */ 16982: li r8, KVM_INST_FETCH_FAILED /* In case lwz faults */ 1699 1700 /* Set guest mode to 'jump over instruction' so if lwz faults 1701 * we'll just continue at the next IP. */ 1702 li r0, KVM_GUEST_MODE_SKIP 1703 stb r0, HSTATE_IN_GUEST(r13) 1704 1705 /* Do the access with MSR:DR enabled */ 1706 mfmsr r3 1707 ori r4, r3, MSR_DR /* Enable paging for data */ 1708 mtmsrd r4 1709 lwz r8, 0(r10) 1710 mtmsrd r3 1711 1712 /* Store the result */ 1713 stw r8, VCPU_LAST_INST(r9) 1714 1715 /* Unset guest mode. */ 1716 li r0, KVM_GUEST_MODE_HOST_HV 1717 stb r0, HSTATE_IN_GUEST(r13) 1718 b guest_exit_cont 1719 1720/* 1721 * Similarly for an HISI, reflect it to the guest as an ISI unless 1722 * it is an HPTE not found fault for a page that we have paged out. 1723 */ 1724kvmppc_hisi: 1725 andis. r0, r11, SRR1_ISI_NOPT@h 1726 beq 1f 1727 andi. r0, r11, MSR_IR /* instruction relocation enabled? */ 1728 beq 3f 1729 clrrdi r0, r10, 28 1730 PPC_SLBFEE_DOT(R5, R0) /* if so, look up SLB */ 1731 li r0, BOOK3S_INTERRUPT_INST_SEGMENT 1732 bne 7f /* if no SLB entry found */ 17334: 1734 /* Search the hash table. */ 1735 mr r3, r9 /* vcpu pointer */ 1736 mr r4, r10 1737 mr r6, r11 1738 li r7, 0 /* instruction fault */ 1739 bl kvmppc_hpte_hv_fault 1740 ld r9, HSTATE_KVM_VCPU(r13) 1741 ld r10, VCPU_PC(r9) 1742 ld r11, VCPU_MSR(r9) 1743 li r12, BOOK3S_INTERRUPT_H_INST_STORAGE 1744 cmpdi r3, 0 /* retry the instruction */ 1745 beq fast_interrupt_c_return 1746 cmpdi r3, -1 /* handle in kernel mode */ 1747 beq guest_exit_cont 1748 1749 /* Synthesize an ISI (or ISegI) for the guest */ 1750 mr r11, r3 17511: li r0, BOOK3S_INTERRUPT_INST_STORAGE 17527: mtspr SPRN_SRR0, r10 1753 mtspr SPRN_SRR1, r11 1754 mr r10, r0 1755 bl kvmppc_msr_interrupt 1756 b fast_interrupt_c_return 1757 17583: ld r6, VCPU_KVM(r9) /* not relocated, use VRMA */ 1759 ld r5, KVM_VRMA_SLB_V(r6) 1760 b 4b 1761 1762/* 1763 * Try to handle an hcall in real mode. 1764 * Returns to the guest if we handle it, or continues on up to 1765 * the kernel if we can't (i.e. if we don't have a handler for 1766 * it, or if the handler returns H_TOO_HARD). 1767 * 1768 * r5 - r8 contain hcall args, 1769 * r9 = vcpu, r10 = pc, r11 = msr, r12 = trap, r13 = paca 1770 */ 1771hcall_try_real_mode: 1772 ld r3,VCPU_GPR(R3)(r9) 1773 andi. r0,r11,MSR_PR 1774 /* sc 1 from userspace - reflect to guest syscall */ 1775 bne sc_1_fast_return 1776 clrrdi r3,r3,2 1777 cmpldi r3,hcall_real_table_end - hcall_real_table 1778 bge guest_exit_cont 1779 /* See if this hcall is enabled for in-kernel handling */ 1780 ld r4, VCPU_KVM(r9) 1781 srdi r0, r3, 8 /* r0 = (r3 / 4) >> 6 */ 1782 sldi r0, r0, 3 /* index into kvm->arch.enabled_hcalls[] */ 1783 add r4, r4, r0 1784 ld r0, KVM_ENABLED_HCALLS(r4) 1785 rlwinm r4, r3, 32-2, 0x3f /* r4 = (r3 / 4) & 0x3f */ 1786 srd r0, r0, r4 1787 andi. r0, r0, 1 1788 beq guest_exit_cont 1789 /* Get pointer to handler, if any, and call it */ 1790 LOAD_REG_ADDR(r4, hcall_real_table) 1791 lwax r3,r3,r4 1792 cmpwi r3,0 1793 beq guest_exit_cont 1794 add r12,r3,r4 1795 mtctr r12 1796 mr r3,r9 /* get vcpu pointer */ 1797 ld r4,VCPU_GPR(R4)(r9) 1798 bctrl 1799 cmpdi r3,H_TOO_HARD 1800 beq hcall_real_fallback 1801 ld r4,HSTATE_KVM_VCPU(r13) 1802 std r3,VCPU_GPR(R3)(r4) 1803 ld r10,VCPU_PC(r4) 1804 ld r11,VCPU_MSR(r4) 1805 b fast_guest_return 1806 1807sc_1_fast_return: 1808 mtspr SPRN_SRR0,r10 1809 mtspr SPRN_SRR1,r11 1810 li r10, BOOK3S_INTERRUPT_SYSCALL 1811 bl kvmppc_msr_interrupt 1812 mr r4,r9 1813 b fast_guest_return 1814 1815 /* We've attempted a real mode hcall, but it's punted it back 1816 * to userspace. We need to restore some clobbered volatiles 1817 * before resuming the pass-it-to-qemu path */ 1818hcall_real_fallback: 1819 li r12,BOOK3S_INTERRUPT_SYSCALL 1820 ld r9, HSTATE_KVM_VCPU(r13) 1821 1822 b guest_exit_cont 1823 1824 .globl hcall_real_table 1825hcall_real_table: 1826 .long 0 /* 0 - unused */ 1827 .long DOTSYM(kvmppc_h_remove) - hcall_real_table 1828 .long DOTSYM(kvmppc_h_enter) - hcall_real_table 1829 .long DOTSYM(kvmppc_h_read) - hcall_real_table 1830 .long DOTSYM(kvmppc_h_clear_mod) - hcall_real_table 1831 .long DOTSYM(kvmppc_h_clear_ref) - hcall_real_table 1832 .long DOTSYM(kvmppc_h_protect) - hcall_real_table 1833#ifdef CONFIG_SPAPR_TCE_IOMMU 1834 .long DOTSYM(kvmppc_h_get_tce) - hcall_real_table 1835 .long DOTSYM(kvmppc_rm_h_put_tce) - hcall_real_table 1836#else 1837 .long 0 /* 0x1c */ 1838 .long 0 /* 0x20 */ 1839#endif 1840 .long 0 /* 0x24 - H_SET_SPRG0 */ 1841 .long DOTSYM(kvmppc_h_set_dabr) - hcall_real_table 1842 .long DOTSYM(kvmppc_rm_h_page_init) - hcall_real_table 1843 .long 0 /* 0x30 */ 1844 .long 0 /* 0x34 */ 1845 .long 0 /* 0x38 */ 1846 .long 0 /* 0x3c */ 1847 .long 0 /* 0x40 */ 1848 .long 0 /* 0x44 */ 1849 .long 0 /* 0x48 */ 1850 .long 0 /* 0x4c */ 1851 .long 0 /* 0x50 */ 1852 .long 0 /* 0x54 */ 1853 .long 0 /* 0x58 */ 1854 .long 0 /* 0x5c */ 1855 .long 0 /* 0x60 */ 1856#ifdef CONFIG_KVM_XICS 1857 .long DOTSYM(kvmppc_rm_h_eoi) - hcall_real_table 1858 .long DOTSYM(kvmppc_rm_h_cppr) - hcall_real_table 1859 .long DOTSYM(kvmppc_rm_h_ipi) - hcall_real_table 1860 .long DOTSYM(kvmppc_rm_h_ipoll) - hcall_real_table 1861 .long DOTSYM(kvmppc_rm_h_xirr) - hcall_real_table 1862#else 1863 .long 0 /* 0x64 - H_EOI */ 1864 .long 0 /* 0x68 - H_CPPR */ 1865 .long 0 /* 0x6c - H_IPI */ 1866 .long 0 /* 0x70 - H_IPOLL */ 1867 .long 0 /* 0x74 - H_XIRR */ 1868#endif 1869 .long 0 /* 0x78 */ 1870 .long 0 /* 0x7c */ 1871 .long 0 /* 0x80 */ 1872 .long 0 /* 0x84 */ 1873 .long 0 /* 0x88 */ 1874 .long 0 /* 0x8c */ 1875 .long 0 /* 0x90 */ 1876 .long 0 /* 0x94 */ 1877 .long 0 /* 0x98 */ 1878 .long 0 /* 0x9c */ 1879 .long 0 /* 0xa0 */ 1880 .long 0 /* 0xa4 */ 1881 .long 0 /* 0xa8 */ 1882 .long 0 /* 0xac */ 1883 .long 0 /* 0xb0 */ 1884 .long 0 /* 0xb4 */ 1885 .long 0 /* 0xb8 */ 1886 .long 0 /* 0xbc */ 1887 .long 0 /* 0xc0 */ 1888 .long 0 /* 0xc4 */ 1889 .long 0 /* 0xc8 */ 1890 .long 0 /* 0xcc */ 1891 .long 0 /* 0xd0 */ 1892 .long 0 /* 0xd4 */ 1893 .long 0 /* 0xd8 */ 1894 .long 0 /* 0xdc */ 1895 .long DOTSYM(kvmppc_h_cede) - hcall_real_table 1896 .long DOTSYM(kvmppc_rm_h_confer) - hcall_real_table 1897 .long 0 /* 0xe8 */ 1898 .long 0 /* 0xec */ 1899 .long 0 /* 0xf0 */ 1900 .long 0 /* 0xf4 */ 1901 .long 0 /* 0xf8 */ 1902 .long 0 /* 0xfc */ 1903 .long 0 /* 0x100 */ 1904 .long 0 /* 0x104 */ 1905 .long 0 /* 0x108 */ 1906 .long 0 /* 0x10c */ 1907 .long 0 /* 0x110 */ 1908 .long 0 /* 0x114 */ 1909 .long 0 /* 0x118 */ 1910 .long 0 /* 0x11c */ 1911 .long 0 /* 0x120 */ 1912 .long DOTSYM(kvmppc_h_bulk_remove) - hcall_real_table 1913 .long 0 /* 0x128 */ 1914 .long 0 /* 0x12c */ 1915 .long 0 /* 0x130 */ 1916 .long DOTSYM(kvmppc_h_set_xdabr) - hcall_real_table 1917#ifdef CONFIG_SPAPR_TCE_IOMMU 1918 .long DOTSYM(kvmppc_rm_h_stuff_tce) - hcall_real_table 1919 .long DOTSYM(kvmppc_rm_h_put_tce_indirect) - hcall_real_table 1920#else 1921 .long 0 /* 0x138 */ 1922 .long 0 /* 0x13c */ 1923#endif 1924 .long 0 /* 0x140 */ 1925 .long 0 /* 0x144 */ 1926 .long 0 /* 0x148 */ 1927 .long 0 /* 0x14c */ 1928 .long 0 /* 0x150 */ 1929 .long 0 /* 0x154 */ 1930 .long 0 /* 0x158 */ 1931 .long 0 /* 0x15c */ 1932 .long 0 /* 0x160 */ 1933 .long 0 /* 0x164 */ 1934 .long 0 /* 0x168 */ 1935 .long 0 /* 0x16c */ 1936 .long 0 /* 0x170 */ 1937 .long 0 /* 0x174 */ 1938 .long 0 /* 0x178 */ 1939 .long 0 /* 0x17c */ 1940 .long 0 /* 0x180 */ 1941 .long 0 /* 0x184 */ 1942 .long 0 /* 0x188 */ 1943 .long 0 /* 0x18c */ 1944 .long 0 /* 0x190 */ 1945 .long 0 /* 0x194 */ 1946 .long 0 /* 0x198 */ 1947 .long 0 /* 0x19c */ 1948 .long 0 /* 0x1a0 */ 1949 .long 0 /* 0x1a4 */ 1950 .long 0 /* 0x1a8 */ 1951 .long 0 /* 0x1ac */ 1952 .long 0 /* 0x1b0 */ 1953 .long 0 /* 0x1b4 */ 1954 .long 0 /* 0x1b8 */ 1955 .long 0 /* 0x1bc */ 1956 .long 0 /* 0x1c0 */ 1957 .long 0 /* 0x1c4 */ 1958 .long 0 /* 0x1c8 */ 1959 .long 0 /* 0x1cc */ 1960 .long 0 /* 0x1d0 */ 1961 .long 0 /* 0x1d4 */ 1962 .long 0 /* 0x1d8 */ 1963 .long 0 /* 0x1dc */ 1964 .long 0 /* 0x1e0 */ 1965 .long 0 /* 0x1e4 */ 1966 .long 0 /* 0x1e8 */ 1967 .long 0 /* 0x1ec */ 1968 .long 0 /* 0x1f0 */ 1969 .long 0 /* 0x1f4 */ 1970 .long 0 /* 0x1f8 */ 1971 .long 0 /* 0x1fc */ 1972 .long 0 /* 0x200 */ 1973 .long 0 /* 0x204 */ 1974 .long 0 /* 0x208 */ 1975 .long 0 /* 0x20c */ 1976 .long 0 /* 0x210 */ 1977 .long 0 /* 0x214 */ 1978 .long 0 /* 0x218 */ 1979 .long 0 /* 0x21c */ 1980 .long 0 /* 0x220 */ 1981 .long 0 /* 0x224 */ 1982 .long 0 /* 0x228 */ 1983 .long 0 /* 0x22c */ 1984 .long 0 /* 0x230 */ 1985 .long 0 /* 0x234 */ 1986 .long 0 /* 0x238 */ 1987 .long 0 /* 0x23c */ 1988 .long 0 /* 0x240 */ 1989 .long 0 /* 0x244 */ 1990 .long 0 /* 0x248 */ 1991 .long 0 /* 0x24c */ 1992 .long 0 /* 0x250 */ 1993 .long 0 /* 0x254 */ 1994 .long 0 /* 0x258 */ 1995 .long 0 /* 0x25c */ 1996 .long 0 /* 0x260 */ 1997 .long 0 /* 0x264 */ 1998 .long 0 /* 0x268 */ 1999 .long 0 /* 0x26c */ 2000 .long 0 /* 0x270 */ 2001 .long 0 /* 0x274 */ 2002 .long 0 /* 0x278 */ 2003 .long 0 /* 0x27c */ 2004 .long 0 /* 0x280 */ 2005 .long 0 /* 0x284 */ 2006 .long 0 /* 0x288 */ 2007 .long 0 /* 0x28c */ 2008 .long 0 /* 0x290 */ 2009 .long 0 /* 0x294 */ 2010 .long 0 /* 0x298 */ 2011 .long 0 /* 0x29c */ 2012 .long 0 /* 0x2a0 */ 2013 .long 0 /* 0x2a4 */ 2014 .long 0 /* 0x2a8 */ 2015 .long 0 /* 0x2ac */ 2016 .long 0 /* 0x2b0 */ 2017 .long 0 /* 0x2b4 */ 2018 .long 0 /* 0x2b8 */ 2019 .long 0 /* 0x2bc */ 2020 .long 0 /* 0x2c0 */ 2021 .long 0 /* 0x2c4 */ 2022 .long 0 /* 0x2c8 */ 2023 .long 0 /* 0x2cc */ 2024 .long 0 /* 0x2d0 */ 2025 .long 0 /* 0x2d4 */ 2026 .long 0 /* 0x2d8 */ 2027 .long 0 /* 0x2dc */ 2028 .long 0 /* 0x2e0 */ 2029 .long 0 /* 0x2e4 */ 2030 .long 0 /* 0x2e8 */ 2031 .long 0 /* 0x2ec */ 2032 .long 0 /* 0x2f0 */ 2033 .long 0 /* 0x2f4 */ 2034 .long 0 /* 0x2f8 */ 2035#ifdef CONFIG_KVM_XICS 2036 .long DOTSYM(kvmppc_rm_h_xirr_x) - hcall_real_table 2037#else 2038 .long 0 /* 0x2fc - H_XIRR_X*/ 2039#endif 2040 .long DOTSYM(kvmppc_rm_h_random) - hcall_real_table 2041 .globl hcall_real_table_end 2042hcall_real_table_end: 2043 2044_GLOBAL(kvmppc_h_set_xdabr) 2045EXPORT_SYMBOL_GPL(kvmppc_h_set_xdabr) 2046 andi. r0, r5, DABRX_USER | DABRX_KERNEL 2047 beq 6f 2048 li r0, DABRX_USER | DABRX_KERNEL | DABRX_BTI 2049 andc. r0, r5, r0 2050 beq 3f 20516: li r3, H_PARAMETER 2052 blr 2053 2054_GLOBAL(kvmppc_h_set_dabr) 2055EXPORT_SYMBOL_GPL(kvmppc_h_set_dabr) 2056 li r5, DABRX_USER | DABRX_KERNEL 20573: 2058BEGIN_FTR_SECTION 2059 b 2f 2060END_FTR_SECTION_IFSET(CPU_FTR_ARCH_207S) 2061 std r4,VCPU_DABR(r3) 2062 stw r5, VCPU_DABRX(r3) 2063 mtspr SPRN_DABRX, r5 2064 /* Work around P7 bug where DABR can get corrupted on mtspr */ 20651: mtspr SPRN_DABR,r4 2066 mfspr r5, SPRN_DABR 2067 cmpd r4, r5 2068 bne 1b 2069 isync 2070 li r3,0 2071 blr 2072 20732: 2074 LOAD_REG_ADDR(r11, dawr_force_enable) 2075 lbz r11, 0(r11) 2076 cmpdi r11, 0 2077 bne 3f 2078 li r3, H_HARDWARE 2079 blr 20803: 2081 /* Emulate H_SET_DABR/X on P8 for the sake of compat mode guests */ 2082 rlwimi r5, r4, 5, DAWRX_DR | DAWRX_DW 2083 rlwimi r5, r4, 2, DAWRX_WT 2084 clrrdi r4, r4, 3 2085 std r4, VCPU_DAWR0(r3) 2086 std r5, VCPU_DAWRX0(r3) 2087 /* 2088 * If came in through the real mode hcall handler then it is necessary 2089 * to write the registers since the return path won't. Otherwise it is 2090 * sufficient to store then in the vcpu struct as they will be loaded 2091 * next time the vcpu is run. 2092 */ 2093 mfmsr r6 2094 andi. r6, r6, MSR_DR /* in real mode? */ 2095 bne 4f 2096 mtspr SPRN_DAWR0, r4 2097 mtspr SPRN_DAWRX0, r5 20984: li r3, 0 2099 blr 2100 2101_GLOBAL(kvmppc_h_cede) /* r3 = vcpu pointer, r11 = msr, r13 = paca */ 2102 ori r11,r11,MSR_EE 2103 std r11,VCPU_MSR(r3) 2104 li r0,1 2105 stb r0,VCPU_CEDED(r3) 2106 sync /* order setting ceded vs. testing prodded */ 2107 lbz r5,VCPU_PRODDED(r3) 2108 cmpwi r5,0 2109 bne kvm_cede_prodded 2110 li r12,0 /* set trap to 0 to say hcall is handled */ 2111 stw r12,VCPU_TRAP(r3) 2112 li r0,H_SUCCESS 2113 std r0,VCPU_GPR(R3)(r3) 2114 2115 /* 2116 * Set our bit in the bitmask of napping threads unless all the 2117 * other threads are already napping, in which case we send this 2118 * up to the host. 2119 */ 2120 ld r5,HSTATE_KVM_VCORE(r13) 2121 lbz r6,HSTATE_PTID(r13) 2122 lwz r8,VCORE_ENTRY_EXIT(r5) 2123 clrldi r8,r8,56 2124 li r0,1 2125 sld r0,r0,r6 2126 addi r6,r5,VCORE_NAPPING_THREADS 212731: lwarx r4,0,r6 2128 or r4,r4,r0 2129 cmpw r4,r8 2130 beq kvm_cede_exit 2131 stwcx. r4,0,r6 2132 bne 31b 2133 /* order napping_threads update vs testing entry_exit_map */ 2134 isync 2135 li r0,NAPPING_CEDE 2136 stb r0,HSTATE_NAPPING(r13) 2137 lwz r7,VCORE_ENTRY_EXIT(r5) 2138 cmpwi r7,0x100 2139 bge 33f /* another thread already exiting */ 2140 2141/* 2142 * Although not specifically required by the architecture, POWER7 2143 * preserves the following registers in nap mode, even if an SMT mode 2144 * switch occurs: SLB entries, PURR, SPURR, AMOR, UAMOR, AMR, SPRG0-3, 2145 * DAR, DSISR, DABR, DABRX, DSCR, PMCx, MMCRx, SIAR, SDAR. 2146 */ 2147 /* Save non-volatile GPRs */ 2148 std r14, VCPU_GPR(R14)(r3) 2149 std r15, VCPU_GPR(R15)(r3) 2150 std r16, VCPU_GPR(R16)(r3) 2151 std r17, VCPU_GPR(R17)(r3) 2152 std r18, VCPU_GPR(R18)(r3) 2153 std r19, VCPU_GPR(R19)(r3) 2154 std r20, VCPU_GPR(R20)(r3) 2155 std r21, VCPU_GPR(R21)(r3) 2156 std r22, VCPU_GPR(R22)(r3) 2157 std r23, VCPU_GPR(R23)(r3) 2158 std r24, VCPU_GPR(R24)(r3) 2159 std r25, VCPU_GPR(R25)(r3) 2160 std r26, VCPU_GPR(R26)(r3) 2161 std r27, VCPU_GPR(R27)(r3) 2162 std r28, VCPU_GPR(R28)(r3) 2163 std r29, VCPU_GPR(R29)(r3) 2164 std r30, VCPU_GPR(R30)(r3) 2165 std r31, VCPU_GPR(R31)(r3) 2166 2167 /* save FP state */ 2168 bl kvmppc_save_fp 2169 2170#ifdef CONFIG_PPC_TRANSACTIONAL_MEM 2171BEGIN_FTR_SECTION 2172 b 91f 2173END_FTR_SECTION_IFCLR(CPU_FTR_TM) 2174 /* 2175 * NOTE THAT THIS TRASHES ALL NON-VOLATILE REGISTERS (but not CR) 2176 */ 2177 ld r3, HSTATE_KVM_VCPU(r13) 2178 ld r4, VCPU_MSR(r3) 2179 li r5, 0 /* don't preserve non-vol regs */ 2180 bl kvmppc_save_tm_hv 2181 nop 218291: 2183#endif 2184 2185 /* 2186 * Set DEC to the smaller of DEC and HDEC, so that we wake 2187 * no later than the end of our timeslice (HDEC interrupts 2188 * don't wake us from nap). 2189 */ 2190 mfspr r3, SPRN_DEC 2191 mfspr r4, SPRN_HDEC 2192 mftb r5 2193 extsw r3, r3 2194 extsw r4, r4 2195 cmpd r3, r4 2196 ble 67f 2197 mtspr SPRN_DEC, r4 219867: 2199 /* save expiry time of guest decrementer */ 2200 add r3, r3, r5 2201 ld r4, HSTATE_KVM_VCPU(r13) 2202 ld r5, HSTATE_KVM_VCORE(r13) 2203 ld r6, VCORE_TB_OFFSET_APPL(r5) 2204 subf r3, r6, r3 /* convert to host TB value */ 2205 std r3, VCPU_DEC_EXPIRES(r4) 2206 2207#ifdef CONFIG_KVM_BOOK3S_HV_EXIT_TIMING 2208 ld r4, HSTATE_KVM_VCPU(r13) 2209 addi r3, r4, VCPU_TB_CEDE 2210 bl kvmhv_accumulate_time 2211#endif 2212 2213 lis r3, LPCR_PECEDP@h /* Do wake on privileged doorbell */ 2214 2215 /* Go back to host stack */ 2216 ld r1, HSTATE_HOST_R1(r13) 2217 2218 /* 2219 * Take a nap until a decrementer or external or doobell interrupt 2220 * occurs, with PECE1 and PECE0 set in LPCR. 2221 * On POWER8, set PECEDH, and if we are ceding, also set PECEDP. 2222 * Also clear the runlatch bit before napping. 2223 */ 2224kvm_do_nap: 2225 mfspr r0, SPRN_CTRLF 2226 clrrdi r0, r0, 1 2227 mtspr SPRN_CTRLT, r0 2228 2229 li r0,1 2230 stb r0,HSTATE_HWTHREAD_REQ(r13) 2231 mfspr r5,SPRN_LPCR 2232 ori r5,r5,LPCR_PECE0 | LPCR_PECE1 2233BEGIN_FTR_SECTION 2234 ori r5, r5, LPCR_PECEDH 2235 rlwimi r5, r3, 0, LPCR_PECEDP 2236END_FTR_SECTION_IFSET(CPU_FTR_ARCH_207S) 2237 2238kvm_nap_sequence: /* desired LPCR value in r5 */ 2239 li r3, PNV_THREAD_NAP 2240 mtspr SPRN_LPCR,r5 2241 isync 2242 2243 bl isa206_idle_insn_mayloss 2244 2245 mfspr r0, SPRN_CTRLF 2246 ori r0, r0, 1 2247 mtspr SPRN_CTRLT, r0 2248 2249 mtspr SPRN_SRR1, r3 2250 2251 li r0, 0 2252 stb r0, PACA_FTRACE_ENABLED(r13) 2253 2254 li r0, KVM_HWTHREAD_IN_KVM 2255 stb r0, HSTATE_HWTHREAD_STATE(r13) 2256 2257 lbz r0, HSTATE_NAPPING(r13) 2258 cmpwi r0, NAPPING_CEDE 2259 beq kvm_end_cede 2260 cmpwi r0, NAPPING_NOVCPU 2261 beq kvm_novcpu_wakeup 2262 cmpwi r0, NAPPING_UNSPLIT 2263 beq kvm_unsplit_wakeup 2264 twi 31,0,0 /* Nap state must not be zero */ 2265 226633: mr r4, r3 2267 li r3, 0 2268 li r12, 0 2269 b 34f 2270 2271kvm_end_cede: 2272 /* Woken by external or decrementer interrupt */ 2273 2274 /* get vcpu pointer */ 2275 ld r4, HSTATE_KVM_VCPU(r13) 2276 2277#ifdef CONFIG_KVM_BOOK3S_HV_EXIT_TIMING 2278 addi r3, r4, VCPU_TB_RMINTR 2279 bl kvmhv_accumulate_time 2280#endif 2281 2282#ifdef CONFIG_PPC_TRANSACTIONAL_MEM 2283BEGIN_FTR_SECTION 2284 b 91f 2285END_FTR_SECTION_IFCLR(CPU_FTR_TM) 2286 /* 2287 * NOTE THAT THIS TRASHES ALL NON-VOLATILE REGISTERS (but not CR) 2288 */ 2289 mr r3, r4 2290 ld r4, VCPU_MSR(r3) 2291 li r5, 0 /* don't preserve non-vol regs */ 2292 bl kvmppc_restore_tm_hv 2293 nop 2294 ld r4, HSTATE_KVM_VCPU(r13) 229591: 2296#endif 2297 2298 /* load up FP state */ 2299 bl kvmppc_load_fp 2300 2301 /* Restore guest decrementer */ 2302 ld r3, VCPU_DEC_EXPIRES(r4) 2303 ld r5, HSTATE_KVM_VCORE(r13) 2304 ld r6, VCORE_TB_OFFSET_APPL(r5) 2305 add r3, r3, r6 /* convert host TB to guest TB value */ 2306 mftb r7 2307 subf r3, r7, r3 2308 mtspr SPRN_DEC, r3 2309 2310 /* Load NV GPRS */ 2311 ld r14, VCPU_GPR(R14)(r4) 2312 ld r15, VCPU_GPR(R15)(r4) 2313 ld r16, VCPU_GPR(R16)(r4) 2314 ld r17, VCPU_GPR(R17)(r4) 2315 ld r18, VCPU_GPR(R18)(r4) 2316 ld r19, VCPU_GPR(R19)(r4) 2317 ld r20, VCPU_GPR(R20)(r4) 2318 ld r21, VCPU_GPR(R21)(r4) 2319 ld r22, VCPU_GPR(R22)(r4) 2320 ld r23, VCPU_GPR(R23)(r4) 2321 ld r24, VCPU_GPR(R24)(r4) 2322 ld r25, VCPU_GPR(R25)(r4) 2323 ld r26, VCPU_GPR(R26)(r4) 2324 ld r27, VCPU_GPR(R27)(r4) 2325 ld r28, VCPU_GPR(R28)(r4) 2326 ld r29, VCPU_GPR(R29)(r4) 2327 ld r30, VCPU_GPR(R30)(r4) 2328 ld r31, VCPU_GPR(R31)(r4) 2329 2330 /* Check the wake reason in SRR1 to see why we got here */ 2331 bl kvmppc_check_wake_reason 2332 2333 /* 2334 * Restore volatile registers since we could have called a 2335 * C routine in kvmppc_check_wake_reason 2336 * r4 = VCPU 2337 * r3 tells us whether we need to return to host or not 2338 * WARNING: it gets checked further down: 2339 * should not modify r3 until this check is done. 2340 */ 2341 ld r4, HSTATE_KVM_VCPU(r13) 2342 2343 /* clear our bit in vcore->napping_threads */ 234434: ld r5,HSTATE_KVM_VCORE(r13) 2345 lbz r7,HSTATE_PTID(r13) 2346 li r0,1 2347 sld r0,r0,r7 2348 addi r6,r5,VCORE_NAPPING_THREADS 234932: lwarx r7,0,r6 2350 andc r7,r7,r0 2351 stwcx. r7,0,r6 2352 bne 32b 2353 li r0,0 2354 stb r0,HSTATE_NAPPING(r13) 2355 2356 /* See if the wake reason saved in r3 means we need to exit */ 2357 stw r12, VCPU_TRAP(r4) 2358 mr r9, r4 2359 cmpdi r3, 0 2360 bgt guest_exit_cont 2361 b maybe_reenter_guest 2362 2363 /* cede when already previously prodded case */ 2364kvm_cede_prodded: 2365 li r0,0 2366 stb r0,VCPU_PRODDED(r3) 2367 sync /* order testing prodded vs. clearing ceded */ 2368 stb r0,VCPU_CEDED(r3) 2369 li r3,H_SUCCESS 2370 blr 2371 2372 /* we've ceded but we want to give control to the host */ 2373kvm_cede_exit: 2374 ld r9, HSTATE_KVM_VCPU(r13) 2375 b guest_exit_cont 2376 2377 /* Try to do machine check recovery in real mode */ 2378machine_check_realmode: 2379 mr r3, r9 /* get vcpu pointer */ 2380 bl kvmppc_realmode_machine_check 2381 nop 2382 /* all machine checks go to virtual mode for further handling */ 2383 ld r9, HSTATE_KVM_VCPU(r13) 2384 li r12, BOOK3S_INTERRUPT_MACHINE_CHECK 2385 b guest_exit_cont 2386 2387/* 2388 * Call C code to handle a HMI in real mode. 2389 * Only the primary thread does the call, secondary threads are handled 2390 * by calling hmi_exception_realmode() after kvmppc_hv_entry returns. 2391 * r9 points to the vcpu on entry 2392 */ 2393hmi_realmode: 2394 lbz r0, HSTATE_PTID(r13) 2395 cmpwi r0, 0 2396 bne guest_exit_cont 2397 bl kvmppc_realmode_hmi_handler 2398 ld r9, HSTATE_KVM_VCPU(r13) 2399 li r12, BOOK3S_INTERRUPT_HMI 2400 b guest_exit_cont 2401 2402/* 2403 * Check the reason we woke from nap, and take appropriate action. 2404 * Returns (in r3): 2405 * 0 if nothing needs to be done 2406 * 1 if something happened that needs to be handled by the host 2407 * -1 if there was a guest wakeup (IPI or msgsnd) 2408 * -2 if we handled a PCI passthrough interrupt (returned by 2409 * kvmppc_read_intr only) 2410 * 2411 * Also sets r12 to the interrupt vector for any interrupt that needs 2412 * to be handled now by the host (0x500 for external interrupt), or zero. 2413 * Modifies all volatile registers (since it may call a C function). 2414 * This routine calls kvmppc_read_intr, a C function, if an external 2415 * interrupt is pending. 2416 */ 2417kvmppc_check_wake_reason: 2418 mfspr r6, SPRN_SRR1 2419BEGIN_FTR_SECTION 2420 rlwinm r6, r6, 45-31, 0xf /* extract wake reason field (P8) */ 2421FTR_SECTION_ELSE 2422 rlwinm r6, r6, 45-31, 0xe /* P7 wake reason field is 3 bits */ 2423ALT_FTR_SECTION_END_IFSET(CPU_FTR_ARCH_207S) 2424 cmpwi r6, 8 /* was it an external interrupt? */ 2425 beq 7f /* if so, see what it was */ 2426 li r3, 0 2427 li r12, 0 2428 cmpwi r6, 6 /* was it the decrementer? */ 2429 beq 0f 2430BEGIN_FTR_SECTION 2431 cmpwi r6, 5 /* privileged doorbell? */ 2432 beq 0f 2433 cmpwi r6, 3 /* hypervisor doorbell? */ 2434 beq 3f 2435END_FTR_SECTION_IFSET(CPU_FTR_ARCH_207S) 2436 cmpwi r6, 0xa /* Hypervisor maintenance ? */ 2437 beq 4f 2438 li r3, 1 /* anything else, return 1 */ 24390: blr 2440 2441 /* hypervisor doorbell */ 24423: li r12, BOOK3S_INTERRUPT_H_DOORBELL 2443 2444 /* 2445 * Clear the doorbell as we will invoke the handler 2446 * explicitly in the guest exit path. 2447 */ 2448 lis r6, (PPC_DBELL_SERVER << (63-36))@h 2449 PPC_MSGCLR(6) 2450 /* see if it's a host IPI */ 2451 li r3, 1 2452 lbz r0, HSTATE_HOST_IPI(r13) 2453 cmpwi r0, 0 2454 bnelr 2455 /* if not, return -1 */ 2456 li r3, -1 2457 blr 2458 2459 /* Woken up due to Hypervisor maintenance interrupt */ 24604: li r12, BOOK3S_INTERRUPT_HMI 2461 li r3, 1 2462 blr 2463 2464 /* external interrupt - create a stack frame so we can call C */ 24657: mflr r0 2466 std r0, PPC_LR_STKOFF(r1) 2467 stdu r1, -PPC_MIN_STKFRM(r1) 2468 bl kvmppc_read_intr 2469 nop 2470 li r12, BOOK3S_INTERRUPT_EXTERNAL 2471 cmpdi r3, 1 2472 ble 1f 2473 2474 /* 2475 * Return code of 2 means PCI passthrough interrupt, but 2476 * we need to return back to host to complete handling the 2477 * interrupt. Trap reason is expected in r12 by guest 2478 * exit code. 2479 */ 2480 li r12, BOOK3S_INTERRUPT_HV_RM_HARD 24811: 2482 ld r0, PPC_MIN_STKFRM+PPC_LR_STKOFF(r1) 2483 addi r1, r1, PPC_MIN_STKFRM 2484 mtlr r0 2485 blr 2486 2487/* 2488 * Save away FP, VMX and VSX registers. 2489 * r3 = vcpu pointer 2490 * N.B. r30 and r31 are volatile across this function, 2491 * thus it is not callable from C. 2492 */ 2493kvmppc_save_fp: 2494 mflr r30 2495 mr r31,r3 2496 mfmsr r5 2497 ori r8,r5,MSR_FP 2498#ifdef CONFIG_ALTIVEC 2499BEGIN_FTR_SECTION 2500 oris r8,r8,MSR_VEC@h 2501END_FTR_SECTION_IFSET(CPU_FTR_ALTIVEC) 2502#endif 2503#ifdef CONFIG_VSX 2504BEGIN_FTR_SECTION 2505 oris r8,r8,MSR_VSX@h 2506END_FTR_SECTION_IFSET(CPU_FTR_VSX) 2507#endif 2508 mtmsrd r8 2509 addi r3,r3,VCPU_FPRS 2510 bl store_fp_state 2511#ifdef CONFIG_ALTIVEC 2512BEGIN_FTR_SECTION 2513 addi r3,r31,VCPU_VRS 2514 bl store_vr_state 2515END_FTR_SECTION_IFSET(CPU_FTR_ALTIVEC) 2516#endif 2517 mfspr r6,SPRN_VRSAVE 2518 stw r6,VCPU_VRSAVE(r31) 2519 mtlr r30 2520 blr 2521 2522/* 2523 * Load up FP, VMX and VSX registers 2524 * r4 = vcpu pointer 2525 * N.B. r30 and r31 are volatile across this function, 2526 * thus it is not callable from C. 2527 */ 2528kvmppc_load_fp: 2529 mflr r30 2530 mr r31,r4 2531 mfmsr r9 2532 ori r8,r9,MSR_FP 2533#ifdef CONFIG_ALTIVEC 2534BEGIN_FTR_SECTION 2535 oris r8,r8,MSR_VEC@h 2536END_FTR_SECTION_IFSET(CPU_FTR_ALTIVEC) 2537#endif 2538#ifdef CONFIG_VSX 2539BEGIN_FTR_SECTION 2540 oris r8,r8,MSR_VSX@h 2541END_FTR_SECTION_IFSET(CPU_FTR_VSX) 2542#endif 2543 mtmsrd r8 2544 addi r3,r4,VCPU_FPRS 2545 bl load_fp_state 2546#ifdef CONFIG_ALTIVEC 2547BEGIN_FTR_SECTION 2548 addi r3,r31,VCPU_VRS 2549 bl load_vr_state 2550END_FTR_SECTION_IFSET(CPU_FTR_ALTIVEC) 2551#endif 2552 lwz r7,VCPU_VRSAVE(r31) 2553 mtspr SPRN_VRSAVE,r7 2554 mtlr r30 2555 mr r4,r31 2556 blr 2557 2558#ifdef CONFIG_PPC_TRANSACTIONAL_MEM 2559/* 2560 * Save transactional state and TM-related registers. 2561 * Called with r3 pointing to the vcpu struct and r4 containing 2562 * the guest MSR value. 2563 * r5 is non-zero iff non-volatile register state needs to be maintained. 2564 * If r5 == 0, this can modify all checkpointed registers, but 2565 * restores r1 and r2 before exit. 2566 */ 2567_GLOBAL_TOC(kvmppc_save_tm_hv) 2568EXPORT_SYMBOL_GPL(kvmppc_save_tm_hv) 2569 /* See if we need to handle fake suspend mode */ 2570BEGIN_FTR_SECTION 2571 b __kvmppc_save_tm 2572END_FTR_SECTION_IFCLR(CPU_FTR_P9_TM_HV_ASSIST) 2573 2574 lbz r0, HSTATE_FAKE_SUSPEND(r13) /* Were we fake suspended? */ 2575 cmpwi r0, 0 2576 beq __kvmppc_save_tm 2577 2578 /* The following code handles the fake_suspend = 1 case */ 2579 mflr r0 2580 std r0, PPC_LR_STKOFF(r1) 2581 stdu r1, -PPC_MIN_STKFRM(r1) 2582 2583 /* Turn on TM. */ 2584 mfmsr r8 2585 li r0, 1 2586 rldimi r8, r0, MSR_TM_LG, 63-MSR_TM_LG 2587 mtmsrd r8 2588 2589 rldicl. r8, r8, 64 - MSR_TS_S_LG, 62 /* Did we actually hrfid? */ 2590 beq 4f 2591BEGIN_FTR_SECTION 2592 bl pnv_power9_force_smt4_catch 2593END_FTR_SECTION_IFSET(CPU_FTR_P9_TM_XER_SO_BUG) 2594 nop 2595 2596 /* We have to treclaim here because that's the only way to do S->N */ 2597 li r3, TM_CAUSE_KVM_RESCHED 2598 TRECLAIM(R3) 2599 2600 /* 2601 * We were in fake suspend, so we are not going to save the 2602 * register state as the guest checkpointed state (since 2603 * we already have it), therefore we can now use any volatile GPR. 2604 * In fact treclaim in fake suspend state doesn't modify 2605 * any registers. 2606 */ 2607 2608BEGIN_FTR_SECTION 2609 bl pnv_power9_force_smt4_release 2610END_FTR_SECTION_IFSET(CPU_FTR_P9_TM_XER_SO_BUG) 2611 nop 2612 26134: 2614 mfspr r3, SPRN_PSSCR 2615 /* PSSCR_FAKE_SUSPEND is a write-only bit, but clear it anyway */ 2616 li r0, PSSCR_FAKE_SUSPEND 2617 andc r3, r3, r0 2618 mtspr SPRN_PSSCR, r3 2619 2620 /* Don't save TEXASR, use value from last exit in real suspend state */ 2621 ld r9, HSTATE_KVM_VCPU(r13) 2622 mfspr r5, SPRN_TFHAR 2623 mfspr r6, SPRN_TFIAR 2624 std r5, VCPU_TFHAR(r9) 2625 std r6, VCPU_TFIAR(r9) 2626 2627 addi r1, r1, PPC_MIN_STKFRM 2628 ld r0, PPC_LR_STKOFF(r1) 2629 mtlr r0 2630 blr 2631 2632/* 2633 * Restore transactional state and TM-related registers. 2634 * Called with r3 pointing to the vcpu struct 2635 * and r4 containing the guest MSR value. 2636 * r5 is non-zero iff non-volatile register state needs to be maintained. 2637 * This potentially modifies all checkpointed registers. 2638 * It restores r1 and r2 from the PACA. 2639 */ 2640_GLOBAL_TOC(kvmppc_restore_tm_hv) 2641EXPORT_SYMBOL_GPL(kvmppc_restore_tm_hv) 2642 /* 2643 * If we are doing TM emulation for the guest on a POWER9 DD2, 2644 * then we don't actually do a trechkpt -- we either set up 2645 * fake-suspend mode, or emulate a TM rollback. 2646 */ 2647BEGIN_FTR_SECTION 2648 b __kvmppc_restore_tm 2649END_FTR_SECTION_IFCLR(CPU_FTR_P9_TM_HV_ASSIST) 2650 mflr r0 2651 std r0, PPC_LR_STKOFF(r1) 2652 2653 li r0, 0 2654 stb r0, HSTATE_FAKE_SUSPEND(r13) 2655 2656 /* Turn on TM so we can restore TM SPRs */ 2657 mfmsr r5 2658 li r0, 1 2659 rldimi r5, r0, MSR_TM_LG, 63-MSR_TM_LG 2660 mtmsrd r5 2661 2662 /* 2663 * The user may change these outside of a transaction, so they must 2664 * always be context switched. 2665 */ 2666 ld r5, VCPU_TFHAR(r3) 2667 ld r6, VCPU_TFIAR(r3) 2668 ld r7, VCPU_TEXASR(r3) 2669 mtspr SPRN_TFHAR, r5 2670 mtspr SPRN_TFIAR, r6 2671 mtspr SPRN_TEXASR, r7 2672 2673 rldicl. r5, r4, 64 - MSR_TS_S_LG, 62 2674 beqlr /* TM not active in guest */ 2675 2676 /* Make sure the failure summary is set */ 2677 oris r7, r7, (TEXASR_FS)@h 2678 mtspr SPRN_TEXASR, r7 2679 2680 cmpwi r5, 1 /* check for suspended state */ 2681 bgt 10f 2682 stb r5, HSTATE_FAKE_SUSPEND(r13) 2683 b 9f /* and return */ 268410: stdu r1, -PPC_MIN_STKFRM(r1) 2685 /* guest is in transactional state, so simulate rollback */ 2686 bl kvmhv_emulate_tm_rollback 2687 nop 2688 addi r1, r1, PPC_MIN_STKFRM 26899: ld r0, PPC_LR_STKOFF(r1) 2690 mtlr r0 2691 blr 2692#endif /* CONFIG_PPC_TRANSACTIONAL_MEM */ 2693 2694/* 2695 * We come here if we get any exception or interrupt while we are 2696 * executing host real mode code while in guest MMU context. 2697 * r12 is (CR << 32) | vector 2698 * r13 points to our PACA 2699 * r12 is saved in HSTATE_SCRATCH0(r13) 2700 * r9 is saved in HSTATE_SCRATCH2(r13) 2701 * r13 is saved in HSPRG1 2702 * cfar is saved in HSTATE_CFAR(r13) 2703 * ppr is saved in HSTATE_PPR(r13) 2704 */ 2705kvmppc_bad_host_intr: 2706 /* 2707 * Switch to the emergency stack, but start half-way down in 2708 * case we were already on it. 2709 */ 2710 mr r9, r1 2711 std r1, PACAR1(r13) 2712 ld r1, PACAEMERGSP(r13) 2713 subi r1, r1, THREAD_SIZE/2 + INT_FRAME_SIZE 2714 std r9, 0(r1) 2715 std r0, GPR0(r1) 2716 std r9, GPR1(r1) 2717 std r2, GPR2(r1) 2718 SAVE_4GPRS(3, r1) 2719 SAVE_2GPRS(7, r1) 2720 srdi r0, r12, 32 2721 clrldi r12, r12, 32 2722 std r0, _CCR(r1) 2723 std r12, _TRAP(r1) 2724 andi. r0, r12, 2 2725 beq 1f 2726 mfspr r3, SPRN_HSRR0 2727 mfspr r4, SPRN_HSRR1 2728 mfspr r5, SPRN_HDAR 2729 mfspr r6, SPRN_HDSISR 2730 b 2f 27311: mfspr r3, SPRN_SRR0 2732 mfspr r4, SPRN_SRR1 2733 mfspr r5, SPRN_DAR 2734 mfspr r6, SPRN_DSISR 27352: std r3, _NIP(r1) 2736 std r4, _MSR(r1) 2737 std r5, _DAR(r1) 2738 std r6, _DSISR(r1) 2739 ld r9, HSTATE_SCRATCH2(r13) 2740 ld r12, HSTATE_SCRATCH0(r13) 2741 GET_SCRATCH0(r0) 2742 SAVE_4GPRS(9, r1) 2743 std r0, GPR13(r1) 2744 SAVE_NVGPRS(r1) 2745 ld r5, HSTATE_CFAR(r13) 2746 std r5, ORIG_GPR3(r1) 2747 mflr r3 2748 mfctr r4 2749 mfxer r5 2750 lbz r6, PACAIRQSOFTMASK(r13) 2751 std r3, _LINK(r1) 2752 std r4, _CTR(r1) 2753 std r5, _XER(r1) 2754 std r6, SOFTE(r1) 2755 ld r2, PACATOC(r13) 2756 LOAD_REG_IMMEDIATE(3, 0x7265677368657265) 2757 std r3, STACK_FRAME_OVERHEAD-16(r1) 2758 2759 /* 2760 * XXX On POWER7 and POWER8, we just spin here since we don't 2761 * know what the other threads are doing (and we don't want to 2762 * coordinate with them) - but at least we now have register state 2763 * in memory that we might be able to look at from another CPU. 2764 */ 2765 b . 2766 2767/* 2768 * This mimics the MSR transition on IRQ delivery. The new guest MSR is taken 2769 * from VCPU_INTR_MSR and is modified based on the required TM state changes. 2770 * r11 has the guest MSR value (in/out) 2771 * r9 has a vcpu pointer (in) 2772 * r0 is used as a scratch register 2773 */ 2774kvmppc_msr_interrupt: 2775 rldicl r0, r11, 64 - MSR_TS_S_LG, 62 2776 cmpwi r0, 2 /* Check if we are in transactional state.. */ 2777 ld r11, VCPU_INTR_MSR(r9) 2778 bne 1f 2779 /* ... if transactional, change to suspended */ 2780 li r0, 1 27811: rldimi r11, r0, MSR_TS_S_LG, 63 - MSR_TS_T_LG 2782 blr 2783 2784/* 2785 * Load up guest PMU state. R3 points to the vcpu struct. 2786 */ 2787_GLOBAL(kvmhv_load_guest_pmu) 2788EXPORT_SYMBOL_GPL(kvmhv_load_guest_pmu) 2789 mr r4, r3 2790 mflr r0 2791 li r3, 1 2792 sldi r3, r3, 31 /* MMCR0_FC (freeze counters) bit */ 2793 mtspr SPRN_MMCR0, r3 /* freeze all counters, disable ints */ 2794 isync 2795BEGIN_FTR_SECTION 2796 ld r3, VCPU_MMCR(r4) 2797 andi. r5, r3, MMCR0_PMAO_SYNC | MMCR0_PMAO 2798 cmpwi r5, MMCR0_PMAO 2799 beql kvmppc_fix_pmao 2800END_FTR_SECTION_IFSET(CPU_FTR_PMAO_BUG) 2801 lwz r3, VCPU_PMC(r4) /* always load up guest PMU registers */ 2802 lwz r5, VCPU_PMC + 4(r4) /* to prevent information leak */ 2803 lwz r6, VCPU_PMC + 8(r4) 2804 lwz r7, VCPU_PMC + 12(r4) 2805 lwz r8, VCPU_PMC + 16(r4) 2806 lwz r9, VCPU_PMC + 20(r4) 2807 mtspr SPRN_PMC1, r3 2808 mtspr SPRN_PMC2, r5 2809 mtspr SPRN_PMC3, r6 2810 mtspr SPRN_PMC4, r7 2811 mtspr SPRN_PMC5, r8 2812 mtspr SPRN_PMC6, r9 2813 ld r3, VCPU_MMCR(r4) 2814 ld r5, VCPU_MMCR + 8(r4) 2815 ld r6, VCPU_MMCRA(r4) 2816 ld r7, VCPU_SIAR(r4) 2817 ld r8, VCPU_SDAR(r4) 2818 mtspr SPRN_MMCR1, r5 2819 mtspr SPRN_MMCRA, r6 2820 mtspr SPRN_SIAR, r7 2821 mtspr SPRN_SDAR, r8 2822BEGIN_FTR_SECTION 2823 ld r5, VCPU_MMCR + 24(r4) 2824 ld r6, VCPU_SIER + 8(r4) 2825 ld r7, VCPU_SIER + 16(r4) 2826 mtspr SPRN_MMCR3, r5 2827 mtspr SPRN_SIER2, r6 2828 mtspr SPRN_SIER3, r7 2829END_FTR_SECTION_IFSET(CPU_FTR_ARCH_31) 2830BEGIN_FTR_SECTION 2831 ld r5, VCPU_MMCR + 16(r4) 2832 ld r6, VCPU_SIER(r4) 2833 mtspr SPRN_MMCR2, r5 2834 mtspr SPRN_SIER, r6 2835BEGIN_FTR_SECTION_NESTED(96) 2836 lwz r7, VCPU_PMC + 24(r4) 2837 lwz r8, VCPU_PMC + 28(r4) 2838 ld r9, VCPU_MMCRS(r4) 2839 mtspr SPRN_SPMC1, r7 2840 mtspr SPRN_SPMC2, r8 2841 mtspr SPRN_MMCRS, r9 2842END_FTR_SECTION_NESTED(CPU_FTR_ARCH_300, 0, 96) 2843END_FTR_SECTION_IFSET(CPU_FTR_ARCH_207S) 2844 mtspr SPRN_MMCR0, r3 2845 isync 2846 mtlr r0 2847 blr 2848 2849/* 2850 * Reload host PMU state saved in the PACA by kvmhv_save_host_pmu. 2851 */ 2852_GLOBAL(kvmhv_load_host_pmu) 2853EXPORT_SYMBOL_GPL(kvmhv_load_host_pmu) 2854 mflr r0 2855 lbz r4, PACA_PMCINUSE(r13) /* is the host using the PMU? */ 2856 cmpwi r4, 0 2857 beq 23f /* skip if not */ 2858BEGIN_FTR_SECTION 2859 ld r3, HSTATE_MMCR0(r13) 2860 andi. r4, r3, MMCR0_PMAO_SYNC | MMCR0_PMAO 2861 cmpwi r4, MMCR0_PMAO 2862 beql kvmppc_fix_pmao 2863END_FTR_SECTION_IFSET(CPU_FTR_PMAO_BUG) 2864 lwz r3, HSTATE_PMC1(r13) 2865 lwz r4, HSTATE_PMC2(r13) 2866 lwz r5, HSTATE_PMC3(r13) 2867 lwz r6, HSTATE_PMC4(r13) 2868 lwz r8, HSTATE_PMC5(r13) 2869 lwz r9, HSTATE_PMC6(r13) 2870 mtspr SPRN_PMC1, r3 2871 mtspr SPRN_PMC2, r4 2872 mtspr SPRN_PMC3, r5 2873 mtspr SPRN_PMC4, r6 2874 mtspr SPRN_PMC5, r8 2875 mtspr SPRN_PMC6, r9 2876 ld r3, HSTATE_MMCR0(r13) 2877 ld r4, HSTATE_MMCR1(r13) 2878 ld r5, HSTATE_MMCRA(r13) 2879 ld r6, HSTATE_SIAR(r13) 2880 ld r7, HSTATE_SDAR(r13) 2881 mtspr SPRN_MMCR1, r4 2882 mtspr SPRN_MMCRA, r5 2883 mtspr SPRN_SIAR, r6 2884 mtspr SPRN_SDAR, r7 2885BEGIN_FTR_SECTION 2886 ld r8, HSTATE_MMCR2(r13) 2887 ld r9, HSTATE_SIER(r13) 2888 mtspr SPRN_MMCR2, r8 2889 mtspr SPRN_SIER, r9 2890END_FTR_SECTION_IFSET(CPU_FTR_ARCH_207S) 2891BEGIN_FTR_SECTION 2892 ld r5, HSTATE_MMCR3(r13) 2893 ld r6, HSTATE_SIER2(r13) 2894 ld r7, HSTATE_SIER3(r13) 2895 mtspr SPRN_MMCR3, r5 2896 mtspr SPRN_SIER2, r6 2897 mtspr SPRN_SIER3, r7 2898END_FTR_SECTION_IFSET(CPU_FTR_ARCH_31) 2899 mtspr SPRN_MMCR0, r3 2900 isync 2901 mtlr r0 290223: blr 2903 2904/* 2905 * Save guest PMU state into the vcpu struct. 2906 * r3 = vcpu, r4 = full save flag (PMU in use flag set in VPA) 2907 */ 2908_GLOBAL(kvmhv_save_guest_pmu) 2909EXPORT_SYMBOL_GPL(kvmhv_save_guest_pmu) 2910 mr r9, r3 2911 mr r8, r4 2912BEGIN_FTR_SECTION 2913 /* 2914 * POWER8 seems to have a hardware bug where setting 2915 * MMCR0[PMAE] along with MMCR0[PMC1CE] and/or MMCR0[PMCjCE] 2916 * when some counters are already negative doesn't seem 2917 * to cause a performance monitor alert (and hence interrupt). 2918 * The effect of this is that when saving the PMU state, 2919 * if there is no PMU alert pending when we read MMCR0 2920 * before freezing the counters, but one becomes pending 2921 * before we read the counters, we lose it. 2922 * To work around this, we need a way to freeze the counters 2923 * before reading MMCR0. Normally, freezing the counters 2924 * is done by writing MMCR0 (to set MMCR0[FC]) which 2925 * unavoidably writes MMCR0[PMA0] as well. On POWER8, 2926 * we can also freeze the counters using MMCR2, by writing 2927 * 1s to all the counter freeze condition bits (there are 2928 * 9 bits each for 6 counters). 2929 */ 2930 li r3, -1 /* set all freeze bits */ 2931 clrrdi r3, r3, 10 2932 mfspr r10, SPRN_MMCR2 2933 mtspr SPRN_MMCR2, r3 2934 isync 2935END_FTR_SECTION_IFSET(CPU_FTR_ARCH_207S) 2936 li r3, 1 2937 sldi r3, r3, 31 /* MMCR0_FC (freeze counters) bit */ 2938 mfspr r4, SPRN_MMCR0 /* save MMCR0 */ 2939 mtspr SPRN_MMCR0, r3 /* freeze all counters, disable ints */ 2940 mfspr r6, SPRN_MMCRA 2941 /* Clear MMCRA in order to disable SDAR updates */ 2942 li r7, 0 2943 mtspr SPRN_MMCRA, r7 2944 isync 2945 cmpwi r8, 0 /* did they ask for PMU stuff to be saved? */ 2946 bne 21f 2947 std r3, VCPU_MMCR(r9) /* if not, set saved MMCR0 to FC */ 2948 b 22f 294921: mfspr r5, SPRN_MMCR1 2950 mfspr r7, SPRN_SIAR 2951 mfspr r8, SPRN_SDAR 2952 std r4, VCPU_MMCR(r9) 2953 std r5, VCPU_MMCR + 8(r9) 2954 std r6, VCPU_MMCRA(r9) 2955BEGIN_FTR_SECTION 2956 std r10, VCPU_MMCR + 16(r9) 2957END_FTR_SECTION_IFSET(CPU_FTR_ARCH_207S) 2958BEGIN_FTR_SECTION 2959 mfspr r5, SPRN_MMCR3 2960 mfspr r6, SPRN_SIER2 2961 mfspr r7, SPRN_SIER3 2962 std r5, VCPU_MMCR + 24(r9) 2963 std r6, VCPU_SIER + 8(r9) 2964 std r7, VCPU_SIER + 16(r9) 2965END_FTR_SECTION_IFSET(CPU_FTR_ARCH_31) 2966 std r7, VCPU_SIAR(r9) 2967 std r8, VCPU_SDAR(r9) 2968 mfspr r3, SPRN_PMC1 2969 mfspr r4, SPRN_PMC2 2970 mfspr r5, SPRN_PMC3 2971 mfspr r6, SPRN_PMC4 2972 mfspr r7, SPRN_PMC5 2973 mfspr r8, SPRN_PMC6 2974 stw r3, VCPU_PMC(r9) 2975 stw r4, VCPU_PMC + 4(r9) 2976 stw r5, VCPU_PMC + 8(r9) 2977 stw r6, VCPU_PMC + 12(r9) 2978 stw r7, VCPU_PMC + 16(r9) 2979 stw r8, VCPU_PMC + 20(r9) 2980BEGIN_FTR_SECTION 2981 mfspr r5, SPRN_SIER 2982 std r5, VCPU_SIER(r9) 2983BEGIN_FTR_SECTION_NESTED(96) 2984 mfspr r6, SPRN_SPMC1 2985 mfspr r7, SPRN_SPMC2 2986 mfspr r8, SPRN_MMCRS 2987 stw r6, VCPU_PMC + 24(r9) 2988 stw r7, VCPU_PMC + 28(r9) 2989 std r8, VCPU_MMCRS(r9) 2990 lis r4, 0x8000 2991 mtspr SPRN_MMCRS, r4 2992END_FTR_SECTION_NESTED(CPU_FTR_ARCH_300, 0, 96) 2993END_FTR_SECTION_IFSET(CPU_FTR_ARCH_207S) 299422: blr 2995 2996/* 2997 * This works around a hardware bug on POWER8E processors, where 2998 * writing a 1 to the MMCR0[PMAO] bit doesn't generate a 2999 * performance monitor interrupt. Instead, when we need to have 3000 * an interrupt pending, we have to arrange for a counter to overflow. 3001 */ 3002kvmppc_fix_pmao: 3003 li r3, 0 3004 mtspr SPRN_MMCR2, r3 3005 lis r3, (MMCR0_PMXE | MMCR0_FCECE)@h 3006 ori r3, r3, MMCR0_PMCjCE | MMCR0_C56RUN 3007 mtspr SPRN_MMCR0, r3 3008 lis r3, 0x7fff 3009 ori r3, r3, 0xffff 3010 mtspr SPRN_PMC6, r3 3011 isync 3012 blr 3013 3014#ifdef CONFIG_KVM_BOOK3S_HV_EXIT_TIMING 3015/* 3016 * Start timing an activity 3017 * r3 = pointer to time accumulation struct, r4 = vcpu 3018 */ 3019kvmhv_start_timing: 3020 ld r5, HSTATE_KVM_VCORE(r13) 3021 ld r6, VCORE_TB_OFFSET_APPL(r5) 3022 mftb r5 3023 subf r5, r6, r5 /* subtract current timebase offset */ 3024 std r3, VCPU_CUR_ACTIVITY(r4) 3025 std r5, VCPU_ACTIVITY_START(r4) 3026 blr 3027 3028/* 3029 * Accumulate time to one activity and start another. 3030 * r3 = pointer to new time accumulation struct, r4 = vcpu 3031 */ 3032kvmhv_accumulate_time: 3033 ld r5, HSTATE_KVM_VCORE(r13) 3034 ld r8, VCORE_TB_OFFSET_APPL(r5) 3035 ld r5, VCPU_CUR_ACTIVITY(r4) 3036 ld r6, VCPU_ACTIVITY_START(r4) 3037 std r3, VCPU_CUR_ACTIVITY(r4) 3038 mftb r7 3039 subf r7, r8, r7 /* subtract current timebase offset */ 3040 std r7, VCPU_ACTIVITY_START(r4) 3041 cmpdi r5, 0 3042 beqlr 3043 subf r3, r6, r7 3044 ld r8, TAS_SEQCOUNT(r5) 3045 cmpdi r8, 0 3046 addi r8, r8, 1 3047 std r8, TAS_SEQCOUNT(r5) 3048 lwsync 3049 ld r7, TAS_TOTAL(r5) 3050 add r7, r7, r3 3051 std r7, TAS_TOTAL(r5) 3052 ld r6, TAS_MIN(r5) 3053 ld r7, TAS_MAX(r5) 3054 beq 3f 3055 cmpd r3, r6 3056 bge 1f 30573: std r3, TAS_MIN(r5) 30581: cmpd r3, r7 3059 ble 2f 3060 std r3, TAS_MAX(r5) 30612: lwsync 3062 addi r8, r8, 1 3063 std r8, TAS_SEQCOUNT(r5) 3064 blr 3065#endif 3066