1/* 2 * This program is free software; you can redistribute it and/or modify 3 * it under the terms of the GNU General Public License, version 2, as 4 * published by the Free Software Foundation. 5 * 6 * This program is distributed in the hope that it will be useful, 7 * but WITHOUT ANY WARRANTY; without even the implied warranty of 8 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 9 * GNU General Public License for more details. 10 * 11 * Copyright 2011 Paul Mackerras, IBM Corp. <paulus@au1.ibm.com> 12 * 13 * Derived from book3s_rmhandlers.S and other files, which are: 14 * 15 * Copyright SUSE Linux Products GmbH 2009 16 * 17 * Authors: Alexander Graf <agraf@suse.de> 18 */ 19 20#include <asm/ppc_asm.h> 21#include <asm/kvm_asm.h> 22#include <asm/reg.h> 23#include <asm/mmu.h> 24#include <asm/page.h> 25#include <asm/ptrace.h> 26#include <asm/hvcall.h> 27#include <asm/asm-offsets.h> 28#include <asm/exception-64s.h> 29#include <asm/kvm_book3s_asm.h> 30#include <asm/book3s/64/mmu-hash.h> 31#include <asm/tm.h> 32#include <asm/opal.h> 33#include <asm/xive-regs.h> 34 35/* Sign-extend HDEC if not on POWER9 */ 36#define EXTEND_HDEC(reg) \ 37BEGIN_FTR_SECTION; \ 38 extsw reg, reg; \ 39END_FTR_SECTION_IFCLR(CPU_FTR_ARCH_300) 40 41#define VCPU_GPRS_TM(reg) (((reg) * ULONG_SIZE) + VCPU_GPR_TM) 42 43/* Values in HSTATE_NAPPING(r13) */ 44#define NAPPING_CEDE 1 45#define NAPPING_NOVCPU 2 46 47/* Stack frame offsets for kvmppc_hv_entry */ 48#define SFS 160 49#define STACK_SLOT_TRAP (SFS-4) 50#define STACK_SLOT_TID (SFS-16) 51#define STACK_SLOT_PSSCR (SFS-24) 52#define STACK_SLOT_PID (SFS-32) 53#define STACK_SLOT_IAMR (SFS-40) 54#define STACK_SLOT_CIABR (SFS-48) 55#define STACK_SLOT_DAWR (SFS-56) 56#define STACK_SLOT_DAWRX (SFS-64) 57#define STACK_SLOT_HFSCR (SFS-72) 58 59/* 60 * Call kvmppc_hv_entry in real mode. 61 * Must be called with interrupts hard-disabled. 62 * 63 * Input Registers: 64 * 65 * LR = return address to continue at after eventually re-enabling MMU 66 */ 67_GLOBAL_TOC(kvmppc_hv_entry_trampoline) 68 mflr r0 69 std r0, PPC_LR_STKOFF(r1) 70 stdu r1, -112(r1) 71 mfmsr r10 72 std r10, HSTATE_HOST_MSR(r13) 73 LOAD_REG_ADDR(r5, kvmppc_call_hv_entry) 74 li r0,MSR_RI 75 andc r0,r10,r0 76 li r6,MSR_IR | MSR_DR 77 andc r6,r10,r6 78 mtmsrd r0,1 /* clear RI in MSR */ 79 mtsrr0 r5 80 mtsrr1 r6 81 RFI 82 83kvmppc_call_hv_entry: 84 ld r4, HSTATE_KVM_VCPU(r13) 85 bl kvmppc_hv_entry 86 87 /* Back from guest - restore host state and return to caller */ 88 89BEGIN_FTR_SECTION 90 /* Restore host DABR and DABRX */ 91 ld r5,HSTATE_DABR(r13) 92 li r6,7 93 mtspr SPRN_DABR,r5 94 mtspr SPRN_DABRX,r6 95END_FTR_SECTION_IFCLR(CPU_FTR_ARCH_207S) 96 97 /* Restore SPRG3 */ 98 ld r3,PACA_SPRG_VDSO(r13) 99 mtspr SPRN_SPRG_VDSO_WRITE,r3 100 101 /* Reload the host's PMU registers */ 102 ld r3, PACALPPACAPTR(r13) /* is the host using the PMU? */ 103 lbz r4, LPPACA_PMCINUSE(r3) 104 cmpwi r4, 0 105 beq 23f /* skip if not */ 106BEGIN_FTR_SECTION 107 ld r3, HSTATE_MMCR0(r13) 108 andi. r4, r3, MMCR0_PMAO_SYNC | MMCR0_PMAO 109 cmpwi r4, MMCR0_PMAO 110 beql kvmppc_fix_pmao 111END_FTR_SECTION_IFSET(CPU_FTR_PMAO_BUG) 112 lwz r3, HSTATE_PMC1(r13) 113 lwz r4, HSTATE_PMC2(r13) 114 lwz r5, HSTATE_PMC3(r13) 115 lwz r6, HSTATE_PMC4(r13) 116 lwz r8, HSTATE_PMC5(r13) 117 lwz r9, HSTATE_PMC6(r13) 118 mtspr SPRN_PMC1, r3 119 mtspr SPRN_PMC2, r4 120 mtspr SPRN_PMC3, r5 121 mtspr SPRN_PMC4, r6 122 mtspr SPRN_PMC5, r8 123 mtspr SPRN_PMC6, r9 124 ld r3, HSTATE_MMCR0(r13) 125 ld r4, HSTATE_MMCR1(r13) 126 ld r5, HSTATE_MMCRA(r13) 127 ld r6, HSTATE_SIAR(r13) 128 ld r7, HSTATE_SDAR(r13) 129 mtspr SPRN_MMCR1, r4 130 mtspr SPRN_MMCRA, r5 131 mtspr SPRN_SIAR, r6 132 mtspr SPRN_SDAR, r7 133BEGIN_FTR_SECTION 134 ld r8, HSTATE_MMCR2(r13) 135 ld r9, HSTATE_SIER(r13) 136 mtspr SPRN_MMCR2, r8 137 mtspr SPRN_SIER, r9 138END_FTR_SECTION_IFSET(CPU_FTR_ARCH_207S) 139 mtspr SPRN_MMCR0, r3 140 isync 14123: 142 143 /* 144 * Reload DEC. HDEC interrupts were disabled when 145 * we reloaded the host's LPCR value. 146 */ 147 ld r3, HSTATE_DECEXP(r13) 148 mftb r4 149 subf r4, r4, r3 150 mtspr SPRN_DEC, r4 151 152BEGIN_FTR_SECTION 153 /* hwthread_req may have got set by cede or no vcpu, so clear it */ 154 li r0, 0 155 stb r0, HSTATE_HWTHREAD_REQ(r13) 156END_FTR_SECTION_IFCLR(CPU_FTR_ARCH_300) 157 158 /* 159 * For external interrupts we need to call the Linux 160 * handler to process the interrupt. We do that by jumping 161 * to absolute address 0x500 for external interrupts. 162 * The [h]rfid at the end of the handler will return to 163 * the book3s_hv_interrupts.S code. For other interrupts 164 * we do the rfid to get back to the book3s_hv_interrupts.S 165 * code here. 166 */ 167 ld r8, 112+PPC_LR_STKOFF(r1) 168 addi r1, r1, 112 169 ld r7, HSTATE_HOST_MSR(r13) 170 171 /* Return the trap number on this thread as the return value */ 172 mr r3, r12 173 174 /* 175 * If we came back from the guest via a relocation-on interrupt, 176 * we will be in virtual mode at this point, which makes it a 177 * little easier to get back to the caller. 178 */ 179 mfmsr r0 180 andi. r0, r0, MSR_IR /* in real mode? */ 181 bne .Lvirt_return 182 183 /* RFI into the highmem handler */ 184 mfmsr r6 185 li r0, MSR_RI 186 andc r6, r6, r0 187 mtmsrd r6, 1 /* Clear RI in MSR */ 188 mtsrr0 r8 189 mtsrr1 r7 190 RFI 191 192 /* Virtual-mode return */ 193.Lvirt_return: 194 mtlr r8 195 blr 196 197kvmppc_primary_no_guest: 198 /* We handle this much like a ceded vcpu */ 199 /* put the HDEC into the DEC, since HDEC interrupts don't wake us */ 200 /* HDEC may be larger than DEC for arch >= v3.00, but since the */ 201 /* HDEC value came from DEC in the first place, it will fit */ 202 mfspr r3, SPRN_HDEC 203 mtspr SPRN_DEC, r3 204 /* 205 * Make sure the primary has finished the MMU switch. 206 * We should never get here on a secondary thread, but 207 * check it for robustness' sake. 208 */ 209 ld r5, HSTATE_KVM_VCORE(r13) 21065: lbz r0, VCORE_IN_GUEST(r5) 211 cmpwi r0, 0 212 beq 65b 213 /* Set LPCR. */ 214 ld r8,VCORE_LPCR(r5) 215 mtspr SPRN_LPCR,r8 216 isync 217 /* set our bit in napping_threads */ 218 ld r5, HSTATE_KVM_VCORE(r13) 219 lbz r7, HSTATE_PTID(r13) 220 li r0, 1 221 sld r0, r0, r7 222 addi r6, r5, VCORE_NAPPING_THREADS 2231: lwarx r3, 0, r6 224 or r3, r3, r0 225 stwcx. r3, 0, r6 226 bne 1b 227 /* order napping_threads update vs testing entry_exit_map */ 228 isync 229 li r12, 0 230 lwz r7, VCORE_ENTRY_EXIT(r5) 231 cmpwi r7, 0x100 232 bge kvm_novcpu_exit /* another thread already exiting */ 233 li r3, NAPPING_NOVCPU 234 stb r3, HSTATE_NAPPING(r13) 235 236 li r3, 0 /* Don't wake on privileged (OS) doorbell */ 237 b kvm_do_nap 238 239/* 240 * kvm_novcpu_wakeup 241 * Entered from kvm_start_guest if kvm_hstate.napping is set 242 * to NAPPING_NOVCPU 243 * r2 = kernel TOC 244 * r13 = paca 245 */ 246kvm_novcpu_wakeup: 247 ld r1, HSTATE_HOST_R1(r13) 248 ld r5, HSTATE_KVM_VCORE(r13) 249 li r0, 0 250 stb r0, HSTATE_NAPPING(r13) 251 252 /* check the wake reason */ 253 bl kvmppc_check_wake_reason 254 255 /* 256 * Restore volatile registers since we could have called 257 * a C routine in kvmppc_check_wake_reason. 258 * r5 = VCORE 259 */ 260 ld r5, HSTATE_KVM_VCORE(r13) 261 262 /* see if any other thread is already exiting */ 263 lwz r0, VCORE_ENTRY_EXIT(r5) 264 cmpwi r0, 0x100 265 bge kvm_novcpu_exit 266 267 /* clear our bit in napping_threads */ 268 lbz r7, HSTATE_PTID(r13) 269 li r0, 1 270 sld r0, r0, r7 271 addi r6, r5, VCORE_NAPPING_THREADS 2724: lwarx r7, 0, r6 273 andc r7, r7, r0 274 stwcx. r7, 0, r6 275 bne 4b 276 277 /* See if the wake reason means we need to exit */ 278 cmpdi r3, 0 279 bge kvm_novcpu_exit 280 281 /* See if our timeslice has expired (HDEC is negative) */ 282 mfspr r0, SPRN_HDEC 283 EXTEND_HDEC(r0) 284 li r12, BOOK3S_INTERRUPT_HV_DECREMENTER 285 cmpdi r0, 0 286 blt kvm_novcpu_exit 287 288 /* Got an IPI but other vcpus aren't yet exiting, must be a latecomer */ 289 ld r4, HSTATE_KVM_VCPU(r13) 290 cmpdi r4, 0 291 beq kvmppc_primary_no_guest 292 293#ifdef CONFIG_KVM_BOOK3S_HV_EXIT_TIMING 294 addi r3, r4, VCPU_TB_RMENTRY 295 bl kvmhv_start_timing 296#endif 297 b kvmppc_got_guest 298 299kvm_novcpu_exit: 300#ifdef CONFIG_KVM_BOOK3S_HV_EXIT_TIMING 301 ld r4, HSTATE_KVM_VCPU(r13) 302 cmpdi r4, 0 303 beq 13f 304 addi r3, r4, VCPU_TB_RMEXIT 305 bl kvmhv_accumulate_time 306#endif 30713: mr r3, r12 308 stw r12, STACK_SLOT_TRAP(r1) 309 bl kvmhv_commence_exit 310 nop 311 lwz r12, STACK_SLOT_TRAP(r1) 312 b kvmhv_switch_to_host 313 314/* 315 * We come in here when wakened from nap mode. 316 * Relocation is off and most register values are lost. 317 * r13 points to the PACA. 318 * r3 contains the SRR1 wakeup value, SRR1 is trashed. 319 * This is not used by ISAv3.0B processors. 320 */ 321 .globl kvm_start_guest 322kvm_start_guest: 323 /* Set runlatch bit the minute you wake up from nap */ 324 mfspr r0, SPRN_CTRLF 325 ori r0, r0, 1 326 mtspr SPRN_CTRLT, r0 327 328 /* 329 * Could avoid this and pass it through in r3. For now, 330 * code expects it to be in SRR1. 331 */ 332 mtspr SPRN_SRR1,r3 333 334 ld r2,PACATOC(r13) 335 336 li r0,KVM_HWTHREAD_IN_KVM 337 stb r0,HSTATE_HWTHREAD_STATE(r13) 338 339 /* NV GPR values from power7_idle() will no longer be valid */ 340 li r0,1 341 stb r0,PACA_NAPSTATELOST(r13) 342 343 /* were we napping due to cede? */ 344 lbz r0,HSTATE_NAPPING(r13) 345 cmpwi r0,NAPPING_CEDE 346 beq kvm_end_cede 347 cmpwi r0,NAPPING_NOVCPU 348 beq kvm_novcpu_wakeup 349 350 ld r1,PACAEMERGSP(r13) 351 subi r1,r1,STACK_FRAME_OVERHEAD 352 353 /* 354 * We weren't napping due to cede, so this must be a secondary 355 * thread being woken up to run a guest, or being woken up due 356 * to a stray IPI. (Or due to some machine check or hypervisor 357 * maintenance interrupt while the core is in KVM.) 358 */ 359 360 /* Check the wake reason in SRR1 to see why we got here */ 361 bl kvmppc_check_wake_reason 362 /* 363 * kvmppc_check_wake_reason could invoke a C routine, but we 364 * have no volatile registers to restore when we return. 365 */ 366 367 cmpdi r3, 0 368 bge kvm_no_guest 369 370 /* get vcore pointer, NULL if we have nothing to run */ 371 ld r5,HSTATE_KVM_VCORE(r13) 372 cmpdi r5,0 373 /* if we have no vcore to run, go back to sleep */ 374 beq kvm_no_guest 375 376kvm_secondary_got_guest: 377 378 /* Set HSTATE_DSCR(r13) to something sensible */ 379 ld r6, PACA_DSCR_DEFAULT(r13) 380 std r6, HSTATE_DSCR(r13) 381 382 /* On thread 0 of a subcore, set HDEC to max */ 383 lbz r4, HSTATE_PTID(r13) 384 cmpwi r4, 0 385 bne 63f 386 LOAD_REG_ADDR(r6, decrementer_max) 387 ld r6, 0(r6) 388 mtspr SPRN_HDEC, r6 389 /* and set per-LPAR registers, if doing dynamic micro-threading */ 390 ld r6, HSTATE_SPLIT_MODE(r13) 391 cmpdi r6, 0 392 beq 63f 393 ld r0, KVM_SPLIT_RPR(r6) 394 mtspr SPRN_RPR, r0 395 ld r0, KVM_SPLIT_PMMAR(r6) 396 mtspr SPRN_PMMAR, r0 397 ld r0, KVM_SPLIT_LDBAR(r6) 398 mtspr SPRN_LDBAR, r0 399 isync 40063: 401 /* Order load of vcpu after load of vcore */ 402 lwsync 403 ld r4, HSTATE_KVM_VCPU(r13) 404 bl kvmppc_hv_entry 405 406 /* Back from the guest, go back to nap */ 407 /* Clear our vcpu and vcore pointers so we don't come back in early */ 408 li r0, 0 409 std r0, HSTATE_KVM_VCPU(r13) 410 /* 411 * Once we clear HSTATE_KVM_VCORE(r13), the code in 412 * kvmppc_run_core() is going to assume that all our vcpu 413 * state is visible in memory. This lwsync makes sure 414 * that that is true. 415 */ 416 lwsync 417 std r0, HSTATE_KVM_VCORE(r13) 418 419 /* 420 * All secondaries exiting guest will fall through this path. 421 * Before proceeding, just check for HMI interrupt and 422 * invoke opal hmi handler. By now we are sure that the 423 * primary thread on this core/subcore has already made partition 424 * switch/TB resync and we are good to call opal hmi handler. 425 */ 426 cmpwi r12, BOOK3S_INTERRUPT_HMI 427 bne kvm_no_guest 428 429 li r3,0 /* NULL argument */ 430 bl hmi_exception_realmode 431/* 432 * At this point we have finished executing in the guest. 433 * We need to wait for hwthread_req to become zero, since 434 * we may not turn on the MMU while hwthread_req is non-zero. 435 * While waiting we also need to check if we get given a vcpu to run. 436 */ 437kvm_no_guest: 438BEGIN_FTR_SECTION 439 twi 31,0,0 440END_FTR_SECTION_IFSET(CPU_FTR_ARCH_300) 441 lbz r3, HSTATE_HWTHREAD_REQ(r13) 442 cmpwi r3, 0 443 bne 53f 444 HMT_MEDIUM 445 li r0, KVM_HWTHREAD_IN_KERNEL 446 stb r0, HSTATE_HWTHREAD_STATE(r13) 447 /* need to recheck hwthread_req after a barrier, to avoid race */ 448 sync 449 lbz r3, HSTATE_HWTHREAD_REQ(r13) 450 cmpwi r3, 0 451 bne 54f 452/* 453 * We jump to pnv_wakeup_loss, which will return to the caller 454 * of power7_nap in the powernv cpu offline loop. The value we 455 * put in r3 becomes the return value for power7_nap. pnv_wakeup_loss 456 * requires SRR1 in r12. 457 */ 458 li r3, LPCR_PECE0 459 mfspr r4, SPRN_LPCR 460 rlwimi r4, r3, 0, LPCR_PECE0 | LPCR_PECE1 461 mtspr SPRN_LPCR, r4 462 li r3, 0 463 mfspr r12,SPRN_SRR1 464 b pnv_wakeup_loss 465 46653: HMT_LOW 467 ld r5, HSTATE_KVM_VCORE(r13) 468 cmpdi r5, 0 469 bne 60f 470 ld r3, HSTATE_SPLIT_MODE(r13) 471 cmpdi r3, 0 472 beq kvm_no_guest 473 lbz r0, KVM_SPLIT_DO_NAP(r3) 474 cmpwi r0, 0 475 beq kvm_no_guest 476 HMT_MEDIUM 477 b kvm_unsplit_nap 47860: HMT_MEDIUM 479 b kvm_secondary_got_guest 480 48154: li r0, KVM_HWTHREAD_IN_KVM 482 stb r0, HSTATE_HWTHREAD_STATE(r13) 483 b kvm_no_guest 484 485/* 486 * Here the primary thread is trying to return the core to 487 * whole-core mode, so we need to nap. 488 */ 489kvm_unsplit_nap: 490 /* 491 * When secondaries are napping in kvm_unsplit_nap() with 492 * hwthread_req = 1, HMI goes ignored even though subcores are 493 * already exited the guest. Hence HMI keeps waking up secondaries 494 * from nap in a loop and secondaries always go back to nap since 495 * no vcore is assigned to them. This makes impossible for primary 496 * thread to get hold of secondary threads resulting into a soft 497 * lockup in KVM path. 498 * 499 * Let us check if HMI is pending and handle it before we go to nap. 500 */ 501 cmpwi r12, BOOK3S_INTERRUPT_HMI 502 bne 55f 503 li r3, 0 /* NULL argument */ 504 bl hmi_exception_realmode 50555: 506 /* 507 * Ensure that secondary doesn't nap when it has 508 * its vcore pointer set. 509 */ 510 sync /* matches smp_mb() before setting split_info.do_nap */ 511 ld r0, HSTATE_KVM_VCORE(r13) 512 cmpdi r0, 0 513 bne kvm_no_guest 514 /* clear any pending message */ 515BEGIN_FTR_SECTION 516 lis r6, (PPC_DBELL_SERVER << (63-36))@h 517 PPC_MSGCLR(6) 518END_FTR_SECTION_IFSET(CPU_FTR_ARCH_207S) 519 /* Set kvm_split_mode.napped[tid] = 1 */ 520 ld r3, HSTATE_SPLIT_MODE(r13) 521 li r0, 1 522 lhz r4, PACAPACAINDEX(r13) 523 clrldi r4, r4, 61 /* micro-threading => P8 => 8 threads/core */ 524 addi r4, r4, KVM_SPLIT_NAPPED 525 stbx r0, r3, r4 526 /* Check the do_nap flag again after setting napped[] */ 527 sync 528 lbz r0, KVM_SPLIT_DO_NAP(r3) 529 cmpwi r0, 0 530 beq 57f 531 li r3, (LPCR_PECEDH | LPCR_PECE0) >> 4 532 mfspr r5, SPRN_LPCR 533 rlwimi r5, r3, 4, (LPCR_PECEDP | LPCR_PECEDH | LPCR_PECE0 | LPCR_PECE1) 534 b kvm_nap_sequence 535 53657: li r0, 0 537 stbx r0, r3, r4 538 b kvm_no_guest 539 540/****************************************************************************** 541 * * 542 * Entry code * 543 * * 544 *****************************************************************************/ 545 546.global kvmppc_hv_entry 547kvmppc_hv_entry: 548 549 /* Required state: 550 * 551 * R4 = vcpu pointer (or NULL) 552 * MSR = ~IR|DR 553 * R13 = PACA 554 * R1 = host R1 555 * R2 = TOC 556 * all other volatile GPRS = free 557 * Does not preserve non-volatile GPRs or CR fields 558 */ 559 mflr r0 560 std r0, PPC_LR_STKOFF(r1) 561 stdu r1, -SFS(r1) 562 563 /* Save R1 in the PACA */ 564 std r1, HSTATE_HOST_R1(r13) 565 566 li r6, KVM_GUEST_MODE_HOST_HV 567 stb r6, HSTATE_IN_GUEST(r13) 568 569#ifdef CONFIG_KVM_BOOK3S_HV_EXIT_TIMING 570 /* Store initial timestamp */ 571 cmpdi r4, 0 572 beq 1f 573 addi r3, r4, VCPU_TB_RMENTRY 574 bl kvmhv_start_timing 5751: 576#endif 577 578 /* Use cr7 as an indication of radix mode */ 579 ld r5, HSTATE_KVM_VCORE(r13) 580 ld r9, VCORE_KVM(r5) /* pointer to struct kvm */ 581 lbz r0, KVM_RADIX(r9) 582 cmpwi cr7, r0, 0 583 584 /* Clear out SLB if hash */ 585 bne cr7, 2f 586 li r6,0 587 slbmte r6,r6 588 slbia 589 ptesync 5902: 591 /* 592 * POWER7/POWER8 host -> guest partition switch code. 593 * We don't have to lock against concurrent tlbies, 594 * but we do have to coordinate across hardware threads. 595 */ 596 /* Set bit in entry map iff exit map is zero. */ 597 li r7, 1 598 lbz r6, HSTATE_PTID(r13) 599 sld r7, r7, r6 600 addi r8, r5, VCORE_ENTRY_EXIT 60121: lwarx r3, 0, r8 602 cmpwi r3, 0x100 /* any threads starting to exit? */ 603 bge secondary_too_late /* if so we're too late to the party */ 604 or r3, r3, r7 605 stwcx. r3, 0, r8 606 bne 21b 607 608 /* Primary thread switches to guest partition. */ 609 cmpwi r6,0 610 bne 10f 611 lwz r7,KVM_LPID(r9) 612BEGIN_FTR_SECTION 613 ld r6,KVM_SDR1(r9) 614 li r0,LPID_RSVD /* switch to reserved LPID */ 615 mtspr SPRN_LPID,r0 616 ptesync 617 mtspr SPRN_SDR1,r6 /* switch to partition page table */ 618END_FTR_SECTION_IFCLR(CPU_FTR_ARCH_300) 619 mtspr SPRN_LPID,r7 620 isync 621 622 /* See if we need to flush the TLB */ 623 lhz r6,PACAPACAINDEX(r13) /* test_bit(cpu, need_tlb_flush) */ 624BEGIN_FTR_SECTION 625 /* 626 * On POWER9, individual threads can come in here, but the 627 * TLB is shared between the 4 threads in a core, hence 628 * invalidating on one thread invalidates for all. 629 * Thus we make all 4 threads use the same bit here. 630 */ 631 clrrdi r6,r6,2 632END_FTR_SECTION_IFSET(CPU_FTR_ARCH_300) 633 clrldi r7,r6,64-6 /* extract bit number (6 bits) */ 634 srdi r6,r6,6 /* doubleword number */ 635 sldi r6,r6,3 /* address offset */ 636 add r6,r6,r9 637 addi r6,r6,KVM_NEED_FLUSH /* dword in kvm->arch.need_tlb_flush */ 638 li r8,1 639 sld r8,r8,r7 640 ld r7,0(r6) 641 and. r7,r7,r8 642 beq 22f 643 /* Flush the TLB of any entries for this LPID */ 644 lwz r0,KVM_TLB_SETS(r9) 645 mtctr r0 646 li r7,0x800 /* IS field = 0b10 */ 647 ptesync 648 li r0,0 /* RS for P9 version of tlbiel */ 649 bne cr7, 29f 65028: tlbiel r7 /* On P9, rs=0, RIC=0, PRS=0, R=0 */ 651 addi r7,r7,0x1000 652 bdnz 28b 653 b 30f 65429: PPC_TLBIEL(7,0,2,1,1) /* for radix, RIC=2, PRS=1, R=1 */ 655 addi r7,r7,0x1000 656 bdnz 29b 65730: ptesync 65823: ldarx r7,0,r6 /* clear the bit after TLB flushed */ 659 andc r7,r7,r8 660 stdcx. r7,0,r6 661 bne 23b 662 663 /* Add timebase offset onto timebase */ 66422: ld r8,VCORE_TB_OFFSET(r5) 665 cmpdi r8,0 666 beq 37f 667 mftb r6 /* current host timebase */ 668 add r8,r8,r6 669 mtspr SPRN_TBU40,r8 /* update upper 40 bits */ 670 mftb r7 /* check if lower 24 bits overflowed */ 671 clrldi r6,r6,40 672 clrldi r7,r7,40 673 cmpld r7,r6 674 bge 37f 675 addis r8,r8,0x100 /* if so, increment upper 40 bits */ 676 mtspr SPRN_TBU40,r8 677 678 /* Load guest PCR value to select appropriate compat mode */ 67937: ld r7, VCORE_PCR(r5) 680 cmpdi r7, 0 681 beq 38f 682 mtspr SPRN_PCR, r7 68338: 684 685BEGIN_FTR_SECTION 686 /* DPDES and VTB are shared between threads */ 687 ld r8, VCORE_DPDES(r5) 688 ld r7, VCORE_VTB(r5) 689 mtspr SPRN_DPDES, r8 690 mtspr SPRN_VTB, r7 691END_FTR_SECTION_IFSET(CPU_FTR_ARCH_207S) 692 693 /* Mark the subcore state as inside guest */ 694 bl kvmppc_subcore_enter_guest 695 nop 696 ld r5, HSTATE_KVM_VCORE(r13) 697 ld r4, HSTATE_KVM_VCPU(r13) 698 li r0,1 699 stb r0,VCORE_IN_GUEST(r5) /* signal secondaries to continue */ 700 701 /* Do we have a guest vcpu to run? */ 70210: cmpdi r4, 0 703 beq kvmppc_primary_no_guest 704kvmppc_got_guest: 705 706 /* Load up guest SLB entries (N.B. slb_max will be 0 for radix) */ 707 lwz r5,VCPU_SLB_MAX(r4) 708 cmpwi r5,0 709 beq 9f 710 mtctr r5 711 addi r6,r4,VCPU_SLB 7121: ld r8,VCPU_SLB_E(r6) 713 ld r9,VCPU_SLB_V(r6) 714 slbmte r9,r8 715 addi r6,r6,VCPU_SLB_SIZE 716 bdnz 1b 7179: 718 /* Increment yield count if they have a VPA */ 719 ld r3, VCPU_VPA(r4) 720 cmpdi r3, 0 721 beq 25f 722 li r6, LPPACA_YIELDCOUNT 723 LWZX_BE r5, r3, r6 724 addi r5, r5, 1 725 STWX_BE r5, r3, r6 726 li r6, 1 727 stb r6, VCPU_VPA_DIRTY(r4) 72825: 729 730 /* Save purr/spurr */ 731 mfspr r5,SPRN_PURR 732 mfspr r6,SPRN_SPURR 733 std r5,HSTATE_PURR(r13) 734 std r6,HSTATE_SPURR(r13) 735 ld r7,VCPU_PURR(r4) 736 ld r8,VCPU_SPURR(r4) 737 mtspr SPRN_PURR,r7 738 mtspr SPRN_SPURR,r8 739 740 /* Save host values of some registers */ 741BEGIN_FTR_SECTION 742 mfspr r5, SPRN_TIDR 743 mfspr r6, SPRN_PSSCR 744 mfspr r7, SPRN_PID 745 mfspr r8, SPRN_IAMR 746 std r5, STACK_SLOT_TID(r1) 747 std r6, STACK_SLOT_PSSCR(r1) 748 std r7, STACK_SLOT_PID(r1) 749 std r8, STACK_SLOT_IAMR(r1) 750 mfspr r5, SPRN_HFSCR 751 std r5, STACK_SLOT_HFSCR(r1) 752END_FTR_SECTION_IFSET(CPU_FTR_ARCH_300) 753BEGIN_FTR_SECTION 754 mfspr r5, SPRN_CIABR 755 mfspr r6, SPRN_DAWR 756 mfspr r7, SPRN_DAWRX 757 std r5, STACK_SLOT_CIABR(r1) 758 std r6, STACK_SLOT_DAWR(r1) 759 std r7, STACK_SLOT_DAWRX(r1) 760END_FTR_SECTION_IFSET(CPU_FTR_ARCH_207S) 761 762BEGIN_FTR_SECTION 763 /* Set partition DABR */ 764 /* Do this before re-enabling PMU to avoid P7 DABR corruption bug */ 765 lwz r5,VCPU_DABRX(r4) 766 ld r6,VCPU_DABR(r4) 767 mtspr SPRN_DABRX,r5 768 mtspr SPRN_DABR,r6 769 isync 770END_FTR_SECTION_IFCLR(CPU_FTR_ARCH_207S) 771 772#ifdef CONFIG_PPC_TRANSACTIONAL_MEM 773BEGIN_FTR_SECTION 774 /* 775 * NOTE THAT THIS TRASHES ALL NON-VOLATILE REGISTERS INCLUDING CR 776 */ 777 bl kvmppc_restore_tm 778END_FTR_SECTION_IFSET(CPU_FTR_TM) 779#endif 780 781 /* Load guest PMU registers */ 782 /* R4 is live here (vcpu pointer) */ 783 li r3, 1 784 sldi r3, r3, 31 /* MMCR0_FC (freeze counters) bit */ 785 mtspr SPRN_MMCR0, r3 /* freeze all counters, disable ints */ 786 isync 787BEGIN_FTR_SECTION 788 ld r3, VCPU_MMCR(r4) 789 andi. r5, r3, MMCR0_PMAO_SYNC | MMCR0_PMAO 790 cmpwi r5, MMCR0_PMAO 791 beql kvmppc_fix_pmao 792END_FTR_SECTION_IFSET(CPU_FTR_PMAO_BUG) 793 lwz r3, VCPU_PMC(r4) /* always load up guest PMU registers */ 794 lwz r5, VCPU_PMC + 4(r4) /* to prevent information leak */ 795 lwz r6, VCPU_PMC + 8(r4) 796 lwz r7, VCPU_PMC + 12(r4) 797 lwz r8, VCPU_PMC + 16(r4) 798 lwz r9, VCPU_PMC + 20(r4) 799 mtspr SPRN_PMC1, r3 800 mtspr SPRN_PMC2, r5 801 mtspr SPRN_PMC3, r6 802 mtspr SPRN_PMC4, r7 803 mtspr SPRN_PMC5, r8 804 mtspr SPRN_PMC6, r9 805 ld r3, VCPU_MMCR(r4) 806 ld r5, VCPU_MMCR + 8(r4) 807 ld r6, VCPU_MMCR + 16(r4) 808 ld r7, VCPU_SIAR(r4) 809 ld r8, VCPU_SDAR(r4) 810 mtspr SPRN_MMCR1, r5 811 mtspr SPRN_MMCRA, r6 812 mtspr SPRN_SIAR, r7 813 mtspr SPRN_SDAR, r8 814BEGIN_FTR_SECTION 815 ld r5, VCPU_MMCR + 24(r4) 816 ld r6, VCPU_SIER(r4) 817 mtspr SPRN_MMCR2, r5 818 mtspr SPRN_SIER, r6 819BEGIN_FTR_SECTION_NESTED(96) 820 lwz r7, VCPU_PMC + 24(r4) 821 lwz r8, VCPU_PMC + 28(r4) 822 ld r9, VCPU_MMCR + 32(r4) 823 mtspr SPRN_SPMC1, r7 824 mtspr SPRN_SPMC2, r8 825 mtspr SPRN_MMCRS, r9 826END_FTR_SECTION_NESTED(CPU_FTR_ARCH_300, 0, 96) 827END_FTR_SECTION_IFSET(CPU_FTR_ARCH_207S) 828 mtspr SPRN_MMCR0, r3 829 isync 830 831 /* Load up FP, VMX and VSX registers */ 832 bl kvmppc_load_fp 833 834 ld r14, VCPU_GPR(R14)(r4) 835 ld r15, VCPU_GPR(R15)(r4) 836 ld r16, VCPU_GPR(R16)(r4) 837 ld r17, VCPU_GPR(R17)(r4) 838 ld r18, VCPU_GPR(R18)(r4) 839 ld r19, VCPU_GPR(R19)(r4) 840 ld r20, VCPU_GPR(R20)(r4) 841 ld r21, VCPU_GPR(R21)(r4) 842 ld r22, VCPU_GPR(R22)(r4) 843 ld r23, VCPU_GPR(R23)(r4) 844 ld r24, VCPU_GPR(R24)(r4) 845 ld r25, VCPU_GPR(R25)(r4) 846 ld r26, VCPU_GPR(R26)(r4) 847 ld r27, VCPU_GPR(R27)(r4) 848 ld r28, VCPU_GPR(R28)(r4) 849 ld r29, VCPU_GPR(R29)(r4) 850 ld r30, VCPU_GPR(R30)(r4) 851 ld r31, VCPU_GPR(R31)(r4) 852 853 /* Switch DSCR to guest value */ 854 ld r5, VCPU_DSCR(r4) 855 mtspr SPRN_DSCR, r5 856 857BEGIN_FTR_SECTION 858 /* Skip next section on POWER7 */ 859 b 8f 860END_FTR_SECTION_IFCLR(CPU_FTR_ARCH_207S) 861 /* Load up POWER8-specific registers */ 862 ld r5, VCPU_IAMR(r4) 863 lwz r6, VCPU_PSPB(r4) 864 ld r7, VCPU_FSCR(r4) 865 mtspr SPRN_IAMR, r5 866 mtspr SPRN_PSPB, r6 867 mtspr SPRN_FSCR, r7 868 ld r5, VCPU_DAWR(r4) 869 ld r6, VCPU_DAWRX(r4) 870 ld r7, VCPU_CIABR(r4) 871 ld r8, VCPU_TAR(r4) 872 mtspr SPRN_DAWR, r5 873 mtspr SPRN_DAWRX, r6 874 mtspr SPRN_CIABR, r7 875 mtspr SPRN_TAR, r8 876 ld r5, VCPU_IC(r4) 877 ld r8, VCPU_EBBHR(r4) 878 mtspr SPRN_IC, r5 879 mtspr SPRN_EBBHR, r8 880 ld r5, VCPU_EBBRR(r4) 881 ld r6, VCPU_BESCR(r4) 882 lwz r7, VCPU_GUEST_PID(r4) 883 ld r8, VCPU_WORT(r4) 884 mtspr SPRN_EBBRR, r5 885 mtspr SPRN_BESCR, r6 886 mtspr SPRN_PID, r7 887 mtspr SPRN_WORT, r8 888BEGIN_FTR_SECTION 889 PPC_INVALIDATE_ERAT 890END_FTR_SECTION_IFSET(CPU_FTR_POWER9_DD1) 891BEGIN_FTR_SECTION 892 /* POWER8-only registers */ 893 ld r5, VCPU_TCSCR(r4) 894 ld r6, VCPU_ACOP(r4) 895 ld r7, VCPU_CSIGR(r4) 896 ld r8, VCPU_TACR(r4) 897 mtspr SPRN_TCSCR, r5 898 mtspr SPRN_ACOP, r6 899 mtspr SPRN_CSIGR, r7 900 mtspr SPRN_TACR, r8 901FTR_SECTION_ELSE 902 /* POWER9-only registers */ 903 ld r5, VCPU_TID(r4) 904 ld r6, VCPU_PSSCR(r4) 905 oris r6, r6, PSSCR_EC@h /* This makes stop trap to HV */ 906 ld r7, VCPU_HFSCR(r4) 907 mtspr SPRN_TIDR, r5 908 mtspr SPRN_PSSCR, r6 909 mtspr SPRN_HFSCR, r7 910ALT_FTR_SECTION_END_IFCLR(CPU_FTR_ARCH_300) 9118: 912 913 /* 914 * Set the decrementer to the guest decrementer. 915 */ 916 ld r8,VCPU_DEC_EXPIRES(r4) 917 /* r8 is a host timebase value here, convert to guest TB */ 918 ld r5,HSTATE_KVM_VCORE(r13) 919 ld r6,VCORE_TB_OFFSET(r5) 920 add r8,r8,r6 921 mftb r7 922 subf r3,r7,r8 923 mtspr SPRN_DEC,r3 924 std r3,VCPU_DEC(r4) 925 926 ld r5, VCPU_SPRG0(r4) 927 ld r6, VCPU_SPRG1(r4) 928 ld r7, VCPU_SPRG2(r4) 929 ld r8, VCPU_SPRG3(r4) 930 mtspr SPRN_SPRG0, r5 931 mtspr SPRN_SPRG1, r6 932 mtspr SPRN_SPRG2, r7 933 mtspr SPRN_SPRG3, r8 934 935 /* Load up DAR and DSISR */ 936 ld r5, VCPU_DAR(r4) 937 lwz r6, VCPU_DSISR(r4) 938 mtspr SPRN_DAR, r5 939 mtspr SPRN_DSISR, r6 940 941 /* Restore AMR and UAMOR, set AMOR to all 1s */ 942 ld r5,VCPU_AMR(r4) 943 ld r6,VCPU_UAMOR(r4) 944 li r7,-1 945 mtspr SPRN_AMR,r5 946 mtspr SPRN_UAMOR,r6 947 mtspr SPRN_AMOR,r7 948 949 /* Restore state of CTRL run bit; assume 1 on entry */ 950 lwz r5,VCPU_CTRL(r4) 951 andi. r5,r5,1 952 bne 4f 953 mfspr r6,SPRN_CTRLF 954 clrrdi r6,r6,1 955 mtspr SPRN_CTRLT,r6 9564: 957 /* Secondary threads wait for primary to have done partition switch */ 958 ld r5, HSTATE_KVM_VCORE(r13) 959 lbz r6, HSTATE_PTID(r13) 960 cmpwi r6, 0 961 beq 21f 962 lbz r0, VCORE_IN_GUEST(r5) 963 cmpwi r0, 0 964 bne 21f 965 HMT_LOW 96620: lwz r3, VCORE_ENTRY_EXIT(r5) 967 cmpwi r3, 0x100 968 bge no_switch_exit 969 lbz r0, VCORE_IN_GUEST(r5) 970 cmpwi r0, 0 971 beq 20b 972 HMT_MEDIUM 97321: 974 /* Set LPCR. */ 975 ld r8,VCORE_LPCR(r5) 976 mtspr SPRN_LPCR,r8 977 isync 978 979 /* Check if HDEC expires soon */ 980 mfspr r3, SPRN_HDEC 981 EXTEND_HDEC(r3) 982 cmpdi r3, 512 /* 1 microsecond */ 983 blt hdec_soon 984 985#ifdef CONFIG_KVM_XICS 986 /* We are entering the guest on that thread, push VCPU to XIVE */ 987 ld r10, HSTATE_XIVE_TIMA_PHYS(r13) 988 cmpldi cr0, r10, 0 989 beq no_xive 990 ld r11, VCPU_XIVE_SAVED_STATE(r4) 991 li r9, TM_QW1_OS 992 stdcix r11,r9,r10 993 eieio 994 lwz r11, VCPU_XIVE_CAM_WORD(r4) 995 li r9, TM_QW1_OS + TM_WORD2 996 stwcix r11,r9,r10 997 li r9, 1 998 stw r9, VCPU_XIVE_PUSHED(r4) 999no_xive: 1000#endif /* CONFIG_KVM_XICS */ 1001 1002deliver_guest_interrupt: 1003 ld r6, VCPU_CTR(r4) 1004 ld r7, VCPU_XER(r4) 1005 1006 mtctr r6 1007 mtxer r7 1008 1009kvmppc_cede_reentry: /* r4 = vcpu, r13 = paca */ 1010 ld r10, VCPU_PC(r4) 1011 ld r11, VCPU_MSR(r4) 1012 ld r6, VCPU_SRR0(r4) 1013 ld r7, VCPU_SRR1(r4) 1014 mtspr SPRN_SRR0, r6 1015 mtspr SPRN_SRR1, r7 1016 1017 /* r11 = vcpu->arch.msr & ~MSR_HV */ 1018 rldicl r11, r11, 63 - MSR_HV_LG, 1 1019 rotldi r11, r11, 1 + MSR_HV_LG 1020 ori r11, r11, MSR_ME 1021 1022 /* Check if we can deliver an external or decrementer interrupt now */ 1023 ld r0, VCPU_PENDING_EXC(r4) 1024 rldicl r0, r0, 64 - BOOK3S_IRQPRIO_EXTERNAL_LEVEL, 63 1025 cmpdi cr1, r0, 0 1026 andi. r8, r11, MSR_EE 1027 mfspr r8, SPRN_LPCR 1028 /* Insert EXTERNAL_LEVEL bit into LPCR at the MER bit position */ 1029 rldimi r8, r0, LPCR_MER_SH, 63 - LPCR_MER_SH 1030 mtspr SPRN_LPCR, r8 1031 isync 1032 beq 5f 1033 li r0, BOOK3S_INTERRUPT_EXTERNAL 1034 bne cr1, 12f 1035 mfspr r0, SPRN_DEC 1036BEGIN_FTR_SECTION 1037 /* On POWER9 check whether the guest has large decrementer enabled */ 1038 andis. r8, r8, LPCR_LD@h 1039 bne 15f 1040END_FTR_SECTION_IFSET(CPU_FTR_ARCH_300) 1041 extsw r0, r0 104215: cmpdi r0, 0 1043 li r0, BOOK3S_INTERRUPT_DECREMENTER 1044 bge 5f 1045 104612: mtspr SPRN_SRR0, r10 1047 mr r10,r0 1048 mtspr SPRN_SRR1, r11 1049 mr r9, r4 1050 bl kvmppc_msr_interrupt 10515: 1052BEGIN_FTR_SECTION 1053 b fast_guest_return 1054END_FTR_SECTION_IFCLR(CPU_FTR_ARCH_300) 1055 /* On POWER9, check for pending doorbell requests */ 1056 lbz r0, VCPU_DBELL_REQ(r4) 1057 cmpwi r0, 0 1058 beq fast_guest_return 1059 ld r5, HSTATE_KVM_VCORE(r13) 1060 /* Set DPDES register so the CPU will take a doorbell interrupt */ 1061 li r0, 1 1062 mtspr SPRN_DPDES, r0 1063 std r0, VCORE_DPDES(r5) 1064 /* Make sure other cpus see vcore->dpdes set before dbell req clear */ 1065 lwsync 1066 /* Clear the pending doorbell request */ 1067 li r0, 0 1068 stb r0, VCPU_DBELL_REQ(r4) 1069 1070/* 1071 * Required state: 1072 * R4 = vcpu 1073 * R10: value for HSRR0 1074 * R11: value for HSRR1 1075 * R13 = PACA 1076 */ 1077fast_guest_return: 1078 li r0,0 1079 stb r0,VCPU_CEDED(r4) /* cancel cede */ 1080 mtspr SPRN_HSRR0,r10 1081 mtspr SPRN_HSRR1,r11 1082 1083 /* Activate guest mode, so faults get handled by KVM */ 1084 li r9, KVM_GUEST_MODE_GUEST_HV 1085 stb r9, HSTATE_IN_GUEST(r13) 1086 1087#ifdef CONFIG_KVM_BOOK3S_HV_EXIT_TIMING 1088 /* Accumulate timing */ 1089 addi r3, r4, VCPU_TB_GUEST 1090 bl kvmhv_accumulate_time 1091#endif 1092 1093 /* Enter guest */ 1094 1095BEGIN_FTR_SECTION 1096 ld r5, VCPU_CFAR(r4) 1097 mtspr SPRN_CFAR, r5 1098END_FTR_SECTION_IFSET(CPU_FTR_CFAR) 1099BEGIN_FTR_SECTION 1100 ld r0, VCPU_PPR(r4) 1101END_FTR_SECTION_IFSET(CPU_FTR_HAS_PPR) 1102 1103 ld r5, VCPU_LR(r4) 1104 lwz r6, VCPU_CR(r4) 1105 mtlr r5 1106 mtcr r6 1107 1108 ld r1, VCPU_GPR(R1)(r4) 1109 ld r2, VCPU_GPR(R2)(r4) 1110 ld r3, VCPU_GPR(R3)(r4) 1111 ld r5, VCPU_GPR(R5)(r4) 1112 ld r6, VCPU_GPR(R6)(r4) 1113 ld r7, VCPU_GPR(R7)(r4) 1114 ld r8, VCPU_GPR(R8)(r4) 1115 ld r9, VCPU_GPR(R9)(r4) 1116 ld r10, VCPU_GPR(R10)(r4) 1117 ld r11, VCPU_GPR(R11)(r4) 1118 ld r12, VCPU_GPR(R12)(r4) 1119 ld r13, VCPU_GPR(R13)(r4) 1120 1121BEGIN_FTR_SECTION 1122 mtspr SPRN_PPR, r0 1123END_FTR_SECTION_IFSET(CPU_FTR_HAS_PPR) 1124 1125/* Move canary into DSISR to check for later */ 1126BEGIN_FTR_SECTION 1127 li r0, 0x7fff 1128 mtspr SPRN_HDSISR, r0 1129END_FTR_SECTION_IFSET(CPU_FTR_ARCH_300) 1130 1131 ld r0, VCPU_GPR(R0)(r4) 1132 ld r4, VCPU_GPR(R4)(r4) 1133 1134 hrfid 1135 b . 1136 1137secondary_too_late: 1138 li r12, 0 1139 cmpdi r4, 0 1140 beq 11f 1141 stw r12, VCPU_TRAP(r4) 1142#ifdef CONFIG_KVM_BOOK3S_HV_EXIT_TIMING 1143 addi r3, r4, VCPU_TB_RMEXIT 1144 bl kvmhv_accumulate_time 1145#endif 114611: b kvmhv_switch_to_host 1147 1148no_switch_exit: 1149 HMT_MEDIUM 1150 li r12, 0 1151 b 12f 1152hdec_soon: 1153 li r12, BOOK3S_INTERRUPT_HV_DECREMENTER 115412: stw r12, VCPU_TRAP(r4) 1155 mr r9, r4 1156#ifdef CONFIG_KVM_BOOK3S_HV_EXIT_TIMING 1157 addi r3, r4, VCPU_TB_RMEXIT 1158 bl kvmhv_accumulate_time 1159#endif 1160 b guest_exit_cont 1161 1162/****************************************************************************** 1163 * * 1164 * Exit code * 1165 * * 1166 *****************************************************************************/ 1167 1168/* 1169 * We come here from the first-level interrupt handlers. 1170 */ 1171 .globl kvmppc_interrupt_hv 1172kvmppc_interrupt_hv: 1173 /* 1174 * Register contents: 1175 * R12 = (guest CR << 32) | interrupt vector 1176 * R13 = PACA 1177 * guest R12 saved in shadow VCPU SCRATCH0 1178 * guest CTR saved in shadow VCPU SCRATCH1 if RELOCATABLE 1179 * guest R13 saved in SPRN_SCRATCH0 1180 */ 1181 std r9, HSTATE_SCRATCH2(r13) 1182 lbz r9, HSTATE_IN_GUEST(r13) 1183 cmpwi r9, KVM_GUEST_MODE_HOST_HV 1184 beq kvmppc_bad_host_intr 1185#ifdef CONFIG_KVM_BOOK3S_PR_POSSIBLE 1186 cmpwi r9, KVM_GUEST_MODE_GUEST 1187 ld r9, HSTATE_SCRATCH2(r13) 1188 beq kvmppc_interrupt_pr 1189#endif 1190 /* We're now back in the host but in guest MMU context */ 1191 li r9, KVM_GUEST_MODE_HOST_HV 1192 stb r9, HSTATE_IN_GUEST(r13) 1193 1194 ld r9, HSTATE_KVM_VCPU(r13) 1195 1196 /* Save registers */ 1197 1198 std r0, VCPU_GPR(R0)(r9) 1199 std r1, VCPU_GPR(R1)(r9) 1200 std r2, VCPU_GPR(R2)(r9) 1201 std r3, VCPU_GPR(R3)(r9) 1202 std r4, VCPU_GPR(R4)(r9) 1203 std r5, VCPU_GPR(R5)(r9) 1204 std r6, VCPU_GPR(R6)(r9) 1205 std r7, VCPU_GPR(R7)(r9) 1206 std r8, VCPU_GPR(R8)(r9) 1207 ld r0, HSTATE_SCRATCH2(r13) 1208 std r0, VCPU_GPR(R9)(r9) 1209 std r10, VCPU_GPR(R10)(r9) 1210 std r11, VCPU_GPR(R11)(r9) 1211 ld r3, HSTATE_SCRATCH0(r13) 1212 std r3, VCPU_GPR(R12)(r9) 1213 /* CR is in the high half of r12 */ 1214 srdi r4, r12, 32 1215 stw r4, VCPU_CR(r9) 1216BEGIN_FTR_SECTION 1217 ld r3, HSTATE_CFAR(r13) 1218 std r3, VCPU_CFAR(r9) 1219END_FTR_SECTION_IFSET(CPU_FTR_CFAR) 1220BEGIN_FTR_SECTION 1221 ld r4, HSTATE_PPR(r13) 1222 std r4, VCPU_PPR(r9) 1223END_FTR_SECTION_IFSET(CPU_FTR_HAS_PPR) 1224 1225 /* Restore R1/R2 so we can handle faults */ 1226 ld r1, HSTATE_HOST_R1(r13) 1227 ld r2, PACATOC(r13) 1228 1229 mfspr r10, SPRN_SRR0 1230 mfspr r11, SPRN_SRR1 1231 std r10, VCPU_SRR0(r9) 1232 std r11, VCPU_SRR1(r9) 1233 /* trap is in the low half of r12, clear CR from the high half */ 1234 clrldi r12, r12, 32 1235 andi. r0, r12, 2 /* need to read HSRR0/1? */ 1236 beq 1f 1237 mfspr r10, SPRN_HSRR0 1238 mfspr r11, SPRN_HSRR1 1239 clrrdi r12, r12, 2 12401: std r10, VCPU_PC(r9) 1241 std r11, VCPU_MSR(r9) 1242 1243 GET_SCRATCH0(r3) 1244 mflr r4 1245 std r3, VCPU_GPR(R13)(r9) 1246 std r4, VCPU_LR(r9) 1247 1248 stw r12,VCPU_TRAP(r9) 1249 1250 /* 1251 * Now that we have saved away SRR0/1 and HSRR0/1, 1252 * interrupts are recoverable in principle, so set MSR_RI. 1253 * This becomes important for relocation-on interrupts from 1254 * the guest, which we can get in radix mode on POWER9. 1255 */ 1256 li r0, MSR_RI 1257 mtmsrd r0, 1 1258 1259#ifdef CONFIG_KVM_BOOK3S_HV_EXIT_TIMING 1260 addi r3, r9, VCPU_TB_RMINTR 1261 mr r4, r9 1262 bl kvmhv_accumulate_time 1263 ld r5, VCPU_GPR(R5)(r9) 1264 ld r6, VCPU_GPR(R6)(r9) 1265 ld r7, VCPU_GPR(R7)(r9) 1266 ld r8, VCPU_GPR(R8)(r9) 1267#endif 1268 1269 /* Save HEIR (HV emulation assist reg) in emul_inst 1270 if this is an HEI (HV emulation interrupt, e40) */ 1271 li r3,KVM_INST_FETCH_FAILED 1272 stw r3,VCPU_LAST_INST(r9) 1273 cmpwi r12,BOOK3S_INTERRUPT_H_EMUL_ASSIST 1274 bne 11f 1275 mfspr r3,SPRN_HEIR 127611: stw r3,VCPU_HEIR(r9) 1277 1278 /* these are volatile across C function calls */ 1279#ifdef CONFIG_RELOCATABLE 1280 ld r3, HSTATE_SCRATCH1(r13) 1281 mtctr r3 1282#else 1283 mfctr r3 1284#endif 1285 mfxer r4 1286 std r3, VCPU_CTR(r9) 1287 std r4, VCPU_XER(r9) 1288 1289 /* If this is a page table miss then see if it's theirs or ours */ 1290 cmpwi r12, BOOK3S_INTERRUPT_H_DATA_STORAGE 1291 beq kvmppc_hdsi 1292 cmpwi r12, BOOK3S_INTERRUPT_H_INST_STORAGE 1293 beq kvmppc_hisi 1294 1295 /* See if this is a leftover HDEC interrupt */ 1296 cmpwi r12,BOOK3S_INTERRUPT_HV_DECREMENTER 1297 bne 2f 1298 mfspr r3,SPRN_HDEC 1299 EXTEND_HDEC(r3) 1300 cmpdi r3,0 1301 mr r4,r9 1302 bge fast_guest_return 13032: 1304 /* See if this is an hcall we can handle in real mode */ 1305 cmpwi r12,BOOK3S_INTERRUPT_SYSCALL 1306 beq hcall_try_real_mode 1307 1308 /* Hypervisor doorbell - exit only if host IPI flag set */ 1309 cmpwi r12, BOOK3S_INTERRUPT_H_DOORBELL 1310 bne 3f 1311BEGIN_FTR_SECTION 1312 PPC_MSGSYNC 1313END_FTR_SECTION_IFSET(CPU_FTR_ARCH_300) 1314 lbz r0, HSTATE_HOST_IPI(r13) 1315 cmpwi r0, 0 1316 beq 4f 1317 b guest_exit_cont 13183: 1319 /* If it's a hypervisor facility unavailable interrupt, save HFSCR */ 1320 cmpwi r12, BOOK3S_INTERRUPT_H_FAC_UNAVAIL 1321 bne 14f 1322 mfspr r3, SPRN_HFSCR 1323 std r3, VCPU_HFSCR(r9) 1324 b guest_exit_cont 132514: 1326 /* External interrupt ? */ 1327 cmpwi r12, BOOK3S_INTERRUPT_EXTERNAL 1328 bne+ guest_exit_cont 1329 1330 /* External interrupt, first check for host_ipi. If this is 1331 * set, we know the host wants us out so let's do it now 1332 */ 1333 bl kvmppc_read_intr 1334 1335 /* 1336 * Restore the active volatile registers after returning from 1337 * a C function. 1338 */ 1339 ld r9, HSTATE_KVM_VCPU(r13) 1340 li r12, BOOK3S_INTERRUPT_EXTERNAL 1341 1342 /* 1343 * kvmppc_read_intr return codes: 1344 * 1345 * Exit to host (r3 > 0) 1346 * 1 An interrupt is pending that needs to be handled by the host 1347 * Exit guest and return to host by branching to guest_exit_cont 1348 * 1349 * 2 Passthrough that needs completion in the host 1350 * Exit guest and return to host by branching to guest_exit_cont 1351 * However, we also set r12 to BOOK3S_INTERRUPT_HV_RM_HARD 1352 * to indicate to the host to complete handling the interrupt 1353 * 1354 * Before returning to guest, we check if any CPU is heading out 1355 * to the host and if so, we head out also. If no CPUs are heading 1356 * check return values <= 0. 1357 * 1358 * Return to guest (r3 <= 0) 1359 * 0 No external interrupt is pending 1360 * -1 A guest wakeup IPI (which has now been cleared) 1361 * In either case, we return to guest to deliver any pending 1362 * guest interrupts. 1363 * 1364 * -2 A PCI passthrough external interrupt was handled 1365 * (interrupt was delivered directly to guest) 1366 * Return to guest to deliver any pending guest interrupts. 1367 */ 1368 1369 cmpdi r3, 1 1370 ble 1f 1371 1372 /* Return code = 2 */ 1373 li r12, BOOK3S_INTERRUPT_HV_RM_HARD 1374 stw r12, VCPU_TRAP(r9) 1375 b guest_exit_cont 1376 13771: /* Return code <= 1 */ 1378 cmpdi r3, 0 1379 bgt guest_exit_cont 1380 1381 /* Return code <= 0 */ 13824: ld r5, HSTATE_KVM_VCORE(r13) 1383 lwz r0, VCORE_ENTRY_EXIT(r5) 1384 cmpwi r0, 0x100 1385 mr r4, r9 1386 blt deliver_guest_interrupt 1387 1388guest_exit_cont: /* r9 = vcpu, r12 = trap, r13 = paca */ 1389#ifdef CONFIG_KVM_XICS 1390 /* We are exiting, pull the VP from the XIVE */ 1391 lwz r0, VCPU_XIVE_PUSHED(r9) 1392 cmpwi cr0, r0, 0 1393 beq 1f 1394 li r7, TM_SPC_PULL_OS_CTX 1395 li r6, TM_QW1_OS 1396 mfmsr r0 1397 andi. r0, r0, MSR_IR /* in real mode? */ 1398 beq 2f 1399 ld r10, HSTATE_XIVE_TIMA_VIRT(r13) 1400 cmpldi cr0, r10, 0 1401 beq 1f 1402 /* First load to pull the context, we ignore the value */ 1403 lwzx r11, r7, r10 1404 eieio 1405 /* Second load to recover the context state (Words 0 and 1) */ 1406 ldx r11, r6, r10 1407 b 3f 14082: ld r10, HSTATE_XIVE_TIMA_PHYS(r13) 1409 cmpldi cr0, r10, 0 1410 beq 1f 1411 /* First load to pull the context, we ignore the value */ 1412 lwzcix r11, r7, r10 1413 eieio 1414 /* Second load to recover the context state (Words 0 and 1) */ 1415 ldcix r11, r6, r10 14163: std r11, VCPU_XIVE_SAVED_STATE(r9) 1417 /* Fixup some of the state for the next load */ 1418 li r10, 0 1419 li r0, 0xff 1420 stw r10, VCPU_XIVE_PUSHED(r9) 1421 stb r10, (VCPU_XIVE_SAVED_STATE+3)(r9) 1422 stb r0, (VCPU_XIVE_SAVED_STATE+4)(r9) 14231: 1424#endif /* CONFIG_KVM_XICS */ 1425 /* Save more register state */ 1426 mfdar r6 1427 mfdsisr r7 1428 std r6, VCPU_DAR(r9) 1429 stw r7, VCPU_DSISR(r9) 1430 /* don't overwrite fault_dar/fault_dsisr if HDSI */ 1431 cmpwi r12,BOOK3S_INTERRUPT_H_DATA_STORAGE 1432 beq mc_cont 1433 std r6, VCPU_FAULT_DAR(r9) 1434 stw r7, VCPU_FAULT_DSISR(r9) 1435 1436 /* See if it is a machine check */ 1437 cmpwi r12, BOOK3S_INTERRUPT_MACHINE_CHECK 1438 beq machine_check_realmode 1439mc_cont: 1440#ifdef CONFIG_KVM_BOOK3S_HV_EXIT_TIMING 1441 addi r3, r9, VCPU_TB_RMEXIT 1442 mr r4, r9 1443 bl kvmhv_accumulate_time 1444#endif 1445 1446 mr r3, r12 1447 /* Increment exit count, poke other threads to exit */ 1448 bl kvmhv_commence_exit 1449 nop 1450 ld r9, HSTATE_KVM_VCPU(r13) 1451 lwz r12, VCPU_TRAP(r9) 1452 1453 /* Stop others sending VCPU interrupts to this physical CPU */ 1454 li r0, -1 1455 stw r0, VCPU_CPU(r9) 1456 stw r0, VCPU_THREAD_CPU(r9) 1457 1458 /* Save guest CTRL register, set runlatch to 1 */ 1459 mfspr r6,SPRN_CTRLF 1460 stw r6,VCPU_CTRL(r9) 1461 andi. r0,r6,1 1462 bne 4f 1463 ori r6,r6,1 1464 mtspr SPRN_CTRLT,r6 14654: 1466 /* Check if we are running hash or radix and store it in cr2 */ 1467 ld r5, VCPU_KVM(r9) 1468 lbz r0, KVM_RADIX(r5) 1469 cmpwi cr2,r0,0 1470 1471 /* Read the guest SLB and save it away */ 1472 li r5, 0 1473 bne cr2, 3f /* for radix, save 0 entries */ 1474 lwz r0,VCPU_SLB_NR(r9) /* number of entries in SLB */ 1475 mtctr r0 1476 li r6,0 1477 addi r7,r9,VCPU_SLB 14781: slbmfee r8,r6 1479 andis. r0,r8,SLB_ESID_V@h 1480 beq 2f 1481 add r8,r8,r6 /* put index in */ 1482 slbmfev r3,r6 1483 std r8,VCPU_SLB_E(r7) 1484 std r3,VCPU_SLB_V(r7) 1485 addi r7,r7,VCPU_SLB_SIZE 1486 addi r5,r5,1 14872: addi r6,r6,1 1488 bdnz 1b 14893: stw r5,VCPU_SLB_MAX(r9) 1490 1491 /* 1492 * Save the guest PURR/SPURR 1493 */ 1494 mfspr r5,SPRN_PURR 1495 mfspr r6,SPRN_SPURR 1496 ld r7,VCPU_PURR(r9) 1497 ld r8,VCPU_SPURR(r9) 1498 std r5,VCPU_PURR(r9) 1499 std r6,VCPU_SPURR(r9) 1500 subf r5,r7,r5 1501 subf r6,r8,r6 1502 1503 /* 1504 * Restore host PURR/SPURR and add guest times 1505 * so that the time in the guest gets accounted. 1506 */ 1507 ld r3,HSTATE_PURR(r13) 1508 ld r4,HSTATE_SPURR(r13) 1509 add r3,r3,r5 1510 add r4,r4,r6 1511 mtspr SPRN_PURR,r3 1512 mtspr SPRN_SPURR,r4 1513 1514 /* Save DEC */ 1515 ld r3, HSTATE_KVM_VCORE(r13) 1516 mfspr r5,SPRN_DEC 1517 mftb r6 1518 /* On P9, if the guest has large decr enabled, don't sign extend */ 1519BEGIN_FTR_SECTION 1520 ld r4, VCORE_LPCR(r3) 1521 andis. r4, r4, LPCR_LD@h 1522 bne 16f 1523END_FTR_SECTION_IFSET(CPU_FTR_ARCH_300) 1524 extsw r5,r5 152516: add r5,r5,r6 1526 /* r5 is a guest timebase value here, convert to host TB */ 1527 ld r4,VCORE_TB_OFFSET(r3) 1528 subf r5,r4,r5 1529 std r5,VCPU_DEC_EXPIRES(r9) 1530 1531BEGIN_FTR_SECTION 1532 b 8f 1533END_FTR_SECTION_IFCLR(CPU_FTR_ARCH_207S) 1534 /* Save POWER8-specific registers */ 1535 mfspr r5, SPRN_IAMR 1536 mfspr r6, SPRN_PSPB 1537 mfspr r7, SPRN_FSCR 1538 std r5, VCPU_IAMR(r9) 1539 stw r6, VCPU_PSPB(r9) 1540 std r7, VCPU_FSCR(r9) 1541 mfspr r5, SPRN_IC 1542 mfspr r7, SPRN_TAR 1543 std r5, VCPU_IC(r9) 1544 std r7, VCPU_TAR(r9) 1545 mfspr r8, SPRN_EBBHR 1546 std r8, VCPU_EBBHR(r9) 1547 mfspr r5, SPRN_EBBRR 1548 mfspr r6, SPRN_BESCR 1549 mfspr r7, SPRN_PID 1550 mfspr r8, SPRN_WORT 1551 std r5, VCPU_EBBRR(r9) 1552 std r6, VCPU_BESCR(r9) 1553 stw r7, VCPU_GUEST_PID(r9) 1554 std r8, VCPU_WORT(r9) 1555BEGIN_FTR_SECTION 1556 mfspr r5, SPRN_TCSCR 1557 mfspr r6, SPRN_ACOP 1558 mfspr r7, SPRN_CSIGR 1559 mfspr r8, SPRN_TACR 1560 std r5, VCPU_TCSCR(r9) 1561 std r6, VCPU_ACOP(r9) 1562 std r7, VCPU_CSIGR(r9) 1563 std r8, VCPU_TACR(r9) 1564FTR_SECTION_ELSE 1565 mfspr r5, SPRN_TIDR 1566 mfspr r6, SPRN_PSSCR 1567 std r5, VCPU_TID(r9) 1568 rldicl r6, r6, 4, 50 /* r6 &= PSSCR_GUEST_VIS */ 1569 rotldi r6, r6, 60 1570 std r6, VCPU_PSSCR(r9) 1571 /* Restore host HFSCR value */ 1572 ld r7, STACK_SLOT_HFSCR(r1) 1573 mtspr SPRN_HFSCR, r7 1574ALT_FTR_SECTION_END_IFCLR(CPU_FTR_ARCH_300) 1575 /* 1576 * Restore various registers to 0, where non-zero values 1577 * set by the guest could disrupt the host. 1578 */ 1579 li r0, 0 1580 mtspr SPRN_PSPB, r0 1581 mtspr SPRN_WORT, r0 1582BEGIN_FTR_SECTION 1583 mtspr SPRN_IAMR, r0 1584 mtspr SPRN_TCSCR, r0 1585 /* Set MMCRS to 1<<31 to freeze and disable the SPMC counters */ 1586 li r0, 1 1587 sldi r0, r0, 31 1588 mtspr SPRN_MMCRS, r0 1589END_FTR_SECTION_IFCLR(CPU_FTR_ARCH_300) 15908: 1591 1592 /* Save and reset AMR and UAMOR before turning on the MMU */ 1593 mfspr r5,SPRN_AMR 1594 mfspr r6,SPRN_UAMOR 1595 std r5,VCPU_AMR(r9) 1596 std r6,VCPU_UAMOR(r9) 1597 li r6,0 1598 mtspr SPRN_AMR,r6 1599 mtspr SPRN_UAMOR, r6 1600 1601 /* Switch DSCR back to host value */ 1602 mfspr r8, SPRN_DSCR 1603 ld r7, HSTATE_DSCR(r13) 1604 std r8, VCPU_DSCR(r9) 1605 mtspr SPRN_DSCR, r7 1606 1607 /* Save non-volatile GPRs */ 1608 std r14, VCPU_GPR(R14)(r9) 1609 std r15, VCPU_GPR(R15)(r9) 1610 std r16, VCPU_GPR(R16)(r9) 1611 std r17, VCPU_GPR(R17)(r9) 1612 std r18, VCPU_GPR(R18)(r9) 1613 std r19, VCPU_GPR(R19)(r9) 1614 std r20, VCPU_GPR(R20)(r9) 1615 std r21, VCPU_GPR(R21)(r9) 1616 std r22, VCPU_GPR(R22)(r9) 1617 std r23, VCPU_GPR(R23)(r9) 1618 std r24, VCPU_GPR(R24)(r9) 1619 std r25, VCPU_GPR(R25)(r9) 1620 std r26, VCPU_GPR(R26)(r9) 1621 std r27, VCPU_GPR(R27)(r9) 1622 std r28, VCPU_GPR(R28)(r9) 1623 std r29, VCPU_GPR(R29)(r9) 1624 std r30, VCPU_GPR(R30)(r9) 1625 std r31, VCPU_GPR(R31)(r9) 1626 1627 /* Save SPRGs */ 1628 mfspr r3, SPRN_SPRG0 1629 mfspr r4, SPRN_SPRG1 1630 mfspr r5, SPRN_SPRG2 1631 mfspr r6, SPRN_SPRG3 1632 std r3, VCPU_SPRG0(r9) 1633 std r4, VCPU_SPRG1(r9) 1634 std r5, VCPU_SPRG2(r9) 1635 std r6, VCPU_SPRG3(r9) 1636 1637 /* save FP state */ 1638 mr r3, r9 1639 bl kvmppc_save_fp 1640 1641#ifdef CONFIG_PPC_TRANSACTIONAL_MEM 1642BEGIN_FTR_SECTION 1643 /* 1644 * NOTE THAT THIS TRASHES ALL NON-VOLATILE REGISTERS INCLUDING CR 1645 */ 1646 bl kvmppc_save_tm 1647END_FTR_SECTION_IFSET(CPU_FTR_TM) 1648#endif 1649 1650 /* Increment yield count if they have a VPA */ 1651 ld r8, VCPU_VPA(r9) /* do they have a VPA? */ 1652 cmpdi r8, 0 1653 beq 25f 1654 li r4, LPPACA_YIELDCOUNT 1655 LWZX_BE r3, r8, r4 1656 addi r3, r3, 1 1657 STWX_BE r3, r8, r4 1658 li r3, 1 1659 stb r3, VCPU_VPA_DIRTY(r9) 166025: 1661 /* Save PMU registers if requested */ 1662 /* r8 and cr0.eq are live here */ 1663BEGIN_FTR_SECTION 1664 /* 1665 * POWER8 seems to have a hardware bug where setting 1666 * MMCR0[PMAE] along with MMCR0[PMC1CE] and/or MMCR0[PMCjCE] 1667 * when some counters are already negative doesn't seem 1668 * to cause a performance monitor alert (and hence interrupt). 1669 * The effect of this is that when saving the PMU state, 1670 * if there is no PMU alert pending when we read MMCR0 1671 * before freezing the counters, but one becomes pending 1672 * before we read the counters, we lose it. 1673 * To work around this, we need a way to freeze the counters 1674 * before reading MMCR0. Normally, freezing the counters 1675 * is done by writing MMCR0 (to set MMCR0[FC]) which 1676 * unavoidably writes MMCR0[PMA0] as well. On POWER8, 1677 * we can also freeze the counters using MMCR2, by writing 1678 * 1s to all the counter freeze condition bits (there are 1679 * 9 bits each for 6 counters). 1680 */ 1681 li r3, -1 /* set all freeze bits */ 1682 clrrdi r3, r3, 10 1683 mfspr r10, SPRN_MMCR2 1684 mtspr SPRN_MMCR2, r3 1685 isync 1686END_FTR_SECTION_IFSET(CPU_FTR_ARCH_207S) 1687 li r3, 1 1688 sldi r3, r3, 31 /* MMCR0_FC (freeze counters) bit */ 1689 mfspr r4, SPRN_MMCR0 /* save MMCR0 */ 1690 mtspr SPRN_MMCR0, r3 /* freeze all counters, disable ints */ 1691 mfspr r6, SPRN_MMCRA 1692 /* Clear MMCRA in order to disable SDAR updates */ 1693 li r7, 0 1694 mtspr SPRN_MMCRA, r7 1695 isync 1696 beq 21f /* if no VPA, save PMU stuff anyway */ 1697 lbz r7, LPPACA_PMCINUSE(r8) 1698 cmpwi r7, 0 /* did they ask for PMU stuff to be saved? */ 1699 bne 21f 1700 std r3, VCPU_MMCR(r9) /* if not, set saved MMCR0 to FC */ 1701 b 22f 170221: mfspr r5, SPRN_MMCR1 1703 mfspr r7, SPRN_SIAR 1704 mfspr r8, SPRN_SDAR 1705 std r4, VCPU_MMCR(r9) 1706 std r5, VCPU_MMCR + 8(r9) 1707 std r6, VCPU_MMCR + 16(r9) 1708BEGIN_FTR_SECTION 1709 std r10, VCPU_MMCR + 24(r9) 1710END_FTR_SECTION_IFSET(CPU_FTR_ARCH_207S) 1711 std r7, VCPU_SIAR(r9) 1712 std r8, VCPU_SDAR(r9) 1713 mfspr r3, SPRN_PMC1 1714 mfspr r4, SPRN_PMC2 1715 mfspr r5, SPRN_PMC3 1716 mfspr r6, SPRN_PMC4 1717 mfspr r7, SPRN_PMC5 1718 mfspr r8, SPRN_PMC6 1719 stw r3, VCPU_PMC(r9) 1720 stw r4, VCPU_PMC + 4(r9) 1721 stw r5, VCPU_PMC + 8(r9) 1722 stw r6, VCPU_PMC + 12(r9) 1723 stw r7, VCPU_PMC + 16(r9) 1724 stw r8, VCPU_PMC + 20(r9) 1725BEGIN_FTR_SECTION 1726 mfspr r5, SPRN_SIER 1727 std r5, VCPU_SIER(r9) 1728BEGIN_FTR_SECTION_NESTED(96) 1729 mfspr r6, SPRN_SPMC1 1730 mfspr r7, SPRN_SPMC2 1731 mfspr r8, SPRN_MMCRS 1732 stw r6, VCPU_PMC + 24(r9) 1733 stw r7, VCPU_PMC + 28(r9) 1734 std r8, VCPU_MMCR + 32(r9) 1735 lis r4, 0x8000 1736 mtspr SPRN_MMCRS, r4 1737END_FTR_SECTION_NESTED(CPU_FTR_ARCH_300, 0, 96) 1738END_FTR_SECTION_IFSET(CPU_FTR_ARCH_207S) 173922: 1740 1741 /* Restore host values of some registers */ 1742BEGIN_FTR_SECTION 1743 ld r5, STACK_SLOT_CIABR(r1) 1744 ld r6, STACK_SLOT_DAWR(r1) 1745 ld r7, STACK_SLOT_DAWRX(r1) 1746 mtspr SPRN_CIABR, r5 1747 mtspr SPRN_DAWR, r6 1748 mtspr SPRN_DAWRX, r7 1749END_FTR_SECTION_IFSET(CPU_FTR_ARCH_207S) 1750BEGIN_FTR_SECTION 1751 ld r5, STACK_SLOT_TID(r1) 1752 ld r6, STACK_SLOT_PSSCR(r1) 1753 ld r7, STACK_SLOT_PID(r1) 1754 ld r8, STACK_SLOT_IAMR(r1) 1755 mtspr SPRN_TIDR, r5 1756 mtspr SPRN_PSSCR, r6 1757 mtspr SPRN_PID, r7 1758 mtspr SPRN_IAMR, r8 1759END_FTR_SECTION_IFSET(CPU_FTR_ARCH_300) 1760 1761#ifdef CONFIG_PPC_RADIX_MMU 1762 /* 1763 * Are we running hash or radix ? 1764 */ 1765 ld r5, VCPU_KVM(r9) 1766 lbz r0, KVM_RADIX(r5) 1767 cmpwi cr2, r0, 0 1768 beq cr2, 3f 1769 1770 /* Radix: Handle the case where the guest used an illegal PID */ 1771 LOAD_REG_ADDR(r4, mmu_base_pid) 1772 lwz r3, VCPU_GUEST_PID(r9) 1773 lwz r5, 0(r4) 1774 cmpw cr0,r3,r5 1775 blt 2f 1776 1777 /* 1778 * Illegal PID, the HW might have prefetched and cached in the TLB 1779 * some translations for the LPID 0 / guest PID combination which 1780 * Linux doesn't know about, so we need to flush that PID out of 1781 * the TLB. First we need to set LPIDR to 0 so tlbiel applies to 1782 * the right context. 1783 */ 1784 li r0,0 1785 mtspr SPRN_LPID,r0 1786 isync 1787 1788 /* Then do a congruence class local flush */ 1789 ld r6,VCPU_KVM(r9) 1790 lwz r0,KVM_TLB_SETS(r6) 1791 mtctr r0 1792 li r7,0x400 /* IS field = 0b01 */ 1793 ptesync 1794 sldi r0,r3,32 /* RS has PID */ 17951: PPC_TLBIEL(7,0,2,1,1) /* RIC=2, PRS=1, R=1 */ 1796 addi r7,r7,0x1000 1797 bdnz 1b 1798 ptesync 1799 18002: /* Flush the ERAT on radix P9 DD1 guest exit */ 1801BEGIN_FTR_SECTION 1802 PPC_INVALIDATE_ERAT 1803END_FTR_SECTION_IFSET(CPU_FTR_POWER9_DD1) 1804 b 4f 1805#endif /* CONFIG_PPC_RADIX_MMU */ 1806 1807 /* Hash: clear out SLB */ 18083: li r5,0 1809 slbmte r5,r5 1810 slbia 1811 ptesync 18124: 1813 /* 1814 * POWER7/POWER8 guest -> host partition switch code. 1815 * We don't have to lock against tlbies but we do 1816 * have to coordinate the hardware threads. 1817 */ 1818kvmhv_switch_to_host: 1819 /* Secondary threads wait for primary to do partition switch */ 1820 ld r5,HSTATE_KVM_VCORE(r13) 1821 ld r4,VCORE_KVM(r5) /* pointer to struct kvm */ 1822 lbz r3,HSTATE_PTID(r13) 1823 cmpwi r3,0 1824 beq 15f 1825 HMT_LOW 182613: lbz r3,VCORE_IN_GUEST(r5) 1827 cmpwi r3,0 1828 bne 13b 1829 HMT_MEDIUM 1830 b 16f 1831 1832 /* Primary thread waits for all the secondaries to exit guest */ 183315: lwz r3,VCORE_ENTRY_EXIT(r5) 1834 rlwinm r0,r3,32-8,0xff 1835 clrldi r3,r3,56 1836 cmpw r3,r0 1837 bne 15b 1838 isync 1839 1840 /* Did we actually switch to the guest at all? */ 1841 lbz r6, VCORE_IN_GUEST(r5) 1842 cmpwi r6, 0 1843 beq 19f 1844 1845 /* Primary thread switches back to host partition */ 1846 lwz r7,KVM_HOST_LPID(r4) 1847BEGIN_FTR_SECTION 1848 ld r6,KVM_HOST_SDR1(r4) 1849 li r8,LPID_RSVD /* switch to reserved LPID */ 1850 mtspr SPRN_LPID,r8 1851 ptesync 1852 mtspr SPRN_SDR1,r6 /* switch to host page table */ 1853END_FTR_SECTION_IFCLR(CPU_FTR_ARCH_300) 1854 mtspr SPRN_LPID,r7 1855 isync 1856 1857BEGIN_FTR_SECTION 1858 /* DPDES and VTB are shared between threads */ 1859 mfspr r7, SPRN_DPDES 1860 mfspr r8, SPRN_VTB 1861 std r7, VCORE_DPDES(r5) 1862 std r8, VCORE_VTB(r5) 1863 /* clear DPDES so we don't get guest doorbells in the host */ 1864 li r8, 0 1865 mtspr SPRN_DPDES, r8 1866END_FTR_SECTION_IFSET(CPU_FTR_ARCH_207S) 1867 1868 /* If HMI, call kvmppc_realmode_hmi_handler() */ 1869 cmpwi r12, BOOK3S_INTERRUPT_HMI 1870 bne 27f 1871 bl kvmppc_realmode_hmi_handler 1872 nop 1873 li r12, BOOK3S_INTERRUPT_HMI 1874 /* 1875 * At this point kvmppc_realmode_hmi_handler would have resync-ed 1876 * the TB. Hence it is not required to subtract guest timebase 1877 * offset from timebase. So, skip it. 1878 * 1879 * Also, do not call kvmppc_subcore_exit_guest() because it has 1880 * been invoked as part of kvmppc_realmode_hmi_handler(). 1881 */ 1882 b 30f 1883 188427: 1885 /* Subtract timebase offset from timebase */ 1886 ld r8,VCORE_TB_OFFSET(r5) 1887 cmpdi r8,0 1888 beq 17f 1889 mftb r6 /* current guest timebase */ 1890 subf r8,r8,r6 1891 mtspr SPRN_TBU40,r8 /* update upper 40 bits */ 1892 mftb r7 /* check if lower 24 bits overflowed */ 1893 clrldi r6,r6,40 1894 clrldi r7,r7,40 1895 cmpld r7,r6 1896 bge 17f 1897 addis r8,r8,0x100 /* if so, increment upper 40 bits */ 1898 mtspr SPRN_TBU40,r8 1899 190017: bl kvmppc_subcore_exit_guest 1901 nop 190230: ld r5,HSTATE_KVM_VCORE(r13) 1903 ld r4,VCORE_KVM(r5) /* pointer to struct kvm */ 1904 1905 /* Reset PCR */ 1906 ld r0, VCORE_PCR(r5) 1907 cmpdi r0, 0 1908 beq 18f 1909 li r0, 0 1910 mtspr SPRN_PCR, r0 191118: 1912 /* Signal secondary CPUs to continue */ 1913 stb r0,VCORE_IN_GUEST(r5) 191419: lis r8,0x7fff /* MAX_INT@h */ 1915 mtspr SPRN_HDEC,r8 1916 191716: ld r8,KVM_HOST_LPCR(r4) 1918 mtspr SPRN_LPCR,r8 1919 isync 1920 1921 /* load host SLB entries */ 1922BEGIN_MMU_FTR_SECTION 1923 b 0f 1924END_MMU_FTR_SECTION_IFSET(MMU_FTR_TYPE_RADIX) 1925 ld r8,PACA_SLBSHADOWPTR(r13) 1926 1927 .rept SLB_NUM_BOLTED 1928 li r3, SLBSHADOW_SAVEAREA 1929 LDX_BE r5, r8, r3 1930 addi r3, r3, 8 1931 LDX_BE r6, r8, r3 1932 andis. r7,r5,SLB_ESID_V@h 1933 beq 1f 1934 slbmte r6,r5 19351: addi r8,r8,16 1936 .endr 19370: 1938#ifdef CONFIG_KVM_BOOK3S_HV_EXIT_TIMING 1939 /* Finish timing, if we have a vcpu */ 1940 ld r4, HSTATE_KVM_VCPU(r13) 1941 cmpdi r4, 0 1942 li r3, 0 1943 beq 2f 1944 bl kvmhv_accumulate_time 19452: 1946#endif 1947 /* Unset guest mode */ 1948 li r0, KVM_GUEST_MODE_NONE 1949 stb r0, HSTATE_IN_GUEST(r13) 1950 1951 ld r0, SFS+PPC_LR_STKOFF(r1) 1952 addi r1, r1, SFS 1953 mtlr r0 1954 blr 1955 1956/* 1957 * Check whether an HDSI is an HPTE not found fault or something else. 1958 * If it is an HPTE not found fault that is due to the guest accessing 1959 * a page that they have mapped but which we have paged out, then 1960 * we continue on with the guest exit path. In all other cases, 1961 * reflect the HDSI to the guest as a DSI. 1962 */ 1963kvmppc_hdsi: 1964 ld r3, VCPU_KVM(r9) 1965 lbz r0, KVM_RADIX(r3) 1966 mfspr r4, SPRN_HDAR 1967 mfspr r6, SPRN_HDSISR 1968BEGIN_FTR_SECTION 1969 /* Look for DSISR canary. If we find it, retry instruction */ 1970 cmpdi r6, 0x7fff 1971 beq 6f 1972END_FTR_SECTION_IFSET(CPU_FTR_ARCH_300) 1973 cmpwi r0, 0 1974 bne .Lradix_hdsi /* on radix, just save DAR/DSISR/ASDR */ 1975 /* HPTE not found fault or protection fault? */ 1976 andis. r0, r6, (DSISR_NOHPTE | DSISR_PROTFAULT)@h 1977 beq 1f /* if not, send it to the guest */ 1978 andi. r0, r11, MSR_DR /* data relocation enabled? */ 1979 beq 3f 1980BEGIN_FTR_SECTION 1981 mfspr r5, SPRN_ASDR /* on POWER9, use ASDR to get VSID */ 1982 b 4f 1983END_FTR_SECTION_IFSET(CPU_FTR_ARCH_300) 1984 clrrdi r0, r4, 28 1985 PPC_SLBFEE_DOT(R5, R0) /* if so, look up SLB */ 1986 li r0, BOOK3S_INTERRUPT_DATA_SEGMENT 1987 bne 7f /* if no SLB entry found */ 19884: std r4, VCPU_FAULT_DAR(r9) 1989 stw r6, VCPU_FAULT_DSISR(r9) 1990 1991 /* Search the hash table. */ 1992 mr r3, r9 /* vcpu pointer */ 1993 li r7, 1 /* data fault */ 1994 bl kvmppc_hpte_hv_fault 1995 ld r9, HSTATE_KVM_VCPU(r13) 1996 ld r10, VCPU_PC(r9) 1997 ld r11, VCPU_MSR(r9) 1998 li r12, BOOK3S_INTERRUPT_H_DATA_STORAGE 1999 cmpdi r3, 0 /* retry the instruction */ 2000 beq 6f 2001 cmpdi r3, -1 /* handle in kernel mode */ 2002 beq guest_exit_cont 2003 cmpdi r3, -2 /* MMIO emulation; need instr word */ 2004 beq 2f 2005 2006 /* Synthesize a DSI (or DSegI) for the guest */ 2007 ld r4, VCPU_FAULT_DAR(r9) 2008 mr r6, r3 20091: li r0, BOOK3S_INTERRUPT_DATA_STORAGE 2010 mtspr SPRN_DSISR, r6 20117: mtspr SPRN_DAR, r4 2012 mtspr SPRN_SRR0, r10 2013 mtspr SPRN_SRR1, r11 2014 mr r10, r0 2015 bl kvmppc_msr_interrupt 2016fast_interrupt_c_return: 20176: ld r7, VCPU_CTR(r9) 2018 ld r8, VCPU_XER(r9) 2019 mtctr r7 2020 mtxer r8 2021 mr r4, r9 2022 b fast_guest_return 2023 20243: ld r5, VCPU_KVM(r9) /* not relocated, use VRMA */ 2025 ld r5, KVM_VRMA_SLB_V(r5) 2026 b 4b 2027 2028 /* If this is for emulated MMIO, load the instruction word */ 20292: li r8, KVM_INST_FETCH_FAILED /* In case lwz faults */ 2030 2031 /* Set guest mode to 'jump over instruction' so if lwz faults 2032 * we'll just continue at the next IP. */ 2033 li r0, KVM_GUEST_MODE_SKIP 2034 stb r0, HSTATE_IN_GUEST(r13) 2035 2036 /* Do the access with MSR:DR enabled */ 2037 mfmsr r3 2038 ori r4, r3, MSR_DR /* Enable paging for data */ 2039 mtmsrd r4 2040 lwz r8, 0(r10) 2041 mtmsrd r3 2042 2043 /* Store the result */ 2044 stw r8, VCPU_LAST_INST(r9) 2045 2046 /* Unset guest mode. */ 2047 li r0, KVM_GUEST_MODE_HOST_HV 2048 stb r0, HSTATE_IN_GUEST(r13) 2049 b guest_exit_cont 2050 2051.Lradix_hdsi: 2052 std r4, VCPU_FAULT_DAR(r9) 2053 stw r6, VCPU_FAULT_DSISR(r9) 2054.Lradix_hisi: 2055 mfspr r5, SPRN_ASDR 2056 std r5, VCPU_FAULT_GPA(r9) 2057 b guest_exit_cont 2058 2059/* 2060 * Similarly for an HISI, reflect it to the guest as an ISI unless 2061 * it is an HPTE not found fault for a page that we have paged out. 2062 */ 2063kvmppc_hisi: 2064 ld r3, VCPU_KVM(r9) 2065 lbz r0, KVM_RADIX(r3) 2066 cmpwi r0, 0 2067 bne .Lradix_hisi /* for radix, just save ASDR */ 2068 andis. r0, r11, SRR1_ISI_NOPT@h 2069 beq 1f 2070 andi. r0, r11, MSR_IR /* instruction relocation enabled? */ 2071 beq 3f 2072BEGIN_FTR_SECTION 2073 mfspr r5, SPRN_ASDR /* on POWER9, use ASDR to get VSID */ 2074 b 4f 2075END_FTR_SECTION_IFSET(CPU_FTR_ARCH_300) 2076 clrrdi r0, r10, 28 2077 PPC_SLBFEE_DOT(R5, R0) /* if so, look up SLB */ 2078 li r0, BOOK3S_INTERRUPT_INST_SEGMENT 2079 bne 7f /* if no SLB entry found */ 20804: 2081 /* Search the hash table. */ 2082 mr r3, r9 /* vcpu pointer */ 2083 mr r4, r10 2084 mr r6, r11 2085 li r7, 0 /* instruction fault */ 2086 bl kvmppc_hpte_hv_fault 2087 ld r9, HSTATE_KVM_VCPU(r13) 2088 ld r10, VCPU_PC(r9) 2089 ld r11, VCPU_MSR(r9) 2090 li r12, BOOK3S_INTERRUPT_H_INST_STORAGE 2091 cmpdi r3, 0 /* retry the instruction */ 2092 beq fast_interrupt_c_return 2093 cmpdi r3, -1 /* handle in kernel mode */ 2094 beq guest_exit_cont 2095 2096 /* Synthesize an ISI (or ISegI) for the guest */ 2097 mr r11, r3 20981: li r0, BOOK3S_INTERRUPT_INST_STORAGE 20997: mtspr SPRN_SRR0, r10 2100 mtspr SPRN_SRR1, r11 2101 mr r10, r0 2102 bl kvmppc_msr_interrupt 2103 b fast_interrupt_c_return 2104 21053: ld r6, VCPU_KVM(r9) /* not relocated, use VRMA */ 2106 ld r5, KVM_VRMA_SLB_V(r6) 2107 b 4b 2108 2109/* 2110 * Try to handle an hcall in real mode. 2111 * Returns to the guest if we handle it, or continues on up to 2112 * the kernel if we can't (i.e. if we don't have a handler for 2113 * it, or if the handler returns H_TOO_HARD). 2114 * 2115 * r5 - r8 contain hcall args, 2116 * r9 = vcpu, r10 = pc, r11 = msr, r12 = trap, r13 = paca 2117 */ 2118hcall_try_real_mode: 2119 ld r3,VCPU_GPR(R3)(r9) 2120 andi. r0,r11,MSR_PR 2121 /* sc 1 from userspace - reflect to guest syscall */ 2122 bne sc_1_fast_return 2123 clrrdi r3,r3,2 2124 cmpldi r3,hcall_real_table_end - hcall_real_table 2125 bge guest_exit_cont 2126 /* See if this hcall is enabled for in-kernel handling */ 2127 ld r4, VCPU_KVM(r9) 2128 srdi r0, r3, 8 /* r0 = (r3 / 4) >> 6 */ 2129 sldi r0, r0, 3 /* index into kvm->arch.enabled_hcalls[] */ 2130 add r4, r4, r0 2131 ld r0, KVM_ENABLED_HCALLS(r4) 2132 rlwinm r4, r3, 32-2, 0x3f /* r4 = (r3 / 4) & 0x3f */ 2133 srd r0, r0, r4 2134 andi. r0, r0, 1 2135 beq guest_exit_cont 2136 /* Get pointer to handler, if any, and call it */ 2137 LOAD_REG_ADDR(r4, hcall_real_table) 2138 lwax r3,r3,r4 2139 cmpwi r3,0 2140 beq guest_exit_cont 2141 add r12,r3,r4 2142 mtctr r12 2143 mr r3,r9 /* get vcpu pointer */ 2144 ld r4,VCPU_GPR(R4)(r9) 2145 bctrl 2146 cmpdi r3,H_TOO_HARD 2147 beq hcall_real_fallback 2148 ld r4,HSTATE_KVM_VCPU(r13) 2149 std r3,VCPU_GPR(R3)(r4) 2150 ld r10,VCPU_PC(r4) 2151 ld r11,VCPU_MSR(r4) 2152 b fast_guest_return 2153 2154sc_1_fast_return: 2155 mtspr SPRN_SRR0,r10 2156 mtspr SPRN_SRR1,r11 2157 li r10, BOOK3S_INTERRUPT_SYSCALL 2158 bl kvmppc_msr_interrupt 2159 mr r4,r9 2160 b fast_guest_return 2161 2162 /* We've attempted a real mode hcall, but it's punted it back 2163 * to userspace. We need to restore some clobbered volatiles 2164 * before resuming the pass-it-to-qemu path */ 2165hcall_real_fallback: 2166 li r12,BOOK3S_INTERRUPT_SYSCALL 2167 ld r9, HSTATE_KVM_VCPU(r13) 2168 2169 b guest_exit_cont 2170 2171 .globl hcall_real_table 2172hcall_real_table: 2173 .long 0 /* 0 - unused */ 2174 .long DOTSYM(kvmppc_h_remove) - hcall_real_table 2175 .long DOTSYM(kvmppc_h_enter) - hcall_real_table 2176 .long DOTSYM(kvmppc_h_read) - hcall_real_table 2177 .long DOTSYM(kvmppc_h_clear_mod) - hcall_real_table 2178 .long DOTSYM(kvmppc_h_clear_ref) - hcall_real_table 2179 .long DOTSYM(kvmppc_h_protect) - hcall_real_table 2180 .long DOTSYM(kvmppc_h_get_tce) - hcall_real_table 2181 .long DOTSYM(kvmppc_rm_h_put_tce) - hcall_real_table 2182 .long 0 /* 0x24 - H_SET_SPRG0 */ 2183 .long DOTSYM(kvmppc_h_set_dabr) - hcall_real_table 2184 .long 0 /* 0x2c */ 2185 .long 0 /* 0x30 */ 2186 .long 0 /* 0x34 */ 2187 .long 0 /* 0x38 */ 2188 .long 0 /* 0x3c */ 2189 .long 0 /* 0x40 */ 2190 .long 0 /* 0x44 */ 2191 .long 0 /* 0x48 */ 2192 .long 0 /* 0x4c */ 2193 .long 0 /* 0x50 */ 2194 .long 0 /* 0x54 */ 2195 .long 0 /* 0x58 */ 2196 .long 0 /* 0x5c */ 2197 .long 0 /* 0x60 */ 2198#ifdef CONFIG_KVM_XICS 2199 .long DOTSYM(kvmppc_rm_h_eoi) - hcall_real_table 2200 .long DOTSYM(kvmppc_rm_h_cppr) - hcall_real_table 2201 .long DOTSYM(kvmppc_rm_h_ipi) - hcall_real_table 2202 .long DOTSYM(kvmppc_rm_h_ipoll) - hcall_real_table 2203 .long DOTSYM(kvmppc_rm_h_xirr) - hcall_real_table 2204#else 2205 .long 0 /* 0x64 - H_EOI */ 2206 .long 0 /* 0x68 - H_CPPR */ 2207 .long 0 /* 0x6c - H_IPI */ 2208 .long 0 /* 0x70 - H_IPOLL */ 2209 .long 0 /* 0x74 - H_XIRR */ 2210#endif 2211 .long 0 /* 0x78 */ 2212 .long 0 /* 0x7c */ 2213 .long 0 /* 0x80 */ 2214 .long 0 /* 0x84 */ 2215 .long 0 /* 0x88 */ 2216 .long 0 /* 0x8c */ 2217 .long 0 /* 0x90 */ 2218 .long 0 /* 0x94 */ 2219 .long 0 /* 0x98 */ 2220 .long 0 /* 0x9c */ 2221 .long 0 /* 0xa0 */ 2222 .long 0 /* 0xa4 */ 2223 .long 0 /* 0xa8 */ 2224 .long 0 /* 0xac */ 2225 .long 0 /* 0xb0 */ 2226 .long 0 /* 0xb4 */ 2227 .long 0 /* 0xb8 */ 2228 .long 0 /* 0xbc */ 2229 .long 0 /* 0xc0 */ 2230 .long 0 /* 0xc4 */ 2231 .long 0 /* 0xc8 */ 2232 .long 0 /* 0xcc */ 2233 .long 0 /* 0xd0 */ 2234 .long 0 /* 0xd4 */ 2235 .long 0 /* 0xd8 */ 2236 .long 0 /* 0xdc */ 2237 .long DOTSYM(kvmppc_h_cede) - hcall_real_table 2238 .long DOTSYM(kvmppc_rm_h_confer) - hcall_real_table 2239 .long 0 /* 0xe8 */ 2240 .long 0 /* 0xec */ 2241 .long 0 /* 0xf0 */ 2242 .long 0 /* 0xf4 */ 2243 .long 0 /* 0xf8 */ 2244 .long 0 /* 0xfc */ 2245 .long 0 /* 0x100 */ 2246 .long 0 /* 0x104 */ 2247 .long 0 /* 0x108 */ 2248 .long 0 /* 0x10c */ 2249 .long 0 /* 0x110 */ 2250 .long 0 /* 0x114 */ 2251 .long 0 /* 0x118 */ 2252 .long 0 /* 0x11c */ 2253 .long 0 /* 0x120 */ 2254 .long DOTSYM(kvmppc_h_bulk_remove) - hcall_real_table 2255 .long 0 /* 0x128 */ 2256 .long 0 /* 0x12c */ 2257 .long 0 /* 0x130 */ 2258 .long DOTSYM(kvmppc_h_set_xdabr) - hcall_real_table 2259 .long DOTSYM(kvmppc_rm_h_stuff_tce) - hcall_real_table 2260 .long DOTSYM(kvmppc_rm_h_put_tce_indirect) - hcall_real_table 2261 .long 0 /* 0x140 */ 2262 .long 0 /* 0x144 */ 2263 .long 0 /* 0x148 */ 2264 .long 0 /* 0x14c */ 2265 .long 0 /* 0x150 */ 2266 .long 0 /* 0x154 */ 2267 .long 0 /* 0x158 */ 2268 .long 0 /* 0x15c */ 2269 .long 0 /* 0x160 */ 2270 .long 0 /* 0x164 */ 2271 .long 0 /* 0x168 */ 2272 .long 0 /* 0x16c */ 2273 .long 0 /* 0x170 */ 2274 .long 0 /* 0x174 */ 2275 .long 0 /* 0x178 */ 2276 .long 0 /* 0x17c */ 2277 .long 0 /* 0x180 */ 2278 .long 0 /* 0x184 */ 2279 .long 0 /* 0x188 */ 2280 .long 0 /* 0x18c */ 2281 .long 0 /* 0x190 */ 2282 .long 0 /* 0x194 */ 2283 .long 0 /* 0x198 */ 2284 .long 0 /* 0x19c */ 2285 .long 0 /* 0x1a0 */ 2286 .long 0 /* 0x1a4 */ 2287 .long 0 /* 0x1a8 */ 2288 .long 0 /* 0x1ac */ 2289 .long 0 /* 0x1b0 */ 2290 .long 0 /* 0x1b4 */ 2291 .long 0 /* 0x1b8 */ 2292 .long 0 /* 0x1bc */ 2293 .long 0 /* 0x1c0 */ 2294 .long 0 /* 0x1c4 */ 2295 .long 0 /* 0x1c8 */ 2296 .long 0 /* 0x1cc */ 2297 .long 0 /* 0x1d0 */ 2298 .long 0 /* 0x1d4 */ 2299 .long 0 /* 0x1d8 */ 2300 .long 0 /* 0x1dc */ 2301 .long 0 /* 0x1e0 */ 2302 .long 0 /* 0x1e4 */ 2303 .long 0 /* 0x1e8 */ 2304 .long 0 /* 0x1ec */ 2305 .long 0 /* 0x1f0 */ 2306 .long 0 /* 0x1f4 */ 2307 .long 0 /* 0x1f8 */ 2308 .long 0 /* 0x1fc */ 2309 .long 0 /* 0x200 */ 2310 .long 0 /* 0x204 */ 2311 .long 0 /* 0x208 */ 2312 .long 0 /* 0x20c */ 2313 .long 0 /* 0x210 */ 2314 .long 0 /* 0x214 */ 2315 .long 0 /* 0x218 */ 2316 .long 0 /* 0x21c */ 2317 .long 0 /* 0x220 */ 2318 .long 0 /* 0x224 */ 2319 .long 0 /* 0x228 */ 2320 .long 0 /* 0x22c */ 2321 .long 0 /* 0x230 */ 2322 .long 0 /* 0x234 */ 2323 .long 0 /* 0x238 */ 2324 .long 0 /* 0x23c */ 2325 .long 0 /* 0x240 */ 2326 .long 0 /* 0x244 */ 2327 .long 0 /* 0x248 */ 2328 .long 0 /* 0x24c */ 2329 .long 0 /* 0x250 */ 2330 .long 0 /* 0x254 */ 2331 .long 0 /* 0x258 */ 2332 .long 0 /* 0x25c */ 2333 .long 0 /* 0x260 */ 2334 .long 0 /* 0x264 */ 2335 .long 0 /* 0x268 */ 2336 .long 0 /* 0x26c */ 2337 .long 0 /* 0x270 */ 2338 .long 0 /* 0x274 */ 2339 .long 0 /* 0x278 */ 2340 .long 0 /* 0x27c */ 2341 .long 0 /* 0x280 */ 2342 .long 0 /* 0x284 */ 2343 .long 0 /* 0x288 */ 2344 .long 0 /* 0x28c */ 2345 .long 0 /* 0x290 */ 2346 .long 0 /* 0x294 */ 2347 .long 0 /* 0x298 */ 2348 .long 0 /* 0x29c */ 2349 .long 0 /* 0x2a0 */ 2350 .long 0 /* 0x2a4 */ 2351 .long 0 /* 0x2a8 */ 2352 .long 0 /* 0x2ac */ 2353 .long 0 /* 0x2b0 */ 2354 .long 0 /* 0x2b4 */ 2355 .long 0 /* 0x2b8 */ 2356 .long 0 /* 0x2bc */ 2357 .long 0 /* 0x2c0 */ 2358 .long 0 /* 0x2c4 */ 2359 .long 0 /* 0x2c8 */ 2360 .long 0 /* 0x2cc */ 2361 .long 0 /* 0x2d0 */ 2362 .long 0 /* 0x2d4 */ 2363 .long 0 /* 0x2d8 */ 2364 .long 0 /* 0x2dc */ 2365 .long 0 /* 0x2e0 */ 2366 .long 0 /* 0x2e4 */ 2367 .long 0 /* 0x2e8 */ 2368 .long 0 /* 0x2ec */ 2369 .long 0 /* 0x2f0 */ 2370 .long 0 /* 0x2f4 */ 2371 .long 0 /* 0x2f8 */ 2372#ifdef CONFIG_KVM_XICS 2373 .long DOTSYM(kvmppc_rm_h_xirr_x) - hcall_real_table 2374#else 2375 .long 0 /* 0x2fc - H_XIRR_X*/ 2376#endif 2377 .long DOTSYM(kvmppc_h_random) - hcall_real_table 2378 .globl hcall_real_table_end 2379hcall_real_table_end: 2380 2381_GLOBAL(kvmppc_h_set_xdabr) 2382 andi. r0, r5, DABRX_USER | DABRX_KERNEL 2383 beq 6f 2384 li r0, DABRX_USER | DABRX_KERNEL | DABRX_BTI 2385 andc. r0, r5, r0 2386 beq 3f 23876: li r3, H_PARAMETER 2388 blr 2389 2390_GLOBAL(kvmppc_h_set_dabr) 2391 li r5, DABRX_USER | DABRX_KERNEL 23923: 2393BEGIN_FTR_SECTION 2394 b 2f 2395END_FTR_SECTION_IFSET(CPU_FTR_ARCH_207S) 2396 std r4,VCPU_DABR(r3) 2397 stw r5, VCPU_DABRX(r3) 2398 mtspr SPRN_DABRX, r5 2399 /* Work around P7 bug where DABR can get corrupted on mtspr */ 24001: mtspr SPRN_DABR,r4 2401 mfspr r5, SPRN_DABR 2402 cmpd r4, r5 2403 bne 1b 2404 isync 2405 li r3,0 2406 blr 2407 2408 /* Emulate H_SET_DABR/X on P8 for the sake of compat mode guests */ 24092: rlwimi r5, r4, 5, DAWRX_DR | DAWRX_DW 2410 rlwimi r5, r4, 2, DAWRX_WT 2411 clrrdi r4, r4, 3 2412 std r4, VCPU_DAWR(r3) 2413 std r5, VCPU_DAWRX(r3) 2414 mtspr SPRN_DAWR, r4 2415 mtspr SPRN_DAWRX, r5 2416 li r3, 0 2417 blr 2418 2419_GLOBAL(kvmppc_h_cede) /* r3 = vcpu pointer, r11 = msr, r13 = paca */ 2420 ori r11,r11,MSR_EE 2421 std r11,VCPU_MSR(r3) 2422 li r0,1 2423 stb r0,VCPU_CEDED(r3) 2424 sync /* order setting ceded vs. testing prodded */ 2425 lbz r5,VCPU_PRODDED(r3) 2426 cmpwi r5,0 2427 bne kvm_cede_prodded 2428 li r12,0 /* set trap to 0 to say hcall is handled */ 2429 stw r12,VCPU_TRAP(r3) 2430 li r0,H_SUCCESS 2431 std r0,VCPU_GPR(R3)(r3) 2432 2433 /* 2434 * Set our bit in the bitmask of napping threads unless all the 2435 * other threads are already napping, in which case we send this 2436 * up to the host. 2437 */ 2438 ld r5,HSTATE_KVM_VCORE(r13) 2439 lbz r6,HSTATE_PTID(r13) 2440 lwz r8,VCORE_ENTRY_EXIT(r5) 2441 clrldi r8,r8,56 2442 li r0,1 2443 sld r0,r0,r6 2444 addi r6,r5,VCORE_NAPPING_THREADS 244531: lwarx r4,0,r6 2446 or r4,r4,r0 2447 cmpw r4,r8 2448 beq kvm_cede_exit 2449 stwcx. r4,0,r6 2450 bne 31b 2451 /* order napping_threads update vs testing entry_exit_map */ 2452 isync 2453 li r0,NAPPING_CEDE 2454 stb r0,HSTATE_NAPPING(r13) 2455 lwz r7,VCORE_ENTRY_EXIT(r5) 2456 cmpwi r7,0x100 2457 bge 33f /* another thread already exiting */ 2458 2459/* 2460 * Although not specifically required by the architecture, POWER7 2461 * preserves the following registers in nap mode, even if an SMT mode 2462 * switch occurs: SLB entries, PURR, SPURR, AMOR, UAMOR, AMR, SPRG0-3, 2463 * DAR, DSISR, DABR, DABRX, DSCR, PMCx, MMCRx, SIAR, SDAR. 2464 */ 2465 /* Save non-volatile GPRs */ 2466 std r14, VCPU_GPR(R14)(r3) 2467 std r15, VCPU_GPR(R15)(r3) 2468 std r16, VCPU_GPR(R16)(r3) 2469 std r17, VCPU_GPR(R17)(r3) 2470 std r18, VCPU_GPR(R18)(r3) 2471 std r19, VCPU_GPR(R19)(r3) 2472 std r20, VCPU_GPR(R20)(r3) 2473 std r21, VCPU_GPR(R21)(r3) 2474 std r22, VCPU_GPR(R22)(r3) 2475 std r23, VCPU_GPR(R23)(r3) 2476 std r24, VCPU_GPR(R24)(r3) 2477 std r25, VCPU_GPR(R25)(r3) 2478 std r26, VCPU_GPR(R26)(r3) 2479 std r27, VCPU_GPR(R27)(r3) 2480 std r28, VCPU_GPR(R28)(r3) 2481 std r29, VCPU_GPR(R29)(r3) 2482 std r30, VCPU_GPR(R30)(r3) 2483 std r31, VCPU_GPR(R31)(r3) 2484 2485 /* save FP state */ 2486 bl kvmppc_save_fp 2487 2488#ifdef CONFIG_PPC_TRANSACTIONAL_MEM 2489BEGIN_FTR_SECTION 2490 /* 2491 * NOTE THAT THIS TRASHES ALL NON-VOLATILE REGISTERS INCLUDING CR 2492 */ 2493 ld r9, HSTATE_KVM_VCPU(r13) 2494 bl kvmppc_save_tm 2495END_FTR_SECTION_IFSET(CPU_FTR_TM) 2496#endif 2497 2498 /* 2499 * Set DEC to the smaller of DEC and HDEC, so that we wake 2500 * no later than the end of our timeslice (HDEC interrupts 2501 * don't wake us from nap). 2502 */ 2503 mfspr r3, SPRN_DEC 2504 mfspr r4, SPRN_HDEC 2505 mftb r5 2506BEGIN_FTR_SECTION 2507 /* On P9 check whether the guest has large decrementer mode enabled */ 2508 ld r6, HSTATE_KVM_VCORE(r13) 2509 ld r6, VCORE_LPCR(r6) 2510 andis. r6, r6, LPCR_LD@h 2511 bne 68f 2512END_FTR_SECTION_IFSET(CPU_FTR_ARCH_300) 2513 extsw r3, r3 251468: EXTEND_HDEC(r4) 2515 cmpd r3, r4 2516 ble 67f 2517 mtspr SPRN_DEC, r4 251867: 2519 /* save expiry time of guest decrementer */ 2520 add r3, r3, r5 2521 ld r4, HSTATE_KVM_VCPU(r13) 2522 ld r5, HSTATE_KVM_VCORE(r13) 2523 ld r6, VCORE_TB_OFFSET(r5) 2524 subf r3, r6, r3 /* convert to host TB value */ 2525 std r3, VCPU_DEC_EXPIRES(r4) 2526 2527#ifdef CONFIG_KVM_BOOK3S_HV_EXIT_TIMING 2528 ld r4, HSTATE_KVM_VCPU(r13) 2529 addi r3, r4, VCPU_TB_CEDE 2530 bl kvmhv_accumulate_time 2531#endif 2532 2533 lis r3, LPCR_PECEDP@h /* Do wake on privileged doorbell */ 2534 2535 /* 2536 * Take a nap until a decrementer or external or doobell interrupt 2537 * occurs, with PECE1 and PECE0 set in LPCR. 2538 * On POWER8, set PECEDH, and if we are ceding, also set PECEDP. 2539 * Also clear the runlatch bit before napping. 2540 */ 2541kvm_do_nap: 2542 mfspr r0, SPRN_CTRLF 2543 clrrdi r0, r0, 1 2544 mtspr SPRN_CTRLT, r0 2545 2546BEGIN_FTR_SECTION 2547 li r0,1 2548 stb r0,HSTATE_HWTHREAD_REQ(r13) 2549END_FTR_SECTION_IFCLR(CPU_FTR_ARCH_300) 2550 mfspr r5,SPRN_LPCR 2551 ori r5,r5,LPCR_PECE0 | LPCR_PECE1 2552BEGIN_FTR_SECTION 2553 ori r5, r5, LPCR_PECEDH 2554 rlwimi r5, r3, 0, LPCR_PECEDP 2555END_FTR_SECTION_IFSET(CPU_FTR_ARCH_207S) 2556 2557kvm_nap_sequence: /* desired LPCR value in r5 */ 2558BEGIN_FTR_SECTION 2559 /* 2560 * PSSCR bits: exit criterion = 1 (wakeup based on LPCR at sreset) 2561 * enable state loss = 1 (allow SMT mode switch) 2562 * requested level = 0 (just stop dispatching) 2563 */ 2564 lis r3, (PSSCR_EC | PSSCR_ESL)@h 2565 mtspr SPRN_PSSCR, r3 2566 /* Set LPCR_PECE_HVEE bit to enable wakeup by HV interrupts */ 2567 li r4, LPCR_PECE_HVEE@higher 2568 sldi r4, r4, 32 2569 or r5, r5, r4 2570END_FTR_SECTION_IFSET(CPU_FTR_ARCH_300) 2571 mtspr SPRN_LPCR,r5 2572 isync 2573 li r0, 0 2574 std r0, HSTATE_SCRATCH0(r13) 2575 ptesync 2576 ld r0, HSTATE_SCRATCH0(r13) 25771: cmpd r0, r0 2578 bne 1b 2579BEGIN_FTR_SECTION 2580 nap 2581FTR_SECTION_ELSE 2582 PPC_STOP 2583ALT_FTR_SECTION_END_IFCLR(CPU_FTR_ARCH_300) 2584 b . 2585 258633: mr r4, r3 2587 li r3, 0 2588 li r12, 0 2589 b 34f 2590 2591kvm_end_cede: 2592 /* get vcpu pointer */ 2593 ld r4, HSTATE_KVM_VCPU(r13) 2594 2595 /* Woken by external or decrementer interrupt */ 2596 ld r1, HSTATE_HOST_R1(r13) 2597 2598#ifdef CONFIG_KVM_BOOK3S_HV_EXIT_TIMING 2599 addi r3, r4, VCPU_TB_RMINTR 2600 bl kvmhv_accumulate_time 2601#endif 2602 2603#ifdef CONFIG_PPC_TRANSACTIONAL_MEM 2604BEGIN_FTR_SECTION 2605 /* 2606 * NOTE THAT THIS TRASHES ALL NON-VOLATILE REGISTERS INCLUDING CR 2607 */ 2608 bl kvmppc_restore_tm 2609END_FTR_SECTION_IFSET(CPU_FTR_TM) 2610#endif 2611 2612 /* load up FP state */ 2613 bl kvmppc_load_fp 2614 2615 /* Restore guest decrementer */ 2616 ld r3, VCPU_DEC_EXPIRES(r4) 2617 ld r5, HSTATE_KVM_VCORE(r13) 2618 ld r6, VCORE_TB_OFFSET(r5) 2619 add r3, r3, r6 /* convert host TB to guest TB value */ 2620 mftb r7 2621 subf r3, r7, r3 2622 mtspr SPRN_DEC, r3 2623 2624 /* Load NV GPRS */ 2625 ld r14, VCPU_GPR(R14)(r4) 2626 ld r15, VCPU_GPR(R15)(r4) 2627 ld r16, VCPU_GPR(R16)(r4) 2628 ld r17, VCPU_GPR(R17)(r4) 2629 ld r18, VCPU_GPR(R18)(r4) 2630 ld r19, VCPU_GPR(R19)(r4) 2631 ld r20, VCPU_GPR(R20)(r4) 2632 ld r21, VCPU_GPR(R21)(r4) 2633 ld r22, VCPU_GPR(R22)(r4) 2634 ld r23, VCPU_GPR(R23)(r4) 2635 ld r24, VCPU_GPR(R24)(r4) 2636 ld r25, VCPU_GPR(R25)(r4) 2637 ld r26, VCPU_GPR(R26)(r4) 2638 ld r27, VCPU_GPR(R27)(r4) 2639 ld r28, VCPU_GPR(R28)(r4) 2640 ld r29, VCPU_GPR(R29)(r4) 2641 ld r30, VCPU_GPR(R30)(r4) 2642 ld r31, VCPU_GPR(R31)(r4) 2643 2644 /* Check the wake reason in SRR1 to see why we got here */ 2645 bl kvmppc_check_wake_reason 2646 2647 /* 2648 * Restore volatile registers since we could have called a 2649 * C routine in kvmppc_check_wake_reason 2650 * r4 = VCPU 2651 * r3 tells us whether we need to return to host or not 2652 * WARNING: it gets checked further down: 2653 * should not modify r3 until this check is done. 2654 */ 2655 ld r4, HSTATE_KVM_VCPU(r13) 2656 2657 /* clear our bit in vcore->napping_threads */ 265834: ld r5,HSTATE_KVM_VCORE(r13) 2659 lbz r7,HSTATE_PTID(r13) 2660 li r0,1 2661 sld r0,r0,r7 2662 addi r6,r5,VCORE_NAPPING_THREADS 266332: lwarx r7,0,r6 2664 andc r7,r7,r0 2665 stwcx. r7,0,r6 2666 bne 32b 2667 li r0,0 2668 stb r0,HSTATE_NAPPING(r13) 2669 2670 /* See if the wake reason saved in r3 means we need to exit */ 2671 stw r12, VCPU_TRAP(r4) 2672 mr r9, r4 2673 cmpdi r3, 0 2674 bgt guest_exit_cont 2675 2676 /* see if any other thread is already exiting */ 2677 lwz r0,VCORE_ENTRY_EXIT(r5) 2678 cmpwi r0,0x100 2679 bge guest_exit_cont 2680 2681 b kvmppc_cede_reentry /* if not go back to guest */ 2682 2683 /* cede when already previously prodded case */ 2684kvm_cede_prodded: 2685 li r0,0 2686 stb r0,VCPU_PRODDED(r3) 2687 sync /* order testing prodded vs. clearing ceded */ 2688 stb r0,VCPU_CEDED(r3) 2689 li r3,H_SUCCESS 2690 blr 2691 2692 /* we've ceded but we want to give control to the host */ 2693kvm_cede_exit: 2694 ld r9, HSTATE_KVM_VCPU(r13) 2695 b guest_exit_cont 2696 2697 /* Try to handle a machine check in real mode */ 2698machine_check_realmode: 2699 mr r3, r9 /* get vcpu pointer */ 2700 bl kvmppc_realmode_machine_check 2701 nop 2702 ld r9, HSTATE_KVM_VCPU(r13) 2703 li r12, BOOK3S_INTERRUPT_MACHINE_CHECK 2704 /* 2705 * For the guest that is FWNMI capable, deliver all the MCE errors 2706 * (handled/unhandled) by exiting the guest with KVM_EXIT_NMI exit 2707 * reason. This new approach injects machine check errors in guest 2708 * address space to guest with additional information in the form 2709 * of RTAS event, thus enabling guest kernel to suitably handle 2710 * such errors. 2711 * 2712 * For the guest that is not FWNMI capable (old QEMU) fallback 2713 * to old behaviour for backward compatibility: 2714 * Deliver unhandled/fatal (e.g. UE) MCE errors to guest either 2715 * through machine check interrupt (set HSRR0 to 0x200). 2716 * For handled errors (no-fatal), just go back to guest execution 2717 * with current HSRR0. 2718 * if we receive machine check with MSR(RI=0) then deliver it to 2719 * guest as machine check causing guest to crash. 2720 */ 2721 ld r11, VCPU_MSR(r9) 2722 rldicl. r0, r11, 64-MSR_HV_LG, 63 /* check if it happened in HV mode */ 2723 bne mc_cont /* if so, exit to host */ 2724 /* Check if guest is capable of handling NMI exit */ 2725 ld r10, VCPU_KVM(r9) 2726 lbz r10, KVM_FWNMI(r10) 2727 cmpdi r10, 1 /* FWNMI capable? */ 2728 beq mc_cont /* if so, exit with KVM_EXIT_NMI. */ 2729 2730 /* if not, fall through for backward compatibility. */ 2731 andi. r10, r11, MSR_RI /* check for unrecoverable exception */ 2732 beq 1f /* Deliver a machine check to guest */ 2733 ld r10, VCPU_PC(r9) 2734 cmpdi r3, 0 /* Did we handle MCE ? */ 2735 bne 2f /* Continue guest execution. */ 2736 /* If not, deliver a machine check. SRR0/1 are already set */ 27371: li r10, BOOK3S_INTERRUPT_MACHINE_CHECK 2738 bl kvmppc_msr_interrupt 27392: b fast_interrupt_c_return 2740 2741/* 2742 * Check the reason we woke from nap, and take appropriate action. 2743 * Returns (in r3): 2744 * 0 if nothing needs to be done 2745 * 1 if something happened that needs to be handled by the host 2746 * -1 if there was a guest wakeup (IPI or msgsnd) 2747 * -2 if we handled a PCI passthrough interrupt (returned by 2748 * kvmppc_read_intr only) 2749 * 2750 * Also sets r12 to the interrupt vector for any interrupt that needs 2751 * to be handled now by the host (0x500 for external interrupt), or zero. 2752 * Modifies all volatile registers (since it may call a C function). 2753 * This routine calls kvmppc_read_intr, a C function, if an external 2754 * interrupt is pending. 2755 */ 2756kvmppc_check_wake_reason: 2757 mfspr r6, SPRN_SRR1 2758BEGIN_FTR_SECTION 2759 rlwinm r6, r6, 45-31, 0xf /* extract wake reason field (P8) */ 2760FTR_SECTION_ELSE 2761 rlwinm r6, r6, 45-31, 0xe /* P7 wake reason field is 3 bits */ 2762ALT_FTR_SECTION_END_IFSET(CPU_FTR_ARCH_207S) 2763 cmpwi r6, 8 /* was it an external interrupt? */ 2764 beq 7f /* if so, see what it was */ 2765 li r3, 0 2766 li r12, 0 2767 cmpwi r6, 6 /* was it the decrementer? */ 2768 beq 0f 2769BEGIN_FTR_SECTION 2770 cmpwi r6, 5 /* privileged doorbell? */ 2771 beq 0f 2772 cmpwi r6, 3 /* hypervisor doorbell? */ 2773 beq 3f 2774END_FTR_SECTION_IFSET(CPU_FTR_ARCH_207S) 2775 cmpwi r6, 0xa /* Hypervisor maintenance ? */ 2776 beq 4f 2777 li r3, 1 /* anything else, return 1 */ 27780: blr 2779 2780 /* hypervisor doorbell */ 27813: li r12, BOOK3S_INTERRUPT_H_DOORBELL 2782 2783 /* 2784 * Clear the doorbell as we will invoke the handler 2785 * explicitly in the guest exit path. 2786 */ 2787 lis r6, (PPC_DBELL_SERVER << (63-36))@h 2788 PPC_MSGCLR(6) 2789 /* see if it's a host IPI */ 2790 li r3, 1 2791 lbz r0, HSTATE_HOST_IPI(r13) 2792 cmpwi r0, 0 2793 bnelr 2794 /* if not, return -1 */ 2795 li r3, -1 2796 blr 2797 2798 /* Woken up due to Hypervisor maintenance interrupt */ 27994: li r12, BOOK3S_INTERRUPT_HMI 2800 li r3, 1 2801 blr 2802 2803 /* external interrupt - create a stack frame so we can call C */ 28047: mflr r0 2805 std r0, PPC_LR_STKOFF(r1) 2806 stdu r1, -PPC_MIN_STKFRM(r1) 2807 bl kvmppc_read_intr 2808 nop 2809 li r12, BOOK3S_INTERRUPT_EXTERNAL 2810 cmpdi r3, 1 2811 ble 1f 2812 2813 /* 2814 * Return code of 2 means PCI passthrough interrupt, but 2815 * we need to return back to host to complete handling the 2816 * interrupt. Trap reason is expected in r12 by guest 2817 * exit code. 2818 */ 2819 li r12, BOOK3S_INTERRUPT_HV_RM_HARD 28201: 2821 ld r0, PPC_MIN_STKFRM+PPC_LR_STKOFF(r1) 2822 addi r1, r1, PPC_MIN_STKFRM 2823 mtlr r0 2824 blr 2825 2826/* 2827 * Save away FP, VMX and VSX registers. 2828 * r3 = vcpu pointer 2829 * N.B. r30 and r31 are volatile across this function, 2830 * thus it is not callable from C. 2831 */ 2832kvmppc_save_fp: 2833 mflr r30 2834 mr r31,r3 2835 mfmsr r5 2836 ori r8,r5,MSR_FP 2837#ifdef CONFIG_ALTIVEC 2838BEGIN_FTR_SECTION 2839 oris r8,r8,MSR_VEC@h 2840END_FTR_SECTION_IFSET(CPU_FTR_ALTIVEC) 2841#endif 2842#ifdef CONFIG_VSX 2843BEGIN_FTR_SECTION 2844 oris r8,r8,MSR_VSX@h 2845END_FTR_SECTION_IFSET(CPU_FTR_VSX) 2846#endif 2847 mtmsrd r8 2848 addi r3,r3,VCPU_FPRS 2849 bl store_fp_state 2850#ifdef CONFIG_ALTIVEC 2851BEGIN_FTR_SECTION 2852 addi r3,r31,VCPU_VRS 2853 bl store_vr_state 2854END_FTR_SECTION_IFSET(CPU_FTR_ALTIVEC) 2855#endif 2856 mfspr r6,SPRN_VRSAVE 2857 stw r6,VCPU_VRSAVE(r31) 2858 mtlr r30 2859 blr 2860 2861/* 2862 * Load up FP, VMX and VSX registers 2863 * r4 = vcpu pointer 2864 * N.B. r30 and r31 are volatile across this function, 2865 * thus it is not callable from C. 2866 */ 2867kvmppc_load_fp: 2868 mflr r30 2869 mr r31,r4 2870 mfmsr r9 2871 ori r8,r9,MSR_FP 2872#ifdef CONFIG_ALTIVEC 2873BEGIN_FTR_SECTION 2874 oris r8,r8,MSR_VEC@h 2875END_FTR_SECTION_IFSET(CPU_FTR_ALTIVEC) 2876#endif 2877#ifdef CONFIG_VSX 2878BEGIN_FTR_SECTION 2879 oris r8,r8,MSR_VSX@h 2880END_FTR_SECTION_IFSET(CPU_FTR_VSX) 2881#endif 2882 mtmsrd r8 2883 addi r3,r4,VCPU_FPRS 2884 bl load_fp_state 2885#ifdef CONFIG_ALTIVEC 2886BEGIN_FTR_SECTION 2887 addi r3,r31,VCPU_VRS 2888 bl load_vr_state 2889END_FTR_SECTION_IFSET(CPU_FTR_ALTIVEC) 2890#endif 2891 lwz r7,VCPU_VRSAVE(r31) 2892 mtspr SPRN_VRSAVE,r7 2893 mtlr r30 2894 mr r4,r31 2895 blr 2896 2897#ifdef CONFIG_PPC_TRANSACTIONAL_MEM 2898/* 2899 * Save transactional state and TM-related registers. 2900 * Called with r9 pointing to the vcpu struct. 2901 * This can modify all checkpointed registers, but 2902 * restores r1, r2 and r9 (vcpu pointer) before exit. 2903 */ 2904kvmppc_save_tm: 2905 mflr r0 2906 std r0, PPC_LR_STKOFF(r1) 2907 2908 /* Turn on TM. */ 2909 mfmsr r8 2910 li r0, 1 2911 rldimi r8, r0, MSR_TM_LG, 63-MSR_TM_LG 2912 mtmsrd r8 2913 2914 ld r5, VCPU_MSR(r9) 2915 rldicl. r5, r5, 64 - MSR_TS_S_LG, 62 2916 beq 1f /* TM not active in guest. */ 2917 2918 std r1, HSTATE_HOST_R1(r13) 2919 li r3, TM_CAUSE_KVM_RESCHED 2920 2921 /* Clear the MSR RI since r1, r13 are all going to be foobar. */ 2922 li r5, 0 2923 mtmsrd r5, 1 2924 2925 /* All GPRs are volatile at this point. */ 2926 TRECLAIM(R3) 2927 2928 /* Temporarily store r13 and r9 so we have some regs to play with */ 2929 SET_SCRATCH0(r13) 2930 GET_PACA(r13) 2931 std r9, PACATMSCRATCH(r13) 2932 ld r9, HSTATE_KVM_VCPU(r13) 2933 2934 /* Get a few more GPRs free. */ 2935 std r29, VCPU_GPRS_TM(29)(r9) 2936 std r30, VCPU_GPRS_TM(30)(r9) 2937 std r31, VCPU_GPRS_TM(31)(r9) 2938 2939 /* Save away PPR and DSCR soon so don't run with user values. */ 2940 mfspr r31, SPRN_PPR 2941 HMT_MEDIUM 2942 mfspr r30, SPRN_DSCR 2943 ld r29, HSTATE_DSCR(r13) 2944 mtspr SPRN_DSCR, r29 2945 2946 /* Save all but r9, r13 & r29-r31 */ 2947 reg = 0 2948 .rept 29 2949 .if (reg != 9) && (reg != 13) 2950 std reg, VCPU_GPRS_TM(reg)(r9) 2951 .endif 2952 reg = reg + 1 2953 .endr 2954 /* ... now save r13 */ 2955 GET_SCRATCH0(r4) 2956 std r4, VCPU_GPRS_TM(13)(r9) 2957 /* ... and save r9 */ 2958 ld r4, PACATMSCRATCH(r13) 2959 std r4, VCPU_GPRS_TM(9)(r9) 2960 2961 /* Reload stack pointer and TOC. */ 2962 ld r1, HSTATE_HOST_R1(r13) 2963 ld r2, PACATOC(r13) 2964 2965 /* Set MSR RI now we have r1 and r13 back. */ 2966 li r5, MSR_RI 2967 mtmsrd r5, 1 2968 2969 /* Save away checkpinted SPRs. */ 2970 std r31, VCPU_PPR_TM(r9) 2971 std r30, VCPU_DSCR_TM(r9) 2972 mflr r5 2973 mfcr r6 2974 mfctr r7 2975 mfspr r8, SPRN_AMR 2976 mfspr r10, SPRN_TAR 2977 mfxer r11 2978 std r5, VCPU_LR_TM(r9) 2979 stw r6, VCPU_CR_TM(r9) 2980 std r7, VCPU_CTR_TM(r9) 2981 std r8, VCPU_AMR_TM(r9) 2982 std r10, VCPU_TAR_TM(r9) 2983 std r11, VCPU_XER_TM(r9) 2984 2985 /* Restore r12 as trap number. */ 2986 lwz r12, VCPU_TRAP(r9) 2987 2988 /* Save FP/VSX. */ 2989 addi r3, r9, VCPU_FPRS_TM 2990 bl store_fp_state 2991 addi r3, r9, VCPU_VRS_TM 2992 bl store_vr_state 2993 mfspr r6, SPRN_VRSAVE 2994 stw r6, VCPU_VRSAVE_TM(r9) 29951: 2996 /* 2997 * We need to save these SPRs after the treclaim so that the software 2998 * error code is recorded correctly in the TEXASR. Also the user may 2999 * change these outside of a transaction, so they must always be 3000 * context switched. 3001 */ 3002 mfspr r5, SPRN_TFHAR 3003 mfspr r6, SPRN_TFIAR 3004 mfspr r7, SPRN_TEXASR 3005 std r5, VCPU_TFHAR(r9) 3006 std r6, VCPU_TFIAR(r9) 3007 std r7, VCPU_TEXASR(r9) 3008 3009 ld r0, PPC_LR_STKOFF(r1) 3010 mtlr r0 3011 blr 3012 3013/* 3014 * Restore transactional state and TM-related registers. 3015 * Called with r4 pointing to the vcpu struct. 3016 * This potentially modifies all checkpointed registers. 3017 * It restores r1, r2, r4 from the PACA. 3018 */ 3019kvmppc_restore_tm: 3020 mflr r0 3021 std r0, PPC_LR_STKOFF(r1) 3022 3023 /* Turn on TM/FP/VSX/VMX so we can restore them. */ 3024 mfmsr r5 3025 li r6, MSR_TM >> 32 3026 sldi r6, r6, 32 3027 or r5, r5, r6 3028 ori r5, r5, MSR_FP 3029 oris r5, r5, (MSR_VEC | MSR_VSX)@h 3030 mtmsrd r5 3031 3032 /* 3033 * The user may change these outside of a transaction, so they must 3034 * always be context switched. 3035 */ 3036 ld r5, VCPU_TFHAR(r4) 3037 ld r6, VCPU_TFIAR(r4) 3038 ld r7, VCPU_TEXASR(r4) 3039 mtspr SPRN_TFHAR, r5 3040 mtspr SPRN_TFIAR, r6 3041 mtspr SPRN_TEXASR, r7 3042 3043 ld r5, VCPU_MSR(r4) 3044 rldicl. r5, r5, 64 - MSR_TS_S_LG, 62 3045 beqlr /* TM not active in guest */ 3046 std r1, HSTATE_HOST_R1(r13) 3047 3048 /* Make sure the failure summary is set, otherwise we'll program check 3049 * when we trechkpt. It's possible that this might have been not set 3050 * on a kvmppc_set_one_reg() call but we shouldn't let this crash the 3051 * host. 3052 */ 3053 oris r7, r7, (TEXASR_FS)@h 3054 mtspr SPRN_TEXASR, r7 3055 3056 /* 3057 * We need to load up the checkpointed state for the guest. 3058 * We need to do this early as it will blow away any GPRs, VSRs and 3059 * some SPRs. 3060 */ 3061 3062 mr r31, r4 3063 addi r3, r31, VCPU_FPRS_TM 3064 bl load_fp_state 3065 addi r3, r31, VCPU_VRS_TM 3066 bl load_vr_state 3067 mr r4, r31 3068 lwz r7, VCPU_VRSAVE_TM(r4) 3069 mtspr SPRN_VRSAVE, r7 3070 3071 ld r5, VCPU_LR_TM(r4) 3072 lwz r6, VCPU_CR_TM(r4) 3073 ld r7, VCPU_CTR_TM(r4) 3074 ld r8, VCPU_AMR_TM(r4) 3075 ld r9, VCPU_TAR_TM(r4) 3076 ld r10, VCPU_XER_TM(r4) 3077 mtlr r5 3078 mtcr r6 3079 mtctr r7 3080 mtspr SPRN_AMR, r8 3081 mtspr SPRN_TAR, r9 3082 mtxer r10 3083 3084 /* 3085 * Load up PPR and DSCR values but don't put them in the actual SPRs 3086 * till the last moment to avoid running with userspace PPR and DSCR for 3087 * too long. 3088 */ 3089 ld r29, VCPU_DSCR_TM(r4) 3090 ld r30, VCPU_PPR_TM(r4) 3091 3092 std r2, PACATMSCRATCH(r13) /* Save TOC */ 3093 3094 /* Clear the MSR RI since r1, r13 are all going to be foobar. */ 3095 li r5, 0 3096 mtmsrd r5, 1 3097 3098 /* Load GPRs r0-r28 */ 3099 reg = 0 3100 .rept 29 3101 ld reg, VCPU_GPRS_TM(reg)(r31) 3102 reg = reg + 1 3103 .endr 3104 3105 mtspr SPRN_DSCR, r29 3106 mtspr SPRN_PPR, r30 3107 3108 /* Load final GPRs */ 3109 ld 29, VCPU_GPRS_TM(29)(r31) 3110 ld 30, VCPU_GPRS_TM(30)(r31) 3111 ld 31, VCPU_GPRS_TM(31)(r31) 3112 3113 /* TM checkpointed state is now setup. All GPRs are now volatile. */ 3114 TRECHKPT 3115 3116 /* Now let's get back the state we need. */ 3117 HMT_MEDIUM 3118 GET_PACA(r13) 3119 ld r29, HSTATE_DSCR(r13) 3120 mtspr SPRN_DSCR, r29 3121 ld r4, HSTATE_KVM_VCPU(r13) 3122 ld r1, HSTATE_HOST_R1(r13) 3123 ld r2, PACATMSCRATCH(r13) 3124 3125 /* Set the MSR RI since we have our registers back. */ 3126 li r5, MSR_RI 3127 mtmsrd r5, 1 3128 3129 ld r0, PPC_LR_STKOFF(r1) 3130 mtlr r0 3131 blr 3132#endif 3133 3134/* 3135 * We come here if we get any exception or interrupt while we are 3136 * executing host real mode code while in guest MMU context. 3137 * For now just spin, but we should do something better. 3138 */ 3139kvmppc_bad_host_intr: 3140 b . 3141 3142/* 3143 * This mimics the MSR transition on IRQ delivery. The new guest MSR is taken 3144 * from VCPU_INTR_MSR and is modified based on the required TM state changes. 3145 * r11 has the guest MSR value (in/out) 3146 * r9 has a vcpu pointer (in) 3147 * r0 is used as a scratch register 3148 */ 3149kvmppc_msr_interrupt: 3150 rldicl r0, r11, 64 - MSR_TS_S_LG, 62 3151 cmpwi r0, 2 /* Check if we are in transactional state.. */ 3152 ld r11, VCPU_INTR_MSR(r9) 3153 bne 1f 3154 /* ... if transactional, change to suspended */ 3155 li r0, 1 31561: rldimi r11, r0, MSR_TS_S_LG, 63 - MSR_TS_T_LG 3157 blr 3158 3159/* 3160 * This works around a hardware bug on POWER8E processors, where 3161 * writing a 1 to the MMCR0[PMAO] bit doesn't generate a 3162 * performance monitor interrupt. Instead, when we need to have 3163 * an interrupt pending, we have to arrange for a counter to overflow. 3164 */ 3165kvmppc_fix_pmao: 3166 li r3, 0 3167 mtspr SPRN_MMCR2, r3 3168 lis r3, (MMCR0_PMXE | MMCR0_FCECE)@h 3169 ori r3, r3, MMCR0_PMCjCE | MMCR0_C56RUN 3170 mtspr SPRN_MMCR0, r3 3171 lis r3, 0x7fff 3172 ori r3, r3, 0xffff 3173 mtspr SPRN_PMC6, r3 3174 isync 3175 blr 3176 3177#ifdef CONFIG_KVM_BOOK3S_HV_EXIT_TIMING 3178/* 3179 * Start timing an activity 3180 * r3 = pointer to time accumulation struct, r4 = vcpu 3181 */ 3182kvmhv_start_timing: 3183 ld r5, HSTATE_KVM_VCORE(r13) 3184 lbz r6, VCORE_IN_GUEST(r5) 3185 cmpwi r6, 0 3186 beq 5f /* if in guest, need to */ 3187 ld r6, VCORE_TB_OFFSET(r5) /* subtract timebase offset */ 31885: mftb r5 3189 subf r5, r6, r5 3190 std r3, VCPU_CUR_ACTIVITY(r4) 3191 std r5, VCPU_ACTIVITY_START(r4) 3192 blr 3193 3194/* 3195 * Accumulate time to one activity and start another. 3196 * r3 = pointer to new time accumulation struct, r4 = vcpu 3197 */ 3198kvmhv_accumulate_time: 3199 ld r5, HSTATE_KVM_VCORE(r13) 3200 lbz r8, VCORE_IN_GUEST(r5) 3201 cmpwi r8, 0 3202 beq 4f /* if in guest, need to */ 3203 ld r8, VCORE_TB_OFFSET(r5) /* subtract timebase offset */ 32044: ld r5, VCPU_CUR_ACTIVITY(r4) 3205 ld r6, VCPU_ACTIVITY_START(r4) 3206 std r3, VCPU_CUR_ACTIVITY(r4) 3207 mftb r7 3208 subf r7, r8, r7 3209 std r7, VCPU_ACTIVITY_START(r4) 3210 cmpdi r5, 0 3211 beqlr 3212 subf r3, r6, r7 3213 ld r8, TAS_SEQCOUNT(r5) 3214 cmpdi r8, 0 3215 addi r8, r8, 1 3216 std r8, TAS_SEQCOUNT(r5) 3217 lwsync 3218 ld r7, TAS_TOTAL(r5) 3219 add r7, r7, r3 3220 std r7, TAS_TOTAL(r5) 3221 ld r6, TAS_MIN(r5) 3222 ld r7, TAS_MAX(r5) 3223 beq 3f 3224 cmpd r3, r6 3225 bge 1f 32263: std r3, TAS_MIN(r5) 32271: cmpd r3, r7 3228 ble 2f 3229 std r3, TAS_MAX(r5) 32302: lwsync 3231 addi r8, r8, 1 3232 std r8, TAS_SEQCOUNT(r5) 3233 blr 3234#endif 3235