1/* 2 * This program is free software; you can redistribute it and/or modify 3 * it under the terms of the GNU General Public License, version 2, as 4 * published by the Free Software Foundation. 5 * 6 * This program is distributed in the hope that it will be useful, 7 * but WITHOUT ANY WARRANTY; without even the implied warranty of 8 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 9 * GNU General Public License for more details. 10 * 11 * Copyright 2011 Paul Mackerras, IBM Corp. <paulus@au1.ibm.com> 12 * 13 * Derived from book3s_rmhandlers.S and other files, which are: 14 * 15 * Copyright SUSE Linux Products GmbH 2009 16 * 17 * Authors: Alexander Graf <agraf@suse.de> 18 */ 19 20#include <asm/ppc_asm.h> 21#include <asm/kvm_asm.h> 22#include <asm/reg.h> 23#include <asm/mmu.h> 24#include <asm/page.h> 25#include <asm/ptrace.h> 26#include <asm/hvcall.h> 27#include <asm/asm-offsets.h> 28#include <asm/exception-64s.h> 29#include <asm/kvm_book3s_asm.h> 30#include <asm/book3s/64/mmu-hash.h> 31#include <asm/tm.h> 32#include <asm/opal.h> 33#include <asm/xive-regs.h> 34 35/* Sign-extend HDEC if not on POWER9 */ 36#define EXTEND_HDEC(reg) \ 37BEGIN_FTR_SECTION; \ 38 extsw reg, reg; \ 39END_FTR_SECTION_IFCLR(CPU_FTR_ARCH_300) 40 41#define VCPU_GPRS_TM(reg) (((reg) * ULONG_SIZE) + VCPU_GPR_TM) 42 43/* Values in HSTATE_NAPPING(r13) */ 44#define NAPPING_CEDE 1 45#define NAPPING_NOVCPU 2 46 47/* Stack frame offsets for kvmppc_hv_entry */ 48#define SFS 160 49#define STACK_SLOT_TRAP (SFS-4) 50#define STACK_SLOT_TID (SFS-16) 51#define STACK_SLOT_PSSCR (SFS-24) 52#define STACK_SLOT_PID (SFS-32) 53#define STACK_SLOT_IAMR (SFS-40) 54#define STACK_SLOT_CIABR (SFS-48) 55#define STACK_SLOT_DAWR (SFS-56) 56#define STACK_SLOT_DAWRX (SFS-64) 57#define STACK_SLOT_HFSCR (SFS-72) 58 59/* 60 * Call kvmppc_hv_entry in real mode. 61 * Must be called with interrupts hard-disabled. 62 * 63 * Input Registers: 64 * 65 * LR = return address to continue at after eventually re-enabling MMU 66 */ 67_GLOBAL_TOC(kvmppc_hv_entry_trampoline) 68 mflr r0 69 std r0, PPC_LR_STKOFF(r1) 70 stdu r1, -112(r1) 71 mfmsr r10 72 std r10, HSTATE_HOST_MSR(r13) 73 LOAD_REG_ADDR(r5, kvmppc_call_hv_entry) 74 li r0,MSR_RI 75 andc r0,r10,r0 76 li r6,MSR_IR | MSR_DR 77 andc r6,r10,r6 78 mtmsrd r0,1 /* clear RI in MSR */ 79 mtsrr0 r5 80 mtsrr1 r6 81 RFI 82 83kvmppc_call_hv_entry: 84 ld r4, HSTATE_KVM_VCPU(r13) 85 bl kvmppc_hv_entry 86 87 /* Back from guest - restore host state and return to caller */ 88 89BEGIN_FTR_SECTION 90 /* Restore host DABR and DABRX */ 91 ld r5,HSTATE_DABR(r13) 92 li r6,7 93 mtspr SPRN_DABR,r5 94 mtspr SPRN_DABRX,r6 95END_FTR_SECTION_IFCLR(CPU_FTR_ARCH_207S) 96 97 /* Restore SPRG3 */ 98 ld r3,PACA_SPRG_VDSO(r13) 99 mtspr SPRN_SPRG_VDSO_WRITE,r3 100 101 /* Reload the host's PMU registers */ 102 ld r3, PACALPPACAPTR(r13) /* is the host using the PMU? */ 103 lbz r4, LPPACA_PMCINUSE(r3) 104 cmpwi r4, 0 105 beq 23f /* skip if not */ 106BEGIN_FTR_SECTION 107 ld r3, HSTATE_MMCR0(r13) 108 andi. r4, r3, MMCR0_PMAO_SYNC | MMCR0_PMAO 109 cmpwi r4, MMCR0_PMAO 110 beql kvmppc_fix_pmao 111END_FTR_SECTION_IFSET(CPU_FTR_PMAO_BUG) 112 lwz r3, HSTATE_PMC1(r13) 113 lwz r4, HSTATE_PMC2(r13) 114 lwz r5, HSTATE_PMC3(r13) 115 lwz r6, HSTATE_PMC4(r13) 116 lwz r8, HSTATE_PMC5(r13) 117 lwz r9, HSTATE_PMC6(r13) 118 mtspr SPRN_PMC1, r3 119 mtspr SPRN_PMC2, r4 120 mtspr SPRN_PMC3, r5 121 mtspr SPRN_PMC4, r6 122 mtspr SPRN_PMC5, r8 123 mtspr SPRN_PMC6, r9 124 ld r3, HSTATE_MMCR0(r13) 125 ld r4, HSTATE_MMCR1(r13) 126 ld r5, HSTATE_MMCRA(r13) 127 ld r6, HSTATE_SIAR(r13) 128 ld r7, HSTATE_SDAR(r13) 129 mtspr SPRN_MMCR1, r4 130 mtspr SPRN_MMCRA, r5 131 mtspr SPRN_SIAR, r6 132 mtspr SPRN_SDAR, r7 133BEGIN_FTR_SECTION 134 ld r8, HSTATE_MMCR2(r13) 135 ld r9, HSTATE_SIER(r13) 136 mtspr SPRN_MMCR2, r8 137 mtspr SPRN_SIER, r9 138END_FTR_SECTION_IFSET(CPU_FTR_ARCH_207S) 139 mtspr SPRN_MMCR0, r3 140 isync 14123: 142 143 /* 144 * Reload DEC. HDEC interrupts were disabled when 145 * we reloaded the host's LPCR value. 146 */ 147 ld r3, HSTATE_DECEXP(r13) 148 mftb r4 149 subf r4, r4, r3 150 mtspr SPRN_DEC, r4 151 152BEGIN_FTR_SECTION 153 /* hwthread_req may have got set by cede or no vcpu, so clear it */ 154 li r0, 0 155 stb r0, HSTATE_HWTHREAD_REQ(r13) 156END_FTR_SECTION_IFCLR(CPU_FTR_ARCH_300) 157 158 /* 159 * For external interrupts we need to call the Linux 160 * handler to process the interrupt. We do that by jumping 161 * to absolute address 0x500 for external interrupts. 162 * The [h]rfid at the end of the handler will return to 163 * the book3s_hv_interrupts.S code. For other interrupts 164 * we do the rfid to get back to the book3s_hv_interrupts.S 165 * code here. 166 */ 167 ld r8, 112+PPC_LR_STKOFF(r1) 168 addi r1, r1, 112 169 ld r7, HSTATE_HOST_MSR(r13) 170 171 /* Return the trap number on this thread as the return value */ 172 mr r3, r12 173 174 /* 175 * If we came back from the guest via a relocation-on interrupt, 176 * we will be in virtual mode at this point, which makes it a 177 * little easier to get back to the caller. 178 */ 179 mfmsr r0 180 andi. r0, r0, MSR_IR /* in real mode? */ 181 bne .Lvirt_return 182 183 /* RFI into the highmem handler */ 184 mfmsr r6 185 li r0, MSR_RI 186 andc r6, r6, r0 187 mtmsrd r6, 1 /* Clear RI in MSR */ 188 mtsrr0 r8 189 mtsrr1 r7 190 RFI 191 192 /* Virtual-mode return */ 193.Lvirt_return: 194 mtlr r8 195 blr 196 197kvmppc_primary_no_guest: 198 /* We handle this much like a ceded vcpu */ 199 /* put the HDEC into the DEC, since HDEC interrupts don't wake us */ 200 /* HDEC may be larger than DEC for arch >= v3.00, but since the */ 201 /* HDEC value came from DEC in the first place, it will fit */ 202 mfspr r3, SPRN_HDEC 203 mtspr SPRN_DEC, r3 204 /* 205 * Make sure the primary has finished the MMU switch. 206 * We should never get here on a secondary thread, but 207 * check it for robustness' sake. 208 */ 209 ld r5, HSTATE_KVM_VCORE(r13) 21065: lbz r0, VCORE_IN_GUEST(r5) 211 cmpwi r0, 0 212 beq 65b 213 /* Set LPCR. */ 214 ld r8,VCORE_LPCR(r5) 215 mtspr SPRN_LPCR,r8 216 isync 217 /* set our bit in napping_threads */ 218 ld r5, HSTATE_KVM_VCORE(r13) 219 lbz r7, HSTATE_PTID(r13) 220 li r0, 1 221 sld r0, r0, r7 222 addi r6, r5, VCORE_NAPPING_THREADS 2231: lwarx r3, 0, r6 224 or r3, r3, r0 225 stwcx. r3, 0, r6 226 bne 1b 227 /* order napping_threads update vs testing entry_exit_map */ 228 isync 229 li r12, 0 230 lwz r7, VCORE_ENTRY_EXIT(r5) 231 cmpwi r7, 0x100 232 bge kvm_novcpu_exit /* another thread already exiting */ 233 li r3, NAPPING_NOVCPU 234 stb r3, HSTATE_NAPPING(r13) 235 236 li r3, 0 /* Don't wake on privileged (OS) doorbell */ 237 b kvm_do_nap 238 239/* 240 * kvm_novcpu_wakeup 241 * Entered from kvm_start_guest if kvm_hstate.napping is set 242 * to NAPPING_NOVCPU 243 * r2 = kernel TOC 244 * r13 = paca 245 */ 246kvm_novcpu_wakeup: 247 ld r1, HSTATE_HOST_R1(r13) 248 ld r5, HSTATE_KVM_VCORE(r13) 249 li r0, 0 250 stb r0, HSTATE_NAPPING(r13) 251 252 /* check the wake reason */ 253 bl kvmppc_check_wake_reason 254 255 /* 256 * Restore volatile registers since we could have called 257 * a C routine in kvmppc_check_wake_reason. 258 * r5 = VCORE 259 */ 260 ld r5, HSTATE_KVM_VCORE(r13) 261 262 /* see if any other thread is already exiting */ 263 lwz r0, VCORE_ENTRY_EXIT(r5) 264 cmpwi r0, 0x100 265 bge kvm_novcpu_exit 266 267 /* clear our bit in napping_threads */ 268 lbz r7, HSTATE_PTID(r13) 269 li r0, 1 270 sld r0, r0, r7 271 addi r6, r5, VCORE_NAPPING_THREADS 2724: lwarx r7, 0, r6 273 andc r7, r7, r0 274 stwcx. r7, 0, r6 275 bne 4b 276 277 /* See if the wake reason means we need to exit */ 278 cmpdi r3, 0 279 bge kvm_novcpu_exit 280 281 /* See if our timeslice has expired (HDEC is negative) */ 282 mfspr r0, SPRN_HDEC 283 EXTEND_HDEC(r0) 284 li r12, BOOK3S_INTERRUPT_HV_DECREMENTER 285 cmpdi r0, 0 286 blt kvm_novcpu_exit 287 288 /* Got an IPI but other vcpus aren't yet exiting, must be a latecomer */ 289 ld r4, HSTATE_KVM_VCPU(r13) 290 cmpdi r4, 0 291 beq kvmppc_primary_no_guest 292 293#ifdef CONFIG_KVM_BOOK3S_HV_EXIT_TIMING 294 addi r3, r4, VCPU_TB_RMENTRY 295 bl kvmhv_start_timing 296#endif 297 b kvmppc_got_guest 298 299kvm_novcpu_exit: 300#ifdef CONFIG_KVM_BOOK3S_HV_EXIT_TIMING 301 ld r4, HSTATE_KVM_VCPU(r13) 302 cmpdi r4, 0 303 beq 13f 304 addi r3, r4, VCPU_TB_RMEXIT 305 bl kvmhv_accumulate_time 306#endif 30713: mr r3, r12 308 stw r12, STACK_SLOT_TRAP(r1) 309 bl kvmhv_commence_exit 310 nop 311 lwz r12, STACK_SLOT_TRAP(r1) 312 b kvmhv_switch_to_host 313 314/* 315 * We come in here when wakened from nap mode. 316 * Relocation is off and most register values are lost. 317 * r13 points to the PACA. 318 * r3 contains the SRR1 wakeup value, SRR1 is trashed. 319 * This is not used by ISAv3.0B processors. 320 */ 321 .globl kvm_start_guest 322kvm_start_guest: 323 /* Set runlatch bit the minute you wake up from nap */ 324 mfspr r0, SPRN_CTRLF 325 ori r0, r0, 1 326 mtspr SPRN_CTRLT, r0 327 328 /* 329 * Could avoid this and pass it through in r3. For now, 330 * code expects it to be in SRR1. 331 */ 332 mtspr SPRN_SRR1,r3 333 334 ld r2,PACATOC(r13) 335 336 li r0,KVM_HWTHREAD_IN_KVM 337 stb r0,HSTATE_HWTHREAD_STATE(r13) 338 339 /* NV GPR values from power7_idle() will no longer be valid */ 340 li r0,1 341 stb r0,PACA_NAPSTATELOST(r13) 342 343 /* were we napping due to cede? */ 344 lbz r0,HSTATE_NAPPING(r13) 345 cmpwi r0,NAPPING_CEDE 346 beq kvm_end_cede 347 cmpwi r0,NAPPING_NOVCPU 348 beq kvm_novcpu_wakeup 349 350 ld r1,PACAEMERGSP(r13) 351 subi r1,r1,STACK_FRAME_OVERHEAD 352 353 /* 354 * We weren't napping due to cede, so this must be a secondary 355 * thread being woken up to run a guest, or being woken up due 356 * to a stray IPI. (Or due to some machine check or hypervisor 357 * maintenance interrupt while the core is in KVM.) 358 */ 359 360 /* Check the wake reason in SRR1 to see why we got here */ 361 bl kvmppc_check_wake_reason 362 /* 363 * kvmppc_check_wake_reason could invoke a C routine, but we 364 * have no volatile registers to restore when we return. 365 */ 366 367 cmpdi r3, 0 368 bge kvm_no_guest 369 370 /* get vcore pointer, NULL if we have nothing to run */ 371 ld r5,HSTATE_KVM_VCORE(r13) 372 cmpdi r5,0 373 /* if we have no vcore to run, go back to sleep */ 374 beq kvm_no_guest 375 376kvm_secondary_got_guest: 377 378 /* Set HSTATE_DSCR(r13) to something sensible */ 379 ld r6, PACA_DSCR_DEFAULT(r13) 380 std r6, HSTATE_DSCR(r13) 381 382 /* On thread 0 of a subcore, set HDEC to max */ 383 lbz r4, HSTATE_PTID(r13) 384 cmpwi r4, 0 385 bne 63f 386 LOAD_REG_ADDR(r6, decrementer_max) 387 ld r6, 0(r6) 388 mtspr SPRN_HDEC, r6 389 /* and set per-LPAR registers, if doing dynamic micro-threading */ 390 ld r6, HSTATE_SPLIT_MODE(r13) 391 cmpdi r6, 0 392 beq 63f 393 ld r0, KVM_SPLIT_RPR(r6) 394 mtspr SPRN_RPR, r0 395 ld r0, KVM_SPLIT_PMMAR(r6) 396 mtspr SPRN_PMMAR, r0 397 ld r0, KVM_SPLIT_LDBAR(r6) 398 mtspr SPRN_LDBAR, r0 399 isync 40063: 401 /* Order load of vcpu after load of vcore */ 402 lwsync 403 ld r4, HSTATE_KVM_VCPU(r13) 404 bl kvmppc_hv_entry 405 406 /* Back from the guest, go back to nap */ 407 /* Clear our vcpu and vcore pointers so we don't come back in early */ 408 li r0, 0 409 std r0, HSTATE_KVM_VCPU(r13) 410 /* 411 * Once we clear HSTATE_KVM_VCORE(r13), the code in 412 * kvmppc_run_core() is going to assume that all our vcpu 413 * state is visible in memory. This lwsync makes sure 414 * that that is true. 415 */ 416 lwsync 417 std r0, HSTATE_KVM_VCORE(r13) 418 419 /* 420 * All secondaries exiting guest will fall through this path. 421 * Before proceeding, just check for HMI interrupt and 422 * invoke opal hmi handler. By now we are sure that the 423 * primary thread on this core/subcore has already made partition 424 * switch/TB resync and we are good to call opal hmi handler. 425 */ 426 cmpwi r12, BOOK3S_INTERRUPT_HMI 427 bne kvm_no_guest 428 429 li r3,0 /* NULL argument */ 430 bl hmi_exception_realmode 431/* 432 * At this point we have finished executing in the guest. 433 * We need to wait for hwthread_req to become zero, since 434 * we may not turn on the MMU while hwthread_req is non-zero. 435 * While waiting we also need to check if we get given a vcpu to run. 436 */ 437kvm_no_guest: 438BEGIN_FTR_SECTION 439 twi 31,0,0 440END_FTR_SECTION_IFSET(CPU_FTR_ARCH_300) 441 lbz r3, HSTATE_HWTHREAD_REQ(r13) 442 cmpwi r3, 0 443 bne 53f 444 HMT_MEDIUM 445 li r0, KVM_HWTHREAD_IN_KERNEL 446 stb r0, HSTATE_HWTHREAD_STATE(r13) 447 /* need to recheck hwthread_req after a barrier, to avoid race */ 448 sync 449 lbz r3, HSTATE_HWTHREAD_REQ(r13) 450 cmpwi r3, 0 451 bne 54f 452/* 453 * We jump to pnv_wakeup_loss, which will return to the caller 454 * of power7_nap in the powernv cpu offline loop. The value we 455 * put in r3 becomes the return value for power7_nap. pnv_wakeup_loss 456 * requires SRR1 in r12. 457 */ 458 li r3, LPCR_PECE0 459 mfspr r4, SPRN_LPCR 460 rlwimi r4, r3, 0, LPCR_PECE0 | LPCR_PECE1 461 mtspr SPRN_LPCR, r4 462 li r3, 0 463 mfspr r12,SPRN_SRR1 464 b pnv_wakeup_loss 465 46653: HMT_LOW 467 ld r5, HSTATE_KVM_VCORE(r13) 468 cmpdi r5, 0 469 bne 60f 470 ld r3, HSTATE_SPLIT_MODE(r13) 471 cmpdi r3, 0 472 beq kvm_no_guest 473 lbz r0, KVM_SPLIT_DO_NAP(r3) 474 cmpwi r0, 0 475 beq kvm_no_guest 476 HMT_MEDIUM 477 b kvm_unsplit_nap 47860: HMT_MEDIUM 479 b kvm_secondary_got_guest 480 48154: li r0, KVM_HWTHREAD_IN_KVM 482 stb r0, HSTATE_HWTHREAD_STATE(r13) 483 b kvm_no_guest 484 485/* 486 * Here the primary thread is trying to return the core to 487 * whole-core mode, so we need to nap. 488 */ 489kvm_unsplit_nap: 490 /* 491 * When secondaries are napping in kvm_unsplit_nap() with 492 * hwthread_req = 1, HMI goes ignored even though subcores are 493 * already exited the guest. Hence HMI keeps waking up secondaries 494 * from nap in a loop and secondaries always go back to nap since 495 * no vcore is assigned to them. This makes impossible for primary 496 * thread to get hold of secondary threads resulting into a soft 497 * lockup in KVM path. 498 * 499 * Let us check if HMI is pending and handle it before we go to nap. 500 */ 501 cmpwi r12, BOOK3S_INTERRUPT_HMI 502 bne 55f 503 li r3, 0 /* NULL argument */ 504 bl hmi_exception_realmode 50555: 506 /* 507 * Ensure that secondary doesn't nap when it has 508 * its vcore pointer set. 509 */ 510 sync /* matches smp_mb() before setting split_info.do_nap */ 511 ld r0, HSTATE_KVM_VCORE(r13) 512 cmpdi r0, 0 513 bne kvm_no_guest 514 /* clear any pending message */ 515BEGIN_FTR_SECTION 516 lis r6, (PPC_DBELL_SERVER << (63-36))@h 517 PPC_MSGCLR(6) 518END_FTR_SECTION_IFSET(CPU_FTR_ARCH_207S) 519 /* Set kvm_split_mode.napped[tid] = 1 */ 520 ld r3, HSTATE_SPLIT_MODE(r13) 521 li r0, 1 522 lhz r4, PACAPACAINDEX(r13) 523 clrldi r4, r4, 61 /* micro-threading => P8 => 8 threads/core */ 524 addi r4, r4, KVM_SPLIT_NAPPED 525 stbx r0, r3, r4 526 /* Check the do_nap flag again after setting napped[] */ 527 sync 528 lbz r0, KVM_SPLIT_DO_NAP(r3) 529 cmpwi r0, 0 530 beq 57f 531 li r3, (LPCR_PECEDH | LPCR_PECE0) >> 4 532 mfspr r5, SPRN_LPCR 533 rlwimi r5, r3, 4, (LPCR_PECEDP | LPCR_PECEDH | LPCR_PECE0 | LPCR_PECE1) 534 b kvm_nap_sequence 535 53657: li r0, 0 537 stbx r0, r3, r4 538 b kvm_no_guest 539 540/****************************************************************************** 541 * * 542 * Entry code * 543 * * 544 *****************************************************************************/ 545 546.global kvmppc_hv_entry 547kvmppc_hv_entry: 548 549 /* Required state: 550 * 551 * R4 = vcpu pointer (or NULL) 552 * MSR = ~IR|DR 553 * R13 = PACA 554 * R1 = host R1 555 * R2 = TOC 556 * all other volatile GPRS = free 557 * Does not preserve non-volatile GPRs or CR fields 558 */ 559 mflr r0 560 std r0, PPC_LR_STKOFF(r1) 561 stdu r1, -SFS(r1) 562 563 /* Save R1 in the PACA */ 564 std r1, HSTATE_HOST_R1(r13) 565 566 li r6, KVM_GUEST_MODE_HOST_HV 567 stb r6, HSTATE_IN_GUEST(r13) 568 569#ifdef CONFIG_KVM_BOOK3S_HV_EXIT_TIMING 570 /* Store initial timestamp */ 571 cmpdi r4, 0 572 beq 1f 573 addi r3, r4, VCPU_TB_RMENTRY 574 bl kvmhv_start_timing 5751: 576#endif 577 578 /* Use cr7 as an indication of radix mode */ 579 ld r5, HSTATE_KVM_VCORE(r13) 580 ld r9, VCORE_KVM(r5) /* pointer to struct kvm */ 581 lbz r0, KVM_RADIX(r9) 582 cmpwi cr7, r0, 0 583 584 /* Clear out SLB if hash */ 585 bne cr7, 2f 586 li r6,0 587 slbmte r6,r6 588 slbia 589 ptesync 5902: 591 /* 592 * POWER7/POWER8 host -> guest partition switch code. 593 * We don't have to lock against concurrent tlbies, 594 * but we do have to coordinate across hardware threads. 595 */ 596 /* Set bit in entry map iff exit map is zero. */ 597 li r7, 1 598 lbz r6, HSTATE_PTID(r13) 599 sld r7, r7, r6 600 addi r8, r5, VCORE_ENTRY_EXIT 60121: lwarx r3, 0, r8 602 cmpwi r3, 0x100 /* any threads starting to exit? */ 603 bge secondary_too_late /* if so we're too late to the party */ 604 or r3, r3, r7 605 stwcx. r3, 0, r8 606 bne 21b 607 608 /* Primary thread switches to guest partition. */ 609 cmpwi r6,0 610 bne 10f 611 lwz r7,KVM_LPID(r9) 612BEGIN_FTR_SECTION 613 ld r6,KVM_SDR1(r9) 614 li r0,LPID_RSVD /* switch to reserved LPID */ 615 mtspr SPRN_LPID,r0 616 ptesync 617 mtspr SPRN_SDR1,r6 /* switch to partition page table */ 618END_FTR_SECTION_IFCLR(CPU_FTR_ARCH_300) 619 mtspr SPRN_LPID,r7 620 isync 621 622 /* See if we need to flush the TLB */ 623 lhz r6,PACAPACAINDEX(r13) /* test_bit(cpu, need_tlb_flush) */ 624BEGIN_FTR_SECTION 625 /* 626 * On POWER9, individual threads can come in here, but the 627 * TLB is shared between the 4 threads in a core, hence 628 * invalidating on one thread invalidates for all. 629 * Thus we make all 4 threads use the same bit here. 630 */ 631 clrrdi r6,r6,2 632END_FTR_SECTION_IFSET(CPU_FTR_ARCH_300) 633 clrldi r7,r6,64-6 /* extract bit number (6 bits) */ 634 srdi r6,r6,6 /* doubleword number */ 635 sldi r6,r6,3 /* address offset */ 636 add r6,r6,r9 637 addi r6,r6,KVM_NEED_FLUSH /* dword in kvm->arch.need_tlb_flush */ 638 li r8,1 639 sld r8,r8,r7 640 ld r7,0(r6) 641 and. r7,r7,r8 642 beq 22f 643 /* Flush the TLB of any entries for this LPID */ 644 lwz r0,KVM_TLB_SETS(r9) 645 mtctr r0 646 li r7,0x800 /* IS field = 0b10 */ 647 ptesync 648 li r0,0 /* RS for P9 version of tlbiel */ 649 bne cr7, 29f 65028: tlbiel r7 /* On P9, rs=0, RIC=0, PRS=0, R=0 */ 651 addi r7,r7,0x1000 652 bdnz 28b 653 b 30f 65429: PPC_TLBIEL(7,0,2,1,1) /* for radix, RIC=2, PRS=1, R=1 */ 655 addi r7,r7,0x1000 656 bdnz 29b 65730: ptesync 65823: ldarx r7,0,r6 /* clear the bit after TLB flushed */ 659 andc r7,r7,r8 660 stdcx. r7,0,r6 661 bne 23b 662 663 /* Add timebase offset onto timebase */ 66422: ld r8,VCORE_TB_OFFSET(r5) 665 cmpdi r8,0 666 beq 37f 667 mftb r6 /* current host timebase */ 668 add r8,r8,r6 669 mtspr SPRN_TBU40,r8 /* update upper 40 bits */ 670 mftb r7 /* check if lower 24 bits overflowed */ 671 clrldi r6,r6,40 672 clrldi r7,r7,40 673 cmpld r7,r6 674 bge 37f 675 addis r8,r8,0x100 /* if so, increment upper 40 bits */ 676 mtspr SPRN_TBU40,r8 677 678 /* Load guest PCR value to select appropriate compat mode */ 67937: ld r7, VCORE_PCR(r5) 680 cmpdi r7, 0 681 beq 38f 682 mtspr SPRN_PCR, r7 68338: 684 685BEGIN_FTR_SECTION 686 /* DPDES and VTB are shared between threads */ 687 ld r8, VCORE_DPDES(r5) 688 ld r7, VCORE_VTB(r5) 689 mtspr SPRN_DPDES, r8 690 mtspr SPRN_VTB, r7 691END_FTR_SECTION_IFSET(CPU_FTR_ARCH_207S) 692 693 /* Mark the subcore state as inside guest */ 694 bl kvmppc_subcore_enter_guest 695 nop 696 ld r5, HSTATE_KVM_VCORE(r13) 697 ld r4, HSTATE_KVM_VCPU(r13) 698 li r0,1 699 stb r0,VCORE_IN_GUEST(r5) /* signal secondaries to continue */ 700 701 /* Do we have a guest vcpu to run? */ 70210: cmpdi r4, 0 703 beq kvmppc_primary_no_guest 704kvmppc_got_guest: 705 706 /* Load up guest SLB entries (N.B. slb_max will be 0 for radix) */ 707 lwz r5,VCPU_SLB_MAX(r4) 708 cmpwi r5,0 709 beq 9f 710 mtctr r5 711 addi r6,r4,VCPU_SLB 7121: ld r8,VCPU_SLB_E(r6) 713 ld r9,VCPU_SLB_V(r6) 714 slbmte r9,r8 715 addi r6,r6,VCPU_SLB_SIZE 716 bdnz 1b 7179: 718 /* Increment yield count if they have a VPA */ 719 ld r3, VCPU_VPA(r4) 720 cmpdi r3, 0 721 beq 25f 722 li r6, LPPACA_YIELDCOUNT 723 LWZX_BE r5, r3, r6 724 addi r5, r5, 1 725 STWX_BE r5, r3, r6 726 li r6, 1 727 stb r6, VCPU_VPA_DIRTY(r4) 72825: 729 730 /* Save purr/spurr */ 731 mfspr r5,SPRN_PURR 732 mfspr r6,SPRN_SPURR 733 std r5,HSTATE_PURR(r13) 734 std r6,HSTATE_SPURR(r13) 735 ld r7,VCPU_PURR(r4) 736 ld r8,VCPU_SPURR(r4) 737 mtspr SPRN_PURR,r7 738 mtspr SPRN_SPURR,r8 739 740 /* Save host values of some registers */ 741BEGIN_FTR_SECTION 742 mfspr r5, SPRN_TIDR 743 mfspr r6, SPRN_PSSCR 744 mfspr r7, SPRN_PID 745 mfspr r8, SPRN_IAMR 746 std r5, STACK_SLOT_TID(r1) 747 std r6, STACK_SLOT_PSSCR(r1) 748 std r7, STACK_SLOT_PID(r1) 749 std r8, STACK_SLOT_IAMR(r1) 750 mfspr r5, SPRN_HFSCR 751 std r5, STACK_SLOT_HFSCR(r1) 752END_FTR_SECTION_IFSET(CPU_FTR_ARCH_300) 753BEGIN_FTR_SECTION 754 mfspr r5, SPRN_CIABR 755 mfspr r6, SPRN_DAWR 756 mfspr r7, SPRN_DAWRX 757 std r5, STACK_SLOT_CIABR(r1) 758 std r6, STACK_SLOT_DAWR(r1) 759 std r7, STACK_SLOT_DAWRX(r1) 760END_FTR_SECTION_IFSET(CPU_FTR_ARCH_207S) 761 762BEGIN_FTR_SECTION 763 /* Set partition DABR */ 764 /* Do this before re-enabling PMU to avoid P7 DABR corruption bug */ 765 lwz r5,VCPU_DABRX(r4) 766 ld r6,VCPU_DABR(r4) 767 mtspr SPRN_DABRX,r5 768 mtspr SPRN_DABR,r6 769 isync 770END_FTR_SECTION_IFCLR(CPU_FTR_ARCH_207S) 771 772#ifdef CONFIG_PPC_TRANSACTIONAL_MEM 773BEGIN_FTR_SECTION 774 /* 775 * NOTE THAT THIS TRASHES ALL NON-VOLATILE REGISTERS INCLUDING CR 776 */ 777 bl kvmppc_restore_tm 778END_FTR_SECTION_IFSET(CPU_FTR_TM) 779#endif 780 781 /* Load guest PMU registers */ 782 /* R4 is live here (vcpu pointer) */ 783 li r3, 1 784 sldi r3, r3, 31 /* MMCR0_FC (freeze counters) bit */ 785 mtspr SPRN_MMCR0, r3 /* freeze all counters, disable ints */ 786 isync 787BEGIN_FTR_SECTION 788 ld r3, VCPU_MMCR(r4) 789 andi. r5, r3, MMCR0_PMAO_SYNC | MMCR0_PMAO 790 cmpwi r5, MMCR0_PMAO 791 beql kvmppc_fix_pmao 792END_FTR_SECTION_IFSET(CPU_FTR_PMAO_BUG) 793 lwz r3, VCPU_PMC(r4) /* always load up guest PMU registers */ 794 lwz r5, VCPU_PMC + 4(r4) /* to prevent information leak */ 795 lwz r6, VCPU_PMC + 8(r4) 796 lwz r7, VCPU_PMC + 12(r4) 797 lwz r8, VCPU_PMC + 16(r4) 798 lwz r9, VCPU_PMC + 20(r4) 799 mtspr SPRN_PMC1, r3 800 mtspr SPRN_PMC2, r5 801 mtspr SPRN_PMC3, r6 802 mtspr SPRN_PMC4, r7 803 mtspr SPRN_PMC5, r8 804 mtspr SPRN_PMC6, r9 805 ld r3, VCPU_MMCR(r4) 806 ld r5, VCPU_MMCR + 8(r4) 807 ld r6, VCPU_MMCR + 16(r4) 808 ld r7, VCPU_SIAR(r4) 809 ld r8, VCPU_SDAR(r4) 810 mtspr SPRN_MMCR1, r5 811 mtspr SPRN_MMCRA, r6 812 mtspr SPRN_SIAR, r7 813 mtspr SPRN_SDAR, r8 814BEGIN_FTR_SECTION 815 ld r5, VCPU_MMCR + 24(r4) 816 ld r6, VCPU_SIER(r4) 817 mtspr SPRN_MMCR2, r5 818 mtspr SPRN_SIER, r6 819BEGIN_FTR_SECTION_NESTED(96) 820 lwz r7, VCPU_PMC + 24(r4) 821 lwz r8, VCPU_PMC + 28(r4) 822 ld r9, VCPU_MMCR + 32(r4) 823 mtspr SPRN_SPMC1, r7 824 mtspr SPRN_SPMC2, r8 825 mtspr SPRN_MMCRS, r9 826END_FTR_SECTION_NESTED(CPU_FTR_ARCH_300, 0, 96) 827END_FTR_SECTION_IFSET(CPU_FTR_ARCH_207S) 828 mtspr SPRN_MMCR0, r3 829 isync 830 831 /* Load up FP, VMX and VSX registers */ 832 bl kvmppc_load_fp 833 834 ld r14, VCPU_GPR(R14)(r4) 835 ld r15, VCPU_GPR(R15)(r4) 836 ld r16, VCPU_GPR(R16)(r4) 837 ld r17, VCPU_GPR(R17)(r4) 838 ld r18, VCPU_GPR(R18)(r4) 839 ld r19, VCPU_GPR(R19)(r4) 840 ld r20, VCPU_GPR(R20)(r4) 841 ld r21, VCPU_GPR(R21)(r4) 842 ld r22, VCPU_GPR(R22)(r4) 843 ld r23, VCPU_GPR(R23)(r4) 844 ld r24, VCPU_GPR(R24)(r4) 845 ld r25, VCPU_GPR(R25)(r4) 846 ld r26, VCPU_GPR(R26)(r4) 847 ld r27, VCPU_GPR(R27)(r4) 848 ld r28, VCPU_GPR(R28)(r4) 849 ld r29, VCPU_GPR(R29)(r4) 850 ld r30, VCPU_GPR(R30)(r4) 851 ld r31, VCPU_GPR(R31)(r4) 852 853 /* Switch DSCR to guest value */ 854 ld r5, VCPU_DSCR(r4) 855 mtspr SPRN_DSCR, r5 856 857BEGIN_FTR_SECTION 858 /* Skip next section on POWER7 */ 859 b 8f 860END_FTR_SECTION_IFCLR(CPU_FTR_ARCH_207S) 861 /* Load up POWER8-specific registers */ 862 ld r5, VCPU_IAMR(r4) 863 lwz r6, VCPU_PSPB(r4) 864 ld r7, VCPU_FSCR(r4) 865 mtspr SPRN_IAMR, r5 866 mtspr SPRN_PSPB, r6 867 mtspr SPRN_FSCR, r7 868 ld r5, VCPU_DAWR(r4) 869 ld r6, VCPU_DAWRX(r4) 870 ld r7, VCPU_CIABR(r4) 871 ld r8, VCPU_TAR(r4) 872 mtspr SPRN_DAWR, r5 873 mtspr SPRN_DAWRX, r6 874 mtspr SPRN_CIABR, r7 875 mtspr SPRN_TAR, r8 876 ld r5, VCPU_IC(r4) 877 ld r8, VCPU_EBBHR(r4) 878 mtspr SPRN_IC, r5 879 mtspr SPRN_EBBHR, r8 880 ld r5, VCPU_EBBRR(r4) 881 ld r6, VCPU_BESCR(r4) 882 lwz r7, VCPU_GUEST_PID(r4) 883 ld r8, VCPU_WORT(r4) 884 mtspr SPRN_EBBRR, r5 885 mtspr SPRN_BESCR, r6 886 mtspr SPRN_PID, r7 887 mtspr SPRN_WORT, r8 888BEGIN_FTR_SECTION 889 PPC_INVALIDATE_ERAT 890END_FTR_SECTION_IFSET(CPU_FTR_POWER9_DD1) 891BEGIN_FTR_SECTION 892 /* POWER8-only registers */ 893 ld r5, VCPU_TCSCR(r4) 894 ld r6, VCPU_ACOP(r4) 895 ld r7, VCPU_CSIGR(r4) 896 ld r8, VCPU_TACR(r4) 897 mtspr SPRN_TCSCR, r5 898 mtspr SPRN_ACOP, r6 899 mtspr SPRN_CSIGR, r7 900 mtspr SPRN_TACR, r8 901FTR_SECTION_ELSE 902 /* POWER9-only registers */ 903 ld r5, VCPU_TID(r4) 904 ld r6, VCPU_PSSCR(r4) 905 oris r6, r6, PSSCR_EC@h /* This makes stop trap to HV */ 906 ld r7, VCPU_HFSCR(r4) 907 mtspr SPRN_TIDR, r5 908 mtspr SPRN_PSSCR, r6 909 mtspr SPRN_HFSCR, r7 910ALT_FTR_SECTION_END_IFCLR(CPU_FTR_ARCH_300) 9118: 912 913 /* 914 * Set the decrementer to the guest decrementer. 915 */ 916 ld r8,VCPU_DEC_EXPIRES(r4) 917 /* r8 is a host timebase value here, convert to guest TB */ 918 ld r5,HSTATE_KVM_VCORE(r13) 919 ld r6,VCORE_TB_OFFSET(r5) 920 add r8,r8,r6 921 mftb r7 922 subf r3,r7,r8 923 mtspr SPRN_DEC,r3 924 std r3,VCPU_DEC(r4) 925 926 ld r5, VCPU_SPRG0(r4) 927 ld r6, VCPU_SPRG1(r4) 928 ld r7, VCPU_SPRG2(r4) 929 ld r8, VCPU_SPRG3(r4) 930 mtspr SPRN_SPRG0, r5 931 mtspr SPRN_SPRG1, r6 932 mtspr SPRN_SPRG2, r7 933 mtspr SPRN_SPRG3, r8 934 935 /* Load up DAR and DSISR */ 936 ld r5, VCPU_DAR(r4) 937 lwz r6, VCPU_DSISR(r4) 938 mtspr SPRN_DAR, r5 939 mtspr SPRN_DSISR, r6 940 941 /* Restore AMR and UAMOR, set AMOR to all 1s */ 942 ld r5,VCPU_AMR(r4) 943 ld r6,VCPU_UAMOR(r4) 944 li r7,-1 945 mtspr SPRN_AMR,r5 946 mtspr SPRN_UAMOR,r6 947 mtspr SPRN_AMOR,r7 948 949 /* Restore state of CTRL run bit; assume 1 on entry */ 950 lwz r5,VCPU_CTRL(r4) 951 andi. r5,r5,1 952 bne 4f 953 mfspr r6,SPRN_CTRLF 954 clrrdi r6,r6,1 955 mtspr SPRN_CTRLT,r6 9564: 957 /* Secondary threads wait for primary to have done partition switch */ 958 ld r5, HSTATE_KVM_VCORE(r13) 959 lbz r6, HSTATE_PTID(r13) 960 cmpwi r6, 0 961 beq 21f 962 lbz r0, VCORE_IN_GUEST(r5) 963 cmpwi r0, 0 964 bne 21f 965 HMT_LOW 96620: lwz r3, VCORE_ENTRY_EXIT(r5) 967 cmpwi r3, 0x100 968 bge no_switch_exit 969 lbz r0, VCORE_IN_GUEST(r5) 970 cmpwi r0, 0 971 beq 20b 972 HMT_MEDIUM 97321: 974 /* Set LPCR. */ 975 ld r8,VCORE_LPCR(r5) 976 mtspr SPRN_LPCR,r8 977 isync 978 979 /* Check if HDEC expires soon */ 980 mfspr r3, SPRN_HDEC 981 EXTEND_HDEC(r3) 982 cmpdi r3, 512 /* 1 microsecond */ 983 blt hdec_soon 984 985#ifdef CONFIG_KVM_XICS 986 /* We are entering the guest on that thread, push VCPU to XIVE */ 987 ld r10, HSTATE_XIVE_TIMA_PHYS(r13) 988 cmpldi cr0, r10, 0 989 beq no_xive 990 ld r11, VCPU_XIVE_SAVED_STATE(r4) 991 li r9, TM_QW1_OS 992 stdcix r11,r9,r10 993 eieio 994 lwz r11, VCPU_XIVE_CAM_WORD(r4) 995 li r9, TM_QW1_OS + TM_WORD2 996 stwcix r11,r9,r10 997 li r9, 1 998 stw r9, VCPU_XIVE_PUSHED(r4) 999no_xive: 1000#endif /* CONFIG_KVM_XICS */ 1001 1002deliver_guest_interrupt: 1003 ld r6, VCPU_CTR(r4) 1004 ld r7, VCPU_XER(r4) 1005 1006 mtctr r6 1007 mtxer r7 1008 1009kvmppc_cede_reentry: /* r4 = vcpu, r13 = paca */ 1010 ld r10, VCPU_PC(r4) 1011 ld r11, VCPU_MSR(r4) 1012 ld r6, VCPU_SRR0(r4) 1013 ld r7, VCPU_SRR1(r4) 1014 mtspr SPRN_SRR0, r6 1015 mtspr SPRN_SRR1, r7 1016 1017 /* r11 = vcpu->arch.msr & ~MSR_HV */ 1018 rldicl r11, r11, 63 - MSR_HV_LG, 1 1019 rotldi r11, r11, 1 + MSR_HV_LG 1020 ori r11, r11, MSR_ME 1021 1022 /* Check if we can deliver an external or decrementer interrupt now */ 1023 ld r0, VCPU_PENDING_EXC(r4) 1024 rldicl r0, r0, 64 - BOOK3S_IRQPRIO_EXTERNAL_LEVEL, 63 1025 cmpdi cr1, r0, 0 1026 andi. r8, r11, MSR_EE 1027 mfspr r8, SPRN_LPCR 1028 /* Insert EXTERNAL_LEVEL bit into LPCR at the MER bit position */ 1029 rldimi r8, r0, LPCR_MER_SH, 63 - LPCR_MER_SH 1030 mtspr SPRN_LPCR, r8 1031 isync 1032 beq 5f 1033 li r0, BOOK3S_INTERRUPT_EXTERNAL 1034 bne cr1, 12f 1035 mfspr r0, SPRN_DEC 1036BEGIN_FTR_SECTION 1037 /* On POWER9 check whether the guest has large decrementer enabled */ 1038 andis. r8, r8, LPCR_LD@h 1039 bne 15f 1040END_FTR_SECTION_IFSET(CPU_FTR_ARCH_300) 1041 extsw r0, r0 104215: cmpdi r0, 0 1043 li r0, BOOK3S_INTERRUPT_DECREMENTER 1044 bge 5f 1045 104612: mtspr SPRN_SRR0, r10 1047 mr r10,r0 1048 mtspr SPRN_SRR1, r11 1049 mr r9, r4 1050 bl kvmppc_msr_interrupt 10515: 1052BEGIN_FTR_SECTION 1053 b fast_guest_return 1054END_FTR_SECTION_IFCLR(CPU_FTR_ARCH_300) 1055 /* On POWER9, check for pending doorbell requests */ 1056 lbz r0, VCPU_DBELL_REQ(r4) 1057 cmpwi r0, 0 1058 beq fast_guest_return 1059 ld r5, HSTATE_KVM_VCORE(r13) 1060 /* Set DPDES register so the CPU will take a doorbell interrupt */ 1061 li r0, 1 1062 mtspr SPRN_DPDES, r0 1063 std r0, VCORE_DPDES(r5) 1064 /* Make sure other cpus see vcore->dpdes set before dbell req clear */ 1065 lwsync 1066 /* Clear the pending doorbell request */ 1067 li r0, 0 1068 stb r0, VCPU_DBELL_REQ(r4) 1069 1070/* 1071 * Required state: 1072 * R4 = vcpu 1073 * R10: value for HSRR0 1074 * R11: value for HSRR1 1075 * R13 = PACA 1076 */ 1077fast_guest_return: 1078 li r0,0 1079 stb r0,VCPU_CEDED(r4) /* cancel cede */ 1080 mtspr SPRN_HSRR0,r10 1081 mtspr SPRN_HSRR1,r11 1082 1083 /* Activate guest mode, so faults get handled by KVM */ 1084 li r9, KVM_GUEST_MODE_GUEST_HV 1085 stb r9, HSTATE_IN_GUEST(r13) 1086 1087#ifdef CONFIG_KVM_BOOK3S_HV_EXIT_TIMING 1088 /* Accumulate timing */ 1089 addi r3, r4, VCPU_TB_GUEST 1090 bl kvmhv_accumulate_time 1091#endif 1092 1093 /* Enter guest */ 1094 1095BEGIN_FTR_SECTION 1096 ld r5, VCPU_CFAR(r4) 1097 mtspr SPRN_CFAR, r5 1098END_FTR_SECTION_IFSET(CPU_FTR_CFAR) 1099BEGIN_FTR_SECTION 1100 ld r0, VCPU_PPR(r4) 1101END_FTR_SECTION_IFSET(CPU_FTR_HAS_PPR) 1102 1103 ld r5, VCPU_LR(r4) 1104 lwz r6, VCPU_CR(r4) 1105 mtlr r5 1106 mtcr r6 1107 1108 ld r1, VCPU_GPR(R1)(r4) 1109 ld r2, VCPU_GPR(R2)(r4) 1110 ld r3, VCPU_GPR(R3)(r4) 1111 ld r5, VCPU_GPR(R5)(r4) 1112 ld r6, VCPU_GPR(R6)(r4) 1113 ld r7, VCPU_GPR(R7)(r4) 1114 ld r8, VCPU_GPR(R8)(r4) 1115 ld r9, VCPU_GPR(R9)(r4) 1116 ld r10, VCPU_GPR(R10)(r4) 1117 ld r11, VCPU_GPR(R11)(r4) 1118 ld r12, VCPU_GPR(R12)(r4) 1119 ld r13, VCPU_GPR(R13)(r4) 1120 1121BEGIN_FTR_SECTION 1122 mtspr SPRN_PPR, r0 1123END_FTR_SECTION_IFSET(CPU_FTR_HAS_PPR) 1124 ld r0, VCPU_GPR(R0)(r4) 1125 ld r4, VCPU_GPR(R4)(r4) 1126 1127 hrfid 1128 b . 1129 1130secondary_too_late: 1131 li r12, 0 1132 cmpdi r4, 0 1133 beq 11f 1134 stw r12, VCPU_TRAP(r4) 1135#ifdef CONFIG_KVM_BOOK3S_HV_EXIT_TIMING 1136 addi r3, r4, VCPU_TB_RMEXIT 1137 bl kvmhv_accumulate_time 1138#endif 113911: b kvmhv_switch_to_host 1140 1141no_switch_exit: 1142 HMT_MEDIUM 1143 li r12, 0 1144 b 12f 1145hdec_soon: 1146 li r12, BOOK3S_INTERRUPT_HV_DECREMENTER 114712: stw r12, VCPU_TRAP(r4) 1148 mr r9, r4 1149#ifdef CONFIG_KVM_BOOK3S_HV_EXIT_TIMING 1150 addi r3, r4, VCPU_TB_RMEXIT 1151 bl kvmhv_accumulate_time 1152#endif 1153 b guest_exit_cont 1154 1155/****************************************************************************** 1156 * * 1157 * Exit code * 1158 * * 1159 *****************************************************************************/ 1160 1161/* 1162 * We come here from the first-level interrupt handlers. 1163 */ 1164 .globl kvmppc_interrupt_hv 1165kvmppc_interrupt_hv: 1166 /* 1167 * Register contents: 1168 * R12 = (guest CR << 32) | interrupt vector 1169 * R13 = PACA 1170 * guest R12 saved in shadow VCPU SCRATCH0 1171 * guest CTR saved in shadow VCPU SCRATCH1 if RELOCATABLE 1172 * guest R13 saved in SPRN_SCRATCH0 1173 */ 1174 std r9, HSTATE_SCRATCH2(r13) 1175 lbz r9, HSTATE_IN_GUEST(r13) 1176 cmpwi r9, KVM_GUEST_MODE_HOST_HV 1177 beq kvmppc_bad_host_intr 1178#ifdef CONFIG_KVM_BOOK3S_PR_POSSIBLE 1179 cmpwi r9, KVM_GUEST_MODE_GUEST 1180 ld r9, HSTATE_SCRATCH2(r13) 1181 beq kvmppc_interrupt_pr 1182#endif 1183 /* We're now back in the host but in guest MMU context */ 1184 li r9, KVM_GUEST_MODE_HOST_HV 1185 stb r9, HSTATE_IN_GUEST(r13) 1186 1187 ld r9, HSTATE_KVM_VCPU(r13) 1188 1189 /* Save registers */ 1190 1191 std r0, VCPU_GPR(R0)(r9) 1192 std r1, VCPU_GPR(R1)(r9) 1193 std r2, VCPU_GPR(R2)(r9) 1194 std r3, VCPU_GPR(R3)(r9) 1195 std r4, VCPU_GPR(R4)(r9) 1196 std r5, VCPU_GPR(R5)(r9) 1197 std r6, VCPU_GPR(R6)(r9) 1198 std r7, VCPU_GPR(R7)(r9) 1199 std r8, VCPU_GPR(R8)(r9) 1200 ld r0, HSTATE_SCRATCH2(r13) 1201 std r0, VCPU_GPR(R9)(r9) 1202 std r10, VCPU_GPR(R10)(r9) 1203 std r11, VCPU_GPR(R11)(r9) 1204 ld r3, HSTATE_SCRATCH0(r13) 1205 std r3, VCPU_GPR(R12)(r9) 1206 /* CR is in the high half of r12 */ 1207 srdi r4, r12, 32 1208 stw r4, VCPU_CR(r9) 1209BEGIN_FTR_SECTION 1210 ld r3, HSTATE_CFAR(r13) 1211 std r3, VCPU_CFAR(r9) 1212END_FTR_SECTION_IFSET(CPU_FTR_CFAR) 1213BEGIN_FTR_SECTION 1214 ld r4, HSTATE_PPR(r13) 1215 std r4, VCPU_PPR(r9) 1216END_FTR_SECTION_IFSET(CPU_FTR_HAS_PPR) 1217 1218 /* Restore R1/R2 so we can handle faults */ 1219 ld r1, HSTATE_HOST_R1(r13) 1220 ld r2, PACATOC(r13) 1221 1222 mfspr r10, SPRN_SRR0 1223 mfspr r11, SPRN_SRR1 1224 std r10, VCPU_SRR0(r9) 1225 std r11, VCPU_SRR1(r9) 1226 /* trap is in the low half of r12, clear CR from the high half */ 1227 clrldi r12, r12, 32 1228 andi. r0, r12, 2 /* need to read HSRR0/1? */ 1229 beq 1f 1230 mfspr r10, SPRN_HSRR0 1231 mfspr r11, SPRN_HSRR1 1232 clrrdi r12, r12, 2 12331: std r10, VCPU_PC(r9) 1234 std r11, VCPU_MSR(r9) 1235 1236 GET_SCRATCH0(r3) 1237 mflr r4 1238 std r3, VCPU_GPR(R13)(r9) 1239 std r4, VCPU_LR(r9) 1240 1241 stw r12,VCPU_TRAP(r9) 1242 1243 /* 1244 * Now that we have saved away SRR0/1 and HSRR0/1, 1245 * interrupts are recoverable in principle, so set MSR_RI. 1246 * This becomes important for relocation-on interrupts from 1247 * the guest, which we can get in radix mode on POWER9. 1248 */ 1249 li r0, MSR_RI 1250 mtmsrd r0, 1 1251 1252#ifdef CONFIG_KVM_BOOK3S_HV_EXIT_TIMING 1253 addi r3, r9, VCPU_TB_RMINTR 1254 mr r4, r9 1255 bl kvmhv_accumulate_time 1256 ld r5, VCPU_GPR(R5)(r9) 1257 ld r6, VCPU_GPR(R6)(r9) 1258 ld r7, VCPU_GPR(R7)(r9) 1259 ld r8, VCPU_GPR(R8)(r9) 1260#endif 1261 1262 /* Save HEIR (HV emulation assist reg) in emul_inst 1263 if this is an HEI (HV emulation interrupt, e40) */ 1264 li r3,KVM_INST_FETCH_FAILED 1265 stw r3,VCPU_LAST_INST(r9) 1266 cmpwi r12,BOOK3S_INTERRUPT_H_EMUL_ASSIST 1267 bne 11f 1268 mfspr r3,SPRN_HEIR 126911: stw r3,VCPU_HEIR(r9) 1270 1271 /* these are volatile across C function calls */ 1272#ifdef CONFIG_RELOCATABLE 1273 ld r3, HSTATE_SCRATCH1(r13) 1274 mtctr r3 1275#else 1276 mfctr r3 1277#endif 1278 mfxer r4 1279 std r3, VCPU_CTR(r9) 1280 std r4, VCPU_XER(r9) 1281 1282 /* If this is a page table miss then see if it's theirs or ours */ 1283 cmpwi r12, BOOK3S_INTERRUPT_H_DATA_STORAGE 1284 beq kvmppc_hdsi 1285 cmpwi r12, BOOK3S_INTERRUPT_H_INST_STORAGE 1286 beq kvmppc_hisi 1287 1288 /* See if this is a leftover HDEC interrupt */ 1289 cmpwi r12,BOOK3S_INTERRUPT_HV_DECREMENTER 1290 bne 2f 1291 mfspr r3,SPRN_HDEC 1292 EXTEND_HDEC(r3) 1293 cmpdi r3,0 1294 mr r4,r9 1295 bge fast_guest_return 12962: 1297 /* See if this is an hcall we can handle in real mode */ 1298 cmpwi r12,BOOK3S_INTERRUPT_SYSCALL 1299 beq hcall_try_real_mode 1300 1301 /* Hypervisor doorbell - exit only if host IPI flag set */ 1302 cmpwi r12, BOOK3S_INTERRUPT_H_DOORBELL 1303 bne 3f 1304BEGIN_FTR_SECTION 1305 PPC_MSGSYNC 1306END_FTR_SECTION_IFSET(CPU_FTR_ARCH_300) 1307 lbz r0, HSTATE_HOST_IPI(r13) 1308 cmpwi r0, 0 1309 beq 4f 1310 b guest_exit_cont 13113: 1312 /* If it's a hypervisor facility unavailable interrupt, save HFSCR */ 1313 cmpwi r12, BOOK3S_INTERRUPT_H_FAC_UNAVAIL 1314 bne 14f 1315 mfspr r3, SPRN_HFSCR 1316 std r3, VCPU_HFSCR(r9) 1317 b guest_exit_cont 131814: 1319 /* External interrupt ? */ 1320 cmpwi r12, BOOK3S_INTERRUPT_EXTERNAL 1321 bne+ guest_exit_cont 1322 1323 /* External interrupt, first check for host_ipi. If this is 1324 * set, we know the host wants us out so let's do it now 1325 */ 1326 bl kvmppc_read_intr 1327 1328 /* 1329 * Restore the active volatile registers after returning from 1330 * a C function. 1331 */ 1332 ld r9, HSTATE_KVM_VCPU(r13) 1333 li r12, BOOK3S_INTERRUPT_EXTERNAL 1334 1335 /* 1336 * kvmppc_read_intr return codes: 1337 * 1338 * Exit to host (r3 > 0) 1339 * 1 An interrupt is pending that needs to be handled by the host 1340 * Exit guest and return to host by branching to guest_exit_cont 1341 * 1342 * 2 Passthrough that needs completion in the host 1343 * Exit guest and return to host by branching to guest_exit_cont 1344 * However, we also set r12 to BOOK3S_INTERRUPT_HV_RM_HARD 1345 * to indicate to the host to complete handling the interrupt 1346 * 1347 * Before returning to guest, we check if any CPU is heading out 1348 * to the host and if so, we head out also. If no CPUs are heading 1349 * check return values <= 0. 1350 * 1351 * Return to guest (r3 <= 0) 1352 * 0 No external interrupt is pending 1353 * -1 A guest wakeup IPI (which has now been cleared) 1354 * In either case, we return to guest to deliver any pending 1355 * guest interrupts. 1356 * 1357 * -2 A PCI passthrough external interrupt was handled 1358 * (interrupt was delivered directly to guest) 1359 * Return to guest to deliver any pending guest interrupts. 1360 */ 1361 1362 cmpdi r3, 1 1363 ble 1f 1364 1365 /* Return code = 2 */ 1366 li r12, BOOK3S_INTERRUPT_HV_RM_HARD 1367 stw r12, VCPU_TRAP(r9) 1368 b guest_exit_cont 1369 13701: /* Return code <= 1 */ 1371 cmpdi r3, 0 1372 bgt guest_exit_cont 1373 1374 /* Return code <= 0 */ 13754: ld r5, HSTATE_KVM_VCORE(r13) 1376 lwz r0, VCORE_ENTRY_EXIT(r5) 1377 cmpwi r0, 0x100 1378 mr r4, r9 1379 blt deliver_guest_interrupt 1380 1381guest_exit_cont: /* r9 = vcpu, r12 = trap, r13 = paca */ 1382#ifdef CONFIG_KVM_XICS 1383 /* We are exiting, pull the VP from the XIVE */ 1384 lwz r0, VCPU_XIVE_PUSHED(r9) 1385 cmpwi cr0, r0, 0 1386 beq 1f 1387 li r7, TM_SPC_PULL_OS_CTX 1388 li r6, TM_QW1_OS 1389 mfmsr r0 1390 andi. r0, r0, MSR_IR /* in real mode? */ 1391 beq 2f 1392 ld r10, HSTATE_XIVE_TIMA_VIRT(r13) 1393 cmpldi cr0, r10, 0 1394 beq 1f 1395 /* First load to pull the context, we ignore the value */ 1396 lwzx r11, r7, r10 1397 eieio 1398 /* Second load to recover the context state (Words 0 and 1) */ 1399 ldx r11, r6, r10 1400 b 3f 14012: ld r10, HSTATE_XIVE_TIMA_PHYS(r13) 1402 cmpldi cr0, r10, 0 1403 beq 1f 1404 /* First load to pull the context, we ignore the value */ 1405 lwzcix r11, r7, r10 1406 eieio 1407 /* Second load to recover the context state (Words 0 and 1) */ 1408 ldcix r11, r6, r10 14093: std r11, VCPU_XIVE_SAVED_STATE(r9) 1410 /* Fixup some of the state for the next load */ 1411 li r10, 0 1412 li r0, 0xff 1413 stw r10, VCPU_XIVE_PUSHED(r9) 1414 stb r10, (VCPU_XIVE_SAVED_STATE+3)(r9) 1415 stb r0, (VCPU_XIVE_SAVED_STATE+4)(r9) 14161: 1417#endif /* CONFIG_KVM_XICS */ 1418 /* Save more register state */ 1419 mfdar r6 1420 mfdsisr r7 1421 std r6, VCPU_DAR(r9) 1422 stw r7, VCPU_DSISR(r9) 1423 /* don't overwrite fault_dar/fault_dsisr if HDSI */ 1424 cmpwi r12,BOOK3S_INTERRUPT_H_DATA_STORAGE 1425 beq mc_cont 1426 std r6, VCPU_FAULT_DAR(r9) 1427 stw r7, VCPU_FAULT_DSISR(r9) 1428 1429 /* See if it is a machine check */ 1430 cmpwi r12, BOOK3S_INTERRUPT_MACHINE_CHECK 1431 beq machine_check_realmode 1432mc_cont: 1433#ifdef CONFIG_KVM_BOOK3S_HV_EXIT_TIMING 1434 addi r3, r9, VCPU_TB_RMEXIT 1435 mr r4, r9 1436 bl kvmhv_accumulate_time 1437#endif 1438 1439 mr r3, r12 1440 /* Increment exit count, poke other threads to exit */ 1441 bl kvmhv_commence_exit 1442 nop 1443 ld r9, HSTATE_KVM_VCPU(r13) 1444 lwz r12, VCPU_TRAP(r9) 1445 1446 /* Stop others sending VCPU interrupts to this physical CPU */ 1447 li r0, -1 1448 stw r0, VCPU_CPU(r9) 1449 stw r0, VCPU_THREAD_CPU(r9) 1450 1451 /* Save guest CTRL register, set runlatch to 1 */ 1452 mfspr r6,SPRN_CTRLF 1453 stw r6,VCPU_CTRL(r9) 1454 andi. r0,r6,1 1455 bne 4f 1456 ori r6,r6,1 1457 mtspr SPRN_CTRLT,r6 14584: 1459 /* Check if we are running hash or radix and store it in cr2 */ 1460 ld r5, VCPU_KVM(r9) 1461 lbz r0, KVM_RADIX(r5) 1462 cmpwi cr2,r0,0 1463 1464 /* Read the guest SLB and save it away */ 1465 li r5, 0 1466 bne cr2, 3f /* for radix, save 0 entries */ 1467 lwz r0,VCPU_SLB_NR(r9) /* number of entries in SLB */ 1468 mtctr r0 1469 li r6,0 1470 addi r7,r9,VCPU_SLB 14711: slbmfee r8,r6 1472 andis. r0,r8,SLB_ESID_V@h 1473 beq 2f 1474 add r8,r8,r6 /* put index in */ 1475 slbmfev r3,r6 1476 std r8,VCPU_SLB_E(r7) 1477 std r3,VCPU_SLB_V(r7) 1478 addi r7,r7,VCPU_SLB_SIZE 1479 addi r5,r5,1 14802: addi r6,r6,1 1481 bdnz 1b 14823: stw r5,VCPU_SLB_MAX(r9) 1483 1484 /* 1485 * Save the guest PURR/SPURR 1486 */ 1487 mfspr r5,SPRN_PURR 1488 mfspr r6,SPRN_SPURR 1489 ld r7,VCPU_PURR(r9) 1490 ld r8,VCPU_SPURR(r9) 1491 std r5,VCPU_PURR(r9) 1492 std r6,VCPU_SPURR(r9) 1493 subf r5,r7,r5 1494 subf r6,r8,r6 1495 1496 /* 1497 * Restore host PURR/SPURR and add guest times 1498 * so that the time in the guest gets accounted. 1499 */ 1500 ld r3,HSTATE_PURR(r13) 1501 ld r4,HSTATE_SPURR(r13) 1502 add r3,r3,r5 1503 add r4,r4,r6 1504 mtspr SPRN_PURR,r3 1505 mtspr SPRN_SPURR,r4 1506 1507 /* Save DEC */ 1508 ld r3, HSTATE_KVM_VCORE(r13) 1509 mfspr r5,SPRN_DEC 1510 mftb r6 1511 /* On P9, if the guest has large decr enabled, don't sign extend */ 1512BEGIN_FTR_SECTION 1513 ld r4, VCORE_LPCR(r3) 1514 andis. r4, r4, LPCR_LD@h 1515 bne 16f 1516END_FTR_SECTION_IFSET(CPU_FTR_ARCH_300) 1517 extsw r5,r5 151816: add r5,r5,r6 1519 /* r5 is a guest timebase value here, convert to host TB */ 1520 ld r4,VCORE_TB_OFFSET(r3) 1521 subf r5,r4,r5 1522 std r5,VCPU_DEC_EXPIRES(r9) 1523 1524BEGIN_FTR_SECTION 1525 b 8f 1526END_FTR_SECTION_IFCLR(CPU_FTR_ARCH_207S) 1527 /* Save POWER8-specific registers */ 1528 mfspr r5, SPRN_IAMR 1529 mfspr r6, SPRN_PSPB 1530 mfspr r7, SPRN_FSCR 1531 std r5, VCPU_IAMR(r9) 1532 stw r6, VCPU_PSPB(r9) 1533 std r7, VCPU_FSCR(r9) 1534 mfspr r5, SPRN_IC 1535 mfspr r7, SPRN_TAR 1536 std r5, VCPU_IC(r9) 1537 std r7, VCPU_TAR(r9) 1538 mfspr r8, SPRN_EBBHR 1539 std r8, VCPU_EBBHR(r9) 1540 mfspr r5, SPRN_EBBRR 1541 mfspr r6, SPRN_BESCR 1542 mfspr r7, SPRN_PID 1543 mfspr r8, SPRN_WORT 1544 std r5, VCPU_EBBRR(r9) 1545 std r6, VCPU_BESCR(r9) 1546 stw r7, VCPU_GUEST_PID(r9) 1547 std r8, VCPU_WORT(r9) 1548BEGIN_FTR_SECTION 1549 mfspr r5, SPRN_TCSCR 1550 mfspr r6, SPRN_ACOP 1551 mfspr r7, SPRN_CSIGR 1552 mfspr r8, SPRN_TACR 1553 std r5, VCPU_TCSCR(r9) 1554 std r6, VCPU_ACOP(r9) 1555 std r7, VCPU_CSIGR(r9) 1556 std r8, VCPU_TACR(r9) 1557FTR_SECTION_ELSE 1558 mfspr r5, SPRN_TIDR 1559 mfspr r6, SPRN_PSSCR 1560 std r5, VCPU_TID(r9) 1561 rldicl r6, r6, 4, 50 /* r6 &= PSSCR_GUEST_VIS */ 1562 rotldi r6, r6, 60 1563 std r6, VCPU_PSSCR(r9) 1564 /* Restore host HFSCR value */ 1565 ld r7, STACK_SLOT_HFSCR(r1) 1566 mtspr SPRN_HFSCR, r7 1567ALT_FTR_SECTION_END_IFCLR(CPU_FTR_ARCH_300) 1568 /* 1569 * Restore various registers to 0, where non-zero values 1570 * set by the guest could disrupt the host. 1571 */ 1572 li r0, 0 1573 mtspr SPRN_PSPB, r0 1574 mtspr SPRN_WORT, r0 1575BEGIN_FTR_SECTION 1576 mtspr SPRN_IAMR, r0 1577 mtspr SPRN_TCSCR, r0 1578 /* Set MMCRS to 1<<31 to freeze and disable the SPMC counters */ 1579 li r0, 1 1580 sldi r0, r0, 31 1581 mtspr SPRN_MMCRS, r0 1582END_FTR_SECTION_IFCLR(CPU_FTR_ARCH_300) 15838: 1584 1585 /* Save and reset AMR and UAMOR before turning on the MMU */ 1586 mfspr r5,SPRN_AMR 1587 mfspr r6,SPRN_UAMOR 1588 std r5,VCPU_AMR(r9) 1589 std r6,VCPU_UAMOR(r9) 1590 li r6,0 1591 mtspr SPRN_AMR,r6 1592 mtspr SPRN_UAMOR, r6 1593 1594 /* Switch DSCR back to host value */ 1595 mfspr r8, SPRN_DSCR 1596 ld r7, HSTATE_DSCR(r13) 1597 std r8, VCPU_DSCR(r9) 1598 mtspr SPRN_DSCR, r7 1599 1600 /* Save non-volatile GPRs */ 1601 std r14, VCPU_GPR(R14)(r9) 1602 std r15, VCPU_GPR(R15)(r9) 1603 std r16, VCPU_GPR(R16)(r9) 1604 std r17, VCPU_GPR(R17)(r9) 1605 std r18, VCPU_GPR(R18)(r9) 1606 std r19, VCPU_GPR(R19)(r9) 1607 std r20, VCPU_GPR(R20)(r9) 1608 std r21, VCPU_GPR(R21)(r9) 1609 std r22, VCPU_GPR(R22)(r9) 1610 std r23, VCPU_GPR(R23)(r9) 1611 std r24, VCPU_GPR(R24)(r9) 1612 std r25, VCPU_GPR(R25)(r9) 1613 std r26, VCPU_GPR(R26)(r9) 1614 std r27, VCPU_GPR(R27)(r9) 1615 std r28, VCPU_GPR(R28)(r9) 1616 std r29, VCPU_GPR(R29)(r9) 1617 std r30, VCPU_GPR(R30)(r9) 1618 std r31, VCPU_GPR(R31)(r9) 1619 1620 /* Save SPRGs */ 1621 mfspr r3, SPRN_SPRG0 1622 mfspr r4, SPRN_SPRG1 1623 mfspr r5, SPRN_SPRG2 1624 mfspr r6, SPRN_SPRG3 1625 std r3, VCPU_SPRG0(r9) 1626 std r4, VCPU_SPRG1(r9) 1627 std r5, VCPU_SPRG2(r9) 1628 std r6, VCPU_SPRG3(r9) 1629 1630 /* save FP state */ 1631 mr r3, r9 1632 bl kvmppc_save_fp 1633 1634#ifdef CONFIG_PPC_TRANSACTIONAL_MEM 1635BEGIN_FTR_SECTION 1636 /* 1637 * NOTE THAT THIS TRASHES ALL NON-VOLATILE REGISTERS INCLUDING CR 1638 */ 1639 bl kvmppc_save_tm 1640END_FTR_SECTION_IFSET(CPU_FTR_TM) 1641#endif 1642 1643 /* Increment yield count if they have a VPA */ 1644 ld r8, VCPU_VPA(r9) /* do they have a VPA? */ 1645 cmpdi r8, 0 1646 beq 25f 1647 li r4, LPPACA_YIELDCOUNT 1648 LWZX_BE r3, r8, r4 1649 addi r3, r3, 1 1650 STWX_BE r3, r8, r4 1651 li r3, 1 1652 stb r3, VCPU_VPA_DIRTY(r9) 165325: 1654 /* Save PMU registers if requested */ 1655 /* r8 and cr0.eq are live here */ 1656BEGIN_FTR_SECTION 1657 /* 1658 * POWER8 seems to have a hardware bug where setting 1659 * MMCR0[PMAE] along with MMCR0[PMC1CE] and/or MMCR0[PMCjCE] 1660 * when some counters are already negative doesn't seem 1661 * to cause a performance monitor alert (and hence interrupt). 1662 * The effect of this is that when saving the PMU state, 1663 * if there is no PMU alert pending when we read MMCR0 1664 * before freezing the counters, but one becomes pending 1665 * before we read the counters, we lose it. 1666 * To work around this, we need a way to freeze the counters 1667 * before reading MMCR0. Normally, freezing the counters 1668 * is done by writing MMCR0 (to set MMCR0[FC]) which 1669 * unavoidably writes MMCR0[PMA0] as well. On POWER8, 1670 * we can also freeze the counters using MMCR2, by writing 1671 * 1s to all the counter freeze condition bits (there are 1672 * 9 bits each for 6 counters). 1673 */ 1674 li r3, -1 /* set all freeze bits */ 1675 clrrdi r3, r3, 10 1676 mfspr r10, SPRN_MMCR2 1677 mtspr SPRN_MMCR2, r3 1678 isync 1679END_FTR_SECTION_IFSET(CPU_FTR_ARCH_207S) 1680 li r3, 1 1681 sldi r3, r3, 31 /* MMCR0_FC (freeze counters) bit */ 1682 mfspr r4, SPRN_MMCR0 /* save MMCR0 */ 1683 mtspr SPRN_MMCR0, r3 /* freeze all counters, disable ints */ 1684 mfspr r6, SPRN_MMCRA 1685 /* Clear MMCRA in order to disable SDAR updates */ 1686 li r7, 0 1687 mtspr SPRN_MMCRA, r7 1688 isync 1689 beq 21f /* if no VPA, save PMU stuff anyway */ 1690 lbz r7, LPPACA_PMCINUSE(r8) 1691 cmpwi r7, 0 /* did they ask for PMU stuff to be saved? */ 1692 bne 21f 1693 std r3, VCPU_MMCR(r9) /* if not, set saved MMCR0 to FC */ 1694 b 22f 169521: mfspr r5, SPRN_MMCR1 1696 mfspr r7, SPRN_SIAR 1697 mfspr r8, SPRN_SDAR 1698 std r4, VCPU_MMCR(r9) 1699 std r5, VCPU_MMCR + 8(r9) 1700 std r6, VCPU_MMCR + 16(r9) 1701BEGIN_FTR_SECTION 1702 std r10, VCPU_MMCR + 24(r9) 1703END_FTR_SECTION_IFSET(CPU_FTR_ARCH_207S) 1704 std r7, VCPU_SIAR(r9) 1705 std r8, VCPU_SDAR(r9) 1706 mfspr r3, SPRN_PMC1 1707 mfspr r4, SPRN_PMC2 1708 mfspr r5, SPRN_PMC3 1709 mfspr r6, SPRN_PMC4 1710 mfspr r7, SPRN_PMC5 1711 mfspr r8, SPRN_PMC6 1712 stw r3, VCPU_PMC(r9) 1713 stw r4, VCPU_PMC + 4(r9) 1714 stw r5, VCPU_PMC + 8(r9) 1715 stw r6, VCPU_PMC + 12(r9) 1716 stw r7, VCPU_PMC + 16(r9) 1717 stw r8, VCPU_PMC + 20(r9) 1718BEGIN_FTR_SECTION 1719 mfspr r5, SPRN_SIER 1720 std r5, VCPU_SIER(r9) 1721BEGIN_FTR_SECTION_NESTED(96) 1722 mfspr r6, SPRN_SPMC1 1723 mfspr r7, SPRN_SPMC2 1724 mfspr r8, SPRN_MMCRS 1725 stw r6, VCPU_PMC + 24(r9) 1726 stw r7, VCPU_PMC + 28(r9) 1727 std r8, VCPU_MMCR + 32(r9) 1728 lis r4, 0x8000 1729 mtspr SPRN_MMCRS, r4 1730END_FTR_SECTION_NESTED(CPU_FTR_ARCH_300, 0, 96) 1731END_FTR_SECTION_IFSET(CPU_FTR_ARCH_207S) 173222: 1733 1734 /* Restore host values of some registers */ 1735BEGIN_FTR_SECTION 1736 ld r5, STACK_SLOT_CIABR(r1) 1737 ld r6, STACK_SLOT_DAWR(r1) 1738 ld r7, STACK_SLOT_DAWRX(r1) 1739 mtspr SPRN_CIABR, r5 1740 mtspr SPRN_DAWR, r6 1741 mtspr SPRN_DAWRX, r7 1742END_FTR_SECTION_IFSET(CPU_FTR_ARCH_207S) 1743BEGIN_FTR_SECTION 1744 ld r5, STACK_SLOT_TID(r1) 1745 ld r6, STACK_SLOT_PSSCR(r1) 1746 ld r7, STACK_SLOT_PID(r1) 1747 ld r8, STACK_SLOT_IAMR(r1) 1748 mtspr SPRN_TIDR, r5 1749 mtspr SPRN_PSSCR, r6 1750 mtspr SPRN_PID, r7 1751 mtspr SPRN_IAMR, r8 1752END_FTR_SECTION_IFSET(CPU_FTR_ARCH_300) 1753 1754#ifdef CONFIG_PPC_RADIX_MMU 1755 /* 1756 * Are we running hash or radix ? 1757 */ 1758 ld r5, VCPU_KVM(r9) 1759 lbz r0, KVM_RADIX(r5) 1760 cmpwi cr2, r0, 0 1761 beq cr2, 3f 1762 1763 /* Radix: Handle the case where the guest used an illegal PID */ 1764 LOAD_REG_ADDR(r4, mmu_base_pid) 1765 lwz r3, VCPU_GUEST_PID(r9) 1766 lwz r5, 0(r4) 1767 cmpw cr0,r3,r5 1768 blt 2f 1769 1770 /* 1771 * Illegal PID, the HW might have prefetched and cached in the TLB 1772 * some translations for the LPID 0 / guest PID combination which 1773 * Linux doesn't know about, so we need to flush that PID out of 1774 * the TLB. First we need to set LPIDR to 0 so tlbiel applies to 1775 * the right context. 1776 */ 1777 li r0,0 1778 mtspr SPRN_LPID,r0 1779 isync 1780 1781 /* Then do a congruence class local flush */ 1782 ld r6,VCPU_KVM(r9) 1783 lwz r0,KVM_TLB_SETS(r6) 1784 mtctr r0 1785 li r7,0x400 /* IS field = 0b01 */ 1786 ptesync 1787 sldi r0,r3,32 /* RS has PID */ 17881: PPC_TLBIEL(7,0,2,1,1) /* RIC=2, PRS=1, R=1 */ 1789 addi r7,r7,0x1000 1790 bdnz 1b 1791 ptesync 1792 17932: /* Flush the ERAT on radix P9 DD1 guest exit */ 1794BEGIN_FTR_SECTION 1795 PPC_INVALIDATE_ERAT 1796END_FTR_SECTION_IFSET(CPU_FTR_POWER9_DD1) 1797 b 4f 1798#endif /* CONFIG_PPC_RADIX_MMU */ 1799 1800 /* Hash: clear out SLB */ 18013: li r5,0 1802 slbmte r5,r5 1803 slbia 1804 ptesync 18054: 1806 /* 1807 * POWER7/POWER8 guest -> host partition switch code. 1808 * We don't have to lock against tlbies but we do 1809 * have to coordinate the hardware threads. 1810 */ 1811kvmhv_switch_to_host: 1812 /* Secondary threads wait for primary to do partition switch */ 1813 ld r5,HSTATE_KVM_VCORE(r13) 1814 ld r4,VCORE_KVM(r5) /* pointer to struct kvm */ 1815 lbz r3,HSTATE_PTID(r13) 1816 cmpwi r3,0 1817 beq 15f 1818 HMT_LOW 181913: lbz r3,VCORE_IN_GUEST(r5) 1820 cmpwi r3,0 1821 bne 13b 1822 HMT_MEDIUM 1823 b 16f 1824 1825 /* Primary thread waits for all the secondaries to exit guest */ 182615: lwz r3,VCORE_ENTRY_EXIT(r5) 1827 rlwinm r0,r3,32-8,0xff 1828 clrldi r3,r3,56 1829 cmpw r3,r0 1830 bne 15b 1831 isync 1832 1833 /* Did we actually switch to the guest at all? */ 1834 lbz r6, VCORE_IN_GUEST(r5) 1835 cmpwi r6, 0 1836 beq 19f 1837 1838 /* Primary thread switches back to host partition */ 1839 lwz r7,KVM_HOST_LPID(r4) 1840BEGIN_FTR_SECTION 1841 ld r6,KVM_HOST_SDR1(r4) 1842 li r8,LPID_RSVD /* switch to reserved LPID */ 1843 mtspr SPRN_LPID,r8 1844 ptesync 1845 mtspr SPRN_SDR1,r6 /* switch to host page table */ 1846END_FTR_SECTION_IFCLR(CPU_FTR_ARCH_300) 1847 mtspr SPRN_LPID,r7 1848 isync 1849 1850BEGIN_FTR_SECTION 1851 /* DPDES and VTB are shared between threads */ 1852 mfspr r7, SPRN_DPDES 1853 mfspr r8, SPRN_VTB 1854 std r7, VCORE_DPDES(r5) 1855 std r8, VCORE_VTB(r5) 1856 /* clear DPDES so we don't get guest doorbells in the host */ 1857 li r8, 0 1858 mtspr SPRN_DPDES, r8 1859END_FTR_SECTION_IFSET(CPU_FTR_ARCH_207S) 1860 1861 /* If HMI, call kvmppc_realmode_hmi_handler() */ 1862 cmpwi r12, BOOK3S_INTERRUPT_HMI 1863 bne 27f 1864 bl kvmppc_realmode_hmi_handler 1865 nop 1866 li r12, BOOK3S_INTERRUPT_HMI 1867 /* 1868 * At this point kvmppc_realmode_hmi_handler would have resync-ed 1869 * the TB. Hence it is not required to subtract guest timebase 1870 * offset from timebase. So, skip it. 1871 * 1872 * Also, do not call kvmppc_subcore_exit_guest() because it has 1873 * been invoked as part of kvmppc_realmode_hmi_handler(). 1874 */ 1875 b 30f 1876 187727: 1878 /* Subtract timebase offset from timebase */ 1879 ld r8,VCORE_TB_OFFSET(r5) 1880 cmpdi r8,0 1881 beq 17f 1882 mftb r6 /* current guest timebase */ 1883 subf r8,r8,r6 1884 mtspr SPRN_TBU40,r8 /* update upper 40 bits */ 1885 mftb r7 /* check if lower 24 bits overflowed */ 1886 clrldi r6,r6,40 1887 clrldi r7,r7,40 1888 cmpld r7,r6 1889 bge 17f 1890 addis r8,r8,0x100 /* if so, increment upper 40 bits */ 1891 mtspr SPRN_TBU40,r8 1892 189317: bl kvmppc_subcore_exit_guest 1894 nop 189530: ld r5,HSTATE_KVM_VCORE(r13) 1896 ld r4,VCORE_KVM(r5) /* pointer to struct kvm */ 1897 1898 /* Reset PCR */ 1899 ld r0, VCORE_PCR(r5) 1900 cmpdi r0, 0 1901 beq 18f 1902 li r0, 0 1903 mtspr SPRN_PCR, r0 190418: 1905 /* Signal secondary CPUs to continue */ 1906 stb r0,VCORE_IN_GUEST(r5) 190719: lis r8,0x7fff /* MAX_INT@h */ 1908 mtspr SPRN_HDEC,r8 1909 191016: ld r8,KVM_HOST_LPCR(r4) 1911 mtspr SPRN_LPCR,r8 1912 isync 1913 1914 /* load host SLB entries */ 1915BEGIN_MMU_FTR_SECTION 1916 b 0f 1917END_MMU_FTR_SECTION_IFSET(MMU_FTR_TYPE_RADIX) 1918 ld r8,PACA_SLBSHADOWPTR(r13) 1919 1920 .rept SLB_NUM_BOLTED 1921 li r3, SLBSHADOW_SAVEAREA 1922 LDX_BE r5, r8, r3 1923 addi r3, r3, 8 1924 LDX_BE r6, r8, r3 1925 andis. r7,r5,SLB_ESID_V@h 1926 beq 1f 1927 slbmte r6,r5 19281: addi r8,r8,16 1929 .endr 19300: 1931#ifdef CONFIG_KVM_BOOK3S_HV_EXIT_TIMING 1932 /* Finish timing, if we have a vcpu */ 1933 ld r4, HSTATE_KVM_VCPU(r13) 1934 cmpdi r4, 0 1935 li r3, 0 1936 beq 2f 1937 bl kvmhv_accumulate_time 19382: 1939#endif 1940 /* Unset guest mode */ 1941 li r0, KVM_GUEST_MODE_NONE 1942 stb r0, HSTATE_IN_GUEST(r13) 1943 1944 ld r0, SFS+PPC_LR_STKOFF(r1) 1945 addi r1, r1, SFS 1946 mtlr r0 1947 blr 1948 1949/* 1950 * Check whether an HDSI is an HPTE not found fault or something else. 1951 * If it is an HPTE not found fault that is due to the guest accessing 1952 * a page that they have mapped but which we have paged out, then 1953 * we continue on with the guest exit path. In all other cases, 1954 * reflect the HDSI to the guest as a DSI. 1955 */ 1956kvmppc_hdsi: 1957 ld r3, VCPU_KVM(r9) 1958 lbz r0, KVM_RADIX(r3) 1959 cmpwi r0, 0 1960 mfspr r4, SPRN_HDAR 1961 mfspr r6, SPRN_HDSISR 1962 bne .Lradix_hdsi /* on radix, just save DAR/DSISR/ASDR */ 1963 /* HPTE not found fault or protection fault? */ 1964 andis. r0, r6, (DSISR_NOHPTE | DSISR_PROTFAULT)@h 1965 beq 1f /* if not, send it to the guest */ 1966 andi. r0, r11, MSR_DR /* data relocation enabled? */ 1967 beq 3f 1968BEGIN_FTR_SECTION 1969 mfspr r5, SPRN_ASDR /* on POWER9, use ASDR to get VSID */ 1970 b 4f 1971END_FTR_SECTION_IFSET(CPU_FTR_ARCH_300) 1972 clrrdi r0, r4, 28 1973 PPC_SLBFEE_DOT(R5, R0) /* if so, look up SLB */ 1974 li r0, BOOK3S_INTERRUPT_DATA_SEGMENT 1975 bne 7f /* if no SLB entry found */ 19764: std r4, VCPU_FAULT_DAR(r9) 1977 stw r6, VCPU_FAULT_DSISR(r9) 1978 1979 /* Search the hash table. */ 1980 mr r3, r9 /* vcpu pointer */ 1981 li r7, 1 /* data fault */ 1982 bl kvmppc_hpte_hv_fault 1983 ld r9, HSTATE_KVM_VCPU(r13) 1984 ld r10, VCPU_PC(r9) 1985 ld r11, VCPU_MSR(r9) 1986 li r12, BOOK3S_INTERRUPT_H_DATA_STORAGE 1987 cmpdi r3, 0 /* retry the instruction */ 1988 beq 6f 1989 cmpdi r3, -1 /* handle in kernel mode */ 1990 beq guest_exit_cont 1991 cmpdi r3, -2 /* MMIO emulation; need instr word */ 1992 beq 2f 1993 1994 /* Synthesize a DSI (or DSegI) for the guest */ 1995 ld r4, VCPU_FAULT_DAR(r9) 1996 mr r6, r3 19971: li r0, BOOK3S_INTERRUPT_DATA_STORAGE 1998 mtspr SPRN_DSISR, r6 19997: mtspr SPRN_DAR, r4 2000 mtspr SPRN_SRR0, r10 2001 mtspr SPRN_SRR1, r11 2002 mr r10, r0 2003 bl kvmppc_msr_interrupt 2004fast_interrupt_c_return: 20056: ld r7, VCPU_CTR(r9) 2006 ld r8, VCPU_XER(r9) 2007 mtctr r7 2008 mtxer r8 2009 mr r4, r9 2010 b fast_guest_return 2011 20123: ld r5, VCPU_KVM(r9) /* not relocated, use VRMA */ 2013 ld r5, KVM_VRMA_SLB_V(r5) 2014 b 4b 2015 2016 /* If this is for emulated MMIO, load the instruction word */ 20172: li r8, KVM_INST_FETCH_FAILED /* In case lwz faults */ 2018 2019 /* Set guest mode to 'jump over instruction' so if lwz faults 2020 * we'll just continue at the next IP. */ 2021 li r0, KVM_GUEST_MODE_SKIP 2022 stb r0, HSTATE_IN_GUEST(r13) 2023 2024 /* Do the access with MSR:DR enabled */ 2025 mfmsr r3 2026 ori r4, r3, MSR_DR /* Enable paging for data */ 2027 mtmsrd r4 2028 lwz r8, 0(r10) 2029 mtmsrd r3 2030 2031 /* Store the result */ 2032 stw r8, VCPU_LAST_INST(r9) 2033 2034 /* Unset guest mode. */ 2035 li r0, KVM_GUEST_MODE_HOST_HV 2036 stb r0, HSTATE_IN_GUEST(r13) 2037 b guest_exit_cont 2038 2039.Lradix_hdsi: 2040 std r4, VCPU_FAULT_DAR(r9) 2041 stw r6, VCPU_FAULT_DSISR(r9) 2042.Lradix_hisi: 2043 mfspr r5, SPRN_ASDR 2044 std r5, VCPU_FAULT_GPA(r9) 2045 b guest_exit_cont 2046 2047/* 2048 * Similarly for an HISI, reflect it to the guest as an ISI unless 2049 * it is an HPTE not found fault for a page that we have paged out. 2050 */ 2051kvmppc_hisi: 2052 ld r3, VCPU_KVM(r9) 2053 lbz r0, KVM_RADIX(r3) 2054 cmpwi r0, 0 2055 bne .Lradix_hisi /* for radix, just save ASDR */ 2056 andis. r0, r11, SRR1_ISI_NOPT@h 2057 beq 1f 2058 andi. r0, r11, MSR_IR /* instruction relocation enabled? */ 2059 beq 3f 2060BEGIN_FTR_SECTION 2061 mfspr r5, SPRN_ASDR /* on POWER9, use ASDR to get VSID */ 2062 b 4f 2063END_FTR_SECTION_IFSET(CPU_FTR_ARCH_300) 2064 clrrdi r0, r10, 28 2065 PPC_SLBFEE_DOT(R5, R0) /* if so, look up SLB */ 2066 li r0, BOOK3S_INTERRUPT_INST_SEGMENT 2067 bne 7f /* if no SLB entry found */ 20684: 2069 /* Search the hash table. */ 2070 mr r3, r9 /* vcpu pointer */ 2071 mr r4, r10 2072 mr r6, r11 2073 li r7, 0 /* instruction fault */ 2074 bl kvmppc_hpte_hv_fault 2075 ld r9, HSTATE_KVM_VCPU(r13) 2076 ld r10, VCPU_PC(r9) 2077 ld r11, VCPU_MSR(r9) 2078 li r12, BOOK3S_INTERRUPT_H_INST_STORAGE 2079 cmpdi r3, 0 /* retry the instruction */ 2080 beq fast_interrupt_c_return 2081 cmpdi r3, -1 /* handle in kernel mode */ 2082 beq guest_exit_cont 2083 2084 /* Synthesize an ISI (or ISegI) for the guest */ 2085 mr r11, r3 20861: li r0, BOOK3S_INTERRUPT_INST_STORAGE 20877: mtspr SPRN_SRR0, r10 2088 mtspr SPRN_SRR1, r11 2089 mr r10, r0 2090 bl kvmppc_msr_interrupt 2091 b fast_interrupt_c_return 2092 20933: ld r6, VCPU_KVM(r9) /* not relocated, use VRMA */ 2094 ld r5, KVM_VRMA_SLB_V(r6) 2095 b 4b 2096 2097/* 2098 * Try to handle an hcall in real mode. 2099 * Returns to the guest if we handle it, or continues on up to 2100 * the kernel if we can't (i.e. if we don't have a handler for 2101 * it, or if the handler returns H_TOO_HARD). 2102 * 2103 * r5 - r8 contain hcall args, 2104 * r9 = vcpu, r10 = pc, r11 = msr, r12 = trap, r13 = paca 2105 */ 2106hcall_try_real_mode: 2107 ld r3,VCPU_GPR(R3)(r9) 2108 andi. r0,r11,MSR_PR 2109 /* sc 1 from userspace - reflect to guest syscall */ 2110 bne sc_1_fast_return 2111 clrrdi r3,r3,2 2112 cmpldi r3,hcall_real_table_end - hcall_real_table 2113 bge guest_exit_cont 2114 /* See if this hcall is enabled for in-kernel handling */ 2115 ld r4, VCPU_KVM(r9) 2116 srdi r0, r3, 8 /* r0 = (r3 / 4) >> 6 */ 2117 sldi r0, r0, 3 /* index into kvm->arch.enabled_hcalls[] */ 2118 add r4, r4, r0 2119 ld r0, KVM_ENABLED_HCALLS(r4) 2120 rlwinm r4, r3, 32-2, 0x3f /* r4 = (r3 / 4) & 0x3f */ 2121 srd r0, r0, r4 2122 andi. r0, r0, 1 2123 beq guest_exit_cont 2124 /* Get pointer to handler, if any, and call it */ 2125 LOAD_REG_ADDR(r4, hcall_real_table) 2126 lwax r3,r3,r4 2127 cmpwi r3,0 2128 beq guest_exit_cont 2129 add r12,r3,r4 2130 mtctr r12 2131 mr r3,r9 /* get vcpu pointer */ 2132 ld r4,VCPU_GPR(R4)(r9) 2133 bctrl 2134 cmpdi r3,H_TOO_HARD 2135 beq hcall_real_fallback 2136 ld r4,HSTATE_KVM_VCPU(r13) 2137 std r3,VCPU_GPR(R3)(r4) 2138 ld r10,VCPU_PC(r4) 2139 ld r11,VCPU_MSR(r4) 2140 b fast_guest_return 2141 2142sc_1_fast_return: 2143 mtspr SPRN_SRR0,r10 2144 mtspr SPRN_SRR1,r11 2145 li r10, BOOK3S_INTERRUPT_SYSCALL 2146 bl kvmppc_msr_interrupt 2147 mr r4,r9 2148 b fast_guest_return 2149 2150 /* We've attempted a real mode hcall, but it's punted it back 2151 * to userspace. We need to restore some clobbered volatiles 2152 * before resuming the pass-it-to-qemu path */ 2153hcall_real_fallback: 2154 li r12,BOOK3S_INTERRUPT_SYSCALL 2155 ld r9, HSTATE_KVM_VCPU(r13) 2156 2157 b guest_exit_cont 2158 2159 .globl hcall_real_table 2160hcall_real_table: 2161 .long 0 /* 0 - unused */ 2162 .long DOTSYM(kvmppc_h_remove) - hcall_real_table 2163 .long DOTSYM(kvmppc_h_enter) - hcall_real_table 2164 .long DOTSYM(kvmppc_h_read) - hcall_real_table 2165 .long DOTSYM(kvmppc_h_clear_mod) - hcall_real_table 2166 .long DOTSYM(kvmppc_h_clear_ref) - hcall_real_table 2167 .long DOTSYM(kvmppc_h_protect) - hcall_real_table 2168 .long DOTSYM(kvmppc_h_get_tce) - hcall_real_table 2169 .long DOTSYM(kvmppc_rm_h_put_tce) - hcall_real_table 2170 .long 0 /* 0x24 - H_SET_SPRG0 */ 2171 .long DOTSYM(kvmppc_h_set_dabr) - hcall_real_table 2172 .long 0 /* 0x2c */ 2173 .long 0 /* 0x30 */ 2174 .long 0 /* 0x34 */ 2175 .long 0 /* 0x38 */ 2176 .long 0 /* 0x3c */ 2177 .long 0 /* 0x40 */ 2178 .long 0 /* 0x44 */ 2179 .long 0 /* 0x48 */ 2180 .long 0 /* 0x4c */ 2181 .long 0 /* 0x50 */ 2182 .long 0 /* 0x54 */ 2183 .long 0 /* 0x58 */ 2184 .long 0 /* 0x5c */ 2185 .long 0 /* 0x60 */ 2186#ifdef CONFIG_KVM_XICS 2187 .long DOTSYM(kvmppc_rm_h_eoi) - hcall_real_table 2188 .long DOTSYM(kvmppc_rm_h_cppr) - hcall_real_table 2189 .long DOTSYM(kvmppc_rm_h_ipi) - hcall_real_table 2190 .long DOTSYM(kvmppc_rm_h_ipoll) - hcall_real_table 2191 .long DOTSYM(kvmppc_rm_h_xirr) - hcall_real_table 2192#else 2193 .long 0 /* 0x64 - H_EOI */ 2194 .long 0 /* 0x68 - H_CPPR */ 2195 .long 0 /* 0x6c - H_IPI */ 2196 .long 0 /* 0x70 - H_IPOLL */ 2197 .long 0 /* 0x74 - H_XIRR */ 2198#endif 2199 .long 0 /* 0x78 */ 2200 .long 0 /* 0x7c */ 2201 .long 0 /* 0x80 */ 2202 .long 0 /* 0x84 */ 2203 .long 0 /* 0x88 */ 2204 .long 0 /* 0x8c */ 2205 .long 0 /* 0x90 */ 2206 .long 0 /* 0x94 */ 2207 .long 0 /* 0x98 */ 2208 .long 0 /* 0x9c */ 2209 .long 0 /* 0xa0 */ 2210 .long 0 /* 0xa4 */ 2211 .long 0 /* 0xa8 */ 2212 .long 0 /* 0xac */ 2213 .long 0 /* 0xb0 */ 2214 .long 0 /* 0xb4 */ 2215 .long 0 /* 0xb8 */ 2216 .long 0 /* 0xbc */ 2217 .long 0 /* 0xc0 */ 2218 .long 0 /* 0xc4 */ 2219 .long 0 /* 0xc8 */ 2220 .long 0 /* 0xcc */ 2221 .long 0 /* 0xd0 */ 2222 .long 0 /* 0xd4 */ 2223 .long 0 /* 0xd8 */ 2224 .long 0 /* 0xdc */ 2225 .long DOTSYM(kvmppc_h_cede) - hcall_real_table 2226 .long DOTSYM(kvmppc_rm_h_confer) - hcall_real_table 2227 .long 0 /* 0xe8 */ 2228 .long 0 /* 0xec */ 2229 .long 0 /* 0xf0 */ 2230 .long 0 /* 0xf4 */ 2231 .long 0 /* 0xf8 */ 2232 .long 0 /* 0xfc */ 2233 .long 0 /* 0x100 */ 2234 .long 0 /* 0x104 */ 2235 .long 0 /* 0x108 */ 2236 .long 0 /* 0x10c */ 2237 .long 0 /* 0x110 */ 2238 .long 0 /* 0x114 */ 2239 .long 0 /* 0x118 */ 2240 .long 0 /* 0x11c */ 2241 .long 0 /* 0x120 */ 2242 .long DOTSYM(kvmppc_h_bulk_remove) - hcall_real_table 2243 .long 0 /* 0x128 */ 2244 .long 0 /* 0x12c */ 2245 .long 0 /* 0x130 */ 2246 .long DOTSYM(kvmppc_h_set_xdabr) - hcall_real_table 2247 .long DOTSYM(kvmppc_rm_h_stuff_tce) - hcall_real_table 2248 .long DOTSYM(kvmppc_rm_h_put_tce_indirect) - hcall_real_table 2249 .long 0 /* 0x140 */ 2250 .long 0 /* 0x144 */ 2251 .long 0 /* 0x148 */ 2252 .long 0 /* 0x14c */ 2253 .long 0 /* 0x150 */ 2254 .long 0 /* 0x154 */ 2255 .long 0 /* 0x158 */ 2256 .long 0 /* 0x15c */ 2257 .long 0 /* 0x160 */ 2258 .long 0 /* 0x164 */ 2259 .long 0 /* 0x168 */ 2260 .long 0 /* 0x16c */ 2261 .long 0 /* 0x170 */ 2262 .long 0 /* 0x174 */ 2263 .long 0 /* 0x178 */ 2264 .long 0 /* 0x17c */ 2265 .long 0 /* 0x180 */ 2266 .long 0 /* 0x184 */ 2267 .long 0 /* 0x188 */ 2268 .long 0 /* 0x18c */ 2269 .long 0 /* 0x190 */ 2270 .long 0 /* 0x194 */ 2271 .long 0 /* 0x198 */ 2272 .long 0 /* 0x19c */ 2273 .long 0 /* 0x1a0 */ 2274 .long 0 /* 0x1a4 */ 2275 .long 0 /* 0x1a8 */ 2276 .long 0 /* 0x1ac */ 2277 .long 0 /* 0x1b0 */ 2278 .long 0 /* 0x1b4 */ 2279 .long 0 /* 0x1b8 */ 2280 .long 0 /* 0x1bc */ 2281 .long 0 /* 0x1c0 */ 2282 .long 0 /* 0x1c4 */ 2283 .long 0 /* 0x1c8 */ 2284 .long 0 /* 0x1cc */ 2285 .long 0 /* 0x1d0 */ 2286 .long 0 /* 0x1d4 */ 2287 .long 0 /* 0x1d8 */ 2288 .long 0 /* 0x1dc */ 2289 .long 0 /* 0x1e0 */ 2290 .long 0 /* 0x1e4 */ 2291 .long 0 /* 0x1e8 */ 2292 .long 0 /* 0x1ec */ 2293 .long 0 /* 0x1f0 */ 2294 .long 0 /* 0x1f4 */ 2295 .long 0 /* 0x1f8 */ 2296 .long 0 /* 0x1fc */ 2297 .long 0 /* 0x200 */ 2298 .long 0 /* 0x204 */ 2299 .long 0 /* 0x208 */ 2300 .long 0 /* 0x20c */ 2301 .long 0 /* 0x210 */ 2302 .long 0 /* 0x214 */ 2303 .long 0 /* 0x218 */ 2304 .long 0 /* 0x21c */ 2305 .long 0 /* 0x220 */ 2306 .long 0 /* 0x224 */ 2307 .long 0 /* 0x228 */ 2308 .long 0 /* 0x22c */ 2309 .long 0 /* 0x230 */ 2310 .long 0 /* 0x234 */ 2311 .long 0 /* 0x238 */ 2312 .long 0 /* 0x23c */ 2313 .long 0 /* 0x240 */ 2314 .long 0 /* 0x244 */ 2315 .long 0 /* 0x248 */ 2316 .long 0 /* 0x24c */ 2317 .long 0 /* 0x250 */ 2318 .long 0 /* 0x254 */ 2319 .long 0 /* 0x258 */ 2320 .long 0 /* 0x25c */ 2321 .long 0 /* 0x260 */ 2322 .long 0 /* 0x264 */ 2323 .long 0 /* 0x268 */ 2324 .long 0 /* 0x26c */ 2325 .long 0 /* 0x270 */ 2326 .long 0 /* 0x274 */ 2327 .long 0 /* 0x278 */ 2328 .long 0 /* 0x27c */ 2329 .long 0 /* 0x280 */ 2330 .long 0 /* 0x284 */ 2331 .long 0 /* 0x288 */ 2332 .long 0 /* 0x28c */ 2333 .long 0 /* 0x290 */ 2334 .long 0 /* 0x294 */ 2335 .long 0 /* 0x298 */ 2336 .long 0 /* 0x29c */ 2337 .long 0 /* 0x2a0 */ 2338 .long 0 /* 0x2a4 */ 2339 .long 0 /* 0x2a8 */ 2340 .long 0 /* 0x2ac */ 2341 .long 0 /* 0x2b0 */ 2342 .long 0 /* 0x2b4 */ 2343 .long 0 /* 0x2b8 */ 2344 .long 0 /* 0x2bc */ 2345 .long 0 /* 0x2c0 */ 2346 .long 0 /* 0x2c4 */ 2347 .long 0 /* 0x2c8 */ 2348 .long 0 /* 0x2cc */ 2349 .long 0 /* 0x2d0 */ 2350 .long 0 /* 0x2d4 */ 2351 .long 0 /* 0x2d8 */ 2352 .long 0 /* 0x2dc */ 2353 .long 0 /* 0x2e0 */ 2354 .long 0 /* 0x2e4 */ 2355 .long 0 /* 0x2e8 */ 2356 .long 0 /* 0x2ec */ 2357 .long 0 /* 0x2f0 */ 2358 .long 0 /* 0x2f4 */ 2359 .long 0 /* 0x2f8 */ 2360#ifdef CONFIG_KVM_XICS 2361 .long DOTSYM(kvmppc_rm_h_xirr_x) - hcall_real_table 2362#else 2363 .long 0 /* 0x2fc - H_XIRR_X*/ 2364#endif 2365 .long DOTSYM(kvmppc_h_random) - hcall_real_table 2366 .globl hcall_real_table_end 2367hcall_real_table_end: 2368 2369_GLOBAL(kvmppc_h_set_xdabr) 2370 andi. r0, r5, DABRX_USER | DABRX_KERNEL 2371 beq 6f 2372 li r0, DABRX_USER | DABRX_KERNEL | DABRX_BTI 2373 andc. r0, r5, r0 2374 beq 3f 23756: li r3, H_PARAMETER 2376 blr 2377 2378_GLOBAL(kvmppc_h_set_dabr) 2379 li r5, DABRX_USER | DABRX_KERNEL 23803: 2381BEGIN_FTR_SECTION 2382 b 2f 2383END_FTR_SECTION_IFSET(CPU_FTR_ARCH_207S) 2384 std r4,VCPU_DABR(r3) 2385 stw r5, VCPU_DABRX(r3) 2386 mtspr SPRN_DABRX, r5 2387 /* Work around P7 bug where DABR can get corrupted on mtspr */ 23881: mtspr SPRN_DABR,r4 2389 mfspr r5, SPRN_DABR 2390 cmpd r4, r5 2391 bne 1b 2392 isync 2393 li r3,0 2394 blr 2395 2396 /* Emulate H_SET_DABR/X on P8 for the sake of compat mode guests */ 23972: rlwimi r5, r4, 5, DAWRX_DR | DAWRX_DW 2398 rlwimi r5, r4, 2, DAWRX_WT 2399 clrrdi r4, r4, 3 2400 std r4, VCPU_DAWR(r3) 2401 std r5, VCPU_DAWRX(r3) 2402 mtspr SPRN_DAWR, r4 2403 mtspr SPRN_DAWRX, r5 2404 li r3, 0 2405 blr 2406 2407_GLOBAL(kvmppc_h_cede) /* r3 = vcpu pointer, r11 = msr, r13 = paca */ 2408 ori r11,r11,MSR_EE 2409 std r11,VCPU_MSR(r3) 2410 li r0,1 2411 stb r0,VCPU_CEDED(r3) 2412 sync /* order setting ceded vs. testing prodded */ 2413 lbz r5,VCPU_PRODDED(r3) 2414 cmpwi r5,0 2415 bne kvm_cede_prodded 2416 li r12,0 /* set trap to 0 to say hcall is handled */ 2417 stw r12,VCPU_TRAP(r3) 2418 li r0,H_SUCCESS 2419 std r0,VCPU_GPR(R3)(r3) 2420 2421 /* 2422 * Set our bit in the bitmask of napping threads unless all the 2423 * other threads are already napping, in which case we send this 2424 * up to the host. 2425 */ 2426 ld r5,HSTATE_KVM_VCORE(r13) 2427 lbz r6,HSTATE_PTID(r13) 2428 lwz r8,VCORE_ENTRY_EXIT(r5) 2429 clrldi r8,r8,56 2430 li r0,1 2431 sld r0,r0,r6 2432 addi r6,r5,VCORE_NAPPING_THREADS 243331: lwarx r4,0,r6 2434 or r4,r4,r0 2435 cmpw r4,r8 2436 beq kvm_cede_exit 2437 stwcx. r4,0,r6 2438 bne 31b 2439 /* order napping_threads update vs testing entry_exit_map */ 2440 isync 2441 li r0,NAPPING_CEDE 2442 stb r0,HSTATE_NAPPING(r13) 2443 lwz r7,VCORE_ENTRY_EXIT(r5) 2444 cmpwi r7,0x100 2445 bge 33f /* another thread already exiting */ 2446 2447/* 2448 * Although not specifically required by the architecture, POWER7 2449 * preserves the following registers in nap mode, even if an SMT mode 2450 * switch occurs: SLB entries, PURR, SPURR, AMOR, UAMOR, AMR, SPRG0-3, 2451 * DAR, DSISR, DABR, DABRX, DSCR, PMCx, MMCRx, SIAR, SDAR. 2452 */ 2453 /* Save non-volatile GPRs */ 2454 std r14, VCPU_GPR(R14)(r3) 2455 std r15, VCPU_GPR(R15)(r3) 2456 std r16, VCPU_GPR(R16)(r3) 2457 std r17, VCPU_GPR(R17)(r3) 2458 std r18, VCPU_GPR(R18)(r3) 2459 std r19, VCPU_GPR(R19)(r3) 2460 std r20, VCPU_GPR(R20)(r3) 2461 std r21, VCPU_GPR(R21)(r3) 2462 std r22, VCPU_GPR(R22)(r3) 2463 std r23, VCPU_GPR(R23)(r3) 2464 std r24, VCPU_GPR(R24)(r3) 2465 std r25, VCPU_GPR(R25)(r3) 2466 std r26, VCPU_GPR(R26)(r3) 2467 std r27, VCPU_GPR(R27)(r3) 2468 std r28, VCPU_GPR(R28)(r3) 2469 std r29, VCPU_GPR(R29)(r3) 2470 std r30, VCPU_GPR(R30)(r3) 2471 std r31, VCPU_GPR(R31)(r3) 2472 2473 /* save FP state */ 2474 bl kvmppc_save_fp 2475 2476#ifdef CONFIG_PPC_TRANSACTIONAL_MEM 2477BEGIN_FTR_SECTION 2478 /* 2479 * NOTE THAT THIS TRASHES ALL NON-VOLATILE REGISTERS INCLUDING CR 2480 */ 2481 ld r9, HSTATE_KVM_VCPU(r13) 2482 bl kvmppc_save_tm 2483END_FTR_SECTION_IFSET(CPU_FTR_TM) 2484#endif 2485 2486 /* 2487 * Set DEC to the smaller of DEC and HDEC, so that we wake 2488 * no later than the end of our timeslice (HDEC interrupts 2489 * don't wake us from nap). 2490 */ 2491 mfspr r3, SPRN_DEC 2492 mfspr r4, SPRN_HDEC 2493 mftb r5 2494BEGIN_FTR_SECTION 2495 /* On P9 check whether the guest has large decrementer mode enabled */ 2496 ld r6, HSTATE_KVM_VCORE(r13) 2497 ld r6, VCORE_LPCR(r6) 2498 andis. r6, r6, LPCR_LD@h 2499 bne 68f 2500END_FTR_SECTION_IFSET(CPU_FTR_ARCH_300) 2501 extsw r3, r3 250268: EXTEND_HDEC(r4) 2503 cmpd r3, r4 2504 ble 67f 2505 mtspr SPRN_DEC, r4 250667: 2507 /* save expiry time of guest decrementer */ 2508 add r3, r3, r5 2509 ld r4, HSTATE_KVM_VCPU(r13) 2510 ld r5, HSTATE_KVM_VCORE(r13) 2511 ld r6, VCORE_TB_OFFSET(r5) 2512 subf r3, r6, r3 /* convert to host TB value */ 2513 std r3, VCPU_DEC_EXPIRES(r4) 2514 2515#ifdef CONFIG_KVM_BOOK3S_HV_EXIT_TIMING 2516 ld r4, HSTATE_KVM_VCPU(r13) 2517 addi r3, r4, VCPU_TB_CEDE 2518 bl kvmhv_accumulate_time 2519#endif 2520 2521 lis r3, LPCR_PECEDP@h /* Do wake on privileged doorbell */ 2522 2523 /* 2524 * Take a nap until a decrementer or external or doobell interrupt 2525 * occurs, with PECE1 and PECE0 set in LPCR. 2526 * On POWER8, set PECEDH, and if we are ceding, also set PECEDP. 2527 * Also clear the runlatch bit before napping. 2528 */ 2529kvm_do_nap: 2530 mfspr r0, SPRN_CTRLF 2531 clrrdi r0, r0, 1 2532 mtspr SPRN_CTRLT, r0 2533 2534BEGIN_FTR_SECTION 2535 li r0,1 2536 stb r0,HSTATE_HWTHREAD_REQ(r13) 2537END_FTR_SECTION_IFCLR(CPU_FTR_ARCH_300) 2538 mfspr r5,SPRN_LPCR 2539 ori r5,r5,LPCR_PECE0 | LPCR_PECE1 2540BEGIN_FTR_SECTION 2541 ori r5, r5, LPCR_PECEDH 2542 rlwimi r5, r3, 0, LPCR_PECEDP 2543END_FTR_SECTION_IFSET(CPU_FTR_ARCH_207S) 2544 2545kvm_nap_sequence: /* desired LPCR value in r5 */ 2546BEGIN_FTR_SECTION 2547 /* 2548 * PSSCR bits: exit criterion = 1 (wakeup based on LPCR at sreset) 2549 * enable state loss = 1 (allow SMT mode switch) 2550 * requested level = 0 (just stop dispatching) 2551 */ 2552 lis r3, (PSSCR_EC | PSSCR_ESL)@h 2553 mtspr SPRN_PSSCR, r3 2554 /* Set LPCR_PECE_HVEE bit to enable wakeup by HV interrupts */ 2555 li r4, LPCR_PECE_HVEE@higher 2556 sldi r4, r4, 32 2557 or r5, r5, r4 2558END_FTR_SECTION_IFSET(CPU_FTR_ARCH_300) 2559 mtspr SPRN_LPCR,r5 2560 isync 2561 li r0, 0 2562 std r0, HSTATE_SCRATCH0(r13) 2563 ptesync 2564 ld r0, HSTATE_SCRATCH0(r13) 25651: cmpd r0, r0 2566 bne 1b 2567BEGIN_FTR_SECTION 2568 nap 2569FTR_SECTION_ELSE 2570 PPC_STOP 2571ALT_FTR_SECTION_END_IFCLR(CPU_FTR_ARCH_300) 2572 b . 2573 257433: mr r4, r3 2575 li r3, 0 2576 li r12, 0 2577 b 34f 2578 2579kvm_end_cede: 2580 /* get vcpu pointer */ 2581 ld r4, HSTATE_KVM_VCPU(r13) 2582 2583 /* Woken by external or decrementer interrupt */ 2584 ld r1, HSTATE_HOST_R1(r13) 2585 2586#ifdef CONFIG_KVM_BOOK3S_HV_EXIT_TIMING 2587 addi r3, r4, VCPU_TB_RMINTR 2588 bl kvmhv_accumulate_time 2589#endif 2590 2591#ifdef CONFIG_PPC_TRANSACTIONAL_MEM 2592BEGIN_FTR_SECTION 2593 /* 2594 * NOTE THAT THIS TRASHES ALL NON-VOLATILE REGISTERS INCLUDING CR 2595 */ 2596 bl kvmppc_restore_tm 2597END_FTR_SECTION_IFSET(CPU_FTR_TM) 2598#endif 2599 2600 /* load up FP state */ 2601 bl kvmppc_load_fp 2602 2603 /* Restore guest decrementer */ 2604 ld r3, VCPU_DEC_EXPIRES(r4) 2605 ld r5, HSTATE_KVM_VCORE(r13) 2606 ld r6, VCORE_TB_OFFSET(r5) 2607 add r3, r3, r6 /* convert host TB to guest TB value */ 2608 mftb r7 2609 subf r3, r7, r3 2610 mtspr SPRN_DEC, r3 2611 2612 /* Load NV GPRS */ 2613 ld r14, VCPU_GPR(R14)(r4) 2614 ld r15, VCPU_GPR(R15)(r4) 2615 ld r16, VCPU_GPR(R16)(r4) 2616 ld r17, VCPU_GPR(R17)(r4) 2617 ld r18, VCPU_GPR(R18)(r4) 2618 ld r19, VCPU_GPR(R19)(r4) 2619 ld r20, VCPU_GPR(R20)(r4) 2620 ld r21, VCPU_GPR(R21)(r4) 2621 ld r22, VCPU_GPR(R22)(r4) 2622 ld r23, VCPU_GPR(R23)(r4) 2623 ld r24, VCPU_GPR(R24)(r4) 2624 ld r25, VCPU_GPR(R25)(r4) 2625 ld r26, VCPU_GPR(R26)(r4) 2626 ld r27, VCPU_GPR(R27)(r4) 2627 ld r28, VCPU_GPR(R28)(r4) 2628 ld r29, VCPU_GPR(R29)(r4) 2629 ld r30, VCPU_GPR(R30)(r4) 2630 ld r31, VCPU_GPR(R31)(r4) 2631 2632 /* Check the wake reason in SRR1 to see why we got here */ 2633 bl kvmppc_check_wake_reason 2634 2635 /* 2636 * Restore volatile registers since we could have called a 2637 * C routine in kvmppc_check_wake_reason 2638 * r4 = VCPU 2639 * r3 tells us whether we need to return to host or not 2640 * WARNING: it gets checked further down: 2641 * should not modify r3 until this check is done. 2642 */ 2643 ld r4, HSTATE_KVM_VCPU(r13) 2644 2645 /* clear our bit in vcore->napping_threads */ 264634: ld r5,HSTATE_KVM_VCORE(r13) 2647 lbz r7,HSTATE_PTID(r13) 2648 li r0,1 2649 sld r0,r0,r7 2650 addi r6,r5,VCORE_NAPPING_THREADS 265132: lwarx r7,0,r6 2652 andc r7,r7,r0 2653 stwcx. r7,0,r6 2654 bne 32b 2655 li r0,0 2656 stb r0,HSTATE_NAPPING(r13) 2657 2658 /* See if the wake reason saved in r3 means we need to exit */ 2659 stw r12, VCPU_TRAP(r4) 2660 mr r9, r4 2661 cmpdi r3, 0 2662 bgt guest_exit_cont 2663 2664 /* see if any other thread is already exiting */ 2665 lwz r0,VCORE_ENTRY_EXIT(r5) 2666 cmpwi r0,0x100 2667 bge guest_exit_cont 2668 2669 b kvmppc_cede_reentry /* if not go back to guest */ 2670 2671 /* cede when already previously prodded case */ 2672kvm_cede_prodded: 2673 li r0,0 2674 stb r0,VCPU_PRODDED(r3) 2675 sync /* order testing prodded vs. clearing ceded */ 2676 stb r0,VCPU_CEDED(r3) 2677 li r3,H_SUCCESS 2678 blr 2679 2680 /* we've ceded but we want to give control to the host */ 2681kvm_cede_exit: 2682 ld r9, HSTATE_KVM_VCPU(r13) 2683 b guest_exit_cont 2684 2685 /* Try to handle a machine check in real mode */ 2686machine_check_realmode: 2687 mr r3, r9 /* get vcpu pointer */ 2688 bl kvmppc_realmode_machine_check 2689 nop 2690 ld r9, HSTATE_KVM_VCPU(r13) 2691 li r12, BOOK3S_INTERRUPT_MACHINE_CHECK 2692 /* 2693 * For the guest that is FWNMI capable, deliver all the MCE errors 2694 * (handled/unhandled) by exiting the guest with KVM_EXIT_NMI exit 2695 * reason. This new approach injects machine check errors in guest 2696 * address space to guest with additional information in the form 2697 * of RTAS event, thus enabling guest kernel to suitably handle 2698 * such errors. 2699 * 2700 * For the guest that is not FWNMI capable (old QEMU) fallback 2701 * to old behaviour for backward compatibility: 2702 * Deliver unhandled/fatal (e.g. UE) MCE errors to guest either 2703 * through machine check interrupt (set HSRR0 to 0x200). 2704 * For handled errors (no-fatal), just go back to guest execution 2705 * with current HSRR0. 2706 * if we receive machine check with MSR(RI=0) then deliver it to 2707 * guest as machine check causing guest to crash. 2708 */ 2709 ld r11, VCPU_MSR(r9) 2710 rldicl. r0, r11, 64-MSR_HV_LG, 63 /* check if it happened in HV mode */ 2711 bne mc_cont /* if so, exit to host */ 2712 /* Check if guest is capable of handling NMI exit */ 2713 ld r10, VCPU_KVM(r9) 2714 lbz r10, KVM_FWNMI(r10) 2715 cmpdi r10, 1 /* FWNMI capable? */ 2716 beq mc_cont /* if so, exit with KVM_EXIT_NMI. */ 2717 2718 /* if not, fall through for backward compatibility. */ 2719 andi. r10, r11, MSR_RI /* check for unrecoverable exception */ 2720 beq 1f /* Deliver a machine check to guest */ 2721 ld r10, VCPU_PC(r9) 2722 cmpdi r3, 0 /* Did we handle MCE ? */ 2723 bne 2f /* Continue guest execution. */ 2724 /* If not, deliver a machine check. SRR0/1 are already set */ 27251: li r10, BOOK3S_INTERRUPT_MACHINE_CHECK 2726 bl kvmppc_msr_interrupt 27272: b fast_interrupt_c_return 2728 2729/* 2730 * Check the reason we woke from nap, and take appropriate action. 2731 * Returns (in r3): 2732 * 0 if nothing needs to be done 2733 * 1 if something happened that needs to be handled by the host 2734 * -1 if there was a guest wakeup (IPI or msgsnd) 2735 * -2 if we handled a PCI passthrough interrupt (returned by 2736 * kvmppc_read_intr only) 2737 * 2738 * Also sets r12 to the interrupt vector for any interrupt that needs 2739 * to be handled now by the host (0x500 for external interrupt), or zero. 2740 * Modifies all volatile registers (since it may call a C function). 2741 * This routine calls kvmppc_read_intr, a C function, if an external 2742 * interrupt is pending. 2743 */ 2744kvmppc_check_wake_reason: 2745 mfspr r6, SPRN_SRR1 2746BEGIN_FTR_SECTION 2747 rlwinm r6, r6, 45-31, 0xf /* extract wake reason field (P8) */ 2748FTR_SECTION_ELSE 2749 rlwinm r6, r6, 45-31, 0xe /* P7 wake reason field is 3 bits */ 2750ALT_FTR_SECTION_END_IFSET(CPU_FTR_ARCH_207S) 2751 cmpwi r6, 8 /* was it an external interrupt? */ 2752 beq 7f /* if so, see what it was */ 2753 li r3, 0 2754 li r12, 0 2755 cmpwi r6, 6 /* was it the decrementer? */ 2756 beq 0f 2757BEGIN_FTR_SECTION 2758 cmpwi r6, 5 /* privileged doorbell? */ 2759 beq 0f 2760 cmpwi r6, 3 /* hypervisor doorbell? */ 2761 beq 3f 2762END_FTR_SECTION_IFSET(CPU_FTR_ARCH_207S) 2763 cmpwi r6, 0xa /* Hypervisor maintenance ? */ 2764 beq 4f 2765 li r3, 1 /* anything else, return 1 */ 27660: blr 2767 2768 /* hypervisor doorbell */ 27693: li r12, BOOK3S_INTERRUPT_H_DOORBELL 2770 2771 /* 2772 * Clear the doorbell as we will invoke the handler 2773 * explicitly in the guest exit path. 2774 */ 2775 lis r6, (PPC_DBELL_SERVER << (63-36))@h 2776 PPC_MSGCLR(6) 2777 /* see if it's a host IPI */ 2778 li r3, 1 2779 lbz r0, HSTATE_HOST_IPI(r13) 2780 cmpwi r0, 0 2781 bnelr 2782 /* if not, return -1 */ 2783 li r3, -1 2784 blr 2785 2786 /* Woken up due to Hypervisor maintenance interrupt */ 27874: li r12, BOOK3S_INTERRUPT_HMI 2788 li r3, 1 2789 blr 2790 2791 /* external interrupt - create a stack frame so we can call C */ 27927: mflr r0 2793 std r0, PPC_LR_STKOFF(r1) 2794 stdu r1, -PPC_MIN_STKFRM(r1) 2795 bl kvmppc_read_intr 2796 nop 2797 li r12, BOOK3S_INTERRUPT_EXTERNAL 2798 cmpdi r3, 1 2799 ble 1f 2800 2801 /* 2802 * Return code of 2 means PCI passthrough interrupt, but 2803 * we need to return back to host to complete handling the 2804 * interrupt. Trap reason is expected in r12 by guest 2805 * exit code. 2806 */ 2807 li r12, BOOK3S_INTERRUPT_HV_RM_HARD 28081: 2809 ld r0, PPC_MIN_STKFRM+PPC_LR_STKOFF(r1) 2810 addi r1, r1, PPC_MIN_STKFRM 2811 mtlr r0 2812 blr 2813 2814/* 2815 * Save away FP, VMX and VSX registers. 2816 * r3 = vcpu pointer 2817 * N.B. r30 and r31 are volatile across this function, 2818 * thus it is not callable from C. 2819 */ 2820kvmppc_save_fp: 2821 mflr r30 2822 mr r31,r3 2823 mfmsr r5 2824 ori r8,r5,MSR_FP 2825#ifdef CONFIG_ALTIVEC 2826BEGIN_FTR_SECTION 2827 oris r8,r8,MSR_VEC@h 2828END_FTR_SECTION_IFSET(CPU_FTR_ALTIVEC) 2829#endif 2830#ifdef CONFIG_VSX 2831BEGIN_FTR_SECTION 2832 oris r8,r8,MSR_VSX@h 2833END_FTR_SECTION_IFSET(CPU_FTR_VSX) 2834#endif 2835 mtmsrd r8 2836 addi r3,r3,VCPU_FPRS 2837 bl store_fp_state 2838#ifdef CONFIG_ALTIVEC 2839BEGIN_FTR_SECTION 2840 addi r3,r31,VCPU_VRS 2841 bl store_vr_state 2842END_FTR_SECTION_IFSET(CPU_FTR_ALTIVEC) 2843#endif 2844 mfspr r6,SPRN_VRSAVE 2845 stw r6,VCPU_VRSAVE(r31) 2846 mtlr r30 2847 blr 2848 2849/* 2850 * Load up FP, VMX and VSX registers 2851 * r4 = vcpu pointer 2852 * N.B. r30 and r31 are volatile across this function, 2853 * thus it is not callable from C. 2854 */ 2855kvmppc_load_fp: 2856 mflr r30 2857 mr r31,r4 2858 mfmsr r9 2859 ori r8,r9,MSR_FP 2860#ifdef CONFIG_ALTIVEC 2861BEGIN_FTR_SECTION 2862 oris r8,r8,MSR_VEC@h 2863END_FTR_SECTION_IFSET(CPU_FTR_ALTIVEC) 2864#endif 2865#ifdef CONFIG_VSX 2866BEGIN_FTR_SECTION 2867 oris r8,r8,MSR_VSX@h 2868END_FTR_SECTION_IFSET(CPU_FTR_VSX) 2869#endif 2870 mtmsrd r8 2871 addi r3,r4,VCPU_FPRS 2872 bl load_fp_state 2873#ifdef CONFIG_ALTIVEC 2874BEGIN_FTR_SECTION 2875 addi r3,r31,VCPU_VRS 2876 bl load_vr_state 2877END_FTR_SECTION_IFSET(CPU_FTR_ALTIVEC) 2878#endif 2879 lwz r7,VCPU_VRSAVE(r31) 2880 mtspr SPRN_VRSAVE,r7 2881 mtlr r30 2882 mr r4,r31 2883 blr 2884 2885#ifdef CONFIG_PPC_TRANSACTIONAL_MEM 2886/* 2887 * Save transactional state and TM-related registers. 2888 * Called with r9 pointing to the vcpu struct. 2889 * This can modify all checkpointed registers, but 2890 * restores r1, r2 and r9 (vcpu pointer) before exit. 2891 */ 2892kvmppc_save_tm: 2893 mflr r0 2894 std r0, PPC_LR_STKOFF(r1) 2895 2896 /* Turn on TM. */ 2897 mfmsr r8 2898 li r0, 1 2899 rldimi r8, r0, MSR_TM_LG, 63-MSR_TM_LG 2900 mtmsrd r8 2901 2902 ld r5, VCPU_MSR(r9) 2903 rldicl. r5, r5, 64 - MSR_TS_S_LG, 62 2904 beq 1f /* TM not active in guest. */ 2905 2906 std r1, HSTATE_HOST_R1(r13) 2907 li r3, TM_CAUSE_KVM_RESCHED 2908 2909 /* Clear the MSR RI since r1, r13 are all going to be foobar. */ 2910 li r5, 0 2911 mtmsrd r5, 1 2912 2913 /* All GPRs are volatile at this point. */ 2914 TRECLAIM(R3) 2915 2916 /* Temporarily store r13 and r9 so we have some regs to play with */ 2917 SET_SCRATCH0(r13) 2918 GET_PACA(r13) 2919 std r9, PACATMSCRATCH(r13) 2920 ld r9, HSTATE_KVM_VCPU(r13) 2921 2922 /* Get a few more GPRs free. */ 2923 std r29, VCPU_GPRS_TM(29)(r9) 2924 std r30, VCPU_GPRS_TM(30)(r9) 2925 std r31, VCPU_GPRS_TM(31)(r9) 2926 2927 /* Save away PPR and DSCR soon so don't run with user values. */ 2928 mfspr r31, SPRN_PPR 2929 HMT_MEDIUM 2930 mfspr r30, SPRN_DSCR 2931 ld r29, HSTATE_DSCR(r13) 2932 mtspr SPRN_DSCR, r29 2933 2934 /* Save all but r9, r13 & r29-r31 */ 2935 reg = 0 2936 .rept 29 2937 .if (reg != 9) && (reg != 13) 2938 std reg, VCPU_GPRS_TM(reg)(r9) 2939 .endif 2940 reg = reg + 1 2941 .endr 2942 /* ... now save r13 */ 2943 GET_SCRATCH0(r4) 2944 std r4, VCPU_GPRS_TM(13)(r9) 2945 /* ... and save r9 */ 2946 ld r4, PACATMSCRATCH(r13) 2947 std r4, VCPU_GPRS_TM(9)(r9) 2948 2949 /* Reload stack pointer and TOC. */ 2950 ld r1, HSTATE_HOST_R1(r13) 2951 ld r2, PACATOC(r13) 2952 2953 /* Set MSR RI now we have r1 and r13 back. */ 2954 li r5, MSR_RI 2955 mtmsrd r5, 1 2956 2957 /* Save away checkpinted SPRs. */ 2958 std r31, VCPU_PPR_TM(r9) 2959 std r30, VCPU_DSCR_TM(r9) 2960 mflr r5 2961 mfcr r6 2962 mfctr r7 2963 mfspr r8, SPRN_AMR 2964 mfspr r10, SPRN_TAR 2965 mfxer r11 2966 std r5, VCPU_LR_TM(r9) 2967 stw r6, VCPU_CR_TM(r9) 2968 std r7, VCPU_CTR_TM(r9) 2969 std r8, VCPU_AMR_TM(r9) 2970 std r10, VCPU_TAR_TM(r9) 2971 std r11, VCPU_XER_TM(r9) 2972 2973 /* Restore r12 as trap number. */ 2974 lwz r12, VCPU_TRAP(r9) 2975 2976 /* Save FP/VSX. */ 2977 addi r3, r9, VCPU_FPRS_TM 2978 bl store_fp_state 2979 addi r3, r9, VCPU_VRS_TM 2980 bl store_vr_state 2981 mfspr r6, SPRN_VRSAVE 2982 stw r6, VCPU_VRSAVE_TM(r9) 29831: 2984 /* 2985 * We need to save these SPRs after the treclaim so that the software 2986 * error code is recorded correctly in the TEXASR. Also the user may 2987 * change these outside of a transaction, so they must always be 2988 * context switched. 2989 */ 2990 mfspr r5, SPRN_TFHAR 2991 mfspr r6, SPRN_TFIAR 2992 mfspr r7, SPRN_TEXASR 2993 std r5, VCPU_TFHAR(r9) 2994 std r6, VCPU_TFIAR(r9) 2995 std r7, VCPU_TEXASR(r9) 2996 2997 ld r0, PPC_LR_STKOFF(r1) 2998 mtlr r0 2999 blr 3000 3001/* 3002 * Restore transactional state and TM-related registers. 3003 * Called with r4 pointing to the vcpu struct. 3004 * This potentially modifies all checkpointed registers. 3005 * It restores r1, r2, r4 from the PACA. 3006 */ 3007kvmppc_restore_tm: 3008 mflr r0 3009 std r0, PPC_LR_STKOFF(r1) 3010 3011 /* Turn on TM/FP/VSX/VMX so we can restore them. */ 3012 mfmsr r5 3013 li r6, MSR_TM >> 32 3014 sldi r6, r6, 32 3015 or r5, r5, r6 3016 ori r5, r5, MSR_FP 3017 oris r5, r5, (MSR_VEC | MSR_VSX)@h 3018 mtmsrd r5 3019 3020 /* 3021 * The user may change these outside of a transaction, so they must 3022 * always be context switched. 3023 */ 3024 ld r5, VCPU_TFHAR(r4) 3025 ld r6, VCPU_TFIAR(r4) 3026 ld r7, VCPU_TEXASR(r4) 3027 mtspr SPRN_TFHAR, r5 3028 mtspr SPRN_TFIAR, r6 3029 mtspr SPRN_TEXASR, r7 3030 3031 ld r5, VCPU_MSR(r4) 3032 rldicl. r5, r5, 64 - MSR_TS_S_LG, 62 3033 beqlr /* TM not active in guest */ 3034 std r1, HSTATE_HOST_R1(r13) 3035 3036 /* Make sure the failure summary is set, otherwise we'll program check 3037 * when we trechkpt. It's possible that this might have been not set 3038 * on a kvmppc_set_one_reg() call but we shouldn't let this crash the 3039 * host. 3040 */ 3041 oris r7, r7, (TEXASR_FS)@h 3042 mtspr SPRN_TEXASR, r7 3043 3044 /* 3045 * We need to load up the checkpointed state for the guest. 3046 * We need to do this early as it will blow away any GPRs, VSRs and 3047 * some SPRs. 3048 */ 3049 3050 mr r31, r4 3051 addi r3, r31, VCPU_FPRS_TM 3052 bl load_fp_state 3053 addi r3, r31, VCPU_VRS_TM 3054 bl load_vr_state 3055 mr r4, r31 3056 lwz r7, VCPU_VRSAVE_TM(r4) 3057 mtspr SPRN_VRSAVE, r7 3058 3059 ld r5, VCPU_LR_TM(r4) 3060 lwz r6, VCPU_CR_TM(r4) 3061 ld r7, VCPU_CTR_TM(r4) 3062 ld r8, VCPU_AMR_TM(r4) 3063 ld r9, VCPU_TAR_TM(r4) 3064 ld r10, VCPU_XER_TM(r4) 3065 mtlr r5 3066 mtcr r6 3067 mtctr r7 3068 mtspr SPRN_AMR, r8 3069 mtspr SPRN_TAR, r9 3070 mtxer r10 3071 3072 /* 3073 * Load up PPR and DSCR values but don't put them in the actual SPRs 3074 * till the last moment to avoid running with userspace PPR and DSCR for 3075 * too long. 3076 */ 3077 ld r29, VCPU_DSCR_TM(r4) 3078 ld r30, VCPU_PPR_TM(r4) 3079 3080 std r2, PACATMSCRATCH(r13) /* Save TOC */ 3081 3082 /* Clear the MSR RI since r1, r13 are all going to be foobar. */ 3083 li r5, 0 3084 mtmsrd r5, 1 3085 3086 /* Load GPRs r0-r28 */ 3087 reg = 0 3088 .rept 29 3089 ld reg, VCPU_GPRS_TM(reg)(r31) 3090 reg = reg + 1 3091 .endr 3092 3093 mtspr SPRN_DSCR, r29 3094 mtspr SPRN_PPR, r30 3095 3096 /* Load final GPRs */ 3097 ld 29, VCPU_GPRS_TM(29)(r31) 3098 ld 30, VCPU_GPRS_TM(30)(r31) 3099 ld 31, VCPU_GPRS_TM(31)(r31) 3100 3101 /* TM checkpointed state is now setup. All GPRs are now volatile. */ 3102 TRECHKPT 3103 3104 /* Now let's get back the state we need. */ 3105 HMT_MEDIUM 3106 GET_PACA(r13) 3107 ld r29, HSTATE_DSCR(r13) 3108 mtspr SPRN_DSCR, r29 3109 ld r4, HSTATE_KVM_VCPU(r13) 3110 ld r1, HSTATE_HOST_R1(r13) 3111 ld r2, PACATMSCRATCH(r13) 3112 3113 /* Set the MSR RI since we have our registers back. */ 3114 li r5, MSR_RI 3115 mtmsrd r5, 1 3116 3117 ld r0, PPC_LR_STKOFF(r1) 3118 mtlr r0 3119 blr 3120#endif 3121 3122/* 3123 * We come here if we get any exception or interrupt while we are 3124 * executing host real mode code while in guest MMU context. 3125 * For now just spin, but we should do something better. 3126 */ 3127kvmppc_bad_host_intr: 3128 b . 3129 3130/* 3131 * This mimics the MSR transition on IRQ delivery. The new guest MSR is taken 3132 * from VCPU_INTR_MSR and is modified based on the required TM state changes. 3133 * r11 has the guest MSR value (in/out) 3134 * r9 has a vcpu pointer (in) 3135 * r0 is used as a scratch register 3136 */ 3137kvmppc_msr_interrupt: 3138 rldicl r0, r11, 64 - MSR_TS_S_LG, 62 3139 cmpwi r0, 2 /* Check if we are in transactional state.. */ 3140 ld r11, VCPU_INTR_MSR(r9) 3141 bne 1f 3142 /* ... if transactional, change to suspended */ 3143 li r0, 1 31441: rldimi r11, r0, MSR_TS_S_LG, 63 - MSR_TS_T_LG 3145 blr 3146 3147/* 3148 * This works around a hardware bug on POWER8E processors, where 3149 * writing a 1 to the MMCR0[PMAO] bit doesn't generate a 3150 * performance monitor interrupt. Instead, when we need to have 3151 * an interrupt pending, we have to arrange for a counter to overflow. 3152 */ 3153kvmppc_fix_pmao: 3154 li r3, 0 3155 mtspr SPRN_MMCR2, r3 3156 lis r3, (MMCR0_PMXE | MMCR0_FCECE)@h 3157 ori r3, r3, MMCR0_PMCjCE | MMCR0_C56RUN 3158 mtspr SPRN_MMCR0, r3 3159 lis r3, 0x7fff 3160 ori r3, r3, 0xffff 3161 mtspr SPRN_PMC6, r3 3162 isync 3163 blr 3164 3165#ifdef CONFIG_KVM_BOOK3S_HV_EXIT_TIMING 3166/* 3167 * Start timing an activity 3168 * r3 = pointer to time accumulation struct, r4 = vcpu 3169 */ 3170kvmhv_start_timing: 3171 ld r5, HSTATE_KVM_VCORE(r13) 3172 lbz r6, VCORE_IN_GUEST(r5) 3173 cmpwi r6, 0 3174 beq 5f /* if in guest, need to */ 3175 ld r6, VCORE_TB_OFFSET(r5) /* subtract timebase offset */ 31765: mftb r5 3177 subf r5, r6, r5 3178 std r3, VCPU_CUR_ACTIVITY(r4) 3179 std r5, VCPU_ACTIVITY_START(r4) 3180 blr 3181 3182/* 3183 * Accumulate time to one activity and start another. 3184 * r3 = pointer to new time accumulation struct, r4 = vcpu 3185 */ 3186kvmhv_accumulate_time: 3187 ld r5, HSTATE_KVM_VCORE(r13) 3188 lbz r8, VCORE_IN_GUEST(r5) 3189 cmpwi r8, 0 3190 beq 4f /* if in guest, need to */ 3191 ld r8, VCORE_TB_OFFSET(r5) /* subtract timebase offset */ 31924: ld r5, VCPU_CUR_ACTIVITY(r4) 3193 ld r6, VCPU_ACTIVITY_START(r4) 3194 std r3, VCPU_CUR_ACTIVITY(r4) 3195 mftb r7 3196 subf r7, r8, r7 3197 std r7, VCPU_ACTIVITY_START(r4) 3198 cmpdi r5, 0 3199 beqlr 3200 subf r3, r6, r7 3201 ld r8, TAS_SEQCOUNT(r5) 3202 cmpdi r8, 0 3203 addi r8, r8, 1 3204 std r8, TAS_SEQCOUNT(r5) 3205 lwsync 3206 ld r7, TAS_TOTAL(r5) 3207 add r7, r7, r3 3208 std r7, TAS_TOTAL(r5) 3209 ld r6, TAS_MIN(r5) 3210 ld r7, TAS_MAX(r5) 3211 beq 3f 3212 cmpd r3, r6 3213 bge 1f 32143: std r3, TAS_MIN(r5) 32151: cmpd r3, r7 3216 ble 2f 3217 std r3, TAS_MAX(r5) 32182: lwsync 3219 addi r8, r8, 1 3220 std r8, TAS_SEQCOUNT(r5) 3221 blr 3222#endif 3223