1/* 2 * This program is free software; you can redistribute it and/or modify 3 * it under the terms of the GNU General Public License, version 2, as 4 * published by the Free Software Foundation. 5 * 6 * This program is distributed in the hope that it will be useful, 7 * but WITHOUT ANY WARRANTY; without even the implied warranty of 8 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 9 * GNU General Public License for more details. 10 * 11 * Copyright 2011 Paul Mackerras, IBM Corp. <paulus@au1.ibm.com> 12 * 13 * Derived from book3s_rmhandlers.S and other files, which are: 14 * 15 * Copyright SUSE Linux Products GmbH 2009 16 * 17 * Authors: Alexander Graf <agraf@suse.de> 18 */ 19 20#include <asm/ppc_asm.h> 21#include <asm/kvm_asm.h> 22#include <asm/reg.h> 23#include <asm/mmu.h> 24#include <asm/page.h> 25#include <asm/ptrace.h> 26#include <asm/hvcall.h> 27#include <asm/asm-offsets.h> 28#include <asm/exception-64s.h> 29#include <asm/kvm_book3s_asm.h> 30#include <asm/book3s/64/mmu-hash.h> 31#include <asm/export.h> 32#include <asm/tm.h> 33#include <asm/opal.h> 34#include <asm/xive-regs.h> 35#include <asm/thread_info.h> 36#include <asm/asm-compat.h> 37#include <asm/feature-fixups.h> 38#include <asm/cpuidle.h> 39 40/* Sign-extend HDEC if not on POWER9 */ 41#define EXTEND_HDEC(reg) \ 42BEGIN_FTR_SECTION; \ 43 extsw reg, reg; \ 44END_FTR_SECTION_IFCLR(CPU_FTR_ARCH_300) 45 46/* Values in HSTATE_NAPPING(r13) */ 47#define NAPPING_CEDE 1 48#define NAPPING_NOVCPU 2 49#define NAPPING_UNSPLIT 3 50 51/* Stack frame offsets for kvmppc_hv_entry */ 52#define SFS 208 53#define STACK_SLOT_TRAP (SFS-4) 54#define STACK_SLOT_SHORT_PATH (SFS-8) 55#define STACK_SLOT_TID (SFS-16) 56#define STACK_SLOT_PSSCR (SFS-24) 57#define STACK_SLOT_PID (SFS-32) 58#define STACK_SLOT_IAMR (SFS-40) 59#define STACK_SLOT_CIABR (SFS-48) 60#define STACK_SLOT_DAWR (SFS-56) 61#define STACK_SLOT_DAWRX (SFS-64) 62#define STACK_SLOT_HFSCR (SFS-72) 63#define STACK_SLOT_AMR (SFS-80) 64#define STACK_SLOT_UAMOR (SFS-88) 65/* the following is used by the P9 short path */ 66#define STACK_SLOT_NVGPRS (SFS-152) /* 18 gprs */ 67 68/* 69 * Call kvmppc_hv_entry in real mode. 70 * Must be called with interrupts hard-disabled. 71 * 72 * Input Registers: 73 * 74 * LR = return address to continue at after eventually re-enabling MMU 75 */ 76_GLOBAL_TOC(kvmppc_hv_entry_trampoline) 77 mflr r0 78 std r0, PPC_LR_STKOFF(r1) 79 stdu r1, -112(r1) 80 mfmsr r10 81 std r10, HSTATE_HOST_MSR(r13) 82 LOAD_REG_ADDR(r5, kvmppc_call_hv_entry) 83 li r0,MSR_RI 84 andc r0,r10,r0 85 li r6,MSR_IR | MSR_DR 86 andc r6,r10,r6 87 mtmsrd r0,1 /* clear RI in MSR */ 88 mtsrr0 r5 89 mtsrr1 r6 90 RFI_TO_KERNEL 91 92kvmppc_call_hv_entry: 93BEGIN_FTR_SECTION 94 /* On P9, do LPCR setting, if necessary */ 95 ld r3, HSTATE_SPLIT_MODE(r13) 96 cmpdi r3, 0 97 beq 46f 98 lwz r4, KVM_SPLIT_DO_SET(r3) 99 cmpwi r4, 0 100 beq 46f 101 bl kvmhv_p9_set_lpcr 102 nop 10346: 104END_FTR_SECTION_IFSET(CPU_FTR_ARCH_300) 105 106 ld r4, HSTATE_KVM_VCPU(r13) 107 bl kvmppc_hv_entry 108 109 /* Back from guest - restore host state and return to caller */ 110 111BEGIN_FTR_SECTION 112 /* Restore host DABR and DABRX */ 113 ld r5,HSTATE_DABR(r13) 114 li r6,7 115 mtspr SPRN_DABR,r5 116 mtspr SPRN_DABRX,r6 117END_FTR_SECTION_IFCLR(CPU_FTR_ARCH_207S) 118 119 /* Restore SPRG3 */ 120 ld r3,PACA_SPRG_VDSO(r13) 121 mtspr SPRN_SPRG_VDSO_WRITE,r3 122 123 /* Reload the host's PMU registers */ 124 bl kvmhv_load_host_pmu 125 126 /* 127 * Reload DEC. HDEC interrupts were disabled when 128 * we reloaded the host's LPCR value. 129 */ 130 ld r3, HSTATE_DECEXP(r13) 131 mftb r4 132 subf r4, r4, r3 133 mtspr SPRN_DEC, r4 134 135 /* hwthread_req may have got set by cede or no vcpu, so clear it */ 136 li r0, 0 137 stb r0, HSTATE_HWTHREAD_REQ(r13) 138 139 /* 140 * For external interrupts we need to call the Linux 141 * handler to process the interrupt. We do that by jumping 142 * to absolute address 0x500 for external interrupts. 143 * The [h]rfid at the end of the handler will return to 144 * the book3s_hv_interrupts.S code. For other interrupts 145 * we do the rfid to get back to the book3s_hv_interrupts.S 146 * code here. 147 */ 148 ld r8, 112+PPC_LR_STKOFF(r1) 149 addi r1, r1, 112 150 ld r7, HSTATE_HOST_MSR(r13) 151 152 /* Return the trap number on this thread as the return value */ 153 mr r3, r12 154 155 /* 156 * If we came back from the guest via a relocation-on interrupt, 157 * we will be in virtual mode at this point, which makes it a 158 * little easier to get back to the caller. 159 */ 160 mfmsr r0 161 andi. r0, r0, MSR_IR /* in real mode? */ 162 bne .Lvirt_return 163 164 /* RFI into the highmem handler */ 165 mfmsr r6 166 li r0, MSR_RI 167 andc r6, r6, r0 168 mtmsrd r6, 1 /* Clear RI in MSR */ 169 mtsrr0 r8 170 mtsrr1 r7 171 RFI_TO_KERNEL 172 173 /* Virtual-mode return */ 174.Lvirt_return: 175 mtlr r8 176 blr 177 178kvmppc_primary_no_guest: 179 /* We handle this much like a ceded vcpu */ 180 /* put the HDEC into the DEC, since HDEC interrupts don't wake us */ 181 /* HDEC may be larger than DEC for arch >= v3.00, but since the */ 182 /* HDEC value came from DEC in the first place, it will fit */ 183 mfspr r3, SPRN_HDEC 184 mtspr SPRN_DEC, r3 185 /* 186 * Make sure the primary has finished the MMU switch. 187 * We should never get here on a secondary thread, but 188 * check it for robustness' sake. 189 */ 190 ld r5, HSTATE_KVM_VCORE(r13) 19165: lbz r0, VCORE_IN_GUEST(r5) 192 cmpwi r0, 0 193 beq 65b 194 /* Set LPCR. */ 195 ld r8,VCORE_LPCR(r5) 196 mtspr SPRN_LPCR,r8 197 isync 198 /* set our bit in napping_threads */ 199 ld r5, HSTATE_KVM_VCORE(r13) 200 lbz r7, HSTATE_PTID(r13) 201 li r0, 1 202 sld r0, r0, r7 203 addi r6, r5, VCORE_NAPPING_THREADS 2041: lwarx r3, 0, r6 205 or r3, r3, r0 206 stwcx. r3, 0, r6 207 bne 1b 208 /* order napping_threads update vs testing entry_exit_map */ 209 isync 210 li r12, 0 211 lwz r7, VCORE_ENTRY_EXIT(r5) 212 cmpwi r7, 0x100 213 bge kvm_novcpu_exit /* another thread already exiting */ 214 li r3, NAPPING_NOVCPU 215 stb r3, HSTATE_NAPPING(r13) 216 217 li r3, 0 /* Don't wake on privileged (OS) doorbell */ 218 b kvm_do_nap 219 220/* 221 * kvm_novcpu_wakeup 222 * Entered from kvm_start_guest if kvm_hstate.napping is set 223 * to NAPPING_NOVCPU 224 * r2 = kernel TOC 225 * r13 = paca 226 */ 227kvm_novcpu_wakeup: 228 ld r1, HSTATE_HOST_R1(r13) 229 ld r5, HSTATE_KVM_VCORE(r13) 230 li r0, 0 231 stb r0, HSTATE_NAPPING(r13) 232 233 /* check the wake reason */ 234 bl kvmppc_check_wake_reason 235 236 /* 237 * Restore volatile registers since we could have called 238 * a C routine in kvmppc_check_wake_reason. 239 * r5 = VCORE 240 */ 241 ld r5, HSTATE_KVM_VCORE(r13) 242 243 /* see if any other thread is already exiting */ 244 lwz r0, VCORE_ENTRY_EXIT(r5) 245 cmpwi r0, 0x100 246 bge kvm_novcpu_exit 247 248 /* clear our bit in napping_threads */ 249 lbz r7, HSTATE_PTID(r13) 250 li r0, 1 251 sld r0, r0, r7 252 addi r6, r5, VCORE_NAPPING_THREADS 2534: lwarx r7, 0, r6 254 andc r7, r7, r0 255 stwcx. r7, 0, r6 256 bne 4b 257 258 /* See if the wake reason means we need to exit */ 259 cmpdi r3, 0 260 bge kvm_novcpu_exit 261 262 /* See if our timeslice has expired (HDEC is negative) */ 263 mfspr r0, SPRN_HDEC 264 EXTEND_HDEC(r0) 265 li r12, BOOK3S_INTERRUPT_HV_DECREMENTER 266 cmpdi r0, 0 267 blt kvm_novcpu_exit 268 269 /* Got an IPI but other vcpus aren't yet exiting, must be a latecomer */ 270 ld r4, HSTATE_KVM_VCPU(r13) 271 cmpdi r4, 0 272 beq kvmppc_primary_no_guest 273 274#ifdef CONFIG_KVM_BOOK3S_HV_EXIT_TIMING 275 addi r3, r4, VCPU_TB_RMENTRY 276 bl kvmhv_start_timing 277#endif 278 b kvmppc_got_guest 279 280kvm_novcpu_exit: 281#ifdef CONFIG_KVM_BOOK3S_HV_EXIT_TIMING 282 ld r4, HSTATE_KVM_VCPU(r13) 283 cmpdi r4, 0 284 beq 13f 285 addi r3, r4, VCPU_TB_RMEXIT 286 bl kvmhv_accumulate_time 287#endif 28813: mr r3, r12 289 stw r12, STACK_SLOT_TRAP(r1) 290 bl kvmhv_commence_exit 291 nop 292 b kvmhv_switch_to_host 293 294/* 295 * We come in here when wakened from Linux offline idle code. 296 * Relocation is off 297 * r3 contains the SRR1 wakeup value, SRR1 is trashed. 298 */ 299_GLOBAL(idle_kvm_start_guest) 300 ld r4,PACAEMERGSP(r13) 301 mfcr r5 302 mflr r0 303 std r1,0(r4) 304 std r5,8(r4) 305 std r0,16(r4) 306 subi r1,r4,STACK_FRAME_OVERHEAD 307 SAVE_NVGPRS(r1) 308 309 /* 310 * Could avoid this and pass it through in r3. For now, 311 * code expects it to be in SRR1. 312 */ 313 mtspr SPRN_SRR1,r3 314 315 li r0,0 316 stb r0,PACA_FTRACE_ENABLED(r13) 317 318 li r0,KVM_HWTHREAD_IN_KVM 319 stb r0,HSTATE_HWTHREAD_STATE(r13) 320 321 /* kvm cede / napping does not come through here */ 322 lbz r0,HSTATE_NAPPING(r13) 323 twnei r0,0 324 325 b 1f 326 327kvm_unsplit_wakeup: 328 li r0, 0 329 stb r0, HSTATE_NAPPING(r13) 330 3311: 332 333 /* 334 * We weren't napping due to cede, so this must be a secondary 335 * thread being woken up to run a guest, or being woken up due 336 * to a stray IPI. (Or due to some machine check or hypervisor 337 * maintenance interrupt while the core is in KVM.) 338 */ 339 340 /* Check the wake reason in SRR1 to see why we got here */ 341 bl kvmppc_check_wake_reason 342 /* 343 * kvmppc_check_wake_reason could invoke a C routine, but we 344 * have no volatile registers to restore when we return. 345 */ 346 347 cmpdi r3, 0 348 bge kvm_no_guest 349 350 /* get vcore pointer, NULL if we have nothing to run */ 351 ld r5,HSTATE_KVM_VCORE(r13) 352 cmpdi r5,0 353 /* if we have no vcore to run, go back to sleep */ 354 beq kvm_no_guest 355 356kvm_secondary_got_guest: 357 358 /* Set HSTATE_DSCR(r13) to something sensible */ 359 ld r6, PACA_DSCR_DEFAULT(r13) 360 std r6, HSTATE_DSCR(r13) 361 362 /* On thread 0 of a subcore, set HDEC to max */ 363 lbz r4, HSTATE_PTID(r13) 364 cmpwi r4, 0 365 bne 63f 366 LOAD_REG_ADDR(r6, decrementer_max) 367 ld r6, 0(r6) 368 mtspr SPRN_HDEC, r6 369 /* and set per-LPAR registers, if doing dynamic micro-threading */ 370 ld r6, HSTATE_SPLIT_MODE(r13) 371 cmpdi r6, 0 372 beq 63f 373BEGIN_FTR_SECTION 374 ld r0, KVM_SPLIT_RPR(r6) 375 mtspr SPRN_RPR, r0 376 ld r0, KVM_SPLIT_PMMAR(r6) 377 mtspr SPRN_PMMAR, r0 378 ld r0, KVM_SPLIT_LDBAR(r6) 379 mtspr SPRN_LDBAR, r0 380 isync 381FTR_SECTION_ELSE 382 /* On P9 we use the split_info for coordinating LPCR changes */ 383 lwz r4, KVM_SPLIT_DO_SET(r6) 384 cmpwi r4, 0 385 beq 1f 386 mr r3, r6 387 bl kvmhv_p9_set_lpcr 388 nop 3891: 390ALT_FTR_SECTION_END_IFCLR(CPU_FTR_ARCH_300) 39163: 392 /* Order load of vcpu after load of vcore */ 393 lwsync 394 ld r4, HSTATE_KVM_VCPU(r13) 395 bl kvmppc_hv_entry 396 397 /* Back from the guest, go back to nap */ 398 /* Clear our vcpu and vcore pointers so we don't come back in early */ 399 li r0, 0 400 std r0, HSTATE_KVM_VCPU(r13) 401 /* 402 * Once we clear HSTATE_KVM_VCORE(r13), the code in 403 * kvmppc_run_core() is going to assume that all our vcpu 404 * state is visible in memory. This lwsync makes sure 405 * that that is true. 406 */ 407 lwsync 408 std r0, HSTATE_KVM_VCORE(r13) 409 410 /* 411 * All secondaries exiting guest will fall through this path. 412 * Before proceeding, just check for HMI interrupt and 413 * invoke opal hmi handler. By now we are sure that the 414 * primary thread on this core/subcore has already made partition 415 * switch/TB resync and we are good to call opal hmi handler. 416 */ 417 cmpwi r12, BOOK3S_INTERRUPT_HMI 418 bne kvm_no_guest 419 420 li r3,0 /* NULL argument */ 421 bl hmi_exception_realmode 422/* 423 * At this point we have finished executing in the guest. 424 * We need to wait for hwthread_req to become zero, since 425 * we may not turn on the MMU while hwthread_req is non-zero. 426 * While waiting we also need to check if we get given a vcpu to run. 427 */ 428kvm_no_guest: 429 lbz r3, HSTATE_HWTHREAD_REQ(r13) 430 cmpwi r3, 0 431 bne 53f 432 HMT_MEDIUM 433 li r0, KVM_HWTHREAD_IN_KERNEL 434 stb r0, HSTATE_HWTHREAD_STATE(r13) 435 /* need to recheck hwthread_req after a barrier, to avoid race */ 436 sync 437 lbz r3, HSTATE_HWTHREAD_REQ(r13) 438 cmpwi r3, 0 439 bne 54f 440 441 /* 442 * Jump to idle_return_gpr_loss, which returns to the 443 * idle_kvm_start_guest caller. 444 */ 445 li r3, LPCR_PECE0 446 mfspr r4, SPRN_LPCR 447 rlwimi r4, r3, 0, LPCR_PECE0 | LPCR_PECE1 448 mtspr SPRN_LPCR, r4 449 /* set up r3 for return */ 450 mfspr r3,SPRN_SRR1 451 REST_NVGPRS(r1) 452 addi r1, r1, STACK_FRAME_OVERHEAD 453 ld r0, 16(r1) 454 ld r5, 8(r1) 455 ld r1, 0(r1) 456 mtlr r0 457 mtcr r5 458 blr 459 46053: HMT_LOW 461 ld r5, HSTATE_KVM_VCORE(r13) 462 cmpdi r5, 0 463 bne 60f 464 ld r3, HSTATE_SPLIT_MODE(r13) 465 cmpdi r3, 0 466 beq kvm_no_guest 467 lwz r0, KVM_SPLIT_DO_SET(r3) 468 cmpwi r0, 0 469 bne kvmhv_do_set 470 lwz r0, KVM_SPLIT_DO_RESTORE(r3) 471 cmpwi r0, 0 472 bne kvmhv_do_restore 473 lbz r0, KVM_SPLIT_DO_NAP(r3) 474 cmpwi r0, 0 475 beq kvm_no_guest 476 HMT_MEDIUM 477 b kvm_unsplit_nap 47860: HMT_MEDIUM 479 b kvm_secondary_got_guest 480 48154: li r0, KVM_HWTHREAD_IN_KVM 482 stb r0, HSTATE_HWTHREAD_STATE(r13) 483 b kvm_no_guest 484 485kvmhv_do_set: 486 /* Set LPCR, LPIDR etc. on P9 */ 487 HMT_MEDIUM 488 bl kvmhv_p9_set_lpcr 489 nop 490 b kvm_no_guest 491 492kvmhv_do_restore: 493 HMT_MEDIUM 494 bl kvmhv_p9_restore_lpcr 495 nop 496 b kvm_no_guest 497 498/* 499 * Here the primary thread is trying to return the core to 500 * whole-core mode, so we need to nap. 501 */ 502kvm_unsplit_nap: 503 /* 504 * When secondaries are napping in kvm_unsplit_nap() with 505 * hwthread_req = 1, HMI goes ignored even though subcores are 506 * already exited the guest. Hence HMI keeps waking up secondaries 507 * from nap in a loop and secondaries always go back to nap since 508 * no vcore is assigned to them. This makes impossible for primary 509 * thread to get hold of secondary threads resulting into a soft 510 * lockup in KVM path. 511 * 512 * Let us check if HMI is pending and handle it before we go to nap. 513 */ 514 cmpwi r12, BOOK3S_INTERRUPT_HMI 515 bne 55f 516 li r3, 0 /* NULL argument */ 517 bl hmi_exception_realmode 51855: 519 /* 520 * Ensure that secondary doesn't nap when it has 521 * its vcore pointer set. 522 */ 523 sync /* matches smp_mb() before setting split_info.do_nap */ 524 ld r0, HSTATE_KVM_VCORE(r13) 525 cmpdi r0, 0 526 bne kvm_no_guest 527 /* clear any pending message */ 528BEGIN_FTR_SECTION 529 lis r6, (PPC_DBELL_SERVER << (63-36))@h 530 PPC_MSGCLR(6) 531END_FTR_SECTION_IFSET(CPU_FTR_ARCH_207S) 532 /* Set kvm_split_mode.napped[tid] = 1 */ 533 ld r3, HSTATE_SPLIT_MODE(r13) 534 li r0, 1 535 lbz r4, HSTATE_TID(r13) 536 addi r4, r4, KVM_SPLIT_NAPPED 537 stbx r0, r3, r4 538 /* Check the do_nap flag again after setting napped[] */ 539 sync 540 lbz r0, KVM_SPLIT_DO_NAP(r3) 541 cmpwi r0, 0 542 beq 57f 543 li r3, NAPPING_UNSPLIT 544 stb r3, HSTATE_NAPPING(r13) 545 li r3, (LPCR_PECEDH | LPCR_PECE0) >> 4 546 mfspr r5, SPRN_LPCR 547 rlwimi r5, r3, 4, (LPCR_PECEDP | LPCR_PECEDH | LPCR_PECE0 | LPCR_PECE1) 548 b kvm_nap_sequence 549 55057: li r0, 0 551 stbx r0, r3, r4 552 b kvm_no_guest 553 554/****************************************************************************** 555 * * 556 * Entry code * 557 * * 558 *****************************************************************************/ 559 560.global kvmppc_hv_entry 561kvmppc_hv_entry: 562 563 /* Required state: 564 * 565 * R4 = vcpu pointer (or NULL) 566 * MSR = ~IR|DR 567 * R13 = PACA 568 * R1 = host R1 569 * R2 = TOC 570 * all other volatile GPRS = free 571 * Does not preserve non-volatile GPRs or CR fields 572 */ 573 mflr r0 574 std r0, PPC_LR_STKOFF(r1) 575 stdu r1, -SFS(r1) 576 577 /* Save R1 in the PACA */ 578 std r1, HSTATE_HOST_R1(r13) 579 580 li r6, KVM_GUEST_MODE_HOST_HV 581 stb r6, HSTATE_IN_GUEST(r13) 582 583#ifdef CONFIG_KVM_BOOK3S_HV_EXIT_TIMING 584 /* Store initial timestamp */ 585 cmpdi r4, 0 586 beq 1f 587 addi r3, r4, VCPU_TB_RMENTRY 588 bl kvmhv_start_timing 5891: 590#endif 591 592 /* Use cr7 as an indication of radix mode */ 593 ld r5, HSTATE_KVM_VCORE(r13) 594 ld r9, VCORE_KVM(r5) /* pointer to struct kvm */ 595 lbz r0, KVM_RADIX(r9) 596 cmpwi cr7, r0, 0 597 598 /* 599 * POWER7/POWER8 host -> guest partition switch code. 600 * We don't have to lock against concurrent tlbies, 601 * but we do have to coordinate across hardware threads. 602 */ 603 /* Set bit in entry map iff exit map is zero. */ 604 li r7, 1 605 lbz r6, HSTATE_PTID(r13) 606 sld r7, r7, r6 607 addi r8, r5, VCORE_ENTRY_EXIT 60821: lwarx r3, 0, r8 609 cmpwi r3, 0x100 /* any threads starting to exit? */ 610 bge secondary_too_late /* if so we're too late to the party */ 611 or r3, r3, r7 612 stwcx. r3, 0, r8 613 bne 21b 614 615 /* Primary thread switches to guest partition. */ 616 cmpwi r6,0 617 bne 10f 618 619 /* Radix has already switched LPID and flushed core TLB */ 620 bne cr7, 22f 621 622 lwz r7,KVM_LPID(r9) 623BEGIN_FTR_SECTION 624 ld r6,KVM_SDR1(r9) 625 li r0,LPID_RSVD /* switch to reserved LPID */ 626 mtspr SPRN_LPID,r0 627 ptesync 628 mtspr SPRN_SDR1,r6 /* switch to partition page table */ 629END_FTR_SECTION_IFCLR(CPU_FTR_ARCH_300) 630 mtspr SPRN_LPID,r7 631 isync 632 633 /* See if we need to flush the TLB. Hash has to be done in RM */ 634 lhz r6,PACAPACAINDEX(r13) /* test_bit(cpu, need_tlb_flush) */ 635BEGIN_FTR_SECTION 636 /* 637 * On POWER9, individual threads can come in here, but the 638 * TLB is shared between the 4 threads in a core, hence 639 * invalidating on one thread invalidates for all. 640 * Thus we make all 4 threads use the same bit here. 641 */ 642 clrrdi r6,r6,2 643END_FTR_SECTION_IFSET(CPU_FTR_ARCH_300) 644 clrldi r7,r6,64-6 /* extract bit number (6 bits) */ 645 srdi r6,r6,6 /* doubleword number */ 646 sldi r6,r6,3 /* address offset */ 647 add r6,r6,r9 648 addi r6,r6,KVM_NEED_FLUSH /* dword in kvm->arch.need_tlb_flush */ 649 li r8,1 650 sld r8,r8,r7 651 ld r7,0(r6) 652 and. r7,r7,r8 653 beq 22f 654 /* Flush the TLB of any entries for this LPID */ 655 lwz r0,KVM_TLB_SETS(r9) 656 mtctr r0 657 li r7,0x800 /* IS field = 0b10 */ 658 ptesync 659 li r0,0 /* RS for P9 version of tlbiel */ 66028: tlbiel r7 /* On P9, rs=0, RIC=0, PRS=0, R=0 */ 661 addi r7,r7,0x1000 662 bdnz 28b 663 ptesync 66423: ldarx r7,0,r6 /* clear the bit after TLB flushed */ 665 andc r7,r7,r8 666 stdcx. r7,0,r6 667 bne 23b 668 669 /* Add timebase offset onto timebase */ 67022: ld r8,VCORE_TB_OFFSET(r5) 671 cmpdi r8,0 672 beq 37f 673 std r8, VCORE_TB_OFFSET_APPL(r5) 674 mftb r6 /* current host timebase */ 675 add r8,r8,r6 676 mtspr SPRN_TBU40,r8 /* update upper 40 bits */ 677 mftb r7 /* check if lower 24 bits overflowed */ 678 clrldi r6,r6,40 679 clrldi r7,r7,40 680 cmpld r7,r6 681 bge 37f 682 addis r8,r8,0x100 /* if so, increment upper 40 bits */ 683 mtspr SPRN_TBU40,r8 684 685 /* Load guest PCR value to select appropriate compat mode */ 68637: ld r7, VCORE_PCR(r5) 687 cmpdi r7, 0 688 beq 38f 689 mtspr SPRN_PCR, r7 69038: 691 692BEGIN_FTR_SECTION 693 /* DPDES and VTB are shared between threads */ 694 ld r8, VCORE_DPDES(r5) 695 ld r7, VCORE_VTB(r5) 696 mtspr SPRN_DPDES, r8 697 mtspr SPRN_VTB, r7 698END_FTR_SECTION_IFSET(CPU_FTR_ARCH_207S) 699 700 /* Mark the subcore state as inside guest */ 701 bl kvmppc_subcore_enter_guest 702 nop 703 ld r5, HSTATE_KVM_VCORE(r13) 704 ld r4, HSTATE_KVM_VCPU(r13) 705 li r0,1 706 stb r0,VCORE_IN_GUEST(r5) /* signal secondaries to continue */ 707 708 /* Do we have a guest vcpu to run? */ 70910: cmpdi r4, 0 710 beq kvmppc_primary_no_guest 711kvmppc_got_guest: 712 /* Increment yield count if they have a VPA */ 713 ld r3, VCPU_VPA(r4) 714 cmpdi r3, 0 715 beq 25f 716 li r6, LPPACA_YIELDCOUNT 717 LWZX_BE r5, r3, r6 718 addi r5, r5, 1 719 STWX_BE r5, r3, r6 720 li r6, 1 721 stb r6, VCPU_VPA_DIRTY(r4) 72225: 723 724 /* Save purr/spurr */ 725 mfspr r5,SPRN_PURR 726 mfspr r6,SPRN_SPURR 727 std r5,HSTATE_PURR(r13) 728 std r6,HSTATE_SPURR(r13) 729 ld r7,VCPU_PURR(r4) 730 ld r8,VCPU_SPURR(r4) 731 mtspr SPRN_PURR,r7 732 mtspr SPRN_SPURR,r8 733 734 /* Save host values of some registers */ 735BEGIN_FTR_SECTION 736 mfspr r5, SPRN_TIDR 737 mfspr r6, SPRN_PSSCR 738 mfspr r7, SPRN_PID 739 std r5, STACK_SLOT_TID(r1) 740 std r6, STACK_SLOT_PSSCR(r1) 741 std r7, STACK_SLOT_PID(r1) 742 mfspr r5, SPRN_HFSCR 743 std r5, STACK_SLOT_HFSCR(r1) 744END_FTR_SECTION_IFSET(CPU_FTR_ARCH_300) 745BEGIN_FTR_SECTION 746 mfspr r5, SPRN_CIABR 747 mfspr r6, SPRN_DAWR 748 mfspr r7, SPRN_DAWRX 749 mfspr r8, SPRN_IAMR 750 std r5, STACK_SLOT_CIABR(r1) 751 std r6, STACK_SLOT_DAWR(r1) 752 std r7, STACK_SLOT_DAWRX(r1) 753 std r8, STACK_SLOT_IAMR(r1) 754END_FTR_SECTION_IFSET(CPU_FTR_ARCH_207S) 755 756 mfspr r5, SPRN_AMR 757 std r5, STACK_SLOT_AMR(r1) 758 mfspr r6, SPRN_UAMOR 759 std r6, STACK_SLOT_UAMOR(r1) 760 761BEGIN_FTR_SECTION 762 /* Set partition DABR */ 763 /* Do this before re-enabling PMU to avoid P7 DABR corruption bug */ 764 lwz r5,VCPU_DABRX(r4) 765 ld r6,VCPU_DABR(r4) 766 mtspr SPRN_DABRX,r5 767 mtspr SPRN_DABR,r6 768 isync 769END_FTR_SECTION_IFCLR(CPU_FTR_ARCH_207S) 770 771#ifdef CONFIG_PPC_TRANSACTIONAL_MEM 772/* 773 * Branch around the call if both CPU_FTR_TM and 774 * CPU_FTR_P9_TM_HV_ASSIST are off. 775 */ 776BEGIN_FTR_SECTION 777 b 91f 778END_FTR_SECTION(CPU_FTR_TM | CPU_FTR_P9_TM_HV_ASSIST, 0) 779 /* 780 * NOTE THAT THIS TRASHES ALL NON-VOLATILE REGISTERS (but not CR) 781 */ 782 mr r3, r4 783 ld r4, VCPU_MSR(r3) 784 li r5, 0 /* don't preserve non-vol regs */ 785 bl kvmppc_restore_tm_hv 786 nop 787 ld r4, HSTATE_KVM_VCPU(r13) 78891: 789#endif 790 791 /* Load guest PMU registers; r4 = vcpu pointer here */ 792 mr r3, r4 793 bl kvmhv_load_guest_pmu 794 795 /* Load up FP, VMX and VSX registers */ 796 ld r4, HSTATE_KVM_VCPU(r13) 797 bl kvmppc_load_fp 798 799 ld r14, VCPU_GPR(R14)(r4) 800 ld r15, VCPU_GPR(R15)(r4) 801 ld r16, VCPU_GPR(R16)(r4) 802 ld r17, VCPU_GPR(R17)(r4) 803 ld r18, VCPU_GPR(R18)(r4) 804 ld r19, VCPU_GPR(R19)(r4) 805 ld r20, VCPU_GPR(R20)(r4) 806 ld r21, VCPU_GPR(R21)(r4) 807 ld r22, VCPU_GPR(R22)(r4) 808 ld r23, VCPU_GPR(R23)(r4) 809 ld r24, VCPU_GPR(R24)(r4) 810 ld r25, VCPU_GPR(R25)(r4) 811 ld r26, VCPU_GPR(R26)(r4) 812 ld r27, VCPU_GPR(R27)(r4) 813 ld r28, VCPU_GPR(R28)(r4) 814 ld r29, VCPU_GPR(R29)(r4) 815 ld r30, VCPU_GPR(R30)(r4) 816 ld r31, VCPU_GPR(R31)(r4) 817 818 /* Switch DSCR to guest value */ 819 ld r5, VCPU_DSCR(r4) 820 mtspr SPRN_DSCR, r5 821 822BEGIN_FTR_SECTION 823 /* Skip next section on POWER7 */ 824 b 8f 825END_FTR_SECTION_IFCLR(CPU_FTR_ARCH_207S) 826 /* Load up POWER8-specific registers */ 827 ld r5, VCPU_IAMR(r4) 828 lwz r6, VCPU_PSPB(r4) 829 ld r7, VCPU_FSCR(r4) 830 mtspr SPRN_IAMR, r5 831 mtspr SPRN_PSPB, r6 832 mtspr SPRN_FSCR, r7 833 /* 834 * Handle broken DAWR case by not writing it. This means we 835 * can still store the DAWR register for migration. 836 */ 837 LOAD_REG_ADDR(r5, dawr_force_enable) 838 lbz r5, 0(r5) 839 cmpdi r5, 0 840 beq 1f 841 ld r5, VCPU_DAWR(r4) 842 ld r6, VCPU_DAWRX(r4) 843 mtspr SPRN_DAWR, r5 844 mtspr SPRN_DAWRX, r6 8451: 846 ld r7, VCPU_CIABR(r4) 847 ld r8, VCPU_TAR(r4) 848 mtspr SPRN_CIABR, r7 849 mtspr SPRN_TAR, r8 850 ld r5, VCPU_IC(r4) 851 ld r8, VCPU_EBBHR(r4) 852 mtspr SPRN_IC, r5 853 mtspr SPRN_EBBHR, r8 854 ld r5, VCPU_EBBRR(r4) 855 ld r6, VCPU_BESCR(r4) 856 lwz r7, VCPU_GUEST_PID(r4) 857 ld r8, VCPU_WORT(r4) 858 mtspr SPRN_EBBRR, r5 859 mtspr SPRN_BESCR, r6 860 mtspr SPRN_PID, r7 861 mtspr SPRN_WORT, r8 862BEGIN_FTR_SECTION 863 /* POWER8-only registers */ 864 ld r5, VCPU_TCSCR(r4) 865 ld r6, VCPU_ACOP(r4) 866 ld r7, VCPU_CSIGR(r4) 867 ld r8, VCPU_TACR(r4) 868 mtspr SPRN_TCSCR, r5 869 mtspr SPRN_ACOP, r6 870 mtspr SPRN_CSIGR, r7 871 mtspr SPRN_TACR, r8 872 nop 873FTR_SECTION_ELSE 874 /* POWER9-only registers */ 875 ld r5, VCPU_TID(r4) 876 ld r6, VCPU_PSSCR(r4) 877 lbz r8, HSTATE_FAKE_SUSPEND(r13) 878 oris r6, r6, PSSCR_EC@h /* This makes stop trap to HV */ 879 rldimi r6, r8, PSSCR_FAKE_SUSPEND_LG, 63 - PSSCR_FAKE_SUSPEND_LG 880 ld r7, VCPU_HFSCR(r4) 881 mtspr SPRN_TIDR, r5 882 mtspr SPRN_PSSCR, r6 883 mtspr SPRN_HFSCR, r7 884ALT_FTR_SECTION_END_IFCLR(CPU_FTR_ARCH_300) 8858: 886 887 ld r5, VCPU_SPRG0(r4) 888 ld r6, VCPU_SPRG1(r4) 889 ld r7, VCPU_SPRG2(r4) 890 ld r8, VCPU_SPRG3(r4) 891 mtspr SPRN_SPRG0, r5 892 mtspr SPRN_SPRG1, r6 893 mtspr SPRN_SPRG2, r7 894 mtspr SPRN_SPRG3, r8 895 896 /* Load up DAR and DSISR */ 897 ld r5, VCPU_DAR(r4) 898 lwz r6, VCPU_DSISR(r4) 899 mtspr SPRN_DAR, r5 900 mtspr SPRN_DSISR, r6 901 902 /* Restore AMR and UAMOR, set AMOR to all 1s */ 903 ld r5,VCPU_AMR(r4) 904 ld r6,VCPU_UAMOR(r4) 905 li r7,-1 906 mtspr SPRN_AMR,r5 907 mtspr SPRN_UAMOR,r6 908 mtspr SPRN_AMOR,r7 909 910 /* Restore state of CTRL run bit; assume 1 on entry */ 911 lwz r5,VCPU_CTRL(r4) 912 andi. r5,r5,1 913 bne 4f 914 mfspr r6,SPRN_CTRLF 915 clrrdi r6,r6,1 916 mtspr SPRN_CTRLT,r6 9174: 918 /* Secondary threads wait for primary to have done partition switch */ 919 ld r5, HSTATE_KVM_VCORE(r13) 920 lbz r6, HSTATE_PTID(r13) 921 cmpwi r6, 0 922 beq 21f 923 lbz r0, VCORE_IN_GUEST(r5) 924 cmpwi r0, 0 925 bne 21f 926 HMT_LOW 92720: lwz r3, VCORE_ENTRY_EXIT(r5) 928 cmpwi r3, 0x100 929 bge no_switch_exit 930 lbz r0, VCORE_IN_GUEST(r5) 931 cmpwi r0, 0 932 beq 20b 933 HMT_MEDIUM 93421: 935 /* Set LPCR. */ 936 ld r8,VCORE_LPCR(r5) 937 mtspr SPRN_LPCR,r8 938 isync 939 940 /* 941 * Set the decrementer to the guest decrementer. 942 */ 943 ld r8,VCPU_DEC_EXPIRES(r4) 944 /* r8 is a host timebase value here, convert to guest TB */ 945 ld r5,HSTATE_KVM_VCORE(r13) 946 ld r6,VCORE_TB_OFFSET_APPL(r5) 947 add r8,r8,r6 948 mftb r7 949 subf r3,r7,r8 950 mtspr SPRN_DEC,r3 951 952 /* Check if HDEC expires soon */ 953 mfspr r3, SPRN_HDEC 954 EXTEND_HDEC(r3) 955 cmpdi r3, 512 /* 1 microsecond */ 956 blt hdec_soon 957 958 /* For hash guest, clear out and reload the SLB */ 959 ld r6, VCPU_KVM(r4) 960 lbz r0, KVM_RADIX(r6) 961 cmpwi r0, 0 962 bne 9f 963 li r6, 0 964 slbmte r6, r6 965 slbia 966 ptesync 967 968 /* Load up guest SLB entries (N.B. slb_max will be 0 for radix) */ 969 lwz r5,VCPU_SLB_MAX(r4) 970 cmpwi r5,0 971 beq 9f 972 mtctr r5 973 addi r6,r4,VCPU_SLB 9741: ld r8,VCPU_SLB_E(r6) 975 ld r9,VCPU_SLB_V(r6) 976 slbmte r9,r8 977 addi r6,r6,VCPU_SLB_SIZE 978 bdnz 1b 9799: 980 981#ifdef CONFIG_KVM_XICS 982 /* We are entering the guest on that thread, push VCPU to XIVE */ 983 ld r10, HSTATE_XIVE_TIMA_PHYS(r13) 984 cmpldi cr0, r10, 0 985 beq no_xive 986 ld r11, VCPU_XIVE_SAVED_STATE(r4) 987 li r9, TM_QW1_OS 988 eieio 989 stdcix r11,r9,r10 990 lwz r11, VCPU_XIVE_CAM_WORD(r4) 991 li r9, TM_QW1_OS + TM_WORD2 992 stwcix r11,r9,r10 993 li r9, 1 994 stb r9, VCPU_XIVE_PUSHED(r4) 995 eieio 996 997 /* 998 * We clear the irq_pending flag. There is a small chance of a 999 * race vs. the escalation interrupt happening on another 1000 * processor setting it again, but the only consequence is to 1001 * cause a spurrious wakeup on the next H_CEDE which is not an 1002 * issue. 1003 */ 1004 li r0,0 1005 stb r0, VCPU_IRQ_PENDING(r4) 1006 1007 /* 1008 * In single escalation mode, if the escalation interrupt is 1009 * on, we mask it. 1010 */ 1011 lbz r0, VCPU_XIVE_ESC_ON(r4) 1012 cmpwi r0,0 1013 beq 1f 1014 ld r10, VCPU_XIVE_ESC_RADDR(r4) 1015 li r9, XIVE_ESB_SET_PQ_01 1016 ldcix r0, r10, r9 1017 sync 1018 1019 /* We have a possible subtle race here: The escalation interrupt might 1020 * have fired and be on its way to the host queue while we mask it, 1021 * and if we unmask it early enough (re-cede right away), there is 1022 * a theorical possibility that it fires again, thus landing in the 1023 * target queue more than once which is a big no-no. 1024 * 1025 * Fortunately, solving this is rather easy. If the above load setting 1026 * PQ to 01 returns a previous value where P is set, then we know the 1027 * escalation interrupt is somewhere on its way to the host. In that 1028 * case we simply don't clear the xive_esc_on flag below. It will be 1029 * eventually cleared by the handler for the escalation interrupt. 1030 * 1031 * Then, when doing a cede, we check that flag again before re-enabling 1032 * the escalation interrupt, and if set, we abort the cede. 1033 */ 1034 andi. r0, r0, XIVE_ESB_VAL_P 1035 bne- 1f 1036 1037 /* Now P is 0, we can clear the flag */ 1038 li r0, 0 1039 stb r0, VCPU_XIVE_ESC_ON(r4) 10401: 1041no_xive: 1042#endif /* CONFIG_KVM_XICS */ 1043 1044 li r0, 0 1045 stw r0, STACK_SLOT_SHORT_PATH(r1) 1046 1047deliver_guest_interrupt: /* r4 = vcpu, r13 = paca */ 1048 /* Check if we can deliver an external or decrementer interrupt now */ 1049 ld r0, VCPU_PENDING_EXC(r4) 1050BEGIN_FTR_SECTION 1051 /* On POWER9, also check for emulated doorbell interrupt */ 1052 lbz r3, VCPU_DBELL_REQ(r4) 1053 or r0, r0, r3 1054END_FTR_SECTION_IFSET(CPU_FTR_ARCH_300) 1055 cmpdi r0, 0 1056 beq 71f 1057 mr r3, r4 1058 bl kvmppc_guest_entry_inject_int 1059 ld r4, HSTATE_KVM_VCPU(r13) 106071: 1061 ld r6, VCPU_SRR0(r4) 1062 ld r7, VCPU_SRR1(r4) 1063 mtspr SPRN_SRR0, r6 1064 mtspr SPRN_SRR1, r7 1065 1066fast_guest_entry_c: 1067 ld r10, VCPU_PC(r4) 1068 ld r11, VCPU_MSR(r4) 1069 /* r11 = vcpu->arch.msr & ~MSR_HV */ 1070 rldicl r11, r11, 63 - MSR_HV_LG, 1 1071 rotldi r11, r11, 1 + MSR_HV_LG 1072 ori r11, r11, MSR_ME 1073 1074 ld r6, VCPU_CTR(r4) 1075 ld r7, VCPU_XER(r4) 1076 mtctr r6 1077 mtxer r7 1078 1079/* 1080 * Required state: 1081 * R4 = vcpu 1082 * R10: value for HSRR0 1083 * R11: value for HSRR1 1084 * R13 = PACA 1085 */ 1086fast_guest_return: 1087 li r0,0 1088 stb r0,VCPU_CEDED(r4) /* cancel cede */ 1089 mtspr SPRN_HSRR0,r10 1090 mtspr SPRN_HSRR1,r11 1091 1092 /* Activate guest mode, so faults get handled by KVM */ 1093 li r9, KVM_GUEST_MODE_GUEST_HV 1094 stb r9, HSTATE_IN_GUEST(r13) 1095 1096#ifdef CONFIG_KVM_BOOK3S_HV_EXIT_TIMING 1097 /* Accumulate timing */ 1098 addi r3, r4, VCPU_TB_GUEST 1099 bl kvmhv_accumulate_time 1100#endif 1101 1102 /* Enter guest */ 1103 1104BEGIN_FTR_SECTION 1105 ld r5, VCPU_CFAR(r4) 1106 mtspr SPRN_CFAR, r5 1107END_FTR_SECTION_IFSET(CPU_FTR_CFAR) 1108BEGIN_FTR_SECTION 1109 ld r0, VCPU_PPR(r4) 1110END_FTR_SECTION_IFSET(CPU_FTR_HAS_PPR) 1111 1112 ld r5, VCPU_LR(r4) 1113 ld r6, VCPU_CR(r4) 1114 mtlr r5 1115 mtcr r6 1116 1117 ld r1, VCPU_GPR(R1)(r4) 1118 ld r2, VCPU_GPR(R2)(r4) 1119 ld r3, VCPU_GPR(R3)(r4) 1120 ld r5, VCPU_GPR(R5)(r4) 1121 ld r6, VCPU_GPR(R6)(r4) 1122 ld r7, VCPU_GPR(R7)(r4) 1123 ld r8, VCPU_GPR(R8)(r4) 1124 ld r9, VCPU_GPR(R9)(r4) 1125 ld r10, VCPU_GPR(R10)(r4) 1126 ld r11, VCPU_GPR(R11)(r4) 1127 ld r12, VCPU_GPR(R12)(r4) 1128 ld r13, VCPU_GPR(R13)(r4) 1129 1130BEGIN_FTR_SECTION 1131 mtspr SPRN_PPR, r0 1132END_FTR_SECTION_IFSET(CPU_FTR_HAS_PPR) 1133 1134/* Move canary into DSISR to check for later */ 1135BEGIN_FTR_SECTION 1136 li r0, 0x7fff 1137 mtspr SPRN_HDSISR, r0 1138END_FTR_SECTION_IFSET(CPU_FTR_ARCH_300) 1139 1140 ld r0, VCPU_GPR(R0)(r4) 1141 ld r4, VCPU_GPR(R4)(r4) 1142 HRFI_TO_GUEST 1143 b . 1144 1145/* 1146 * Enter the guest on a P9 or later system where we have exactly 1147 * one vcpu per vcore and we don't need to go to real mode 1148 * (which implies that host and guest are both using radix MMU mode). 1149 * r3 = vcpu pointer 1150 * Most SPRs and all the VSRs have been loaded already. 1151 */ 1152_GLOBAL(__kvmhv_vcpu_entry_p9) 1153EXPORT_SYMBOL_GPL(__kvmhv_vcpu_entry_p9) 1154 mflr r0 1155 std r0, PPC_LR_STKOFF(r1) 1156 stdu r1, -SFS(r1) 1157 1158 li r0, 1 1159 stw r0, STACK_SLOT_SHORT_PATH(r1) 1160 1161 std r3, HSTATE_KVM_VCPU(r13) 1162 mfcr r4 1163 stw r4, SFS+8(r1) 1164 1165 std r1, HSTATE_HOST_R1(r13) 1166 1167 reg = 14 1168 .rept 18 1169 std reg, STACK_SLOT_NVGPRS + ((reg - 14) * 8)(r1) 1170 reg = reg + 1 1171 .endr 1172 1173 reg = 14 1174 .rept 18 1175 ld reg, __VCPU_GPR(reg)(r3) 1176 reg = reg + 1 1177 .endr 1178 1179 mfmsr r10 1180 std r10, HSTATE_HOST_MSR(r13) 1181 1182 mr r4, r3 1183 b fast_guest_entry_c 1184guest_exit_short_path: 1185 1186 li r0, KVM_GUEST_MODE_NONE 1187 stb r0, HSTATE_IN_GUEST(r13) 1188 1189 reg = 14 1190 .rept 18 1191 std reg, __VCPU_GPR(reg)(r9) 1192 reg = reg + 1 1193 .endr 1194 1195 reg = 14 1196 .rept 18 1197 ld reg, STACK_SLOT_NVGPRS + ((reg - 14) * 8)(r1) 1198 reg = reg + 1 1199 .endr 1200 1201 lwz r4, SFS+8(r1) 1202 mtcr r4 1203 1204 mr r3, r12 /* trap number */ 1205 1206 addi r1, r1, SFS 1207 ld r0, PPC_LR_STKOFF(r1) 1208 mtlr r0 1209 1210 /* If we are in real mode, do a rfid to get back to the caller */ 1211 mfmsr r4 1212 andi. r5, r4, MSR_IR 1213 bnelr 1214 rldicl r5, r4, 64 - MSR_TS_S_LG, 62 /* extract TS field */ 1215 mtspr SPRN_SRR0, r0 1216 ld r10, HSTATE_HOST_MSR(r13) 1217 rldimi r10, r5, MSR_TS_S_LG, 63 - MSR_TS_T_LG 1218 mtspr SPRN_SRR1, r10 1219 RFI_TO_KERNEL 1220 b . 1221 1222secondary_too_late: 1223 li r12, 0 1224 stw r12, STACK_SLOT_TRAP(r1) 1225 cmpdi r4, 0 1226 beq 11f 1227 stw r12, VCPU_TRAP(r4) 1228#ifdef CONFIG_KVM_BOOK3S_HV_EXIT_TIMING 1229 addi r3, r4, VCPU_TB_RMEXIT 1230 bl kvmhv_accumulate_time 1231#endif 123211: b kvmhv_switch_to_host 1233 1234no_switch_exit: 1235 HMT_MEDIUM 1236 li r12, 0 1237 b 12f 1238hdec_soon: 1239 li r12, BOOK3S_INTERRUPT_HV_DECREMENTER 124012: stw r12, VCPU_TRAP(r4) 1241 mr r9, r4 1242#ifdef CONFIG_KVM_BOOK3S_HV_EXIT_TIMING 1243 addi r3, r4, VCPU_TB_RMEXIT 1244 bl kvmhv_accumulate_time 1245#endif 1246 b guest_bypass 1247 1248/****************************************************************************** 1249 * * 1250 * Exit code * 1251 * * 1252 *****************************************************************************/ 1253 1254/* 1255 * We come here from the first-level interrupt handlers. 1256 */ 1257 .globl kvmppc_interrupt_hv 1258kvmppc_interrupt_hv: 1259 /* 1260 * Register contents: 1261 * R12 = (guest CR << 32) | interrupt vector 1262 * R13 = PACA 1263 * guest R12 saved in shadow VCPU SCRATCH0 1264 * guest CTR saved in shadow VCPU SCRATCH1 if RELOCATABLE 1265 * guest R13 saved in SPRN_SCRATCH0 1266 */ 1267 std r9, HSTATE_SCRATCH2(r13) 1268 lbz r9, HSTATE_IN_GUEST(r13) 1269 cmpwi r9, KVM_GUEST_MODE_HOST_HV 1270 beq kvmppc_bad_host_intr 1271#ifdef CONFIG_KVM_BOOK3S_PR_POSSIBLE 1272 cmpwi r9, KVM_GUEST_MODE_GUEST 1273 ld r9, HSTATE_SCRATCH2(r13) 1274 beq kvmppc_interrupt_pr 1275#endif 1276 /* We're now back in the host but in guest MMU context */ 1277 li r9, KVM_GUEST_MODE_HOST_HV 1278 stb r9, HSTATE_IN_GUEST(r13) 1279 1280 ld r9, HSTATE_KVM_VCPU(r13) 1281 1282 /* Save registers */ 1283 1284 std r0, VCPU_GPR(R0)(r9) 1285 std r1, VCPU_GPR(R1)(r9) 1286 std r2, VCPU_GPR(R2)(r9) 1287 std r3, VCPU_GPR(R3)(r9) 1288 std r4, VCPU_GPR(R4)(r9) 1289 std r5, VCPU_GPR(R5)(r9) 1290 std r6, VCPU_GPR(R6)(r9) 1291 std r7, VCPU_GPR(R7)(r9) 1292 std r8, VCPU_GPR(R8)(r9) 1293 ld r0, HSTATE_SCRATCH2(r13) 1294 std r0, VCPU_GPR(R9)(r9) 1295 std r10, VCPU_GPR(R10)(r9) 1296 std r11, VCPU_GPR(R11)(r9) 1297 ld r3, HSTATE_SCRATCH0(r13) 1298 std r3, VCPU_GPR(R12)(r9) 1299 /* CR is in the high half of r12 */ 1300 srdi r4, r12, 32 1301 std r4, VCPU_CR(r9) 1302BEGIN_FTR_SECTION 1303 ld r3, HSTATE_CFAR(r13) 1304 std r3, VCPU_CFAR(r9) 1305END_FTR_SECTION_IFSET(CPU_FTR_CFAR) 1306BEGIN_FTR_SECTION 1307 ld r4, HSTATE_PPR(r13) 1308 std r4, VCPU_PPR(r9) 1309END_FTR_SECTION_IFSET(CPU_FTR_HAS_PPR) 1310 1311 /* Restore R1/R2 so we can handle faults */ 1312 ld r1, HSTATE_HOST_R1(r13) 1313 ld r2, PACATOC(r13) 1314 1315 mfspr r10, SPRN_SRR0 1316 mfspr r11, SPRN_SRR1 1317 std r10, VCPU_SRR0(r9) 1318 std r11, VCPU_SRR1(r9) 1319 /* trap is in the low half of r12, clear CR from the high half */ 1320 clrldi r12, r12, 32 1321 andi. r0, r12, 2 /* need to read HSRR0/1? */ 1322 beq 1f 1323 mfspr r10, SPRN_HSRR0 1324 mfspr r11, SPRN_HSRR1 1325 clrrdi r12, r12, 2 13261: std r10, VCPU_PC(r9) 1327 std r11, VCPU_MSR(r9) 1328 1329 GET_SCRATCH0(r3) 1330 mflr r4 1331 std r3, VCPU_GPR(R13)(r9) 1332 std r4, VCPU_LR(r9) 1333 1334 stw r12,VCPU_TRAP(r9) 1335 1336 /* 1337 * Now that we have saved away SRR0/1 and HSRR0/1, 1338 * interrupts are recoverable in principle, so set MSR_RI. 1339 * This becomes important for relocation-on interrupts from 1340 * the guest, which we can get in radix mode on POWER9. 1341 */ 1342 li r0, MSR_RI 1343 mtmsrd r0, 1 1344 1345#ifdef CONFIG_KVM_BOOK3S_HV_EXIT_TIMING 1346 addi r3, r9, VCPU_TB_RMINTR 1347 mr r4, r9 1348 bl kvmhv_accumulate_time 1349 ld r5, VCPU_GPR(R5)(r9) 1350 ld r6, VCPU_GPR(R6)(r9) 1351 ld r7, VCPU_GPR(R7)(r9) 1352 ld r8, VCPU_GPR(R8)(r9) 1353#endif 1354 1355 /* Save HEIR (HV emulation assist reg) in emul_inst 1356 if this is an HEI (HV emulation interrupt, e40) */ 1357 li r3,KVM_INST_FETCH_FAILED 1358 stw r3,VCPU_LAST_INST(r9) 1359 cmpwi r12,BOOK3S_INTERRUPT_H_EMUL_ASSIST 1360 bne 11f 1361 mfspr r3,SPRN_HEIR 136211: stw r3,VCPU_HEIR(r9) 1363 1364 /* these are volatile across C function calls */ 1365#ifdef CONFIG_RELOCATABLE 1366 ld r3, HSTATE_SCRATCH1(r13) 1367 mtctr r3 1368#else 1369 mfctr r3 1370#endif 1371 mfxer r4 1372 std r3, VCPU_CTR(r9) 1373 std r4, VCPU_XER(r9) 1374 1375 /* Save more register state */ 1376 mfdar r3 1377 mfdsisr r4 1378 std r3, VCPU_DAR(r9) 1379 stw r4, VCPU_DSISR(r9) 1380 1381 /* If this is a page table miss then see if it's theirs or ours */ 1382 cmpwi r12, BOOK3S_INTERRUPT_H_DATA_STORAGE 1383 beq kvmppc_hdsi 1384 std r3, VCPU_FAULT_DAR(r9) 1385 stw r4, VCPU_FAULT_DSISR(r9) 1386 cmpwi r12, BOOK3S_INTERRUPT_H_INST_STORAGE 1387 beq kvmppc_hisi 1388 1389#ifdef CONFIG_PPC_TRANSACTIONAL_MEM 1390 /* For softpatch interrupt, go off and do TM instruction emulation */ 1391 cmpwi r12, BOOK3S_INTERRUPT_HV_SOFTPATCH 1392 beq kvmppc_tm_emul 1393#endif 1394 1395 /* See if this is a leftover HDEC interrupt */ 1396 cmpwi r12,BOOK3S_INTERRUPT_HV_DECREMENTER 1397 bne 2f 1398 mfspr r3,SPRN_HDEC 1399 EXTEND_HDEC(r3) 1400 cmpdi r3,0 1401 mr r4,r9 1402 bge fast_guest_return 14032: 1404 /* See if this is an hcall we can handle in real mode */ 1405 cmpwi r12,BOOK3S_INTERRUPT_SYSCALL 1406 beq hcall_try_real_mode 1407 1408 /* Hypervisor doorbell - exit only if host IPI flag set */ 1409 cmpwi r12, BOOK3S_INTERRUPT_H_DOORBELL 1410 bne 3f 1411BEGIN_FTR_SECTION 1412 PPC_MSGSYNC 1413 lwsync 1414 /* always exit if we're running a nested guest */ 1415 ld r0, VCPU_NESTED(r9) 1416 cmpdi r0, 0 1417 bne guest_exit_cont 1418END_FTR_SECTION_IFSET(CPU_FTR_ARCH_300) 1419 lbz r0, HSTATE_HOST_IPI(r13) 1420 cmpwi r0, 0 1421 beq maybe_reenter_guest 1422 b guest_exit_cont 14233: 1424 /* If it's a hypervisor facility unavailable interrupt, save HFSCR */ 1425 cmpwi r12, BOOK3S_INTERRUPT_H_FAC_UNAVAIL 1426 bne 14f 1427 mfspr r3, SPRN_HFSCR 1428 std r3, VCPU_HFSCR(r9) 1429 b guest_exit_cont 143014: 1431 /* External interrupt ? */ 1432 cmpwi r12, BOOK3S_INTERRUPT_EXTERNAL 1433 beq kvmppc_guest_external 1434 /* See if it is a machine check */ 1435 cmpwi r12, BOOK3S_INTERRUPT_MACHINE_CHECK 1436 beq machine_check_realmode 1437 /* Or a hypervisor maintenance interrupt */ 1438 cmpwi r12, BOOK3S_INTERRUPT_HMI 1439 beq hmi_realmode 1440 1441guest_exit_cont: /* r9 = vcpu, r12 = trap, r13 = paca */ 1442 1443#ifdef CONFIG_KVM_BOOK3S_HV_EXIT_TIMING 1444 addi r3, r9, VCPU_TB_RMEXIT 1445 mr r4, r9 1446 bl kvmhv_accumulate_time 1447#endif 1448#ifdef CONFIG_KVM_XICS 1449 /* We are exiting, pull the VP from the XIVE */ 1450 lbz r0, VCPU_XIVE_PUSHED(r9) 1451 cmpwi cr0, r0, 0 1452 beq 1f 1453 li r7, TM_SPC_PULL_OS_CTX 1454 li r6, TM_QW1_OS 1455 mfmsr r0 1456 andi. r0, r0, MSR_DR /* in real mode? */ 1457 beq 2f 1458 ld r10, HSTATE_XIVE_TIMA_VIRT(r13) 1459 cmpldi cr0, r10, 0 1460 beq 1f 1461 /* First load to pull the context, we ignore the value */ 1462 eieio 1463 lwzx r11, r7, r10 1464 /* Second load to recover the context state (Words 0 and 1) */ 1465 ldx r11, r6, r10 1466 b 3f 14672: ld r10, HSTATE_XIVE_TIMA_PHYS(r13) 1468 cmpldi cr0, r10, 0 1469 beq 1f 1470 /* First load to pull the context, we ignore the value */ 1471 eieio 1472 lwzcix r11, r7, r10 1473 /* Second load to recover the context state (Words 0 and 1) */ 1474 ldcix r11, r6, r10 14753: std r11, VCPU_XIVE_SAVED_STATE(r9) 1476 /* Fixup some of the state for the next load */ 1477 li r10, 0 1478 li r0, 0xff 1479 stb r10, VCPU_XIVE_PUSHED(r9) 1480 stb r10, (VCPU_XIVE_SAVED_STATE+3)(r9) 1481 stb r0, (VCPU_XIVE_SAVED_STATE+4)(r9) 1482 eieio 14831: 1484#endif /* CONFIG_KVM_XICS */ 1485 1486 /* If we came in through the P9 short path, go back out to C now */ 1487 lwz r0, STACK_SLOT_SHORT_PATH(r1) 1488 cmpwi r0, 0 1489 bne guest_exit_short_path 1490 1491 /* For hash guest, read the guest SLB and save it away */ 1492 ld r5, VCPU_KVM(r9) 1493 lbz r0, KVM_RADIX(r5) 1494 li r5, 0 1495 cmpwi r0, 0 1496 bne 3f /* for radix, save 0 entries */ 1497 lwz r0,VCPU_SLB_NR(r9) /* number of entries in SLB */ 1498 mtctr r0 1499 li r6,0 1500 addi r7,r9,VCPU_SLB 15011: slbmfee r8,r6 1502 andis. r0,r8,SLB_ESID_V@h 1503 beq 2f 1504 add r8,r8,r6 /* put index in */ 1505 slbmfev r3,r6 1506 std r8,VCPU_SLB_E(r7) 1507 std r3,VCPU_SLB_V(r7) 1508 addi r7,r7,VCPU_SLB_SIZE 1509 addi r5,r5,1 15102: addi r6,r6,1 1511 bdnz 1b 1512 /* Finally clear out the SLB */ 1513 li r0,0 1514 slbmte r0,r0 1515 slbia 1516 ptesync 15173: stw r5,VCPU_SLB_MAX(r9) 1518 1519 /* load host SLB entries */ 1520BEGIN_MMU_FTR_SECTION 1521 b 0f 1522END_MMU_FTR_SECTION_IFSET(MMU_FTR_TYPE_RADIX) 1523 ld r8,PACA_SLBSHADOWPTR(r13) 1524 1525 .rept SLB_NUM_BOLTED 1526 li r3, SLBSHADOW_SAVEAREA 1527 LDX_BE r5, r8, r3 1528 addi r3, r3, 8 1529 LDX_BE r6, r8, r3 1530 andis. r7,r5,SLB_ESID_V@h 1531 beq 1f 1532 slbmte r6,r5 15331: addi r8,r8,16 1534 .endr 15350: 1536 1537guest_bypass: 1538 stw r12, STACK_SLOT_TRAP(r1) 1539 1540 /* Save DEC */ 1541 /* Do this before kvmhv_commence_exit so we know TB is guest TB */ 1542 ld r3, HSTATE_KVM_VCORE(r13) 1543 mfspr r5,SPRN_DEC 1544 mftb r6 1545 /* On P9, if the guest has large decr enabled, don't sign extend */ 1546BEGIN_FTR_SECTION 1547 ld r4, VCORE_LPCR(r3) 1548 andis. r4, r4, LPCR_LD@h 1549 bne 16f 1550END_FTR_SECTION_IFSET(CPU_FTR_ARCH_300) 1551 extsw r5,r5 155216: add r5,r5,r6 1553 /* r5 is a guest timebase value here, convert to host TB */ 1554 ld r4,VCORE_TB_OFFSET_APPL(r3) 1555 subf r5,r4,r5 1556 std r5,VCPU_DEC_EXPIRES(r9) 1557 1558 /* Increment exit count, poke other threads to exit */ 1559 mr r3, r12 1560 bl kvmhv_commence_exit 1561 nop 1562 ld r9, HSTATE_KVM_VCPU(r13) 1563 1564 /* Stop others sending VCPU interrupts to this physical CPU */ 1565 li r0, -1 1566 stw r0, VCPU_CPU(r9) 1567 stw r0, VCPU_THREAD_CPU(r9) 1568 1569 /* Save guest CTRL register, set runlatch to 1 */ 1570 mfspr r6,SPRN_CTRLF 1571 stw r6,VCPU_CTRL(r9) 1572 andi. r0,r6,1 1573 bne 4f 1574 ori r6,r6,1 1575 mtspr SPRN_CTRLT,r6 15764: 1577 /* 1578 * Save the guest PURR/SPURR 1579 */ 1580 mfspr r5,SPRN_PURR 1581 mfspr r6,SPRN_SPURR 1582 ld r7,VCPU_PURR(r9) 1583 ld r8,VCPU_SPURR(r9) 1584 std r5,VCPU_PURR(r9) 1585 std r6,VCPU_SPURR(r9) 1586 subf r5,r7,r5 1587 subf r6,r8,r6 1588 1589 /* 1590 * Restore host PURR/SPURR and add guest times 1591 * so that the time in the guest gets accounted. 1592 */ 1593 ld r3,HSTATE_PURR(r13) 1594 ld r4,HSTATE_SPURR(r13) 1595 add r3,r3,r5 1596 add r4,r4,r6 1597 mtspr SPRN_PURR,r3 1598 mtspr SPRN_SPURR,r4 1599 1600BEGIN_FTR_SECTION 1601 b 8f 1602END_FTR_SECTION_IFCLR(CPU_FTR_ARCH_207S) 1603 /* Save POWER8-specific registers */ 1604 mfspr r5, SPRN_IAMR 1605 mfspr r6, SPRN_PSPB 1606 mfspr r7, SPRN_FSCR 1607 std r5, VCPU_IAMR(r9) 1608 stw r6, VCPU_PSPB(r9) 1609 std r7, VCPU_FSCR(r9) 1610 mfspr r5, SPRN_IC 1611 mfspr r7, SPRN_TAR 1612 std r5, VCPU_IC(r9) 1613 std r7, VCPU_TAR(r9) 1614 mfspr r8, SPRN_EBBHR 1615 std r8, VCPU_EBBHR(r9) 1616 mfspr r5, SPRN_EBBRR 1617 mfspr r6, SPRN_BESCR 1618 mfspr r7, SPRN_PID 1619 mfspr r8, SPRN_WORT 1620 std r5, VCPU_EBBRR(r9) 1621 std r6, VCPU_BESCR(r9) 1622 stw r7, VCPU_GUEST_PID(r9) 1623 std r8, VCPU_WORT(r9) 1624BEGIN_FTR_SECTION 1625 mfspr r5, SPRN_TCSCR 1626 mfspr r6, SPRN_ACOP 1627 mfspr r7, SPRN_CSIGR 1628 mfspr r8, SPRN_TACR 1629 std r5, VCPU_TCSCR(r9) 1630 std r6, VCPU_ACOP(r9) 1631 std r7, VCPU_CSIGR(r9) 1632 std r8, VCPU_TACR(r9) 1633FTR_SECTION_ELSE 1634 mfspr r5, SPRN_TIDR 1635 mfspr r6, SPRN_PSSCR 1636 std r5, VCPU_TID(r9) 1637 rldicl r6, r6, 4, 50 /* r6 &= PSSCR_GUEST_VIS */ 1638 rotldi r6, r6, 60 1639 std r6, VCPU_PSSCR(r9) 1640 /* Restore host HFSCR value */ 1641 ld r7, STACK_SLOT_HFSCR(r1) 1642 mtspr SPRN_HFSCR, r7 1643ALT_FTR_SECTION_END_IFCLR(CPU_FTR_ARCH_300) 1644 /* 1645 * Restore various registers to 0, where non-zero values 1646 * set by the guest could disrupt the host. 1647 */ 1648 li r0, 0 1649 mtspr SPRN_PSPB, r0 1650 mtspr SPRN_WORT, r0 1651BEGIN_FTR_SECTION 1652 mtspr SPRN_TCSCR, r0 1653 /* Set MMCRS to 1<<31 to freeze and disable the SPMC counters */ 1654 li r0, 1 1655 sldi r0, r0, 31 1656 mtspr SPRN_MMCRS, r0 1657END_FTR_SECTION_IFCLR(CPU_FTR_ARCH_300) 1658 1659 /* Save and restore AMR, IAMR and UAMOR before turning on the MMU */ 1660 ld r8, STACK_SLOT_IAMR(r1) 1661 mtspr SPRN_IAMR, r8 1662 16638: /* Power7 jumps back in here */ 1664 mfspr r5,SPRN_AMR 1665 mfspr r6,SPRN_UAMOR 1666 std r5,VCPU_AMR(r9) 1667 std r6,VCPU_UAMOR(r9) 1668 ld r5,STACK_SLOT_AMR(r1) 1669 ld r6,STACK_SLOT_UAMOR(r1) 1670 mtspr SPRN_AMR, r5 1671 mtspr SPRN_UAMOR, r6 1672 1673 /* Switch DSCR back to host value */ 1674 mfspr r8, SPRN_DSCR 1675 ld r7, HSTATE_DSCR(r13) 1676 std r8, VCPU_DSCR(r9) 1677 mtspr SPRN_DSCR, r7 1678 1679 /* Save non-volatile GPRs */ 1680 std r14, VCPU_GPR(R14)(r9) 1681 std r15, VCPU_GPR(R15)(r9) 1682 std r16, VCPU_GPR(R16)(r9) 1683 std r17, VCPU_GPR(R17)(r9) 1684 std r18, VCPU_GPR(R18)(r9) 1685 std r19, VCPU_GPR(R19)(r9) 1686 std r20, VCPU_GPR(R20)(r9) 1687 std r21, VCPU_GPR(R21)(r9) 1688 std r22, VCPU_GPR(R22)(r9) 1689 std r23, VCPU_GPR(R23)(r9) 1690 std r24, VCPU_GPR(R24)(r9) 1691 std r25, VCPU_GPR(R25)(r9) 1692 std r26, VCPU_GPR(R26)(r9) 1693 std r27, VCPU_GPR(R27)(r9) 1694 std r28, VCPU_GPR(R28)(r9) 1695 std r29, VCPU_GPR(R29)(r9) 1696 std r30, VCPU_GPR(R30)(r9) 1697 std r31, VCPU_GPR(R31)(r9) 1698 1699 /* Save SPRGs */ 1700 mfspr r3, SPRN_SPRG0 1701 mfspr r4, SPRN_SPRG1 1702 mfspr r5, SPRN_SPRG2 1703 mfspr r6, SPRN_SPRG3 1704 std r3, VCPU_SPRG0(r9) 1705 std r4, VCPU_SPRG1(r9) 1706 std r5, VCPU_SPRG2(r9) 1707 std r6, VCPU_SPRG3(r9) 1708 1709 /* save FP state */ 1710 mr r3, r9 1711 bl kvmppc_save_fp 1712 1713#ifdef CONFIG_PPC_TRANSACTIONAL_MEM 1714/* 1715 * Branch around the call if both CPU_FTR_TM and 1716 * CPU_FTR_P9_TM_HV_ASSIST are off. 1717 */ 1718BEGIN_FTR_SECTION 1719 b 91f 1720END_FTR_SECTION(CPU_FTR_TM | CPU_FTR_P9_TM_HV_ASSIST, 0) 1721 /* 1722 * NOTE THAT THIS TRASHES ALL NON-VOLATILE REGISTERS (but not CR) 1723 */ 1724 mr r3, r9 1725 ld r4, VCPU_MSR(r3) 1726 li r5, 0 /* don't preserve non-vol regs */ 1727 bl kvmppc_save_tm_hv 1728 nop 1729 ld r9, HSTATE_KVM_VCPU(r13) 173091: 1731#endif 1732 1733 /* Increment yield count if they have a VPA */ 1734 ld r8, VCPU_VPA(r9) /* do they have a VPA? */ 1735 cmpdi r8, 0 1736 beq 25f 1737 li r4, LPPACA_YIELDCOUNT 1738 LWZX_BE r3, r8, r4 1739 addi r3, r3, 1 1740 STWX_BE r3, r8, r4 1741 li r3, 1 1742 stb r3, VCPU_VPA_DIRTY(r9) 174325: 1744 /* Save PMU registers if requested */ 1745 /* r8 and cr0.eq are live here */ 1746 mr r3, r9 1747 li r4, 1 1748 beq 21f /* if no VPA, save PMU stuff anyway */ 1749 lbz r4, LPPACA_PMCINUSE(r8) 175021: bl kvmhv_save_guest_pmu 1751 ld r9, HSTATE_KVM_VCPU(r13) 1752 1753 /* Restore host values of some registers */ 1754BEGIN_FTR_SECTION 1755 ld r5, STACK_SLOT_CIABR(r1) 1756 ld r6, STACK_SLOT_DAWR(r1) 1757 ld r7, STACK_SLOT_DAWRX(r1) 1758 mtspr SPRN_CIABR, r5 1759 /* 1760 * If the DAWR doesn't work, it's ok to write these here as 1761 * this value should always be zero 1762 */ 1763 mtspr SPRN_DAWR, r6 1764 mtspr SPRN_DAWRX, r7 1765END_FTR_SECTION_IFSET(CPU_FTR_ARCH_207S) 1766BEGIN_FTR_SECTION 1767 ld r5, STACK_SLOT_TID(r1) 1768 ld r6, STACK_SLOT_PSSCR(r1) 1769 ld r7, STACK_SLOT_PID(r1) 1770 mtspr SPRN_TIDR, r5 1771 mtspr SPRN_PSSCR, r6 1772 mtspr SPRN_PID, r7 1773END_FTR_SECTION_IFSET(CPU_FTR_ARCH_300) 1774 1775#ifdef CONFIG_PPC_RADIX_MMU 1776 /* 1777 * Are we running hash or radix ? 1778 */ 1779 ld r5, VCPU_KVM(r9) 1780 lbz r0, KVM_RADIX(r5) 1781 cmpwi cr2, r0, 0 1782 beq cr2, 2f 1783 1784 /* 1785 * Radix: do eieio; tlbsync; ptesync sequence in case we 1786 * interrupted the guest between a tlbie and a ptesync. 1787 */ 1788 eieio 1789 tlbsync 1790 ptesync 1791 1792 /* Radix: Handle the case where the guest used an illegal PID */ 1793 LOAD_REG_ADDR(r4, mmu_base_pid) 1794 lwz r3, VCPU_GUEST_PID(r9) 1795 lwz r5, 0(r4) 1796 cmpw cr0,r3,r5 1797 blt 2f 1798 1799 /* 1800 * Illegal PID, the HW might have prefetched and cached in the TLB 1801 * some translations for the LPID 0 / guest PID combination which 1802 * Linux doesn't know about, so we need to flush that PID out of 1803 * the TLB. First we need to set LPIDR to 0 so tlbiel applies to 1804 * the right context. 1805 */ 1806 li r0,0 1807 mtspr SPRN_LPID,r0 1808 isync 1809 1810 /* Then do a congruence class local flush */ 1811 ld r6,VCPU_KVM(r9) 1812 lwz r0,KVM_TLB_SETS(r6) 1813 mtctr r0 1814 li r7,0x400 /* IS field = 0b01 */ 1815 ptesync 1816 sldi r0,r3,32 /* RS has PID */ 18171: PPC_TLBIEL(7,0,2,1,1) /* RIC=2, PRS=1, R=1 */ 1818 addi r7,r7,0x1000 1819 bdnz 1b 1820 ptesync 1821 18222: 1823#endif /* CONFIG_PPC_RADIX_MMU */ 1824 1825 /* 1826 * POWER7/POWER8 guest -> host partition switch code. 1827 * We don't have to lock against tlbies but we do 1828 * have to coordinate the hardware threads. 1829 * Here STACK_SLOT_TRAP(r1) contains the trap number. 1830 */ 1831kvmhv_switch_to_host: 1832 /* Secondary threads wait for primary to do partition switch */ 1833 ld r5,HSTATE_KVM_VCORE(r13) 1834 ld r4,VCORE_KVM(r5) /* pointer to struct kvm */ 1835 lbz r3,HSTATE_PTID(r13) 1836 cmpwi r3,0 1837 beq 15f 1838 HMT_LOW 183913: lbz r3,VCORE_IN_GUEST(r5) 1840 cmpwi r3,0 1841 bne 13b 1842 HMT_MEDIUM 1843 b 16f 1844 1845 /* Primary thread waits for all the secondaries to exit guest */ 184615: lwz r3,VCORE_ENTRY_EXIT(r5) 1847 rlwinm r0,r3,32-8,0xff 1848 clrldi r3,r3,56 1849 cmpw r3,r0 1850 bne 15b 1851 isync 1852 1853 /* Did we actually switch to the guest at all? */ 1854 lbz r6, VCORE_IN_GUEST(r5) 1855 cmpwi r6, 0 1856 beq 19f 1857 1858 /* Primary thread switches back to host partition */ 1859 lwz r7,KVM_HOST_LPID(r4) 1860BEGIN_FTR_SECTION 1861 ld r6,KVM_HOST_SDR1(r4) 1862 li r8,LPID_RSVD /* switch to reserved LPID */ 1863 mtspr SPRN_LPID,r8 1864 ptesync 1865 mtspr SPRN_SDR1,r6 /* switch to host page table */ 1866END_FTR_SECTION_IFCLR(CPU_FTR_ARCH_300) 1867 mtspr SPRN_LPID,r7 1868 isync 1869 1870BEGIN_FTR_SECTION 1871 /* DPDES and VTB are shared between threads */ 1872 mfspr r7, SPRN_DPDES 1873 mfspr r8, SPRN_VTB 1874 std r7, VCORE_DPDES(r5) 1875 std r8, VCORE_VTB(r5) 1876 /* clear DPDES so we don't get guest doorbells in the host */ 1877 li r8, 0 1878 mtspr SPRN_DPDES, r8 1879END_FTR_SECTION_IFSET(CPU_FTR_ARCH_207S) 1880 1881 /* Subtract timebase offset from timebase */ 1882 ld r8, VCORE_TB_OFFSET_APPL(r5) 1883 cmpdi r8,0 1884 beq 17f 1885 li r0, 0 1886 std r0, VCORE_TB_OFFSET_APPL(r5) 1887 mftb r6 /* current guest timebase */ 1888 subf r8,r8,r6 1889 mtspr SPRN_TBU40,r8 /* update upper 40 bits */ 1890 mftb r7 /* check if lower 24 bits overflowed */ 1891 clrldi r6,r6,40 1892 clrldi r7,r7,40 1893 cmpld r7,r6 1894 bge 17f 1895 addis r8,r8,0x100 /* if so, increment upper 40 bits */ 1896 mtspr SPRN_TBU40,r8 1897 189817: 1899 /* 1900 * If this is an HMI, we called kvmppc_realmode_hmi_handler 1901 * above, which may or may not have already called 1902 * kvmppc_subcore_exit_guest. Fortunately, all that 1903 * kvmppc_subcore_exit_guest does is clear a flag, so calling 1904 * it again here is benign even if kvmppc_realmode_hmi_handler 1905 * has already called it. 1906 */ 1907 bl kvmppc_subcore_exit_guest 1908 nop 190930: ld r5,HSTATE_KVM_VCORE(r13) 1910 ld r4,VCORE_KVM(r5) /* pointer to struct kvm */ 1911 1912 /* Reset PCR */ 1913 ld r0, VCORE_PCR(r5) 1914 cmpdi r0, 0 1915 beq 18f 1916 li r0, 0 1917 mtspr SPRN_PCR, r0 191818: 1919 /* Signal secondary CPUs to continue */ 1920 stb r0,VCORE_IN_GUEST(r5) 192119: lis r8,0x7fff /* MAX_INT@h */ 1922 mtspr SPRN_HDEC,r8 1923 192416: 1925BEGIN_FTR_SECTION 1926 /* On POWER9 with HPT-on-radix we need to wait for all other threads */ 1927 ld r3, HSTATE_SPLIT_MODE(r13) 1928 cmpdi r3, 0 1929 beq 47f 1930 lwz r8, KVM_SPLIT_DO_RESTORE(r3) 1931 cmpwi r8, 0 1932 beq 47f 1933 bl kvmhv_p9_restore_lpcr 1934 nop 1935 b 48f 193647: 1937END_FTR_SECTION_IFSET(CPU_FTR_ARCH_300) 1938 ld r8,KVM_HOST_LPCR(r4) 1939 mtspr SPRN_LPCR,r8 1940 isync 194148: 1942#ifdef CONFIG_KVM_BOOK3S_HV_EXIT_TIMING 1943 /* Finish timing, if we have a vcpu */ 1944 ld r4, HSTATE_KVM_VCPU(r13) 1945 cmpdi r4, 0 1946 li r3, 0 1947 beq 2f 1948 bl kvmhv_accumulate_time 19492: 1950#endif 1951 /* Unset guest mode */ 1952 li r0, KVM_GUEST_MODE_NONE 1953 stb r0, HSTATE_IN_GUEST(r13) 1954 1955 lwz r12, STACK_SLOT_TRAP(r1) /* return trap # in r12 */ 1956 ld r0, SFS+PPC_LR_STKOFF(r1) 1957 addi r1, r1, SFS 1958 mtlr r0 1959 blr 1960 1961kvmppc_guest_external: 1962 /* External interrupt, first check for host_ipi. If this is 1963 * set, we know the host wants us out so let's do it now 1964 */ 1965 bl kvmppc_read_intr 1966 1967 /* 1968 * Restore the active volatile registers after returning from 1969 * a C function. 1970 */ 1971 ld r9, HSTATE_KVM_VCPU(r13) 1972 li r12, BOOK3S_INTERRUPT_EXTERNAL 1973 1974 /* 1975 * kvmppc_read_intr return codes: 1976 * 1977 * Exit to host (r3 > 0) 1978 * 1 An interrupt is pending that needs to be handled by the host 1979 * Exit guest and return to host by branching to guest_exit_cont 1980 * 1981 * 2 Passthrough that needs completion in the host 1982 * Exit guest and return to host by branching to guest_exit_cont 1983 * However, we also set r12 to BOOK3S_INTERRUPT_HV_RM_HARD 1984 * to indicate to the host to complete handling the interrupt 1985 * 1986 * Before returning to guest, we check if any CPU is heading out 1987 * to the host and if so, we head out also. If no CPUs are heading 1988 * check return values <= 0. 1989 * 1990 * Return to guest (r3 <= 0) 1991 * 0 No external interrupt is pending 1992 * -1 A guest wakeup IPI (which has now been cleared) 1993 * In either case, we return to guest to deliver any pending 1994 * guest interrupts. 1995 * 1996 * -2 A PCI passthrough external interrupt was handled 1997 * (interrupt was delivered directly to guest) 1998 * Return to guest to deliver any pending guest interrupts. 1999 */ 2000 2001 cmpdi r3, 1 2002 ble 1f 2003 2004 /* Return code = 2 */ 2005 li r12, BOOK3S_INTERRUPT_HV_RM_HARD 2006 stw r12, VCPU_TRAP(r9) 2007 b guest_exit_cont 2008 20091: /* Return code <= 1 */ 2010 cmpdi r3, 0 2011 bgt guest_exit_cont 2012 2013 /* Return code <= 0 */ 2014maybe_reenter_guest: 2015 ld r5, HSTATE_KVM_VCORE(r13) 2016 lwz r0, VCORE_ENTRY_EXIT(r5) 2017 cmpwi r0, 0x100 2018 mr r4, r9 2019 blt deliver_guest_interrupt 2020 b guest_exit_cont 2021 2022#ifdef CONFIG_PPC_TRANSACTIONAL_MEM 2023/* 2024 * Softpatch interrupt for transactional memory emulation cases 2025 * on POWER9 DD2.2. This is early in the guest exit path - we 2026 * haven't saved registers or done a treclaim yet. 2027 */ 2028kvmppc_tm_emul: 2029 /* Save instruction image in HEIR */ 2030 mfspr r3, SPRN_HEIR 2031 stw r3, VCPU_HEIR(r9) 2032 2033 /* 2034 * The cases we want to handle here are those where the guest 2035 * is in real suspend mode and is trying to transition to 2036 * transactional mode. 2037 */ 2038 lbz r0, HSTATE_FAKE_SUSPEND(r13) 2039 cmpwi r0, 0 /* keep exiting guest if in fake suspend */ 2040 bne guest_exit_cont 2041 rldicl r3, r11, 64 - MSR_TS_S_LG, 62 2042 cmpwi r3, 1 /* or if not in suspend state */ 2043 bne guest_exit_cont 2044 2045 /* Call C code to do the emulation */ 2046 mr r3, r9 2047 bl kvmhv_p9_tm_emulation_early 2048 nop 2049 ld r9, HSTATE_KVM_VCPU(r13) 2050 li r12, BOOK3S_INTERRUPT_HV_SOFTPATCH 2051 cmpwi r3, 0 2052 beq guest_exit_cont /* continue exiting if not handled */ 2053 ld r10, VCPU_PC(r9) 2054 ld r11, VCPU_MSR(r9) 2055 b fast_interrupt_c_return /* go back to guest if handled */ 2056#endif /* CONFIG_PPC_TRANSACTIONAL_MEM */ 2057 2058/* 2059 * Check whether an HDSI is an HPTE not found fault or something else. 2060 * If it is an HPTE not found fault that is due to the guest accessing 2061 * a page that they have mapped but which we have paged out, then 2062 * we continue on with the guest exit path. In all other cases, 2063 * reflect the HDSI to the guest as a DSI. 2064 */ 2065kvmppc_hdsi: 2066 ld r3, VCPU_KVM(r9) 2067 lbz r0, KVM_RADIX(r3) 2068 mfspr r4, SPRN_HDAR 2069 mfspr r6, SPRN_HDSISR 2070BEGIN_FTR_SECTION 2071 /* Look for DSISR canary. If we find it, retry instruction */ 2072 cmpdi r6, 0x7fff 2073 beq 6f 2074END_FTR_SECTION_IFSET(CPU_FTR_ARCH_300) 2075 cmpwi r0, 0 2076 bne .Lradix_hdsi /* on radix, just save DAR/DSISR/ASDR */ 2077 /* HPTE not found fault or protection fault? */ 2078 andis. r0, r6, (DSISR_NOHPTE | DSISR_PROTFAULT)@h 2079 beq 1f /* if not, send it to the guest */ 2080 andi. r0, r11, MSR_DR /* data relocation enabled? */ 2081 beq 3f 2082BEGIN_FTR_SECTION 2083 mfspr r5, SPRN_ASDR /* on POWER9, use ASDR to get VSID */ 2084 b 4f 2085END_FTR_SECTION_IFSET(CPU_FTR_ARCH_300) 2086 clrrdi r0, r4, 28 2087 PPC_SLBFEE_DOT(R5, R0) /* if so, look up SLB */ 2088 li r0, BOOK3S_INTERRUPT_DATA_SEGMENT 2089 bne 7f /* if no SLB entry found */ 20904: std r4, VCPU_FAULT_DAR(r9) 2091 stw r6, VCPU_FAULT_DSISR(r9) 2092 2093 /* Search the hash table. */ 2094 mr r3, r9 /* vcpu pointer */ 2095 li r7, 1 /* data fault */ 2096 bl kvmppc_hpte_hv_fault 2097 ld r9, HSTATE_KVM_VCPU(r13) 2098 ld r10, VCPU_PC(r9) 2099 ld r11, VCPU_MSR(r9) 2100 li r12, BOOK3S_INTERRUPT_H_DATA_STORAGE 2101 cmpdi r3, 0 /* retry the instruction */ 2102 beq 6f 2103 cmpdi r3, -1 /* handle in kernel mode */ 2104 beq guest_exit_cont 2105 cmpdi r3, -2 /* MMIO emulation; need instr word */ 2106 beq 2f 2107 2108 /* Synthesize a DSI (or DSegI) for the guest */ 2109 ld r4, VCPU_FAULT_DAR(r9) 2110 mr r6, r3 21111: li r0, BOOK3S_INTERRUPT_DATA_STORAGE 2112 mtspr SPRN_DSISR, r6 21137: mtspr SPRN_DAR, r4 2114 mtspr SPRN_SRR0, r10 2115 mtspr SPRN_SRR1, r11 2116 mr r10, r0 2117 bl kvmppc_msr_interrupt 2118fast_interrupt_c_return: 21196: ld r7, VCPU_CTR(r9) 2120 ld r8, VCPU_XER(r9) 2121 mtctr r7 2122 mtxer r8 2123 mr r4, r9 2124 b fast_guest_return 2125 21263: ld r5, VCPU_KVM(r9) /* not relocated, use VRMA */ 2127 ld r5, KVM_VRMA_SLB_V(r5) 2128 b 4b 2129 2130 /* If this is for emulated MMIO, load the instruction word */ 21312: li r8, KVM_INST_FETCH_FAILED /* In case lwz faults */ 2132 2133 /* Set guest mode to 'jump over instruction' so if lwz faults 2134 * we'll just continue at the next IP. */ 2135 li r0, KVM_GUEST_MODE_SKIP 2136 stb r0, HSTATE_IN_GUEST(r13) 2137 2138 /* Do the access with MSR:DR enabled */ 2139 mfmsr r3 2140 ori r4, r3, MSR_DR /* Enable paging for data */ 2141 mtmsrd r4 2142 lwz r8, 0(r10) 2143 mtmsrd r3 2144 2145 /* Store the result */ 2146 stw r8, VCPU_LAST_INST(r9) 2147 2148 /* Unset guest mode. */ 2149 li r0, KVM_GUEST_MODE_HOST_HV 2150 stb r0, HSTATE_IN_GUEST(r13) 2151 b guest_exit_cont 2152 2153.Lradix_hdsi: 2154 std r4, VCPU_FAULT_DAR(r9) 2155 stw r6, VCPU_FAULT_DSISR(r9) 2156.Lradix_hisi: 2157 mfspr r5, SPRN_ASDR 2158 std r5, VCPU_FAULT_GPA(r9) 2159 b guest_exit_cont 2160 2161/* 2162 * Similarly for an HISI, reflect it to the guest as an ISI unless 2163 * it is an HPTE not found fault for a page that we have paged out. 2164 */ 2165kvmppc_hisi: 2166 ld r3, VCPU_KVM(r9) 2167 lbz r0, KVM_RADIX(r3) 2168 cmpwi r0, 0 2169 bne .Lradix_hisi /* for radix, just save ASDR */ 2170 andis. r0, r11, SRR1_ISI_NOPT@h 2171 beq 1f 2172 andi. r0, r11, MSR_IR /* instruction relocation enabled? */ 2173 beq 3f 2174BEGIN_FTR_SECTION 2175 mfspr r5, SPRN_ASDR /* on POWER9, use ASDR to get VSID */ 2176 b 4f 2177END_FTR_SECTION_IFSET(CPU_FTR_ARCH_300) 2178 clrrdi r0, r10, 28 2179 PPC_SLBFEE_DOT(R5, R0) /* if so, look up SLB */ 2180 li r0, BOOK3S_INTERRUPT_INST_SEGMENT 2181 bne 7f /* if no SLB entry found */ 21824: 2183 /* Search the hash table. */ 2184 mr r3, r9 /* vcpu pointer */ 2185 mr r4, r10 2186 mr r6, r11 2187 li r7, 0 /* instruction fault */ 2188 bl kvmppc_hpte_hv_fault 2189 ld r9, HSTATE_KVM_VCPU(r13) 2190 ld r10, VCPU_PC(r9) 2191 ld r11, VCPU_MSR(r9) 2192 li r12, BOOK3S_INTERRUPT_H_INST_STORAGE 2193 cmpdi r3, 0 /* retry the instruction */ 2194 beq fast_interrupt_c_return 2195 cmpdi r3, -1 /* handle in kernel mode */ 2196 beq guest_exit_cont 2197 2198 /* Synthesize an ISI (or ISegI) for the guest */ 2199 mr r11, r3 22001: li r0, BOOK3S_INTERRUPT_INST_STORAGE 22017: mtspr SPRN_SRR0, r10 2202 mtspr SPRN_SRR1, r11 2203 mr r10, r0 2204 bl kvmppc_msr_interrupt 2205 b fast_interrupt_c_return 2206 22073: ld r6, VCPU_KVM(r9) /* not relocated, use VRMA */ 2208 ld r5, KVM_VRMA_SLB_V(r6) 2209 b 4b 2210 2211/* 2212 * Try to handle an hcall in real mode. 2213 * Returns to the guest if we handle it, or continues on up to 2214 * the kernel if we can't (i.e. if we don't have a handler for 2215 * it, or if the handler returns H_TOO_HARD). 2216 * 2217 * r5 - r8 contain hcall args, 2218 * r9 = vcpu, r10 = pc, r11 = msr, r12 = trap, r13 = paca 2219 */ 2220hcall_try_real_mode: 2221 ld r3,VCPU_GPR(R3)(r9) 2222 andi. r0,r11,MSR_PR 2223 /* sc 1 from userspace - reflect to guest syscall */ 2224 bne sc_1_fast_return 2225 /* sc 1 from nested guest - give it to L1 to handle */ 2226 ld r0, VCPU_NESTED(r9) 2227 cmpdi r0, 0 2228 bne guest_exit_cont 2229 clrrdi r3,r3,2 2230 cmpldi r3,hcall_real_table_end - hcall_real_table 2231 bge guest_exit_cont 2232 /* See if this hcall is enabled for in-kernel handling */ 2233 ld r4, VCPU_KVM(r9) 2234 srdi r0, r3, 8 /* r0 = (r3 / 4) >> 6 */ 2235 sldi r0, r0, 3 /* index into kvm->arch.enabled_hcalls[] */ 2236 add r4, r4, r0 2237 ld r0, KVM_ENABLED_HCALLS(r4) 2238 rlwinm r4, r3, 32-2, 0x3f /* r4 = (r3 / 4) & 0x3f */ 2239 srd r0, r0, r4 2240 andi. r0, r0, 1 2241 beq guest_exit_cont 2242 /* Get pointer to handler, if any, and call it */ 2243 LOAD_REG_ADDR(r4, hcall_real_table) 2244 lwax r3,r3,r4 2245 cmpwi r3,0 2246 beq guest_exit_cont 2247 add r12,r3,r4 2248 mtctr r12 2249 mr r3,r9 /* get vcpu pointer */ 2250 ld r4,VCPU_GPR(R4)(r9) 2251 bctrl 2252 cmpdi r3,H_TOO_HARD 2253 beq hcall_real_fallback 2254 ld r4,HSTATE_KVM_VCPU(r13) 2255 std r3,VCPU_GPR(R3)(r4) 2256 ld r10,VCPU_PC(r4) 2257 ld r11,VCPU_MSR(r4) 2258 b fast_guest_return 2259 2260sc_1_fast_return: 2261 mtspr SPRN_SRR0,r10 2262 mtspr SPRN_SRR1,r11 2263 li r10, BOOK3S_INTERRUPT_SYSCALL 2264 bl kvmppc_msr_interrupt 2265 mr r4,r9 2266 b fast_guest_return 2267 2268 /* We've attempted a real mode hcall, but it's punted it back 2269 * to userspace. We need to restore some clobbered volatiles 2270 * before resuming the pass-it-to-qemu path */ 2271hcall_real_fallback: 2272 li r12,BOOK3S_INTERRUPT_SYSCALL 2273 ld r9, HSTATE_KVM_VCPU(r13) 2274 2275 b guest_exit_cont 2276 2277 .globl hcall_real_table 2278hcall_real_table: 2279 .long 0 /* 0 - unused */ 2280 .long DOTSYM(kvmppc_h_remove) - hcall_real_table 2281 .long DOTSYM(kvmppc_h_enter) - hcall_real_table 2282 .long DOTSYM(kvmppc_h_read) - hcall_real_table 2283 .long DOTSYM(kvmppc_h_clear_mod) - hcall_real_table 2284 .long DOTSYM(kvmppc_h_clear_ref) - hcall_real_table 2285 .long DOTSYM(kvmppc_h_protect) - hcall_real_table 2286#ifdef CONFIG_SPAPR_TCE_IOMMU 2287 .long DOTSYM(kvmppc_h_get_tce) - hcall_real_table 2288 .long DOTSYM(kvmppc_rm_h_put_tce) - hcall_real_table 2289#else 2290 .long 0 /* 0x1c */ 2291 .long 0 /* 0x20 */ 2292#endif 2293 .long 0 /* 0x24 - H_SET_SPRG0 */ 2294 .long DOTSYM(kvmppc_h_set_dabr) - hcall_real_table 2295 .long 0 /* 0x2c */ 2296 .long 0 /* 0x30 */ 2297 .long 0 /* 0x34 */ 2298 .long 0 /* 0x38 */ 2299 .long 0 /* 0x3c */ 2300 .long 0 /* 0x40 */ 2301 .long 0 /* 0x44 */ 2302 .long 0 /* 0x48 */ 2303 .long 0 /* 0x4c */ 2304 .long 0 /* 0x50 */ 2305 .long 0 /* 0x54 */ 2306 .long 0 /* 0x58 */ 2307 .long 0 /* 0x5c */ 2308 .long 0 /* 0x60 */ 2309#ifdef CONFIG_KVM_XICS 2310 .long DOTSYM(kvmppc_rm_h_eoi) - hcall_real_table 2311 .long DOTSYM(kvmppc_rm_h_cppr) - hcall_real_table 2312 .long DOTSYM(kvmppc_rm_h_ipi) - hcall_real_table 2313 .long DOTSYM(kvmppc_rm_h_ipoll) - hcall_real_table 2314 .long DOTSYM(kvmppc_rm_h_xirr) - hcall_real_table 2315#else 2316 .long 0 /* 0x64 - H_EOI */ 2317 .long 0 /* 0x68 - H_CPPR */ 2318 .long 0 /* 0x6c - H_IPI */ 2319 .long 0 /* 0x70 - H_IPOLL */ 2320 .long 0 /* 0x74 - H_XIRR */ 2321#endif 2322 .long 0 /* 0x78 */ 2323 .long 0 /* 0x7c */ 2324 .long 0 /* 0x80 */ 2325 .long 0 /* 0x84 */ 2326 .long 0 /* 0x88 */ 2327 .long 0 /* 0x8c */ 2328 .long 0 /* 0x90 */ 2329 .long 0 /* 0x94 */ 2330 .long 0 /* 0x98 */ 2331 .long 0 /* 0x9c */ 2332 .long 0 /* 0xa0 */ 2333 .long 0 /* 0xa4 */ 2334 .long 0 /* 0xa8 */ 2335 .long 0 /* 0xac */ 2336 .long 0 /* 0xb0 */ 2337 .long 0 /* 0xb4 */ 2338 .long 0 /* 0xb8 */ 2339 .long 0 /* 0xbc */ 2340 .long 0 /* 0xc0 */ 2341 .long 0 /* 0xc4 */ 2342 .long 0 /* 0xc8 */ 2343 .long 0 /* 0xcc */ 2344 .long 0 /* 0xd0 */ 2345 .long 0 /* 0xd4 */ 2346 .long 0 /* 0xd8 */ 2347 .long 0 /* 0xdc */ 2348 .long DOTSYM(kvmppc_h_cede) - hcall_real_table 2349 .long DOTSYM(kvmppc_rm_h_confer) - hcall_real_table 2350 .long 0 /* 0xe8 */ 2351 .long 0 /* 0xec */ 2352 .long 0 /* 0xf0 */ 2353 .long 0 /* 0xf4 */ 2354 .long 0 /* 0xf8 */ 2355 .long 0 /* 0xfc */ 2356 .long 0 /* 0x100 */ 2357 .long 0 /* 0x104 */ 2358 .long 0 /* 0x108 */ 2359 .long 0 /* 0x10c */ 2360 .long 0 /* 0x110 */ 2361 .long 0 /* 0x114 */ 2362 .long 0 /* 0x118 */ 2363 .long 0 /* 0x11c */ 2364 .long 0 /* 0x120 */ 2365 .long DOTSYM(kvmppc_h_bulk_remove) - hcall_real_table 2366 .long 0 /* 0x128 */ 2367 .long 0 /* 0x12c */ 2368 .long 0 /* 0x130 */ 2369 .long DOTSYM(kvmppc_h_set_xdabr) - hcall_real_table 2370#ifdef CONFIG_SPAPR_TCE_IOMMU 2371 .long DOTSYM(kvmppc_rm_h_stuff_tce) - hcall_real_table 2372 .long DOTSYM(kvmppc_rm_h_put_tce_indirect) - hcall_real_table 2373#else 2374 .long 0 /* 0x138 */ 2375 .long 0 /* 0x13c */ 2376#endif 2377 .long 0 /* 0x140 */ 2378 .long 0 /* 0x144 */ 2379 .long 0 /* 0x148 */ 2380 .long 0 /* 0x14c */ 2381 .long 0 /* 0x150 */ 2382 .long 0 /* 0x154 */ 2383 .long 0 /* 0x158 */ 2384 .long 0 /* 0x15c */ 2385 .long 0 /* 0x160 */ 2386 .long 0 /* 0x164 */ 2387 .long 0 /* 0x168 */ 2388 .long 0 /* 0x16c */ 2389 .long 0 /* 0x170 */ 2390 .long 0 /* 0x174 */ 2391 .long 0 /* 0x178 */ 2392 .long 0 /* 0x17c */ 2393 .long 0 /* 0x180 */ 2394 .long 0 /* 0x184 */ 2395 .long 0 /* 0x188 */ 2396 .long 0 /* 0x18c */ 2397 .long 0 /* 0x190 */ 2398 .long 0 /* 0x194 */ 2399 .long 0 /* 0x198 */ 2400 .long 0 /* 0x19c */ 2401 .long 0 /* 0x1a0 */ 2402 .long 0 /* 0x1a4 */ 2403 .long 0 /* 0x1a8 */ 2404 .long 0 /* 0x1ac */ 2405 .long 0 /* 0x1b0 */ 2406 .long 0 /* 0x1b4 */ 2407 .long 0 /* 0x1b8 */ 2408 .long 0 /* 0x1bc */ 2409 .long 0 /* 0x1c0 */ 2410 .long 0 /* 0x1c4 */ 2411 .long 0 /* 0x1c8 */ 2412 .long 0 /* 0x1cc */ 2413 .long 0 /* 0x1d0 */ 2414 .long 0 /* 0x1d4 */ 2415 .long 0 /* 0x1d8 */ 2416 .long 0 /* 0x1dc */ 2417 .long 0 /* 0x1e0 */ 2418 .long 0 /* 0x1e4 */ 2419 .long 0 /* 0x1e8 */ 2420 .long 0 /* 0x1ec */ 2421 .long 0 /* 0x1f0 */ 2422 .long 0 /* 0x1f4 */ 2423 .long 0 /* 0x1f8 */ 2424 .long 0 /* 0x1fc */ 2425 .long 0 /* 0x200 */ 2426 .long 0 /* 0x204 */ 2427 .long 0 /* 0x208 */ 2428 .long 0 /* 0x20c */ 2429 .long 0 /* 0x210 */ 2430 .long 0 /* 0x214 */ 2431 .long 0 /* 0x218 */ 2432 .long 0 /* 0x21c */ 2433 .long 0 /* 0x220 */ 2434 .long 0 /* 0x224 */ 2435 .long 0 /* 0x228 */ 2436 .long 0 /* 0x22c */ 2437 .long 0 /* 0x230 */ 2438 .long 0 /* 0x234 */ 2439 .long 0 /* 0x238 */ 2440 .long 0 /* 0x23c */ 2441 .long 0 /* 0x240 */ 2442 .long 0 /* 0x244 */ 2443 .long 0 /* 0x248 */ 2444 .long 0 /* 0x24c */ 2445 .long 0 /* 0x250 */ 2446 .long 0 /* 0x254 */ 2447 .long 0 /* 0x258 */ 2448 .long 0 /* 0x25c */ 2449 .long 0 /* 0x260 */ 2450 .long 0 /* 0x264 */ 2451 .long 0 /* 0x268 */ 2452 .long 0 /* 0x26c */ 2453 .long 0 /* 0x270 */ 2454 .long 0 /* 0x274 */ 2455 .long 0 /* 0x278 */ 2456 .long 0 /* 0x27c */ 2457 .long 0 /* 0x280 */ 2458 .long 0 /* 0x284 */ 2459 .long 0 /* 0x288 */ 2460 .long 0 /* 0x28c */ 2461 .long 0 /* 0x290 */ 2462 .long 0 /* 0x294 */ 2463 .long 0 /* 0x298 */ 2464 .long 0 /* 0x29c */ 2465 .long 0 /* 0x2a0 */ 2466 .long 0 /* 0x2a4 */ 2467 .long 0 /* 0x2a8 */ 2468 .long 0 /* 0x2ac */ 2469 .long 0 /* 0x2b0 */ 2470 .long 0 /* 0x2b4 */ 2471 .long 0 /* 0x2b8 */ 2472 .long 0 /* 0x2bc */ 2473 .long 0 /* 0x2c0 */ 2474 .long 0 /* 0x2c4 */ 2475 .long 0 /* 0x2c8 */ 2476 .long 0 /* 0x2cc */ 2477 .long 0 /* 0x2d0 */ 2478 .long 0 /* 0x2d4 */ 2479 .long 0 /* 0x2d8 */ 2480 .long 0 /* 0x2dc */ 2481 .long 0 /* 0x2e0 */ 2482 .long 0 /* 0x2e4 */ 2483 .long 0 /* 0x2e8 */ 2484 .long 0 /* 0x2ec */ 2485 .long 0 /* 0x2f0 */ 2486 .long 0 /* 0x2f4 */ 2487 .long 0 /* 0x2f8 */ 2488#ifdef CONFIG_KVM_XICS 2489 .long DOTSYM(kvmppc_rm_h_xirr_x) - hcall_real_table 2490#else 2491 .long 0 /* 0x2fc - H_XIRR_X*/ 2492#endif 2493 .long DOTSYM(kvmppc_h_random) - hcall_real_table 2494 .globl hcall_real_table_end 2495hcall_real_table_end: 2496 2497_GLOBAL(kvmppc_h_set_xdabr) 2498EXPORT_SYMBOL_GPL(kvmppc_h_set_xdabr) 2499 andi. r0, r5, DABRX_USER | DABRX_KERNEL 2500 beq 6f 2501 li r0, DABRX_USER | DABRX_KERNEL | DABRX_BTI 2502 andc. r0, r5, r0 2503 beq 3f 25046: li r3, H_PARAMETER 2505 blr 2506 2507_GLOBAL(kvmppc_h_set_dabr) 2508EXPORT_SYMBOL_GPL(kvmppc_h_set_dabr) 2509 li r5, DABRX_USER | DABRX_KERNEL 25103: 2511BEGIN_FTR_SECTION 2512 b 2f 2513END_FTR_SECTION_IFSET(CPU_FTR_ARCH_207S) 2514 std r4,VCPU_DABR(r3) 2515 stw r5, VCPU_DABRX(r3) 2516 mtspr SPRN_DABRX, r5 2517 /* Work around P7 bug where DABR can get corrupted on mtspr */ 25181: mtspr SPRN_DABR,r4 2519 mfspr r5, SPRN_DABR 2520 cmpd r4, r5 2521 bne 1b 2522 isync 2523 li r3,0 2524 blr 2525 25262: 2527 LOAD_REG_ADDR(r11, dawr_force_enable) 2528 lbz r11, 0(r11) 2529 cmpdi r11, 0 2530 li r3, H_HARDWARE 2531 beqlr 2532 /* Emulate H_SET_DABR/X on P8 for the sake of compat mode guests */ 2533 rlwimi r5, r4, 5, DAWRX_DR | DAWRX_DW 2534 rlwimi r5, r4, 2, DAWRX_WT 2535 clrrdi r4, r4, 3 2536 std r4, VCPU_DAWR(r3) 2537 std r5, VCPU_DAWRX(r3) 2538 mtspr SPRN_DAWR, r4 2539 mtspr SPRN_DAWRX, r5 2540 li r3, 0 2541 blr 2542 2543_GLOBAL(kvmppc_h_cede) /* r3 = vcpu pointer, r11 = msr, r13 = paca */ 2544 ori r11,r11,MSR_EE 2545 std r11,VCPU_MSR(r3) 2546 li r0,1 2547 stb r0,VCPU_CEDED(r3) 2548 sync /* order setting ceded vs. testing prodded */ 2549 lbz r5,VCPU_PRODDED(r3) 2550 cmpwi r5,0 2551 bne kvm_cede_prodded 2552 li r12,0 /* set trap to 0 to say hcall is handled */ 2553 stw r12,VCPU_TRAP(r3) 2554 li r0,H_SUCCESS 2555 std r0,VCPU_GPR(R3)(r3) 2556 2557 /* 2558 * Set our bit in the bitmask of napping threads unless all the 2559 * other threads are already napping, in which case we send this 2560 * up to the host. 2561 */ 2562 ld r5,HSTATE_KVM_VCORE(r13) 2563 lbz r6,HSTATE_PTID(r13) 2564 lwz r8,VCORE_ENTRY_EXIT(r5) 2565 clrldi r8,r8,56 2566 li r0,1 2567 sld r0,r0,r6 2568 addi r6,r5,VCORE_NAPPING_THREADS 256931: lwarx r4,0,r6 2570 or r4,r4,r0 2571 cmpw r4,r8 2572 beq kvm_cede_exit 2573 stwcx. r4,0,r6 2574 bne 31b 2575 /* order napping_threads update vs testing entry_exit_map */ 2576 isync 2577 li r0,NAPPING_CEDE 2578 stb r0,HSTATE_NAPPING(r13) 2579 lwz r7,VCORE_ENTRY_EXIT(r5) 2580 cmpwi r7,0x100 2581 bge 33f /* another thread already exiting */ 2582 2583/* 2584 * Although not specifically required by the architecture, POWER7 2585 * preserves the following registers in nap mode, even if an SMT mode 2586 * switch occurs: SLB entries, PURR, SPURR, AMOR, UAMOR, AMR, SPRG0-3, 2587 * DAR, DSISR, DABR, DABRX, DSCR, PMCx, MMCRx, SIAR, SDAR. 2588 */ 2589 /* Save non-volatile GPRs */ 2590 std r14, VCPU_GPR(R14)(r3) 2591 std r15, VCPU_GPR(R15)(r3) 2592 std r16, VCPU_GPR(R16)(r3) 2593 std r17, VCPU_GPR(R17)(r3) 2594 std r18, VCPU_GPR(R18)(r3) 2595 std r19, VCPU_GPR(R19)(r3) 2596 std r20, VCPU_GPR(R20)(r3) 2597 std r21, VCPU_GPR(R21)(r3) 2598 std r22, VCPU_GPR(R22)(r3) 2599 std r23, VCPU_GPR(R23)(r3) 2600 std r24, VCPU_GPR(R24)(r3) 2601 std r25, VCPU_GPR(R25)(r3) 2602 std r26, VCPU_GPR(R26)(r3) 2603 std r27, VCPU_GPR(R27)(r3) 2604 std r28, VCPU_GPR(R28)(r3) 2605 std r29, VCPU_GPR(R29)(r3) 2606 std r30, VCPU_GPR(R30)(r3) 2607 std r31, VCPU_GPR(R31)(r3) 2608 2609 /* save FP state */ 2610 bl kvmppc_save_fp 2611 2612#ifdef CONFIG_PPC_TRANSACTIONAL_MEM 2613/* 2614 * Branch around the call if both CPU_FTR_TM and 2615 * CPU_FTR_P9_TM_HV_ASSIST are off. 2616 */ 2617BEGIN_FTR_SECTION 2618 b 91f 2619END_FTR_SECTION(CPU_FTR_TM | CPU_FTR_P9_TM_HV_ASSIST, 0) 2620 /* 2621 * NOTE THAT THIS TRASHES ALL NON-VOLATILE REGISTERS (but not CR) 2622 */ 2623 ld r3, HSTATE_KVM_VCPU(r13) 2624 ld r4, VCPU_MSR(r3) 2625 li r5, 0 /* don't preserve non-vol regs */ 2626 bl kvmppc_save_tm_hv 2627 nop 262891: 2629#endif 2630 2631 /* 2632 * Set DEC to the smaller of DEC and HDEC, so that we wake 2633 * no later than the end of our timeslice (HDEC interrupts 2634 * don't wake us from nap). 2635 */ 2636 mfspr r3, SPRN_DEC 2637 mfspr r4, SPRN_HDEC 2638 mftb r5 2639BEGIN_FTR_SECTION 2640 /* On P9 check whether the guest has large decrementer mode enabled */ 2641 ld r6, HSTATE_KVM_VCORE(r13) 2642 ld r6, VCORE_LPCR(r6) 2643 andis. r6, r6, LPCR_LD@h 2644 bne 68f 2645END_FTR_SECTION_IFSET(CPU_FTR_ARCH_300) 2646 extsw r3, r3 264768: EXTEND_HDEC(r4) 2648 cmpd r3, r4 2649 ble 67f 2650 mtspr SPRN_DEC, r4 265167: 2652 /* save expiry time of guest decrementer */ 2653 add r3, r3, r5 2654 ld r4, HSTATE_KVM_VCPU(r13) 2655 ld r5, HSTATE_KVM_VCORE(r13) 2656 ld r6, VCORE_TB_OFFSET_APPL(r5) 2657 subf r3, r6, r3 /* convert to host TB value */ 2658 std r3, VCPU_DEC_EXPIRES(r4) 2659 2660#ifdef CONFIG_KVM_BOOK3S_HV_EXIT_TIMING 2661 ld r4, HSTATE_KVM_VCPU(r13) 2662 addi r3, r4, VCPU_TB_CEDE 2663 bl kvmhv_accumulate_time 2664#endif 2665 2666 lis r3, LPCR_PECEDP@h /* Do wake on privileged doorbell */ 2667 2668 /* Go back to host stack */ 2669 ld r1, HSTATE_HOST_R1(r13) 2670 2671 /* 2672 * Take a nap until a decrementer or external or doobell interrupt 2673 * occurs, with PECE1 and PECE0 set in LPCR. 2674 * On POWER8, set PECEDH, and if we are ceding, also set PECEDP. 2675 * Also clear the runlatch bit before napping. 2676 */ 2677kvm_do_nap: 2678 mfspr r0, SPRN_CTRLF 2679 clrrdi r0, r0, 1 2680 mtspr SPRN_CTRLT, r0 2681 2682 li r0,1 2683 stb r0,HSTATE_HWTHREAD_REQ(r13) 2684 mfspr r5,SPRN_LPCR 2685 ori r5,r5,LPCR_PECE0 | LPCR_PECE1 2686BEGIN_FTR_SECTION 2687 ori r5, r5, LPCR_PECEDH 2688 rlwimi r5, r3, 0, LPCR_PECEDP 2689END_FTR_SECTION_IFSET(CPU_FTR_ARCH_207S) 2690 2691kvm_nap_sequence: /* desired LPCR value in r5 */ 2692BEGIN_FTR_SECTION 2693 /* 2694 * PSSCR bits: exit criterion = 1 (wakeup based on LPCR at sreset) 2695 * enable state loss = 1 (allow SMT mode switch) 2696 * requested level = 0 (just stop dispatching) 2697 */ 2698 lis r3, (PSSCR_EC | PSSCR_ESL)@h 2699 /* Set LPCR_PECE_HVEE bit to enable wakeup by HV interrupts */ 2700 li r4, LPCR_PECE_HVEE@higher 2701 sldi r4, r4, 32 2702 or r5, r5, r4 2703FTR_SECTION_ELSE 2704 li r3, PNV_THREAD_NAP 2705ALT_FTR_SECTION_END_IFSET(CPU_FTR_ARCH_300) 2706 mtspr SPRN_LPCR,r5 2707 isync 2708 2709BEGIN_FTR_SECTION 2710 bl isa300_idle_stop_mayloss 2711FTR_SECTION_ELSE 2712 bl isa206_idle_insn_mayloss 2713ALT_FTR_SECTION_END_IFSET(CPU_FTR_ARCH_300) 2714 2715 mfspr r0, SPRN_CTRLF 2716 ori r0, r0, 1 2717 mtspr SPRN_CTRLT, r0 2718 2719 mtspr SPRN_SRR1, r3 2720 2721 li r0, 0 2722 stb r0, PACA_FTRACE_ENABLED(r13) 2723 2724 li r0, KVM_HWTHREAD_IN_KVM 2725 stb r0, HSTATE_HWTHREAD_STATE(r13) 2726 2727 lbz r0, HSTATE_NAPPING(r13) 2728 cmpwi r0, NAPPING_CEDE 2729 beq kvm_end_cede 2730 cmpwi r0, NAPPING_NOVCPU 2731 beq kvm_novcpu_wakeup 2732 cmpwi r0, NAPPING_UNSPLIT 2733 beq kvm_unsplit_wakeup 2734 twi 31,0,0 /* Nap state must not be zero */ 2735 273633: mr r4, r3 2737 li r3, 0 2738 li r12, 0 2739 b 34f 2740 2741kvm_end_cede: 2742 /* Woken by external or decrementer interrupt */ 2743 2744 /* get vcpu pointer */ 2745 ld r4, HSTATE_KVM_VCPU(r13) 2746 2747#ifdef CONFIG_KVM_BOOK3S_HV_EXIT_TIMING 2748 addi r3, r4, VCPU_TB_RMINTR 2749 bl kvmhv_accumulate_time 2750#endif 2751 2752#ifdef CONFIG_PPC_TRANSACTIONAL_MEM 2753/* 2754 * Branch around the call if both CPU_FTR_TM and 2755 * CPU_FTR_P9_TM_HV_ASSIST are off. 2756 */ 2757BEGIN_FTR_SECTION 2758 b 91f 2759END_FTR_SECTION(CPU_FTR_TM | CPU_FTR_P9_TM_HV_ASSIST, 0) 2760 /* 2761 * NOTE THAT THIS TRASHES ALL NON-VOLATILE REGISTERS (but not CR) 2762 */ 2763 mr r3, r4 2764 ld r4, VCPU_MSR(r3) 2765 li r5, 0 /* don't preserve non-vol regs */ 2766 bl kvmppc_restore_tm_hv 2767 nop 2768 ld r4, HSTATE_KVM_VCPU(r13) 276991: 2770#endif 2771 2772 /* load up FP state */ 2773 bl kvmppc_load_fp 2774 2775 /* Restore guest decrementer */ 2776 ld r3, VCPU_DEC_EXPIRES(r4) 2777 ld r5, HSTATE_KVM_VCORE(r13) 2778 ld r6, VCORE_TB_OFFSET_APPL(r5) 2779 add r3, r3, r6 /* convert host TB to guest TB value */ 2780 mftb r7 2781 subf r3, r7, r3 2782 mtspr SPRN_DEC, r3 2783 2784 /* Load NV GPRS */ 2785 ld r14, VCPU_GPR(R14)(r4) 2786 ld r15, VCPU_GPR(R15)(r4) 2787 ld r16, VCPU_GPR(R16)(r4) 2788 ld r17, VCPU_GPR(R17)(r4) 2789 ld r18, VCPU_GPR(R18)(r4) 2790 ld r19, VCPU_GPR(R19)(r4) 2791 ld r20, VCPU_GPR(R20)(r4) 2792 ld r21, VCPU_GPR(R21)(r4) 2793 ld r22, VCPU_GPR(R22)(r4) 2794 ld r23, VCPU_GPR(R23)(r4) 2795 ld r24, VCPU_GPR(R24)(r4) 2796 ld r25, VCPU_GPR(R25)(r4) 2797 ld r26, VCPU_GPR(R26)(r4) 2798 ld r27, VCPU_GPR(R27)(r4) 2799 ld r28, VCPU_GPR(R28)(r4) 2800 ld r29, VCPU_GPR(R29)(r4) 2801 ld r30, VCPU_GPR(R30)(r4) 2802 ld r31, VCPU_GPR(R31)(r4) 2803 2804 /* Check the wake reason in SRR1 to see why we got here */ 2805 bl kvmppc_check_wake_reason 2806 2807 /* 2808 * Restore volatile registers since we could have called a 2809 * C routine in kvmppc_check_wake_reason 2810 * r4 = VCPU 2811 * r3 tells us whether we need to return to host or not 2812 * WARNING: it gets checked further down: 2813 * should not modify r3 until this check is done. 2814 */ 2815 ld r4, HSTATE_KVM_VCPU(r13) 2816 2817 /* clear our bit in vcore->napping_threads */ 281834: ld r5,HSTATE_KVM_VCORE(r13) 2819 lbz r7,HSTATE_PTID(r13) 2820 li r0,1 2821 sld r0,r0,r7 2822 addi r6,r5,VCORE_NAPPING_THREADS 282332: lwarx r7,0,r6 2824 andc r7,r7,r0 2825 stwcx. r7,0,r6 2826 bne 32b 2827 li r0,0 2828 stb r0,HSTATE_NAPPING(r13) 2829 2830 /* See if the wake reason saved in r3 means we need to exit */ 2831 stw r12, VCPU_TRAP(r4) 2832 mr r9, r4 2833 cmpdi r3, 0 2834 bgt guest_exit_cont 2835 b maybe_reenter_guest 2836 2837 /* cede when already previously prodded case */ 2838kvm_cede_prodded: 2839 li r0,0 2840 stb r0,VCPU_PRODDED(r3) 2841 sync /* order testing prodded vs. clearing ceded */ 2842 stb r0,VCPU_CEDED(r3) 2843 li r3,H_SUCCESS 2844 blr 2845 2846 /* we've ceded but we want to give control to the host */ 2847kvm_cede_exit: 2848 ld r9, HSTATE_KVM_VCPU(r13) 2849#ifdef CONFIG_KVM_XICS 2850 /* Abort if we still have a pending escalation */ 2851 lbz r5, VCPU_XIVE_ESC_ON(r9) 2852 cmpwi r5, 0 2853 beq 1f 2854 li r0, 0 2855 stb r0, VCPU_CEDED(r9) 28561: /* Enable XIVE escalation */ 2857 li r5, XIVE_ESB_SET_PQ_00 2858 mfmsr r0 2859 andi. r0, r0, MSR_DR /* in real mode? */ 2860 beq 1f 2861 ld r10, VCPU_XIVE_ESC_VADDR(r9) 2862 cmpdi r10, 0 2863 beq 3f 2864 ldx r0, r10, r5 2865 b 2f 28661: ld r10, VCPU_XIVE_ESC_RADDR(r9) 2867 cmpdi r10, 0 2868 beq 3f 2869 ldcix r0, r10, r5 28702: sync 2871 li r0, 1 2872 stb r0, VCPU_XIVE_ESC_ON(r9) 2873#endif /* CONFIG_KVM_XICS */ 28743: b guest_exit_cont 2875 2876 /* Try to do machine check recovery in real mode */ 2877machine_check_realmode: 2878 mr r3, r9 /* get vcpu pointer */ 2879 bl kvmppc_realmode_machine_check 2880 nop 2881 /* all machine checks go to virtual mode for further handling */ 2882 ld r9, HSTATE_KVM_VCPU(r13) 2883 li r12, BOOK3S_INTERRUPT_MACHINE_CHECK 2884 b guest_exit_cont 2885 2886/* 2887 * Call C code to handle a HMI in real mode. 2888 * Only the primary thread does the call, secondary threads are handled 2889 * by calling hmi_exception_realmode() after kvmppc_hv_entry returns. 2890 * r9 points to the vcpu on entry 2891 */ 2892hmi_realmode: 2893 lbz r0, HSTATE_PTID(r13) 2894 cmpwi r0, 0 2895 bne guest_exit_cont 2896 bl kvmppc_realmode_hmi_handler 2897 ld r9, HSTATE_KVM_VCPU(r13) 2898 li r12, BOOK3S_INTERRUPT_HMI 2899 b guest_exit_cont 2900 2901/* 2902 * Check the reason we woke from nap, and take appropriate action. 2903 * Returns (in r3): 2904 * 0 if nothing needs to be done 2905 * 1 if something happened that needs to be handled by the host 2906 * -1 if there was a guest wakeup (IPI or msgsnd) 2907 * -2 if we handled a PCI passthrough interrupt (returned by 2908 * kvmppc_read_intr only) 2909 * 2910 * Also sets r12 to the interrupt vector for any interrupt that needs 2911 * to be handled now by the host (0x500 for external interrupt), or zero. 2912 * Modifies all volatile registers (since it may call a C function). 2913 * This routine calls kvmppc_read_intr, a C function, if an external 2914 * interrupt is pending. 2915 */ 2916kvmppc_check_wake_reason: 2917 mfspr r6, SPRN_SRR1 2918BEGIN_FTR_SECTION 2919 rlwinm r6, r6, 45-31, 0xf /* extract wake reason field (P8) */ 2920FTR_SECTION_ELSE 2921 rlwinm r6, r6, 45-31, 0xe /* P7 wake reason field is 3 bits */ 2922ALT_FTR_SECTION_END_IFSET(CPU_FTR_ARCH_207S) 2923 cmpwi r6, 8 /* was it an external interrupt? */ 2924 beq 7f /* if so, see what it was */ 2925 li r3, 0 2926 li r12, 0 2927 cmpwi r6, 6 /* was it the decrementer? */ 2928 beq 0f 2929BEGIN_FTR_SECTION 2930 cmpwi r6, 5 /* privileged doorbell? */ 2931 beq 0f 2932 cmpwi r6, 3 /* hypervisor doorbell? */ 2933 beq 3f 2934END_FTR_SECTION_IFSET(CPU_FTR_ARCH_207S) 2935 cmpwi r6, 0xa /* Hypervisor maintenance ? */ 2936 beq 4f 2937 li r3, 1 /* anything else, return 1 */ 29380: blr 2939 2940 /* hypervisor doorbell */ 29413: li r12, BOOK3S_INTERRUPT_H_DOORBELL 2942 2943 /* 2944 * Clear the doorbell as we will invoke the handler 2945 * explicitly in the guest exit path. 2946 */ 2947 lis r6, (PPC_DBELL_SERVER << (63-36))@h 2948 PPC_MSGCLR(6) 2949 /* see if it's a host IPI */ 2950 li r3, 1 2951BEGIN_FTR_SECTION 2952 PPC_MSGSYNC 2953 lwsync 2954END_FTR_SECTION_IFSET(CPU_FTR_ARCH_300) 2955 lbz r0, HSTATE_HOST_IPI(r13) 2956 cmpwi r0, 0 2957 bnelr 2958 /* if not, return -1 */ 2959 li r3, -1 2960 blr 2961 2962 /* Woken up due to Hypervisor maintenance interrupt */ 29634: li r12, BOOK3S_INTERRUPT_HMI 2964 li r3, 1 2965 blr 2966 2967 /* external interrupt - create a stack frame so we can call C */ 29687: mflr r0 2969 std r0, PPC_LR_STKOFF(r1) 2970 stdu r1, -PPC_MIN_STKFRM(r1) 2971 bl kvmppc_read_intr 2972 nop 2973 li r12, BOOK3S_INTERRUPT_EXTERNAL 2974 cmpdi r3, 1 2975 ble 1f 2976 2977 /* 2978 * Return code of 2 means PCI passthrough interrupt, but 2979 * we need to return back to host to complete handling the 2980 * interrupt. Trap reason is expected in r12 by guest 2981 * exit code. 2982 */ 2983 li r12, BOOK3S_INTERRUPT_HV_RM_HARD 29841: 2985 ld r0, PPC_MIN_STKFRM+PPC_LR_STKOFF(r1) 2986 addi r1, r1, PPC_MIN_STKFRM 2987 mtlr r0 2988 blr 2989 2990/* 2991 * Save away FP, VMX and VSX registers. 2992 * r3 = vcpu pointer 2993 * N.B. r30 and r31 are volatile across this function, 2994 * thus it is not callable from C. 2995 */ 2996kvmppc_save_fp: 2997 mflr r30 2998 mr r31,r3 2999 mfmsr r5 3000 ori r8,r5,MSR_FP 3001#ifdef CONFIG_ALTIVEC 3002BEGIN_FTR_SECTION 3003 oris r8,r8,MSR_VEC@h 3004END_FTR_SECTION_IFSET(CPU_FTR_ALTIVEC) 3005#endif 3006#ifdef CONFIG_VSX 3007BEGIN_FTR_SECTION 3008 oris r8,r8,MSR_VSX@h 3009END_FTR_SECTION_IFSET(CPU_FTR_VSX) 3010#endif 3011 mtmsrd r8 3012 addi r3,r3,VCPU_FPRS 3013 bl store_fp_state 3014#ifdef CONFIG_ALTIVEC 3015BEGIN_FTR_SECTION 3016 addi r3,r31,VCPU_VRS 3017 bl store_vr_state 3018END_FTR_SECTION_IFSET(CPU_FTR_ALTIVEC) 3019#endif 3020 mfspr r6,SPRN_VRSAVE 3021 stw r6,VCPU_VRSAVE(r31) 3022 mtlr r30 3023 blr 3024 3025/* 3026 * Load up FP, VMX and VSX registers 3027 * r4 = vcpu pointer 3028 * N.B. r30 and r31 are volatile across this function, 3029 * thus it is not callable from C. 3030 */ 3031kvmppc_load_fp: 3032 mflr r30 3033 mr r31,r4 3034 mfmsr r9 3035 ori r8,r9,MSR_FP 3036#ifdef CONFIG_ALTIVEC 3037BEGIN_FTR_SECTION 3038 oris r8,r8,MSR_VEC@h 3039END_FTR_SECTION_IFSET(CPU_FTR_ALTIVEC) 3040#endif 3041#ifdef CONFIG_VSX 3042BEGIN_FTR_SECTION 3043 oris r8,r8,MSR_VSX@h 3044END_FTR_SECTION_IFSET(CPU_FTR_VSX) 3045#endif 3046 mtmsrd r8 3047 addi r3,r4,VCPU_FPRS 3048 bl load_fp_state 3049#ifdef CONFIG_ALTIVEC 3050BEGIN_FTR_SECTION 3051 addi r3,r31,VCPU_VRS 3052 bl load_vr_state 3053END_FTR_SECTION_IFSET(CPU_FTR_ALTIVEC) 3054#endif 3055 lwz r7,VCPU_VRSAVE(r31) 3056 mtspr SPRN_VRSAVE,r7 3057 mtlr r30 3058 mr r4,r31 3059 blr 3060 3061#ifdef CONFIG_PPC_TRANSACTIONAL_MEM 3062/* 3063 * Save transactional state and TM-related registers. 3064 * Called with r3 pointing to the vcpu struct and r4 containing 3065 * the guest MSR value. 3066 * r5 is non-zero iff non-volatile register state needs to be maintained. 3067 * If r5 == 0, this can modify all checkpointed registers, but 3068 * restores r1 and r2 before exit. 3069 */ 3070_GLOBAL_TOC(kvmppc_save_tm_hv) 3071EXPORT_SYMBOL_GPL(kvmppc_save_tm_hv) 3072 /* See if we need to handle fake suspend mode */ 3073BEGIN_FTR_SECTION 3074 b __kvmppc_save_tm 3075END_FTR_SECTION_IFCLR(CPU_FTR_P9_TM_HV_ASSIST) 3076 3077 lbz r0, HSTATE_FAKE_SUSPEND(r13) /* Were we fake suspended? */ 3078 cmpwi r0, 0 3079 beq __kvmppc_save_tm 3080 3081 /* The following code handles the fake_suspend = 1 case */ 3082 mflr r0 3083 std r0, PPC_LR_STKOFF(r1) 3084 stdu r1, -PPC_MIN_STKFRM(r1) 3085 3086 /* Turn on TM. */ 3087 mfmsr r8 3088 li r0, 1 3089 rldimi r8, r0, MSR_TM_LG, 63-MSR_TM_LG 3090 mtmsrd r8 3091 3092 rldicl. r8, r8, 64 - MSR_TS_S_LG, 62 /* Did we actually hrfid? */ 3093 beq 4f 3094BEGIN_FTR_SECTION 3095 bl pnv_power9_force_smt4_catch 3096END_FTR_SECTION_IFSET(CPU_FTR_P9_TM_XER_SO_BUG) 3097 nop 3098 3099 /* We have to treclaim here because that's the only way to do S->N */ 3100 li r3, TM_CAUSE_KVM_RESCHED 3101 TRECLAIM(R3) 3102 3103 /* 3104 * We were in fake suspend, so we are not going to save the 3105 * register state as the guest checkpointed state (since 3106 * we already have it), therefore we can now use any volatile GPR. 3107 * In fact treclaim in fake suspend state doesn't modify 3108 * any registers. 3109 */ 3110 3111BEGIN_FTR_SECTION 3112 bl pnv_power9_force_smt4_release 3113END_FTR_SECTION_IFSET(CPU_FTR_P9_TM_XER_SO_BUG) 3114 nop 3115 31164: 3117 mfspr r3, SPRN_PSSCR 3118 /* PSSCR_FAKE_SUSPEND is a write-only bit, but clear it anyway */ 3119 li r0, PSSCR_FAKE_SUSPEND 3120 andc r3, r3, r0 3121 mtspr SPRN_PSSCR, r3 3122 3123 /* Don't save TEXASR, use value from last exit in real suspend state */ 3124 ld r9, HSTATE_KVM_VCPU(r13) 3125 mfspr r5, SPRN_TFHAR 3126 mfspr r6, SPRN_TFIAR 3127 std r5, VCPU_TFHAR(r9) 3128 std r6, VCPU_TFIAR(r9) 3129 3130 addi r1, r1, PPC_MIN_STKFRM 3131 ld r0, PPC_LR_STKOFF(r1) 3132 mtlr r0 3133 blr 3134 3135/* 3136 * Restore transactional state and TM-related registers. 3137 * Called with r3 pointing to the vcpu struct 3138 * and r4 containing the guest MSR value. 3139 * r5 is non-zero iff non-volatile register state needs to be maintained. 3140 * This potentially modifies all checkpointed registers. 3141 * It restores r1 and r2 from the PACA. 3142 */ 3143_GLOBAL_TOC(kvmppc_restore_tm_hv) 3144EXPORT_SYMBOL_GPL(kvmppc_restore_tm_hv) 3145 /* 3146 * If we are doing TM emulation for the guest on a POWER9 DD2, 3147 * then we don't actually do a trechkpt -- we either set up 3148 * fake-suspend mode, or emulate a TM rollback. 3149 */ 3150BEGIN_FTR_SECTION 3151 b __kvmppc_restore_tm 3152END_FTR_SECTION_IFCLR(CPU_FTR_P9_TM_HV_ASSIST) 3153 mflr r0 3154 std r0, PPC_LR_STKOFF(r1) 3155 3156 li r0, 0 3157 stb r0, HSTATE_FAKE_SUSPEND(r13) 3158 3159 /* Turn on TM so we can restore TM SPRs */ 3160 mfmsr r5 3161 li r0, 1 3162 rldimi r5, r0, MSR_TM_LG, 63-MSR_TM_LG 3163 mtmsrd r5 3164 3165 /* 3166 * The user may change these outside of a transaction, so they must 3167 * always be context switched. 3168 */ 3169 ld r5, VCPU_TFHAR(r3) 3170 ld r6, VCPU_TFIAR(r3) 3171 ld r7, VCPU_TEXASR(r3) 3172 mtspr SPRN_TFHAR, r5 3173 mtspr SPRN_TFIAR, r6 3174 mtspr SPRN_TEXASR, r7 3175 3176 rldicl. r5, r4, 64 - MSR_TS_S_LG, 62 3177 beqlr /* TM not active in guest */ 3178 3179 /* Make sure the failure summary is set */ 3180 oris r7, r7, (TEXASR_FS)@h 3181 mtspr SPRN_TEXASR, r7 3182 3183 cmpwi r5, 1 /* check for suspended state */ 3184 bgt 10f 3185 stb r5, HSTATE_FAKE_SUSPEND(r13) 3186 b 9f /* and return */ 318710: stdu r1, -PPC_MIN_STKFRM(r1) 3188 /* guest is in transactional state, so simulate rollback */ 3189 bl kvmhv_emulate_tm_rollback 3190 nop 3191 addi r1, r1, PPC_MIN_STKFRM 31929: ld r0, PPC_LR_STKOFF(r1) 3193 mtlr r0 3194 blr 3195#endif /* CONFIG_PPC_TRANSACTIONAL_MEM */ 3196 3197/* 3198 * We come here if we get any exception or interrupt while we are 3199 * executing host real mode code while in guest MMU context. 3200 * r12 is (CR << 32) | vector 3201 * r13 points to our PACA 3202 * r12 is saved in HSTATE_SCRATCH0(r13) 3203 * ctr is saved in HSTATE_SCRATCH1(r13) if RELOCATABLE 3204 * r9 is saved in HSTATE_SCRATCH2(r13) 3205 * r13 is saved in HSPRG1 3206 * cfar is saved in HSTATE_CFAR(r13) 3207 * ppr is saved in HSTATE_PPR(r13) 3208 */ 3209kvmppc_bad_host_intr: 3210 /* 3211 * Switch to the emergency stack, but start half-way down in 3212 * case we were already on it. 3213 */ 3214 mr r9, r1 3215 std r1, PACAR1(r13) 3216 ld r1, PACAEMERGSP(r13) 3217 subi r1, r1, THREAD_SIZE/2 + INT_FRAME_SIZE 3218 std r9, 0(r1) 3219 std r0, GPR0(r1) 3220 std r9, GPR1(r1) 3221 std r2, GPR2(r1) 3222 SAVE_4GPRS(3, r1) 3223 SAVE_2GPRS(7, r1) 3224 srdi r0, r12, 32 3225 clrldi r12, r12, 32 3226 std r0, _CCR(r1) 3227 std r12, _TRAP(r1) 3228 andi. r0, r12, 2 3229 beq 1f 3230 mfspr r3, SPRN_HSRR0 3231 mfspr r4, SPRN_HSRR1 3232 mfspr r5, SPRN_HDAR 3233 mfspr r6, SPRN_HDSISR 3234 b 2f 32351: mfspr r3, SPRN_SRR0 3236 mfspr r4, SPRN_SRR1 3237 mfspr r5, SPRN_DAR 3238 mfspr r6, SPRN_DSISR 32392: std r3, _NIP(r1) 3240 std r4, _MSR(r1) 3241 std r5, _DAR(r1) 3242 std r6, _DSISR(r1) 3243 ld r9, HSTATE_SCRATCH2(r13) 3244 ld r12, HSTATE_SCRATCH0(r13) 3245 GET_SCRATCH0(r0) 3246 SAVE_4GPRS(9, r1) 3247 std r0, GPR13(r1) 3248 SAVE_NVGPRS(r1) 3249 ld r5, HSTATE_CFAR(r13) 3250 std r5, ORIG_GPR3(r1) 3251 mflr r3 3252#ifdef CONFIG_RELOCATABLE 3253 ld r4, HSTATE_SCRATCH1(r13) 3254#else 3255 mfctr r4 3256#endif 3257 mfxer r5 3258 lbz r6, PACAIRQSOFTMASK(r13) 3259 std r3, _LINK(r1) 3260 std r4, _CTR(r1) 3261 std r5, _XER(r1) 3262 std r6, SOFTE(r1) 3263 ld r2, PACATOC(r13) 3264 LOAD_REG_IMMEDIATE(3, 0x7265677368657265) 3265 std r3, STACK_FRAME_OVERHEAD-16(r1) 3266 3267 /* 3268 * On POWER9 do a minimal restore of the MMU and call C code, 3269 * which will print a message and panic. 3270 * XXX On POWER7 and POWER8, we just spin here since we don't 3271 * know what the other threads are doing (and we don't want to 3272 * coordinate with them) - but at least we now have register state 3273 * in memory that we might be able to look at from another CPU. 3274 */ 3275BEGIN_FTR_SECTION 3276 b . 3277END_FTR_SECTION_IFCLR(CPU_FTR_ARCH_300) 3278 ld r9, HSTATE_KVM_VCPU(r13) 3279 ld r10, VCPU_KVM(r9) 3280 3281 li r0, 0 3282 mtspr SPRN_AMR, r0 3283 mtspr SPRN_IAMR, r0 3284 mtspr SPRN_CIABR, r0 3285 mtspr SPRN_DAWRX, r0 3286 3287BEGIN_MMU_FTR_SECTION 3288 b 4f 3289END_MMU_FTR_SECTION_IFSET(MMU_FTR_TYPE_RADIX) 3290 3291 slbmte r0, r0 3292 slbia 3293 ptesync 3294 ld r8, PACA_SLBSHADOWPTR(r13) 3295 .rept SLB_NUM_BOLTED 3296 li r3, SLBSHADOW_SAVEAREA 3297 LDX_BE r5, r8, r3 3298 addi r3, r3, 8 3299 LDX_BE r6, r8, r3 3300 andis. r7, r5, SLB_ESID_V@h 3301 beq 3f 3302 slbmte r6, r5 33033: addi r8, r8, 16 3304 .endr 3305 33064: lwz r7, KVM_HOST_LPID(r10) 3307 mtspr SPRN_LPID, r7 3308 mtspr SPRN_PID, r0 3309 ld r8, KVM_HOST_LPCR(r10) 3310 mtspr SPRN_LPCR, r8 3311 isync 3312 li r0, KVM_GUEST_MODE_NONE 3313 stb r0, HSTATE_IN_GUEST(r13) 3314 3315 /* 3316 * Turn on the MMU and jump to C code 3317 */ 3318 bcl 20, 31, .+4 33195: mflr r3 3320 addi r3, r3, 9f - 5b 3321 li r4, -1 3322 rldimi r3, r4, 62, 0 /* ensure 0xc000000000000000 bits are set */ 3323 ld r4, PACAKMSR(r13) 3324 mtspr SPRN_SRR0, r3 3325 mtspr SPRN_SRR1, r4 3326 RFI_TO_KERNEL 33279: addi r3, r1, STACK_FRAME_OVERHEAD 3328 bl kvmppc_bad_interrupt 3329 b 9b 3330 3331/* 3332 * This mimics the MSR transition on IRQ delivery. The new guest MSR is taken 3333 * from VCPU_INTR_MSR and is modified based on the required TM state changes. 3334 * r11 has the guest MSR value (in/out) 3335 * r9 has a vcpu pointer (in) 3336 * r0 is used as a scratch register 3337 */ 3338kvmppc_msr_interrupt: 3339 rldicl r0, r11, 64 - MSR_TS_S_LG, 62 3340 cmpwi r0, 2 /* Check if we are in transactional state.. */ 3341 ld r11, VCPU_INTR_MSR(r9) 3342 bne 1f 3343 /* ... if transactional, change to suspended */ 3344 li r0, 1 33451: rldimi r11, r0, MSR_TS_S_LG, 63 - MSR_TS_T_LG 3346 blr 3347 3348/* 3349 * Load up guest PMU state. R3 points to the vcpu struct. 3350 */ 3351_GLOBAL(kvmhv_load_guest_pmu) 3352EXPORT_SYMBOL_GPL(kvmhv_load_guest_pmu) 3353 mr r4, r3 3354 mflr r0 3355 li r3, 1 3356 sldi r3, r3, 31 /* MMCR0_FC (freeze counters) bit */ 3357 mtspr SPRN_MMCR0, r3 /* freeze all counters, disable ints */ 3358 isync 3359BEGIN_FTR_SECTION 3360 ld r3, VCPU_MMCR(r4) 3361 andi. r5, r3, MMCR0_PMAO_SYNC | MMCR0_PMAO 3362 cmpwi r5, MMCR0_PMAO 3363 beql kvmppc_fix_pmao 3364END_FTR_SECTION_IFSET(CPU_FTR_PMAO_BUG) 3365 lwz r3, VCPU_PMC(r4) /* always load up guest PMU registers */ 3366 lwz r5, VCPU_PMC + 4(r4) /* to prevent information leak */ 3367 lwz r6, VCPU_PMC + 8(r4) 3368 lwz r7, VCPU_PMC + 12(r4) 3369 lwz r8, VCPU_PMC + 16(r4) 3370 lwz r9, VCPU_PMC + 20(r4) 3371 mtspr SPRN_PMC1, r3 3372 mtspr SPRN_PMC2, r5 3373 mtspr SPRN_PMC3, r6 3374 mtspr SPRN_PMC4, r7 3375 mtspr SPRN_PMC5, r8 3376 mtspr SPRN_PMC6, r9 3377 ld r3, VCPU_MMCR(r4) 3378 ld r5, VCPU_MMCR + 8(r4) 3379 ld r6, VCPU_MMCR + 16(r4) 3380 ld r7, VCPU_SIAR(r4) 3381 ld r8, VCPU_SDAR(r4) 3382 mtspr SPRN_MMCR1, r5 3383 mtspr SPRN_MMCRA, r6 3384 mtspr SPRN_SIAR, r7 3385 mtspr SPRN_SDAR, r8 3386BEGIN_FTR_SECTION 3387 ld r5, VCPU_MMCR + 24(r4) 3388 ld r6, VCPU_SIER(r4) 3389 mtspr SPRN_MMCR2, r5 3390 mtspr SPRN_SIER, r6 3391BEGIN_FTR_SECTION_NESTED(96) 3392 lwz r7, VCPU_PMC + 24(r4) 3393 lwz r8, VCPU_PMC + 28(r4) 3394 ld r9, VCPU_MMCR + 32(r4) 3395 mtspr SPRN_SPMC1, r7 3396 mtspr SPRN_SPMC2, r8 3397 mtspr SPRN_MMCRS, r9 3398END_FTR_SECTION_NESTED(CPU_FTR_ARCH_300, 0, 96) 3399END_FTR_SECTION_IFSET(CPU_FTR_ARCH_207S) 3400 mtspr SPRN_MMCR0, r3 3401 isync 3402 mtlr r0 3403 blr 3404 3405/* 3406 * Reload host PMU state saved in the PACA by kvmhv_save_host_pmu. 3407 */ 3408_GLOBAL(kvmhv_load_host_pmu) 3409EXPORT_SYMBOL_GPL(kvmhv_load_host_pmu) 3410 mflr r0 3411 lbz r4, PACA_PMCINUSE(r13) /* is the host using the PMU? */ 3412 cmpwi r4, 0 3413 beq 23f /* skip if not */ 3414BEGIN_FTR_SECTION 3415 ld r3, HSTATE_MMCR0(r13) 3416 andi. r4, r3, MMCR0_PMAO_SYNC | MMCR0_PMAO 3417 cmpwi r4, MMCR0_PMAO 3418 beql kvmppc_fix_pmao 3419END_FTR_SECTION_IFSET(CPU_FTR_PMAO_BUG) 3420 lwz r3, HSTATE_PMC1(r13) 3421 lwz r4, HSTATE_PMC2(r13) 3422 lwz r5, HSTATE_PMC3(r13) 3423 lwz r6, HSTATE_PMC4(r13) 3424 lwz r8, HSTATE_PMC5(r13) 3425 lwz r9, HSTATE_PMC6(r13) 3426 mtspr SPRN_PMC1, r3 3427 mtspr SPRN_PMC2, r4 3428 mtspr SPRN_PMC3, r5 3429 mtspr SPRN_PMC4, r6 3430 mtspr SPRN_PMC5, r8 3431 mtspr SPRN_PMC6, r9 3432 ld r3, HSTATE_MMCR0(r13) 3433 ld r4, HSTATE_MMCR1(r13) 3434 ld r5, HSTATE_MMCRA(r13) 3435 ld r6, HSTATE_SIAR(r13) 3436 ld r7, HSTATE_SDAR(r13) 3437 mtspr SPRN_MMCR1, r4 3438 mtspr SPRN_MMCRA, r5 3439 mtspr SPRN_SIAR, r6 3440 mtspr SPRN_SDAR, r7 3441BEGIN_FTR_SECTION 3442 ld r8, HSTATE_MMCR2(r13) 3443 ld r9, HSTATE_SIER(r13) 3444 mtspr SPRN_MMCR2, r8 3445 mtspr SPRN_SIER, r9 3446END_FTR_SECTION_IFSET(CPU_FTR_ARCH_207S) 3447 mtspr SPRN_MMCR0, r3 3448 isync 3449 mtlr r0 345023: blr 3451 3452/* 3453 * Save guest PMU state into the vcpu struct. 3454 * r3 = vcpu, r4 = full save flag (PMU in use flag set in VPA) 3455 */ 3456_GLOBAL(kvmhv_save_guest_pmu) 3457EXPORT_SYMBOL_GPL(kvmhv_save_guest_pmu) 3458 mr r9, r3 3459 mr r8, r4 3460BEGIN_FTR_SECTION 3461 /* 3462 * POWER8 seems to have a hardware bug where setting 3463 * MMCR0[PMAE] along with MMCR0[PMC1CE] and/or MMCR0[PMCjCE] 3464 * when some counters are already negative doesn't seem 3465 * to cause a performance monitor alert (and hence interrupt). 3466 * The effect of this is that when saving the PMU state, 3467 * if there is no PMU alert pending when we read MMCR0 3468 * before freezing the counters, but one becomes pending 3469 * before we read the counters, we lose it. 3470 * To work around this, we need a way to freeze the counters 3471 * before reading MMCR0. Normally, freezing the counters 3472 * is done by writing MMCR0 (to set MMCR0[FC]) which 3473 * unavoidably writes MMCR0[PMA0] as well. On POWER8, 3474 * we can also freeze the counters using MMCR2, by writing 3475 * 1s to all the counter freeze condition bits (there are 3476 * 9 bits each for 6 counters). 3477 */ 3478 li r3, -1 /* set all freeze bits */ 3479 clrrdi r3, r3, 10 3480 mfspr r10, SPRN_MMCR2 3481 mtspr SPRN_MMCR2, r3 3482 isync 3483END_FTR_SECTION_IFSET(CPU_FTR_ARCH_207S) 3484 li r3, 1 3485 sldi r3, r3, 31 /* MMCR0_FC (freeze counters) bit */ 3486 mfspr r4, SPRN_MMCR0 /* save MMCR0 */ 3487 mtspr SPRN_MMCR0, r3 /* freeze all counters, disable ints */ 3488 mfspr r6, SPRN_MMCRA 3489 /* Clear MMCRA in order to disable SDAR updates */ 3490 li r7, 0 3491 mtspr SPRN_MMCRA, r7 3492 isync 3493 cmpwi r8, 0 /* did they ask for PMU stuff to be saved? */ 3494 bne 21f 3495 std r3, VCPU_MMCR(r9) /* if not, set saved MMCR0 to FC */ 3496 b 22f 349721: mfspr r5, SPRN_MMCR1 3498 mfspr r7, SPRN_SIAR 3499 mfspr r8, SPRN_SDAR 3500 std r4, VCPU_MMCR(r9) 3501 std r5, VCPU_MMCR + 8(r9) 3502 std r6, VCPU_MMCR + 16(r9) 3503BEGIN_FTR_SECTION 3504 std r10, VCPU_MMCR + 24(r9) 3505END_FTR_SECTION_IFSET(CPU_FTR_ARCH_207S) 3506 std r7, VCPU_SIAR(r9) 3507 std r8, VCPU_SDAR(r9) 3508 mfspr r3, SPRN_PMC1 3509 mfspr r4, SPRN_PMC2 3510 mfspr r5, SPRN_PMC3 3511 mfspr r6, SPRN_PMC4 3512 mfspr r7, SPRN_PMC5 3513 mfspr r8, SPRN_PMC6 3514 stw r3, VCPU_PMC(r9) 3515 stw r4, VCPU_PMC + 4(r9) 3516 stw r5, VCPU_PMC + 8(r9) 3517 stw r6, VCPU_PMC + 12(r9) 3518 stw r7, VCPU_PMC + 16(r9) 3519 stw r8, VCPU_PMC + 20(r9) 3520BEGIN_FTR_SECTION 3521 mfspr r5, SPRN_SIER 3522 std r5, VCPU_SIER(r9) 3523BEGIN_FTR_SECTION_NESTED(96) 3524 mfspr r6, SPRN_SPMC1 3525 mfspr r7, SPRN_SPMC2 3526 mfspr r8, SPRN_MMCRS 3527 stw r6, VCPU_PMC + 24(r9) 3528 stw r7, VCPU_PMC + 28(r9) 3529 std r8, VCPU_MMCR + 32(r9) 3530 lis r4, 0x8000 3531 mtspr SPRN_MMCRS, r4 3532END_FTR_SECTION_NESTED(CPU_FTR_ARCH_300, 0, 96) 3533END_FTR_SECTION_IFSET(CPU_FTR_ARCH_207S) 353422: blr 3535 3536/* 3537 * This works around a hardware bug on POWER8E processors, where 3538 * writing a 1 to the MMCR0[PMAO] bit doesn't generate a 3539 * performance monitor interrupt. Instead, when we need to have 3540 * an interrupt pending, we have to arrange for a counter to overflow. 3541 */ 3542kvmppc_fix_pmao: 3543 li r3, 0 3544 mtspr SPRN_MMCR2, r3 3545 lis r3, (MMCR0_PMXE | MMCR0_FCECE)@h 3546 ori r3, r3, MMCR0_PMCjCE | MMCR0_C56RUN 3547 mtspr SPRN_MMCR0, r3 3548 lis r3, 0x7fff 3549 ori r3, r3, 0xffff 3550 mtspr SPRN_PMC6, r3 3551 isync 3552 blr 3553 3554#ifdef CONFIG_KVM_BOOK3S_HV_EXIT_TIMING 3555/* 3556 * Start timing an activity 3557 * r3 = pointer to time accumulation struct, r4 = vcpu 3558 */ 3559kvmhv_start_timing: 3560 ld r5, HSTATE_KVM_VCORE(r13) 3561 ld r6, VCORE_TB_OFFSET_APPL(r5) 3562 mftb r5 3563 subf r5, r6, r5 /* subtract current timebase offset */ 3564 std r3, VCPU_CUR_ACTIVITY(r4) 3565 std r5, VCPU_ACTIVITY_START(r4) 3566 blr 3567 3568/* 3569 * Accumulate time to one activity and start another. 3570 * r3 = pointer to new time accumulation struct, r4 = vcpu 3571 */ 3572kvmhv_accumulate_time: 3573 ld r5, HSTATE_KVM_VCORE(r13) 3574 ld r8, VCORE_TB_OFFSET_APPL(r5) 3575 ld r5, VCPU_CUR_ACTIVITY(r4) 3576 ld r6, VCPU_ACTIVITY_START(r4) 3577 std r3, VCPU_CUR_ACTIVITY(r4) 3578 mftb r7 3579 subf r7, r8, r7 /* subtract current timebase offset */ 3580 std r7, VCPU_ACTIVITY_START(r4) 3581 cmpdi r5, 0 3582 beqlr 3583 subf r3, r6, r7 3584 ld r8, TAS_SEQCOUNT(r5) 3585 cmpdi r8, 0 3586 addi r8, r8, 1 3587 std r8, TAS_SEQCOUNT(r5) 3588 lwsync 3589 ld r7, TAS_TOTAL(r5) 3590 add r7, r7, r3 3591 std r7, TAS_TOTAL(r5) 3592 ld r6, TAS_MIN(r5) 3593 ld r7, TAS_MAX(r5) 3594 beq 3f 3595 cmpd r3, r6 3596 bge 1f 35973: std r3, TAS_MIN(r5) 35981: cmpd r3, r7 3599 ble 2f 3600 std r3, TAS_MAX(r5) 36012: lwsync 3602 addi r8, r8, 1 3603 std r8, TAS_SEQCOUNT(r5) 3604 blr 3605#endif 3606