1/* 2 * This program is free software; you can redistribute it and/or modify 3 * it under the terms of the GNU General Public License, version 2, as 4 * published by the Free Software Foundation. 5 * 6 * This program is distributed in the hope that it will be useful, 7 * but WITHOUT ANY WARRANTY; without even the implied warranty of 8 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 9 * GNU General Public License for more details. 10 * 11 * Copyright 2011 Paul Mackerras, IBM Corp. <paulus@au1.ibm.com> 12 * 13 * Derived from book3s_rmhandlers.S and other files, which are: 14 * 15 * Copyright SUSE Linux Products GmbH 2009 16 * 17 * Authors: Alexander Graf <agraf@suse.de> 18 */ 19 20#include <asm/ppc_asm.h> 21#include <asm/kvm_asm.h> 22#include <asm/reg.h> 23#include <asm/mmu.h> 24#include <asm/page.h> 25#include <asm/ptrace.h> 26#include <asm/hvcall.h> 27#include <asm/asm-offsets.h> 28#include <asm/exception-64s.h> 29#include <asm/kvm_book3s_asm.h> 30#include <asm/book3s/64/mmu-hash.h> 31#include <asm/export.h> 32#include <asm/tm.h> 33#include <asm/opal.h> 34#include <asm/xive-regs.h> 35#include <asm/thread_info.h> 36#include <asm/asm-compat.h> 37#include <asm/feature-fixups.h> 38 39/* Sign-extend HDEC if not on POWER9 */ 40#define EXTEND_HDEC(reg) \ 41BEGIN_FTR_SECTION; \ 42 extsw reg, reg; \ 43END_FTR_SECTION_IFCLR(CPU_FTR_ARCH_300) 44 45/* Values in HSTATE_NAPPING(r13) */ 46#define NAPPING_CEDE 1 47#define NAPPING_NOVCPU 2 48 49/* Stack frame offsets for kvmppc_hv_entry */ 50#define SFS 208 51#define STACK_SLOT_TRAP (SFS-4) 52#define STACK_SLOT_SHORT_PATH (SFS-8) 53#define STACK_SLOT_TID (SFS-16) 54#define STACK_SLOT_PSSCR (SFS-24) 55#define STACK_SLOT_PID (SFS-32) 56#define STACK_SLOT_IAMR (SFS-40) 57#define STACK_SLOT_CIABR (SFS-48) 58#define STACK_SLOT_DAWR (SFS-56) 59#define STACK_SLOT_DAWRX (SFS-64) 60#define STACK_SLOT_HFSCR (SFS-72) 61/* the following is used by the P9 short path */ 62#define STACK_SLOT_NVGPRS (SFS-152) /* 18 gprs */ 63 64/* 65 * Call kvmppc_hv_entry in real mode. 66 * Must be called with interrupts hard-disabled. 67 * 68 * Input Registers: 69 * 70 * LR = return address to continue at after eventually re-enabling MMU 71 */ 72_GLOBAL_TOC(kvmppc_hv_entry_trampoline) 73 mflr r0 74 std r0, PPC_LR_STKOFF(r1) 75 stdu r1, -112(r1) 76 mfmsr r10 77 std r10, HSTATE_HOST_MSR(r13) 78 LOAD_REG_ADDR(r5, kvmppc_call_hv_entry) 79 li r0,MSR_RI 80 andc r0,r10,r0 81 li r6,MSR_IR | MSR_DR 82 andc r6,r10,r6 83 mtmsrd r0,1 /* clear RI in MSR */ 84 mtsrr0 r5 85 mtsrr1 r6 86 RFI_TO_KERNEL 87 88kvmppc_call_hv_entry: 89BEGIN_FTR_SECTION 90 /* On P9, do LPCR setting, if necessary */ 91 ld r3, HSTATE_SPLIT_MODE(r13) 92 cmpdi r3, 0 93 beq 46f 94 lwz r4, KVM_SPLIT_DO_SET(r3) 95 cmpwi r4, 0 96 beq 46f 97 bl kvmhv_p9_set_lpcr 98 nop 9946: 100END_FTR_SECTION_IFSET(CPU_FTR_ARCH_300) 101 102 ld r4, HSTATE_KVM_VCPU(r13) 103 bl kvmppc_hv_entry 104 105 /* Back from guest - restore host state and return to caller */ 106 107BEGIN_FTR_SECTION 108 /* Restore host DABR and DABRX */ 109 ld r5,HSTATE_DABR(r13) 110 li r6,7 111 mtspr SPRN_DABR,r5 112 mtspr SPRN_DABRX,r6 113END_FTR_SECTION_IFCLR(CPU_FTR_ARCH_207S) 114 115 /* Restore SPRG3 */ 116 ld r3,PACA_SPRG_VDSO(r13) 117 mtspr SPRN_SPRG_VDSO_WRITE,r3 118 119 /* Reload the host's PMU registers */ 120 bl kvmhv_load_host_pmu 121 122 /* 123 * Reload DEC. HDEC interrupts were disabled when 124 * we reloaded the host's LPCR value. 125 */ 126 ld r3, HSTATE_DECEXP(r13) 127 mftb r4 128 subf r4, r4, r3 129 mtspr SPRN_DEC, r4 130 131 /* hwthread_req may have got set by cede or no vcpu, so clear it */ 132 li r0, 0 133 stb r0, HSTATE_HWTHREAD_REQ(r13) 134 135 /* 136 * For external interrupts we need to call the Linux 137 * handler to process the interrupt. We do that by jumping 138 * to absolute address 0x500 for external interrupts. 139 * The [h]rfid at the end of the handler will return to 140 * the book3s_hv_interrupts.S code. For other interrupts 141 * we do the rfid to get back to the book3s_hv_interrupts.S 142 * code here. 143 */ 144 ld r8, 112+PPC_LR_STKOFF(r1) 145 addi r1, r1, 112 146 ld r7, HSTATE_HOST_MSR(r13) 147 148 /* Return the trap number on this thread as the return value */ 149 mr r3, r12 150 151 /* 152 * If we came back from the guest via a relocation-on interrupt, 153 * we will be in virtual mode at this point, which makes it a 154 * little easier to get back to the caller. 155 */ 156 mfmsr r0 157 andi. r0, r0, MSR_IR /* in real mode? */ 158 bne .Lvirt_return 159 160 /* RFI into the highmem handler */ 161 mfmsr r6 162 li r0, MSR_RI 163 andc r6, r6, r0 164 mtmsrd r6, 1 /* Clear RI in MSR */ 165 mtsrr0 r8 166 mtsrr1 r7 167 RFI_TO_KERNEL 168 169 /* Virtual-mode return */ 170.Lvirt_return: 171 mtlr r8 172 blr 173 174kvmppc_primary_no_guest: 175 /* We handle this much like a ceded vcpu */ 176 /* put the HDEC into the DEC, since HDEC interrupts don't wake us */ 177 /* HDEC may be larger than DEC for arch >= v3.00, but since the */ 178 /* HDEC value came from DEC in the first place, it will fit */ 179 mfspr r3, SPRN_HDEC 180 mtspr SPRN_DEC, r3 181 /* 182 * Make sure the primary has finished the MMU switch. 183 * We should never get here on a secondary thread, but 184 * check it for robustness' sake. 185 */ 186 ld r5, HSTATE_KVM_VCORE(r13) 18765: lbz r0, VCORE_IN_GUEST(r5) 188 cmpwi r0, 0 189 beq 65b 190 /* Set LPCR. */ 191 ld r8,VCORE_LPCR(r5) 192 mtspr SPRN_LPCR,r8 193 isync 194 /* set our bit in napping_threads */ 195 ld r5, HSTATE_KVM_VCORE(r13) 196 lbz r7, HSTATE_PTID(r13) 197 li r0, 1 198 sld r0, r0, r7 199 addi r6, r5, VCORE_NAPPING_THREADS 2001: lwarx r3, 0, r6 201 or r3, r3, r0 202 stwcx. r3, 0, r6 203 bne 1b 204 /* order napping_threads update vs testing entry_exit_map */ 205 isync 206 li r12, 0 207 lwz r7, VCORE_ENTRY_EXIT(r5) 208 cmpwi r7, 0x100 209 bge kvm_novcpu_exit /* another thread already exiting */ 210 li r3, NAPPING_NOVCPU 211 stb r3, HSTATE_NAPPING(r13) 212 213 li r3, 0 /* Don't wake on privileged (OS) doorbell */ 214 b kvm_do_nap 215 216/* 217 * kvm_novcpu_wakeup 218 * Entered from kvm_start_guest if kvm_hstate.napping is set 219 * to NAPPING_NOVCPU 220 * r2 = kernel TOC 221 * r13 = paca 222 */ 223kvm_novcpu_wakeup: 224 ld r1, HSTATE_HOST_R1(r13) 225 ld r5, HSTATE_KVM_VCORE(r13) 226 li r0, 0 227 stb r0, HSTATE_NAPPING(r13) 228 229 /* check the wake reason */ 230 bl kvmppc_check_wake_reason 231 232 /* 233 * Restore volatile registers since we could have called 234 * a C routine in kvmppc_check_wake_reason. 235 * r5 = VCORE 236 */ 237 ld r5, HSTATE_KVM_VCORE(r13) 238 239 /* see if any other thread is already exiting */ 240 lwz r0, VCORE_ENTRY_EXIT(r5) 241 cmpwi r0, 0x100 242 bge kvm_novcpu_exit 243 244 /* clear our bit in napping_threads */ 245 lbz r7, HSTATE_PTID(r13) 246 li r0, 1 247 sld r0, r0, r7 248 addi r6, r5, VCORE_NAPPING_THREADS 2494: lwarx r7, 0, r6 250 andc r7, r7, r0 251 stwcx. r7, 0, r6 252 bne 4b 253 254 /* See if the wake reason means we need to exit */ 255 cmpdi r3, 0 256 bge kvm_novcpu_exit 257 258 /* See if our timeslice has expired (HDEC is negative) */ 259 mfspr r0, SPRN_HDEC 260 EXTEND_HDEC(r0) 261 li r12, BOOK3S_INTERRUPT_HV_DECREMENTER 262 cmpdi r0, 0 263 blt kvm_novcpu_exit 264 265 /* Got an IPI but other vcpus aren't yet exiting, must be a latecomer */ 266 ld r4, HSTATE_KVM_VCPU(r13) 267 cmpdi r4, 0 268 beq kvmppc_primary_no_guest 269 270#ifdef CONFIG_KVM_BOOK3S_HV_EXIT_TIMING 271 addi r3, r4, VCPU_TB_RMENTRY 272 bl kvmhv_start_timing 273#endif 274 b kvmppc_got_guest 275 276kvm_novcpu_exit: 277#ifdef CONFIG_KVM_BOOK3S_HV_EXIT_TIMING 278 ld r4, HSTATE_KVM_VCPU(r13) 279 cmpdi r4, 0 280 beq 13f 281 addi r3, r4, VCPU_TB_RMEXIT 282 bl kvmhv_accumulate_time 283#endif 28413: mr r3, r12 285 stw r12, STACK_SLOT_TRAP(r1) 286 bl kvmhv_commence_exit 287 nop 288 b kvmhv_switch_to_host 289 290/* 291 * We come in here when wakened from nap mode. 292 * Relocation is off and most register values are lost. 293 * r13 points to the PACA. 294 * r3 contains the SRR1 wakeup value, SRR1 is trashed. 295 */ 296 .globl kvm_start_guest 297kvm_start_guest: 298 /* Set runlatch bit the minute you wake up from nap */ 299 mfspr r0, SPRN_CTRLF 300 ori r0, r0, 1 301 mtspr SPRN_CTRLT, r0 302 303 /* 304 * Could avoid this and pass it through in r3. For now, 305 * code expects it to be in SRR1. 306 */ 307 mtspr SPRN_SRR1,r3 308 309 ld r2,PACATOC(r13) 310 311 li r0,0 312 stb r0,PACA_FTRACE_ENABLED(r13) 313 314 li r0,KVM_HWTHREAD_IN_KVM 315 stb r0,HSTATE_HWTHREAD_STATE(r13) 316 317 /* NV GPR values from power7_idle() will no longer be valid */ 318 li r0,1 319 stb r0,PACA_NAPSTATELOST(r13) 320 321 /* were we napping due to cede? */ 322 lbz r0,HSTATE_NAPPING(r13) 323 cmpwi r0,NAPPING_CEDE 324 beq kvm_end_cede 325 cmpwi r0,NAPPING_NOVCPU 326 beq kvm_novcpu_wakeup 327 328 ld r1,PACAEMERGSP(r13) 329 subi r1,r1,STACK_FRAME_OVERHEAD 330 331 /* 332 * We weren't napping due to cede, so this must be a secondary 333 * thread being woken up to run a guest, or being woken up due 334 * to a stray IPI. (Or due to some machine check or hypervisor 335 * maintenance interrupt while the core is in KVM.) 336 */ 337 338 /* Check the wake reason in SRR1 to see why we got here */ 339 bl kvmppc_check_wake_reason 340 /* 341 * kvmppc_check_wake_reason could invoke a C routine, but we 342 * have no volatile registers to restore when we return. 343 */ 344 345 cmpdi r3, 0 346 bge kvm_no_guest 347 348 /* get vcore pointer, NULL if we have nothing to run */ 349 ld r5,HSTATE_KVM_VCORE(r13) 350 cmpdi r5,0 351 /* if we have no vcore to run, go back to sleep */ 352 beq kvm_no_guest 353 354kvm_secondary_got_guest: 355 356 /* Set HSTATE_DSCR(r13) to something sensible */ 357 ld r6, PACA_DSCR_DEFAULT(r13) 358 std r6, HSTATE_DSCR(r13) 359 360 /* On thread 0 of a subcore, set HDEC to max */ 361 lbz r4, HSTATE_PTID(r13) 362 cmpwi r4, 0 363 bne 63f 364 LOAD_REG_ADDR(r6, decrementer_max) 365 ld r6, 0(r6) 366 mtspr SPRN_HDEC, r6 367 /* and set per-LPAR registers, if doing dynamic micro-threading */ 368 ld r6, HSTATE_SPLIT_MODE(r13) 369 cmpdi r6, 0 370 beq 63f 371BEGIN_FTR_SECTION 372 ld r0, KVM_SPLIT_RPR(r6) 373 mtspr SPRN_RPR, r0 374 ld r0, KVM_SPLIT_PMMAR(r6) 375 mtspr SPRN_PMMAR, r0 376 ld r0, KVM_SPLIT_LDBAR(r6) 377 mtspr SPRN_LDBAR, r0 378 isync 379FTR_SECTION_ELSE 380 /* On P9 we use the split_info for coordinating LPCR changes */ 381 lwz r4, KVM_SPLIT_DO_SET(r6) 382 cmpwi r4, 0 383 beq 1f 384 mr r3, r6 385 bl kvmhv_p9_set_lpcr 386 nop 3871: 388ALT_FTR_SECTION_END_IFCLR(CPU_FTR_ARCH_300) 38963: 390 /* Order load of vcpu after load of vcore */ 391 lwsync 392 ld r4, HSTATE_KVM_VCPU(r13) 393 bl kvmppc_hv_entry 394 395 /* Back from the guest, go back to nap */ 396 /* Clear our vcpu and vcore pointers so we don't come back in early */ 397 li r0, 0 398 std r0, HSTATE_KVM_VCPU(r13) 399 /* 400 * Once we clear HSTATE_KVM_VCORE(r13), the code in 401 * kvmppc_run_core() is going to assume that all our vcpu 402 * state is visible in memory. This lwsync makes sure 403 * that that is true. 404 */ 405 lwsync 406 std r0, HSTATE_KVM_VCORE(r13) 407 408 /* 409 * All secondaries exiting guest will fall through this path. 410 * Before proceeding, just check for HMI interrupt and 411 * invoke opal hmi handler. By now we are sure that the 412 * primary thread on this core/subcore has already made partition 413 * switch/TB resync and we are good to call opal hmi handler. 414 */ 415 cmpwi r12, BOOK3S_INTERRUPT_HMI 416 bne kvm_no_guest 417 418 li r3,0 /* NULL argument */ 419 bl hmi_exception_realmode 420/* 421 * At this point we have finished executing in the guest. 422 * We need to wait for hwthread_req to become zero, since 423 * we may not turn on the MMU while hwthread_req is non-zero. 424 * While waiting we also need to check if we get given a vcpu to run. 425 */ 426kvm_no_guest: 427 lbz r3, HSTATE_HWTHREAD_REQ(r13) 428 cmpwi r3, 0 429 bne 53f 430 HMT_MEDIUM 431 li r0, KVM_HWTHREAD_IN_KERNEL 432 stb r0, HSTATE_HWTHREAD_STATE(r13) 433 /* need to recheck hwthread_req after a barrier, to avoid race */ 434 sync 435 lbz r3, HSTATE_HWTHREAD_REQ(r13) 436 cmpwi r3, 0 437 bne 54f 438/* 439 * We jump to pnv_wakeup_loss, which will return to the caller 440 * of power7_nap in the powernv cpu offline loop. The value we 441 * put in r3 becomes the return value for power7_nap. pnv_wakeup_loss 442 * requires SRR1 in r12. 443 */ 444 li r3, LPCR_PECE0 445 mfspr r4, SPRN_LPCR 446 rlwimi r4, r3, 0, LPCR_PECE0 | LPCR_PECE1 447 mtspr SPRN_LPCR, r4 448 li r3, 0 449 mfspr r12,SPRN_SRR1 450 b pnv_wakeup_loss 451 45253: HMT_LOW 453 ld r5, HSTATE_KVM_VCORE(r13) 454 cmpdi r5, 0 455 bne 60f 456 ld r3, HSTATE_SPLIT_MODE(r13) 457 cmpdi r3, 0 458 beq kvm_no_guest 459 lwz r0, KVM_SPLIT_DO_SET(r3) 460 cmpwi r0, 0 461 bne kvmhv_do_set 462 lwz r0, KVM_SPLIT_DO_RESTORE(r3) 463 cmpwi r0, 0 464 bne kvmhv_do_restore 465 lbz r0, KVM_SPLIT_DO_NAP(r3) 466 cmpwi r0, 0 467 beq kvm_no_guest 468 HMT_MEDIUM 469 b kvm_unsplit_nap 47060: HMT_MEDIUM 471 b kvm_secondary_got_guest 472 47354: li r0, KVM_HWTHREAD_IN_KVM 474 stb r0, HSTATE_HWTHREAD_STATE(r13) 475 b kvm_no_guest 476 477kvmhv_do_set: 478 /* Set LPCR, LPIDR etc. on P9 */ 479 HMT_MEDIUM 480 bl kvmhv_p9_set_lpcr 481 nop 482 b kvm_no_guest 483 484kvmhv_do_restore: 485 HMT_MEDIUM 486 bl kvmhv_p9_restore_lpcr 487 nop 488 b kvm_no_guest 489 490/* 491 * Here the primary thread is trying to return the core to 492 * whole-core mode, so we need to nap. 493 */ 494kvm_unsplit_nap: 495 /* 496 * When secondaries are napping in kvm_unsplit_nap() with 497 * hwthread_req = 1, HMI goes ignored even though subcores are 498 * already exited the guest. Hence HMI keeps waking up secondaries 499 * from nap in a loop and secondaries always go back to nap since 500 * no vcore is assigned to them. This makes impossible for primary 501 * thread to get hold of secondary threads resulting into a soft 502 * lockup in KVM path. 503 * 504 * Let us check if HMI is pending and handle it before we go to nap. 505 */ 506 cmpwi r12, BOOK3S_INTERRUPT_HMI 507 bne 55f 508 li r3, 0 /* NULL argument */ 509 bl hmi_exception_realmode 51055: 511 /* 512 * Ensure that secondary doesn't nap when it has 513 * its vcore pointer set. 514 */ 515 sync /* matches smp_mb() before setting split_info.do_nap */ 516 ld r0, HSTATE_KVM_VCORE(r13) 517 cmpdi r0, 0 518 bne kvm_no_guest 519 /* clear any pending message */ 520BEGIN_FTR_SECTION 521 lis r6, (PPC_DBELL_SERVER << (63-36))@h 522 PPC_MSGCLR(6) 523END_FTR_SECTION_IFSET(CPU_FTR_ARCH_207S) 524 /* Set kvm_split_mode.napped[tid] = 1 */ 525 ld r3, HSTATE_SPLIT_MODE(r13) 526 li r0, 1 527 lbz r4, HSTATE_TID(r13) 528 addi r4, r4, KVM_SPLIT_NAPPED 529 stbx r0, r3, r4 530 /* Check the do_nap flag again after setting napped[] */ 531 sync 532 lbz r0, KVM_SPLIT_DO_NAP(r3) 533 cmpwi r0, 0 534 beq 57f 535 li r3, (LPCR_PECEDH | LPCR_PECE0) >> 4 536 mfspr r5, SPRN_LPCR 537 rlwimi r5, r3, 4, (LPCR_PECEDP | LPCR_PECEDH | LPCR_PECE0 | LPCR_PECE1) 538 b kvm_nap_sequence 539 54057: li r0, 0 541 stbx r0, r3, r4 542 b kvm_no_guest 543 544/****************************************************************************** 545 * * 546 * Entry code * 547 * * 548 *****************************************************************************/ 549 550.global kvmppc_hv_entry 551kvmppc_hv_entry: 552 553 /* Required state: 554 * 555 * R4 = vcpu pointer (or NULL) 556 * MSR = ~IR|DR 557 * R13 = PACA 558 * R1 = host R1 559 * R2 = TOC 560 * all other volatile GPRS = free 561 * Does not preserve non-volatile GPRs or CR fields 562 */ 563 mflr r0 564 std r0, PPC_LR_STKOFF(r1) 565 stdu r1, -SFS(r1) 566 567 /* Save R1 in the PACA */ 568 std r1, HSTATE_HOST_R1(r13) 569 570 li r6, KVM_GUEST_MODE_HOST_HV 571 stb r6, HSTATE_IN_GUEST(r13) 572 573#ifdef CONFIG_KVM_BOOK3S_HV_EXIT_TIMING 574 /* Store initial timestamp */ 575 cmpdi r4, 0 576 beq 1f 577 addi r3, r4, VCPU_TB_RMENTRY 578 bl kvmhv_start_timing 5791: 580#endif 581 582 /* Use cr7 as an indication of radix mode */ 583 ld r5, HSTATE_KVM_VCORE(r13) 584 ld r9, VCORE_KVM(r5) /* pointer to struct kvm */ 585 lbz r0, KVM_RADIX(r9) 586 cmpwi cr7, r0, 0 587 588 /* 589 * POWER7/POWER8 host -> guest partition switch code. 590 * We don't have to lock against concurrent tlbies, 591 * but we do have to coordinate across hardware threads. 592 */ 593 /* Set bit in entry map iff exit map is zero. */ 594 li r7, 1 595 lbz r6, HSTATE_PTID(r13) 596 sld r7, r7, r6 597 addi r8, r5, VCORE_ENTRY_EXIT 59821: lwarx r3, 0, r8 599 cmpwi r3, 0x100 /* any threads starting to exit? */ 600 bge secondary_too_late /* if so we're too late to the party */ 601 or r3, r3, r7 602 stwcx. r3, 0, r8 603 bne 21b 604 605 /* Primary thread switches to guest partition. */ 606 cmpwi r6,0 607 bne 10f 608 609 /* Radix has already switched LPID and flushed core TLB */ 610 bne cr7, 22f 611 612 lwz r7,KVM_LPID(r9) 613BEGIN_FTR_SECTION 614 ld r6,KVM_SDR1(r9) 615 li r0,LPID_RSVD /* switch to reserved LPID */ 616 mtspr SPRN_LPID,r0 617 ptesync 618 mtspr SPRN_SDR1,r6 /* switch to partition page table */ 619END_FTR_SECTION_IFCLR(CPU_FTR_ARCH_300) 620 mtspr SPRN_LPID,r7 621 isync 622 623 /* See if we need to flush the TLB. Hash has to be done in RM */ 624 lhz r6,PACAPACAINDEX(r13) /* test_bit(cpu, need_tlb_flush) */ 625BEGIN_FTR_SECTION 626 /* 627 * On POWER9, individual threads can come in here, but the 628 * TLB is shared between the 4 threads in a core, hence 629 * invalidating on one thread invalidates for all. 630 * Thus we make all 4 threads use the same bit here. 631 */ 632 clrrdi r6,r6,2 633END_FTR_SECTION_IFSET(CPU_FTR_ARCH_300) 634 clrldi r7,r6,64-6 /* extract bit number (6 bits) */ 635 srdi r6,r6,6 /* doubleword number */ 636 sldi r6,r6,3 /* address offset */ 637 add r6,r6,r9 638 addi r6,r6,KVM_NEED_FLUSH /* dword in kvm->arch.need_tlb_flush */ 639 li r8,1 640 sld r8,r8,r7 641 ld r7,0(r6) 642 and. r7,r7,r8 643 beq 22f 644 /* Flush the TLB of any entries for this LPID */ 645 lwz r0,KVM_TLB_SETS(r9) 646 mtctr r0 647 li r7,0x800 /* IS field = 0b10 */ 648 ptesync 649 li r0,0 /* RS for P9 version of tlbiel */ 65028: tlbiel r7 /* On P9, rs=0, RIC=0, PRS=0, R=0 */ 651 addi r7,r7,0x1000 652 bdnz 28b 653 ptesync 65423: ldarx r7,0,r6 /* clear the bit after TLB flushed */ 655 andc r7,r7,r8 656 stdcx. r7,0,r6 657 bne 23b 658 659 /* Add timebase offset onto timebase */ 66022: ld r8,VCORE_TB_OFFSET(r5) 661 cmpdi r8,0 662 beq 37f 663 std r8, VCORE_TB_OFFSET_APPL(r5) 664 mftb r6 /* current host timebase */ 665 add r8,r8,r6 666 mtspr SPRN_TBU40,r8 /* update upper 40 bits */ 667 mftb r7 /* check if lower 24 bits overflowed */ 668 clrldi r6,r6,40 669 clrldi r7,r7,40 670 cmpld r7,r6 671 bge 37f 672 addis r8,r8,0x100 /* if so, increment upper 40 bits */ 673 mtspr SPRN_TBU40,r8 674 675 /* Load guest PCR value to select appropriate compat mode */ 67637: ld r7, VCORE_PCR(r5) 677 cmpdi r7, 0 678 beq 38f 679 mtspr SPRN_PCR, r7 68038: 681 682BEGIN_FTR_SECTION 683 /* DPDES and VTB are shared between threads */ 684 ld r8, VCORE_DPDES(r5) 685 ld r7, VCORE_VTB(r5) 686 mtspr SPRN_DPDES, r8 687 mtspr SPRN_VTB, r7 688END_FTR_SECTION_IFSET(CPU_FTR_ARCH_207S) 689 690 /* Mark the subcore state as inside guest */ 691 bl kvmppc_subcore_enter_guest 692 nop 693 ld r5, HSTATE_KVM_VCORE(r13) 694 ld r4, HSTATE_KVM_VCPU(r13) 695 li r0,1 696 stb r0,VCORE_IN_GUEST(r5) /* signal secondaries to continue */ 697 698 /* Do we have a guest vcpu to run? */ 69910: cmpdi r4, 0 700 beq kvmppc_primary_no_guest 701kvmppc_got_guest: 702 /* Increment yield count if they have a VPA */ 703 ld r3, VCPU_VPA(r4) 704 cmpdi r3, 0 705 beq 25f 706 li r6, LPPACA_YIELDCOUNT 707 LWZX_BE r5, r3, r6 708 addi r5, r5, 1 709 STWX_BE r5, r3, r6 710 li r6, 1 711 stb r6, VCPU_VPA_DIRTY(r4) 71225: 713 714 /* Save purr/spurr */ 715 mfspr r5,SPRN_PURR 716 mfspr r6,SPRN_SPURR 717 std r5,HSTATE_PURR(r13) 718 std r6,HSTATE_SPURR(r13) 719 ld r7,VCPU_PURR(r4) 720 ld r8,VCPU_SPURR(r4) 721 mtspr SPRN_PURR,r7 722 mtspr SPRN_SPURR,r8 723 724 /* Save host values of some registers */ 725BEGIN_FTR_SECTION 726 mfspr r5, SPRN_TIDR 727 mfspr r6, SPRN_PSSCR 728 mfspr r7, SPRN_PID 729 mfspr r8, SPRN_IAMR 730 std r5, STACK_SLOT_TID(r1) 731 std r6, STACK_SLOT_PSSCR(r1) 732 std r7, STACK_SLOT_PID(r1) 733 std r8, STACK_SLOT_IAMR(r1) 734 mfspr r5, SPRN_HFSCR 735 std r5, STACK_SLOT_HFSCR(r1) 736END_FTR_SECTION_IFSET(CPU_FTR_ARCH_300) 737BEGIN_FTR_SECTION 738 mfspr r5, SPRN_CIABR 739 mfspr r6, SPRN_DAWR 740 mfspr r7, SPRN_DAWRX 741 std r5, STACK_SLOT_CIABR(r1) 742 std r6, STACK_SLOT_DAWR(r1) 743 std r7, STACK_SLOT_DAWRX(r1) 744END_FTR_SECTION_IFSET(CPU_FTR_ARCH_207S) 745 746BEGIN_FTR_SECTION 747 /* Set partition DABR */ 748 /* Do this before re-enabling PMU to avoid P7 DABR corruption bug */ 749 lwz r5,VCPU_DABRX(r4) 750 ld r6,VCPU_DABR(r4) 751 mtspr SPRN_DABRX,r5 752 mtspr SPRN_DABR,r6 753 isync 754END_FTR_SECTION_IFCLR(CPU_FTR_ARCH_207S) 755 756#ifdef CONFIG_PPC_TRANSACTIONAL_MEM 757/* 758 * Branch around the call if both CPU_FTR_TM and 759 * CPU_FTR_P9_TM_HV_ASSIST are off. 760 */ 761BEGIN_FTR_SECTION 762 b 91f 763END_FTR_SECTION(CPU_FTR_TM | CPU_FTR_P9_TM_HV_ASSIST, 0) 764 /* 765 * NOTE THAT THIS TRASHES ALL NON-VOLATILE REGISTERS (but not CR) 766 */ 767 mr r3, r4 768 ld r4, VCPU_MSR(r3) 769 li r5, 0 /* don't preserve non-vol regs */ 770 bl kvmppc_restore_tm_hv 771 nop 772 ld r4, HSTATE_KVM_VCPU(r13) 77391: 774#endif 775 776 /* Load guest PMU registers; r4 = vcpu pointer here */ 777 mr r3, r4 778 bl kvmhv_load_guest_pmu 779 780 /* Load up FP, VMX and VSX registers */ 781 ld r4, HSTATE_KVM_VCPU(r13) 782 bl kvmppc_load_fp 783 784 ld r14, VCPU_GPR(R14)(r4) 785 ld r15, VCPU_GPR(R15)(r4) 786 ld r16, VCPU_GPR(R16)(r4) 787 ld r17, VCPU_GPR(R17)(r4) 788 ld r18, VCPU_GPR(R18)(r4) 789 ld r19, VCPU_GPR(R19)(r4) 790 ld r20, VCPU_GPR(R20)(r4) 791 ld r21, VCPU_GPR(R21)(r4) 792 ld r22, VCPU_GPR(R22)(r4) 793 ld r23, VCPU_GPR(R23)(r4) 794 ld r24, VCPU_GPR(R24)(r4) 795 ld r25, VCPU_GPR(R25)(r4) 796 ld r26, VCPU_GPR(R26)(r4) 797 ld r27, VCPU_GPR(R27)(r4) 798 ld r28, VCPU_GPR(R28)(r4) 799 ld r29, VCPU_GPR(R29)(r4) 800 ld r30, VCPU_GPR(R30)(r4) 801 ld r31, VCPU_GPR(R31)(r4) 802 803 /* Switch DSCR to guest value */ 804 ld r5, VCPU_DSCR(r4) 805 mtspr SPRN_DSCR, r5 806 807BEGIN_FTR_SECTION 808 /* Skip next section on POWER7 */ 809 b 8f 810END_FTR_SECTION_IFCLR(CPU_FTR_ARCH_207S) 811 /* Load up POWER8-specific registers */ 812 ld r5, VCPU_IAMR(r4) 813 lwz r6, VCPU_PSPB(r4) 814 ld r7, VCPU_FSCR(r4) 815 mtspr SPRN_IAMR, r5 816 mtspr SPRN_PSPB, r6 817 mtspr SPRN_FSCR, r7 818 ld r5, VCPU_DAWR(r4) 819 ld r6, VCPU_DAWRX(r4) 820 ld r7, VCPU_CIABR(r4) 821 ld r8, VCPU_TAR(r4) 822 /* 823 * Handle broken DAWR case by not writing it. This means we 824 * can still store the DAWR register for migration. 825 */ 826BEGIN_FTR_SECTION 827 mtspr SPRN_DAWR, r5 828 mtspr SPRN_DAWRX, r6 829END_FTR_SECTION_IFSET(CPU_FTR_DAWR) 830 mtspr SPRN_CIABR, r7 831 mtspr SPRN_TAR, r8 832 ld r5, VCPU_IC(r4) 833 ld r8, VCPU_EBBHR(r4) 834 mtspr SPRN_IC, r5 835 mtspr SPRN_EBBHR, r8 836 ld r5, VCPU_EBBRR(r4) 837 ld r6, VCPU_BESCR(r4) 838 lwz r7, VCPU_GUEST_PID(r4) 839 ld r8, VCPU_WORT(r4) 840 mtspr SPRN_EBBRR, r5 841 mtspr SPRN_BESCR, r6 842 mtspr SPRN_PID, r7 843 mtspr SPRN_WORT, r8 844BEGIN_FTR_SECTION 845 /* POWER8-only registers */ 846 ld r5, VCPU_TCSCR(r4) 847 ld r6, VCPU_ACOP(r4) 848 ld r7, VCPU_CSIGR(r4) 849 ld r8, VCPU_TACR(r4) 850 mtspr SPRN_TCSCR, r5 851 mtspr SPRN_ACOP, r6 852 mtspr SPRN_CSIGR, r7 853 mtspr SPRN_TACR, r8 854 nop 855FTR_SECTION_ELSE 856 /* POWER9-only registers */ 857 ld r5, VCPU_TID(r4) 858 ld r6, VCPU_PSSCR(r4) 859 lbz r8, HSTATE_FAKE_SUSPEND(r13) 860 oris r6, r6, PSSCR_EC@h /* This makes stop trap to HV */ 861 rldimi r6, r8, PSSCR_FAKE_SUSPEND_LG, 63 - PSSCR_FAKE_SUSPEND_LG 862 ld r7, VCPU_HFSCR(r4) 863 mtspr SPRN_TIDR, r5 864 mtspr SPRN_PSSCR, r6 865 mtspr SPRN_HFSCR, r7 866ALT_FTR_SECTION_END_IFCLR(CPU_FTR_ARCH_300) 8678: 868 869 ld r5, VCPU_SPRG0(r4) 870 ld r6, VCPU_SPRG1(r4) 871 ld r7, VCPU_SPRG2(r4) 872 ld r8, VCPU_SPRG3(r4) 873 mtspr SPRN_SPRG0, r5 874 mtspr SPRN_SPRG1, r6 875 mtspr SPRN_SPRG2, r7 876 mtspr SPRN_SPRG3, r8 877 878 /* Load up DAR and DSISR */ 879 ld r5, VCPU_DAR(r4) 880 lwz r6, VCPU_DSISR(r4) 881 mtspr SPRN_DAR, r5 882 mtspr SPRN_DSISR, r6 883 884 /* Restore AMR and UAMOR, set AMOR to all 1s */ 885 ld r5,VCPU_AMR(r4) 886 ld r6,VCPU_UAMOR(r4) 887 li r7,-1 888 mtspr SPRN_AMR,r5 889 mtspr SPRN_UAMOR,r6 890 mtspr SPRN_AMOR,r7 891 892 /* Restore state of CTRL run bit; assume 1 on entry */ 893 lwz r5,VCPU_CTRL(r4) 894 andi. r5,r5,1 895 bne 4f 896 mfspr r6,SPRN_CTRLF 897 clrrdi r6,r6,1 898 mtspr SPRN_CTRLT,r6 8994: 900 /* Secondary threads wait for primary to have done partition switch */ 901 ld r5, HSTATE_KVM_VCORE(r13) 902 lbz r6, HSTATE_PTID(r13) 903 cmpwi r6, 0 904 beq 21f 905 lbz r0, VCORE_IN_GUEST(r5) 906 cmpwi r0, 0 907 bne 21f 908 HMT_LOW 90920: lwz r3, VCORE_ENTRY_EXIT(r5) 910 cmpwi r3, 0x100 911 bge no_switch_exit 912 lbz r0, VCORE_IN_GUEST(r5) 913 cmpwi r0, 0 914 beq 20b 915 HMT_MEDIUM 91621: 917 /* Set LPCR. */ 918 ld r8,VCORE_LPCR(r5) 919 mtspr SPRN_LPCR,r8 920 isync 921 922 /* 923 * Set the decrementer to the guest decrementer. 924 */ 925 ld r8,VCPU_DEC_EXPIRES(r4) 926 /* r8 is a host timebase value here, convert to guest TB */ 927 ld r5,HSTATE_KVM_VCORE(r13) 928 ld r6,VCORE_TB_OFFSET_APPL(r5) 929 add r8,r8,r6 930 mftb r7 931 subf r3,r7,r8 932 mtspr SPRN_DEC,r3 933 934 /* Check if HDEC expires soon */ 935 mfspr r3, SPRN_HDEC 936 EXTEND_HDEC(r3) 937 cmpdi r3, 512 /* 1 microsecond */ 938 blt hdec_soon 939 940 /* For hash guest, clear out and reload the SLB */ 941 ld r6, VCPU_KVM(r4) 942 lbz r0, KVM_RADIX(r6) 943 cmpwi r0, 0 944 bne 9f 945 li r6, 0 946 slbmte r6, r6 947 slbia 948 ptesync 949 950 /* Load up guest SLB entries (N.B. slb_max will be 0 for radix) */ 951 lwz r5,VCPU_SLB_MAX(r4) 952 cmpwi r5,0 953 beq 9f 954 mtctr r5 955 addi r6,r4,VCPU_SLB 9561: ld r8,VCPU_SLB_E(r6) 957 ld r9,VCPU_SLB_V(r6) 958 slbmte r9,r8 959 addi r6,r6,VCPU_SLB_SIZE 960 bdnz 1b 9619: 962 963#ifdef CONFIG_KVM_XICS 964 /* We are entering the guest on that thread, push VCPU to XIVE */ 965 ld r10, HSTATE_XIVE_TIMA_PHYS(r13) 966 cmpldi cr0, r10, 0 967 beq no_xive 968 ld r11, VCPU_XIVE_SAVED_STATE(r4) 969 li r9, TM_QW1_OS 970 eieio 971 stdcix r11,r9,r10 972 lwz r11, VCPU_XIVE_CAM_WORD(r4) 973 li r9, TM_QW1_OS + TM_WORD2 974 stwcix r11,r9,r10 975 li r9, 1 976 stb r9, VCPU_XIVE_PUSHED(r4) 977 eieio 978 979 /* 980 * We clear the irq_pending flag. There is a small chance of a 981 * race vs. the escalation interrupt happening on another 982 * processor setting it again, but the only consequence is to 983 * cause a spurrious wakeup on the next H_CEDE which is not an 984 * issue. 985 */ 986 li r0,0 987 stb r0, VCPU_IRQ_PENDING(r4) 988 989 /* 990 * In single escalation mode, if the escalation interrupt is 991 * on, we mask it. 992 */ 993 lbz r0, VCPU_XIVE_ESC_ON(r4) 994 cmpwi r0,0 995 beq 1f 996 ld r10, VCPU_XIVE_ESC_RADDR(r4) 997 li r9, XIVE_ESB_SET_PQ_01 998 ldcix r0, r10, r9 999 sync 1000 1001 /* We have a possible subtle race here: The escalation interrupt might 1002 * have fired and be on its way to the host queue while we mask it, 1003 * and if we unmask it early enough (re-cede right away), there is 1004 * a theorical possibility that it fires again, thus landing in the 1005 * target queue more than once which is a big no-no. 1006 * 1007 * Fortunately, solving this is rather easy. If the above load setting 1008 * PQ to 01 returns a previous value where P is set, then we know the 1009 * escalation interrupt is somewhere on its way to the host. In that 1010 * case we simply don't clear the xive_esc_on flag below. It will be 1011 * eventually cleared by the handler for the escalation interrupt. 1012 * 1013 * Then, when doing a cede, we check that flag again before re-enabling 1014 * the escalation interrupt, and if set, we abort the cede. 1015 */ 1016 andi. r0, r0, XIVE_ESB_VAL_P 1017 bne- 1f 1018 1019 /* Now P is 0, we can clear the flag */ 1020 li r0, 0 1021 stb r0, VCPU_XIVE_ESC_ON(r4) 10221: 1023no_xive: 1024#endif /* CONFIG_KVM_XICS */ 1025 1026 li r0, 0 1027 stw r0, STACK_SLOT_SHORT_PATH(r1) 1028 1029deliver_guest_interrupt: /* r4 = vcpu, r13 = paca */ 1030 /* Check if we can deliver an external or decrementer interrupt now */ 1031 ld r0, VCPU_PENDING_EXC(r4) 1032BEGIN_FTR_SECTION 1033 /* On POWER9, also check for emulated doorbell interrupt */ 1034 lbz r3, VCPU_DBELL_REQ(r4) 1035 or r0, r0, r3 1036END_FTR_SECTION_IFSET(CPU_FTR_ARCH_300) 1037 cmpdi r0, 0 1038 beq 71f 1039 mr r3, r4 1040 bl kvmppc_guest_entry_inject_int 1041 ld r4, HSTATE_KVM_VCPU(r13) 104271: 1043 ld r6, VCPU_SRR0(r4) 1044 ld r7, VCPU_SRR1(r4) 1045 mtspr SPRN_SRR0, r6 1046 mtspr SPRN_SRR1, r7 1047 1048fast_guest_entry_c: 1049 ld r10, VCPU_PC(r4) 1050 ld r11, VCPU_MSR(r4) 1051 /* r11 = vcpu->arch.msr & ~MSR_HV */ 1052 rldicl r11, r11, 63 - MSR_HV_LG, 1 1053 rotldi r11, r11, 1 + MSR_HV_LG 1054 ori r11, r11, MSR_ME 1055 1056 ld r6, VCPU_CTR(r4) 1057 ld r7, VCPU_XER(r4) 1058 mtctr r6 1059 mtxer r7 1060 1061/* 1062 * Required state: 1063 * R4 = vcpu 1064 * R10: value for HSRR0 1065 * R11: value for HSRR1 1066 * R13 = PACA 1067 */ 1068fast_guest_return: 1069 li r0,0 1070 stb r0,VCPU_CEDED(r4) /* cancel cede */ 1071 mtspr SPRN_HSRR0,r10 1072 mtspr SPRN_HSRR1,r11 1073 1074 /* Activate guest mode, so faults get handled by KVM */ 1075 li r9, KVM_GUEST_MODE_GUEST_HV 1076 stb r9, HSTATE_IN_GUEST(r13) 1077 1078#ifdef CONFIG_KVM_BOOK3S_HV_EXIT_TIMING 1079 /* Accumulate timing */ 1080 addi r3, r4, VCPU_TB_GUEST 1081 bl kvmhv_accumulate_time 1082#endif 1083 1084 /* Enter guest */ 1085 1086BEGIN_FTR_SECTION 1087 ld r5, VCPU_CFAR(r4) 1088 mtspr SPRN_CFAR, r5 1089END_FTR_SECTION_IFSET(CPU_FTR_CFAR) 1090BEGIN_FTR_SECTION 1091 ld r0, VCPU_PPR(r4) 1092END_FTR_SECTION_IFSET(CPU_FTR_HAS_PPR) 1093 1094 ld r5, VCPU_LR(r4) 1095 ld r6, VCPU_CR(r4) 1096 mtlr r5 1097 mtcr r6 1098 1099 ld r1, VCPU_GPR(R1)(r4) 1100 ld r2, VCPU_GPR(R2)(r4) 1101 ld r3, VCPU_GPR(R3)(r4) 1102 ld r5, VCPU_GPR(R5)(r4) 1103 ld r6, VCPU_GPR(R6)(r4) 1104 ld r7, VCPU_GPR(R7)(r4) 1105 ld r8, VCPU_GPR(R8)(r4) 1106 ld r9, VCPU_GPR(R9)(r4) 1107 ld r10, VCPU_GPR(R10)(r4) 1108 ld r11, VCPU_GPR(R11)(r4) 1109 ld r12, VCPU_GPR(R12)(r4) 1110 ld r13, VCPU_GPR(R13)(r4) 1111 1112BEGIN_FTR_SECTION 1113 mtspr SPRN_PPR, r0 1114END_FTR_SECTION_IFSET(CPU_FTR_HAS_PPR) 1115 1116/* Move canary into DSISR to check for later */ 1117BEGIN_FTR_SECTION 1118 li r0, 0x7fff 1119 mtspr SPRN_HDSISR, r0 1120END_FTR_SECTION_IFSET(CPU_FTR_ARCH_300) 1121 1122 ld r0, VCPU_GPR(R0)(r4) 1123 ld r4, VCPU_GPR(R4)(r4) 1124 HRFI_TO_GUEST 1125 b . 1126 1127/* 1128 * Enter the guest on a P9 or later system where we have exactly 1129 * one vcpu per vcore and we don't need to go to real mode 1130 * (which implies that host and guest are both using radix MMU mode). 1131 * r3 = vcpu pointer 1132 * Most SPRs and all the VSRs have been loaded already. 1133 */ 1134_GLOBAL(__kvmhv_vcpu_entry_p9) 1135EXPORT_SYMBOL_GPL(__kvmhv_vcpu_entry_p9) 1136 mflr r0 1137 std r0, PPC_LR_STKOFF(r1) 1138 stdu r1, -SFS(r1) 1139 1140 li r0, 1 1141 stw r0, STACK_SLOT_SHORT_PATH(r1) 1142 1143 std r3, HSTATE_KVM_VCPU(r13) 1144 mfcr r4 1145 stw r4, SFS+8(r1) 1146 1147 std r1, HSTATE_HOST_R1(r13) 1148 1149 reg = 14 1150 .rept 18 1151 std reg, STACK_SLOT_NVGPRS + ((reg - 14) * 8)(r1) 1152 reg = reg + 1 1153 .endr 1154 1155 reg = 14 1156 .rept 18 1157 ld reg, __VCPU_GPR(reg)(r3) 1158 reg = reg + 1 1159 .endr 1160 1161 mfmsr r10 1162 std r10, HSTATE_HOST_MSR(r13) 1163 1164 mr r4, r3 1165 b fast_guest_entry_c 1166guest_exit_short_path: 1167 1168 li r0, KVM_GUEST_MODE_NONE 1169 stb r0, HSTATE_IN_GUEST(r13) 1170 1171 reg = 14 1172 .rept 18 1173 std reg, __VCPU_GPR(reg)(r9) 1174 reg = reg + 1 1175 .endr 1176 1177 reg = 14 1178 .rept 18 1179 ld reg, STACK_SLOT_NVGPRS + ((reg - 14) * 8)(r1) 1180 reg = reg + 1 1181 .endr 1182 1183 lwz r4, SFS+8(r1) 1184 mtcr r4 1185 1186 mr r3, r12 /* trap number */ 1187 1188 addi r1, r1, SFS 1189 ld r0, PPC_LR_STKOFF(r1) 1190 mtlr r0 1191 1192 /* If we are in real mode, do a rfid to get back to the caller */ 1193 mfmsr r4 1194 andi. r5, r4, MSR_IR 1195 bnelr 1196 rldicl r5, r4, 64 - MSR_TS_S_LG, 62 /* extract TS field */ 1197 mtspr SPRN_SRR0, r0 1198 ld r10, HSTATE_HOST_MSR(r13) 1199 rldimi r10, r5, MSR_TS_S_LG, 63 - MSR_TS_T_LG 1200 mtspr SPRN_SRR1, r10 1201 RFI_TO_KERNEL 1202 b . 1203 1204secondary_too_late: 1205 li r12, 0 1206 stw r12, STACK_SLOT_TRAP(r1) 1207 cmpdi r4, 0 1208 beq 11f 1209 stw r12, VCPU_TRAP(r4) 1210#ifdef CONFIG_KVM_BOOK3S_HV_EXIT_TIMING 1211 addi r3, r4, VCPU_TB_RMEXIT 1212 bl kvmhv_accumulate_time 1213#endif 121411: b kvmhv_switch_to_host 1215 1216no_switch_exit: 1217 HMT_MEDIUM 1218 li r12, 0 1219 b 12f 1220hdec_soon: 1221 li r12, BOOK3S_INTERRUPT_HV_DECREMENTER 122212: stw r12, VCPU_TRAP(r4) 1223 mr r9, r4 1224#ifdef CONFIG_KVM_BOOK3S_HV_EXIT_TIMING 1225 addi r3, r4, VCPU_TB_RMEXIT 1226 bl kvmhv_accumulate_time 1227#endif 1228 b guest_bypass 1229 1230/****************************************************************************** 1231 * * 1232 * Exit code * 1233 * * 1234 *****************************************************************************/ 1235 1236/* 1237 * We come here from the first-level interrupt handlers. 1238 */ 1239 .globl kvmppc_interrupt_hv 1240kvmppc_interrupt_hv: 1241 /* 1242 * Register contents: 1243 * R12 = (guest CR << 32) | interrupt vector 1244 * R13 = PACA 1245 * guest R12 saved in shadow VCPU SCRATCH0 1246 * guest CTR saved in shadow VCPU SCRATCH1 if RELOCATABLE 1247 * guest R13 saved in SPRN_SCRATCH0 1248 */ 1249 std r9, HSTATE_SCRATCH2(r13) 1250 lbz r9, HSTATE_IN_GUEST(r13) 1251 cmpwi r9, KVM_GUEST_MODE_HOST_HV 1252 beq kvmppc_bad_host_intr 1253#ifdef CONFIG_KVM_BOOK3S_PR_POSSIBLE 1254 cmpwi r9, KVM_GUEST_MODE_GUEST 1255 ld r9, HSTATE_SCRATCH2(r13) 1256 beq kvmppc_interrupt_pr 1257#endif 1258 /* We're now back in the host but in guest MMU context */ 1259 li r9, KVM_GUEST_MODE_HOST_HV 1260 stb r9, HSTATE_IN_GUEST(r13) 1261 1262 ld r9, HSTATE_KVM_VCPU(r13) 1263 1264 /* Save registers */ 1265 1266 std r0, VCPU_GPR(R0)(r9) 1267 std r1, VCPU_GPR(R1)(r9) 1268 std r2, VCPU_GPR(R2)(r9) 1269 std r3, VCPU_GPR(R3)(r9) 1270 std r4, VCPU_GPR(R4)(r9) 1271 std r5, VCPU_GPR(R5)(r9) 1272 std r6, VCPU_GPR(R6)(r9) 1273 std r7, VCPU_GPR(R7)(r9) 1274 std r8, VCPU_GPR(R8)(r9) 1275 ld r0, HSTATE_SCRATCH2(r13) 1276 std r0, VCPU_GPR(R9)(r9) 1277 std r10, VCPU_GPR(R10)(r9) 1278 std r11, VCPU_GPR(R11)(r9) 1279 ld r3, HSTATE_SCRATCH0(r13) 1280 std r3, VCPU_GPR(R12)(r9) 1281 /* CR is in the high half of r12 */ 1282 srdi r4, r12, 32 1283 std r4, VCPU_CR(r9) 1284BEGIN_FTR_SECTION 1285 ld r3, HSTATE_CFAR(r13) 1286 std r3, VCPU_CFAR(r9) 1287END_FTR_SECTION_IFSET(CPU_FTR_CFAR) 1288BEGIN_FTR_SECTION 1289 ld r4, HSTATE_PPR(r13) 1290 std r4, VCPU_PPR(r9) 1291END_FTR_SECTION_IFSET(CPU_FTR_HAS_PPR) 1292 1293 /* Restore R1/R2 so we can handle faults */ 1294 ld r1, HSTATE_HOST_R1(r13) 1295 ld r2, PACATOC(r13) 1296 1297 mfspr r10, SPRN_SRR0 1298 mfspr r11, SPRN_SRR1 1299 std r10, VCPU_SRR0(r9) 1300 std r11, VCPU_SRR1(r9) 1301 /* trap is in the low half of r12, clear CR from the high half */ 1302 clrldi r12, r12, 32 1303 andi. r0, r12, 2 /* need to read HSRR0/1? */ 1304 beq 1f 1305 mfspr r10, SPRN_HSRR0 1306 mfspr r11, SPRN_HSRR1 1307 clrrdi r12, r12, 2 13081: std r10, VCPU_PC(r9) 1309 std r11, VCPU_MSR(r9) 1310 1311 GET_SCRATCH0(r3) 1312 mflr r4 1313 std r3, VCPU_GPR(R13)(r9) 1314 std r4, VCPU_LR(r9) 1315 1316 stw r12,VCPU_TRAP(r9) 1317 1318 /* 1319 * Now that we have saved away SRR0/1 and HSRR0/1, 1320 * interrupts are recoverable in principle, so set MSR_RI. 1321 * This becomes important for relocation-on interrupts from 1322 * the guest, which we can get in radix mode on POWER9. 1323 */ 1324 li r0, MSR_RI 1325 mtmsrd r0, 1 1326 1327#ifdef CONFIG_KVM_BOOK3S_HV_EXIT_TIMING 1328 addi r3, r9, VCPU_TB_RMINTR 1329 mr r4, r9 1330 bl kvmhv_accumulate_time 1331 ld r5, VCPU_GPR(R5)(r9) 1332 ld r6, VCPU_GPR(R6)(r9) 1333 ld r7, VCPU_GPR(R7)(r9) 1334 ld r8, VCPU_GPR(R8)(r9) 1335#endif 1336 1337 /* Save HEIR (HV emulation assist reg) in emul_inst 1338 if this is an HEI (HV emulation interrupt, e40) */ 1339 li r3,KVM_INST_FETCH_FAILED 1340 stw r3,VCPU_LAST_INST(r9) 1341 cmpwi r12,BOOK3S_INTERRUPT_H_EMUL_ASSIST 1342 bne 11f 1343 mfspr r3,SPRN_HEIR 134411: stw r3,VCPU_HEIR(r9) 1345 1346 /* these are volatile across C function calls */ 1347#ifdef CONFIG_RELOCATABLE 1348 ld r3, HSTATE_SCRATCH1(r13) 1349 mtctr r3 1350#else 1351 mfctr r3 1352#endif 1353 mfxer r4 1354 std r3, VCPU_CTR(r9) 1355 std r4, VCPU_XER(r9) 1356 1357 /* Save more register state */ 1358 mfdar r3 1359 mfdsisr r4 1360 std r3, VCPU_DAR(r9) 1361 stw r4, VCPU_DSISR(r9) 1362 1363 /* If this is a page table miss then see if it's theirs or ours */ 1364 cmpwi r12, BOOK3S_INTERRUPT_H_DATA_STORAGE 1365 beq kvmppc_hdsi 1366 std r3, VCPU_FAULT_DAR(r9) 1367 stw r4, VCPU_FAULT_DSISR(r9) 1368 cmpwi r12, BOOK3S_INTERRUPT_H_INST_STORAGE 1369 beq kvmppc_hisi 1370 1371#ifdef CONFIG_PPC_TRANSACTIONAL_MEM 1372 /* For softpatch interrupt, go off and do TM instruction emulation */ 1373 cmpwi r12, BOOK3S_INTERRUPT_HV_SOFTPATCH 1374 beq kvmppc_tm_emul 1375#endif 1376 1377 /* See if this is a leftover HDEC interrupt */ 1378 cmpwi r12,BOOK3S_INTERRUPT_HV_DECREMENTER 1379 bne 2f 1380 mfspr r3,SPRN_HDEC 1381 EXTEND_HDEC(r3) 1382 cmpdi r3,0 1383 mr r4,r9 1384 bge fast_guest_return 13852: 1386 /* See if this is an hcall we can handle in real mode */ 1387 cmpwi r12,BOOK3S_INTERRUPT_SYSCALL 1388 beq hcall_try_real_mode 1389 1390 /* Hypervisor doorbell - exit only if host IPI flag set */ 1391 cmpwi r12, BOOK3S_INTERRUPT_H_DOORBELL 1392 bne 3f 1393BEGIN_FTR_SECTION 1394 PPC_MSGSYNC 1395 lwsync 1396 /* always exit if we're running a nested guest */ 1397 ld r0, VCPU_NESTED(r9) 1398 cmpdi r0, 0 1399 bne guest_exit_cont 1400END_FTR_SECTION_IFSET(CPU_FTR_ARCH_300) 1401 lbz r0, HSTATE_HOST_IPI(r13) 1402 cmpwi r0, 0 1403 beq maybe_reenter_guest 1404 b guest_exit_cont 14053: 1406 /* If it's a hypervisor facility unavailable interrupt, save HFSCR */ 1407 cmpwi r12, BOOK3S_INTERRUPT_H_FAC_UNAVAIL 1408 bne 14f 1409 mfspr r3, SPRN_HFSCR 1410 std r3, VCPU_HFSCR(r9) 1411 b guest_exit_cont 141214: 1413 /* External interrupt ? */ 1414 cmpwi r12, BOOK3S_INTERRUPT_EXTERNAL 1415 beq kvmppc_guest_external 1416 /* See if it is a machine check */ 1417 cmpwi r12, BOOK3S_INTERRUPT_MACHINE_CHECK 1418 beq machine_check_realmode 1419 /* Or a hypervisor maintenance interrupt */ 1420 cmpwi r12, BOOK3S_INTERRUPT_HMI 1421 beq hmi_realmode 1422 1423guest_exit_cont: /* r9 = vcpu, r12 = trap, r13 = paca */ 1424 1425#ifdef CONFIG_KVM_BOOK3S_HV_EXIT_TIMING 1426 addi r3, r9, VCPU_TB_RMEXIT 1427 mr r4, r9 1428 bl kvmhv_accumulate_time 1429#endif 1430#ifdef CONFIG_KVM_XICS 1431 /* We are exiting, pull the VP from the XIVE */ 1432 lbz r0, VCPU_XIVE_PUSHED(r9) 1433 cmpwi cr0, r0, 0 1434 beq 1f 1435 li r7, TM_SPC_PULL_OS_CTX 1436 li r6, TM_QW1_OS 1437 mfmsr r0 1438 andi. r0, r0, MSR_DR /* in real mode? */ 1439 beq 2f 1440 ld r10, HSTATE_XIVE_TIMA_VIRT(r13) 1441 cmpldi cr0, r10, 0 1442 beq 1f 1443 /* First load to pull the context, we ignore the value */ 1444 eieio 1445 lwzx r11, r7, r10 1446 /* Second load to recover the context state (Words 0 and 1) */ 1447 ldx r11, r6, r10 1448 b 3f 14492: ld r10, HSTATE_XIVE_TIMA_PHYS(r13) 1450 cmpldi cr0, r10, 0 1451 beq 1f 1452 /* First load to pull the context, we ignore the value */ 1453 eieio 1454 lwzcix r11, r7, r10 1455 /* Second load to recover the context state (Words 0 and 1) */ 1456 ldcix r11, r6, r10 14573: std r11, VCPU_XIVE_SAVED_STATE(r9) 1458 /* Fixup some of the state for the next load */ 1459 li r10, 0 1460 li r0, 0xff 1461 stb r10, VCPU_XIVE_PUSHED(r9) 1462 stb r10, (VCPU_XIVE_SAVED_STATE+3)(r9) 1463 stb r0, (VCPU_XIVE_SAVED_STATE+4)(r9) 1464 eieio 14651: 1466#endif /* CONFIG_KVM_XICS */ 1467 1468 /* If we came in through the P9 short path, go back out to C now */ 1469 lwz r0, STACK_SLOT_SHORT_PATH(r1) 1470 cmpwi r0, 0 1471 bne guest_exit_short_path 1472 1473 /* For hash guest, read the guest SLB and save it away */ 1474 ld r5, VCPU_KVM(r9) 1475 lbz r0, KVM_RADIX(r5) 1476 li r5, 0 1477 cmpwi r0, 0 1478 bne 3f /* for radix, save 0 entries */ 1479 lwz r0,VCPU_SLB_NR(r9) /* number of entries in SLB */ 1480 mtctr r0 1481 li r6,0 1482 addi r7,r9,VCPU_SLB 14831: slbmfee r8,r6 1484 andis. r0,r8,SLB_ESID_V@h 1485 beq 2f 1486 add r8,r8,r6 /* put index in */ 1487 slbmfev r3,r6 1488 std r8,VCPU_SLB_E(r7) 1489 std r3,VCPU_SLB_V(r7) 1490 addi r7,r7,VCPU_SLB_SIZE 1491 addi r5,r5,1 14922: addi r6,r6,1 1493 bdnz 1b 1494 /* Finally clear out the SLB */ 1495 li r0,0 1496 slbmte r0,r0 1497 slbia 1498 ptesync 14993: stw r5,VCPU_SLB_MAX(r9) 1500 1501 /* load host SLB entries */ 1502BEGIN_MMU_FTR_SECTION 1503 b 0f 1504END_MMU_FTR_SECTION_IFSET(MMU_FTR_TYPE_RADIX) 1505 ld r8,PACA_SLBSHADOWPTR(r13) 1506 1507 .rept SLB_NUM_BOLTED 1508 li r3, SLBSHADOW_SAVEAREA 1509 LDX_BE r5, r8, r3 1510 addi r3, r3, 8 1511 LDX_BE r6, r8, r3 1512 andis. r7,r5,SLB_ESID_V@h 1513 beq 1f 1514 slbmte r6,r5 15151: addi r8,r8,16 1516 .endr 15170: 1518 1519guest_bypass: 1520 stw r12, STACK_SLOT_TRAP(r1) 1521 1522 /* Save DEC */ 1523 /* Do this before kvmhv_commence_exit so we know TB is guest TB */ 1524 ld r3, HSTATE_KVM_VCORE(r13) 1525 mfspr r5,SPRN_DEC 1526 mftb r6 1527 /* On P9, if the guest has large decr enabled, don't sign extend */ 1528BEGIN_FTR_SECTION 1529 ld r4, VCORE_LPCR(r3) 1530 andis. r4, r4, LPCR_LD@h 1531 bne 16f 1532END_FTR_SECTION_IFSET(CPU_FTR_ARCH_300) 1533 extsw r5,r5 153416: add r5,r5,r6 1535 /* r5 is a guest timebase value here, convert to host TB */ 1536 ld r4,VCORE_TB_OFFSET_APPL(r3) 1537 subf r5,r4,r5 1538 std r5,VCPU_DEC_EXPIRES(r9) 1539 1540 /* Increment exit count, poke other threads to exit */ 1541 mr r3, r12 1542 bl kvmhv_commence_exit 1543 nop 1544 ld r9, HSTATE_KVM_VCPU(r13) 1545 1546 /* Stop others sending VCPU interrupts to this physical CPU */ 1547 li r0, -1 1548 stw r0, VCPU_CPU(r9) 1549 stw r0, VCPU_THREAD_CPU(r9) 1550 1551 /* Save guest CTRL register, set runlatch to 1 */ 1552 mfspr r6,SPRN_CTRLF 1553 stw r6,VCPU_CTRL(r9) 1554 andi. r0,r6,1 1555 bne 4f 1556 ori r6,r6,1 1557 mtspr SPRN_CTRLT,r6 15584: 1559 /* 1560 * Save the guest PURR/SPURR 1561 */ 1562 mfspr r5,SPRN_PURR 1563 mfspr r6,SPRN_SPURR 1564 ld r7,VCPU_PURR(r9) 1565 ld r8,VCPU_SPURR(r9) 1566 std r5,VCPU_PURR(r9) 1567 std r6,VCPU_SPURR(r9) 1568 subf r5,r7,r5 1569 subf r6,r8,r6 1570 1571 /* 1572 * Restore host PURR/SPURR and add guest times 1573 * so that the time in the guest gets accounted. 1574 */ 1575 ld r3,HSTATE_PURR(r13) 1576 ld r4,HSTATE_SPURR(r13) 1577 add r3,r3,r5 1578 add r4,r4,r6 1579 mtspr SPRN_PURR,r3 1580 mtspr SPRN_SPURR,r4 1581 1582BEGIN_FTR_SECTION 1583 b 8f 1584END_FTR_SECTION_IFCLR(CPU_FTR_ARCH_207S) 1585 /* Save POWER8-specific registers */ 1586 mfspr r5, SPRN_IAMR 1587 mfspr r6, SPRN_PSPB 1588 mfspr r7, SPRN_FSCR 1589 std r5, VCPU_IAMR(r9) 1590 stw r6, VCPU_PSPB(r9) 1591 std r7, VCPU_FSCR(r9) 1592 mfspr r5, SPRN_IC 1593 mfspr r7, SPRN_TAR 1594 std r5, VCPU_IC(r9) 1595 std r7, VCPU_TAR(r9) 1596 mfspr r8, SPRN_EBBHR 1597 std r8, VCPU_EBBHR(r9) 1598 mfspr r5, SPRN_EBBRR 1599 mfspr r6, SPRN_BESCR 1600 mfspr r7, SPRN_PID 1601 mfspr r8, SPRN_WORT 1602 std r5, VCPU_EBBRR(r9) 1603 std r6, VCPU_BESCR(r9) 1604 stw r7, VCPU_GUEST_PID(r9) 1605 std r8, VCPU_WORT(r9) 1606BEGIN_FTR_SECTION 1607 mfspr r5, SPRN_TCSCR 1608 mfspr r6, SPRN_ACOP 1609 mfspr r7, SPRN_CSIGR 1610 mfspr r8, SPRN_TACR 1611 std r5, VCPU_TCSCR(r9) 1612 std r6, VCPU_ACOP(r9) 1613 std r7, VCPU_CSIGR(r9) 1614 std r8, VCPU_TACR(r9) 1615FTR_SECTION_ELSE 1616 mfspr r5, SPRN_TIDR 1617 mfspr r6, SPRN_PSSCR 1618 std r5, VCPU_TID(r9) 1619 rldicl r6, r6, 4, 50 /* r6 &= PSSCR_GUEST_VIS */ 1620 rotldi r6, r6, 60 1621 std r6, VCPU_PSSCR(r9) 1622 /* Restore host HFSCR value */ 1623 ld r7, STACK_SLOT_HFSCR(r1) 1624 mtspr SPRN_HFSCR, r7 1625ALT_FTR_SECTION_END_IFCLR(CPU_FTR_ARCH_300) 1626 /* 1627 * Restore various registers to 0, where non-zero values 1628 * set by the guest could disrupt the host. 1629 */ 1630 li r0, 0 1631 mtspr SPRN_PSPB, r0 1632 mtspr SPRN_WORT, r0 1633BEGIN_FTR_SECTION 1634 mtspr SPRN_IAMR, r0 1635 mtspr SPRN_TCSCR, r0 1636 /* Set MMCRS to 1<<31 to freeze and disable the SPMC counters */ 1637 li r0, 1 1638 sldi r0, r0, 31 1639 mtspr SPRN_MMCRS, r0 1640END_FTR_SECTION_IFCLR(CPU_FTR_ARCH_300) 16418: 1642 1643 /* Save and reset AMR and UAMOR before turning on the MMU */ 1644 mfspr r5,SPRN_AMR 1645 mfspr r6,SPRN_UAMOR 1646 std r5,VCPU_AMR(r9) 1647 std r6,VCPU_UAMOR(r9) 1648 li r6,0 1649 mtspr SPRN_AMR,r6 1650 mtspr SPRN_UAMOR, r6 1651 1652 /* Switch DSCR back to host value */ 1653 mfspr r8, SPRN_DSCR 1654 ld r7, HSTATE_DSCR(r13) 1655 std r8, VCPU_DSCR(r9) 1656 mtspr SPRN_DSCR, r7 1657 1658 /* Save non-volatile GPRs */ 1659 std r14, VCPU_GPR(R14)(r9) 1660 std r15, VCPU_GPR(R15)(r9) 1661 std r16, VCPU_GPR(R16)(r9) 1662 std r17, VCPU_GPR(R17)(r9) 1663 std r18, VCPU_GPR(R18)(r9) 1664 std r19, VCPU_GPR(R19)(r9) 1665 std r20, VCPU_GPR(R20)(r9) 1666 std r21, VCPU_GPR(R21)(r9) 1667 std r22, VCPU_GPR(R22)(r9) 1668 std r23, VCPU_GPR(R23)(r9) 1669 std r24, VCPU_GPR(R24)(r9) 1670 std r25, VCPU_GPR(R25)(r9) 1671 std r26, VCPU_GPR(R26)(r9) 1672 std r27, VCPU_GPR(R27)(r9) 1673 std r28, VCPU_GPR(R28)(r9) 1674 std r29, VCPU_GPR(R29)(r9) 1675 std r30, VCPU_GPR(R30)(r9) 1676 std r31, VCPU_GPR(R31)(r9) 1677 1678 /* Save SPRGs */ 1679 mfspr r3, SPRN_SPRG0 1680 mfspr r4, SPRN_SPRG1 1681 mfspr r5, SPRN_SPRG2 1682 mfspr r6, SPRN_SPRG3 1683 std r3, VCPU_SPRG0(r9) 1684 std r4, VCPU_SPRG1(r9) 1685 std r5, VCPU_SPRG2(r9) 1686 std r6, VCPU_SPRG3(r9) 1687 1688 /* save FP state */ 1689 mr r3, r9 1690 bl kvmppc_save_fp 1691 1692#ifdef CONFIG_PPC_TRANSACTIONAL_MEM 1693/* 1694 * Branch around the call if both CPU_FTR_TM and 1695 * CPU_FTR_P9_TM_HV_ASSIST are off. 1696 */ 1697BEGIN_FTR_SECTION 1698 b 91f 1699END_FTR_SECTION(CPU_FTR_TM | CPU_FTR_P9_TM_HV_ASSIST, 0) 1700 /* 1701 * NOTE THAT THIS TRASHES ALL NON-VOLATILE REGISTERS (but not CR) 1702 */ 1703 mr r3, r9 1704 ld r4, VCPU_MSR(r3) 1705 li r5, 0 /* don't preserve non-vol regs */ 1706 bl kvmppc_save_tm_hv 1707 nop 1708 ld r9, HSTATE_KVM_VCPU(r13) 170991: 1710#endif 1711 1712 /* Increment yield count if they have a VPA */ 1713 ld r8, VCPU_VPA(r9) /* do they have a VPA? */ 1714 cmpdi r8, 0 1715 beq 25f 1716 li r4, LPPACA_YIELDCOUNT 1717 LWZX_BE r3, r8, r4 1718 addi r3, r3, 1 1719 STWX_BE r3, r8, r4 1720 li r3, 1 1721 stb r3, VCPU_VPA_DIRTY(r9) 172225: 1723 /* Save PMU registers if requested */ 1724 /* r8 and cr0.eq are live here */ 1725 mr r3, r9 1726 li r4, 1 1727 beq 21f /* if no VPA, save PMU stuff anyway */ 1728 lbz r4, LPPACA_PMCINUSE(r8) 172921: bl kvmhv_save_guest_pmu 1730 ld r9, HSTATE_KVM_VCPU(r13) 1731 1732 /* Restore host values of some registers */ 1733BEGIN_FTR_SECTION 1734 ld r5, STACK_SLOT_CIABR(r1) 1735 ld r6, STACK_SLOT_DAWR(r1) 1736 ld r7, STACK_SLOT_DAWRX(r1) 1737 mtspr SPRN_CIABR, r5 1738 /* 1739 * If the DAWR doesn't work, it's ok to write these here as 1740 * this value should always be zero 1741 */ 1742 mtspr SPRN_DAWR, r6 1743 mtspr SPRN_DAWRX, r7 1744END_FTR_SECTION_IFSET(CPU_FTR_ARCH_207S) 1745BEGIN_FTR_SECTION 1746 ld r5, STACK_SLOT_TID(r1) 1747 ld r6, STACK_SLOT_PSSCR(r1) 1748 ld r7, STACK_SLOT_PID(r1) 1749 ld r8, STACK_SLOT_IAMR(r1) 1750 mtspr SPRN_TIDR, r5 1751 mtspr SPRN_PSSCR, r6 1752 mtspr SPRN_PID, r7 1753 mtspr SPRN_IAMR, r8 1754END_FTR_SECTION_IFSET(CPU_FTR_ARCH_300) 1755 1756#ifdef CONFIG_PPC_RADIX_MMU 1757 /* 1758 * Are we running hash or radix ? 1759 */ 1760 ld r5, VCPU_KVM(r9) 1761 lbz r0, KVM_RADIX(r5) 1762 cmpwi cr2, r0, 0 1763 beq cr2, 2f 1764 1765 /* 1766 * Radix: do eieio; tlbsync; ptesync sequence in case we 1767 * interrupted the guest between a tlbie and a ptesync. 1768 */ 1769 eieio 1770 tlbsync 1771 ptesync 1772 1773 /* Radix: Handle the case where the guest used an illegal PID */ 1774 LOAD_REG_ADDR(r4, mmu_base_pid) 1775 lwz r3, VCPU_GUEST_PID(r9) 1776 lwz r5, 0(r4) 1777 cmpw cr0,r3,r5 1778 blt 2f 1779 1780 /* 1781 * Illegal PID, the HW might have prefetched and cached in the TLB 1782 * some translations for the LPID 0 / guest PID combination which 1783 * Linux doesn't know about, so we need to flush that PID out of 1784 * the TLB. First we need to set LPIDR to 0 so tlbiel applies to 1785 * the right context. 1786 */ 1787 li r0,0 1788 mtspr SPRN_LPID,r0 1789 isync 1790 1791 /* Then do a congruence class local flush */ 1792 ld r6,VCPU_KVM(r9) 1793 lwz r0,KVM_TLB_SETS(r6) 1794 mtctr r0 1795 li r7,0x400 /* IS field = 0b01 */ 1796 ptesync 1797 sldi r0,r3,32 /* RS has PID */ 17981: PPC_TLBIEL(7,0,2,1,1) /* RIC=2, PRS=1, R=1 */ 1799 addi r7,r7,0x1000 1800 bdnz 1b 1801 ptesync 1802 18032: 1804#endif /* CONFIG_PPC_RADIX_MMU */ 1805 1806 /* 1807 * POWER7/POWER8 guest -> host partition switch code. 1808 * We don't have to lock against tlbies but we do 1809 * have to coordinate the hardware threads. 1810 * Here STACK_SLOT_TRAP(r1) contains the trap number. 1811 */ 1812kvmhv_switch_to_host: 1813 /* Secondary threads wait for primary to do partition switch */ 1814 ld r5,HSTATE_KVM_VCORE(r13) 1815 ld r4,VCORE_KVM(r5) /* pointer to struct kvm */ 1816 lbz r3,HSTATE_PTID(r13) 1817 cmpwi r3,0 1818 beq 15f 1819 HMT_LOW 182013: lbz r3,VCORE_IN_GUEST(r5) 1821 cmpwi r3,0 1822 bne 13b 1823 HMT_MEDIUM 1824 b 16f 1825 1826 /* Primary thread waits for all the secondaries to exit guest */ 182715: lwz r3,VCORE_ENTRY_EXIT(r5) 1828 rlwinm r0,r3,32-8,0xff 1829 clrldi r3,r3,56 1830 cmpw r3,r0 1831 bne 15b 1832 isync 1833 1834 /* Did we actually switch to the guest at all? */ 1835 lbz r6, VCORE_IN_GUEST(r5) 1836 cmpwi r6, 0 1837 beq 19f 1838 1839 /* Primary thread switches back to host partition */ 1840 lwz r7,KVM_HOST_LPID(r4) 1841BEGIN_FTR_SECTION 1842 ld r6,KVM_HOST_SDR1(r4) 1843 li r8,LPID_RSVD /* switch to reserved LPID */ 1844 mtspr SPRN_LPID,r8 1845 ptesync 1846 mtspr SPRN_SDR1,r6 /* switch to host page table */ 1847END_FTR_SECTION_IFCLR(CPU_FTR_ARCH_300) 1848 mtspr SPRN_LPID,r7 1849 isync 1850 1851BEGIN_FTR_SECTION 1852 /* DPDES and VTB are shared between threads */ 1853 mfspr r7, SPRN_DPDES 1854 mfspr r8, SPRN_VTB 1855 std r7, VCORE_DPDES(r5) 1856 std r8, VCORE_VTB(r5) 1857 /* clear DPDES so we don't get guest doorbells in the host */ 1858 li r8, 0 1859 mtspr SPRN_DPDES, r8 1860END_FTR_SECTION_IFSET(CPU_FTR_ARCH_207S) 1861 1862 /* Subtract timebase offset from timebase */ 1863 ld r8, VCORE_TB_OFFSET_APPL(r5) 1864 cmpdi r8,0 1865 beq 17f 1866 li r0, 0 1867 std r0, VCORE_TB_OFFSET_APPL(r5) 1868 mftb r6 /* current guest timebase */ 1869 subf r8,r8,r6 1870 mtspr SPRN_TBU40,r8 /* update upper 40 bits */ 1871 mftb r7 /* check if lower 24 bits overflowed */ 1872 clrldi r6,r6,40 1873 clrldi r7,r7,40 1874 cmpld r7,r6 1875 bge 17f 1876 addis r8,r8,0x100 /* if so, increment upper 40 bits */ 1877 mtspr SPRN_TBU40,r8 1878 187917: 1880 /* 1881 * If this is an HMI, we called kvmppc_realmode_hmi_handler 1882 * above, which may or may not have already called 1883 * kvmppc_subcore_exit_guest. Fortunately, all that 1884 * kvmppc_subcore_exit_guest does is clear a flag, so calling 1885 * it again here is benign even if kvmppc_realmode_hmi_handler 1886 * has already called it. 1887 */ 1888 bl kvmppc_subcore_exit_guest 1889 nop 189030: ld r5,HSTATE_KVM_VCORE(r13) 1891 ld r4,VCORE_KVM(r5) /* pointer to struct kvm */ 1892 1893 /* Reset PCR */ 1894 ld r0, VCORE_PCR(r5) 1895 cmpdi r0, 0 1896 beq 18f 1897 li r0, 0 1898 mtspr SPRN_PCR, r0 189918: 1900 /* Signal secondary CPUs to continue */ 1901 stb r0,VCORE_IN_GUEST(r5) 190219: lis r8,0x7fff /* MAX_INT@h */ 1903 mtspr SPRN_HDEC,r8 1904 190516: 1906BEGIN_FTR_SECTION 1907 /* On POWER9 with HPT-on-radix we need to wait for all other threads */ 1908 ld r3, HSTATE_SPLIT_MODE(r13) 1909 cmpdi r3, 0 1910 beq 47f 1911 lwz r8, KVM_SPLIT_DO_RESTORE(r3) 1912 cmpwi r8, 0 1913 beq 47f 1914 bl kvmhv_p9_restore_lpcr 1915 nop 1916 b 48f 191747: 1918END_FTR_SECTION_IFSET(CPU_FTR_ARCH_300) 1919 ld r8,KVM_HOST_LPCR(r4) 1920 mtspr SPRN_LPCR,r8 1921 isync 192248: 1923#ifdef CONFIG_KVM_BOOK3S_HV_EXIT_TIMING 1924 /* Finish timing, if we have a vcpu */ 1925 ld r4, HSTATE_KVM_VCPU(r13) 1926 cmpdi r4, 0 1927 li r3, 0 1928 beq 2f 1929 bl kvmhv_accumulate_time 19302: 1931#endif 1932 /* Unset guest mode */ 1933 li r0, KVM_GUEST_MODE_NONE 1934 stb r0, HSTATE_IN_GUEST(r13) 1935 1936 lwz r12, STACK_SLOT_TRAP(r1) /* return trap # in r12 */ 1937 ld r0, SFS+PPC_LR_STKOFF(r1) 1938 addi r1, r1, SFS 1939 mtlr r0 1940 blr 1941 1942kvmppc_guest_external: 1943 /* External interrupt, first check for host_ipi. If this is 1944 * set, we know the host wants us out so let's do it now 1945 */ 1946 bl kvmppc_read_intr 1947 1948 /* 1949 * Restore the active volatile registers after returning from 1950 * a C function. 1951 */ 1952 ld r9, HSTATE_KVM_VCPU(r13) 1953 li r12, BOOK3S_INTERRUPT_EXTERNAL 1954 1955 /* 1956 * kvmppc_read_intr return codes: 1957 * 1958 * Exit to host (r3 > 0) 1959 * 1 An interrupt is pending that needs to be handled by the host 1960 * Exit guest and return to host by branching to guest_exit_cont 1961 * 1962 * 2 Passthrough that needs completion in the host 1963 * Exit guest and return to host by branching to guest_exit_cont 1964 * However, we also set r12 to BOOK3S_INTERRUPT_HV_RM_HARD 1965 * to indicate to the host to complete handling the interrupt 1966 * 1967 * Before returning to guest, we check if any CPU is heading out 1968 * to the host and if so, we head out also. If no CPUs are heading 1969 * check return values <= 0. 1970 * 1971 * Return to guest (r3 <= 0) 1972 * 0 No external interrupt is pending 1973 * -1 A guest wakeup IPI (which has now been cleared) 1974 * In either case, we return to guest to deliver any pending 1975 * guest interrupts. 1976 * 1977 * -2 A PCI passthrough external interrupt was handled 1978 * (interrupt was delivered directly to guest) 1979 * Return to guest to deliver any pending guest interrupts. 1980 */ 1981 1982 cmpdi r3, 1 1983 ble 1f 1984 1985 /* Return code = 2 */ 1986 li r12, BOOK3S_INTERRUPT_HV_RM_HARD 1987 stw r12, VCPU_TRAP(r9) 1988 b guest_exit_cont 1989 19901: /* Return code <= 1 */ 1991 cmpdi r3, 0 1992 bgt guest_exit_cont 1993 1994 /* Return code <= 0 */ 1995maybe_reenter_guest: 1996 ld r5, HSTATE_KVM_VCORE(r13) 1997 lwz r0, VCORE_ENTRY_EXIT(r5) 1998 cmpwi r0, 0x100 1999 mr r4, r9 2000 blt deliver_guest_interrupt 2001 b guest_exit_cont 2002 2003#ifdef CONFIG_PPC_TRANSACTIONAL_MEM 2004/* 2005 * Softpatch interrupt for transactional memory emulation cases 2006 * on POWER9 DD2.2. This is early in the guest exit path - we 2007 * haven't saved registers or done a treclaim yet. 2008 */ 2009kvmppc_tm_emul: 2010 /* Save instruction image in HEIR */ 2011 mfspr r3, SPRN_HEIR 2012 stw r3, VCPU_HEIR(r9) 2013 2014 /* 2015 * The cases we want to handle here are those where the guest 2016 * is in real suspend mode and is trying to transition to 2017 * transactional mode. 2018 */ 2019 lbz r0, HSTATE_FAKE_SUSPEND(r13) 2020 cmpwi r0, 0 /* keep exiting guest if in fake suspend */ 2021 bne guest_exit_cont 2022 rldicl r3, r11, 64 - MSR_TS_S_LG, 62 2023 cmpwi r3, 1 /* or if not in suspend state */ 2024 bne guest_exit_cont 2025 2026 /* Call C code to do the emulation */ 2027 mr r3, r9 2028 bl kvmhv_p9_tm_emulation_early 2029 nop 2030 ld r9, HSTATE_KVM_VCPU(r13) 2031 li r12, BOOK3S_INTERRUPT_HV_SOFTPATCH 2032 cmpwi r3, 0 2033 beq guest_exit_cont /* continue exiting if not handled */ 2034 ld r10, VCPU_PC(r9) 2035 ld r11, VCPU_MSR(r9) 2036 b fast_interrupt_c_return /* go back to guest if handled */ 2037#endif /* CONFIG_PPC_TRANSACTIONAL_MEM */ 2038 2039/* 2040 * Check whether an HDSI is an HPTE not found fault or something else. 2041 * If it is an HPTE not found fault that is due to the guest accessing 2042 * a page that they have mapped but which we have paged out, then 2043 * we continue on with the guest exit path. In all other cases, 2044 * reflect the HDSI to the guest as a DSI. 2045 */ 2046kvmppc_hdsi: 2047 ld r3, VCPU_KVM(r9) 2048 lbz r0, KVM_RADIX(r3) 2049 mfspr r4, SPRN_HDAR 2050 mfspr r6, SPRN_HDSISR 2051BEGIN_FTR_SECTION 2052 /* Look for DSISR canary. If we find it, retry instruction */ 2053 cmpdi r6, 0x7fff 2054 beq 6f 2055END_FTR_SECTION_IFSET(CPU_FTR_ARCH_300) 2056 cmpwi r0, 0 2057 bne .Lradix_hdsi /* on radix, just save DAR/DSISR/ASDR */ 2058 /* HPTE not found fault or protection fault? */ 2059 andis. r0, r6, (DSISR_NOHPTE | DSISR_PROTFAULT)@h 2060 beq 1f /* if not, send it to the guest */ 2061 andi. r0, r11, MSR_DR /* data relocation enabled? */ 2062 beq 3f 2063BEGIN_FTR_SECTION 2064 mfspr r5, SPRN_ASDR /* on POWER9, use ASDR to get VSID */ 2065 b 4f 2066END_FTR_SECTION_IFSET(CPU_FTR_ARCH_300) 2067 clrrdi r0, r4, 28 2068 PPC_SLBFEE_DOT(R5, R0) /* if so, look up SLB */ 2069 li r0, BOOK3S_INTERRUPT_DATA_SEGMENT 2070 bne 7f /* if no SLB entry found */ 20714: std r4, VCPU_FAULT_DAR(r9) 2072 stw r6, VCPU_FAULT_DSISR(r9) 2073 2074 /* Search the hash table. */ 2075 mr r3, r9 /* vcpu pointer */ 2076 li r7, 1 /* data fault */ 2077 bl kvmppc_hpte_hv_fault 2078 ld r9, HSTATE_KVM_VCPU(r13) 2079 ld r10, VCPU_PC(r9) 2080 ld r11, VCPU_MSR(r9) 2081 li r12, BOOK3S_INTERRUPT_H_DATA_STORAGE 2082 cmpdi r3, 0 /* retry the instruction */ 2083 beq 6f 2084 cmpdi r3, -1 /* handle in kernel mode */ 2085 beq guest_exit_cont 2086 cmpdi r3, -2 /* MMIO emulation; need instr word */ 2087 beq 2f 2088 2089 /* Synthesize a DSI (or DSegI) for the guest */ 2090 ld r4, VCPU_FAULT_DAR(r9) 2091 mr r6, r3 20921: li r0, BOOK3S_INTERRUPT_DATA_STORAGE 2093 mtspr SPRN_DSISR, r6 20947: mtspr SPRN_DAR, r4 2095 mtspr SPRN_SRR0, r10 2096 mtspr SPRN_SRR1, r11 2097 mr r10, r0 2098 bl kvmppc_msr_interrupt 2099fast_interrupt_c_return: 21006: ld r7, VCPU_CTR(r9) 2101 ld r8, VCPU_XER(r9) 2102 mtctr r7 2103 mtxer r8 2104 mr r4, r9 2105 b fast_guest_return 2106 21073: ld r5, VCPU_KVM(r9) /* not relocated, use VRMA */ 2108 ld r5, KVM_VRMA_SLB_V(r5) 2109 b 4b 2110 2111 /* If this is for emulated MMIO, load the instruction word */ 21122: li r8, KVM_INST_FETCH_FAILED /* In case lwz faults */ 2113 2114 /* Set guest mode to 'jump over instruction' so if lwz faults 2115 * we'll just continue at the next IP. */ 2116 li r0, KVM_GUEST_MODE_SKIP 2117 stb r0, HSTATE_IN_GUEST(r13) 2118 2119 /* Do the access with MSR:DR enabled */ 2120 mfmsr r3 2121 ori r4, r3, MSR_DR /* Enable paging for data */ 2122 mtmsrd r4 2123 lwz r8, 0(r10) 2124 mtmsrd r3 2125 2126 /* Store the result */ 2127 stw r8, VCPU_LAST_INST(r9) 2128 2129 /* Unset guest mode. */ 2130 li r0, KVM_GUEST_MODE_HOST_HV 2131 stb r0, HSTATE_IN_GUEST(r13) 2132 b guest_exit_cont 2133 2134.Lradix_hdsi: 2135 std r4, VCPU_FAULT_DAR(r9) 2136 stw r6, VCPU_FAULT_DSISR(r9) 2137.Lradix_hisi: 2138 mfspr r5, SPRN_ASDR 2139 std r5, VCPU_FAULT_GPA(r9) 2140 b guest_exit_cont 2141 2142/* 2143 * Similarly for an HISI, reflect it to the guest as an ISI unless 2144 * it is an HPTE not found fault for a page that we have paged out. 2145 */ 2146kvmppc_hisi: 2147 ld r3, VCPU_KVM(r9) 2148 lbz r0, KVM_RADIX(r3) 2149 cmpwi r0, 0 2150 bne .Lradix_hisi /* for radix, just save ASDR */ 2151 andis. r0, r11, SRR1_ISI_NOPT@h 2152 beq 1f 2153 andi. r0, r11, MSR_IR /* instruction relocation enabled? */ 2154 beq 3f 2155BEGIN_FTR_SECTION 2156 mfspr r5, SPRN_ASDR /* on POWER9, use ASDR to get VSID */ 2157 b 4f 2158END_FTR_SECTION_IFSET(CPU_FTR_ARCH_300) 2159 clrrdi r0, r10, 28 2160 PPC_SLBFEE_DOT(R5, R0) /* if so, look up SLB */ 2161 li r0, BOOK3S_INTERRUPT_INST_SEGMENT 2162 bne 7f /* if no SLB entry found */ 21634: 2164 /* Search the hash table. */ 2165 mr r3, r9 /* vcpu pointer */ 2166 mr r4, r10 2167 mr r6, r11 2168 li r7, 0 /* instruction fault */ 2169 bl kvmppc_hpte_hv_fault 2170 ld r9, HSTATE_KVM_VCPU(r13) 2171 ld r10, VCPU_PC(r9) 2172 ld r11, VCPU_MSR(r9) 2173 li r12, BOOK3S_INTERRUPT_H_INST_STORAGE 2174 cmpdi r3, 0 /* retry the instruction */ 2175 beq fast_interrupt_c_return 2176 cmpdi r3, -1 /* handle in kernel mode */ 2177 beq guest_exit_cont 2178 2179 /* Synthesize an ISI (or ISegI) for the guest */ 2180 mr r11, r3 21811: li r0, BOOK3S_INTERRUPT_INST_STORAGE 21827: mtspr SPRN_SRR0, r10 2183 mtspr SPRN_SRR1, r11 2184 mr r10, r0 2185 bl kvmppc_msr_interrupt 2186 b fast_interrupt_c_return 2187 21883: ld r6, VCPU_KVM(r9) /* not relocated, use VRMA */ 2189 ld r5, KVM_VRMA_SLB_V(r6) 2190 b 4b 2191 2192/* 2193 * Try to handle an hcall in real mode. 2194 * Returns to the guest if we handle it, or continues on up to 2195 * the kernel if we can't (i.e. if we don't have a handler for 2196 * it, or if the handler returns H_TOO_HARD). 2197 * 2198 * r5 - r8 contain hcall args, 2199 * r9 = vcpu, r10 = pc, r11 = msr, r12 = trap, r13 = paca 2200 */ 2201hcall_try_real_mode: 2202 ld r3,VCPU_GPR(R3)(r9) 2203 andi. r0,r11,MSR_PR 2204 /* sc 1 from userspace - reflect to guest syscall */ 2205 bne sc_1_fast_return 2206 /* sc 1 from nested guest - give it to L1 to handle */ 2207 ld r0, VCPU_NESTED(r9) 2208 cmpdi r0, 0 2209 bne guest_exit_cont 2210 clrrdi r3,r3,2 2211 cmpldi r3,hcall_real_table_end - hcall_real_table 2212 bge guest_exit_cont 2213 /* See if this hcall is enabled for in-kernel handling */ 2214 ld r4, VCPU_KVM(r9) 2215 srdi r0, r3, 8 /* r0 = (r3 / 4) >> 6 */ 2216 sldi r0, r0, 3 /* index into kvm->arch.enabled_hcalls[] */ 2217 add r4, r4, r0 2218 ld r0, KVM_ENABLED_HCALLS(r4) 2219 rlwinm r4, r3, 32-2, 0x3f /* r4 = (r3 / 4) & 0x3f */ 2220 srd r0, r0, r4 2221 andi. r0, r0, 1 2222 beq guest_exit_cont 2223 /* Get pointer to handler, if any, and call it */ 2224 LOAD_REG_ADDR(r4, hcall_real_table) 2225 lwax r3,r3,r4 2226 cmpwi r3,0 2227 beq guest_exit_cont 2228 add r12,r3,r4 2229 mtctr r12 2230 mr r3,r9 /* get vcpu pointer */ 2231 ld r4,VCPU_GPR(R4)(r9) 2232 bctrl 2233 cmpdi r3,H_TOO_HARD 2234 beq hcall_real_fallback 2235 ld r4,HSTATE_KVM_VCPU(r13) 2236 std r3,VCPU_GPR(R3)(r4) 2237 ld r10,VCPU_PC(r4) 2238 ld r11,VCPU_MSR(r4) 2239 b fast_guest_return 2240 2241sc_1_fast_return: 2242 mtspr SPRN_SRR0,r10 2243 mtspr SPRN_SRR1,r11 2244 li r10, BOOK3S_INTERRUPT_SYSCALL 2245 bl kvmppc_msr_interrupt 2246 mr r4,r9 2247 b fast_guest_return 2248 2249 /* We've attempted a real mode hcall, but it's punted it back 2250 * to userspace. We need to restore some clobbered volatiles 2251 * before resuming the pass-it-to-qemu path */ 2252hcall_real_fallback: 2253 li r12,BOOK3S_INTERRUPT_SYSCALL 2254 ld r9, HSTATE_KVM_VCPU(r13) 2255 2256 b guest_exit_cont 2257 2258 .globl hcall_real_table 2259hcall_real_table: 2260 .long 0 /* 0 - unused */ 2261 .long DOTSYM(kvmppc_h_remove) - hcall_real_table 2262 .long DOTSYM(kvmppc_h_enter) - hcall_real_table 2263 .long DOTSYM(kvmppc_h_read) - hcall_real_table 2264 .long DOTSYM(kvmppc_h_clear_mod) - hcall_real_table 2265 .long DOTSYM(kvmppc_h_clear_ref) - hcall_real_table 2266 .long DOTSYM(kvmppc_h_protect) - hcall_real_table 2267 .long DOTSYM(kvmppc_h_get_tce) - hcall_real_table 2268 .long DOTSYM(kvmppc_rm_h_put_tce) - hcall_real_table 2269 .long 0 /* 0x24 - H_SET_SPRG0 */ 2270 .long DOTSYM(kvmppc_h_set_dabr) - hcall_real_table 2271 .long 0 /* 0x2c */ 2272 .long 0 /* 0x30 */ 2273 .long 0 /* 0x34 */ 2274 .long 0 /* 0x38 */ 2275 .long 0 /* 0x3c */ 2276 .long 0 /* 0x40 */ 2277 .long 0 /* 0x44 */ 2278 .long 0 /* 0x48 */ 2279 .long 0 /* 0x4c */ 2280 .long 0 /* 0x50 */ 2281 .long 0 /* 0x54 */ 2282 .long 0 /* 0x58 */ 2283 .long 0 /* 0x5c */ 2284 .long 0 /* 0x60 */ 2285#ifdef CONFIG_KVM_XICS 2286 .long DOTSYM(kvmppc_rm_h_eoi) - hcall_real_table 2287 .long DOTSYM(kvmppc_rm_h_cppr) - hcall_real_table 2288 .long DOTSYM(kvmppc_rm_h_ipi) - hcall_real_table 2289 .long DOTSYM(kvmppc_rm_h_ipoll) - hcall_real_table 2290 .long DOTSYM(kvmppc_rm_h_xirr) - hcall_real_table 2291#else 2292 .long 0 /* 0x64 - H_EOI */ 2293 .long 0 /* 0x68 - H_CPPR */ 2294 .long 0 /* 0x6c - H_IPI */ 2295 .long 0 /* 0x70 - H_IPOLL */ 2296 .long 0 /* 0x74 - H_XIRR */ 2297#endif 2298 .long 0 /* 0x78 */ 2299 .long 0 /* 0x7c */ 2300 .long 0 /* 0x80 */ 2301 .long 0 /* 0x84 */ 2302 .long 0 /* 0x88 */ 2303 .long 0 /* 0x8c */ 2304 .long 0 /* 0x90 */ 2305 .long 0 /* 0x94 */ 2306 .long 0 /* 0x98 */ 2307 .long 0 /* 0x9c */ 2308 .long 0 /* 0xa0 */ 2309 .long 0 /* 0xa4 */ 2310 .long 0 /* 0xa8 */ 2311 .long 0 /* 0xac */ 2312 .long 0 /* 0xb0 */ 2313 .long 0 /* 0xb4 */ 2314 .long 0 /* 0xb8 */ 2315 .long 0 /* 0xbc */ 2316 .long 0 /* 0xc0 */ 2317 .long 0 /* 0xc4 */ 2318 .long 0 /* 0xc8 */ 2319 .long 0 /* 0xcc */ 2320 .long 0 /* 0xd0 */ 2321 .long 0 /* 0xd4 */ 2322 .long 0 /* 0xd8 */ 2323 .long 0 /* 0xdc */ 2324 .long DOTSYM(kvmppc_h_cede) - hcall_real_table 2325 .long DOTSYM(kvmppc_rm_h_confer) - hcall_real_table 2326 .long 0 /* 0xe8 */ 2327 .long 0 /* 0xec */ 2328 .long 0 /* 0xf0 */ 2329 .long 0 /* 0xf4 */ 2330 .long 0 /* 0xf8 */ 2331 .long 0 /* 0xfc */ 2332 .long 0 /* 0x100 */ 2333 .long 0 /* 0x104 */ 2334 .long 0 /* 0x108 */ 2335 .long 0 /* 0x10c */ 2336 .long 0 /* 0x110 */ 2337 .long 0 /* 0x114 */ 2338 .long 0 /* 0x118 */ 2339 .long 0 /* 0x11c */ 2340 .long 0 /* 0x120 */ 2341 .long DOTSYM(kvmppc_h_bulk_remove) - hcall_real_table 2342 .long 0 /* 0x128 */ 2343 .long 0 /* 0x12c */ 2344 .long 0 /* 0x130 */ 2345 .long DOTSYM(kvmppc_h_set_xdabr) - hcall_real_table 2346 .long DOTSYM(kvmppc_rm_h_stuff_tce) - hcall_real_table 2347 .long DOTSYM(kvmppc_rm_h_put_tce_indirect) - hcall_real_table 2348 .long 0 /* 0x140 */ 2349 .long 0 /* 0x144 */ 2350 .long 0 /* 0x148 */ 2351 .long 0 /* 0x14c */ 2352 .long 0 /* 0x150 */ 2353 .long 0 /* 0x154 */ 2354 .long 0 /* 0x158 */ 2355 .long 0 /* 0x15c */ 2356 .long 0 /* 0x160 */ 2357 .long 0 /* 0x164 */ 2358 .long 0 /* 0x168 */ 2359 .long 0 /* 0x16c */ 2360 .long 0 /* 0x170 */ 2361 .long 0 /* 0x174 */ 2362 .long 0 /* 0x178 */ 2363 .long 0 /* 0x17c */ 2364 .long 0 /* 0x180 */ 2365 .long 0 /* 0x184 */ 2366 .long 0 /* 0x188 */ 2367 .long 0 /* 0x18c */ 2368 .long 0 /* 0x190 */ 2369 .long 0 /* 0x194 */ 2370 .long 0 /* 0x198 */ 2371 .long 0 /* 0x19c */ 2372 .long 0 /* 0x1a0 */ 2373 .long 0 /* 0x1a4 */ 2374 .long 0 /* 0x1a8 */ 2375 .long 0 /* 0x1ac */ 2376 .long 0 /* 0x1b0 */ 2377 .long 0 /* 0x1b4 */ 2378 .long 0 /* 0x1b8 */ 2379 .long 0 /* 0x1bc */ 2380 .long 0 /* 0x1c0 */ 2381 .long 0 /* 0x1c4 */ 2382 .long 0 /* 0x1c8 */ 2383 .long 0 /* 0x1cc */ 2384 .long 0 /* 0x1d0 */ 2385 .long 0 /* 0x1d4 */ 2386 .long 0 /* 0x1d8 */ 2387 .long 0 /* 0x1dc */ 2388 .long 0 /* 0x1e0 */ 2389 .long 0 /* 0x1e4 */ 2390 .long 0 /* 0x1e8 */ 2391 .long 0 /* 0x1ec */ 2392 .long 0 /* 0x1f0 */ 2393 .long 0 /* 0x1f4 */ 2394 .long 0 /* 0x1f8 */ 2395 .long 0 /* 0x1fc */ 2396 .long 0 /* 0x200 */ 2397 .long 0 /* 0x204 */ 2398 .long 0 /* 0x208 */ 2399 .long 0 /* 0x20c */ 2400 .long 0 /* 0x210 */ 2401 .long 0 /* 0x214 */ 2402 .long 0 /* 0x218 */ 2403 .long 0 /* 0x21c */ 2404 .long 0 /* 0x220 */ 2405 .long 0 /* 0x224 */ 2406 .long 0 /* 0x228 */ 2407 .long 0 /* 0x22c */ 2408 .long 0 /* 0x230 */ 2409 .long 0 /* 0x234 */ 2410 .long 0 /* 0x238 */ 2411 .long 0 /* 0x23c */ 2412 .long 0 /* 0x240 */ 2413 .long 0 /* 0x244 */ 2414 .long 0 /* 0x248 */ 2415 .long 0 /* 0x24c */ 2416 .long 0 /* 0x250 */ 2417 .long 0 /* 0x254 */ 2418 .long 0 /* 0x258 */ 2419 .long 0 /* 0x25c */ 2420 .long 0 /* 0x260 */ 2421 .long 0 /* 0x264 */ 2422 .long 0 /* 0x268 */ 2423 .long 0 /* 0x26c */ 2424 .long 0 /* 0x270 */ 2425 .long 0 /* 0x274 */ 2426 .long 0 /* 0x278 */ 2427 .long 0 /* 0x27c */ 2428 .long 0 /* 0x280 */ 2429 .long 0 /* 0x284 */ 2430 .long 0 /* 0x288 */ 2431 .long 0 /* 0x28c */ 2432 .long 0 /* 0x290 */ 2433 .long 0 /* 0x294 */ 2434 .long 0 /* 0x298 */ 2435 .long 0 /* 0x29c */ 2436 .long 0 /* 0x2a0 */ 2437 .long 0 /* 0x2a4 */ 2438 .long 0 /* 0x2a8 */ 2439 .long 0 /* 0x2ac */ 2440 .long 0 /* 0x2b0 */ 2441 .long 0 /* 0x2b4 */ 2442 .long 0 /* 0x2b8 */ 2443 .long 0 /* 0x2bc */ 2444 .long 0 /* 0x2c0 */ 2445 .long 0 /* 0x2c4 */ 2446 .long 0 /* 0x2c8 */ 2447 .long 0 /* 0x2cc */ 2448 .long 0 /* 0x2d0 */ 2449 .long 0 /* 0x2d4 */ 2450 .long 0 /* 0x2d8 */ 2451 .long 0 /* 0x2dc */ 2452 .long 0 /* 0x2e0 */ 2453 .long 0 /* 0x2e4 */ 2454 .long 0 /* 0x2e8 */ 2455 .long 0 /* 0x2ec */ 2456 .long 0 /* 0x2f0 */ 2457 .long 0 /* 0x2f4 */ 2458 .long 0 /* 0x2f8 */ 2459#ifdef CONFIG_KVM_XICS 2460 .long DOTSYM(kvmppc_rm_h_xirr_x) - hcall_real_table 2461#else 2462 .long 0 /* 0x2fc - H_XIRR_X*/ 2463#endif 2464 .long DOTSYM(kvmppc_h_random) - hcall_real_table 2465 .globl hcall_real_table_end 2466hcall_real_table_end: 2467 2468_GLOBAL(kvmppc_h_set_xdabr) 2469EXPORT_SYMBOL_GPL(kvmppc_h_set_xdabr) 2470 andi. r0, r5, DABRX_USER | DABRX_KERNEL 2471 beq 6f 2472 li r0, DABRX_USER | DABRX_KERNEL | DABRX_BTI 2473 andc. r0, r5, r0 2474 beq 3f 24756: li r3, H_PARAMETER 2476 blr 2477 2478_GLOBAL(kvmppc_h_set_dabr) 2479EXPORT_SYMBOL_GPL(kvmppc_h_set_dabr) 2480 li r5, DABRX_USER | DABRX_KERNEL 24813: 2482BEGIN_FTR_SECTION 2483 b 2f 2484END_FTR_SECTION_IFSET(CPU_FTR_ARCH_207S) 2485 std r4,VCPU_DABR(r3) 2486 stw r5, VCPU_DABRX(r3) 2487 mtspr SPRN_DABRX, r5 2488 /* Work around P7 bug where DABR can get corrupted on mtspr */ 24891: mtspr SPRN_DABR,r4 2490 mfspr r5, SPRN_DABR 2491 cmpd r4, r5 2492 bne 1b 2493 isync 2494 li r3,0 2495 blr 2496 24972: 2498BEGIN_FTR_SECTION 2499 /* POWER9 with disabled DAWR */ 2500 li r3, H_HARDWARE 2501 blr 2502END_FTR_SECTION_IFCLR(CPU_FTR_DAWR) 2503 /* Emulate H_SET_DABR/X on P8 for the sake of compat mode guests */ 2504 rlwimi r5, r4, 5, DAWRX_DR | DAWRX_DW 2505 rlwimi r5, r4, 2, DAWRX_WT 2506 clrrdi r4, r4, 3 2507 std r4, VCPU_DAWR(r3) 2508 std r5, VCPU_DAWRX(r3) 2509 mtspr SPRN_DAWR, r4 2510 mtspr SPRN_DAWRX, r5 2511 li r3, 0 2512 blr 2513 2514_GLOBAL(kvmppc_h_cede) /* r3 = vcpu pointer, r11 = msr, r13 = paca */ 2515 ori r11,r11,MSR_EE 2516 std r11,VCPU_MSR(r3) 2517 li r0,1 2518 stb r0,VCPU_CEDED(r3) 2519 sync /* order setting ceded vs. testing prodded */ 2520 lbz r5,VCPU_PRODDED(r3) 2521 cmpwi r5,0 2522 bne kvm_cede_prodded 2523 li r12,0 /* set trap to 0 to say hcall is handled */ 2524 stw r12,VCPU_TRAP(r3) 2525 li r0,H_SUCCESS 2526 std r0,VCPU_GPR(R3)(r3) 2527 2528 /* 2529 * Set our bit in the bitmask of napping threads unless all the 2530 * other threads are already napping, in which case we send this 2531 * up to the host. 2532 */ 2533 ld r5,HSTATE_KVM_VCORE(r13) 2534 lbz r6,HSTATE_PTID(r13) 2535 lwz r8,VCORE_ENTRY_EXIT(r5) 2536 clrldi r8,r8,56 2537 li r0,1 2538 sld r0,r0,r6 2539 addi r6,r5,VCORE_NAPPING_THREADS 254031: lwarx r4,0,r6 2541 or r4,r4,r0 2542 cmpw r4,r8 2543 beq kvm_cede_exit 2544 stwcx. r4,0,r6 2545 bne 31b 2546 /* order napping_threads update vs testing entry_exit_map */ 2547 isync 2548 li r0,NAPPING_CEDE 2549 stb r0,HSTATE_NAPPING(r13) 2550 lwz r7,VCORE_ENTRY_EXIT(r5) 2551 cmpwi r7,0x100 2552 bge 33f /* another thread already exiting */ 2553 2554/* 2555 * Although not specifically required by the architecture, POWER7 2556 * preserves the following registers in nap mode, even if an SMT mode 2557 * switch occurs: SLB entries, PURR, SPURR, AMOR, UAMOR, AMR, SPRG0-3, 2558 * DAR, DSISR, DABR, DABRX, DSCR, PMCx, MMCRx, SIAR, SDAR. 2559 */ 2560 /* Save non-volatile GPRs */ 2561 std r14, VCPU_GPR(R14)(r3) 2562 std r15, VCPU_GPR(R15)(r3) 2563 std r16, VCPU_GPR(R16)(r3) 2564 std r17, VCPU_GPR(R17)(r3) 2565 std r18, VCPU_GPR(R18)(r3) 2566 std r19, VCPU_GPR(R19)(r3) 2567 std r20, VCPU_GPR(R20)(r3) 2568 std r21, VCPU_GPR(R21)(r3) 2569 std r22, VCPU_GPR(R22)(r3) 2570 std r23, VCPU_GPR(R23)(r3) 2571 std r24, VCPU_GPR(R24)(r3) 2572 std r25, VCPU_GPR(R25)(r3) 2573 std r26, VCPU_GPR(R26)(r3) 2574 std r27, VCPU_GPR(R27)(r3) 2575 std r28, VCPU_GPR(R28)(r3) 2576 std r29, VCPU_GPR(R29)(r3) 2577 std r30, VCPU_GPR(R30)(r3) 2578 std r31, VCPU_GPR(R31)(r3) 2579 2580 /* save FP state */ 2581 bl kvmppc_save_fp 2582 2583#ifdef CONFIG_PPC_TRANSACTIONAL_MEM 2584/* 2585 * Branch around the call if both CPU_FTR_TM and 2586 * CPU_FTR_P9_TM_HV_ASSIST are off. 2587 */ 2588BEGIN_FTR_SECTION 2589 b 91f 2590END_FTR_SECTION(CPU_FTR_TM | CPU_FTR_P9_TM_HV_ASSIST, 0) 2591 /* 2592 * NOTE THAT THIS TRASHES ALL NON-VOLATILE REGISTERS (but not CR) 2593 */ 2594 ld r3, HSTATE_KVM_VCPU(r13) 2595 ld r4, VCPU_MSR(r3) 2596 li r5, 0 /* don't preserve non-vol regs */ 2597 bl kvmppc_save_tm_hv 2598 nop 259991: 2600#endif 2601 2602 /* 2603 * Set DEC to the smaller of DEC and HDEC, so that we wake 2604 * no later than the end of our timeslice (HDEC interrupts 2605 * don't wake us from nap). 2606 */ 2607 mfspr r3, SPRN_DEC 2608 mfspr r4, SPRN_HDEC 2609 mftb r5 2610BEGIN_FTR_SECTION 2611 /* On P9 check whether the guest has large decrementer mode enabled */ 2612 ld r6, HSTATE_KVM_VCORE(r13) 2613 ld r6, VCORE_LPCR(r6) 2614 andis. r6, r6, LPCR_LD@h 2615 bne 68f 2616END_FTR_SECTION_IFSET(CPU_FTR_ARCH_300) 2617 extsw r3, r3 261868: EXTEND_HDEC(r4) 2619 cmpd r3, r4 2620 ble 67f 2621 mtspr SPRN_DEC, r4 262267: 2623 /* save expiry time of guest decrementer */ 2624 add r3, r3, r5 2625 ld r4, HSTATE_KVM_VCPU(r13) 2626 ld r5, HSTATE_KVM_VCORE(r13) 2627 ld r6, VCORE_TB_OFFSET_APPL(r5) 2628 subf r3, r6, r3 /* convert to host TB value */ 2629 std r3, VCPU_DEC_EXPIRES(r4) 2630 2631#ifdef CONFIG_KVM_BOOK3S_HV_EXIT_TIMING 2632 ld r4, HSTATE_KVM_VCPU(r13) 2633 addi r3, r4, VCPU_TB_CEDE 2634 bl kvmhv_accumulate_time 2635#endif 2636 2637 lis r3, LPCR_PECEDP@h /* Do wake on privileged doorbell */ 2638 2639 /* 2640 * Take a nap until a decrementer or external or doobell interrupt 2641 * occurs, with PECE1 and PECE0 set in LPCR. 2642 * On POWER8, set PECEDH, and if we are ceding, also set PECEDP. 2643 * Also clear the runlatch bit before napping. 2644 */ 2645kvm_do_nap: 2646 mfspr r0, SPRN_CTRLF 2647 clrrdi r0, r0, 1 2648 mtspr SPRN_CTRLT, r0 2649 2650 li r0,1 2651 stb r0,HSTATE_HWTHREAD_REQ(r13) 2652 mfspr r5,SPRN_LPCR 2653 ori r5,r5,LPCR_PECE0 | LPCR_PECE1 2654BEGIN_FTR_SECTION 2655 ori r5, r5, LPCR_PECEDH 2656 rlwimi r5, r3, 0, LPCR_PECEDP 2657END_FTR_SECTION_IFSET(CPU_FTR_ARCH_207S) 2658 2659kvm_nap_sequence: /* desired LPCR value in r5 */ 2660BEGIN_FTR_SECTION 2661 /* 2662 * PSSCR bits: exit criterion = 1 (wakeup based on LPCR at sreset) 2663 * enable state loss = 1 (allow SMT mode switch) 2664 * requested level = 0 (just stop dispatching) 2665 */ 2666 lis r3, (PSSCR_EC | PSSCR_ESL)@h 2667 mtspr SPRN_PSSCR, r3 2668 /* Set LPCR_PECE_HVEE bit to enable wakeup by HV interrupts */ 2669 li r4, LPCR_PECE_HVEE@higher 2670 sldi r4, r4, 32 2671 or r5, r5, r4 2672END_FTR_SECTION_IFSET(CPU_FTR_ARCH_300) 2673 mtspr SPRN_LPCR,r5 2674 isync 2675 li r0, 0 2676 std r0, HSTATE_SCRATCH0(r13) 2677 ptesync 2678 ld r0, HSTATE_SCRATCH0(r13) 26791: cmpd r0, r0 2680 bne 1b 2681BEGIN_FTR_SECTION 2682 nap 2683FTR_SECTION_ELSE 2684 PPC_STOP 2685ALT_FTR_SECTION_END_IFCLR(CPU_FTR_ARCH_300) 2686 b . 2687 268833: mr r4, r3 2689 li r3, 0 2690 li r12, 0 2691 b 34f 2692 2693kvm_end_cede: 2694 /* get vcpu pointer */ 2695 ld r4, HSTATE_KVM_VCPU(r13) 2696 2697 /* Woken by external or decrementer interrupt */ 2698 ld r1, HSTATE_HOST_R1(r13) 2699 2700#ifdef CONFIG_KVM_BOOK3S_HV_EXIT_TIMING 2701 addi r3, r4, VCPU_TB_RMINTR 2702 bl kvmhv_accumulate_time 2703#endif 2704 2705#ifdef CONFIG_PPC_TRANSACTIONAL_MEM 2706/* 2707 * Branch around the call if both CPU_FTR_TM and 2708 * CPU_FTR_P9_TM_HV_ASSIST are off. 2709 */ 2710BEGIN_FTR_SECTION 2711 b 91f 2712END_FTR_SECTION(CPU_FTR_TM | CPU_FTR_P9_TM_HV_ASSIST, 0) 2713 /* 2714 * NOTE THAT THIS TRASHES ALL NON-VOLATILE REGISTERS (but not CR) 2715 */ 2716 mr r3, r4 2717 ld r4, VCPU_MSR(r3) 2718 li r5, 0 /* don't preserve non-vol regs */ 2719 bl kvmppc_restore_tm_hv 2720 nop 2721 ld r4, HSTATE_KVM_VCPU(r13) 272291: 2723#endif 2724 2725 /* load up FP state */ 2726 bl kvmppc_load_fp 2727 2728 /* Restore guest decrementer */ 2729 ld r3, VCPU_DEC_EXPIRES(r4) 2730 ld r5, HSTATE_KVM_VCORE(r13) 2731 ld r6, VCORE_TB_OFFSET_APPL(r5) 2732 add r3, r3, r6 /* convert host TB to guest TB value */ 2733 mftb r7 2734 subf r3, r7, r3 2735 mtspr SPRN_DEC, r3 2736 2737 /* Load NV GPRS */ 2738 ld r14, VCPU_GPR(R14)(r4) 2739 ld r15, VCPU_GPR(R15)(r4) 2740 ld r16, VCPU_GPR(R16)(r4) 2741 ld r17, VCPU_GPR(R17)(r4) 2742 ld r18, VCPU_GPR(R18)(r4) 2743 ld r19, VCPU_GPR(R19)(r4) 2744 ld r20, VCPU_GPR(R20)(r4) 2745 ld r21, VCPU_GPR(R21)(r4) 2746 ld r22, VCPU_GPR(R22)(r4) 2747 ld r23, VCPU_GPR(R23)(r4) 2748 ld r24, VCPU_GPR(R24)(r4) 2749 ld r25, VCPU_GPR(R25)(r4) 2750 ld r26, VCPU_GPR(R26)(r4) 2751 ld r27, VCPU_GPR(R27)(r4) 2752 ld r28, VCPU_GPR(R28)(r4) 2753 ld r29, VCPU_GPR(R29)(r4) 2754 ld r30, VCPU_GPR(R30)(r4) 2755 ld r31, VCPU_GPR(R31)(r4) 2756 2757 /* Check the wake reason in SRR1 to see why we got here */ 2758 bl kvmppc_check_wake_reason 2759 2760 /* 2761 * Restore volatile registers since we could have called a 2762 * C routine in kvmppc_check_wake_reason 2763 * r4 = VCPU 2764 * r3 tells us whether we need to return to host or not 2765 * WARNING: it gets checked further down: 2766 * should not modify r3 until this check is done. 2767 */ 2768 ld r4, HSTATE_KVM_VCPU(r13) 2769 2770 /* clear our bit in vcore->napping_threads */ 277134: ld r5,HSTATE_KVM_VCORE(r13) 2772 lbz r7,HSTATE_PTID(r13) 2773 li r0,1 2774 sld r0,r0,r7 2775 addi r6,r5,VCORE_NAPPING_THREADS 277632: lwarx r7,0,r6 2777 andc r7,r7,r0 2778 stwcx. r7,0,r6 2779 bne 32b 2780 li r0,0 2781 stb r0,HSTATE_NAPPING(r13) 2782 2783 /* See if the wake reason saved in r3 means we need to exit */ 2784 stw r12, VCPU_TRAP(r4) 2785 mr r9, r4 2786 cmpdi r3, 0 2787 bgt guest_exit_cont 2788 b maybe_reenter_guest 2789 2790 /* cede when already previously prodded case */ 2791kvm_cede_prodded: 2792 li r0,0 2793 stb r0,VCPU_PRODDED(r3) 2794 sync /* order testing prodded vs. clearing ceded */ 2795 stb r0,VCPU_CEDED(r3) 2796 li r3,H_SUCCESS 2797 blr 2798 2799 /* we've ceded but we want to give control to the host */ 2800kvm_cede_exit: 2801 ld r9, HSTATE_KVM_VCPU(r13) 2802#ifdef CONFIG_KVM_XICS 2803 /* Abort if we still have a pending escalation */ 2804 lbz r5, VCPU_XIVE_ESC_ON(r9) 2805 cmpwi r5, 0 2806 beq 1f 2807 li r0, 0 2808 stb r0, VCPU_CEDED(r9) 28091: /* Enable XIVE escalation */ 2810 li r5, XIVE_ESB_SET_PQ_00 2811 mfmsr r0 2812 andi. r0, r0, MSR_DR /* in real mode? */ 2813 beq 1f 2814 ld r10, VCPU_XIVE_ESC_VADDR(r9) 2815 cmpdi r10, 0 2816 beq 3f 2817 ldx r0, r10, r5 2818 b 2f 28191: ld r10, VCPU_XIVE_ESC_RADDR(r9) 2820 cmpdi r10, 0 2821 beq 3f 2822 ldcix r0, r10, r5 28232: sync 2824 li r0, 1 2825 stb r0, VCPU_XIVE_ESC_ON(r9) 2826#endif /* CONFIG_KVM_XICS */ 28273: b guest_exit_cont 2828 2829 /* Try to handle a machine check in real mode */ 2830machine_check_realmode: 2831 mr r3, r9 /* get vcpu pointer */ 2832 bl kvmppc_realmode_machine_check 2833 nop 2834 ld r9, HSTATE_KVM_VCPU(r13) 2835 li r12, BOOK3S_INTERRUPT_MACHINE_CHECK 2836 /* 2837 * For the guest that is FWNMI capable, deliver all the MCE errors 2838 * (handled/unhandled) by exiting the guest with KVM_EXIT_NMI exit 2839 * reason. This new approach injects machine check errors in guest 2840 * address space to guest with additional information in the form 2841 * of RTAS event, thus enabling guest kernel to suitably handle 2842 * such errors. 2843 * 2844 * For the guest that is not FWNMI capable (old QEMU) fallback 2845 * to old behaviour for backward compatibility: 2846 * Deliver unhandled/fatal (e.g. UE) MCE errors to guest either 2847 * through machine check interrupt (set HSRR0 to 0x200). 2848 * For handled errors (no-fatal), just go back to guest execution 2849 * with current HSRR0. 2850 * if we receive machine check with MSR(RI=0) then deliver it to 2851 * guest as machine check causing guest to crash. 2852 */ 2853 ld r11, VCPU_MSR(r9) 2854 rldicl. r0, r11, 64-MSR_HV_LG, 63 /* check if it happened in HV mode */ 2855 bne guest_exit_cont /* if so, exit to host */ 2856 /* Check if guest is capable of handling NMI exit */ 2857 ld r10, VCPU_KVM(r9) 2858 lbz r10, KVM_FWNMI(r10) 2859 cmpdi r10, 1 /* FWNMI capable? */ 2860 beq guest_exit_cont /* if so, exit with KVM_EXIT_NMI. */ 2861 2862 /* if not, fall through for backward compatibility. */ 2863 andi. r10, r11, MSR_RI /* check for unrecoverable exception */ 2864 beq 1f /* Deliver a machine check to guest */ 2865 ld r10, VCPU_PC(r9) 2866 cmpdi r3, 0 /* Did we handle MCE ? */ 2867 bne 2f /* Continue guest execution. */ 2868 /* If not, deliver a machine check. SRR0/1 are already set */ 28691: li r10, BOOK3S_INTERRUPT_MACHINE_CHECK 2870 bl kvmppc_msr_interrupt 28712: b fast_interrupt_c_return 2872 2873/* 2874 * Call C code to handle a HMI in real mode. 2875 * Only the primary thread does the call, secondary threads are handled 2876 * by calling hmi_exception_realmode() after kvmppc_hv_entry returns. 2877 * r9 points to the vcpu on entry 2878 */ 2879hmi_realmode: 2880 lbz r0, HSTATE_PTID(r13) 2881 cmpwi r0, 0 2882 bne guest_exit_cont 2883 bl kvmppc_realmode_hmi_handler 2884 ld r9, HSTATE_KVM_VCPU(r13) 2885 li r12, BOOK3S_INTERRUPT_HMI 2886 b guest_exit_cont 2887 2888/* 2889 * Check the reason we woke from nap, and take appropriate action. 2890 * Returns (in r3): 2891 * 0 if nothing needs to be done 2892 * 1 if something happened that needs to be handled by the host 2893 * -1 if there was a guest wakeup (IPI or msgsnd) 2894 * -2 if we handled a PCI passthrough interrupt (returned by 2895 * kvmppc_read_intr only) 2896 * 2897 * Also sets r12 to the interrupt vector for any interrupt that needs 2898 * to be handled now by the host (0x500 for external interrupt), or zero. 2899 * Modifies all volatile registers (since it may call a C function). 2900 * This routine calls kvmppc_read_intr, a C function, if an external 2901 * interrupt is pending. 2902 */ 2903kvmppc_check_wake_reason: 2904 mfspr r6, SPRN_SRR1 2905BEGIN_FTR_SECTION 2906 rlwinm r6, r6, 45-31, 0xf /* extract wake reason field (P8) */ 2907FTR_SECTION_ELSE 2908 rlwinm r6, r6, 45-31, 0xe /* P7 wake reason field is 3 bits */ 2909ALT_FTR_SECTION_END_IFSET(CPU_FTR_ARCH_207S) 2910 cmpwi r6, 8 /* was it an external interrupt? */ 2911 beq 7f /* if so, see what it was */ 2912 li r3, 0 2913 li r12, 0 2914 cmpwi r6, 6 /* was it the decrementer? */ 2915 beq 0f 2916BEGIN_FTR_SECTION 2917 cmpwi r6, 5 /* privileged doorbell? */ 2918 beq 0f 2919 cmpwi r6, 3 /* hypervisor doorbell? */ 2920 beq 3f 2921END_FTR_SECTION_IFSET(CPU_FTR_ARCH_207S) 2922 cmpwi r6, 0xa /* Hypervisor maintenance ? */ 2923 beq 4f 2924 li r3, 1 /* anything else, return 1 */ 29250: blr 2926 2927 /* hypervisor doorbell */ 29283: li r12, BOOK3S_INTERRUPT_H_DOORBELL 2929 2930 /* 2931 * Clear the doorbell as we will invoke the handler 2932 * explicitly in the guest exit path. 2933 */ 2934 lis r6, (PPC_DBELL_SERVER << (63-36))@h 2935 PPC_MSGCLR(6) 2936 /* see if it's a host IPI */ 2937 li r3, 1 2938BEGIN_FTR_SECTION 2939 PPC_MSGSYNC 2940 lwsync 2941END_FTR_SECTION_IFSET(CPU_FTR_ARCH_300) 2942 lbz r0, HSTATE_HOST_IPI(r13) 2943 cmpwi r0, 0 2944 bnelr 2945 /* if not, return -1 */ 2946 li r3, -1 2947 blr 2948 2949 /* Woken up due to Hypervisor maintenance interrupt */ 29504: li r12, BOOK3S_INTERRUPT_HMI 2951 li r3, 1 2952 blr 2953 2954 /* external interrupt - create a stack frame so we can call C */ 29557: mflr r0 2956 std r0, PPC_LR_STKOFF(r1) 2957 stdu r1, -PPC_MIN_STKFRM(r1) 2958 bl kvmppc_read_intr 2959 nop 2960 li r12, BOOK3S_INTERRUPT_EXTERNAL 2961 cmpdi r3, 1 2962 ble 1f 2963 2964 /* 2965 * Return code of 2 means PCI passthrough interrupt, but 2966 * we need to return back to host to complete handling the 2967 * interrupt. Trap reason is expected in r12 by guest 2968 * exit code. 2969 */ 2970 li r12, BOOK3S_INTERRUPT_HV_RM_HARD 29711: 2972 ld r0, PPC_MIN_STKFRM+PPC_LR_STKOFF(r1) 2973 addi r1, r1, PPC_MIN_STKFRM 2974 mtlr r0 2975 blr 2976 2977/* 2978 * Save away FP, VMX and VSX registers. 2979 * r3 = vcpu pointer 2980 * N.B. r30 and r31 are volatile across this function, 2981 * thus it is not callable from C. 2982 */ 2983kvmppc_save_fp: 2984 mflr r30 2985 mr r31,r3 2986 mfmsr r5 2987 ori r8,r5,MSR_FP 2988#ifdef CONFIG_ALTIVEC 2989BEGIN_FTR_SECTION 2990 oris r8,r8,MSR_VEC@h 2991END_FTR_SECTION_IFSET(CPU_FTR_ALTIVEC) 2992#endif 2993#ifdef CONFIG_VSX 2994BEGIN_FTR_SECTION 2995 oris r8,r8,MSR_VSX@h 2996END_FTR_SECTION_IFSET(CPU_FTR_VSX) 2997#endif 2998 mtmsrd r8 2999 addi r3,r3,VCPU_FPRS 3000 bl store_fp_state 3001#ifdef CONFIG_ALTIVEC 3002BEGIN_FTR_SECTION 3003 addi r3,r31,VCPU_VRS 3004 bl store_vr_state 3005END_FTR_SECTION_IFSET(CPU_FTR_ALTIVEC) 3006#endif 3007 mfspr r6,SPRN_VRSAVE 3008 stw r6,VCPU_VRSAVE(r31) 3009 mtlr r30 3010 blr 3011 3012/* 3013 * Load up FP, VMX and VSX registers 3014 * r4 = vcpu pointer 3015 * N.B. r30 and r31 are volatile across this function, 3016 * thus it is not callable from C. 3017 */ 3018kvmppc_load_fp: 3019 mflr r30 3020 mr r31,r4 3021 mfmsr r9 3022 ori r8,r9,MSR_FP 3023#ifdef CONFIG_ALTIVEC 3024BEGIN_FTR_SECTION 3025 oris r8,r8,MSR_VEC@h 3026END_FTR_SECTION_IFSET(CPU_FTR_ALTIVEC) 3027#endif 3028#ifdef CONFIG_VSX 3029BEGIN_FTR_SECTION 3030 oris r8,r8,MSR_VSX@h 3031END_FTR_SECTION_IFSET(CPU_FTR_VSX) 3032#endif 3033 mtmsrd r8 3034 addi r3,r4,VCPU_FPRS 3035 bl load_fp_state 3036#ifdef CONFIG_ALTIVEC 3037BEGIN_FTR_SECTION 3038 addi r3,r31,VCPU_VRS 3039 bl load_vr_state 3040END_FTR_SECTION_IFSET(CPU_FTR_ALTIVEC) 3041#endif 3042 lwz r7,VCPU_VRSAVE(r31) 3043 mtspr SPRN_VRSAVE,r7 3044 mtlr r30 3045 mr r4,r31 3046 blr 3047 3048#ifdef CONFIG_PPC_TRANSACTIONAL_MEM 3049/* 3050 * Save transactional state and TM-related registers. 3051 * Called with r3 pointing to the vcpu struct and r4 containing 3052 * the guest MSR value. 3053 * r5 is non-zero iff non-volatile register state needs to be maintained. 3054 * If r5 == 0, this can modify all checkpointed registers, but 3055 * restores r1 and r2 before exit. 3056 */ 3057_GLOBAL_TOC(kvmppc_save_tm_hv) 3058EXPORT_SYMBOL_GPL(kvmppc_save_tm_hv) 3059 /* See if we need to handle fake suspend mode */ 3060BEGIN_FTR_SECTION 3061 b __kvmppc_save_tm 3062END_FTR_SECTION_IFCLR(CPU_FTR_P9_TM_HV_ASSIST) 3063 3064 lbz r0, HSTATE_FAKE_SUSPEND(r13) /* Were we fake suspended? */ 3065 cmpwi r0, 0 3066 beq __kvmppc_save_tm 3067 3068 /* The following code handles the fake_suspend = 1 case */ 3069 mflr r0 3070 std r0, PPC_LR_STKOFF(r1) 3071 stdu r1, -PPC_MIN_STKFRM(r1) 3072 3073 /* Turn on TM. */ 3074 mfmsr r8 3075 li r0, 1 3076 rldimi r8, r0, MSR_TM_LG, 63-MSR_TM_LG 3077 mtmsrd r8 3078 3079 rldicl. r8, r8, 64 - MSR_TS_S_LG, 62 /* Did we actually hrfid? */ 3080 beq 4f 3081BEGIN_FTR_SECTION 3082 bl pnv_power9_force_smt4_catch 3083END_FTR_SECTION_IFSET(CPU_FTR_P9_TM_XER_SO_BUG) 3084 nop 3085 3086 /* We have to treclaim here because that's the only way to do S->N */ 3087 li r3, TM_CAUSE_KVM_RESCHED 3088 TRECLAIM(R3) 3089 3090 /* 3091 * We were in fake suspend, so we are not going to save the 3092 * register state as the guest checkpointed state (since 3093 * we already have it), therefore we can now use any volatile GPR. 3094 * In fact treclaim in fake suspend state doesn't modify 3095 * any registers. 3096 */ 3097 3098BEGIN_FTR_SECTION 3099 bl pnv_power9_force_smt4_release 3100END_FTR_SECTION_IFSET(CPU_FTR_P9_TM_XER_SO_BUG) 3101 nop 3102 31034: 3104 mfspr r3, SPRN_PSSCR 3105 /* PSSCR_FAKE_SUSPEND is a write-only bit, but clear it anyway */ 3106 li r0, PSSCR_FAKE_SUSPEND 3107 andc r3, r3, r0 3108 mtspr SPRN_PSSCR, r3 3109 3110 /* Don't save TEXASR, use value from last exit in real suspend state */ 3111 ld r9, HSTATE_KVM_VCPU(r13) 3112 mfspr r5, SPRN_TFHAR 3113 mfspr r6, SPRN_TFIAR 3114 std r5, VCPU_TFHAR(r9) 3115 std r6, VCPU_TFIAR(r9) 3116 3117 addi r1, r1, PPC_MIN_STKFRM 3118 ld r0, PPC_LR_STKOFF(r1) 3119 mtlr r0 3120 blr 3121 3122/* 3123 * Restore transactional state and TM-related registers. 3124 * Called with r3 pointing to the vcpu struct 3125 * and r4 containing the guest MSR value. 3126 * r5 is non-zero iff non-volatile register state needs to be maintained. 3127 * This potentially modifies all checkpointed registers. 3128 * It restores r1 and r2 from the PACA. 3129 */ 3130_GLOBAL_TOC(kvmppc_restore_tm_hv) 3131EXPORT_SYMBOL_GPL(kvmppc_restore_tm_hv) 3132 /* 3133 * If we are doing TM emulation for the guest on a POWER9 DD2, 3134 * then we don't actually do a trechkpt -- we either set up 3135 * fake-suspend mode, or emulate a TM rollback. 3136 */ 3137BEGIN_FTR_SECTION 3138 b __kvmppc_restore_tm 3139END_FTR_SECTION_IFCLR(CPU_FTR_P9_TM_HV_ASSIST) 3140 mflr r0 3141 std r0, PPC_LR_STKOFF(r1) 3142 3143 li r0, 0 3144 stb r0, HSTATE_FAKE_SUSPEND(r13) 3145 3146 /* Turn on TM so we can restore TM SPRs */ 3147 mfmsr r5 3148 li r0, 1 3149 rldimi r5, r0, MSR_TM_LG, 63-MSR_TM_LG 3150 mtmsrd r5 3151 3152 /* 3153 * The user may change these outside of a transaction, so they must 3154 * always be context switched. 3155 */ 3156 ld r5, VCPU_TFHAR(r3) 3157 ld r6, VCPU_TFIAR(r3) 3158 ld r7, VCPU_TEXASR(r3) 3159 mtspr SPRN_TFHAR, r5 3160 mtspr SPRN_TFIAR, r6 3161 mtspr SPRN_TEXASR, r7 3162 3163 rldicl. r5, r4, 64 - MSR_TS_S_LG, 62 3164 beqlr /* TM not active in guest */ 3165 3166 /* Make sure the failure summary is set */ 3167 oris r7, r7, (TEXASR_FS)@h 3168 mtspr SPRN_TEXASR, r7 3169 3170 cmpwi r5, 1 /* check for suspended state */ 3171 bgt 10f 3172 stb r5, HSTATE_FAKE_SUSPEND(r13) 3173 b 9f /* and return */ 317410: stdu r1, -PPC_MIN_STKFRM(r1) 3175 /* guest is in transactional state, so simulate rollback */ 3176 bl kvmhv_emulate_tm_rollback 3177 nop 3178 addi r1, r1, PPC_MIN_STKFRM 31799: ld r0, PPC_LR_STKOFF(r1) 3180 mtlr r0 3181 blr 3182#endif /* CONFIG_PPC_TRANSACTIONAL_MEM */ 3183 3184/* 3185 * We come here if we get any exception or interrupt while we are 3186 * executing host real mode code while in guest MMU context. 3187 * r12 is (CR << 32) | vector 3188 * r13 points to our PACA 3189 * r12 is saved in HSTATE_SCRATCH0(r13) 3190 * ctr is saved in HSTATE_SCRATCH1(r13) if RELOCATABLE 3191 * r9 is saved in HSTATE_SCRATCH2(r13) 3192 * r13 is saved in HSPRG1 3193 * cfar is saved in HSTATE_CFAR(r13) 3194 * ppr is saved in HSTATE_PPR(r13) 3195 */ 3196kvmppc_bad_host_intr: 3197 /* 3198 * Switch to the emergency stack, but start half-way down in 3199 * case we were already on it. 3200 */ 3201 mr r9, r1 3202 std r1, PACAR1(r13) 3203 ld r1, PACAEMERGSP(r13) 3204 subi r1, r1, THREAD_SIZE/2 + INT_FRAME_SIZE 3205 std r9, 0(r1) 3206 std r0, GPR0(r1) 3207 std r9, GPR1(r1) 3208 std r2, GPR2(r1) 3209 SAVE_4GPRS(3, r1) 3210 SAVE_2GPRS(7, r1) 3211 srdi r0, r12, 32 3212 clrldi r12, r12, 32 3213 std r0, _CCR(r1) 3214 std r12, _TRAP(r1) 3215 andi. r0, r12, 2 3216 beq 1f 3217 mfspr r3, SPRN_HSRR0 3218 mfspr r4, SPRN_HSRR1 3219 mfspr r5, SPRN_HDAR 3220 mfspr r6, SPRN_HDSISR 3221 b 2f 32221: mfspr r3, SPRN_SRR0 3223 mfspr r4, SPRN_SRR1 3224 mfspr r5, SPRN_DAR 3225 mfspr r6, SPRN_DSISR 32262: std r3, _NIP(r1) 3227 std r4, _MSR(r1) 3228 std r5, _DAR(r1) 3229 std r6, _DSISR(r1) 3230 ld r9, HSTATE_SCRATCH2(r13) 3231 ld r12, HSTATE_SCRATCH0(r13) 3232 GET_SCRATCH0(r0) 3233 SAVE_4GPRS(9, r1) 3234 std r0, GPR13(r1) 3235 SAVE_NVGPRS(r1) 3236 ld r5, HSTATE_CFAR(r13) 3237 std r5, ORIG_GPR3(r1) 3238 mflr r3 3239#ifdef CONFIG_RELOCATABLE 3240 ld r4, HSTATE_SCRATCH1(r13) 3241#else 3242 mfctr r4 3243#endif 3244 mfxer r5 3245 lbz r6, PACAIRQSOFTMASK(r13) 3246 std r3, _LINK(r1) 3247 std r4, _CTR(r1) 3248 std r5, _XER(r1) 3249 std r6, SOFTE(r1) 3250 ld r2, PACATOC(r13) 3251 LOAD_REG_IMMEDIATE(3, 0x7265677368657265) 3252 std r3, STACK_FRAME_OVERHEAD-16(r1) 3253 3254 /* 3255 * On POWER9 do a minimal restore of the MMU and call C code, 3256 * which will print a message and panic. 3257 * XXX On POWER7 and POWER8, we just spin here since we don't 3258 * know what the other threads are doing (and we don't want to 3259 * coordinate with them) - but at least we now have register state 3260 * in memory that we might be able to look at from another CPU. 3261 */ 3262BEGIN_FTR_SECTION 3263 b . 3264END_FTR_SECTION_IFCLR(CPU_FTR_ARCH_300) 3265 ld r9, HSTATE_KVM_VCPU(r13) 3266 ld r10, VCPU_KVM(r9) 3267 3268 li r0, 0 3269 mtspr SPRN_AMR, r0 3270 mtspr SPRN_IAMR, r0 3271 mtspr SPRN_CIABR, r0 3272 mtspr SPRN_DAWRX, r0 3273 3274BEGIN_MMU_FTR_SECTION 3275 b 4f 3276END_MMU_FTR_SECTION_IFSET(MMU_FTR_TYPE_RADIX) 3277 3278 slbmte r0, r0 3279 slbia 3280 ptesync 3281 ld r8, PACA_SLBSHADOWPTR(r13) 3282 .rept SLB_NUM_BOLTED 3283 li r3, SLBSHADOW_SAVEAREA 3284 LDX_BE r5, r8, r3 3285 addi r3, r3, 8 3286 LDX_BE r6, r8, r3 3287 andis. r7, r5, SLB_ESID_V@h 3288 beq 3f 3289 slbmte r6, r5 32903: addi r8, r8, 16 3291 .endr 3292 32934: lwz r7, KVM_HOST_LPID(r10) 3294 mtspr SPRN_LPID, r7 3295 mtspr SPRN_PID, r0 3296 ld r8, KVM_HOST_LPCR(r10) 3297 mtspr SPRN_LPCR, r8 3298 isync 3299 li r0, KVM_GUEST_MODE_NONE 3300 stb r0, HSTATE_IN_GUEST(r13) 3301 3302 /* 3303 * Turn on the MMU and jump to C code 3304 */ 3305 bcl 20, 31, .+4 33065: mflr r3 3307 addi r3, r3, 9f - 5b 3308 li r4, -1 3309 rldimi r3, r4, 62, 0 /* ensure 0xc000000000000000 bits are set */ 3310 ld r4, PACAKMSR(r13) 3311 mtspr SPRN_SRR0, r3 3312 mtspr SPRN_SRR1, r4 3313 RFI_TO_KERNEL 33149: addi r3, r1, STACK_FRAME_OVERHEAD 3315 bl kvmppc_bad_interrupt 3316 b 9b 3317 3318/* 3319 * This mimics the MSR transition on IRQ delivery. The new guest MSR is taken 3320 * from VCPU_INTR_MSR and is modified based on the required TM state changes. 3321 * r11 has the guest MSR value (in/out) 3322 * r9 has a vcpu pointer (in) 3323 * r0 is used as a scratch register 3324 */ 3325kvmppc_msr_interrupt: 3326 rldicl r0, r11, 64 - MSR_TS_S_LG, 62 3327 cmpwi r0, 2 /* Check if we are in transactional state.. */ 3328 ld r11, VCPU_INTR_MSR(r9) 3329 bne 1f 3330 /* ... if transactional, change to suspended */ 3331 li r0, 1 33321: rldimi r11, r0, MSR_TS_S_LG, 63 - MSR_TS_T_LG 3333 blr 3334 3335/* 3336 * Load up guest PMU state. R3 points to the vcpu struct. 3337 */ 3338_GLOBAL(kvmhv_load_guest_pmu) 3339EXPORT_SYMBOL_GPL(kvmhv_load_guest_pmu) 3340 mr r4, r3 3341 mflr r0 3342 li r3, 1 3343 sldi r3, r3, 31 /* MMCR0_FC (freeze counters) bit */ 3344 mtspr SPRN_MMCR0, r3 /* freeze all counters, disable ints */ 3345 isync 3346BEGIN_FTR_SECTION 3347 ld r3, VCPU_MMCR(r4) 3348 andi. r5, r3, MMCR0_PMAO_SYNC | MMCR0_PMAO 3349 cmpwi r5, MMCR0_PMAO 3350 beql kvmppc_fix_pmao 3351END_FTR_SECTION_IFSET(CPU_FTR_PMAO_BUG) 3352 lwz r3, VCPU_PMC(r4) /* always load up guest PMU registers */ 3353 lwz r5, VCPU_PMC + 4(r4) /* to prevent information leak */ 3354 lwz r6, VCPU_PMC + 8(r4) 3355 lwz r7, VCPU_PMC + 12(r4) 3356 lwz r8, VCPU_PMC + 16(r4) 3357 lwz r9, VCPU_PMC + 20(r4) 3358 mtspr SPRN_PMC1, r3 3359 mtspr SPRN_PMC2, r5 3360 mtspr SPRN_PMC3, r6 3361 mtspr SPRN_PMC4, r7 3362 mtspr SPRN_PMC5, r8 3363 mtspr SPRN_PMC6, r9 3364 ld r3, VCPU_MMCR(r4) 3365 ld r5, VCPU_MMCR + 8(r4) 3366 ld r6, VCPU_MMCR + 16(r4) 3367 ld r7, VCPU_SIAR(r4) 3368 ld r8, VCPU_SDAR(r4) 3369 mtspr SPRN_MMCR1, r5 3370 mtspr SPRN_MMCRA, r6 3371 mtspr SPRN_SIAR, r7 3372 mtspr SPRN_SDAR, r8 3373BEGIN_FTR_SECTION 3374 ld r5, VCPU_MMCR + 24(r4) 3375 ld r6, VCPU_SIER(r4) 3376 mtspr SPRN_MMCR2, r5 3377 mtspr SPRN_SIER, r6 3378BEGIN_FTR_SECTION_NESTED(96) 3379 lwz r7, VCPU_PMC + 24(r4) 3380 lwz r8, VCPU_PMC + 28(r4) 3381 ld r9, VCPU_MMCR + 32(r4) 3382 mtspr SPRN_SPMC1, r7 3383 mtspr SPRN_SPMC2, r8 3384 mtspr SPRN_MMCRS, r9 3385END_FTR_SECTION_NESTED(CPU_FTR_ARCH_300, 0, 96) 3386END_FTR_SECTION_IFSET(CPU_FTR_ARCH_207S) 3387 mtspr SPRN_MMCR0, r3 3388 isync 3389 mtlr r0 3390 blr 3391 3392/* 3393 * Reload host PMU state saved in the PACA by kvmhv_save_host_pmu. 3394 */ 3395_GLOBAL(kvmhv_load_host_pmu) 3396EXPORT_SYMBOL_GPL(kvmhv_load_host_pmu) 3397 mflr r0 3398 lbz r4, PACA_PMCINUSE(r13) /* is the host using the PMU? */ 3399 cmpwi r4, 0 3400 beq 23f /* skip if not */ 3401BEGIN_FTR_SECTION 3402 ld r3, HSTATE_MMCR0(r13) 3403 andi. r4, r3, MMCR0_PMAO_SYNC | MMCR0_PMAO 3404 cmpwi r4, MMCR0_PMAO 3405 beql kvmppc_fix_pmao 3406END_FTR_SECTION_IFSET(CPU_FTR_PMAO_BUG) 3407 lwz r3, HSTATE_PMC1(r13) 3408 lwz r4, HSTATE_PMC2(r13) 3409 lwz r5, HSTATE_PMC3(r13) 3410 lwz r6, HSTATE_PMC4(r13) 3411 lwz r8, HSTATE_PMC5(r13) 3412 lwz r9, HSTATE_PMC6(r13) 3413 mtspr SPRN_PMC1, r3 3414 mtspr SPRN_PMC2, r4 3415 mtspr SPRN_PMC3, r5 3416 mtspr SPRN_PMC4, r6 3417 mtspr SPRN_PMC5, r8 3418 mtspr SPRN_PMC6, r9 3419 ld r3, HSTATE_MMCR0(r13) 3420 ld r4, HSTATE_MMCR1(r13) 3421 ld r5, HSTATE_MMCRA(r13) 3422 ld r6, HSTATE_SIAR(r13) 3423 ld r7, HSTATE_SDAR(r13) 3424 mtspr SPRN_MMCR1, r4 3425 mtspr SPRN_MMCRA, r5 3426 mtspr SPRN_SIAR, r6 3427 mtspr SPRN_SDAR, r7 3428BEGIN_FTR_SECTION 3429 ld r8, HSTATE_MMCR2(r13) 3430 ld r9, HSTATE_SIER(r13) 3431 mtspr SPRN_MMCR2, r8 3432 mtspr SPRN_SIER, r9 3433END_FTR_SECTION_IFSET(CPU_FTR_ARCH_207S) 3434 mtspr SPRN_MMCR0, r3 3435 isync 3436 mtlr r0 343723: blr 3438 3439/* 3440 * Save guest PMU state into the vcpu struct. 3441 * r3 = vcpu, r4 = full save flag (PMU in use flag set in VPA) 3442 */ 3443_GLOBAL(kvmhv_save_guest_pmu) 3444EXPORT_SYMBOL_GPL(kvmhv_save_guest_pmu) 3445 mr r9, r3 3446 mr r8, r4 3447BEGIN_FTR_SECTION 3448 /* 3449 * POWER8 seems to have a hardware bug where setting 3450 * MMCR0[PMAE] along with MMCR0[PMC1CE] and/or MMCR0[PMCjCE] 3451 * when some counters are already negative doesn't seem 3452 * to cause a performance monitor alert (and hence interrupt). 3453 * The effect of this is that when saving the PMU state, 3454 * if there is no PMU alert pending when we read MMCR0 3455 * before freezing the counters, but one becomes pending 3456 * before we read the counters, we lose it. 3457 * To work around this, we need a way to freeze the counters 3458 * before reading MMCR0. Normally, freezing the counters 3459 * is done by writing MMCR0 (to set MMCR0[FC]) which 3460 * unavoidably writes MMCR0[PMA0] as well. On POWER8, 3461 * we can also freeze the counters using MMCR2, by writing 3462 * 1s to all the counter freeze condition bits (there are 3463 * 9 bits each for 6 counters). 3464 */ 3465 li r3, -1 /* set all freeze bits */ 3466 clrrdi r3, r3, 10 3467 mfspr r10, SPRN_MMCR2 3468 mtspr SPRN_MMCR2, r3 3469 isync 3470END_FTR_SECTION_IFSET(CPU_FTR_ARCH_207S) 3471 li r3, 1 3472 sldi r3, r3, 31 /* MMCR0_FC (freeze counters) bit */ 3473 mfspr r4, SPRN_MMCR0 /* save MMCR0 */ 3474 mtspr SPRN_MMCR0, r3 /* freeze all counters, disable ints */ 3475 mfspr r6, SPRN_MMCRA 3476 /* Clear MMCRA in order to disable SDAR updates */ 3477 li r7, 0 3478 mtspr SPRN_MMCRA, r7 3479 isync 3480 cmpwi r8, 0 /* did they ask for PMU stuff to be saved? */ 3481 bne 21f 3482 std r3, VCPU_MMCR(r9) /* if not, set saved MMCR0 to FC */ 3483 b 22f 348421: mfspr r5, SPRN_MMCR1 3485 mfspr r7, SPRN_SIAR 3486 mfspr r8, SPRN_SDAR 3487 std r4, VCPU_MMCR(r9) 3488 std r5, VCPU_MMCR + 8(r9) 3489 std r6, VCPU_MMCR + 16(r9) 3490BEGIN_FTR_SECTION 3491 std r10, VCPU_MMCR + 24(r9) 3492END_FTR_SECTION_IFSET(CPU_FTR_ARCH_207S) 3493 std r7, VCPU_SIAR(r9) 3494 std r8, VCPU_SDAR(r9) 3495 mfspr r3, SPRN_PMC1 3496 mfspr r4, SPRN_PMC2 3497 mfspr r5, SPRN_PMC3 3498 mfspr r6, SPRN_PMC4 3499 mfspr r7, SPRN_PMC5 3500 mfspr r8, SPRN_PMC6 3501 stw r3, VCPU_PMC(r9) 3502 stw r4, VCPU_PMC + 4(r9) 3503 stw r5, VCPU_PMC + 8(r9) 3504 stw r6, VCPU_PMC + 12(r9) 3505 stw r7, VCPU_PMC + 16(r9) 3506 stw r8, VCPU_PMC + 20(r9) 3507BEGIN_FTR_SECTION 3508 mfspr r5, SPRN_SIER 3509 std r5, VCPU_SIER(r9) 3510BEGIN_FTR_SECTION_NESTED(96) 3511 mfspr r6, SPRN_SPMC1 3512 mfspr r7, SPRN_SPMC2 3513 mfspr r8, SPRN_MMCRS 3514 stw r6, VCPU_PMC + 24(r9) 3515 stw r7, VCPU_PMC + 28(r9) 3516 std r8, VCPU_MMCR + 32(r9) 3517 lis r4, 0x8000 3518 mtspr SPRN_MMCRS, r4 3519END_FTR_SECTION_NESTED(CPU_FTR_ARCH_300, 0, 96) 3520END_FTR_SECTION_IFSET(CPU_FTR_ARCH_207S) 352122: blr 3522 3523/* 3524 * This works around a hardware bug on POWER8E processors, where 3525 * writing a 1 to the MMCR0[PMAO] bit doesn't generate a 3526 * performance monitor interrupt. Instead, when we need to have 3527 * an interrupt pending, we have to arrange for a counter to overflow. 3528 */ 3529kvmppc_fix_pmao: 3530 li r3, 0 3531 mtspr SPRN_MMCR2, r3 3532 lis r3, (MMCR0_PMXE | MMCR0_FCECE)@h 3533 ori r3, r3, MMCR0_PMCjCE | MMCR0_C56RUN 3534 mtspr SPRN_MMCR0, r3 3535 lis r3, 0x7fff 3536 ori r3, r3, 0xffff 3537 mtspr SPRN_PMC6, r3 3538 isync 3539 blr 3540 3541#ifdef CONFIG_KVM_BOOK3S_HV_EXIT_TIMING 3542/* 3543 * Start timing an activity 3544 * r3 = pointer to time accumulation struct, r4 = vcpu 3545 */ 3546kvmhv_start_timing: 3547 ld r5, HSTATE_KVM_VCORE(r13) 3548 ld r6, VCORE_TB_OFFSET_APPL(r5) 3549 mftb r5 3550 subf r5, r6, r5 /* subtract current timebase offset */ 3551 std r3, VCPU_CUR_ACTIVITY(r4) 3552 std r5, VCPU_ACTIVITY_START(r4) 3553 blr 3554 3555/* 3556 * Accumulate time to one activity and start another. 3557 * r3 = pointer to new time accumulation struct, r4 = vcpu 3558 */ 3559kvmhv_accumulate_time: 3560 ld r5, HSTATE_KVM_VCORE(r13) 3561 ld r8, VCORE_TB_OFFSET_APPL(r5) 3562 ld r5, VCPU_CUR_ACTIVITY(r4) 3563 ld r6, VCPU_ACTIVITY_START(r4) 3564 std r3, VCPU_CUR_ACTIVITY(r4) 3565 mftb r7 3566 subf r7, r8, r7 /* subtract current timebase offset */ 3567 std r7, VCPU_ACTIVITY_START(r4) 3568 cmpdi r5, 0 3569 beqlr 3570 subf r3, r6, r7 3571 ld r8, TAS_SEQCOUNT(r5) 3572 cmpdi r8, 0 3573 addi r8, r8, 1 3574 std r8, TAS_SEQCOUNT(r5) 3575 lwsync 3576 ld r7, TAS_TOTAL(r5) 3577 add r7, r7, r3 3578 std r7, TAS_TOTAL(r5) 3579 ld r6, TAS_MIN(r5) 3580 ld r7, TAS_MAX(r5) 3581 beq 3f 3582 cmpd r3, r6 3583 bge 1f 35843: std r3, TAS_MIN(r5) 35851: cmpd r3, r7 3586 ble 2f 3587 std r3, TAS_MAX(r5) 35882: lwsync 3589 addi r8, r8, 1 3590 std r8, TAS_SEQCOUNT(r5) 3591 blr 3592#endif 3593