1/* 2 * This program is free software; you can redistribute it and/or modify 3 * it under the terms of the GNU General Public License, version 2, as 4 * published by the Free Software Foundation. 5 * 6 * This program is distributed in the hope that it will be useful, 7 * but WITHOUT ANY WARRANTY; without even the implied warranty of 8 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 9 * GNU General Public License for more details. 10 * 11 * Copyright 2011 Paul Mackerras, IBM Corp. <paulus@au1.ibm.com> 12 * 13 * Derived from book3s_rmhandlers.S and other files, which are: 14 * 15 * Copyright SUSE Linux Products GmbH 2009 16 * 17 * Authors: Alexander Graf <agraf@suse.de> 18 */ 19 20#include <asm/ppc_asm.h> 21#include <asm/kvm_asm.h> 22#include <asm/reg.h> 23#include <asm/mmu.h> 24#include <asm/page.h> 25#include <asm/ptrace.h> 26#include <asm/hvcall.h> 27#include <asm/asm-offsets.h> 28#include <asm/exception-64s.h> 29#include <asm/kvm_book3s_asm.h> 30#include <asm/book3s/64/mmu-hash.h> 31#include <asm/tm.h> 32#include <asm/opal.h> 33#include <asm/xive-regs.h> 34#include <asm/thread_info.h> 35 36/* Sign-extend HDEC if not on POWER9 */ 37#define EXTEND_HDEC(reg) \ 38BEGIN_FTR_SECTION; \ 39 extsw reg, reg; \ 40END_FTR_SECTION_IFCLR(CPU_FTR_ARCH_300) 41 42#define VCPU_GPRS_TM(reg) (((reg) * ULONG_SIZE) + VCPU_GPR_TM) 43 44/* Values in HSTATE_NAPPING(r13) */ 45#define NAPPING_CEDE 1 46#define NAPPING_NOVCPU 2 47 48/* Stack frame offsets for kvmppc_hv_entry */ 49#define SFS 160 50#define STACK_SLOT_TRAP (SFS-4) 51#define STACK_SLOT_TID (SFS-16) 52#define STACK_SLOT_PSSCR (SFS-24) 53#define STACK_SLOT_PID (SFS-32) 54#define STACK_SLOT_IAMR (SFS-40) 55#define STACK_SLOT_CIABR (SFS-48) 56#define STACK_SLOT_DAWR (SFS-56) 57#define STACK_SLOT_DAWRX (SFS-64) 58#define STACK_SLOT_HFSCR (SFS-72) 59 60/* 61 * Call kvmppc_hv_entry in real mode. 62 * Must be called with interrupts hard-disabled. 63 * 64 * Input Registers: 65 * 66 * LR = return address to continue at after eventually re-enabling MMU 67 */ 68_GLOBAL_TOC(kvmppc_hv_entry_trampoline) 69 mflr r0 70 std r0, PPC_LR_STKOFF(r1) 71 stdu r1, -112(r1) 72 mfmsr r10 73 std r10, HSTATE_HOST_MSR(r13) 74 LOAD_REG_ADDR(r5, kvmppc_call_hv_entry) 75 li r0,MSR_RI 76 andc r0,r10,r0 77 li r6,MSR_IR | MSR_DR 78 andc r6,r10,r6 79 mtmsrd r0,1 /* clear RI in MSR */ 80 mtsrr0 r5 81 mtsrr1 r6 82 RFI_TO_KERNEL 83 84kvmppc_call_hv_entry: 85BEGIN_FTR_SECTION 86 /* On P9, do LPCR setting, if necessary */ 87 ld r3, HSTATE_SPLIT_MODE(r13) 88 cmpdi r3, 0 89 beq 46f 90 lwz r4, KVM_SPLIT_DO_SET(r3) 91 cmpwi r4, 0 92 beq 46f 93 bl kvmhv_p9_set_lpcr 94 nop 9546: 96END_FTR_SECTION_IFSET(CPU_FTR_ARCH_300) 97 98 ld r4, HSTATE_KVM_VCPU(r13) 99 bl kvmppc_hv_entry 100 101 /* Back from guest - restore host state and return to caller */ 102 103BEGIN_FTR_SECTION 104 /* Restore host DABR and DABRX */ 105 ld r5,HSTATE_DABR(r13) 106 li r6,7 107 mtspr SPRN_DABR,r5 108 mtspr SPRN_DABRX,r6 109END_FTR_SECTION_IFCLR(CPU_FTR_ARCH_207S) 110 111 /* Restore SPRG3 */ 112 ld r3,PACA_SPRG_VDSO(r13) 113 mtspr SPRN_SPRG_VDSO_WRITE,r3 114 115 /* Reload the host's PMU registers */ 116 ld r3, PACALPPACAPTR(r13) /* is the host using the PMU? */ 117 lbz r4, LPPACA_PMCINUSE(r3) 118 cmpwi r4, 0 119 beq 23f /* skip if not */ 120BEGIN_FTR_SECTION 121 ld r3, HSTATE_MMCR0(r13) 122 andi. r4, r3, MMCR0_PMAO_SYNC | MMCR0_PMAO 123 cmpwi r4, MMCR0_PMAO 124 beql kvmppc_fix_pmao 125END_FTR_SECTION_IFSET(CPU_FTR_PMAO_BUG) 126 lwz r3, HSTATE_PMC1(r13) 127 lwz r4, HSTATE_PMC2(r13) 128 lwz r5, HSTATE_PMC3(r13) 129 lwz r6, HSTATE_PMC4(r13) 130 lwz r8, HSTATE_PMC5(r13) 131 lwz r9, HSTATE_PMC6(r13) 132 mtspr SPRN_PMC1, r3 133 mtspr SPRN_PMC2, r4 134 mtspr SPRN_PMC3, r5 135 mtspr SPRN_PMC4, r6 136 mtspr SPRN_PMC5, r8 137 mtspr SPRN_PMC6, r9 138 ld r3, HSTATE_MMCR0(r13) 139 ld r4, HSTATE_MMCR1(r13) 140 ld r5, HSTATE_MMCRA(r13) 141 ld r6, HSTATE_SIAR(r13) 142 ld r7, HSTATE_SDAR(r13) 143 mtspr SPRN_MMCR1, r4 144 mtspr SPRN_MMCRA, r5 145 mtspr SPRN_SIAR, r6 146 mtspr SPRN_SDAR, r7 147BEGIN_FTR_SECTION 148 ld r8, HSTATE_MMCR2(r13) 149 ld r9, HSTATE_SIER(r13) 150 mtspr SPRN_MMCR2, r8 151 mtspr SPRN_SIER, r9 152END_FTR_SECTION_IFSET(CPU_FTR_ARCH_207S) 153 mtspr SPRN_MMCR0, r3 154 isync 15523: 156 157 /* 158 * Reload DEC. HDEC interrupts were disabled when 159 * we reloaded the host's LPCR value. 160 */ 161 ld r3, HSTATE_DECEXP(r13) 162 mftb r4 163 subf r4, r4, r3 164 mtspr SPRN_DEC, r4 165 166 /* hwthread_req may have got set by cede or no vcpu, so clear it */ 167 li r0, 0 168 stb r0, HSTATE_HWTHREAD_REQ(r13) 169 170 /* 171 * For external interrupts we need to call the Linux 172 * handler to process the interrupt. We do that by jumping 173 * to absolute address 0x500 for external interrupts. 174 * The [h]rfid at the end of the handler will return to 175 * the book3s_hv_interrupts.S code. For other interrupts 176 * we do the rfid to get back to the book3s_hv_interrupts.S 177 * code here. 178 */ 179 ld r8, 112+PPC_LR_STKOFF(r1) 180 addi r1, r1, 112 181 ld r7, HSTATE_HOST_MSR(r13) 182 183 /* Return the trap number on this thread as the return value */ 184 mr r3, r12 185 186 /* 187 * If we came back from the guest via a relocation-on interrupt, 188 * we will be in virtual mode at this point, which makes it a 189 * little easier to get back to the caller. 190 */ 191 mfmsr r0 192 andi. r0, r0, MSR_IR /* in real mode? */ 193 bne .Lvirt_return 194 195 /* RFI into the highmem handler */ 196 mfmsr r6 197 li r0, MSR_RI 198 andc r6, r6, r0 199 mtmsrd r6, 1 /* Clear RI in MSR */ 200 mtsrr0 r8 201 mtsrr1 r7 202 RFI_TO_KERNEL 203 204 /* Virtual-mode return */ 205.Lvirt_return: 206 mtlr r8 207 blr 208 209kvmppc_primary_no_guest: 210 /* We handle this much like a ceded vcpu */ 211 /* put the HDEC into the DEC, since HDEC interrupts don't wake us */ 212 /* HDEC may be larger than DEC for arch >= v3.00, but since the */ 213 /* HDEC value came from DEC in the first place, it will fit */ 214 mfspr r3, SPRN_HDEC 215 mtspr SPRN_DEC, r3 216 /* 217 * Make sure the primary has finished the MMU switch. 218 * We should never get here on a secondary thread, but 219 * check it for robustness' sake. 220 */ 221 ld r5, HSTATE_KVM_VCORE(r13) 22265: lbz r0, VCORE_IN_GUEST(r5) 223 cmpwi r0, 0 224 beq 65b 225 /* Set LPCR. */ 226 ld r8,VCORE_LPCR(r5) 227 mtspr SPRN_LPCR,r8 228 isync 229 /* set our bit in napping_threads */ 230 ld r5, HSTATE_KVM_VCORE(r13) 231 lbz r7, HSTATE_PTID(r13) 232 li r0, 1 233 sld r0, r0, r7 234 addi r6, r5, VCORE_NAPPING_THREADS 2351: lwarx r3, 0, r6 236 or r3, r3, r0 237 stwcx. r3, 0, r6 238 bne 1b 239 /* order napping_threads update vs testing entry_exit_map */ 240 isync 241 li r12, 0 242 lwz r7, VCORE_ENTRY_EXIT(r5) 243 cmpwi r7, 0x100 244 bge kvm_novcpu_exit /* another thread already exiting */ 245 li r3, NAPPING_NOVCPU 246 stb r3, HSTATE_NAPPING(r13) 247 248 li r3, 0 /* Don't wake on privileged (OS) doorbell */ 249 b kvm_do_nap 250 251/* 252 * kvm_novcpu_wakeup 253 * Entered from kvm_start_guest if kvm_hstate.napping is set 254 * to NAPPING_NOVCPU 255 * r2 = kernel TOC 256 * r13 = paca 257 */ 258kvm_novcpu_wakeup: 259 ld r1, HSTATE_HOST_R1(r13) 260 ld r5, HSTATE_KVM_VCORE(r13) 261 li r0, 0 262 stb r0, HSTATE_NAPPING(r13) 263 264 /* check the wake reason */ 265 bl kvmppc_check_wake_reason 266 267 /* 268 * Restore volatile registers since we could have called 269 * a C routine in kvmppc_check_wake_reason. 270 * r5 = VCORE 271 */ 272 ld r5, HSTATE_KVM_VCORE(r13) 273 274 /* see if any other thread is already exiting */ 275 lwz r0, VCORE_ENTRY_EXIT(r5) 276 cmpwi r0, 0x100 277 bge kvm_novcpu_exit 278 279 /* clear our bit in napping_threads */ 280 lbz r7, HSTATE_PTID(r13) 281 li r0, 1 282 sld r0, r0, r7 283 addi r6, r5, VCORE_NAPPING_THREADS 2844: lwarx r7, 0, r6 285 andc r7, r7, r0 286 stwcx. r7, 0, r6 287 bne 4b 288 289 /* See if the wake reason means we need to exit */ 290 cmpdi r3, 0 291 bge kvm_novcpu_exit 292 293 /* See if our timeslice has expired (HDEC is negative) */ 294 mfspr r0, SPRN_HDEC 295 EXTEND_HDEC(r0) 296 li r12, BOOK3S_INTERRUPT_HV_DECREMENTER 297 cmpdi r0, 0 298 blt kvm_novcpu_exit 299 300 /* Got an IPI but other vcpus aren't yet exiting, must be a latecomer */ 301 ld r4, HSTATE_KVM_VCPU(r13) 302 cmpdi r4, 0 303 beq kvmppc_primary_no_guest 304 305#ifdef CONFIG_KVM_BOOK3S_HV_EXIT_TIMING 306 addi r3, r4, VCPU_TB_RMENTRY 307 bl kvmhv_start_timing 308#endif 309 b kvmppc_got_guest 310 311kvm_novcpu_exit: 312#ifdef CONFIG_KVM_BOOK3S_HV_EXIT_TIMING 313 ld r4, HSTATE_KVM_VCPU(r13) 314 cmpdi r4, 0 315 beq 13f 316 addi r3, r4, VCPU_TB_RMEXIT 317 bl kvmhv_accumulate_time 318#endif 31913: mr r3, r12 320 stw r12, STACK_SLOT_TRAP(r1) 321 bl kvmhv_commence_exit 322 nop 323 b kvmhv_switch_to_host 324 325/* 326 * We come in here when wakened from nap mode. 327 * Relocation is off and most register values are lost. 328 * r13 points to the PACA. 329 * r3 contains the SRR1 wakeup value, SRR1 is trashed. 330 */ 331 .globl kvm_start_guest 332kvm_start_guest: 333 /* Set runlatch bit the minute you wake up from nap */ 334 mfspr r0, SPRN_CTRLF 335 ori r0, r0, 1 336 mtspr SPRN_CTRLT, r0 337 338 /* 339 * Could avoid this and pass it through in r3. For now, 340 * code expects it to be in SRR1. 341 */ 342 mtspr SPRN_SRR1,r3 343 344 ld r2,PACATOC(r13) 345 346 li r0,KVM_HWTHREAD_IN_KVM 347 stb r0,HSTATE_HWTHREAD_STATE(r13) 348 349 /* NV GPR values from power7_idle() will no longer be valid */ 350 li r0,1 351 stb r0,PACA_NAPSTATELOST(r13) 352 353 /* were we napping due to cede? */ 354 lbz r0,HSTATE_NAPPING(r13) 355 cmpwi r0,NAPPING_CEDE 356 beq kvm_end_cede 357 cmpwi r0,NAPPING_NOVCPU 358 beq kvm_novcpu_wakeup 359 360 ld r1,PACAEMERGSP(r13) 361 subi r1,r1,STACK_FRAME_OVERHEAD 362 363 /* 364 * We weren't napping due to cede, so this must be a secondary 365 * thread being woken up to run a guest, or being woken up due 366 * to a stray IPI. (Or due to some machine check or hypervisor 367 * maintenance interrupt while the core is in KVM.) 368 */ 369 370 /* Check the wake reason in SRR1 to see why we got here */ 371 bl kvmppc_check_wake_reason 372 /* 373 * kvmppc_check_wake_reason could invoke a C routine, but we 374 * have no volatile registers to restore when we return. 375 */ 376 377 cmpdi r3, 0 378 bge kvm_no_guest 379 380 /* get vcore pointer, NULL if we have nothing to run */ 381 ld r5,HSTATE_KVM_VCORE(r13) 382 cmpdi r5,0 383 /* if we have no vcore to run, go back to sleep */ 384 beq kvm_no_guest 385 386kvm_secondary_got_guest: 387 388 /* Set HSTATE_DSCR(r13) to something sensible */ 389 ld r6, PACA_DSCR_DEFAULT(r13) 390 std r6, HSTATE_DSCR(r13) 391 392 /* On thread 0 of a subcore, set HDEC to max */ 393 lbz r4, HSTATE_PTID(r13) 394 cmpwi r4, 0 395 bne 63f 396 LOAD_REG_ADDR(r6, decrementer_max) 397 ld r6, 0(r6) 398 mtspr SPRN_HDEC, r6 399 /* and set per-LPAR registers, if doing dynamic micro-threading */ 400 ld r6, HSTATE_SPLIT_MODE(r13) 401 cmpdi r6, 0 402 beq 63f 403BEGIN_FTR_SECTION 404 ld r0, KVM_SPLIT_RPR(r6) 405 mtspr SPRN_RPR, r0 406 ld r0, KVM_SPLIT_PMMAR(r6) 407 mtspr SPRN_PMMAR, r0 408 ld r0, KVM_SPLIT_LDBAR(r6) 409 mtspr SPRN_LDBAR, r0 410 isync 411FTR_SECTION_ELSE 412 /* On P9 we use the split_info for coordinating LPCR changes */ 413 lwz r4, KVM_SPLIT_DO_SET(r6) 414 cmpwi r4, 0 415 beq 1f 416 mr r3, r6 417 bl kvmhv_p9_set_lpcr 418 nop 4191: 420ALT_FTR_SECTION_END_IFCLR(CPU_FTR_ARCH_300) 42163: 422 /* Order load of vcpu after load of vcore */ 423 lwsync 424 ld r4, HSTATE_KVM_VCPU(r13) 425 bl kvmppc_hv_entry 426 427 /* Back from the guest, go back to nap */ 428 /* Clear our vcpu and vcore pointers so we don't come back in early */ 429 li r0, 0 430 std r0, HSTATE_KVM_VCPU(r13) 431 /* 432 * Once we clear HSTATE_KVM_VCORE(r13), the code in 433 * kvmppc_run_core() is going to assume that all our vcpu 434 * state is visible in memory. This lwsync makes sure 435 * that that is true. 436 */ 437 lwsync 438 std r0, HSTATE_KVM_VCORE(r13) 439 440 /* 441 * All secondaries exiting guest will fall through this path. 442 * Before proceeding, just check for HMI interrupt and 443 * invoke opal hmi handler. By now we are sure that the 444 * primary thread on this core/subcore has already made partition 445 * switch/TB resync and we are good to call opal hmi handler. 446 */ 447 cmpwi r12, BOOK3S_INTERRUPT_HMI 448 bne kvm_no_guest 449 450 li r3,0 /* NULL argument */ 451 bl hmi_exception_realmode 452/* 453 * At this point we have finished executing in the guest. 454 * We need to wait for hwthread_req to become zero, since 455 * we may not turn on the MMU while hwthread_req is non-zero. 456 * While waiting we also need to check if we get given a vcpu to run. 457 */ 458kvm_no_guest: 459 lbz r3, HSTATE_HWTHREAD_REQ(r13) 460 cmpwi r3, 0 461 bne 53f 462 HMT_MEDIUM 463 li r0, KVM_HWTHREAD_IN_KERNEL 464 stb r0, HSTATE_HWTHREAD_STATE(r13) 465 /* need to recheck hwthread_req after a barrier, to avoid race */ 466 sync 467 lbz r3, HSTATE_HWTHREAD_REQ(r13) 468 cmpwi r3, 0 469 bne 54f 470/* 471 * We jump to pnv_wakeup_loss, which will return to the caller 472 * of power7_nap in the powernv cpu offline loop. The value we 473 * put in r3 becomes the return value for power7_nap. pnv_wakeup_loss 474 * requires SRR1 in r12. 475 */ 476 li r3, LPCR_PECE0 477 mfspr r4, SPRN_LPCR 478 rlwimi r4, r3, 0, LPCR_PECE0 | LPCR_PECE1 479 mtspr SPRN_LPCR, r4 480 li r3, 0 481 mfspr r12,SPRN_SRR1 482 b pnv_wakeup_loss 483 48453: HMT_LOW 485 ld r5, HSTATE_KVM_VCORE(r13) 486 cmpdi r5, 0 487 bne 60f 488 ld r3, HSTATE_SPLIT_MODE(r13) 489 cmpdi r3, 0 490 beq kvm_no_guest 491 lwz r0, KVM_SPLIT_DO_SET(r3) 492 cmpwi r0, 0 493 bne kvmhv_do_set 494 lwz r0, KVM_SPLIT_DO_RESTORE(r3) 495 cmpwi r0, 0 496 bne kvmhv_do_restore 497 lbz r0, KVM_SPLIT_DO_NAP(r3) 498 cmpwi r0, 0 499 beq kvm_no_guest 500 HMT_MEDIUM 501 b kvm_unsplit_nap 50260: HMT_MEDIUM 503 b kvm_secondary_got_guest 504 50554: li r0, KVM_HWTHREAD_IN_KVM 506 stb r0, HSTATE_HWTHREAD_STATE(r13) 507 b kvm_no_guest 508 509kvmhv_do_set: 510 /* Set LPCR, LPIDR etc. on P9 */ 511 HMT_MEDIUM 512 bl kvmhv_p9_set_lpcr 513 nop 514 b kvm_no_guest 515 516kvmhv_do_restore: 517 HMT_MEDIUM 518 bl kvmhv_p9_restore_lpcr 519 nop 520 b kvm_no_guest 521 522/* 523 * Here the primary thread is trying to return the core to 524 * whole-core mode, so we need to nap. 525 */ 526kvm_unsplit_nap: 527 /* 528 * When secondaries are napping in kvm_unsplit_nap() with 529 * hwthread_req = 1, HMI goes ignored even though subcores are 530 * already exited the guest. Hence HMI keeps waking up secondaries 531 * from nap in a loop and secondaries always go back to nap since 532 * no vcore is assigned to them. This makes impossible for primary 533 * thread to get hold of secondary threads resulting into a soft 534 * lockup in KVM path. 535 * 536 * Let us check if HMI is pending and handle it before we go to nap. 537 */ 538 cmpwi r12, BOOK3S_INTERRUPT_HMI 539 bne 55f 540 li r3, 0 /* NULL argument */ 541 bl hmi_exception_realmode 54255: 543 /* 544 * Ensure that secondary doesn't nap when it has 545 * its vcore pointer set. 546 */ 547 sync /* matches smp_mb() before setting split_info.do_nap */ 548 ld r0, HSTATE_KVM_VCORE(r13) 549 cmpdi r0, 0 550 bne kvm_no_guest 551 /* clear any pending message */ 552BEGIN_FTR_SECTION 553 lis r6, (PPC_DBELL_SERVER << (63-36))@h 554 PPC_MSGCLR(6) 555END_FTR_SECTION_IFSET(CPU_FTR_ARCH_207S) 556 /* Set kvm_split_mode.napped[tid] = 1 */ 557 ld r3, HSTATE_SPLIT_MODE(r13) 558 li r0, 1 559 lbz r4, HSTATE_TID(r13) 560 addi r4, r4, KVM_SPLIT_NAPPED 561 stbx r0, r3, r4 562 /* Check the do_nap flag again after setting napped[] */ 563 sync 564 lbz r0, KVM_SPLIT_DO_NAP(r3) 565 cmpwi r0, 0 566 beq 57f 567 li r3, (LPCR_PECEDH | LPCR_PECE0) >> 4 568 mfspr r5, SPRN_LPCR 569 rlwimi r5, r3, 4, (LPCR_PECEDP | LPCR_PECEDH | LPCR_PECE0 | LPCR_PECE1) 570 b kvm_nap_sequence 571 57257: li r0, 0 573 stbx r0, r3, r4 574 b kvm_no_guest 575 576/****************************************************************************** 577 * * 578 * Entry code * 579 * * 580 *****************************************************************************/ 581 582.global kvmppc_hv_entry 583kvmppc_hv_entry: 584 585 /* Required state: 586 * 587 * R4 = vcpu pointer (or NULL) 588 * MSR = ~IR|DR 589 * R13 = PACA 590 * R1 = host R1 591 * R2 = TOC 592 * all other volatile GPRS = free 593 * Does not preserve non-volatile GPRs or CR fields 594 */ 595 mflr r0 596 std r0, PPC_LR_STKOFF(r1) 597 stdu r1, -SFS(r1) 598 599 /* Save R1 in the PACA */ 600 std r1, HSTATE_HOST_R1(r13) 601 602 li r6, KVM_GUEST_MODE_HOST_HV 603 stb r6, HSTATE_IN_GUEST(r13) 604 605#ifdef CONFIG_KVM_BOOK3S_HV_EXIT_TIMING 606 /* Store initial timestamp */ 607 cmpdi r4, 0 608 beq 1f 609 addi r3, r4, VCPU_TB_RMENTRY 610 bl kvmhv_start_timing 6111: 612#endif 613 614 /* Use cr7 as an indication of radix mode */ 615 ld r5, HSTATE_KVM_VCORE(r13) 616 ld r9, VCORE_KVM(r5) /* pointer to struct kvm */ 617 lbz r0, KVM_RADIX(r9) 618 cmpwi cr7, r0, 0 619 620 /* 621 * POWER7/POWER8 host -> guest partition switch code. 622 * We don't have to lock against concurrent tlbies, 623 * but we do have to coordinate across hardware threads. 624 */ 625 /* Set bit in entry map iff exit map is zero. */ 626 li r7, 1 627 lbz r6, HSTATE_PTID(r13) 628 sld r7, r7, r6 629 addi r8, r5, VCORE_ENTRY_EXIT 63021: lwarx r3, 0, r8 631 cmpwi r3, 0x100 /* any threads starting to exit? */ 632 bge secondary_too_late /* if so we're too late to the party */ 633 or r3, r3, r7 634 stwcx. r3, 0, r8 635 bne 21b 636 637 /* Primary thread switches to guest partition. */ 638 cmpwi r6,0 639 bne 10f 640 lwz r7,KVM_LPID(r9) 641BEGIN_FTR_SECTION 642 ld r6,KVM_SDR1(r9) 643 li r0,LPID_RSVD /* switch to reserved LPID */ 644 mtspr SPRN_LPID,r0 645 ptesync 646 mtspr SPRN_SDR1,r6 /* switch to partition page table */ 647END_FTR_SECTION_IFCLR(CPU_FTR_ARCH_300) 648 mtspr SPRN_LPID,r7 649 isync 650 651 /* See if we need to flush the TLB */ 652 lhz r6,PACAPACAINDEX(r13) /* test_bit(cpu, need_tlb_flush) */ 653BEGIN_FTR_SECTION 654 /* 655 * On POWER9, individual threads can come in here, but the 656 * TLB is shared between the 4 threads in a core, hence 657 * invalidating on one thread invalidates for all. 658 * Thus we make all 4 threads use the same bit here. 659 */ 660 clrrdi r6,r6,2 661END_FTR_SECTION_IFSET(CPU_FTR_ARCH_300) 662 clrldi r7,r6,64-6 /* extract bit number (6 bits) */ 663 srdi r6,r6,6 /* doubleword number */ 664 sldi r6,r6,3 /* address offset */ 665 add r6,r6,r9 666 addi r6,r6,KVM_NEED_FLUSH /* dword in kvm->arch.need_tlb_flush */ 667 li r8,1 668 sld r8,r8,r7 669 ld r7,0(r6) 670 and. r7,r7,r8 671 beq 22f 672 /* Flush the TLB of any entries for this LPID */ 673 lwz r0,KVM_TLB_SETS(r9) 674 mtctr r0 675 li r7,0x800 /* IS field = 0b10 */ 676 ptesync 677 li r0,0 /* RS for P9 version of tlbiel */ 678 bne cr7, 29f 67928: tlbiel r7 /* On P9, rs=0, RIC=0, PRS=0, R=0 */ 680 addi r7,r7,0x1000 681 bdnz 28b 682 b 30f 68329: PPC_TLBIEL(7,0,2,1,1) /* for radix, RIC=2, PRS=1, R=1 */ 684 addi r7,r7,0x1000 685 bdnz 29b 68630: ptesync 68723: ldarx r7,0,r6 /* clear the bit after TLB flushed */ 688 andc r7,r7,r8 689 stdcx. r7,0,r6 690 bne 23b 691 692 /* Add timebase offset onto timebase */ 69322: ld r8,VCORE_TB_OFFSET(r5) 694 cmpdi r8,0 695 beq 37f 696 mftb r6 /* current host timebase */ 697 add r8,r8,r6 698 mtspr SPRN_TBU40,r8 /* update upper 40 bits */ 699 mftb r7 /* check if lower 24 bits overflowed */ 700 clrldi r6,r6,40 701 clrldi r7,r7,40 702 cmpld r7,r6 703 bge 37f 704 addis r8,r8,0x100 /* if so, increment upper 40 bits */ 705 mtspr SPRN_TBU40,r8 706 707 /* Load guest PCR value to select appropriate compat mode */ 70837: ld r7, VCORE_PCR(r5) 709 cmpdi r7, 0 710 beq 38f 711 mtspr SPRN_PCR, r7 71238: 713 714BEGIN_FTR_SECTION 715 /* DPDES and VTB are shared between threads */ 716 ld r8, VCORE_DPDES(r5) 717 ld r7, VCORE_VTB(r5) 718 mtspr SPRN_DPDES, r8 719 mtspr SPRN_VTB, r7 720END_FTR_SECTION_IFSET(CPU_FTR_ARCH_207S) 721 722 /* Mark the subcore state as inside guest */ 723 bl kvmppc_subcore_enter_guest 724 nop 725 ld r5, HSTATE_KVM_VCORE(r13) 726 ld r4, HSTATE_KVM_VCPU(r13) 727 li r0,1 728 stb r0,VCORE_IN_GUEST(r5) /* signal secondaries to continue */ 729 730 /* Do we have a guest vcpu to run? */ 73110: cmpdi r4, 0 732 beq kvmppc_primary_no_guest 733kvmppc_got_guest: 734 /* Increment yield count if they have a VPA */ 735 ld r3, VCPU_VPA(r4) 736 cmpdi r3, 0 737 beq 25f 738 li r6, LPPACA_YIELDCOUNT 739 LWZX_BE r5, r3, r6 740 addi r5, r5, 1 741 STWX_BE r5, r3, r6 742 li r6, 1 743 stb r6, VCPU_VPA_DIRTY(r4) 74425: 745 746 /* Save purr/spurr */ 747 mfspr r5,SPRN_PURR 748 mfspr r6,SPRN_SPURR 749 std r5,HSTATE_PURR(r13) 750 std r6,HSTATE_SPURR(r13) 751 ld r7,VCPU_PURR(r4) 752 ld r8,VCPU_SPURR(r4) 753 mtspr SPRN_PURR,r7 754 mtspr SPRN_SPURR,r8 755 756 /* Save host values of some registers */ 757BEGIN_FTR_SECTION 758 mfspr r5, SPRN_TIDR 759 mfspr r6, SPRN_PSSCR 760 mfspr r7, SPRN_PID 761 mfspr r8, SPRN_IAMR 762 std r5, STACK_SLOT_TID(r1) 763 std r6, STACK_SLOT_PSSCR(r1) 764 std r7, STACK_SLOT_PID(r1) 765 std r8, STACK_SLOT_IAMR(r1) 766 mfspr r5, SPRN_HFSCR 767 std r5, STACK_SLOT_HFSCR(r1) 768END_FTR_SECTION_IFSET(CPU_FTR_ARCH_300) 769BEGIN_FTR_SECTION 770 mfspr r5, SPRN_CIABR 771 mfspr r6, SPRN_DAWR 772 mfspr r7, SPRN_DAWRX 773 std r5, STACK_SLOT_CIABR(r1) 774 std r6, STACK_SLOT_DAWR(r1) 775 std r7, STACK_SLOT_DAWRX(r1) 776END_FTR_SECTION_IFSET(CPU_FTR_ARCH_207S) 777 778BEGIN_FTR_SECTION 779 /* Set partition DABR */ 780 /* Do this before re-enabling PMU to avoid P7 DABR corruption bug */ 781 lwz r5,VCPU_DABRX(r4) 782 ld r6,VCPU_DABR(r4) 783 mtspr SPRN_DABRX,r5 784 mtspr SPRN_DABR,r6 785 isync 786END_FTR_SECTION_IFCLR(CPU_FTR_ARCH_207S) 787 788#ifdef CONFIG_PPC_TRANSACTIONAL_MEM 789BEGIN_FTR_SECTION 790 /* 791 * NOTE THAT THIS TRASHES ALL NON-VOLATILE REGISTERS INCLUDING CR 792 */ 793 bl kvmppc_restore_tm 794END_FTR_SECTION_IFSET(CPU_FTR_TM) 795#endif 796 797 /* Load guest PMU registers */ 798 /* R4 is live here (vcpu pointer) */ 799 li r3, 1 800 sldi r3, r3, 31 /* MMCR0_FC (freeze counters) bit */ 801 mtspr SPRN_MMCR0, r3 /* freeze all counters, disable ints */ 802 isync 803BEGIN_FTR_SECTION 804 ld r3, VCPU_MMCR(r4) 805 andi. r5, r3, MMCR0_PMAO_SYNC | MMCR0_PMAO 806 cmpwi r5, MMCR0_PMAO 807 beql kvmppc_fix_pmao 808END_FTR_SECTION_IFSET(CPU_FTR_PMAO_BUG) 809 lwz r3, VCPU_PMC(r4) /* always load up guest PMU registers */ 810 lwz r5, VCPU_PMC + 4(r4) /* to prevent information leak */ 811 lwz r6, VCPU_PMC + 8(r4) 812 lwz r7, VCPU_PMC + 12(r4) 813 lwz r8, VCPU_PMC + 16(r4) 814 lwz r9, VCPU_PMC + 20(r4) 815 mtspr SPRN_PMC1, r3 816 mtspr SPRN_PMC2, r5 817 mtspr SPRN_PMC3, r6 818 mtspr SPRN_PMC4, r7 819 mtspr SPRN_PMC5, r8 820 mtspr SPRN_PMC6, r9 821 ld r3, VCPU_MMCR(r4) 822 ld r5, VCPU_MMCR + 8(r4) 823 ld r6, VCPU_MMCR + 16(r4) 824 ld r7, VCPU_SIAR(r4) 825 ld r8, VCPU_SDAR(r4) 826 mtspr SPRN_MMCR1, r5 827 mtspr SPRN_MMCRA, r6 828 mtspr SPRN_SIAR, r7 829 mtspr SPRN_SDAR, r8 830BEGIN_FTR_SECTION 831 ld r5, VCPU_MMCR + 24(r4) 832 ld r6, VCPU_SIER(r4) 833 mtspr SPRN_MMCR2, r5 834 mtspr SPRN_SIER, r6 835BEGIN_FTR_SECTION_NESTED(96) 836 lwz r7, VCPU_PMC + 24(r4) 837 lwz r8, VCPU_PMC + 28(r4) 838 ld r9, VCPU_MMCR + 32(r4) 839 mtspr SPRN_SPMC1, r7 840 mtspr SPRN_SPMC2, r8 841 mtspr SPRN_MMCRS, r9 842END_FTR_SECTION_NESTED(CPU_FTR_ARCH_300, 0, 96) 843END_FTR_SECTION_IFSET(CPU_FTR_ARCH_207S) 844 mtspr SPRN_MMCR0, r3 845 isync 846 847 /* Load up FP, VMX and VSX registers */ 848 bl kvmppc_load_fp 849 850 ld r14, VCPU_GPR(R14)(r4) 851 ld r15, VCPU_GPR(R15)(r4) 852 ld r16, VCPU_GPR(R16)(r4) 853 ld r17, VCPU_GPR(R17)(r4) 854 ld r18, VCPU_GPR(R18)(r4) 855 ld r19, VCPU_GPR(R19)(r4) 856 ld r20, VCPU_GPR(R20)(r4) 857 ld r21, VCPU_GPR(R21)(r4) 858 ld r22, VCPU_GPR(R22)(r4) 859 ld r23, VCPU_GPR(R23)(r4) 860 ld r24, VCPU_GPR(R24)(r4) 861 ld r25, VCPU_GPR(R25)(r4) 862 ld r26, VCPU_GPR(R26)(r4) 863 ld r27, VCPU_GPR(R27)(r4) 864 ld r28, VCPU_GPR(R28)(r4) 865 ld r29, VCPU_GPR(R29)(r4) 866 ld r30, VCPU_GPR(R30)(r4) 867 ld r31, VCPU_GPR(R31)(r4) 868 869 /* Switch DSCR to guest value */ 870 ld r5, VCPU_DSCR(r4) 871 mtspr SPRN_DSCR, r5 872 873BEGIN_FTR_SECTION 874 /* Skip next section on POWER7 */ 875 b 8f 876END_FTR_SECTION_IFCLR(CPU_FTR_ARCH_207S) 877 /* Load up POWER8-specific registers */ 878 ld r5, VCPU_IAMR(r4) 879 lwz r6, VCPU_PSPB(r4) 880 ld r7, VCPU_FSCR(r4) 881 mtspr SPRN_IAMR, r5 882 mtspr SPRN_PSPB, r6 883 mtspr SPRN_FSCR, r7 884 ld r5, VCPU_DAWR(r4) 885 ld r6, VCPU_DAWRX(r4) 886 ld r7, VCPU_CIABR(r4) 887 ld r8, VCPU_TAR(r4) 888 mtspr SPRN_DAWR, r5 889 mtspr SPRN_DAWRX, r6 890 mtspr SPRN_CIABR, r7 891 mtspr SPRN_TAR, r8 892 ld r5, VCPU_IC(r4) 893 ld r8, VCPU_EBBHR(r4) 894 mtspr SPRN_IC, r5 895 mtspr SPRN_EBBHR, r8 896 ld r5, VCPU_EBBRR(r4) 897 ld r6, VCPU_BESCR(r4) 898 lwz r7, VCPU_GUEST_PID(r4) 899 ld r8, VCPU_WORT(r4) 900 mtspr SPRN_EBBRR, r5 901 mtspr SPRN_BESCR, r6 902 mtspr SPRN_PID, r7 903 mtspr SPRN_WORT, r8 904BEGIN_FTR_SECTION 905 PPC_INVALIDATE_ERAT 906END_FTR_SECTION_IFSET(CPU_FTR_POWER9_DD1) 907BEGIN_FTR_SECTION 908 /* POWER8-only registers */ 909 ld r5, VCPU_TCSCR(r4) 910 ld r6, VCPU_ACOP(r4) 911 ld r7, VCPU_CSIGR(r4) 912 ld r8, VCPU_TACR(r4) 913 mtspr SPRN_TCSCR, r5 914 mtspr SPRN_ACOP, r6 915 mtspr SPRN_CSIGR, r7 916 mtspr SPRN_TACR, r8 917FTR_SECTION_ELSE 918 /* POWER9-only registers */ 919 ld r5, VCPU_TID(r4) 920 ld r6, VCPU_PSSCR(r4) 921 oris r6, r6, PSSCR_EC@h /* This makes stop trap to HV */ 922 ld r7, VCPU_HFSCR(r4) 923 mtspr SPRN_TIDR, r5 924 mtspr SPRN_PSSCR, r6 925 mtspr SPRN_HFSCR, r7 926ALT_FTR_SECTION_END_IFCLR(CPU_FTR_ARCH_300) 9278: 928 929 /* 930 * Set the decrementer to the guest decrementer. 931 */ 932 ld r8,VCPU_DEC_EXPIRES(r4) 933 /* r8 is a host timebase value here, convert to guest TB */ 934 ld r5,HSTATE_KVM_VCORE(r13) 935 ld r6,VCORE_TB_OFFSET(r5) 936 add r8,r8,r6 937 mftb r7 938 subf r3,r7,r8 939 mtspr SPRN_DEC,r3 940 941 ld r5, VCPU_SPRG0(r4) 942 ld r6, VCPU_SPRG1(r4) 943 ld r7, VCPU_SPRG2(r4) 944 ld r8, VCPU_SPRG3(r4) 945 mtspr SPRN_SPRG0, r5 946 mtspr SPRN_SPRG1, r6 947 mtspr SPRN_SPRG2, r7 948 mtspr SPRN_SPRG3, r8 949 950 /* Load up DAR and DSISR */ 951 ld r5, VCPU_DAR(r4) 952 lwz r6, VCPU_DSISR(r4) 953 mtspr SPRN_DAR, r5 954 mtspr SPRN_DSISR, r6 955 956 /* Restore AMR and UAMOR, set AMOR to all 1s */ 957 ld r5,VCPU_AMR(r4) 958 ld r6,VCPU_UAMOR(r4) 959 li r7,-1 960 mtspr SPRN_AMR,r5 961 mtspr SPRN_UAMOR,r6 962 mtspr SPRN_AMOR,r7 963 964 /* Restore state of CTRL run bit; assume 1 on entry */ 965 lwz r5,VCPU_CTRL(r4) 966 andi. r5,r5,1 967 bne 4f 968 mfspr r6,SPRN_CTRLF 969 clrrdi r6,r6,1 970 mtspr SPRN_CTRLT,r6 9714: 972 /* Secondary threads wait for primary to have done partition switch */ 973 ld r5, HSTATE_KVM_VCORE(r13) 974 lbz r6, HSTATE_PTID(r13) 975 cmpwi r6, 0 976 beq 21f 977 lbz r0, VCORE_IN_GUEST(r5) 978 cmpwi r0, 0 979 bne 21f 980 HMT_LOW 98120: lwz r3, VCORE_ENTRY_EXIT(r5) 982 cmpwi r3, 0x100 983 bge no_switch_exit 984 lbz r0, VCORE_IN_GUEST(r5) 985 cmpwi r0, 0 986 beq 20b 987 HMT_MEDIUM 98821: 989 /* Set LPCR. */ 990 ld r8,VCORE_LPCR(r5) 991 mtspr SPRN_LPCR,r8 992 isync 993 994 /* Check if HDEC expires soon */ 995 mfspr r3, SPRN_HDEC 996 EXTEND_HDEC(r3) 997 cmpdi r3, 512 /* 1 microsecond */ 998 blt hdec_soon 999 1000 /* For hash guest, clear out and reload the SLB */ 1001 ld r6, VCPU_KVM(r4) 1002 lbz r0, KVM_RADIX(r6) 1003 cmpwi r0, 0 1004 bne 9f 1005 li r6, 0 1006 slbmte r6, r6 1007 slbia 1008 ptesync 1009 1010 /* Load up guest SLB entries (N.B. slb_max will be 0 for radix) */ 1011 lwz r5,VCPU_SLB_MAX(r4) 1012 cmpwi r5,0 1013 beq 9f 1014 mtctr r5 1015 addi r6,r4,VCPU_SLB 10161: ld r8,VCPU_SLB_E(r6) 1017 ld r9,VCPU_SLB_V(r6) 1018 slbmte r9,r8 1019 addi r6,r6,VCPU_SLB_SIZE 1020 bdnz 1b 10219: 1022 1023#ifdef CONFIG_KVM_XICS 1024 /* We are entering the guest on that thread, push VCPU to XIVE */ 1025 ld r10, HSTATE_XIVE_TIMA_PHYS(r13) 1026 cmpldi cr0, r10, 0 1027 beq no_xive 1028 ld r11, VCPU_XIVE_SAVED_STATE(r4) 1029 li r9, TM_QW1_OS 1030 eieio 1031 stdcix r11,r9,r10 1032 lwz r11, VCPU_XIVE_CAM_WORD(r4) 1033 li r9, TM_QW1_OS + TM_WORD2 1034 stwcix r11,r9,r10 1035 li r9, 1 1036 stb r9, VCPU_XIVE_PUSHED(r4) 1037 eieio 1038 1039 /* 1040 * We clear the irq_pending flag. There is a small chance of a 1041 * race vs. the escalation interrupt happening on another 1042 * processor setting it again, but the only consequence is to 1043 * cause a spurrious wakeup on the next H_CEDE which is not an 1044 * issue. 1045 */ 1046 li r0,0 1047 stb r0, VCPU_IRQ_PENDING(r4) 1048 1049 /* 1050 * In single escalation mode, if the escalation interrupt is 1051 * on, we mask it. 1052 */ 1053 lbz r0, VCPU_XIVE_ESC_ON(r4) 1054 cmpwi r0,0 1055 beq 1f 1056 ld r10, VCPU_XIVE_ESC_RADDR(r4) 1057 li r9, XIVE_ESB_SET_PQ_01 1058 ldcix r0, r10, r9 1059 sync 1060 1061 /* We have a possible subtle race here: The escalation interrupt might 1062 * have fired and be on its way to the host queue while we mask it, 1063 * and if we unmask it early enough (re-cede right away), there is 1064 * a theorical possibility that it fires again, thus landing in the 1065 * target queue more than once which is a big no-no. 1066 * 1067 * Fortunately, solving this is rather easy. If the above load setting 1068 * PQ to 01 returns a previous value where P is set, then we know the 1069 * escalation interrupt is somewhere on its way to the host. In that 1070 * case we simply don't clear the xive_esc_on flag below. It will be 1071 * eventually cleared by the handler for the escalation interrupt. 1072 * 1073 * Then, when doing a cede, we check that flag again before re-enabling 1074 * the escalation interrupt, and if set, we abort the cede. 1075 */ 1076 andi. r0, r0, XIVE_ESB_VAL_P 1077 bne- 1f 1078 1079 /* Now P is 0, we can clear the flag */ 1080 li r0, 0 1081 stb r0, VCPU_XIVE_ESC_ON(r4) 10821: 1083no_xive: 1084#endif /* CONFIG_KVM_XICS */ 1085 1086deliver_guest_interrupt: 1087 ld r6, VCPU_CTR(r4) 1088 ld r7, VCPU_XER(r4) 1089 1090 mtctr r6 1091 mtxer r7 1092 1093kvmppc_cede_reentry: /* r4 = vcpu, r13 = paca */ 1094 ld r10, VCPU_PC(r4) 1095 ld r11, VCPU_MSR(r4) 1096 ld r6, VCPU_SRR0(r4) 1097 ld r7, VCPU_SRR1(r4) 1098 mtspr SPRN_SRR0, r6 1099 mtspr SPRN_SRR1, r7 1100 1101 /* r11 = vcpu->arch.msr & ~MSR_HV */ 1102 rldicl r11, r11, 63 - MSR_HV_LG, 1 1103 rotldi r11, r11, 1 + MSR_HV_LG 1104 ori r11, r11, MSR_ME 1105 1106 /* Check if we can deliver an external or decrementer interrupt now */ 1107 ld r0, VCPU_PENDING_EXC(r4) 1108 rldicl r0, r0, 64 - BOOK3S_IRQPRIO_EXTERNAL_LEVEL, 63 1109 cmpdi cr1, r0, 0 1110 andi. r8, r11, MSR_EE 1111 mfspr r8, SPRN_LPCR 1112 /* Insert EXTERNAL_LEVEL bit into LPCR at the MER bit position */ 1113 rldimi r8, r0, LPCR_MER_SH, 63 - LPCR_MER_SH 1114 mtspr SPRN_LPCR, r8 1115 isync 1116 beq 5f 1117 li r0, BOOK3S_INTERRUPT_EXTERNAL 1118 bne cr1, 12f 1119 mfspr r0, SPRN_DEC 1120BEGIN_FTR_SECTION 1121 /* On POWER9 check whether the guest has large decrementer enabled */ 1122 andis. r8, r8, LPCR_LD@h 1123 bne 15f 1124END_FTR_SECTION_IFSET(CPU_FTR_ARCH_300) 1125 extsw r0, r0 112615: cmpdi r0, 0 1127 li r0, BOOK3S_INTERRUPT_DECREMENTER 1128 bge 5f 1129 113012: mtspr SPRN_SRR0, r10 1131 mr r10,r0 1132 mtspr SPRN_SRR1, r11 1133 mr r9, r4 1134 bl kvmppc_msr_interrupt 11355: 1136BEGIN_FTR_SECTION 1137 b fast_guest_return 1138END_FTR_SECTION_IFCLR(CPU_FTR_ARCH_300) 1139 /* On POWER9, check for pending doorbell requests */ 1140 lbz r0, VCPU_DBELL_REQ(r4) 1141 cmpwi r0, 0 1142 beq fast_guest_return 1143 ld r5, HSTATE_KVM_VCORE(r13) 1144 /* Set DPDES register so the CPU will take a doorbell interrupt */ 1145 li r0, 1 1146 mtspr SPRN_DPDES, r0 1147 std r0, VCORE_DPDES(r5) 1148 /* Make sure other cpus see vcore->dpdes set before dbell req clear */ 1149 lwsync 1150 /* Clear the pending doorbell request */ 1151 li r0, 0 1152 stb r0, VCPU_DBELL_REQ(r4) 1153 1154/* 1155 * Required state: 1156 * R4 = vcpu 1157 * R10: value for HSRR0 1158 * R11: value for HSRR1 1159 * R13 = PACA 1160 */ 1161fast_guest_return: 1162 li r0,0 1163 stb r0,VCPU_CEDED(r4) /* cancel cede */ 1164 mtspr SPRN_HSRR0,r10 1165 mtspr SPRN_HSRR1,r11 1166 1167 /* Activate guest mode, so faults get handled by KVM */ 1168 li r9, KVM_GUEST_MODE_GUEST_HV 1169 stb r9, HSTATE_IN_GUEST(r13) 1170 1171#ifdef CONFIG_KVM_BOOK3S_HV_EXIT_TIMING 1172 /* Accumulate timing */ 1173 addi r3, r4, VCPU_TB_GUEST 1174 bl kvmhv_accumulate_time 1175#endif 1176 1177 /* Enter guest */ 1178 1179BEGIN_FTR_SECTION 1180 ld r5, VCPU_CFAR(r4) 1181 mtspr SPRN_CFAR, r5 1182END_FTR_SECTION_IFSET(CPU_FTR_CFAR) 1183BEGIN_FTR_SECTION 1184 ld r0, VCPU_PPR(r4) 1185END_FTR_SECTION_IFSET(CPU_FTR_HAS_PPR) 1186 1187 ld r5, VCPU_LR(r4) 1188 lwz r6, VCPU_CR(r4) 1189 mtlr r5 1190 mtcr r6 1191 1192 ld r1, VCPU_GPR(R1)(r4) 1193 ld r2, VCPU_GPR(R2)(r4) 1194 ld r3, VCPU_GPR(R3)(r4) 1195 ld r5, VCPU_GPR(R5)(r4) 1196 ld r6, VCPU_GPR(R6)(r4) 1197 ld r7, VCPU_GPR(R7)(r4) 1198 ld r8, VCPU_GPR(R8)(r4) 1199 ld r9, VCPU_GPR(R9)(r4) 1200 ld r10, VCPU_GPR(R10)(r4) 1201 ld r11, VCPU_GPR(R11)(r4) 1202 ld r12, VCPU_GPR(R12)(r4) 1203 ld r13, VCPU_GPR(R13)(r4) 1204 1205BEGIN_FTR_SECTION 1206 mtspr SPRN_PPR, r0 1207END_FTR_SECTION_IFSET(CPU_FTR_HAS_PPR) 1208 1209/* Move canary into DSISR to check for later */ 1210BEGIN_FTR_SECTION 1211 li r0, 0x7fff 1212 mtspr SPRN_HDSISR, r0 1213END_FTR_SECTION_IFSET(CPU_FTR_ARCH_300) 1214 1215 ld r0, VCPU_GPR(R0)(r4) 1216 ld r4, VCPU_GPR(R4)(r4) 1217 HRFI_TO_GUEST 1218 b . 1219 1220secondary_too_late: 1221 li r12, 0 1222 stw r12, STACK_SLOT_TRAP(r1) 1223 cmpdi r4, 0 1224 beq 11f 1225 stw r12, VCPU_TRAP(r4) 1226#ifdef CONFIG_KVM_BOOK3S_HV_EXIT_TIMING 1227 addi r3, r4, VCPU_TB_RMEXIT 1228 bl kvmhv_accumulate_time 1229#endif 123011: b kvmhv_switch_to_host 1231 1232no_switch_exit: 1233 HMT_MEDIUM 1234 li r12, 0 1235 b 12f 1236hdec_soon: 1237 li r12, BOOK3S_INTERRUPT_HV_DECREMENTER 123812: stw r12, VCPU_TRAP(r4) 1239 mr r9, r4 1240#ifdef CONFIG_KVM_BOOK3S_HV_EXIT_TIMING 1241 addi r3, r4, VCPU_TB_RMEXIT 1242 bl kvmhv_accumulate_time 1243#endif 1244 b guest_bypass 1245 1246/****************************************************************************** 1247 * * 1248 * Exit code * 1249 * * 1250 *****************************************************************************/ 1251 1252/* 1253 * We come here from the first-level interrupt handlers. 1254 */ 1255 .globl kvmppc_interrupt_hv 1256kvmppc_interrupt_hv: 1257 /* 1258 * Register contents: 1259 * R12 = (guest CR << 32) | interrupt vector 1260 * R13 = PACA 1261 * guest R12 saved in shadow VCPU SCRATCH0 1262 * guest CTR saved in shadow VCPU SCRATCH1 if RELOCATABLE 1263 * guest R13 saved in SPRN_SCRATCH0 1264 */ 1265 std r9, HSTATE_SCRATCH2(r13) 1266 lbz r9, HSTATE_IN_GUEST(r13) 1267 cmpwi r9, KVM_GUEST_MODE_HOST_HV 1268 beq kvmppc_bad_host_intr 1269#ifdef CONFIG_KVM_BOOK3S_PR_POSSIBLE 1270 cmpwi r9, KVM_GUEST_MODE_GUEST 1271 ld r9, HSTATE_SCRATCH2(r13) 1272 beq kvmppc_interrupt_pr 1273#endif 1274 /* We're now back in the host but in guest MMU context */ 1275 li r9, KVM_GUEST_MODE_HOST_HV 1276 stb r9, HSTATE_IN_GUEST(r13) 1277 1278 ld r9, HSTATE_KVM_VCPU(r13) 1279 1280 /* Save registers */ 1281 1282 std r0, VCPU_GPR(R0)(r9) 1283 std r1, VCPU_GPR(R1)(r9) 1284 std r2, VCPU_GPR(R2)(r9) 1285 std r3, VCPU_GPR(R3)(r9) 1286 std r4, VCPU_GPR(R4)(r9) 1287 std r5, VCPU_GPR(R5)(r9) 1288 std r6, VCPU_GPR(R6)(r9) 1289 std r7, VCPU_GPR(R7)(r9) 1290 std r8, VCPU_GPR(R8)(r9) 1291 ld r0, HSTATE_SCRATCH2(r13) 1292 std r0, VCPU_GPR(R9)(r9) 1293 std r10, VCPU_GPR(R10)(r9) 1294 std r11, VCPU_GPR(R11)(r9) 1295 ld r3, HSTATE_SCRATCH0(r13) 1296 std r3, VCPU_GPR(R12)(r9) 1297 /* CR is in the high half of r12 */ 1298 srdi r4, r12, 32 1299 stw r4, VCPU_CR(r9) 1300BEGIN_FTR_SECTION 1301 ld r3, HSTATE_CFAR(r13) 1302 std r3, VCPU_CFAR(r9) 1303END_FTR_SECTION_IFSET(CPU_FTR_CFAR) 1304BEGIN_FTR_SECTION 1305 ld r4, HSTATE_PPR(r13) 1306 std r4, VCPU_PPR(r9) 1307END_FTR_SECTION_IFSET(CPU_FTR_HAS_PPR) 1308 1309 /* Restore R1/R2 so we can handle faults */ 1310 ld r1, HSTATE_HOST_R1(r13) 1311 ld r2, PACATOC(r13) 1312 1313 mfspr r10, SPRN_SRR0 1314 mfspr r11, SPRN_SRR1 1315 std r10, VCPU_SRR0(r9) 1316 std r11, VCPU_SRR1(r9) 1317 /* trap is in the low half of r12, clear CR from the high half */ 1318 clrldi r12, r12, 32 1319 andi. r0, r12, 2 /* need to read HSRR0/1? */ 1320 beq 1f 1321 mfspr r10, SPRN_HSRR0 1322 mfspr r11, SPRN_HSRR1 1323 clrrdi r12, r12, 2 13241: std r10, VCPU_PC(r9) 1325 std r11, VCPU_MSR(r9) 1326 1327 GET_SCRATCH0(r3) 1328 mflr r4 1329 std r3, VCPU_GPR(R13)(r9) 1330 std r4, VCPU_LR(r9) 1331 1332 stw r12,VCPU_TRAP(r9) 1333 1334 /* 1335 * Now that we have saved away SRR0/1 and HSRR0/1, 1336 * interrupts are recoverable in principle, so set MSR_RI. 1337 * This becomes important for relocation-on interrupts from 1338 * the guest, which we can get in radix mode on POWER9. 1339 */ 1340 li r0, MSR_RI 1341 mtmsrd r0, 1 1342 1343#ifdef CONFIG_KVM_BOOK3S_HV_EXIT_TIMING 1344 addi r3, r9, VCPU_TB_RMINTR 1345 mr r4, r9 1346 bl kvmhv_accumulate_time 1347 ld r5, VCPU_GPR(R5)(r9) 1348 ld r6, VCPU_GPR(R6)(r9) 1349 ld r7, VCPU_GPR(R7)(r9) 1350 ld r8, VCPU_GPR(R8)(r9) 1351#endif 1352 1353 /* Save HEIR (HV emulation assist reg) in emul_inst 1354 if this is an HEI (HV emulation interrupt, e40) */ 1355 li r3,KVM_INST_FETCH_FAILED 1356 stw r3,VCPU_LAST_INST(r9) 1357 cmpwi r12,BOOK3S_INTERRUPT_H_EMUL_ASSIST 1358 bne 11f 1359 mfspr r3,SPRN_HEIR 136011: stw r3,VCPU_HEIR(r9) 1361 1362 /* these are volatile across C function calls */ 1363#ifdef CONFIG_RELOCATABLE 1364 ld r3, HSTATE_SCRATCH1(r13) 1365 mtctr r3 1366#else 1367 mfctr r3 1368#endif 1369 mfxer r4 1370 std r3, VCPU_CTR(r9) 1371 std r4, VCPU_XER(r9) 1372 1373 /* If this is a page table miss then see if it's theirs or ours */ 1374 cmpwi r12, BOOK3S_INTERRUPT_H_DATA_STORAGE 1375 beq kvmppc_hdsi 1376 cmpwi r12, BOOK3S_INTERRUPT_H_INST_STORAGE 1377 beq kvmppc_hisi 1378 1379 /* See if this is a leftover HDEC interrupt */ 1380 cmpwi r12,BOOK3S_INTERRUPT_HV_DECREMENTER 1381 bne 2f 1382 mfspr r3,SPRN_HDEC 1383 EXTEND_HDEC(r3) 1384 cmpdi r3,0 1385 mr r4,r9 1386 bge fast_guest_return 13872: 1388 /* See if this is an hcall we can handle in real mode */ 1389 cmpwi r12,BOOK3S_INTERRUPT_SYSCALL 1390 beq hcall_try_real_mode 1391 1392 /* Hypervisor doorbell - exit only if host IPI flag set */ 1393 cmpwi r12, BOOK3S_INTERRUPT_H_DOORBELL 1394 bne 3f 1395BEGIN_FTR_SECTION 1396 PPC_MSGSYNC 1397 lwsync 1398END_FTR_SECTION_IFSET(CPU_FTR_ARCH_300) 1399 lbz r0, HSTATE_HOST_IPI(r13) 1400 cmpwi r0, 0 1401 beq 4f 1402 b guest_exit_cont 14033: 1404 /* If it's a hypervisor facility unavailable interrupt, save HFSCR */ 1405 cmpwi r12, BOOK3S_INTERRUPT_H_FAC_UNAVAIL 1406 bne 14f 1407 mfspr r3, SPRN_HFSCR 1408 std r3, VCPU_HFSCR(r9) 1409 b guest_exit_cont 141014: 1411 /* External interrupt ? */ 1412 cmpwi r12, BOOK3S_INTERRUPT_EXTERNAL 1413 bne+ guest_exit_cont 1414 1415 /* External interrupt, first check for host_ipi. If this is 1416 * set, we know the host wants us out so let's do it now 1417 */ 1418 bl kvmppc_read_intr 1419 1420 /* 1421 * Restore the active volatile registers after returning from 1422 * a C function. 1423 */ 1424 ld r9, HSTATE_KVM_VCPU(r13) 1425 li r12, BOOK3S_INTERRUPT_EXTERNAL 1426 1427 /* 1428 * kvmppc_read_intr return codes: 1429 * 1430 * Exit to host (r3 > 0) 1431 * 1 An interrupt is pending that needs to be handled by the host 1432 * Exit guest and return to host by branching to guest_exit_cont 1433 * 1434 * 2 Passthrough that needs completion in the host 1435 * Exit guest and return to host by branching to guest_exit_cont 1436 * However, we also set r12 to BOOK3S_INTERRUPT_HV_RM_HARD 1437 * to indicate to the host to complete handling the interrupt 1438 * 1439 * Before returning to guest, we check if any CPU is heading out 1440 * to the host and if so, we head out also. If no CPUs are heading 1441 * check return values <= 0. 1442 * 1443 * Return to guest (r3 <= 0) 1444 * 0 No external interrupt is pending 1445 * -1 A guest wakeup IPI (which has now been cleared) 1446 * In either case, we return to guest to deliver any pending 1447 * guest interrupts. 1448 * 1449 * -2 A PCI passthrough external interrupt was handled 1450 * (interrupt was delivered directly to guest) 1451 * Return to guest to deliver any pending guest interrupts. 1452 */ 1453 1454 cmpdi r3, 1 1455 ble 1f 1456 1457 /* Return code = 2 */ 1458 li r12, BOOK3S_INTERRUPT_HV_RM_HARD 1459 stw r12, VCPU_TRAP(r9) 1460 b guest_exit_cont 1461 14621: /* Return code <= 1 */ 1463 cmpdi r3, 0 1464 bgt guest_exit_cont 1465 1466 /* Return code <= 0 */ 14674: ld r5, HSTATE_KVM_VCORE(r13) 1468 lwz r0, VCORE_ENTRY_EXIT(r5) 1469 cmpwi r0, 0x100 1470 mr r4, r9 1471 blt deliver_guest_interrupt 1472 1473guest_exit_cont: /* r9 = vcpu, r12 = trap, r13 = paca */ 1474 /* Save more register state */ 1475 mfdar r6 1476 mfdsisr r7 1477 std r6, VCPU_DAR(r9) 1478 stw r7, VCPU_DSISR(r9) 1479 /* don't overwrite fault_dar/fault_dsisr if HDSI */ 1480 cmpwi r12,BOOK3S_INTERRUPT_H_DATA_STORAGE 1481 beq mc_cont 1482 std r6, VCPU_FAULT_DAR(r9) 1483 stw r7, VCPU_FAULT_DSISR(r9) 1484 1485 /* See if it is a machine check */ 1486 cmpwi r12, BOOK3S_INTERRUPT_MACHINE_CHECK 1487 beq machine_check_realmode 1488mc_cont: 1489#ifdef CONFIG_KVM_BOOK3S_HV_EXIT_TIMING 1490 addi r3, r9, VCPU_TB_RMEXIT 1491 mr r4, r9 1492 bl kvmhv_accumulate_time 1493#endif 1494#ifdef CONFIG_KVM_XICS 1495 /* We are exiting, pull the VP from the XIVE */ 1496 lbz r0, VCPU_XIVE_PUSHED(r9) 1497 cmpwi cr0, r0, 0 1498 beq 1f 1499 li r7, TM_SPC_PULL_OS_CTX 1500 li r6, TM_QW1_OS 1501 mfmsr r0 1502 andi. r0, r0, MSR_DR /* in real mode? */ 1503 beq 2f 1504 ld r10, HSTATE_XIVE_TIMA_VIRT(r13) 1505 cmpldi cr0, r10, 0 1506 beq 1f 1507 /* First load to pull the context, we ignore the value */ 1508 eieio 1509 lwzx r11, r7, r10 1510 /* Second load to recover the context state (Words 0 and 1) */ 1511 ldx r11, r6, r10 1512 b 3f 15132: ld r10, HSTATE_XIVE_TIMA_PHYS(r13) 1514 cmpldi cr0, r10, 0 1515 beq 1f 1516 /* First load to pull the context, we ignore the value */ 1517 eieio 1518 lwzcix r11, r7, r10 1519 /* Second load to recover the context state (Words 0 and 1) */ 1520 ldcix r11, r6, r10 15213: std r11, VCPU_XIVE_SAVED_STATE(r9) 1522 /* Fixup some of the state for the next load */ 1523 li r10, 0 1524 li r0, 0xff 1525 stb r10, VCPU_XIVE_PUSHED(r9) 1526 stb r10, (VCPU_XIVE_SAVED_STATE+3)(r9) 1527 stb r0, (VCPU_XIVE_SAVED_STATE+4)(r9) 1528 eieio 15291: 1530#endif /* CONFIG_KVM_XICS */ 1531 1532 /* For hash guest, read the guest SLB and save it away */ 1533 ld r5, VCPU_KVM(r9) 1534 lbz r0, KVM_RADIX(r5) 1535 li r5, 0 1536 cmpwi r0, 0 1537 bne 3f /* for radix, save 0 entries */ 1538 lwz r0,VCPU_SLB_NR(r9) /* number of entries in SLB */ 1539 mtctr r0 1540 li r6,0 1541 addi r7,r9,VCPU_SLB 15421: slbmfee r8,r6 1543 andis. r0,r8,SLB_ESID_V@h 1544 beq 2f 1545 add r8,r8,r6 /* put index in */ 1546 slbmfev r3,r6 1547 std r8,VCPU_SLB_E(r7) 1548 std r3,VCPU_SLB_V(r7) 1549 addi r7,r7,VCPU_SLB_SIZE 1550 addi r5,r5,1 15512: addi r6,r6,1 1552 bdnz 1b 1553 /* Finally clear out the SLB */ 1554 li r0,0 1555 slbmte r0,r0 1556 slbia 1557 ptesync 15583: stw r5,VCPU_SLB_MAX(r9) 1559 1560 /* load host SLB entries */ 1561BEGIN_MMU_FTR_SECTION 1562 b 0f 1563END_MMU_FTR_SECTION_IFSET(MMU_FTR_TYPE_RADIX) 1564 ld r8,PACA_SLBSHADOWPTR(r13) 1565 1566 .rept SLB_NUM_BOLTED 1567 li r3, SLBSHADOW_SAVEAREA 1568 LDX_BE r5, r8, r3 1569 addi r3, r3, 8 1570 LDX_BE r6, r8, r3 1571 andis. r7,r5,SLB_ESID_V@h 1572 beq 1f 1573 slbmte r6,r5 15741: addi r8,r8,16 1575 .endr 15760: 1577 1578guest_bypass: 1579 stw r12, STACK_SLOT_TRAP(r1) 1580 mr r3, r12 1581 /* Increment exit count, poke other threads to exit */ 1582 bl kvmhv_commence_exit 1583 nop 1584 ld r9, HSTATE_KVM_VCPU(r13) 1585 1586 /* Stop others sending VCPU interrupts to this physical CPU */ 1587 li r0, -1 1588 stw r0, VCPU_CPU(r9) 1589 stw r0, VCPU_THREAD_CPU(r9) 1590 1591 /* Save guest CTRL register, set runlatch to 1 */ 1592 mfspr r6,SPRN_CTRLF 1593 stw r6,VCPU_CTRL(r9) 1594 andi. r0,r6,1 1595 bne 4f 1596 ori r6,r6,1 1597 mtspr SPRN_CTRLT,r6 15984: 1599 /* 1600 * Save the guest PURR/SPURR 1601 */ 1602 mfspr r5,SPRN_PURR 1603 mfspr r6,SPRN_SPURR 1604 ld r7,VCPU_PURR(r9) 1605 ld r8,VCPU_SPURR(r9) 1606 std r5,VCPU_PURR(r9) 1607 std r6,VCPU_SPURR(r9) 1608 subf r5,r7,r5 1609 subf r6,r8,r6 1610 1611 /* 1612 * Restore host PURR/SPURR and add guest times 1613 * so that the time in the guest gets accounted. 1614 */ 1615 ld r3,HSTATE_PURR(r13) 1616 ld r4,HSTATE_SPURR(r13) 1617 add r3,r3,r5 1618 add r4,r4,r6 1619 mtspr SPRN_PURR,r3 1620 mtspr SPRN_SPURR,r4 1621 1622 /* Save DEC */ 1623 ld r3, HSTATE_KVM_VCORE(r13) 1624 mfspr r5,SPRN_DEC 1625 mftb r6 1626 /* On P9, if the guest has large decr enabled, don't sign extend */ 1627BEGIN_FTR_SECTION 1628 ld r4, VCORE_LPCR(r3) 1629 andis. r4, r4, LPCR_LD@h 1630 bne 16f 1631END_FTR_SECTION_IFSET(CPU_FTR_ARCH_300) 1632 extsw r5,r5 163316: add r5,r5,r6 1634 /* r5 is a guest timebase value here, convert to host TB */ 1635 ld r4,VCORE_TB_OFFSET(r3) 1636 subf r5,r4,r5 1637 std r5,VCPU_DEC_EXPIRES(r9) 1638 1639BEGIN_FTR_SECTION 1640 b 8f 1641END_FTR_SECTION_IFCLR(CPU_FTR_ARCH_207S) 1642 /* Save POWER8-specific registers */ 1643 mfspr r5, SPRN_IAMR 1644 mfspr r6, SPRN_PSPB 1645 mfspr r7, SPRN_FSCR 1646 std r5, VCPU_IAMR(r9) 1647 stw r6, VCPU_PSPB(r9) 1648 std r7, VCPU_FSCR(r9) 1649 mfspr r5, SPRN_IC 1650 mfspr r7, SPRN_TAR 1651 std r5, VCPU_IC(r9) 1652 std r7, VCPU_TAR(r9) 1653 mfspr r8, SPRN_EBBHR 1654 std r8, VCPU_EBBHR(r9) 1655 mfspr r5, SPRN_EBBRR 1656 mfspr r6, SPRN_BESCR 1657 mfspr r7, SPRN_PID 1658 mfspr r8, SPRN_WORT 1659 std r5, VCPU_EBBRR(r9) 1660 std r6, VCPU_BESCR(r9) 1661 stw r7, VCPU_GUEST_PID(r9) 1662 std r8, VCPU_WORT(r9) 1663BEGIN_FTR_SECTION 1664 mfspr r5, SPRN_TCSCR 1665 mfspr r6, SPRN_ACOP 1666 mfspr r7, SPRN_CSIGR 1667 mfspr r8, SPRN_TACR 1668 std r5, VCPU_TCSCR(r9) 1669 std r6, VCPU_ACOP(r9) 1670 std r7, VCPU_CSIGR(r9) 1671 std r8, VCPU_TACR(r9) 1672FTR_SECTION_ELSE 1673 mfspr r5, SPRN_TIDR 1674 mfspr r6, SPRN_PSSCR 1675 std r5, VCPU_TID(r9) 1676 rldicl r6, r6, 4, 50 /* r6 &= PSSCR_GUEST_VIS */ 1677 rotldi r6, r6, 60 1678 std r6, VCPU_PSSCR(r9) 1679 /* Restore host HFSCR value */ 1680 ld r7, STACK_SLOT_HFSCR(r1) 1681 mtspr SPRN_HFSCR, r7 1682ALT_FTR_SECTION_END_IFCLR(CPU_FTR_ARCH_300) 1683 /* 1684 * Restore various registers to 0, where non-zero values 1685 * set by the guest could disrupt the host. 1686 */ 1687 li r0, 0 1688 mtspr SPRN_PSPB, r0 1689 mtspr SPRN_WORT, r0 1690BEGIN_FTR_SECTION 1691 mtspr SPRN_IAMR, r0 1692 mtspr SPRN_TCSCR, r0 1693 /* Set MMCRS to 1<<31 to freeze and disable the SPMC counters */ 1694 li r0, 1 1695 sldi r0, r0, 31 1696 mtspr SPRN_MMCRS, r0 1697END_FTR_SECTION_IFCLR(CPU_FTR_ARCH_300) 16988: 1699 1700 /* Save and reset AMR and UAMOR before turning on the MMU */ 1701 mfspr r5,SPRN_AMR 1702 mfspr r6,SPRN_UAMOR 1703 std r5,VCPU_AMR(r9) 1704 std r6,VCPU_UAMOR(r9) 1705 li r6,0 1706 mtspr SPRN_AMR,r6 1707 mtspr SPRN_UAMOR, r6 1708 1709 /* Switch DSCR back to host value */ 1710 mfspr r8, SPRN_DSCR 1711 ld r7, HSTATE_DSCR(r13) 1712 std r8, VCPU_DSCR(r9) 1713 mtspr SPRN_DSCR, r7 1714 1715 /* Save non-volatile GPRs */ 1716 std r14, VCPU_GPR(R14)(r9) 1717 std r15, VCPU_GPR(R15)(r9) 1718 std r16, VCPU_GPR(R16)(r9) 1719 std r17, VCPU_GPR(R17)(r9) 1720 std r18, VCPU_GPR(R18)(r9) 1721 std r19, VCPU_GPR(R19)(r9) 1722 std r20, VCPU_GPR(R20)(r9) 1723 std r21, VCPU_GPR(R21)(r9) 1724 std r22, VCPU_GPR(R22)(r9) 1725 std r23, VCPU_GPR(R23)(r9) 1726 std r24, VCPU_GPR(R24)(r9) 1727 std r25, VCPU_GPR(R25)(r9) 1728 std r26, VCPU_GPR(R26)(r9) 1729 std r27, VCPU_GPR(R27)(r9) 1730 std r28, VCPU_GPR(R28)(r9) 1731 std r29, VCPU_GPR(R29)(r9) 1732 std r30, VCPU_GPR(R30)(r9) 1733 std r31, VCPU_GPR(R31)(r9) 1734 1735 /* Save SPRGs */ 1736 mfspr r3, SPRN_SPRG0 1737 mfspr r4, SPRN_SPRG1 1738 mfspr r5, SPRN_SPRG2 1739 mfspr r6, SPRN_SPRG3 1740 std r3, VCPU_SPRG0(r9) 1741 std r4, VCPU_SPRG1(r9) 1742 std r5, VCPU_SPRG2(r9) 1743 std r6, VCPU_SPRG3(r9) 1744 1745 /* save FP state */ 1746 mr r3, r9 1747 bl kvmppc_save_fp 1748 1749#ifdef CONFIG_PPC_TRANSACTIONAL_MEM 1750BEGIN_FTR_SECTION 1751 /* 1752 * NOTE THAT THIS TRASHES ALL NON-VOLATILE REGISTERS INCLUDING CR 1753 */ 1754 bl kvmppc_save_tm 1755END_FTR_SECTION_IFSET(CPU_FTR_TM) 1756#endif 1757 1758 /* Increment yield count if they have a VPA */ 1759 ld r8, VCPU_VPA(r9) /* do they have a VPA? */ 1760 cmpdi r8, 0 1761 beq 25f 1762 li r4, LPPACA_YIELDCOUNT 1763 LWZX_BE r3, r8, r4 1764 addi r3, r3, 1 1765 STWX_BE r3, r8, r4 1766 li r3, 1 1767 stb r3, VCPU_VPA_DIRTY(r9) 176825: 1769 /* Save PMU registers if requested */ 1770 /* r8 and cr0.eq are live here */ 1771BEGIN_FTR_SECTION 1772 /* 1773 * POWER8 seems to have a hardware bug where setting 1774 * MMCR0[PMAE] along with MMCR0[PMC1CE] and/or MMCR0[PMCjCE] 1775 * when some counters are already negative doesn't seem 1776 * to cause a performance monitor alert (and hence interrupt). 1777 * The effect of this is that when saving the PMU state, 1778 * if there is no PMU alert pending when we read MMCR0 1779 * before freezing the counters, but one becomes pending 1780 * before we read the counters, we lose it. 1781 * To work around this, we need a way to freeze the counters 1782 * before reading MMCR0. Normally, freezing the counters 1783 * is done by writing MMCR0 (to set MMCR0[FC]) which 1784 * unavoidably writes MMCR0[PMA0] as well. On POWER8, 1785 * we can also freeze the counters using MMCR2, by writing 1786 * 1s to all the counter freeze condition bits (there are 1787 * 9 bits each for 6 counters). 1788 */ 1789 li r3, -1 /* set all freeze bits */ 1790 clrrdi r3, r3, 10 1791 mfspr r10, SPRN_MMCR2 1792 mtspr SPRN_MMCR2, r3 1793 isync 1794END_FTR_SECTION_IFSET(CPU_FTR_ARCH_207S) 1795 li r3, 1 1796 sldi r3, r3, 31 /* MMCR0_FC (freeze counters) bit */ 1797 mfspr r4, SPRN_MMCR0 /* save MMCR0 */ 1798 mtspr SPRN_MMCR0, r3 /* freeze all counters, disable ints */ 1799 mfspr r6, SPRN_MMCRA 1800 /* Clear MMCRA in order to disable SDAR updates */ 1801 li r7, 0 1802 mtspr SPRN_MMCRA, r7 1803 isync 1804 beq 21f /* if no VPA, save PMU stuff anyway */ 1805 lbz r7, LPPACA_PMCINUSE(r8) 1806 cmpwi r7, 0 /* did they ask for PMU stuff to be saved? */ 1807 bne 21f 1808 std r3, VCPU_MMCR(r9) /* if not, set saved MMCR0 to FC */ 1809 b 22f 181021: mfspr r5, SPRN_MMCR1 1811 mfspr r7, SPRN_SIAR 1812 mfspr r8, SPRN_SDAR 1813 std r4, VCPU_MMCR(r9) 1814 std r5, VCPU_MMCR + 8(r9) 1815 std r6, VCPU_MMCR + 16(r9) 1816BEGIN_FTR_SECTION 1817 std r10, VCPU_MMCR + 24(r9) 1818END_FTR_SECTION_IFSET(CPU_FTR_ARCH_207S) 1819 std r7, VCPU_SIAR(r9) 1820 std r8, VCPU_SDAR(r9) 1821 mfspr r3, SPRN_PMC1 1822 mfspr r4, SPRN_PMC2 1823 mfspr r5, SPRN_PMC3 1824 mfspr r6, SPRN_PMC4 1825 mfspr r7, SPRN_PMC5 1826 mfspr r8, SPRN_PMC6 1827 stw r3, VCPU_PMC(r9) 1828 stw r4, VCPU_PMC + 4(r9) 1829 stw r5, VCPU_PMC + 8(r9) 1830 stw r6, VCPU_PMC + 12(r9) 1831 stw r7, VCPU_PMC + 16(r9) 1832 stw r8, VCPU_PMC + 20(r9) 1833BEGIN_FTR_SECTION 1834 mfspr r5, SPRN_SIER 1835 std r5, VCPU_SIER(r9) 1836BEGIN_FTR_SECTION_NESTED(96) 1837 mfspr r6, SPRN_SPMC1 1838 mfspr r7, SPRN_SPMC2 1839 mfspr r8, SPRN_MMCRS 1840 stw r6, VCPU_PMC + 24(r9) 1841 stw r7, VCPU_PMC + 28(r9) 1842 std r8, VCPU_MMCR + 32(r9) 1843 lis r4, 0x8000 1844 mtspr SPRN_MMCRS, r4 1845END_FTR_SECTION_NESTED(CPU_FTR_ARCH_300, 0, 96) 1846END_FTR_SECTION_IFSET(CPU_FTR_ARCH_207S) 184722: 1848 1849 /* Restore host values of some registers */ 1850BEGIN_FTR_SECTION 1851 ld r5, STACK_SLOT_CIABR(r1) 1852 ld r6, STACK_SLOT_DAWR(r1) 1853 ld r7, STACK_SLOT_DAWRX(r1) 1854 mtspr SPRN_CIABR, r5 1855 mtspr SPRN_DAWR, r6 1856 mtspr SPRN_DAWRX, r7 1857END_FTR_SECTION_IFSET(CPU_FTR_ARCH_207S) 1858BEGIN_FTR_SECTION 1859 ld r5, STACK_SLOT_TID(r1) 1860 ld r6, STACK_SLOT_PSSCR(r1) 1861 ld r7, STACK_SLOT_PID(r1) 1862 ld r8, STACK_SLOT_IAMR(r1) 1863 mtspr SPRN_TIDR, r5 1864 mtspr SPRN_PSSCR, r6 1865 mtspr SPRN_PID, r7 1866 mtspr SPRN_IAMR, r8 1867END_FTR_SECTION_IFSET(CPU_FTR_ARCH_300) 1868 1869#ifdef CONFIG_PPC_RADIX_MMU 1870 /* 1871 * Are we running hash or radix ? 1872 */ 1873 ld r5, VCPU_KVM(r9) 1874 lbz r0, KVM_RADIX(r5) 1875 cmpwi cr2, r0, 0 1876 beq cr2, 4f 1877 1878 /* Radix: Handle the case where the guest used an illegal PID */ 1879 LOAD_REG_ADDR(r4, mmu_base_pid) 1880 lwz r3, VCPU_GUEST_PID(r9) 1881 lwz r5, 0(r4) 1882 cmpw cr0,r3,r5 1883 blt 2f 1884 1885 /* 1886 * Illegal PID, the HW might have prefetched and cached in the TLB 1887 * some translations for the LPID 0 / guest PID combination which 1888 * Linux doesn't know about, so we need to flush that PID out of 1889 * the TLB. First we need to set LPIDR to 0 so tlbiel applies to 1890 * the right context. 1891 */ 1892 li r0,0 1893 mtspr SPRN_LPID,r0 1894 isync 1895 1896 /* Then do a congruence class local flush */ 1897 ld r6,VCPU_KVM(r9) 1898 lwz r0,KVM_TLB_SETS(r6) 1899 mtctr r0 1900 li r7,0x400 /* IS field = 0b01 */ 1901 ptesync 1902 sldi r0,r3,32 /* RS has PID */ 19031: PPC_TLBIEL(7,0,2,1,1) /* RIC=2, PRS=1, R=1 */ 1904 addi r7,r7,0x1000 1905 bdnz 1b 1906 ptesync 1907 19082: /* Flush the ERAT on radix P9 DD1 guest exit */ 1909BEGIN_FTR_SECTION 1910 PPC_INVALIDATE_ERAT 1911END_FTR_SECTION_IFSET(CPU_FTR_POWER9_DD1) 19124: 1913#endif /* CONFIG_PPC_RADIX_MMU */ 1914 1915 /* 1916 * POWER7/POWER8 guest -> host partition switch code. 1917 * We don't have to lock against tlbies but we do 1918 * have to coordinate the hardware threads. 1919 * Here STACK_SLOT_TRAP(r1) contains the trap number. 1920 */ 1921kvmhv_switch_to_host: 1922 /* Secondary threads wait for primary to do partition switch */ 1923 ld r5,HSTATE_KVM_VCORE(r13) 1924 ld r4,VCORE_KVM(r5) /* pointer to struct kvm */ 1925 lbz r3,HSTATE_PTID(r13) 1926 cmpwi r3,0 1927 beq 15f 1928 HMT_LOW 192913: lbz r3,VCORE_IN_GUEST(r5) 1930 cmpwi r3,0 1931 bne 13b 1932 HMT_MEDIUM 1933 b 16f 1934 1935 /* Primary thread waits for all the secondaries to exit guest */ 193615: lwz r3,VCORE_ENTRY_EXIT(r5) 1937 rlwinm r0,r3,32-8,0xff 1938 clrldi r3,r3,56 1939 cmpw r3,r0 1940 bne 15b 1941 isync 1942 1943 /* Did we actually switch to the guest at all? */ 1944 lbz r6, VCORE_IN_GUEST(r5) 1945 cmpwi r6, 0 1946 beq 19f 1947 1948 /* Primary thread switches back to host partition */ 1949 lwz r7,KVM_HOST_LPID(r4) 1950BEGIN_FTR_SECTION 1951 ld r6,KVM_HOST_SDR1(r4) 1952 li r8,LPID_RSVD /* switch to reserved LPID */ 1953 mtspr SPRN_LPID,r8 1954 ptesync 1955 mtspr SPRN_SDR1,r6 /* switch to host page table */ 1956END_FTR_SECTION_IFCLR(CPU_FTR_ARCH_300) 1957 mtspr SPRN_LPID,r7 1958 isync 1959 1960BEGIN_FTR_SECTION 1961 /* DPDES and VTB are shared between threads */ 1962 mfspr r7, SPRN_DPDES 1963 mfspr r8, SPRN_VTB 1964 std r7, VCORE_DPDES(r5) 1965 std r8, VCORE_VTB(r5) 1966 /* clear DPDES so we don't get guest doorbells in the host */ 1967 li r8, 0 1968 mtspr SPRN_DPDES, r8 1969END_FTR_SECTION_IFSET(CPU_FTR_ARCH_207S) 1970 1971 /* If HMI, call kvmppc_realmode_hmi_handler() */ 1972 lwz r12, STACK_SLOT_TRAP(r1) 1973 cmpwi r12, BOOK3S_INTERRUPT_HMI 1974 bne 27f 1975 bl kvmppc_realmode_hmi_handler 1976 nop 1977 cmpdi r3, 0 1978 /* 1979 * At this point kvmppc_realmode_hmi_handler may have resync-ed 1980 * the TB, and if it has, we must not subtract the guest timebase 1981 * offset from the timebase. So, skip it. 1982 * 1983 * Also, do not call kvmppc_subcore_exit_guest() because it has 1984 * been invoked as part of kvmppc_realmode_hmi_handler(). 1985 */ 1986 beq 30f 1987 198827: 1989 /* Subtract timebase offset from timebase */ 1990 ld r8,VCORE_TB_OFFSET(r5) 1991 cmpdi r8,0 1992 beq 17f 1993 mftb r6 /* current guest timebase */ 1994 subf r8,r8,r6 1995 mtspr SPRN_TBU40,r8 /* update upper 40 bits */ 1996 mftb r7 /* check if lower 24 bits overflowed */ 1997 clrldi r6,r6,40 1998 clrldi r7,r7,40 1999 cmpld r7,r6 2000 bge 17f 2001 addis r8,r8,0x100 /* if so, increment upper 40 bits */ 2002 mtspr SPRN_TBU40,r8 2003 200417: bl kvmppc_subcore_exit_guest 2005 nop 200630: ld r5,HSTATE_KVM_VCORE(r13) 2007 ld r4,VCORE_KVM(r5) /* pointer to struct kvm */ 2008 2009 /* Reset PCR */ 2010 ld r0, VCORE_PCR(r5) 2011 cmpdi r0, 0 2012 beq 18f 2013 li r0, 0 2014 mtspr SPRN_PCR, r0 201518: 2016 /* Signal secondary CPUs to continue */ 2017 stb r0,VCORE_IN_GUEST(r5) 201819: lis r8,0x7fff /* MAX_INT@h */ 2019 mtspr SPRN_HDEC,r8 2020 202116: 2022BEGIN_FTR_SECTION 2023 /* On POWER9 with HPT-on-radix we need to wait for all other threads */ 2024 ld r3, HSTATE_SPLIT_MODE(r13) 2025 cmpdi r3, 0 2026 beq 47f 2027 lwz r8, KVM_SPLIT_DO_RESTORE(r3) 2028 cmpwi r8, 0 2029 beq 47f 2030 bl kvmhv_p9_restore_lpcr 2031 nop 2032 b 48f 203347: 2034END_FTR_SECTION_IFSET(CPU_FTR_ARCH_300) 2035 ld r8,KVM_HOST_LPCR(r4) 2036 mtspr SPRN_LPCR,r8 2037 isync 203848: 2039#ifdef CONFIG_KVM_BOOK3S_HV_EXIT_TIMING 2040 /* Finish timing, if we have a vcpu */ 2041 ld r4, HSTATE_KVM_VCPU(r13) 2042 cmpdi r4, 0 2043 li r3, 0 2044 beq 2f 2045 bl kvmhv_accumulate_time 20462: 2047#endif 2048 /* Unset guest mode */ 2049 li r0, KVM_GUEST_MODE_NONE 2050 stb r0, HSTATE_IN_GUEST(r13) 2051 2052 lwz r12, STACK_SLOT_TRAP(r1) /* return trap # in r12 */ 2053 ld r0, SFS+PPC_LR_STKOFF(r1) 2054 addi r1, r1, SFS 2055 mtlr r0 2056 blr 2057 2058/* 2059 * Check whether an HDSI is an HPTE not found fault or something else. 2060 * If it is an HPTE not found fault that is due to the guest accessing 2061 * a page that they have mapped but which we have paged out, then 2062 * we continue on with the guest exit path. In all other cases, 2063 * reflect the HDSI to the guest as a DSI. 2064 */ 2065kvmppc_hdsi: 2066 ld r3, VCPU_KVM(r9) 2067 lbz r0, KVM_RADIX(r3) 2068 mfspr r4, SPRN_HDAR 2069 mfspr r6, SPRN_HDSISR 2070BEGIN_FTR_SECTION 2071 /* Look for DSISR canary. If we find it, retry instruction */ 2072 cmpdi r6, 0x7fff 2073 beq 6f 2074END_FTR_SECTION_IFSET(CPU_FTR_ARCH_300) 2075 cmpwi r0, 0 2076 bne .Lradix_hdsi /* on radix, just save DAR/DSISR/ASDR */ 2077 /* HPTE not found fault or protection fault? */ 2078 andis. r0, r6, (DSISR_NOHPTE | DSISR_PROTFAULT)@h 2079 beq 1f /* if not, send it to the guest */ 2080 andi. r0, r11, MSR_DR /* data relocation enabled? */ 2081 beq 3f 2082BEGIN_FTR_SECTION 2083 mfspr r5, SPRN_ASDR /* on POWER9, use ASDR to get VSID */ 2084 b 4f 2085END_FTR_SECTION_IFSET(CPU_FTR_ARCH_300) 2086 clrrdi r0, r4, 28 2087 PPC_SLBFEE_DOT(R5, R0) /* if so, look up SLB */ 2088 li r0, BOOK3S_INTERRUPT_DATA_SEGMENT 2089 bne 7f /* if no SLB entry found */ 20904: std r4, VCPU_FAULT_DAR(r9) 2091 stw r6, VCPU_FAULT_DSISR(r9) 2092 2093 /* Search the hash table. */ 2094 mr r3, r9 /* vcpu pointer */ 2095 li r7, 1 /* data fault */ 2096 bl kvmppc_hpte_hv_fault 2097 ld r9, HSTATE_KVM_VCPU(r13) 2098 ld r10, VCPU_PC(r9) 2099 ld r11, VCPU_MSR(r9) 2100 li r12, BOOK3S_INTERRUPT_H_DATA_STORAGE 2101 cmpdi r3, 0 /* retry the instruction */ 2102 beq 6f 2103 cmpdi r3, -1 /* handle in kernel mode */ 2104 beq guest_exit_cont 2105 cmpdi r3, -2 /* MMIO emulation; need instr word */ 2106 beq 2f 2107 2108 /* Synthesize a DSI (or DSegI) for the guest */ 2109 ld r4, VCPU_FAULT_DAR(r9) 2110 mr r6, r3 21111: li r0, BOOK3S_INTERRUPT_DATA_STORAGE 2112 mtspr SPRN_DSISR, r6 21137: mtspr SPRN_DAR, r4 2114 mtspr SPRN_SRR0, r10 2115 mtspr SPRN_SRR1, r11 2116 mr r10, r0 2117 bl kvmppc_msr_interrupt 2118fast_interrupt_c_return: 21196: ld r7, VCPU_CTR(r9) 2120 ld r8, VCPU_XER(r9) 2121 mtctr r7 2122 mtxer r8 2123 mr r4, r9 2124 b fast_guest_return 2125 21263: ld r5, VCPU_KVM(r9) /* not relocated, use VRMA */ 2127 ld r5, KVM_VRMA_SLB_V(r5) 2128 b 4b 2129 2130 /* If this is for emulated MMIO, load the instruction word */ 21312: li r8, KVM_INST_FETCH_FAILED /* In case lwz faults */ 2132 2133 /* Set guest mode to 'jump over instruction' so if lwz faults 2134 * we'll just continue at the next IP. */ 2135 li r0, KVM_GUEST_MODE_SKIP 2136 stb r0, HSTATE_IN_GUEST(r13) 2137 2138 /* Do the access with MSR:DR enabled */ 2139 mfmsr r3 2140 ori r4, r3, MSR_DR /* Enable paging for data */ 2141 mtmsrd r4 2142 lwz r8, 0(r10) 2143 mtmsrd r3 2144 2145 /* Store the result */ 2146 stw r8, VCPU_LAST_INST(r9) 2147 2148 /* Unset guest mode. */ 2149 li r0, KVM_GUEST_MODE_HOST_HV 2150 stb r0, HSTATE_IN_GUEST(r13) 2151 b guest_exit_cont 2152 2153.Lradix_hdsi: 2154 std r4, VCPU_FAULT_DAR(r9) 2155 stw r6, VCPU_FAULT_DSISR(r9) 2156.Lradix_hisi: 2157 mfspr r5, SPRN_ASDR 2158 std r5, VCPU_FAULT_GPA(r9) 2159 b guest_exit_cont 2160 2161/* 2162 * Similarly for an HISI, reflect it to the guest as an ISI unless 2163 * it is an HPTE not found fault for a page that we have paged out. 2164 */ 2165kvmppc_hisi: 2166 ld r3, VCPU_KVM(r9) 2167 lbz r0, KVM_RADIX(r3) 2168 cmpwi r0, 0 2169 bne .Lradix_hisi /* for radix, just save ASDR */ 2170 andis. r0, r11, SRR1_ISI_NOPT@h 2171 beq 1f 2172 andi. r0, r11, MSR_IR /* instruction relocation enabled? */ 2173 beq 3f 2174BEGIN_FTR_SECTION 2175 mfspr r5, SPRN_ASDR /* on POWER9, use ASDR to get VSID */ 2176 b 4f 2177END_FTR_SECTION_IFSET(CPU_FTR_ARCH_300) 2178 clrrdi r0, r10, 28 2179 PPC_SLBFEE_DOT(R5, R0) /* if so, look up SLB */ 2180 li r0, BOOK3S_INTERRUPT_INST_SEGMENT 2181 bne 7f /* if no SLB entry found */ 21824: 2183 /* Search the hash table. */ 2184 mr r3, r9 /* vcpu pointer */ 2185 mr r4, r10 2186 mr r6, r11 2187 li r7, 0 /* instruction fault */ 2188 bl kvmppc_hpte_hv_fault 2189 ld r9, HSTATE_KVM_VCPU(r13) 2190 ld r10, VCPU_PC(r9) 2191 ld r11, VCPU_MSR(r9) 2192 li r12, BOOK3S_INTERRUPT_H_INST_STORAGE 2193 cmpdi r3, 0 /* retry the instruction */ 2194 beq fast_interrupt_c_return 2195 cmpdi r3, -1 /* handle in kernel mode */ 2196 beq guest_exit_cont 2197 2198 /* Synthesize an ISI (or ISegI) for the guest */ 2199 mr r11, r3 22001: li r0, BOOK3S_INTERRUPT_INST_STORAGE 22017: mtspr SPRN_SRR0, r10 2202 mtspr SPRN_SRR1, r11 2203 mr r10, r0 2204 bl kvmppc_msr_interrupt 2205 b fast_interrupt_c_return 2206 22073: ld r6, VCPU_KVM(r9) /* not relocated, use VRMA */ 2208 ld r5, KVM_VRMA_SLB_V(r6) 2209 b 4b 2210 2211/* 2212 * Try to handle an hcall in real mode. 2213 * Returns to the guest if we handle it, or continues on up to 2214 * the kernel if we can't (i.e. if we don't have a handler for 2215 * it, or if the handler returns H_TOO_HARD). 2216 * 2217 * r5 - r8 contain hcall args, 2218 * r9 = vcpu, r10 = pc, r11 = msr, r12 = trap, r13 = paca 2219 */ 2220hcall_try_real_mode: 2221 ld r3,VCPU_GPR(R3)(r9) 2222 andi. r0,r11,MSR_PR 2223 /* sc 1 from userspace - reflect to guest syscall */ 2224 bne sc_1_fast_return 2225 clrrdi r3,r3,2 2226 cmpldi r3,hcall_real_table_end - hcall_real_table 2227 bge guest_exit_cont 2228 /* See if this hcall is enabled for in-kernel handling */ 2229 ld r4, VCPU_KVM(r9) 2230 srdi r0, r3, 8 /* r0 = (r3 / 4) >> 6 */ 2231 sldi r0, r0, 3 /* index into kvm->arch.enabled_hcalls[] */ 2232 add r4, r4, r0 2233 ld r0, KVM_ENABLED_HCALLS(r4) 2234 rlwinm r4, r3, 32-2, 0x3f /* r4 = (r3 / 4) & 0x3f */ 2235 srd r0, r0, r4 2236 andi. r0, r0, 1 2237 beq guest_exit_cont 2238 /* Get pointer to handler, if any, and call it */ 2239 LOAD_REG_ADDR(r4, hcall_real_table) 2240 lwax r3,r3,r4 2241 cmpwi r3,0 2242 beq guest_exit_cont 2243 add r12,r3,r4 2244 mtctr r12 2245 mr r3,r9 /* get vcpu pointer */ 2246 ld r4,VCPU_GPR(R4)(r9) 2247 bctrl 2248 cmpdi r3,H_TOO_HARD 2249 beq hcall_real_fallback 2250 ld r4,HSTATE_KVM_VCPU(r13) 2251 std r3,VCPU_GPR(R3)(r4) 2252 ld r10,VCPU_PC(r4) 2253 ld r11,VCPU_MSR(r4) 2254 b fast_guest_return 2255 2256sc_1_fast_return: 2257 mtspr SPRN_SRR0,r10 2258 mtspr SPRN_SRR1,r11 2259 li r10, BOOK3S_INTERRUPT_SYSCALL 2260 bl kvmppc_msr_interrupt 2261 mr r4,r9 2262 b fast_guest_return 2263 2264 /* We've attempted a real mode hcall, but it's punted it back 2265 * to userspace. We need to restore some clobbered volatiles 2266 * before resuming the pass-it-to-qemu path */ 2267hcall_real_fallback: 2268 li r12,BOOK3S_INTERRUPT_SYSCALL 2269 ld r9, HSTATE_KVM_VCPU(r13) 2270 2271 b guest_exit_cont 2272 2273 .globl hcall_real_table 2274hcall_real_table: 2275 .long 0 /* 0 - unused */ 2276 .long DOTSYM(kvmppc_h_remove) - hcall_real_table 2277 .long DOTSYM(kvmppc_h_enter) - hcall_real_table 2278 .long DOTSYM(kvmppc_h_read) - hcall_real_table 2279 .long DOTSYM(kvmppc_h_clear_mod) - hcall_real_table 2280 .long DOTSYM(kvmppc_h_clear_ref) - hcall_real_table 2281 .long DOTSYM(kvmppc_h_protect) - hcall_real_table 2282 .long DOTSYM(kvmppc_h_get_tce) - hcall_real_table 2283 .long DOTSYM(kvmppc_rm_h_put_tce) - hcall_real_table 2284 .long 0 /* 0x24 - H_SET_SPRG0 */ 2285 .long DOTSYM(kvmppc_h_set_dabr) - hcall_real_table 2286 .long 0 /* 0x2c */ 2287 .long 0 /* 0x30 */ 2288 .long 0 /* 0x34 */ 2289 .long 0 /* 0x38 */ 2290 .long 0 /* 0x3c */ 2291 .long 0 /* 0x40 */ 2292 .long 0 /* 0x44 */ 2293 .long 0 /* 0x48 */ 2294 .long 0 /* 0x4c */ 2295 .long 0 /* 0x50 */ 2296 .long 0 /* 0x54 */ 2297 .long 0 /* 0x58 */ 2298 .long 0 /* 0x5c */ 2299 .long 0 /* 0x60 */ 2300#ifdef CONFIG_KVM_XICS 2301 .long DOTSYM(kvmppc_rm_h_eoi) - hcall_real_table 2302 .long DOTSYM(kvmppc_rm_h_cppr) - hcall_real_table 2303 .long DOTSYM(kvmppc_rm_h_ipi) - hcall_real_table 2304 .long DOTSYM(kvmppc_rm_h_ipoll) - hcall_real_table 2305 .long DOTSYM(kvmppc_rm_h_xirr) - hcall_real_table 2306#else 2307 .long 0 /* 0x64 - H_EOI */ 2308 .long 0 /* 0x68 - H_CPPR */ 2309 .long 0 /* 0x6c - H_IPI */ 2310 .long 0 /* 0x70 - H_IPOLL */ 2311 .long 0 /* 0x74 - H_XIRR */ 2312#endif 2313 .long 0 /* 0x78 */ 2314 .long 0 /* 0x7c */ 2315 .long 0 /* 0x80 */ 2316 .long 0 /* 0x84 */ 2317 .long 0 /* 0x88 */ 2318 .long 0 /* 0x8c */ 2319 .long 0 /* 0x90 */ 2320 .long 0 /* 0x94 */ 2321 .long 0 /* 0x98 */ 2322 .long 0 /* 0x9c */ 2323 .long 0 /* 0xa0 */ 2324 .long 0 /* 0xa4 */ 2325 .long 0 /* 0xa8 */ 2326 .long 0 /* 0xac */ 2327 .long 0 /* 0xb0 */ 2328 .long 0 /* 0xb4 */ 2329 .long 0 /* 0xb8 */ 2330 .long 0 /* 0xbc */ 2331 .long 0 /* 0xc0 */ 2332 .long 0 /* 0xc4 */ 2333 .long 0 /* 0xc8 */ 2334 .long 0 /* 0xcc */ 2335 .long 0 /* 0xd0 */ 2336 .long 0 /* 0xd4 */ 2337 .long 0 /* 0xd8 */ 2338 .long 0 /* 0xdc */ 2339 .long DOTSYM(kvmppc_h_cede) - hcall_real_table 2340 .long DOTSYM(kvmppc_rm_h_confer) - hcall_real_table 2341 .long 0 /* 0xe8 */ 2342 .long 0 /* 0xec */ 2343 .long 0 /* 0xf0 */ 2344 .long 0 /* 0xf4 */ 2345 .long 0 /* 0xf8 */ 2346 .long 0 /* 0xfc */ 2347 .long 0 /* 0x100 */ 2348 .long 0 /* 0x104 */ 2349 .long 0 /* 0x108 */ 2350 .long 0 /* 0x10c */ 2351 .long 0 /* 0x110 */ 2352 .long 0 /* 0x114 */ 2353 .long 0 /* 0x118 */ 2354 .long 0 /* 0x11c */ 2355 .long 0 /* 0x120 */ 2356 .long DOTSYM(kvmppc_h_bulk_remove) - hcall_real_table 2357 .long 0 /* 0x128 */ 2358 .long 0 /* 0x12c */ 2359 .long 0 /* 0x130 */ 2360 .long DOTSYM(kvmppc_h_set_xdabr) - hcall_real_table 2361 .long DOTSYM(kvmppc_rm_h_stuff_tce) - hcall_real_table 2362 .long DOTSYM(kvmppc_rm_h_put_tce_indirect) - hcall_real_table 2363 .long 0 /* 0x140 */ 2364 .long 0 /* 0x144 */ 2365 .long 0 /* 0x148 */ 2366 .long 0 /* 0x14c */ 2367 .long 0 /* 0x150 */ 2368 .long 0 /* 0x154 */ 2369 .long 0 /* 0x158 */ 2370 .long 0 /* 0x15c */ 2371 .long 0 /* 0x160 */ 2372 .long 0 /* 0x164 */ 2373 .long 0 /* 0x168 */ 2374 .long 0 /* 0x16c */ 2375 .long 0 /* 0x170 */ 2376 .long 0 /* 0x174 */ 2377 .long 0 /* 0x178 */ 2378 .long 0 /* 0x17c */ 2379 .long 0 /* 0x180 */ 2380 .long 0 /* 0x184 */ 2381 .long 0 /* 0x188 */ 2382 .long 0 /* 0x18c */ 2383 .long 0 /* 0x190 */ 2384 .long 0 /* 0x194 */ 2385 .long 0 /* 0x198 */ 2386 .long 0 /* 0x19c */ 2387 .long 0 /* 0x1a0 */ 2388 .long 0 /* 0x1a4 */ 2389 .long 0 /* 0x1a8 */ 2390 .long 0 /* 0x1ac */ 2391 .long 0 /* 0x1b0 */ 2392 .long 0 /* 0x1b4 */ 2393 .long 0 /* 0x1b8 */ 2394 .long 0 /* 0x1bc */ 2395 .long 0 /* 0x1c0 */ 2396 .long 0 /* 0x1c4 */ 2397 .long 0 /* 0x1c8 */ 2398 .long 0 /* 0x1cc */ 2399 .long 0 /* 0x1d0 */ 2400 .long 0 /* 0x1d4 */ 2401 .long 0 /* 0x1d8 */ 2402 .long 0 /* 0x1dc */ 2403 .long 0 /* 0x1e0 */ 2404 .long 0 /* 0x1e4 */ 2405 .long 0 /* 0x1e8 */ 2406 .long 0 /* 0x1ec */ 2407 .long 0 /* 0x1f0 */ 2408 .long 0 /* 0x1f4 */ 2409 .long 0 /* 0x1f8 */ 2410 .long 0 /* 0x1fc */ 2411 .long 0 /* 0x200 */ 2412 .long 0 /* 0x204 */ 2413 .long 0 /* 0x208 */ 2414 .long 0 /* 0x20c */ 2415 .long 0 /* 0x210 */ 2416 .long 0 /* 0x214 */ 2417 .long 0 /* 0x218 */ 2418 .long 0 /* 0x21c */ 2419 .long 0 /* 0x220 */ 2420 .long 0 /* 0x224 */ 2421 .long 0 /* 0x228 */ 2422 .long 0 /* 0x22c */ 2423 .long 0 /* 0x230 */ 2424 .long 0 /* 0x234 */ 2425 .long 0 /* 0x238 */ 2426 .long 0 /* 0x23c */ 2427 .long 0 /* 0x240 */ 2428 .long 0 /* 0x244 */ 2429 .long 0 /* 0x248 */ 2430 .long 0 /* 0x24c */ 2431 .long 0 /* 0x250 */ 2432 .long 0 /* 0x254 */ 2433 .long 0 /* 0x258 */ 2434 .long 0 /* 0x25c */ 2435 .long 0 /* 0x260 */ 2436 .long 0 /* 0x264 */ 2437 .long 0 /* 0x268 */ 2438 .long 0 /* 0x26c */ 2439 .long 0 /* 0x270 */ 2440 .long 0 /* 0x274 */ 2441 .long 0 /* 0x278 */ 2442 .long 0 /* 0x27c */ 2443 .long 0 /* 0x280 */ 2444 .long 0 /* 0x284 */ 2445 .long 0 /* 0x288 */ 2446 .long 0 /* 0x28c */ 2447 .long 0 /* 0x290 */ 2448 .long 0 /* 0x294 */ 2449 .long 0 /* 0x298 */ 2450 .long 0 /* 0x29c */ 2451 .long 0 /* 0x2a0 */ 2452 .long 0 /* 0x2a4 */ 2453 .long 0 /* 0x2a8 */ 2454 .long 0 /* 0x2ac */ 2455 .long 0 /* 0x2b0 */ 2456 .long 0 /* 0x2b4 */ 2457 .long 0 /* 0x2b8 */ 2458 .long 0 /* 0x2bc */ 2459 .long 0 /* 0x2c0 */ 2460 .long 0 /* 0x2c4 */ 2461 .long 0 /* 0x2c8 */ 2462 .long 0 /* 0x2cc */ 2463 .long 0 /* 0x2d0 */ 2464 .long 0 /* 0x2d4 */ 2465 .long 0 /* 0x2d8 */ 2466 .long 0 /* 0x2dc */ 2467 .long 0 /* 0x2e0 */ 2468 .long 0 /* 0x2e4 */ 2469 .long 0 /* 0x2e8 */ 2470 .long 0 /* 0x2ec */ 2471 .long 0 /* 0x2f0 */ 2472 .long 0 /* 0x2f4 */ 2473 .long 0 /* 0x2f8 */ 2474#ifdef CONFIG_KVM_XICS 2475 .long DOTSYM(kvmppc_rm_h_xirr_x) - hcall_real_table 2476#else 2477 .long 0 /* 0x2fc - H_XIRR_X*/ 2478#endif 2479 .long DOTSYM(kvmppc_h_random) - hcall_real_table 2480 .globl hcall_real_table_end 2481hcall_real_table_end: 2482 2483_GLOBAL(kvmppc_h_set_xdabr) 2484 andi. r0, r5, DABRX_USER | DABRX_KERNEL 2485 beq 6f 2486 li r0, DABRX_USER | DABRX_KERNEL | DABRX_BTI 2487 andc. r0, r5, r0 2488 beq 3f 24896: li r3, H_PARAMETER 2490 blr 2491 2492_GLOBAL(kvmppc_h_set_dabr) 2493 li r5, DABRX_USER | DABRX_KERNEL 24943: 2495BEGIN_FTR_SECTION 2496 b 2f 2497END_FTR_SECTION_IFSET(CPU_FTR_ARCH_207S) 2498 std r4,VCPU_DABR(r3) 2499 stw r5, VCPU_DABRX(r3) 2500 mtspr SPRN_DABRX, r5 2501 /* Work around P7 bug where DABR can get corrupted on mtspr */ 25021: mtspr SPRN_DABR,r4 2503 mfspr r5, SPRN_DABR 2504 cmpd r4, r5 2505 bne 1b 2506 isync 2507 li r3,0 2508 blr 2509 2510 /* Emulate H_SET_DABR/X on P8 for the sake of compat mode guests */ 25112: rlwimi r5, r4, 5, DAWRX_DR | DAWRX_DW 2512 rlwimi r5, r4, 2, DAWRX_WT 2513 clrrdi r4, r4, 3 2514 std r4, VCPU_DAWR(r3) 2515 std r5, VCPU_DAWRX(r3) 2516 mtspr SPRN_DAWR, r4 2517 mtspr SPRN_DAWRX, r5 2518 li r3, 0 2519 blr 2520 2521_GLOBAL(kvmppc_h_cede) /* r3 = vcpu pointer, r11 = msr, r13 = paca */ 2522 ori r11,r11,MSR_EE 2523 std r11,VCPU_MSR(r3) 2524 li r0,1 2525 stb r0,VCPU_CEDED(r3) 2526 sync /* order setting ceded vs. testing prodded */ 2527 lbz r5,VCPU_PRODDED(r3) 2528 cmpwi r5,0 2529 bne kvm_cede_prodded 2530 li r12,0 /* set trap to 0 to say hcall is handled */ 2531 stw r12,VCPU_TRAP(r3) 2532 li r0,H_SUCCESS 2533 std r0,VCPU_GPR(R3)(r3) 2534 2535 /* 2536 * Set our bit in the bitmask of napping threads unless all the 2537 * other threads are already napping, in which case we send this 2538 * up to the host. 2539 */ 2540 ld r5,HSTATE_KVM_VCORE(r13) 2541 lbz r6,HSTATE_PTID(r13) 2542 lwz r8,VCORE_ENTRY_EXIT(r5) 2543 clrldi r8,r8,56 2544 li r0,1 2545 sld r0,r0,r6 2546 addi r6,r5,VCORE_NAPPING_THREADS 254731: lwarx r4,0,r6 2548 or r4,r4,r0 2549 cmpw r4,r8 2550 beq kvm_cede_exit 2551 stwcx. r4,0,r6 2552 bne 31b 2553 /* order napping_threads update vs testing entry_exit_map */ 2554 isync 2555 li r0,NAPPING_CEDE 2556 stb r0,HSTATE_NAPPING(r13) 2557 lwz r7,VCORE_ENTRY_EXIT(r5) 2558 cmpwi r7,0x100 2559 bge 33f /* another thread already exiting */ 2560 2561/* 2562 * Although not specifically required by the architecture, POWER7 2563 * preserves the following registers in nap mode, even if an SMT mode 2564 * switch occurs: SLB entries, PURR, SPURR, AMOR, UAMOR, AMR, SPRG0-3, 2565 * DAR, DSISR, DABR, DABRX, DSCR, PMCx, MMCRx, SIAR, SDAR. 2566 */ 2567 /* Save non-volatile GPRs */ 2568 std r14, VCPU_GPR(R14)(r3) 2569 std r15, VCPU_GPR(R15)(r3) 2570 std r16, VCPU_GPR(R16)(r3) 2571 std r17, VCPU_GPR(R17)(r3) 2572 std r18, VCPU_GPR(R18)(r3) 2573 std r19, VCPU_GPR(R19)(r3) 2574 std r20, VCPU_GPR(R20)(r3) 2575 std r21, VCPU_GPR(R21)(r3) 2576 std r22, VCPU_GPR(R22)(r3) 2577 std r23, VCPU_GPR(R23)(r3) 2578 std r24, VCPU_GPR(R24)(r3) 2579 std r25, VCPU_GPR(R25)(r3) 2580 std r26, VCPU_GPR(R26)(r3) 2581 std r27, VCPU_GPR(R27)(r3) 2582 std r28, VCPU_GPR(R28)(r3) 2583 std r29, VCPU_GPR(R29)(r3) 2584 std r30, VCPU_GPR(R30)(r3) 2585 std r31, VCPU_GPR(R31)(r3) 2586 2587 /* save FP state */ 2588 bl kvmppc_save_fp 2589 2590#ifdef CONFIG_PPC_TRANSACTIONAL_MEM 2591BEGIN_FTR_SECTION 2592 /* 2593 * NOTE THAT THIS TRASHES ALL NON-VOLATILE REGISTERS INCLUDING CR 2594 */ 2595 ld r9, HSTATE_KVM_VCPU(r13) 2596 bl kvmppc_save_tm 2597END_FTR_SECTION_IFSET(CPU_FTR_TM) 2598#endif 2599 2600 /* 2601 * Set DEC to the smaller of DEC and HDEC, so that we wake 2602 * no later than the end of our timeslice (HDEC interrupts 2603 * don't wake us from nap). 2604 */ 2605 mfspr r3, SPRN_DEC 2606 mfspr r4, SPRN_HDEC 2607 mftb r5 2608BEGIN_FTR_SECTION 2609 /* On P9 check whether the guest has large decrementer mode enabled */ 2610 ld r6, HSTATE_KVM_VCORE(r13) 2611 ld r6, VCORE_LPCR(r6) 2612 andis. r6, r6, LPCR_LD@h 2613 bne 68f 2614END_FTR_SECTION_IFSET(CPU_FTR_ARCH_300) 2615 extsw r3, r3 261668: EXTEND_HDEC(r4) 2617 cmpd r3, r4 2618 ble 67f 2619 mtspr SPRN_DEC, r4 262067: 2621 /* save expiry time of guest decrementer */ 2622 add r3, r3, r5 2623 ld r4, HSTATE_KVM_VCPU(r13) 2624 ld r5, HSTATE_KVM_VCORE(r13) 2625 ld r6, VCORE_TB_OFFSET(r5) 2626 subf r3, r6, r3 /* convert to host TB value */ 2627 std r3, VCPU_DEC_EXPIRES(r4) 2628 2629#ifdef CONFIG_KVM_BOOK3S_HV_EXIT_TIMING 2630 ld r4, HSTATE_KVM_VCPU(r13) 2631 addi r3, r4, VCPU_TB_CEDE 2632 bl kvmhv_accumulate_time 2633#endif 2634 2635 lis r3, LPCR_PECEDP@h /* Do wake on privileged doorbell */ 2636 2637 /* 2638 * Take a nap until a decrementer or external or doobell interrupt 2639 * occurs, with PECE1 and PECE0 set in LPCR. 2640 * On POWER8, set PECEDH, and if we are ceding, also set PECEDP. 2641 * Also clear the runlatch bit before napping. 2642 */ 2643kvm_do_nap: 2644 mfspr r0, SPRN_CTRLF 2645 clrrdi r0, r0, 1 2646 mtspr SPRN_CTRLT, r0 2647 2648 li r0,1 2649 stb r0,HSTATE_HWTHREAD_REQ(r13) 2650 mfspr r5,SPRN_LPCR 2651 ori r5,r5,LPCR_PECE0 | LPCR_PECE1 2652BEGIN_FTR_SECTION 2653 ori r5, r5, LPCR_PECEDH 2654 rlwimi r5, r3, 0, LPCR_PECEDP 2655END_FTR_SECTION_IFSET(CPU_FTR_ARCH_207S) 2656 2657kvm_nap_sequence: /* desired LPCR value in r5 */ 2658BEGIN_FTR_SECTION 2659 /* 2660 * PSSCR bits: exit criterion = 1 (wakeup based on LPCR at sreset) 2661 * enable state loss = 1 (allow SMT mode switch) 2662 * requested level = 0 (just stop dispatching) 2663 */ 2664 lis r3, (PSSCR_EC | PSSCR_ESL)@h 2665 mtspr SPRN_PSSCR, r3 2666 /* Set LPCR_PECE_HVEE bit to enable wakeup by HV interrupts */ 2667 li r4, LPCR_PECE_HVEE@higher 2668 sldi r4, r4, 32 2669 or r5, r5, r4 2670END_FTR_SECTION_IFSET(CPU_FTR_ARCH_300) 2671 mtspr SPRN_LPCR,r5 2672 isync 2673 li r0, 0 2674 std r0, HSTATE_SCRATCH0(r13) 2675 ptesync 2676 ld r0, HSTATE_SCRATCH0(r13) 26771: cmpd r0, r0 2678 bne 1b 2679BEGIN_FTR_SECTION 2680 nap 2681FTR_SECTION_ELSE 2682 PPC_STOP 2683ALT_FTR_SECTION_END_IFCLR(CPU_FTR_ARCH_300) 2684 b . 2685 268633: mr r4, r3 2687 li r3, 0 2688 li r12, 0 2689 b 34f 2690 2691kvm_end_cede: 2692 /* get vcpu pointer */ 2693 ld r4, HSTATE_KVM_VCPU(r13) 2694 2695 /* Woken by external or decrementer interrupt */ 2696 ld r1, HSTATE_HOST_R1(r13) 2697 2698#ifdef CONFIG_KVM_BOOK3S_HV_EXIT_TIMING 2699 addi r3, r4, VCPU_TB_RMINTR 2700 bl kvmhv_accumulate_time 2701#endif 2702 2703#ifdef CONFIG_PPC_TRANSACTIONAL_MEM 2704BEGIN_FTR_SECTION 2705 /* 2706 * NOTE THAT THIS TRASHES ALL NON-VOLATILE REGISTERS INCLUDING CR 2707 */ 2708 bl kvmppc_restore_tm 2709END_FTR_SECTION_IFSET(CPU_FTR_TM) 2710#endif 2711 2712 /* load up FP state */ 2713 bl kvmppc_load_fp 2714 2715 /* Restore guest decrementer */ 2716 ld r3, VCPU_DEC_EXPIRES(r4) 2717 ld r5, HSTATE_KVM_VCORE(r13) 2718 ld r6, VCORE_TB_OFFSET(r5) 2719 add r3, r3, r6 /* convert host TB to guest TB value */ 2720 mftb r7 2721 subf r3, r7, r3 2722 mtspr SPRN_DEC, r3 2723 2724 /* Load NV GPRS */ 2725 ld r14, VCPU_GPR(R14)(r4) 2726 ld r15, VCPU_GPR(R15)(r4) 2727 ld r16, VCPU_GPR(R16)(r4) 2728 ld r17, VCPU_GPR(R17)(r4) 2729 ld r18, VCPU_GPR(R18)(r4) 2730 ld r19, VCPU_GPR(R19)(r4) 2731 ld r20, VCPU_GPR(R20)(r4) 2732 ld r21, VCPU_GPR(R21)(r4) 2733 ld r22, VCPU_GPR(R22)(r4) 2734 ld r23, VCPU_GPR(R23)(r4) 2735 ld r24, VCPU_GPR(R24)(r4) 2736 ld r25, VCPU_GPR(R25)(r4) 2737 ld r26, VCPU_GPR(R26)(r4) 2738 ld r27, VCPU_GPR(R27)(r4) 2739 ld r28, VCPU_GPR(R28)(r4) 2740 ld r29, VCPU_GPR(R29)(r4) 2741 ld r30, VCPU_GPR(R30)(r4) 2742 ld r31, VCPU_GPR(R31)(r4) 2743 2744 /* Check the wake reason in SRR1 to see why we got here */ 2745 bl kvmppc_check_wake_reason 2746 2747 /* 2748 * Restore volatile registers since we could have called a 2749 * C routine in kvmppc_check_wake_reason 2750 * r4 = VCPU 2751 * r3 tells us whether we need to return to host or not 2752 * WARNING: it gets checked further down: 2753 * should not modify r3 until this check is done. 2754 */ 2755 ld r4, HSTATE_KVM_VCPU(r13) 2756 2757 /* clear our bit in vcore->napping_threads */ 275834: ld r5,HSTATE_KVM_VCORE(r13) 2759 lbz r7,HSTATE_PTID(r13) 2760 li r0,1 2761 sld r0,r0,r7 2762 addi r6,r5,VCORE_NAPPING_THREADS 276332: lwarx r7,0,r6 2764 andc r7,r7,r0 2765 stwcx. r7,0,r6 2766 bne 32b 2767 li r0,0 2768 stb r0,HSTATE_NAPPING(r13) 2769 2770 /* See if the wake reason saved in r3 means we need to exit */ 2771 stw r12, VCPU_TRAP(r4) 2772 mr r9, r4 2773 cmpdi r3, 0 2774 bgt guest_exit_cont 2775 2776 /* see if any other thread is already exiting */ 2777 lwz r0,VCORE_ENTRY_EXIT(r5) 2778 cmpwi r0,0x100 2779 bge guest_exit_cont 2780 2781 b kvmppc_cede_reentry /* if not go back to guest */ 2782 2783 /* cede when already previously prodded case */ 2784kvm_cede_prodded: 2785 li r0,0 2786 stb r0,VCPU_PRODDED(r3) 2787 sync /* order testing prodded vs. clearing ceded */ 2788 stb r0,VCPU_CEDED(r3) 2789 li r3,H_SUCCESS 2790 blr 2791 2792 /* we've ceded but we want to give control to the host */ 2793kvm_cede_exit: 2794 ld r9, HSTATE_KVM_VCPU(r13) 2795#ifdef CONFIG_KVM_XICS 2796 /* Abort if we still have a pending escalation */ 2797 lbz r5, VCPU_XIVE_ESC_ON(r9) 2798 cmpwi r5, 0 2799 beq 1f 2800 li r0, 0 2801 stb r0, VCPU_CEDED(r9) 28021: /* Enable XIVE escalation */ 2803 li r5, XIVE_ESB_SET_PQ_00 2804 mfmsr r0 2805 andi. r0, r0, MSR_DR /* in real mode? */ 2806 beq 1f 2807 ld r10, VCPU_XIVE_ESC_VADDR(r9) 2808 cmpdi r10, 0 2809 beq 3f 2810 ldx r0, r10, r5 2811 b 2f 28121: ld r10, VCPU_XIVE_ESC_RADDR(r9) 2813 cmpdi r10, 0 2814 beq 3f 2815 ldcix r0, r10, r5 28162: sync 2817 li r0, 1 2818 stb r0, VCPU_XIVE_ESC_ON(r9) 2819#endif /* CONFIG_KVM_XICS */ 28203: b guest_exit_cont 2821 2822 /* Try to handle a machine check in real mode */ 2823machine_check_realmode: 2824 mr r3, r9 /* get vcpu pointer */ 2825 bl kvmppc_realmode_machine_check 2826 nop 2827 ld r9, HSTATE_KVM_VCPU(r13) 2828 li r12, BOOK3S_INTERRUPT_MACHINE_CHECK 2829 /* 2830 * For the guest that is FWNMI capable, deliver all the MCE errors 2831 * (handled/unhandled) by exiting the guest with KVM_EXIT_NMI exit 2832 * reason. This new approach injects machine check errors in guest 2833 * address space to guest with additional information in the form 2834 * of RTAS event, thus enabling guest kernel to suitably handle 2835 * such errors. 2836 * 2837 * For the guest that is not FWNMI capable (old QEMU) fallback 2838 * to old behaviour for backward compatibility: 2839 * Deliver unhandled/fatal (e.g. UE) MCE errors to guest either 2840 * through machine check interrupt (set HSRR0 to 0x200). 2841 * For handled errors (no-fatal), just go back to guest execution 2842 * with current HSRR0. 2843 * if we receive machine check with MSR(RI=0) then deliver it to 2844 * guest as machine check causing guest to crash. 2845 */ 2846 ld r11, VCPU_MSR(r9) 2847 rldicl. r0, r11, 64-MSR_HV_LG, 63 /* check if it happened in HV mode */ 2848 bne mc_cont /* if so, exit to host */ 2849 /* Check if guest is capable of handling NMI exit */ 2850 ld r10, VCPU_KVM(r9) 2851 lbz r10, KVM_FWNMI(r10) 2852 cmpdi r10, 1 /* FWNMI capable? */ 2853 beq mc_cont /* if so, exit with KVM_EXIT_NMI. */ 2854 2855 /* if not, fall through for backward compatibility. */ 2856 andi. r10, r11, MSR_RI /* check for unrecoverable exception */ 2857 beq 1f /* Deliver a machine check to guest */ 2858 ld r10, VCPU_PC(r9) 2859 cmpdi r3, 0 /* Did we handle MCE ? */ 2860 bne 2f /* Continue guest execution. */ 2861 /* If not, deliver a machine check. SRR0/1 are already set */ 28621: li r10, BOOK3S_INTERRUPT_MACHINE_CHECK 2863 bl kvmppc_msr_interrupt 28642: b fast_interrupt_c_return 2865 2866/* 2867 * Check the reason we woke from nap, and take appropriate action. 2868 * Returns (in r3): 2869 * 0 if nothing needs to be done 2870 * 1 if something happened that needs to be handled by the host 2871 * -1 if there was a guest wakeup (IPI or msgsnd) 2872 * -2 if we handled a PCI passthrough interrupt (returned by 2873 * kvmppc_read_intr only) 2874 * 2875 * Also sets r12 to the interrupt vector for any interrupt that needs 2876 * to be handled now by the host (0x500 for external interrupt), or zero. 2877 * Modifies all volatile registers (since it may call a C function). 2878 * This routine calls kvmppc_read_intr, a C function, if an external 2879 * interrupt is pending. 2880 */ 2881kvmppc_check_wake_reason: 2882 mfspr r6, SPRN_SRR1 2883BEGIN_FTR_SECTION 2884 rlwinm r6, r6, 45-31, 0xf /* extract wake reason field (P8) */ 2885FTR_SECTION_ELSE 2886 rlwinm r6, r6, 45-31, 0xe /* P7 wake reason field is 3 bits */ 2887ALT_FTR_SECTION_END_IFSET(CPU_FTR_ARCH_207S) 2888 cmpwi r6, 8 /* was it an external interrupt? */ 2889 beq 7f /* if so, see what it was */ 2890 li r3, 0 2891 li r12, 0 2892 cmpwi r6, 6 /* was it the decrementer? */ 2893 beq 0f 2894BEGIN_FTR_SECTION 2895 cmpwi r6, 5 /* privileged doorbell? */ 2896 beq 0f 2897 cmpwi r6, 3 /* hypervisor doorbell? */ 2898 beq 3f 2899END_FTR_SECTION_IFSET(CPU_FTR_ARCH_207S) 2900 cmpwi r6, 0xa /* Hypervisor maintenance ? */ 2901 beq 4f 2902 li r3, 1 /* anything else, return 1 */ 29030: blr 2904 2905 /* hypervisor doorbell */ 29063: li r12, BOOK3S_INTERRUPT_H_DOORBELL 2907 2908 /* 2909 * Clear the doorbell as we will invoke the handler 2910 * explicitly in the guest exit path. 2911 */ 2912 lis r6, (PPC_DBELL_SERVER << (63-36))@h 2913 PPC_MSGCLR(6) 2914 /* see if it's a host IPI */ 2915 li r3, 1 2916BEGIN_FTR_SECTION 2917 PPC_MSGSYNC 2918 lwsync 2919END_FTR_SECTION_IFSET(CPU_FTR_ARCH_300) 2920 lbz r0, HSTATE_HOST_IPI(r13) 2921 cmpwi r0, 0 2922 bnelr 2923 /* if not, return -1 */ 2924 li r3, -1 2925 blr 2926 2927 /* Woken up due to Hypervisor maintenance interrupt */ 29284: li r12, BOOK3S_INTERRUPT_HMI 2929 li r3, 1 2930 blr 2931 2932 /* external interrupt - create a stack frame so we can call C */ 29337: mflr r0 2934 std r0, PPC_LR_STKOFF(r1) 2935 stdu r1, -PPC_MIN_STKFRM(r1) 2936 bl kvmppc_read_intr 2937 nop 2938 li r12, BOOK3S_INTERRUPT_EXTERNAL 2939 cmpdi r3, 1 2940 ble 1f 2941 2942 /* 2943 * Return code of 2 means PCI passthrough interrupt, but 2944 * we need to return back to host to complete handling the 2945 * interrupt. Trap reason is expected in r12 by guest 2946 * exit code. 2947 */ 2948 li r12, BOOK3S_INTERRUPT_HV_RM_HARD 29491: 2950 ld r0, PPC_MIN_STKFRM+PPC_LR_STKOFF(r1) 2951 addi r1, r1, PPC_MIN_STKFRM 2952 mtlr r0 2953 blr 2954 2955/* 2956 * Save away FP, VMX and VSX registers. 2957 * r3 = vcpu pointer 2958 * N.B. r30 and r31 are volatile across this function, 2959 * thus it is not callable from C. 2960 */ 2961kvmppc_save_fp: 2962 mflr r30 2963 mr r31,r3 2964 mfmsr r5 2965 ori r8,r5,MSR_FP 2966#ifdef CONFIG_ALTIVEC 2967BEGIN_FTR_SECTION 2968 oris r8,r8,MSR_VEC@h 2969END_FTR_SECTION_IFSET(CPU_FTR_ALTIVEC) 2970#endif 2971#ifdef CONFIG_VSX 2972BEGIN_FTR_SECTION 2973 oris r8,r8,MSR_VSX@h 2974END_FTR_SECTION_IFSET(CPU_FTR_VSX) 2975#endif 2976 mtmsrd r8 2977 addi r3,r3,VCPU_FPRS 2978 bl store_fp_state 2979#ifdef CONFIG_ALTIVEC 2980BEGIN_FTR_SECTION 2981 addi r3,r31,VCPU_VRS 2982 bl store_vr_state 2983END_FTR_SECTION_IFSET(CPU_FTR_ALTIVEC) 2984#endif 2985 mfspr r6,SPRN_VRSAVE 2986 stw r6,VCPU_VRSAVE(r31) 2987 mtlr r30 2988 blr 2989 2990/* 2991 * Load up FP, VMX and VSX registers 2992 * r4 = vcpu pointer 2993 * N.B. r30 and r31 are volatile across this function, 2994 * thus it is not callable from C. 2995 */ 2996kvmppc_load_fp: 2997 mflr r30 2998 mr r31,r4 2999 mfmsr r9 3000 ori r8,r9,MSR_FP 3001#ifdef CONFIG_ALTIVEC 3002BEGIN_FTR_SECTION 3003 oris r8,r8,MSR_VEC@h 3004END_FTR_SECTION_IFSET(CPU_FTR_ALTIVEC) 3005#endif 3006#ifdef CONFIG_VSX 3007BEGIN_FTR_SECTION 3008 oris r8,r8,MSR_VSX@h 3009END_FTR_SECTION_IFSET(CPU_FTR_VSX) 3010#endif 3011 mtmsrd r8 3012 addi r3,r4,VCPU_FPRS 3013 bl load_fp_state 3014#ifdef CONFIG_ALTIVEC 3015BEGIN_FTR_SECTION 3016 addi r3,r31,VCPU_VRS 3017 bl load_vr_state 3018END_FTR_SECTION_IFSET(CPU_FTR_ALTIVEC) 3019#endif 3020 lwz r7,VCPU_VRSAVE(r31) 3021 mtspr SPRN_VRSAVE,r7 3022 mtlr r30 3023 mr r4,r31 3024 blr 3025 3026#ifdef CONFIG_PPC_TRANSACTIONAL_MEM 3027/* 3028 * Save transactional state and TM-related registers. 3029 * Called with r9 pointing to the vcpu struct. 3030 * This can modify all checkpointed registers, but 3031 * restores r1, r2 and r9 (vcpu pointer) before exit. 3032 */ 3033kvmppc_save_tm: 3034 mflr r0 3035 std r0, PPC_LR_STKOFF(r1) 3036 3037 /* Turn on TM. */ 3038 mfmsr r8 3039 li r0, 1 3040 rldimi r8, r0, MSR_TM_LG, 63-MSR_TM_LG 3041 mtmsrd r8 3042 3043 ld r5, VCPU_MSR(r9) 3044 rldicl. r5, r5, 64 - MSR_TS_S_LG, 62 3045 beq 1f /* TM not active in guest. */ 3046 3047 std r1, HSTATE_HOST_R1(r13) 3048 li r3, TM_CAUSE_KVM_RESCHED 3049 3050 /* Clear the MSR RI since r1, r13 are all going to be foobar. */ 3051 li r5, 0 3052 mtmsrd r5, 1 3053 3054 /* All GPRs are volatile at this point. */ 3055 TRECLAIM(R3) 3056 3057 /* Temporarily store r13 and r9 so we have some regs to play with */ 3058 SET_SCRATCH0(r13) 3059 GET_PACA(r13) 3060 std r9, PACATMSCRATCH(r13) 3061 ld r9, HSTATE_KVM_VCPU(r13) 3062 3063 /* Get a few more GPRs free. */ 3064 std r29, VCPU_GPRS_TM(29)(r9) 3065 std r30, VCPU_GPRS_TM(30)(r9) 3066 std r31, VCPU_GPRS_TM(31)(r9) 3067 3068 /* Save away PPR and DSCR soon so don't run with user values. */ 3069 mfspr r31, SPRN_PPR 3070 HMT_MEDIUM 3071 mfspr r30, SPRN_DSCR 3072 ld r29, HSTATE_DSCR(r13) 3073 mtspr SPRN_DSCR, r29 3074 3075 /* Save all but r9, r13 & r29-r31 */ 3076 reg = 0 3077 .rept 29 3078 .if (reg != 9) && (reg != 13) 3079 std reg, VCPU_GPRS_TM(reg)(r9) 3080 .endif 3081 reg = reg + 1 3082 .endr 3083 /* ... now save r13 */ 3084 GET_SCRATCH0(r4) 3085 std r4, VCPU_GPRS_TM(13)(r9) 3086 /* ... and save r9 */ 3087 ld r4, PACATMSCRATCH(r13) 3088 std r4, VCPU_GPRS_TM(9)(r9) 3089 3090 /* Reload stack pointer and TOC. */ 3091 ld r1, HSTATE_HOST_R1(r13) 3092 ld r2, PACATOC(r13) 3093 3094 /* Set MSR RI now we have r1 and r13 back. */ 3095 li r5, MSR_RI 3096 mtmsrd r5, 1 3097 3098 /* Save away checkpinted SPRs. */ 3099 std r31, VCPU_PPR_TM(r9) 3100 std r30, VCPU_DSCR_TM(r9) 3101 mflr r5 3102 mfcr r6 3103 mfctr r7 3104 mfspr r8, SPRN_AMR 3105 mfspr r10, SPRN_TAR 3106 mfxer r11 3107 std r5, VCPU_LR_TM(r9) 3108 stw r6, VCPU_CR_TM(r9) 3109 std r7, VCPU_CTR_TM(r9) 3110 std r8, VCPU_AMR_TM(r9) 3111 std r10, VCPU_TAR_TM(r9) 3112 std r11, VCPU_XER_TM(r9) 3113 3114 /* Restore r12 as trap number. */ 3115 lwz r12, VCPU_TRAP(r9) 3116 3117 /* Save FP/VSX. */ 3118 addi r3, r9, VCPU_FPRS_TM 3119 bl store_fp_state 3120 addi r3, r9, VCPU_VRS_TM 3121 bl store_vr_state 3122 mfspr r6, SPRN_VRSAVE 3123 stw r6, VCPU_VRSAVE_TM(r9) 31241: 3125 /* 3126 * We need to save these SPRs after the treclaim so that the software 3127 * error code is recorded correctly in the TEXASR. Also the user may 3128 * change these outside of a transaction, so they must always be 3129 * context switched. 3130 */ 3131 mfspr r5, SPRN_TFHAR 3132 mfspr r6, SPRN_TFIAR 3133 mfspr r7, SPRN_TEXASR 3134 std r5, VCPU_TFHAR(r9) 3135 std r6, VCPU_TFIAR(r9) 3136 std r7, VCPU_TEXASR(r9) 3137 3138 ld r0, PPC_LR_STKOFF(r1) 3139 mtlr r0 3140 blr 3141 3142/* 3143 * Restore transactional state and TM-related registers. 3144 * Called with r4 pointing to the vcpu struct. 3145 * This potentially modifies all checkpointed registers. 3146 * It restores r1, r2, r4 from the PACA. 3147 */ 3148kvmppc_restore_tm: 3149 mflr r0 3150 std r0, PPC_LR_STKOFF(r1) 3151 3152 /* Turn on TM/FP/VSX/VMX so we can restore them. */ 3153 mfmsr r5 3154 li r6, MSR_TM >> 32 3155 sldi r6, r6, 32 3156 or r5, r5, r6 3157 ori r5, r5, MSR_FP 3158 oris r5, r5, (MSR_VEC | MSR_VSX)@h 3159 mtmsrd r5 3160 3161 /* 3162 * The user may change these outside of a transaction, so they must 3163 * always be context switched. 3164 */ 3165 ld r5, VCPU_TFHAR(r4) 3166 ld r6, VCPU_TFIAR(r4) 3167 ld r7, VCPU_TEXASR(r4) 3168 mtspr SPRN_TFHAR, r5 3169 mtspr SPRN_TFIAR, r6 3170 mtspr SPRN_TEXASR, r7 3171 3172 ld r5, VCPU_MSR(r4) 3173 rldicl. r5, r5, 64 - MSR_TS_S_LG, 62 3174 beqlr /* TM not active in guest */ 3175 std r1, HSTATE_HOST_R1(r13) 3176 3177 /* Make sure the failure summary is set, otherwise we'll program check 3178 * when we trechkpt. It's possible that this might have been not set 3179 * on a kvmppc_set_one_reg() call but we shouldn't let this crash the 3180 * host. 3181 */ 3182 oris r7, r7, (TEXASR_FS)@h 3183 mtspr SPRN_TEXASR, r7 3184 3185 /* 3186 * We need to load up the checkpointed state for the guest. 3187 * We need to do this early as it will blow away any GPRs, VSRs and 3188 * some SPRs. 3189 */ 3190 3191 mr r31, r4 3192 addi r3, r31, VCPU_FPRS_TM 3193 bl load_fp_state 3194 addi r3, r31, VCPU_VRS_TM 3195 bl load_vr_state 3196 mr r4, r31 3197 lwz r7, VCPU_VRSAVE_TM(r4) 3198 mtspr SPRN_VRSAVE, r7 3199 3200 ld r5, VCPU_LR_TM(r4) 3201 lwz r6, VCPU_CR_TM(r4) 3202 ld r7, VCPU_CTR_TM(r4) 3203 ld r8, VCPU_AMR_TM(r4) 3204 ld r9, VCPU_TAR_TM(r4) 3205 ld r10, VCPU_XER_TM(r4) 3206 mtlr r5 3207 mtcr r6 3208 mtctr r7 3209 mtspr SPRN_AMR, r8 3210 mtspr SPRN_TAR, r9 3211 mtxer r10 3212 3213 /* 3214 * Load up PPR and DSCR values but don't put them in the actual SPRs 3215 * till the last moment to avoid running with userspace PPR and DSCR for 3216 * too long. 3217 */ 3218 ld r29, VCPU_DSCR_TM(r4) 3219 ld r30, VCPU_PPR_TM(r4) 3220 3221 std r2, PACATMSCRATCH(r13) /* Save TOC */ 3222 3223 /* Clear the MSR RI since r1, r13 are all going to be foobar. */ 3224 li r5, 0 3225 mtmsrd r5, 1 3226 3227 /* Load GPRs r0-r28 */ 3228 reg = 0 3229 .rept 29 3230 ld reg, VCPU_GPRS_TM(reg)(r31) 3231 reg = reg + 1 3232 .endr 3233 3234 mtspr SPRN_DSCR, r29 3235 mtspr SPRN_PPR, r30 3236 3237 /* Load final GPRs */ 3238 ld 29, VCPU_GPRS_TM(29)(r31) 3239 ld 30, VCPU_GPRS_TM(30)(r31) 3240 ld 31, VCPU_GPRS_TM(31)(r31) 3241 3242 /* TM checkpointed state is now setup. All GPRs are now volatile. */ 3243 TRECHKPT 3244 3245 /* Now let's get back the state we need. */ 3246 HMT_MEDIUM 3247 GET_PACA(r13) 3248 ld r29, HSTATE_DSCR(r13) 3249 mtspr SPRN_DSCR, r29 3250 ld r4, HSTATE_KVM_VCPU(r13) 3251 ld r1, HSTATE_HOST_R1(r13) 3252 ld r2, PACATMSCRATCH(r13) 3253 3254 /* Set the MSR RI since we have our registers back. */ 3255 li r5, MSR_RI 3256 mtmsrd r5, 1 3257 3258 ld r0, PPC_LR_STKOFF(r1) 3259 mtlr r0 3260 blr 3261#endif 3262 3263/* 3264 * We come here if we get any exception or interrupt while we are 3265 * executing host real mode code while in guest MMU context. 3266 * r12 is (CR << 32) | vector 3267 * r13 points to our PACA 3268 * r12 is saved in HSTATE_SCRATCH0(r13) 3269 * ctr is saved in HSTATE_SCRATCH1(r13) if RELOCATABLE 3270 * r9 is saved in HSTATE_SCRATCH2(r13) 3271 * r13 is saved in HSPRG1 3272 * cfar is saved in HSTATE_CFAR(r13) 3273 * ppr is saved in HSTATE_PPR(r13) 3274 */ 3275kvmppc_bad_host_intr: 3276 /* 3277 * Switch to the emergency stack, but start half-way down in 3278 * case we were already on it. 3279 */ 3280 mr r9, r1 3281 std r1, PACAR1(r13) 3282 ld r1, PACAEMERGSP(r13) 3283 subi r1, r1, THREAD_SIZE/2 + INT_FRAME_SIZE 3284 std r9, 0(r1) 3285 std r0, GPR0(r1) 3286 std r9, GPR1(r1) 3287 std r2, GPR2(r1) 3288 SAVE_4GPRS(3, r1) 3289 SAVE_2GPRS(7, r1) 3290 srdi r0, r12, 32 3291 clrldi r12, r12, 32 3292 std r0, _CCR(r1) 3293 std r12, _TRAP(r1) 3294 andi. r0, r12, 2 3295 beq 1f 3296 mfspr r3, SPRN_HSRR0 3297 mfspr r4, SPRN_HSRR1 3298 mfspr r5, SPRN_HDAR 3299 mfspr r6, SPRN_HDSISR 3300 b 2f 33011: mfspr r3, SPRN_SRR0 3302 mfspr r4, SPRN_SRR1 3303 mfspr r5, SPRN_DAR 3304 mfspr r6, SPRN_DSISR 33052: std r3, _NIP(r1) 3306 std r4, _MSR(r1) 3307 std r5, _DAR(r1) 3308 std r6, _DSISR(r1) 3309 ld r9, HSTATE_SCRATCH2(r13) 3310 ld r12, HSTATE_SCRATCH0(r13) 3311 GET_SCRATCH0(r0) 3312 SAVE_4GPRS(9, r1) 3313 std r0, GPR13(r1) 3314 SAVE_NVGPRS(r1) 3315 ld r5, HSTATE_CFAR(r13) 3316 std r5, ORIG_GPR3(r1) 3317 mflr r3 3318#ifdef CONFIG_RELOCATABLE 3319 ld r4, HSTATE_SCRATCH1(r13) 3320#else 3321 mfctr r4 3322#endif 3323 mfxer r5 3324 lbz r6, PACAIRQSOFTMASK(r13) 3325 std r3, _LINK(r1) 3326 std r4, _CTR(r1) 3327 std r5, _XER(r1) 3328 std r6, SOFTE(r1) 3329 ld r2, PACATOC(r13) 3330 LOAD_REG_IMMEDIATE(3, 0x7265677368657265) 3331 std r3, STACK_FRAME_OVERHEAD-16(r1) 3332 3333 /* 3334 * On POWER9 do a minimal restore of the MMU and call C code, 3335 * which will print a message and panic. 3336 * XXX On POWER7 and POWER8, we just spin here since we don't 3337 * know what the other threads are doing (and we don't want to 3338 * coordinate with them) - but at least we now have register state 3339 * in memory that we might be able to look at from another CPU. 3340 */ 3341BEGIN_FTR_SECTION 3342 b . 3343END_FTR_SECTION_IFCLR(CPU_FTR_ARCH_300) 3344 ld r9, HSTATE_KVM_VCPU(r13) 3345 ld r10, VCPU_KVM(r9) 3346 3347 li r0, 0 3348 mtspr SPRN_AMR, r0 3349 mtspr SPRN_IAMR, r0 3350 mtspr SPRN_CIABR, r0 3351 mtspr SPRN_DAWRX, r0 3352 3353 /* Flush the ERAT on radix P9 DD1 guest exit */ 3354BEGIN_FTR_SECTION 3355 PPC_INVALIDATE_ERAT 3356END_FTR_SECTION_IFSET(CPU_FTR_POWER9_DD1) 3357 3358BEGIN_MMU_FTR_SECTION 3359 b 4f 3360END_MMU_FTR_SECTION_IFSET(MMU_FTR_TYPE_RADIX) 3361 3362 slbmte r0, r0 3363 slbia 3364 ptesync 3365 ld r8, PACA_SLBSHADOWPTR(r13) 3366 .rept SLB_NUM_BOLTED 3367 li r3, SLBSHADOW_SAVEAREA 3368 LDX_BE r5, r8, r3 3369 addi r3, r3, 8 3370 LDX_BE r6, r8, r3 3371 andis. r7, r5, SLB_ESID_V@h 3372 beq 3f 3373 slbmte r6, r5 33743: addi r8, r8, 16 3375 .endr 3376 33774: lwz r7, KVM_HOST_LPID(r10) 3378 mtspr SPRN_LPID, r7 3379 mtspr SPRN_PID, r0 3380 ld r8, KVM_HOST_LPCR(r10) 3381 mtspr SPRN_LPCR, r8 3382 isync 3383 li r0, KVM_GUEST_MODE_NONE 3384 stb r0, HSTATE_IN_GUEST(r13) 3385 3386 /* 3387 * Turn on the MMU and jump to C code 3388 */ 3389 bcl 20, 31, .+4 33905: mflr r3 3391 addi r3, r3, 9f - 5b 3392 ld r4, PACAKMSR(r13) 3393 mtspr SPRN_SRR0, r3 3394 mtspr SPRN_SRR1, r4 3395 RFI_TO_KERNEL 33969: addi r3, r1, STACK_FRAME_OVERHEAD 3397 bl kvmppc_bad_interrupt 3398 b 9b 3399 3400/* 3401 * This mimics the MSR transition on IRQ delivery. The new guest MSR is taken 3402 * from VCPU_INTR_MSR and is modified based on the required TM state changes. 3403 * r11 has the guest MSR value (in/out) 3404 * r9 has a vcpu pointer (in) 3405 * r0 is used as a scratch register 3406 */ 3407kvmppc_msr_interrupt: 3408 rldicl r0, r11, 64 - MSR_TS_S_LG, 62 3409 cmpwi r0, 2 /* Check if we are in transactional state.. */ 3410 ld r11, VCPU_INTR_MSR(r9) 3411 bne 1f 3412 /* ... if transactional, change to suspended */ 3413 li r0, 1 34141: rldimi r11, r0, MSR_TS_S_LG, 63 - MSR_TS_T_LG 3415 blr 3416 3417/* 3418 * This works around a hardware bug on POWER8E processors, where 3419 * writing a 1 to the MMCR0[PMAO] bit doesn't generate a 3420 * performance monitor interrupt. Instead, when we need to have 3421 * an interrupt pending, we have to arrange for a counter to overflow. 3422 */ 3423kvmppc_fix_pmao: 3424 li r3, 0 3425 mtspr SPRN_MMCR2, r3 3426 lis r3, (MMCR0_PMXE | MMCR0_FCECE)@h 3427 ori r3, r3, MMCR0_PMCjCE | MMCR0_C56RUN 3428 mtspr SPRN_MMCR0, r3 3429 lis r3, 0x7fff 3430 ori r3, r3, 0xffff 3431 mtspr SPRN_PMC6, r3 3432 isync 3433 blr 3434 3435#ifdef CONFIG_KVM_BOOK3S_HV_EXIT_TIMING 3436/* 3437 * Start timing an activity 3438 * r3 = pointer to time accumulation struct, r4 = vcpu 3439 */ 3440kvmhv_start_timing: 3441 ld r5, HSTATE_KVM_VCORE(r13) 3442 lbz r6, VCORE_IN_GUEST(r5) 3443 cmpwi r6, 0 3444 beq 5f /* if in guest, need to */ 3445 ld r6, VCORE_TB_OFFSET(r5) /* subtract timebase offset */ 34465: mftb r5 3447 subf r5, r6, r5 3448 std r3, VCPU_CUR_ACTIVITY(r4) 3449 std r5, VCPU_ACTIVITY_START(r4) 3450 blr 3451 3452/* 3453 * Accumulate time to one activity and start another. 3454 * r3 = pointer to new time accumulation struct, r4 = vcpu 3455 */ 3456kvmhv_accumulate_time: 3457 ld r5, HSTATE_KVM_VCORE(r13) 3458 lbz r8, VCORE_IN_GUEST(r5) 3459 cmpwi r8, 0 3460 beq 4f /* if in guest, need to */ 3461 ld r8, VCORE_TB_OFFSET(r5) /* subtract timebase offset */ 34624: ld r5, VCPU_CUR_ACTIVITY(r4) 3463 ld r6, VCPU_ACTIVITY_START(r4) 3464 std r3, VCPU_CUR_ACTIVITY(r4) 3465 mftb r7 3466 subf r7, r8, r7 3467 std r7, VCPU_ACTIVITY_START(r4) 3468 cmpdi r5, 0 3469 beqlr 3470 subf r3, r6, r7 3471 ld r8, TAS_SEQCOUNT(r5) 3472 cmpdi r8, 0 3473 addi r8, r8, 1 3474 std r8, TAS_SEQCOUNT(r5) 3475 lwsync 3476 ld r7, TAS_TOTAL(r5) 3477 add r7, r7, r3 3478 std r7, TAS_TOTAL(r5) 3479 ld r6, TAS_MIN(r5) 3480 ld r7, TAS_MAX(r5) 3481 beq 3f 3482 cmpd r3, r6 3483 bge 1f 34843: std r3, TAS_MIN(r5) 34851: cmpd r3, r7 3486 ble 2f 3487 std r3, TAS_MAX(r5) 34882: lwsync 3489 addi r8, r8, 1 3490 std r8, TAS_SEQCOUNT(r5) 3491 blr 3492#endif 3493