109cf57ebSDavid Brazdil // SPDX-License-Identifier: GPL-2.0-only 209cf57ebSDavid Brazdil /* 309cf57ebSDavid Brazdil * Copyright (C) 2015 - ARM Ltd 409cf57ebSDavid Brazdil * Author: Marc Zyngier <marc.zyngier@arm.com> 509cf57ebSDavid Brazdil */ 609cf57ebSDavid Brazdil 7cdb5e02eSMarc Zyngier #include <hyp/adjust_pc.h> 809cf57ebSDavid Brazdil #include <hyp/switch.h> 913aeb9b4SDavid Brazdil #include <hyp/sysreg-sr.h> 1009cf57ebSDavid Brazdil 1109cf57ebSDavid Brazdil #include <linux/arm-smccc.h> 1209cf57ebSDavid Brazdil #include <linux/kvm_host.h> 1309cf57ebSDavid Brazdil #include <linux/types.h> 1409cf57ebSDavid Brazdil #include <linux/jump_label.h> 1509cf57ebSDavid Brazdil #include <uapi/linux/psci.h> 1609cf57ebSDavid Brazdil 1709cf57ebSDavid Brazdil #include <kvm/arm_psci.h> 1809cf57ebSDavid Brazdil 1909cf57ebSDavid Brazdil #include <asm/barrier.h> 2009cf57ebSDavid Brazdil #include <asm/cpufeature.h> 2109cf57ebSDavid Brazdil #include <asm/kprobes.h> 2209cf57ebSDavid Brazdil #include <asm/kvm_asm.h> 2309cf57ebSDavid Brazdil #include <asm/kvm_emulate.h> 2409cf57ebSDavid Brazdil #include <asm/kvm_hyp.h> 2509cf57ebSDavid Brazdil #include <asm/kvm_mmu.h> 2609cf57ebSDavid Brazdil #include <asm/fpsimd.h> 2709cf57ebSDavid Brazdil #include <asm/debug-monitors.h> 2809cf57ebSDavid Brazdil #include <asm/processor.h> 2909cf57ebSDavid Brazdil #include <asm/thread_info.h> 3009cf57ebSDavid Brazdil 3114ef9d04SMarc Zyngier /* Non-VHE specific context */ 3214ef9d04SMarc Zyngier DEFINE_PER_CPU(struct kvm_host_data, kvm_host_data); 3314ef9d04SMarc Zyngier DEFINE_PER_CPU(struct kvm_cpu_context, kvm_hyp_ctxt); 3414ef9d04SMarc Zyngier DEFINE_PER_CPU(unsigned long, kvm_hyp_vector); 352a1198c9SDavid Brazdil 36c50cb043SDavid Brazdil static void __activate_traps(struct kvm_vcpu *vcpu) 3709cf57ebSDavid Brazdil { 3809cf57ebSDavid Brazdil u64 val; 3909cf57ebSDavid Brazdil 4009cf57ebSDavid Brazdil ___activate_traps(vcpu); 4109cf57ebSDavid Brazdil __activate_traps_common(vcpu); 4209cf57ebSDavid Brazdil 4309cf57ebSDavid Brazdil val = CPTR_EL2_DEFAULT; 4409cf57ebSDavid Brazdil val |= CPTR_EL2_TTA | CPTR_EL2_TZ | CPTR_EL2_TAM; 4509cf57ebSDavid Brazdil if (!update_fp_enabled(vcpu)) { 4609cf57ebSDavid Brazdil val |= CPTR_EL2_TFP; 4709cf57ebSDavid Brazdil __activate_traps_fpsimd32(vcpu); 4809cf57ebSDavid Brazdil } 4909cf57ebSDavid Brazdil 5009cf57ebSDavid Brazdil write_sysreg(val, cptr_el2); 5114ef9d04SMarc Zyngier write_sysreg(__this_cpu_read(kvm_hyp_vector), vbar_el2); 5209cf57ebSDavid Brazdil 5309cf57ebSDavid Brazdil if (cpus_have_final_cap(ARM64_WORKAROUND_SPECULATIVE_AT)) { 5409cf57ebSDavid Brazdil struct kvm_cpu_context *ctxt = &vcpu->arch.ctxt; 5509cf57ebSDavid Brazdil 5609cf57ebSDavid Brazdil isb(); 5709cf57ebSDavid Brazdil /* 5809cf57ebSDavid Brazdil * At this stage, and thanks to the above isb(), S2 is 5909cf57ebSDavid Brazdil * configured and enabled. We can now restore the guest's S1 6009cf57ebSDavid Brazdil * configuration: SCTLR, and only then TCR. 6109cf57ebSDavid Brazdil */ 6271071acfSMarc Zyngier write_sysreg_el1(ctxt_sys_reg(ctxt, SCTLR_EL1), SYS_SCTLR); 6309cf57ebSDavid Brazdil isb(); 6471071acfSMarc Zyngier write_sysreg_el1(ctxt_sys_reg(ctxt, TCR_EL1), SYS_TCR); 6509cf57ebSDavid Brazdil } 6609cf57ebSDavid Brazdil } 6709cf57ebSDavid Brazdil 68c50cb043SDavid Brazdil static void __deactivate_traps(struct kvm_vcpu *vcpu) 6909cf57ebSDavid Brazdil { 706e3bfbb2SAndrew Scull extern char __kvm_hyp_host_vector[]; 7109cf57ebSDavid Brazdil u64 mdcr_el2; 7209cf57ebSDavid Brazdil 7309cf57ebSDavid Brazdil ___deactivate_traps(vcpu); 7409cf57ebSDavid Brazdil 7509cf57ebSDavid Brazdil mdcr_el2 = read_sysreg(mdcr_el2); 7609cf57ebSDavid Brazdil 7709cf57ebSDavid Brazdil if (cpus_have_final_cap(ARM64_WORKAROUND_SPECULATIVE_AT)) { 7809cf57ebSDavid Brazdil u64 val; 7909cf57ebSDavid Brazdil 8009cf57ebSDavid Brazdil /* 8109cf57ebSDavid Brazdil * Set the TCR and SCTLR registers in the exact opposite 8209cf57ebSDavid Brazdil * sequence as __activate_traps (first prevent walks, 8309cf57ebSDavid Brazdil * then force the MMU on). A generous sprinkling of isb() 8409cf57ebSDavid Brazdil * ensure that things happen in this exact order. 8509cf57ebSDavid Brazdil */ 8609cf57ebSDavid Brazdil val = read_sysreg_el1(SYS_TCR); 8709cf57ebSDavid Brazdil write_sysreg_el1(val | TCR_EPD1_MASK | TCR_EPD0_MASK, SYS_TCR); 8809cf57ebSDavid Brazdil isb(); 8909cf57ebSDavid Brazdil val = read_sysreg_el1(SYS_SCTLR); 9009cf57ebSDavid Brazdil write_sysreg_el1(val | SCTLR_ELx_M, SYS_SCTLR); 9109cf57ebSDavid Brazdil isb(); 9209cf57ebSDavid Brazdil } 9309cf57ebSDavid Brazdil 9409cf57ebSDavid Brazdil __deactivate_traps_common(); 9509cf57ebSDavid Brazdil 9609cf57ebSDavid Brazdil mdcr_el2 &= MDCR_EL2_HPMN_MASK; 9709cf57ebSDavid Brazdil mdcr_el2 |= MDCR_EL2_E2PB_MASK << MDCR_EL2_E2PB_SHIFT; 9809cf57ebSDavid Brazdil 9909cf57ebSDavid Brazdil write_sysreg(mdcr_el2, mdcr_el2); 100b93c17c4SDavid Brazdil if (is_protected_kvm_enabled()) 101b93c17c4SDavid Brazdil write_sysreg(HCR_HOST_NVHE_PROTECTED_FLAGS, hcr_el2); 102b93c17c4SDavid Brazdil else 10309cf57ebSDavid Brazdil write_sysreg(HCR_HOST_NVHE_FLAGS, hcr_el2); 10409cf57ebSDavid Brazdil write_sysreg(CPTR_EL2_DEFAULT, cptr_el2); 1056e3bfbb2SAndrew Scull write_sysreg(__kvm_hyp_host_vector, vbar_el2); 10609cf57ebSDavid Brazdil } 10709cf57ebSDavid Brazdil 108501a67a2SAndrew Scull static void __load_host_stage2(void) 10909cf57ebSDavid Brazdil { 11009cf57ebSDavid Brazdil write_sysreg(0, vttbr_el2); 11109cf57ebSDavid Brazdil } 11209cf57ebSDavid Brazdil 11309cf57ebSDavid Brazdil /* Save VGICv3 state on non-VHE systems */ 114c50cb043SDavid Brazdil static void __hyp_vgic_save_state(struct kvm_vcpu *vcpu) 11509cf57ebSDavid Brazdil { 11609cf57ebSDavid Brazdil if (static_branch_unlikely(&kvm_vgic_global_state.gicv3_cpuif)) { 11709cf57ebSDavid Brazdil __vgic_v3_save_state(&vcpu->arch.vgic_cpu.vgic_v3); 11809cf57ebSDavid Brazdil __vgic_v3_deactivate_traps(&vcpu->arch.vgic_cpu.vgic_v3); 11909cf57ebSDavid Brazdil } 12009cf57ebSDavid Brazdil } 12109cf57ebSDavid Brazdil 12209cf57ebSDavid Brazdil /* Restore VGICv3 state on non_VEH systems */ 123c50cb043SDavid Brazdil static void __hyp_vgic_restore_state(struct kvm_vcpu *vcpu) 12409cf57ebSDavid Brazdil { 12509cf57ebSDavid Brazdil if (static_branch_unlikely(&kvm_vgic_global_state.gicv3_cpuif)) { 12609cf57ebSDavid Brazdil __vgic_v3_activate_traps(&vcpu->arch.vgic_cpu.vgic_v3); 12709cf57ebSDavid Brazdil __vgic_v3_restore_state(&vcpu->arch.vgic_cpu.vgic_v3); 12809cf57ebSDavid Brazdil } 12909cf57ebSDavid Brazdil } 13009cf57ebSDavid Brazdil 13109cf57ebSDavid Brazdil /** 13209cf57ebSDavid Brazdil * Disable host events, enable guest events 13309cf57ebSDavid Brazdil */ 134c50cb043SDavid Brazdil static bool __pmu_switch_to_guest(struct kvm_cpu_context *host_ctxt) 13509cf57ebSDavid Brazdil { 13609cf57ebSDavid Brazdil struct kvm_host_data *host; 13709cf57ebSDavid Brazdil struct kvm_pmu_events *pmu; 13809cf57ebSDavid Brazdil 13909cf57ebSDavid Brazdil host = container_of(host_ctxt, struct kvm_host_data, host_ctxt); 14009cf57ebSDavid Brazdil pmu = &host->pmu_events; 14109cf57ebSDavid Brazdil 14209cf57ebSDavid Brazdil if (pmu->events_host) 14309cf57ebSDavid Brazdil write_sysreg(pmu->events_host, pmcntenclr_el0); 14409cf57ebSDavid Brazdil 14509cf57ebSDavid Brazdil if (pmu->events_guest) 14609cf57ebSDavid Brazdil write_sysreg(pmu->events_guest, pmcntenset_el0); 14709cf57ebSDavid Brazdil 14809cf57ebSDavid Brazdil return (pmu->events_host || pmu->events_guest); 14909cf57ebSDavid Brazdil } 15009cf57ebSDavid Brazdil 15109cf57ebSDavid Brazdil /** 15209cf57ebSDavid Brazdil * Disable guest events, enable host events 15309cf57ebSDavid Brazdil */ 154c50cb043SDavid Brazdil static void __pmu_switch_to_host(struct kvm_cpu_context *host_ctxt) 15509cf57ebSDavid Brazdil { 15609cf57ebSDavid Brazdil struct kvm_host_data *host; 15709cf57ebSDavid Brazdil struct kvm_pmu_events *pmu; 15809cf57ebSDavid Brazdil 15909cf57ebSDavid Brazdil host = container_of(host_ctxt, struct kvm_host_data, host_ctxt); 16009cf57ebSDavid Brazdil pmu = &host->pmu_events; 16109cf57ebSDavid Brazdil 16209cf57ebSDavid Brazdil if (pmu->events_guest) 16309cf57ebSDavid Brazdil write_sysreg(pmu->events_guest, pmcntenclr_el0); 16409cf57ebSDavid Brazdil 16509cf57ebSDavid Brazdil if (pmu->events_host) 16609cf57ebSDavid Brazdil write_sysreg(pmu->events_host, pmcntenset_el0); 16709cf57ebSDavid Brazdil } 16809cf57ebSDavid Brazdil 16909cf57ebSDavid Brazdil /* Switch to the guest for legacy non-VHE systems */ 170c50cb043SDavid Brazdil int __kvm_vcpu_run(struct kvm_vcpu *vcpu) 17109cf57ebSDavid Brazdil { 17209cf57ebSDavid Brazdil struct kvm_cpu_context *host_ctxt; 17309cf57ebSDavid Brazdil struct kvm_cpu_context *guest_ctxt; 17409cf57ebSDavid Brazdil bool pmu_switch_needed; 17509cf57ebSDavid Brazdil u64 exit_code; 17609cf57ebSDavid Brazdil 17709cf57ebSDavid Brazdil /* 17809cf57ebSDavid Brazdil * Having IRQs masked via PMR when entering the guest means the GIC 17909cf57ebSDavid Brazdil * will not signal the CPU of interrupts of lower priority, and the 18009cf57ebSDavid Brazdil * only way to get out will be via guest exceptions. 18109cf57ebSDavid Brazdil * Naturally, we want to avoid this. 18209cf57ebSDavid Brazdil */ 18309cf57ebSDavid Brazdil if (system_uses_irq_prio_masking()) { 18409cf57ebSDavid Brazdil gic_write_pmr(GIC_PRIO_IRQON | GIC_PRIO_PSR_I_SET); 18509cf57ebSDavid Brazdil pmr_sync(); 18609cf57ebSDavid Brazdil } 18709cf57ebSDavid Brazdil 188717cf94aSDavid Brazdil host_ctxt = &this_cpu_ptr(&kvm_host_data)->host_ctxt; 18909cf57ebSDavid Brazdil host_ctxt->__hyp_running_vcpu = vcpu; 19009cf57ebSDavid Brazdil guest_ctxt = &vcpu->arch.ctxt; 19109cf57ebSDavid Brazdil 19209cf57ebSDavid Brazdil pmu_switch_needed = __pmu_switch_to_guest(host_ctxt); 19309cf57ebSDavid Brazdil 19409cf57ebSDavid Brazdil __sysreg_save_state_nvhe(host_ctxt); 195*b96b0c5dSSuzuki K Poulose /* 196*b96b0c5dSSuzuki K Poulose * We must flush and disable the SPE buffer for nVHE, as 197*b96b0c5dSSuzuki K Poulose * the translation regime(EL1&0) is going to be loaded with 198*b96b0c5dSSuzuki K Poulose * that of the guest. And we must do this before we change the 199*b96b0c5dSSuzuki K Poulose * translation regime to EL2 (via MDCR_EL2_E2PB == 0) and 200*b96b0c5dSSuzuki K Poulose * before we load guest Stage1. 201*b96b0c5dSSuzuki K Poulose */ 202*b96b0c5dSSuzuki K Poulose __debug_save_host_buffers_nvhe(vcpu); 20309cf57ebSDavid Brazdil 204cdb5e02eSMarc Zyngier __adjust_pc(vcpu); 205cdb5e02eSMarc Zyngier 20609cf57ebSDavid Brazdil /* 20709cf57ebSDavid Brazdil * We must restore the 32-bit state before the sysregs, thanks 20809cf57ebSDavid Brazdil * to erratum #852523 (Cortex-A57) or #853709 (Cortex-A72). 20909cf57ebSDavid Brazdil * 21009cf57ebSDavid Brazdil * Also, and in order to be able to deal with erratum #1319537 (A57) 21109cf57ebSDavid Brazdil * and #1319367 (A72), we must ensure that all VM-related sysreg are 21209cf57ebSDavid Brazdil * restored before we enable S2 translation. 21309cf57ebSDavid Brazdil */ 21409cf57ebSDavid Brazdil __sysreg32_restore_state(vcpu); 21509cf57ebSDavid Brazdil __sysreg_restore_state_nvhe(guest_ctxt); 21609cf57ebSDavid Brazdil 217501a67a2SAndrew Scull __load_guest_stage2(kern_hyp_va(vcpu->arch.hw_mmu)); 21809cf57ebSDavid Brazdil __activate_traps(vcpu); 21909cf57ebSDavid Brazdil 22009cf57ebSDavid Brazdil __hyp_vgic_restore_state(vcpu); 22109cf57ebSDavid Brazdil __timer_enable_traps(vcpu); 22209cf57ebSDavid Brazdil 22309cf57ebSDavid Brazdil __debug_switch_to_guest(vcpu); 22409cf57ebSDavid Brazdil 22509cf57ebSDavid Brazdil do { 22609cf57ebSDavid Brazdil /* Jump in the fire! */ 227b619d9aaSAndrew Scull exit_code = __guest_enter(vcpu); 22809cf57ebSDavid Brazdil 22909cf57ebSDavid Brazdil /* And we're baaack! */ 23009cf57ebSDavid Brazdil } while (fixup_guest_exit(vcpu, &exit_code)); 23109cf57ebSDavid Brazdil 23209cf57ebSDavid Brazdil __sysreg_save_state_nvhe(guest_ctxt); 23309cf57ebSDavid Brazdil __sysreg32_save_state(vcpu); 23409cf57ebSDavid Brazdil __timer_disable_traps(vcpu); 23509cf57ebSDavid Brazdil __hyp_vgic_save_state(vcpu); 23609cf57ebSDavid Brazdil 23709cf57ebSDavid Brazdil __deactivate_traps(vcpu); 238501a67a2SAndrew Scull __load_host_stage2(); 23909cf57ebSDavid Brazdil 24009cf57ebSDavid Brazdil __sysreg_restore_state_nvhe(host_ctxt); 24109cf57ebSDavid Brazdil 24209cf57ebSDavid Brazdil if (vcpu->arch.flags & KVM_ARM64_FP_ENABLED) 24309cf57ebSDavid Brazdil __fpsimd_save_fpexc32(vcpu); 24409cf57ebSDavid Brazdil 245*b96b0c5dSSuzuki K Poulose __debug_switch_to_host(vcpu); 24609cf57ebSDavid Brazdil /* 24709cf57ebSDavid Brazdil * This must come after restoring the host sysregs, since a non-VHE 24809cf57ebSDavid Brazdil * system may enable SPE here and make use of the TTBRs. 24909cf57ebSDavid Brazdil */ 250*b96b0c5dSSuzuki K Poulose __debug_restore_host_buffers_nvhe(vcpu); 25109cf57ebSDavid Brazdil 25209cf57ebSDavid Brazdil if (pmu_switch_needed) 25309cf57ebSDavid Brazdil __pmu_switch_to_host(host_ctxt); 25409cf57ebSDavid Brazdil 25509cf57ebSDavid Brazdil /* Returning to host will clear PSR.I, remask PMR if needed */ 25609cf57ebSDavid Brazdil if (system_uses_irq_prio_masking()) 25709cf57ebSDavid Brazdil gic_write_pmr(GIC_PRIO_IRQOFF); 25809cf57ebSDavid Brazdil 259a2e102e2SAndrew Scull host_ctxt->__hyp_running_vcpu = NULL; 260a2e102e2SAndrew Scull 26109cf57ebSDavid Brazdil return exit_code; 26209cf57ebSDavid Brazdil } 26309cf57ebSDavid Brazdil 2646a0259edSAndrew Scull void __noreturn hyp_panic(void) 26509cf57ebSDavid Brazdil { 26609cf57ebSDavid Brazdil u64 spsr = read_sysreg_el2(SYS_SPSR); 26709cf57ebSDavid Brazdil u64 elr = read_sysreg_el2(SYS_ELR); 26896d389caSRob Herring u64 par = read_sysreg_par(); 269a2e102e2SAndrew Scull bool restore_host = true; 2706a0259edSAndrew Scull struct kvm_cpu_context *host_ctxt; 2716a0259edSAndrew Scull struct kvm_vcpu *vcpu; 27209cf57ebSDavid Brazdil 27314ef9d04SMarc Zyngier host_ctxt = &this_cpu_ptr(&kvm_host_data)->host_ctxt; 2746a0259edSAndrew Scull vcpu = host_ctxt->__hyp_running_vcpu; 2756a0259edSAndrew Scull 276a2e102e2SAndrew Scull if (vcpu) { 27709cf57ebSDavid Brazdil __timer_disable_traps(vcpu); 27809cf57ebSDavid Brazdil __deactivate_traps(vcpu); 279501a67a2SAndrew Scull __load_host_stage2(); 28009cf57ebSDavid Brazdil __sysreg_restore_state_nvhe(host_ctxt); 28109cf57ebSDavid Brazdil } 28209cf57ebSDavid Brazdil 283a2e102e2SAndrew Scull __hyp_do_panic(restore_host, spsr, elr, par); 28409cf57ebSDavid Brazdil unreachable(); 28509cf57ebSDavid Brazdil } 286e9ee186bSJames Morse 287e9ee186bSJames Morse asmlinkage void kvm_unexpected_el2_exception(void) 288e9ee186bSJames Morse { 289e9ee186bSJames Morse return __kvm_unexpected_el2_exception(); 290e9ee186bSJames Morse } 291