109cf57ebSDavid Brazdil // SPDX-License-Identifier: GPL-2.0-only 209cf57ebSDavid Brazdil /* 309cf57ebSDavid Brazdil * Copyright (C) 2015 - ARM Ltd 409cf57ebSDavid Brazdil * Author: Marc Zyngier <marc.zyngier@arm.com> 509cf57ebSDavid Brazdil */ 609cf57ebSDavid Brazdil 709cf57ebSDavid Brazdil #include <hyp/switch.h> 809cf57ebSDavid Brazdil 909cf57ebSDavid Brazdil #include <linux/arm-smccc.h> 1009cf57ebSDavid Brazdil #include <linux/kvm_host.h> 1109cf57ebSDavid Brazdil #include <linux/types.h> 1209cf57ebSDavid Brazdil #include <linux/jump_label.h> 13bd09128dSJames Morse #include <linux/percpu.h> 1409cf57ebSDavid Brazdil #include <uapi/linux/psci.h> 1509cf57ebSDavid Brazdil 1609cf57ebSDavid Brazdil #include <kvm/arm_psci.h> 1709cf57ebSDavid Brazdil 1809cf57ebSDavid Brazdil #include <asm/barrier.h> 1909cf57ebSDavid Brazdil #include <asm/cpufeature.h> 2009cf57ebSDavid Brazdil #include <asm/kprobes.h> 2109cf57ebSDavid Brazdil #include <asm/kvm_asm.h> 2209cf57ebSDavid Brazdil #include <asm/kvm_emulate.h> 2309cf57ebSDavid Brazdil #include <asm/kvm_hyp.h> 2409cf57ebSDavid Brazdil #include <asm/kvm_mmu.h> 2509cf57ebSDavid Brazdil #include <asm/fpsimd.h> 2609cf57ebSDavid Brazdil #include <asm/debug-monitors.h> 2709cf57ebSDavid Brazdil #include <asm/processor.h> 28bd09128dSJames Morse #include <asm/thread_info.h> 29bd09128dSJames Morse #include <asm/vectors.h> 3009cf57ebSDavid Brazdil 3114ef9d04SMarc Zyngier /* VHE specific context */ 3214ef9d04SMarc Zyngier DEFINE_PER_CPU(struct kvm_host_data, kvm_host_data); 3314ef9d04SMarc Zyngier DEFINE_PER_CPU(struct kvm_cpu_context, kvm_hyp_ctxt); 3414ef9d04SMarc Zyngier DEFINE_PER_CPU(unsigned long, kvm_hyp_vector); 352a1198c9SDavid Brazdil 3609cf57ebSDavid Brazdil static void __activate_traps(struct kvm_vcpu *vcpu) 3709cf57ebSDavid Brazdil { 3809cf57ebSDavid Brazdil u64 val; 3909cf57ebSDavid Brazdil 4009cf57ebSDavid Brazdil ___activate_traps(vcpu); 4109cf57ebSDavid Brazdil 4209cf57ebSDavid Brazdil val = read_sysreg(cpacr_el1); 437a5e9c8fSMarc Zyngier val |= CPACR_ELx_TTA; 4451729fb1SMark Brown val &= ~(CPACR_EL1_ZEN_EL0EN | CPACR_EL1_ZEN_EL1EN | 4551729fb1SMark Brown CPACR_EL1_SMEN_EL0EN | CPACR_EL1_SMEN_EL1EN); 4609cf57ebSDavid Brazdil 4709cf57ebSDavid Brazdil /* 4809cf57ebSDavid Brazdil * With VHE (HCR.E2H == 1), accesses to CPACR_EL1 are routed to 4909cf57ebSDavid Brazdil * CPTR_EL2. In general, CPACR_EL1 has the same layout as CPTR_EL2, 5009cf57ebSDavid Brazdil * except for some missing controls, such as TAM. 5109cf57ebSDavid Brazdil * In this case, CPTR_EL2.TAM has the same position with or without 5209cf57ebSDavid Brazdil * VHE (HCR.E2H == 1) which allows us to use here the CPTR_EL2.TAM 5309cf57ebSDavid Brazdil * shift value for trapping the AMU accesses. 5409cf57ebSDavid Brazdil */ 5509cf57ebSDavid Brazdil 5609cf57ebSDavid Brazdil val |= CPTR_EL2_TAM; 5709cf57ebSDavid Brazdil 58e9ada6c2SMarc Zyngier if (guest_owns_fp_regs(vcpu)) { 5909cf57ebSDavid Brazdil if (vcpu_has_sve(vcpu)) 603bb72d86SMark Brown val |= CPACR_EL1_ZEN_EL0EN | CPACR_EL1_ZEN_EL1EN; 6109cf57ebSDavid Brazdil } else { 623bb72d86SMark Brown val &= ~(CPACR_EL1_FPEN_EL0EN | CPACR_EL1_FPEN_EL1EN); 6309cf57ebSDavid Brazdil __activate_traps_fpsimd32(vcpu); 6409cf57ebSDavid Brazdil } 6509cf57ebSDavid Brazdil 6609cf57ebSDavid Brazdil write_sysreg(val, cpacr_el1); 6709cf57ebSDavid Brazdil 68a0e47952SAndrew Scull write_sysreg(__this_cpu_read(kvm_hyp_vector), vbar_el1); 6909cf57ebSDavid Brazdil } 7009cf57ebSDavid Brazdil NOKPROBE_SYMBOL(__activate_traps); 7109cf57ebSDavid Brazdil 7209cf57ebSDavid Brazdil static void __deactivate_traps(struct kvm_vcpu *vcpu) 7309cf57ebSDavid Brazdil { 74bd09128dSJames Morse const char *host_vectors = vectors; 7509cf57ebSDavid Brazdil 7609cf57ebSDavid Brazdil ___deactivate_traps(vcpu); 7709cf57ebSDavid Brazdil 7809cf57ebSDavid Brazdil write_sysreg(HCR_HOST_VHE_FLAGS, hcr_el2); 7909cf57ebSDavid Brazdil 8009cf57ebSDavid Brazdil /* 8109cf57ebSDavid Brazdil * ARM errata 1165522 and 1530923 require the actual execution of the 8209cf57ebSDavid Brazdil * above before we can switch to the EL2/EL0 translation regime used by 8309cf57ebSDavid Brazdil * the host. 8409cf57ebSDavid Brazdil */ 8509cf57ebSDavid Brazdil asm(ALTERNATIVE("nop", "isb", ARM64_WORKAROUND_SPECULATIVE_AT)); 8609cf57ebSDavid Brazdil 8709cf57ebSDavid Brazdil write_sysreg(CPACR_EL1_DEFAULT, cpacr_el1); 88bd09128dSJames Morse 89bd09128dSJames Morse if (!arm64_kernel_unmapped_at_el0()) 90bd09128dSJames Morse host_vectors = __this_cpu_read(this_cpu_vector); 91bd09128dSJames Morse write_sysreg(host_vectors, vbar_el1); 9209cf57ebSDavid Brazdil } 9309cf57ebSDavid Brazdil NOKPROBE_SYMBOL(__deactivate_traps); 9409cf57ebSDavid Brazdil 9509cf57ebSDavid Brazdil void activate_traps_vhe_load(struct kvm_vcpu *vcpu) 9609cf57ebSDavid Brazdil { 9709cf57ebSDavid Brazdil __activate_traps_common(vcpu); 9809cf57ebSDavid Brazdil } 9909cf57ebSDavid Brazdil 1001460b4b2SFuad Tabba void deactivate_traps_vhe_put(struct kvm_vcpu *vcpu) 10109cf57ebSDavid Brazdil { 1021460b4b2SFuad Tabba __deactivate_traps_common(vcpu); 10309cf57ebSDavid Brazdil } 10409cf57ebSDavid Brazdil 1058fb20461SMarc Zyngier static const exit_handler_fn hyp_exit_handlers[] = { 1068fb20461SMarc Zyngier [0 ... ESR_ELx_EC_MAX] = NULL, 1078fb20461SMarc Zyngier [ESR_ELx_EC_CP15_32] = kvm_hyp_handle_cp15_32, 1088fb20461SMarc Zyngier [ESR_ELx_EC_SYS64] = kvm_hyp_handle_sysreg, 1098fb20461SMarc Zyngier [ESR_ELx_EC_SVE] = kvm_hyp_handle_fpsimd, 1108fb20461SMarc Zyngier [ESR_ELx_EC_FP_ASIMD] = kvm_hyp_handle_fpsimd, 1118fb20461SMarc Zyngier [ESR_ELx_EC_IABT_LOW] = kvm_hyp_handle_iabt_low, 1128fb20461SMarc Zyngier [ESR_ELx_EC_DABT_LOW] = kvm_hyp_handle_dabt_low, 1138fb20461SMarc Zyngier [ESR_ELx_EC_PAC] = kvm_hyp_handle_ptrauth, 1148fb20461SMarc Zyngier }; 1158fb20461SMarc Zyngier 1160c7639ccSMarc Zyngier static const exit_handler_fn *kvm_get_exit_handler_array(struct kvm_vcpu *vcpu) 1178fb20461SMarc Zyngier { 1188fb20461SMarc Zyngier return hyp_exit_handlers; 1198fb20461SMarc Zyngier } 1208fb20461SMarc Zyngier 1217183b2b5SMarc Zyngier static void early_exit_filter(struct kvm_vcpu *vcpu, u64 *exit_code) 1227183b2b5SMarc Zyngier { 123d9552fe1SMarc Zyngier /* 124d9552fe1SMarc Zyngier * If we were in HYP context on entry, adjust the PSTATE view 125d9552fe1SMarc Zyngier * so that the usual helpers work correctly. 126d9552fe1SMarc Zyngier */ 127d9552fe1SMarc Zyngier if (unlikely(vcpu_get_flag(vcpu, VCPU_HYP_CONTEXT))) { 128d9552fe1SMarc Zyngier u64 mode = *vcpu_cpsr(vcpu) & (PSR_MODE_MASK | PSR_MODE32_BIT); 129d9552fe1SMarc Zyngier 130d9552fe1SMarc Zyngier switch (mode) { 131d9552fe1SMarc Zyngier case PSR_MODE_EL1t: 132d9552fe1SMarc Zyngier mode = PSR_MODE_EL2t; 133d9552fe1SMarc Zyngier break; 134d9552fe1SMarc Zyngier case PSR_MODE_EL1h: 135d9552fe1SMarc Zyngier mode = PSR_MODE_EL2h; 136d9552fe1SMarc Zyngier break; 137d9552fe1SMarc Zyngier } 138d9552fe1SMarc Zyngier 139d9552fe1SMarc Zyngier *vcpu_cpsr(vcpu) &= ~(PSR_MODE_MASK | PSR_MODE32_BIT); 140d9552fe1SMarc Zyngier *vcpu_cpsr(vcpu) |= mode; 141d9552fe1SMarc Zyngier } 1427183b2b5SMarc Zyngier } 1437183b2b5SMarc Zyngier 14409cf57ebSDavid Brazdil /* Switch to the guest for VHE systems running in EL2 */ 14509cf57ebSDavid Brazdil static int __kvm_vcpu_run_vhe(struct kvm_vcpu *vcpu) 14609cf57ebSDavid Brazdil { 14709cf57ebSDavid Brazdil struct kvm_cpu_context *host_ctxt; 14809cf57ebSDavid Brazdil struct kvm_cpu_context *guest_ctxt; 14909cf57ebSDavid Brazdil u64 exit_code; 15009cf57ebSDavid Brazdil 151717cf94aSDavid Brazdil host_ctxt = &this_cpu_ptr(&kvm_host_data)->host_ctxt; 15209cf57ebSDavid Brazdil host_ctxt->__hyp_running_vcpu = vcpu; 15309cf57ebSDavid Brazdil guest_ctxt = &vcpu->arch.ctxt; 15409cf57ebSDavid Brazdil 15509cf57ebSDavid Brazdil sysreg_save_host_state_vhe(host_ctxt); 15609cf57ebSDavid Brazdil 15709cf57ebSDavid Brazdil /* 15809cf57ebSDavid Brazdil * ARM erratum 1165522 requires us to configure both stage 1 and 15909cf57ebSDavid Brazdil * stage 2 translation for the guest context before we clear 16009cf57ebSDavid Brazdil * HCR_EL2.TGE. 16109cf57ebSDavid Brazdil * 16209cf57ebSDavid Brazdil * We have already configured the guest's stage 1 translation in 163501a67a2SAndrew Scull * kvm_vcpu_load_sysregs_vhe above. We must now call 1644efc0edeSMarc Zyngier * __load_stage2 before __activate_traps, because 1654efc0edeSMarc Zyngier * __load_stage2 configures stage 2 translation, and 166501a67a2SAndrew Scull * __activate_traps clear HCR_EL2.TGE (among other things). 16709cf57ebSDavid Brazdil */ 1684efc0edeSMarc Zyngier __load_stage2(vcpu->arch.hw_mmu, vcpu->arch.hw_mmu->arch); 16909cf57ebSDavid Brazdil __activate_traps(vcpu); 17009cf57ebSDavid Brazdil 171f5e30680SMarc Zyngier __kvm_adjust_pc(vcpu); 172cdb5e02eSMarc Zyngier 17309cf57ebSDavid Brazdil sysreg_restore_guest_state_vhe(guest_ctxt); 17409cf57ebSDavid Brazdil __debug_switch_to_guest(vcpu); 17509cf57ebSDavid Brazdil 176d9552fe1SMarc Zyngier if (is_hyp_ctxt(vcpu)) 177d9552fe1SMarc Zyngier vcpu_set_flag(vcpu, VCPU_HYP_CONTEXT); 178d9552fe1SMarc Zyngier else 179d9552fe1SMarc Zyngier vcpu_clear_flag(vcpu, VCPU_HYP_CONTEXT); 180d9552fe1SMarc Zyngier 18109cf57ebSDavid Brazdil do { 18209cf57ebSDavid Brazdil /* Jump in the fire! */ 183b619d9aaSAndrew Scull exit_code = __guest_enter(vcpu); 18409cf57ebSDavid Brazdil 18509cf57ebSDavid Brazdil /* And we're baaack! */ 18609cf57ebSDavid Brazdil } while (fixup_guest_exit(vcpu, &exit_code)); 18709cf57ebSDavid Brazdil 18809cf57ebSDavid Brazdil sysreg_save_guest_state_vhe(guest_ctxt); 18909cf57ebSDavid Brazdil 19009cf57ebSDavid Brazdil __deactivate_traps(vcpu); 19109cf57ebSDavid Brazdil 19209cf57ebSDavid Brazdil sysreg_restore_host_state_vhe(host_ctxt); 19309cf57ebSDavid Brazdil 194f8077b0dSMarc Zyngier if (vcpu->arch.fp_state == FP_STATE_GUEST_OWNED) 19509cf57ebSDavid Brazdil __fpsimd_save_fpexc32(vcpu); 19609cf57ebSDavid Brazdil 19709cf57ebSDavid Brazdil __debug_switch_to_host(vcpu); 19809cf57ebSDavid Brazdil 19909cf57ebSDavid Brazdil return exit_code; 20009cf57ebSDavid Brazdil } 20109cf57ebSDavid Brazdil NOKPROBE_SYMBOL(__kvm_vcpu_run_vhe); 20209cf57ebSDavid Brazdil 20309cf57ebSDavid Brazdil int __kvm_vcpu_run(struct kvm_vcpu *vcpu) 20409cf57ebSDavid Brazdil { 20509cf57ebSDavid Brazdil int ret; 20609cf57ebSDavid Brazdil 20709cf57ebSDavid Brazdil local_daif_mask(); 20809cf57ebSDavid Brazdil 20909cf57ebSDavid Brazdil /* 21009cf57ebSDavid Brazdil * Having IRQs masked via PMR when entering the guest means the GIC 21109cf57ebSDavid Brazdil * will not signal the CPU of interrupts of lower priority, and the 21209cf57ebSDavid Brazdil * only way to get out will be via guest exceptions. 21309cf57ebSDavid Brazdil * Naturally, we want to avoid this. 21409cf57ebSDavid Brazdil * 21509cf57ebSDavid Brazdil * local_daif_mask() already sets GIC_PRIO_PSR_I_SET, we just need a 21609cf57ebSDavid Brazdil * dsb to ensure the redistributor is forwards EL2 IRQs to the CPU. 21709cf57ebSDavid Brazdil */ 21809cf57ebSDavid Brazdil pmr_sync(); 21909cf57ebSDavid Brazdil 22009cf57ebSDavid Brazdil ret = __kvm_vcpu_run_vhe(vcpu); 22109cf57ebSDavid Brazdil 22209cf57ebSDavid Brazdil /* 22309cf57ebSDavid Brazdil * local_daif_restore() takes care to properly restore PSTATE.DAIF 22409cf57ebSDavid Brazdil * and the GIC PMR if the host is using IRQ priorities. 22509cf57ebSDavid Brazdil */ 22609cf57ebSDavid Brazdil local_daif_restore(DAIF_PROCCTX_NOIRQ); 22709cf57ebSDavid Brazdil 22809cf57ebSDavid Brazdil /* 22909cf57ebSDavid Brazdil * When we exit from the guest we change a number of CPU configuration 230*bcf3e7daSMarc Zyngier * parameters, such as traps. We rely on the isb() in kvm_call_hyp*() 231*bcf3e7daSMarc Zyngier * to make sure these changes take effect before running the host or 232*bcf3e7daSMarc Zyngier * additional guests. 23309cf57ebSDavid Brazdil */ 23409cf57ebSDavid Brazdil return ret; 23509cf57ebSDavid Brazdil } 23609cf57ebSDavid Brazdil 2376a0259edSAndrew Scull static void __hyp_call_panic(u64 spsr, u64 elr, u64 par) 23809cf57ebSDavid Brazdil { 2396a0259edSAndrew Scull struct kvm_cpu_context *host_ctxt; 24009cf57ebSDavid Brazdil struct kvm_vcpu *vcpu; 2416a0259edSAndrew Scull 24214ef9d04SMarc Zyngier host_ctxt = &this_cpu_ptr(&kvm_host_data)->host_ctxt; 24309cf57ebSDavid Brazdil vcpu = host_ctxt->__hyp_running_vcpu; 24409cf57ebSDavid Brazdil 24509cf57ebSDavid Brazdil __deactivate_traps(vcpu); 24609cf57ebSDavid Brazdil sysreg_restore_host_state_vhe(host_ctxt); 24709cf57ebSDavid Brazdil 248aec0fae6SAndrew Scull panic("HYP panic:\nPS:%08llx PC:%016llx ESR:%08llx\nFAR:%016llx HPFAR:%016llx PAR:%016llx\nVCPU:%p\n", 24909cf57ebSDavid Brazdil spsr, elr, 25009cf57ebSDavid Brazdil read_sysreg_el2(SYS_ESR), read_sysreg_el2(SYS_FAR), 25109cf57ebSDavid Brazdil read_sysreg(hpfar_el2), par, vcpu); 25209cf57ebSDavid Brazdil } 25309cf57ebSDavid Brazdil NOKPROBE_SYMBOL(__hyp_call_panic); 25409cf57ebSDavid Brazdil 2556a0259edSAndrew Scull void __noreturn hyp_panic(void) 25609cf57ebSDavid Brazdil { 25709cf57ebSDavid Brazdil u64 spsr = read_sysreg_el2(SYS_SPSR); 25809cf57ebSDavid Brazdil u64 elr = read_sysreg_el2(SYS_ELR); 25996d389caSRob Herring u64 par = read_sysreg_par(); 26009cf57ebSDavid Brazdil 2616a0259edSAndrew Scull __hyp_call_panic(spsr, elr, par); 26209cf57ebSDavid Brazdil unreachable(); 26309cf57ebSDavid Brazdil } 264e9ee186bSJames Morse 265e9ee186bSJames Morse asmlinkage void kvm_unexpected_el2_exception(void) 266e9ee186bSJames Morse { 2671c3ace2bSQuentin Perret __kvm_unexpected_el2_exception(); 268e9ee186bSJames Morse } 269