109cf57ebSDavid Brazdil // SPDX-License-Identifier: GPL-2.0-only 209cf57ebSDavid Brazdil /* 309cf57ebSDavid Brazdil * Copyright (C) 2015 - ARM Ltd 409cf57ebSDavid Brazdil * Author: Marc Zyngier <marc.zyngier@arm.com> 509cf57ebSDavid Brazdil */ 609cf57ebSDavid Brazdil 709cf57ebSDavid Brazdil #include <hyp/switch.h> 809cf57ebSDavid Brazdil 909cf57ebSDavid Brazdil #include <linux/arm-smccc.h> 1009cf57ebSDavid Brazdil #include <linux/kvm_host.h> 1109cf57ebSDavid Brazdil #include <linux/types.h> 1209cf57ebSDavid Brazdil #include <linux/jump_label.h> 1309cf57ebSDavid Brazdil #include <uapi/linux/psci.h> 1409cf57ebSDavid Brazdil 1509cf57ebSDavid Brazdil #include <kvm/arm_psci.h> 1609cf57ebSDavid Brazdil 1709cf57ebSDavid Brazdil #include <asm/barrier.h> 1809cf57ebSDavid Brazdil #include <asm/cpufeature.h> 1909cf57ebSDavid Brazdil #include <asm/kprobes.h> 2009cf57ebSDavid Brazdil #include <asm/kvm_asm.h> 2109cf57ebSDavid Brazdil #include <asm/kvm_emulate.h> 2209cf57ebSDavid Brazdil #include <asm/kvm_hyp.h> 2309cf57ebSDavid Brazdil #include <asm/kvm_mmu.h> 2409cf57ebSDavid Brazdil #include <asm/fpsimd.h> 2509cf57ebSDavid Brazdil #include <asm/debug-monitors.h> 2609cf57ebSDavid Brazdil #include <asm/processor.h> 2709cf57ebSDavid Brazdil #include <asm/thread_info.h> 2809cf57ebSDavid Brazdil 2909cf57ebSDavid Brazdil const char __hyp_panic_string[] = "HYP panic:\nPS:%08llx PC:%016llx ESR:%08llx\nFAR:%016llx HPFAR:%016llx PAR:%016llx\nVCPU:%p\n"; 3009cf57ebSDavid Brazdil 3109cf57ebSDavid Brazdil static void __activate_traps(struct kvm_vcpu *vcpu) 3209cf57ebSDavid Brazdil { 3309cf57ebSDavid Brazdil u64 val; 3409cf57ebSDavid Brazdil 3509cf57ebSDavid Brazdil ___activate_traps(vcpu); 3609cf57ebSDavid Brazdil 3709cf57ebSDavid Brazdil val = read_sysreg(cpacr_el1); 3809cf57ebSDavid Brazdil val |= CPACR_EL1_TTA; 3909cf57ebSDavid Brazdil val &= ~CPACR_EL1_ZEN; 4009cf57ebSDavid Brazdil 4109cf57ebSDavid Brazdil /* 4209cf57ebSDavid Brazdil * With VHE (HCR.E2H == 1), accesses to CPACR_EL1 are routed to 4309cf57ebSDavid Brazdil * CPTR_EL2. In general, CPACR_EL1 has the same layout as CPTR_EL2, 4409cf57ebSDavid Brazdil * except for some missing controls, such as TAM. 4509cf57ebSDavid Brazdil * In this case, CPTR_EL2.TAM has the same position with or without 4609cf57ebSDavid Brazdil * VHE (HCR.E2H == 1) which allows us to use here the CPTR_EL2.TAM 4709cf57ebSDavid Brazdil * shift value for trapping the AMU accesses. 4809cf57ebSDavid Brazdil */ 4909cf57ebSDavid Brazdil 5009cf57ebSDavid Brazdil val |= CPTR_EL2_TAM; 5109cf57ebSDavid Brazdil 5209cf57ebSDavid Brazdil if (update_fp_enabled(vcpu)) { 5309cf57ebSDavid Brazdil if (vcpu_has_sve(vcpu)) 5409cf57ebSDavid Brazdil val |= CPACR_EL1_ZEN; 5509cf57ebSDavid Brazdil } else { 5609cf57ebSDavid Brazdil val &= ~CPACR_EL1_FPEN; 5709cf57ebSDavid Brazdil __activate_traps_fpsimd32(vcpu); 5809cf57ebSDavid Brazdil } 5909cf57ebSDavid Brazdil 6009cf57ebSDavid Brazdil write_sysreg(val, cpacr_el1); 6109cf57ebSDavid Brazdil 6209cf57ebSDavid Brazdil write_sysreg(kvm_get_hyp_vector(), vbar_el1); 6309cf57ebSDavid Brazdil } 6409cf57ebSDavid Brazdil NOKPROBE_SYMBOL(__activate_traps); 6509cf57ebSDavid Brazdil 6609cf57ebSDavid Brazdil static void __deactivate_traps(struct kvm_vcpu *vcpu) 6709cf57ebSDavid Brazdil { 6809cf57ebSDavid Brazdil extern char vectors[]; /* kernel exception vectors */ 6909cf57ebSDavid Brazdil 7009cf57ebSDavid Brazdil ___deactivate_traps(vcpu); 7109cf57ebSDavid Brazdil 7209cf57ebSDavid Brazdil write_sysreg(HCR_HOST_VHE_FLAGS, hcr_el2); 7309cf57ebSDavid Brazdil 7409cf57ebSDavid Brazdil /* 7509cf57ebSDavid Brazdil * ARM errata 1165522 and 1530923 require the actual execution of the 7609cf57ebSDavid Brazdil * above before we can switch to the EL2/EL0 translation regime used by 7709cf57ebSDavid Brazdil * the host. 7809cf57ebSDavid Brazdil */ 7909cf57ebSDavid Brazdil asm(ALTERNATIVE("nop", "isb", ARM64_WORKAROUND_SPECULATIVE_AT)); 8009cf57ebSDavid Brazdil 8109cf57ebSDavid Brazdil write_sysreg(CPACR_EL1_DEFAULT, cpacr_el1); 8209cf57ebSDavid Brazdil write_sysreg(vectors, vbar_el1); 8309cf57ebSDavid Brazdil } 8409cf57ebSDavid Brazdil NOKPROBE_SYMBOL(__deactivate_traps); 8509cf57ebSDavid Brazdil 8609cf57ebSDavid Brazdil void activate_traps_vhe_load(struct kvm_vcpu *vcpu) 8709cf57ebSDavid Brazdil { 8809cf57ebSDavid Brazdil __activate_traps_common(vcpu); 8909cf57ebSDavid Brazdil } 9009cf57ebSDavid Brazdil 9109cf57ebSDavid Brazdil void deactivate_traps_vhe_put(void) 9209cf57ebSDavid Brazdil { 9309cf57ebSDavid Brazdil u64 mdcr_el2 = read_sysreg(mdcr_el2); 9409cf57ebSDavid Brazdil 9509cf57ebSDavid Brazdil mdcr_el2 &= MDCR_EL2_HPMN_MASK | 9609cf57ebSDavid Brazdil MDCR_EL2_E2PB_MASK << MDCR_EL2_E2PB_SHIFT | 9709cf57ebSDavid Brazdil MDCR_EL2_TPMS; 9809cf57ebSDavid Brazdil 9909cf57ebSDavid Brazdil write_sysreg(mdcr_el2, mdcr_el2); 10009cf57ebSDavid Brazdil 10109cf57ebSDavid Brazdil __deactivate_traps_common(); 10209cf57ebSDavid Brazdil } 10309cf57ebSDavid Brazdil 10409cf57ebSDavid Brazdil /* Switch to the guest for VHE systems running in EL2 */ 10509cf57ebSDavid Brazdil static int __kvm_vcpu_run_vhe(struct kvm_vcpu *vcpu) 10609cf57ebSDavid Brazdil { 10709cf57ebSDavid Brazdil struct kvm_cpu_context *host_ctxt; 10809cf57ebSDavid Brazdil struct kvm_cpu_context *guest_ctxt; 10909cf57ebSDavid Brazdil u64 exit_code; 11009cf57ebSDavid Brazdil 11109cf57ebSDavid Brazdil host_ctxt = &__hyp_this_cpu_ptr(kvm_host_data)->host_ctxt; 11209cf57ebSDavid Brazdil host_ctxt->__hyp_running_vcpu = vcpu; 11309cf57ebSDavid Brazdil guest_ctxt = &vcpu->arch.ctxt; 11409cf57ebSDavid Brazdil 11509cf57ebSDavid Brazdil sysreg_save_host_state_vhe(host_ctxt); 11609cf57ebSDavid Brazdil 11709cf57ebSDavid Brazdil /* 11809cf57ebSDavid Brazdil * ARM erratum 1165522 requires us to configure both stage 1 and 11909cf57ebSDavid Brazdil * stage 2 translation for the guest context before we clear 12009cf57ebSDavid Brazdil * HCR_EL2.TGE. 12109cf57ebSDavid Brazdil * 12209cf57ebSDavid Brazdil * We have already configured the guest's stage 1 translation in 123501a67a2SAndrew Scull * kvm_vcpu_load_sysregs_vhe above. We must now call 124501a67a2SAndrew Scull * __load_guest_stage2 before __activate_traps, because 125501a67a2SAndrew Scull * __load_guest_stage2 configures stage 2 translation, and 126501a67a2SAndrew Scull * __activate_traps clear HCR_EL2.TGE (among other things). 12709cf57ebSDavid Brazdil */ 128501a67a2SAndrew Scull __load_guest_stage2(vcpu->arch.hw_mmu); 12909cf57ebSDavid Brazdil __activate_traps(vcpu); 13009cf57ebSDavid Brazdil 13109cf57ebSDavid Brazdil sysreg_restore_guest_state_vhe(guest_ctxt); 13209cf57ebSDavid Brazdil __debug_switch_to_guest(vcpu); 13309cf57ebSDavid Brazdil 13409cf57ebSDavid Brazdil __set_guest_arch_workaround_state(vcpu); 13509cf57ebSDavid Brazdil 13609cf57ebSDavid Brazdil do { 13709cf57ebSDavid Brazdil /* Jump in the fire! */ 13809cf57ebSDavid Brazdil exit_code = __guest_enter(vcpu, host_ctxt); 13909cf57ebSDavid Brazdil 14009cf57ebSDavid Brazdil /* And we're baaack! */ 14109cf57ebSDavid Brazdil } while (fixup_guest_exit(vcpu, &exit_code)); 14209cf57ebSDavid Brazdil 14309cf57ebSDavid Brazdil __set_host_arch_workaround_state(vcpu); 14409cf57ebSDavid Brazdil 14509cf57ebSDavid Brazdil sysreg_save_guest_state_vhe(guest_ctxt); 14609cf57ebSDavid Brazdil 14709cf57ebSDavid Brazdil __deactivate_traps(vcpu); 14809cf57ebSDavid Brazdil 14909cf57ebSDavid Brazdil sysreg_restore_host_state_vhe(host_ctxt); 15009cf57ebSDavid Brazdil 15109cf57ebSDavid Brazdil if (vcpu->arch.flags & KVM_ARM64_FP_ENABLED) 15209cf57ebSDavid Brazdil __fpsimd_save_fpexc32(vcpu); 15309cf57ebSDavid Brazdil 15409cf57ebSDavid Brazdil __debug_switch_to_host(vcpu); 15509cf57ebSDavid Brazdil 15609cf57ebSDavid Brazdil return exit_code; 15709cf57ebSDavid Brazdil } 15809cf57ebSDavid Brazdil NOKPROBE_SYMBOL(__kvm_vcpu_run_vhe); 15909cf57ebSDavid Brazdil 16009cf57ebSDavid Brazdil int __kvm_vcpu_run(struct kvm_vcpu *vcpu) 16109cf57ebSDavid Brazdil { 16209cf57ebSDavid Brazdil int ret; 16309cf57ebSDavid Brazdil 16409cf57ebSDavid Brazdil local_daif_mask(); 16509cf57ebSDavid Brazdil 16609cf57ebSDavid Brazdil /* 16709cf57ebSDavid Brazdil * Having IRQs masked via PMR when entering the guest means the GIC 16809cf57ebSDavid Brazdil * will not signal the CPU of interrupts of lower priority, and the 16909cf57ebSDavid Brazdil * only way to get out will be via guest exceptions. 17009cf57ebSDavid Brazdil * Naturally, we want to avoid this. 17109cf57ebSDavid Brazdil * 17209cf57ebSDavid Brazdil * local_daif_mask() already sets GIC_PRIO_PSR_I_SET, we just need a 17309cf57ebSDavid Brazdil * dsb to ensure the redistributor is forwards EL2 IRQs to the CPU. 17409cf57ebSDavid Brazdil */ 17509cf57ebSDavid Brazdil pmr_sync(); 17609cf57ebSDavid Brazdil 17709cf57ebSDavid Brazdil ret = __kvm_vcpu_run_vhe(vcpu); 17809cf57ebSDavid Brazdil 17909cf57ebSDavid Brazdil /* 18009cf57ebSDavid Brazdil * local_daif_restore() takes care to properly restore PSTATE.DAIF 18109cf57ebSDavid Brazdil * and the GIC PMR if the host is using IRQ priorities. 18209cf57ebSDavid Brazdil */ 18309cf57ebSDavid Brazdil local_daif_restore(DAIF_PROCCTX_NOIRQ); 18409cf57ebSDavid Brazdil 18509cf57ebSDavid Brazdil /* 18609cf57ebSDavid Brazdil * When we exit from the guest we change a number of CPU configuration 18709cf57ebSDavid Brazdil * parameters, such as traps. Make sure these changes take effect 18809cf57ebSDavid Brazdil * before running the host or additional guests. 18909cf57ebSDavid Brazdil */ 19009cf57ebSDavid Brazdil isb(); 19109cf57ebSDavid Brazdil 19209cf57ebSDavid Brazdil return ret; 19309cf57ebSDavid Brazdil } 19409cf57ebSDavid Brazdil 1956a0259edSAndrew Scull static void __hyp_call_panic(u64 spsr, u64 elr, u64 par) 19609cf57ebSDavid Brazdil { 1976a0259edSAndrew Scull struct kvm_cpu_context *host_ctxt; 19809cf57ebSDavid Brazdil struct kvm_vcpu *vcpu; 1996a0259edSAndrew Scull 2006a0259edSAndrew Scull host_ctxt = &__hyp_this_cpu_ptr(kvm_host_data)->host_ctxt; 20109cf57ebSDavid Brazdil vcpu = host_ctxt->__hyp_running_vcpu; 20209cf57ebSDavid Brazdil 20309cf57ebSDavid Brazdil __deactivate_traps(vcpu); 20409cf57ebSDavid Brazdil sysreg_restore_host_state_vhe(host_ctxt); 20509cf57ebSDavid Brazdil 20609cf57ebSDavid Brazdil panic(__hyp_panic_string, 20709cf57ebSDavid Brazdil spsr, elr, 20809cf57ebSDavid Brazdil read_sysreg_el2(SYS_ESR), read_sysreg_el2(SYS_FAR), 20909cf57ebSDavid Brazdil read_sysreg(hpfar_el2), par, vcpu); 21009cf57ebSDavid Brazdil } 21109cf57ebSDavid Brazdil NOKPROBE_SYMBOL(__hyp_call_panic); 21209cf57ebSDavid Brazdil 2136a0259edSAndrew Scull void __noreturn hyp_panic(void) 21409cf57ebSDavid Brazdil { 21509cf57ebSDavid Brazdil u64 spsr = read_sysreg_el2(SYS_SPSR); 21609cf57ebSDavid Brazdil u64 elr = read_sysreg_el2(SYS_ELR); 21709cf57ebSDavid Brazdil u64 par = read_sysreg(par_el1); 21809cf57ebSDavid Brazdil 2196a0259edSAndrew Scull __hyp_call_panic(spsr, elr, par); 22009cf57ebSDavid Brazdil unreachable(); 22109cf57ebSDavid Brazdil } 222e9ee186bSJames Morse 223e9ee186bSJames Morse asmlinkage void kvm_unexpected_el2_exception(void) 224e9ee186bSJames Morse { 225e9ee186bSJames Morse return __kvm_unexpected_el2_exception(); 226e9ee186bSJames Morse } 227