109cf57ebSDavid Brazdil // SPDX-License-Identifier: GPL-2.0-only 209cf57ebSDavid Brazdil /* 309cf57ebSDavid Brazdil * Copyright (C) 2015 - ARM Ltd 409cf57ebSDavid Brazdil * Author: Marc Zyngier <marc.zyngier@arm.com> 509cf57ebSDavid Brazdil */ 609cf57ebSDavid Brazdil 709cf57ebSDavid Brazdil #include <hyp/switch.h> 813aeb9b4SDavid Brazdil #include <hyp/sysreg-sr.h> 909cf57ebSDavid Brazdil 1009cf57ebSDavid Brazdil #include <linux/arm-smccc.h> 1109cf57ebSDavid Brazdil #include <linux/kvm_host.h> 1209cf57ebSDavid Brazdil #include <linux/types.h> 1309cf57ebSDavid Brazdil #include <linux/jump_label.h> 1409cf57ebSDavid Brazdil #include <uapi/linux/psci.h> 1509cf57ebSDavid Brazdil 1609cf57ebSDavid Brazdil #include <kvm/arm_psci.h> 1709cf57ebSDavid Brazdil 1809cf57ebSDavid Brazdil #include <asm/barrier.h> 1909cf57ebSDavid Brazdil #include <asm/cpufeature.h> 2009cf57ebSDavid Brazdil #include <asm/kprobes.h> 2109cf57ebSDavid Brazdil #include <asm/kvm_asm.h> 2209cf57ebSDavid Brazdil #include <asm/kvm_emulate.h> 2309cf57ebSDavid Brazdil #include <asm/kvm_hyp.h> 2409cf57ebSDavid Brazdil #include <asm/kvm_mmu.h> 2509cf57ebSDavid Brazdil #include <asm/fpsimd.h> 2609cf57ebSDavid Brazdil #include <asm/debug-monitors.h> 2709cf57ebSDavid Brazdil #include <asm/processor.h> 2809cf57ebSDavid Brazdil #include <asm/thread_info.h> 2909cf57ebSDavid Brazdil 30c50cb043SDavid Brazdil static void __activate_traps(struct kvm_vcpu *vcpu) 3109cf57ebSDavid Brazdil { 3209cf57ebSDavid Brazdil u64 val; 3309cf57ebSDavid Brazdil 3409cf57ebSDavid Brazdil ___activate_traps(vcpu); 3509cf57ebSDavid Brazdil __activate_traps_common(vcpu); 3609cf57ebSDavid Brazdil 3709cf57ebSDavid Brazdil val = CPTR_EL2_DEFAULT; 3809cf57ebSDavid Brazdil val |= CPTR_EL2_TTA | CPTR_EL2_TZ | CPTR_EL2_TAM; 3909cf57ebSDavid Brazdil if (!update_fp_enabled(vcpu)) { 4009cf57ebSDavid Brazdil val |= CPTR_EL2_TFP; 4109cf57ebSDavid Brazdil __activate_traps_fpsimd32(vcpu); 4209cf57ebSDavid Brazdil } 4309cf57ebSDavid Brazdil 4409cf57ebSDavid Brazdil write_sysreg(val, cptr_el2); 4509cf57ebSDavid Brazdil 4609cf57ebSDavid Brazdil if (cpus_have_final_cap(ARM64_WORKAROUND_SPECULATIVE_AT)) { 4709cf57ebSDavid Brazdil struct kvm_cpu_context *ctxt = &vcpu->arch.ctxt; 4809cf57ebSDavid Brazdil 4909cf57ebSDavid Brazdil isb(); 5009cf57ebSDavid Brazdil /* 5109cf57ebSDavid Brazdil * At this stage, and thanks to the above isb(), S2 is 5209cf57ebSDavid Brazdil * configured and enabled. We can now restore the guest's S1 5309cf57ebSDavid Brazdil * configuration: SCTLR, and only then TCR. 5409cf57ebSDavid Brazdil */ 5571071acfSMarc Zyngier write_sysreg_el1(ctxt_sys_reg(ctxt, SCTLR_EL1), SYS_SCTLR); 5609cf57ebSDavid Brazdil isb(); 5771071acfSMarc Zyngier write_sysreg_el1(ctxt_sys_reg(ctxt, TCR_EL1), SYS_TCR); 5809cf57ebSDavid Brazdil } 5909cf57ebSDavid Brazdil } 6009cf57ebSDavid Brazdil 61c50cb043SDavid Brazdil static void __deactivate_traps(struct kvm_vcpu *vcpu) 6209cf57ebSDavid Brazdil { 6309cf57ebSDavid Brazdil u64 mdcr_el2; 6409cf57ebSDavid Brazdil 6509cf57ebSDavid Brazdil ___deactivate_traps(vcpu); 6609cf57ebSDavid Brazdil 6709cf57ebSDavid Brazdil mdcr_el2 = read_sysreg(mdcr_el2); 6809cf57ebSDavid Brazdil 6909cf57ebSDavid Brazdil if (cpus_have_final_cap(ARM64_WORKAROUND_SPECULATIVE_AT)) { 7009cf57ebSDavid Brazdil u64 val; 7109cf57ebSDavid Brazdil 7209cf57ebSDavid Brazdil /* 7309cf57ebSDavid Brazdil * Set the TCR and SCTLR registers in the exact opposite 7409cf57ebSDavid Brazdil * sequence as __activate_traps (first prevent walks, 7509cf57ebSDavid Brazdil * then force the MMU on). A generous sprinkling of isb() 7609cf57ebSDavid Brazdil * ensure that things happen in this exact order. 7709cf57ebSDavid Brazdil */ 7809cf57ebSDavid Brazdil val = read_sysreg_el1(SYS_TCR); 7909cf57ebSDavid Brazdil write_sysreg_el1(val | TCR_EPD1_MASK | TCR_EPD0_MASK, SYS_TCR); 8009cf57ebSDavid Brazdil isb(); 8109cf57ebSDavid Brazdil val = read_sysreg_el1(SYS_SCTLR); 8209cf57ebSDavid Brazdil write_sysreg_el1(val | SCTLR_ELx_M, SYS_SCTLR); 8309cf57ebSDavid Brazdil isb(); 8409cf57ebSDavid Brazdil } 8509cf57ebSDavid Brazdil 8609cf57ebSDavid Brazdil __deactivate_traps_common(); 8709cf57ebSDavid Brazdil 8809cf57ebSDavid Brazdil mdcr_el2 &= MDCR_EL2_HPMN_MASK; 8909cf57ebSDavid Brazdil mdcr_el2 |= MDCR_EL2_E2PB_MASK << MDCR_EL2_E2PB_SHIFT; 9009cf57ebSDavid Brazdil 9109cf57ebSDavid Brazdil write_sysreg(mdcr_el2, mdcr_el2); 9209cf57ebSDavid Brazdil write_sysreg(HCR_HOST_NVHE_FLAGS, hcr_el2); 9309cf57ebSDavid Brazdil write_sysreg(CPTR_EL2_DEFAULT, cptr_el2); 9409cf57ebSDavid Brazdil } 9509cf57ebSDavid Brazdil 96c50cb043SDavid Brazdil static void __deactivate_vm(struct kvm_vcpu *vcpu) 9709cf57ebSDavid Brazdil { 9809cf57ebSDavid Brazdil write_sysreg(0, vttbr_el2); 9909cf57ebSDavid Brazdil } 10009cf57ebSDavid Brazdil 10109cf57ebSDavid Brazdil /* Save VGICv3 state on non-VHE systems */ 102c50cb043SDavid Brazdil static void __hyp_vgic_save_state(struct kvm_vcpu *vcpu) 10309cf57ebSDavid Brazdil { 10409cf57ebSDavid Brazdil if (static_branch_unlikely(&kvm_vgic_global_state.gicv3_cpuif)) { 10509cf57ebSDavid Brazdil __vgic_v3_save_state(&vcpu->arch.vgic_cpu.vgic_v3); 10609cf57ebSDavid Brazdil __vgic_v3_deactivate_traps(&vcpu->arch.vgic_cpu.vgic_v3); 10709cf57ebSDavid Brazdil } 10809cf57ebSDavid Brazdil } 10909cf57ebSDavid Brazdil 11009cf57ebSDavid Brazdil /* Restore VGICv3 state on non_VEH systems */ 111c50cb043SDavid Brazdil static void __hyp_vgic_restore_state(struct kvm_vcpu *vcpu) 11209cf57ebSDavid Brazdil { 11309cf57ebSDavid Brazdil if (static_branch_unlikely(&kvm_vgic_global_state.gicv3_cpuif)) { 11409cf57ebSDavid Brazdil __vgic_v3_activate_traps(&vcpu->arch.vgic_cpu.vgic_v3); 11509cf57ebSDavid Brazdil __vgic_v3_restore_state(&vcpu->arch.vgic_cpu.vgic_v3); 11609cf57ebSDavid Brazdil } 11709cf57ebSDavid Brazdil } 11809cf57ebSDavid Brazdil 11909cf57ebSDavid Brazdil /** 12009cf57ebSDavid Brazdil * Disable host events, enable guest events 12109cf57ebSDavid Brazdil */ 122c50cb043SDavid Brazdil static bool __pmu_switch_to_guest(struct kvm_cpu_context *host_ctxt) 12309cf57ebSDavid Brazdil { 12409cf57ebSDavid Brazdil struct kvm_host_data *host; 12509cf57ebSDavid Brazdil struct kvm_pmu_events *pmu; 12609cf57ebSDavid Brazdil 12709cf57ebSDavid Brazdil host = container_of(host_ctxt, struct kvm_host_data, host_ctxt); 12809cf57ebSDavid Brazdil pmu = &host->pmu_events; 12909cf57ebSDavid Brazdil 13009cf57ebSDavid Brazdil if (pmu->events_host) 13109cf57ebSDavid Brazdil write_sysreg(pmu->events_host, pmcntenclr_el0); 13209cf57ebSDavid Brazdil 13309cf57ebSDavid Brazdil if (pmu->events_guest) 13409cf57ebSDavid Brazdil write_sysreg(pmu->events_guest, pmcntenset_el0); 13509cf57ebSDavid Brazdil 13609cf57ebSDavid Brazdil return (pmu->events_host || pmu->events_guest); 13709cf57ebSDavid Brazdil } 13809cf57ebSDavid Brazdil 13909cf57ebSDavid Brazdil /** 14009cf57ebSDavid Brazdil * Disable guest events, enable host events 14109cf57ebSDavid Brazdil */ 142c50cb043SDavid Brazdil static void __pmu_switch_to_host(struct kvm_cpu_context *host_ctxt) 14309cf57ebSDavid Brazdil { 14409cf57ebSDavid Brazdil struct kvm_host_data *host; 14509cf57ebSDavid Brazdil struct kvm_pmu_events *pmu; 14609cf57ebSDavid Brazdil 14709cf57ebSDavid Brazdil host = container_of(host_ctxt, struct kvm_host_data, host_ctxt); 14809cf57ebSDavid Brazdil pmu = &host->pmu_events; 14909cf57ebSDavid Brazdil 15009cf57ebSDavid Brazdil if (pmu->events_guest) 15109cf57ebSDavid Brazdil write_sysreg(pmu->events_guest, pmcntenclr_el0); 15209cf57ebSDavid Brazdil 15309cf57ebSDavid Brazdil if (pmu->events_host) 15409cf57ebSDavid Brazdil write_sysreg(pmu->events_host, pmcntenset_el0); 15509cf57ebSDavid Brazdil } 15609cf57ebSDavid Brazdil 15709cf57ebSDavid Brazdil /* Switch to the guest for legacy non-VHE systems */ 158c50cb043SDavid Brazdil int __kvm_vcpu_run(struct kvm_vcpu *vcpu) 15909cf57ebSDavid Brazdil { 16009cf57ebSDavid Brazdil struct kvm_cpu_context *host_ctxt; 16109cf57ebSDavid Brazdil struct kvm_cpu_context *guest_ctxt; 16209cf57ebSDavid Brazdil bool pmu_switch_needed; 16309cf57ebSDavid Brazdil u64 exit_code; 16409cf57ebSDavid Brazdil 16509cf57ebSDavid Brazdil /* 16609cf57ebSDavid Brazdil * Having IRQs masked via PMR when entering the guest means the GIC 16709cf57ebSDavid Brazdil * will not signal the CPU of interrupts of lower priority, and the 16809cf57ebSDavid Brazdil * only way to get out will be via guest exceptions. 16909cf57ebSDavid Brazdil * Naturally, we want to avoid this. 17009cf57ebSDavid Brazdil */ 17109cf57ebSDavid Brazdil if (system_uses_irq_prio_masking()) { 17209cf57ebSDavid Brazdil gic_write_pmr(GIC_PRIO_IRQON | GIC_PRIO_PSR_I_SET); 17309cf57ebSDavid Brazdil pmr_sync(); 17409cf57ebSDavid Brazdil } 17509cf57ebSDavid Brazdil 17609cf57ebSDavid Brazdil vcpu = kern_hyp_va(vcpu); 17709cf57ebSDavid Brazdil 17809cf57ebSDavid Brazdil host_ctxt = &__hyp_this_cpu_ptr(kvm_host_data)->host_ctxt; 17909cf57ebSDavid Brazdil host_ctxt->__hyp_running_vcpu = vcpu; 18009cf57ebSDavid Brazdil guest_ctxt = &vcpu->arch.ctxt; 18109cf57ebSDavid Brazdil 18209cf57ebSDavid Brazdil pmu_switch_needed = __pmu_switch_to_guest(host_ctxt); 18309cf57ebSDavid Brazdil 18409cf57ebSDavid Brazdil __sysreg_save_state_nvhe(host_ctxt); 18509cf57ebSDavid Brazdil 18609cf57ebSDavid Brazdil /* 18709cf57ebSDavid Brazdil * We must restore the 32-bit state before the sysregs, thanks 18809cf57ebSDavid Brazdil * to erratum #852523 (Cortex-A57) or #853709 (Cortex-A72). 18909cf57ebSDavid Brazdil * 19009cf57ebSDavid Brazdil * Also, and in order to be able to deal with erratum #1319537 (A57) 19109cf57ebSDavid Brazdil * and #1319367 (A72), we must ensure that all VM-related sysreg are 19209cf57ebSDavid Brazdil * restored before we enable S2 translation. 19309cf57ebSDavid Brazdil */ 19409cf57ebSDavid Brazdil __sysreg32_restore_state(vcpu); 19509cf57ebSDavid Brazdil __sysreg_restore_state_nvhe(guest_ctxt); 19609cf57ebSDavid Brazdil 197a0e50aa3SChristoffer Dall __activate_vm(kern_hyp_va(vcpu->arch.hw_mmu)); 19809cf57ebSDavid Brazdil __activate_traps(vcpu); 19909cf57ebSDavid Brazdil 20009cf57ebSDavid Brazdil __hyp_vgic_restore_state(vcpu); 20109cf57ebSDavid Brazdil __timer_enable_traps(vcpu); 20209cf57ebSDavid Brazdil 20309cf57ebSDavid Brazdil __debug_switch_to_guest(vcpu); 20409cf57ebSDavid Brazdil 20509cf57ebSDavid Brazdil __set_guest_arch_workaround_state(vcpu); 20609cf57ebSDavid Brazdil 20709cf57ebSDavid Brazdil do { 20809cf57ebSDavid Brazdil /* Jump in the fire! */ 20909cf57ebSDavid Brazdil exit_code = __guest_enter(vcpu, host_ctxt); 21009cf57ebSDavid Brazdil 21109cf57ebSDavid Brazdil /* And we're baaack! */ 21209cf57ebSDavid Brazdil } while (fixup_guest_exit(vcpu, &exit_code)); 21309cf57ebSDavid Brazdil 21409cf57ebSDavid Brazdil __set_host_arch_workaround_state(vcpu); 21509cf57ebSDavid Brazdil 21609cf57ebSDavid Brazdil __sysreg_save_state_nvhe(guest_ctxt); 21709cf57ebSDavid Brazdil __sysreg32_save_state(vcpu); 21809cf57ebSDavid Brazdil __timer_disable_traps(vcpu); 21909cf57ebSDavid Brazdil __hyp_vgic_save_state(vcpu); 22009cf57ebSDavid Brazdil 22109cf57ebSDavid Brazdil __deactivate_traps(vcpu); 22209cf57ebSDavid Brazdil __deactivate_vm(vcpu); 22309cf57ebSDavid Brazdil 22409cf57ebSDavid Brazdil __sysreg_restore_state_nvhe(host_ctxt); 22509cf57ebSDavid Brazdil 22609cf57ebSDavid Brazdil if (vcpu->arch.flags & KVM_ARM64_FP_ENABLED) 22709cf57ebSDavid Brazdil __fpsimd_save_fpexc32(vcpu); 22809cf57ebSDavid Brazdil 22909cf57ebSDavid Brazdil /* 23009cf57ebSDavid Brazdil * This must come after restoring the host sysregs, since a non-VHE 23109cf57ebSDavid Brazdil * system may enable SPE here and make use of the TTBRs. 23209cf57ebSDavid Brazdil */ 23309cf57ebSDavid Brazdil __debug_switch_to_host(vcpu); 23409cf57ebSDavid Brazdil 23509cf57ebSDavid Brazdil if (pmu_switch_needed) 23609cf57ebSDavid Brazdil __pmu_switch_to_host(host_ctxt); 23709cf57ebSDavid Brazdil 23809cf57ebSDavid Brazdil /* Returning to host will clear PSR.I, remask PMR if needed */ 23909cf57ebSDavid Brazdil if (system_uses_irq_prio_masking()) 24009cf57ebSDavid Brazdil gic_write_pmr(GIC_PRIO_IRQOFF); 24109cf57ebSDavid Brazdil 24209cf57ebSDavid Brazdil return exit_code; 24309cf57ebSDavid Brazdil } 24409cf57ebSDavid Brazdil 245c50cb043SDavid Brazdil void __noreturn hyp_panic(struct kvm_cpu_context *host_ctxt) 24609cf57ebSDavid Brazdil { 24709cf57ebSDavid Brazdil u64 spsr = read_sysreg_el2(SYS_SPSR); 24809cf57ebSDavid Brazdil u64 elr = read_sysreg_el2(SYS_ELR); 24909cf57ebSDavid Brazdil u64 par = read_sysreg(par_el1); 25009cf57ebSDavid Brazdil struct kvm_vcpu *vcpu = host_ctxt->__hyp_running_vcpu; 25109cf57ebSDavid Brazdil unsigned long str_va; 25209cf57ebSDavid Brazdil 25309cf57ebSDavid Brazdil if (read_sysreg(vttbr_el2)) { 25409cf57ebSDavid Brazdil __timer_disable_traps(vcpu); 25509cf57ebSDavid Brazdil __deactivate_traps(vcpu); 25609cf57ebSDavid Brazdil __deactivate_vm(vcpu); 25709cf57ebSDavid Brazdil __sysreg_restore_state_nvhe(host_ctxt); 25809cf57ebSDavid Brazdil } 25909cf57ebSDavid Brazdil 26009cf57ebSDavid Brazdil /* 26109cf57ebSDavid Brazdil * Force the panic string to be loaded from the literal pool, 26209cf57ebSDavid Brazdil * making sure it is a kernel address and not a PC-relative 26309cf57ebSDavid Brazdil * reference. 26409cf57ebSDavid Brazdil */ 26509cf57ebSDavid Brazdil asm volatile("ldr %0, =%1" : "=r" (str_va) : "S" (__hyp_panic_string)); 26609cf57ebSDavid Brazdil 26709cf57ebSDavid Brazdil __hyp_do_panic(str_va, 26809cf57ebSDavid Brazdil spsr, elr, 26909cf57ebSDavid Brazdil read_sysreg(esr_el2), read_sysreg_el2(SYS_FAR), 27009cf57ebSDavid Brazdil read_sysreg(hpfar_el2), par, vcpu); 27109cf57ebSDavid Brazdil unreachable(); 27209cf57ebSDavid Brazdil } 273e9ee186bSJames Morse 274e9ee186bSJames Morse asmlinkage void kvm_unexpected_el2_exception(void) 275e9ee186bSJames Morse { 276e9ee186bSJames Morse return __kvm_unexpected_el2_exception(); 277e9ee186bSJames Morse } 278