109cf57ebSDavid Brazdil // SPDX-License-Identifier: GPL-2.0-only
209cf57ebSDavid Brazdil /*
309cf57ebSDavid Brazdil * Copyright (C) 2015 - ARM Ltd
409cf57ebSDavid Brazdil * Author: Marc Zyngier <marc.zyngier@arm.com>
509cf57ebSDavid Brazdil */
609cf57ebSDavid Brazdil
709cf57ebSDavid Brazdil #include <hyp/switch.h>
809cf57ebSDavid Brazdil
909cf57ebSDavid Brazdil #include <linux/arm-smccc.h>
1009cf57ebSDavid Brazdil #include <linux/kvm_host.h>
1109cf57ebSDavid Brazdil #include <linux/types.h>
1209cf57ebSDavid Brazdil #include <linux/jump_label.h>
13bd09128dSJames Morse #include <linux/percpu.h>
1409cf57ebSDavid Brazdil #include <uapi/linux/psci.h>
1509cf57ebSDavid Brazdil
1609cf57ebSDavid Brazdil #include <kvm/arm_psci.h>
1709cf57ebSDavid Brazdil
1809cf57ebSDavid Brazdil #include <asm/barrier.h>
1909cf57ebSDavid Brazdil #include <asm/cpufeature.h>
2009cf57ebSDavid Brazdil #include <asm/kprobes.h>
2109cf57ebSDavid Brazdil #include <asm/kvm_asm.h>
2209cf57ebSDavid Brazdil #include <asm/kvm_emulate.h>
2309cf57ebSDavid Brazdil #include <asm/kvm_hyp.h>
2409cf57ebSDavid Brazdil #include <asm/kvm_mmu.h>
2509cf57ebSDavid Brazdil #include <asm/fpsimd.h>
2609cf57ebSDavid Brazdil #include <asm/debug-monitors.h>
2709cf57ebSDavid Brazdil #include <asm/processor.h>
28bd09128dSJames Morse #include <asm/thread_info.h>
29bd09128dSJames Morse #include <asm/vectors.h>
3009cf57ebSDavid Brazdil
3114ef9d04SMarc Zyngier /* VHE specific context */
3214ef9d04SMarc Zyngier DEFINE_PER_CPU(struct kvm_host_data, kvm_host_data);
3314ef9d04SMarc Zyngier DEFINE_PER_CPU(struct kvm_cpu_context, kvm_hyp_ctxt);
3414ef9d04SMarc Zyngier DEFINE_PER_CPU(unsigned long, kvm_hyp_vector);
352a1198c9SDavid Brazdil
__activate_traps(struct kvm_vcpu * vcpu)3609cf57ebSDavid Brazdil static void __activate_traps(struct kvm_vcpu *vcpu)
3709cf57ebSDavid Brazdil {
3809cf57ebSDavid Brazdil u64 val;
3909cf57ebSDavid Brazdil
4009cf57ebSDavid Brazdil ___activate_traps(vcpu);
4109cf57ebSDavid Brazdil
42*94046732SMarc Zyngier if (has_cntpoff()) {
43*94046732SMarc Zyngier struct timer_map map;
44*94046732SMarc Zyngier
45*94046732SMarc Zyngier get_timer_map(vcpu, &map);
46*94046732SMarc Zyngier
47*94046732SMarc Zyngier /*
48*94046732SMarc Zyngier * We're entrering the guest. Reload the correct
49*94046732SMarc Zyngier * values from memory now that TGE is clear.
50*94046732SMarc Zyngier */
51*94046732SMarc Zyngier if (map.direct_ptimer == vcpu_ptimer(vcpu))
52*94046732SMarc Zyngier val = __vcpu_sys_reg(vcpu, CNTP_CVAL_EL0);
53*94046732SMarc Zyngier if (map.direct_ptimer == vcpu_hptimer(vcpu))
54*94046732SMarc Zyngier val = __vcpu_sys_reg(vcpu, CNTHP_CVAL_EL2);
55*94046732SMarc Zyngier
56*94046732SMarc Zyngier if (map.direct_ptimer) {
57*94046732SMarc Zyngier write_sysreg_el0(val, SYS_CNTP_CVAL);
58*94046732SMarc Zyngier isb();
59*94046732SMarc Zyngier }
60*94046732SMarc Zyngier }
61*94046732SMarc Zyngier
6209cf57ebSDavid Brazdil val = read_sysreg(cpacr_el1);
637a5e9c8fSMarc Zyngier val |= CPACR_ELx_TTA;
6451729fb1SMark Brown val &= ~(CPACR_EL1_ZEN_EL0EN | CPACR_EL1_ZEN_EL1EN |
6551729fb1SMark Brown CPACR_EL1_SMEN_EL0EN | CPACR_EL1_SMEN_EL1EN);
6609cf57ebSDavid Brazdil
6709cf57ebSDavid Brazdil /*
6809cf57ebSDavid Brazdil * With VHE (HCR.E2H == 1), accesses to CPACR_EL1 are routed to
6909cf57ebSDavid Brazdil * CPTR_EL2. In general, CPACR_EL1 has the same layout as CPTR_EL2,
7009cf57ebSDavid Brazdil * except for some missing controls, such as TAM.
7109cf57ebSDavid Brazdil * In this case, CPTR_EL2.TAM has the same position with or without
7209cf57ebSDavid Brazdil * VHE (HCR.E2H == 1) which allows us to use here the CPTR_EL2.TAM
7309cf57ebSDavid Brazdil * shift value for trapping the AMU accesses.
7409cf57ebSDavid Brazdil */
7509cf57ebSDavid Brazdil
7609cf57ebSDavid Brazdil val |= CPTR_EL2_TAM;
7709cf57ebSDavid Brazdil
78e9ada6c2SMarc Zyngier if (guest_owns_fp_regs(vcpu)) {
7909cf57ebSDavid Brazdil if (vcpu_has_sve(vcpu))
803bb72d86SMark Brown val |= CPACR_EL1_ZEN_EL0EN | CPACR_EL1_ZEN_EL1EN;
8109cf57ebSDavid Brazdil } else {
823bb72d86SMark Brown val &= ~(CPACR_EL1_FPEN_EL0EN | CPACR_EL1_FPEN_EL1EN);
8309cf57ebSDavid Brazdil __activate_traps_fpsimd32(vcpu);
8409cf57ebSDavid Brazdil }
8509cf57ebSDavid Brazdil
8609cf57ebSDavid Brazdil write_sysreg(val, cpacr_el1);
8709cf57ebSDavid Brazdil
88a0e47952SAndrew Scull write_sysreg(__this_cpu_read(kvm_hyp_vector), vbar_el1);
8909cf57ebSDavid Brazdil }
9009cf57ebSDavid Brazdil NOKPROBE_SYMBOL(__activate_traps);
9109cf57ebSDavid Brazdil
__deactivate_traps(struct kvm_vcpu * vcpu)9209cf57ebSDavid Brazdil static void __deactivate_traps(struct kvm_vcpu *vcpu)
9309cf57ebSDavid Brazdil {
94bd09128dSJames Morse const char *host_vectors = vectors;
9509cf57ebSDavid Brazdil
9609cf57ebSDavid Brazdil ___deactivate_traps(vcpu);
9709cf57ebSDavid Brazdil
9809cf57ebSDavid Brazdil write_sysreg(HCR_HOST_VHE_FLAGS, hcr_el2);
9909cf57ebSDavid Brazdil
100*94046732SMarc Zyngier if (has_cntpoff()) {
101*94046732SMarc Zyngier struct timer_map map;
102*94046732SMarc Zyngier u64 val, offset;
103*94046732SMarc Zyngier
104*94046732SMarc Zyngier get_timer_map(vcpu, &map);
105*94046732SMarc Zyngier
106*94046732SMarc Zyngier /*
107*94046732SMarc Zyngier * We're exiting the guest. Save the latest CVAL value
108*94046732SMarc Zyngier * to memory and apply the offset now that TGE is set.
109*94046732SMarc Zyngier */
110*94046732SMarc Zyngier val = read_sysreg_el0(SYS_CNTP_CVAL);
111*94046732SMarc Zyngier if (map.direct_ptimer == vcpu_ptimer(vcpu))
112*94046732SMarc Zyngier __vcpu_sys_reg(vcpu, CNTP_CVAL_EL0) = val;
113*94046732SMarc Zyngier if (map.direct_ptimer == vcpu_hptimer(vcpu))
114*94046732SMarc Zyngier __vcpu_sys_reg(vcpu, CNTHP_CVAL_EL2) = val;
115*94046732SMarc Zyngier
116*94046732SMarc Zyngier offset = read_sysreg_s(SYS_CNTPOFF_EL2);
117*94046732SMarc Zyngier
118*94046732SMarc Zyngier if (map.direct_ptimer && offset) {
119*94046732SMarc Zyngier write_sysreg_el0(val + offset, SYS_CNTP_CVAL);
120*94046732SMarc Zyngier isb();
121*94046732SMarc Zyngier }
122*94046732SMarc Zyngier }
123*94046732SMarc Zyngier
12409cf57ebSDavid Brazdil /*
12509cf57ebSDavid Brazdil * ARM errata 1165522 and 1530923 require the actual execution of the
12609cf57ebSDavid Brazdil * above before we can switch to the EL2/EL0 translation regime used by
12709cf57ebSDavid Brazdil * the host.
12809cf57ebSDavid Brazdil */
12909cf57ebSDavid Brazdil asm(ALTERNATIVE("nop", "isb", ARM64_WORKAROUND_SPECULATIVE_AT));
13009cf57ebSDavid Brazdil
13175c76ab5SMarc Zyngier kvm_reset_cptr_el2(vcpu);
132bd09128dSJames Morse
133bd09128dSJames Morse if (!arm64_kernel_unmapped_at_el0())
134bd09128dSJames Morse host_vectors = __this_cpu_read(this_cpu_vector);
135bd09128dSJames Morse write_sysreg(host_vectors, vbar_el1);
13609cf57ebSDavid Brazdil }
13709cf57ebSDavid Brazdil NOKPROBE_SYMBOL(__deactivate_traps);
13809cf57ebSDavid Brazdil
1390c2f9acfSReiji Watanabe /*
1400c2f9acfSReiji Watanabe * Disable IRQs in {activate,deactivate}_traps_vhe_{load,put}() to
1410c2f9acfSReiji Watanabe * prevent a race condition between context switching of PMUSERENR_EL0
1420c2f9acfSReiji Watanabe * in __{activate,deactivate}_traps_common() and IPIs that attempts to
1430c2f9acfSReiji Watanabe * update PMUSERENR_EL0. See also kvm_set_pmuserenr().
1440c2f9acfSReiji Watanabe */
activate_traps_vhe_load(struct kvm_vcpu * vcpu)14509cf57ebSDavid Brazdil void activate_traps_vhe_load(struct kvm_vcpu *vcpu)
14609cf57ebSDavid Brazdil {
1470c2f9acfSReiji Watanabe unsigned long flags;
1480c2f9acfSReiji Watanabe
1490c2f9acfSReiji Watanabe local_irq_save(flags);
15009cf57ebSDavid Brazdil __activate_traps_common(vcpu);
1510c2f9acfSReiji Watanabe local_irq_restore(flags);
15209cf57ebSDavid Brazdil }
15309cf57ebSDavid Brazdil
deactivate_traps_vhe_put(struct kvm_vcpu * vcpu)1541460b4b2SFuad Tabba void deactivate_traps_vhe_put(struct kvm_vcpu *vcpu)
15509cf57ebSDavid Brazdil {
1560c2f9acfSReiji Watanabe unsigned long flags;
1570c2f9acfSReiji Watanabe
1580c2f9acfSReiji Watanabe local_irq_save(flags);
1591460b4b2SFuad Tabba __deactivate_traps_common(vcpu);
1600c2f9acfSReiji Watanabe local_irq_restore(flags);
16109cf57ebSDavid Brazdil }
16209cf57ebSDavid Brazdil
1638fb20461SMarc Zyngier static const exit_handler_fn hyp_exit_handlers[] = {
1648fb20461SMarc Zyngier [0 ... ESR_ELx_EC_MAX] = NULL,
1658fb20461SMarc Zyngier [ESR_ELx_EC_CP15_32] = kvm_hyp_handle_cp15_32,
1668fb20461SMarc Zyngier [ESR_ELx_EC_SYS64] = kvm_hyp_handle_sysreg,
1678fb20461SMarc Zyngier [ESR_ELx_EC_SVE] = kvm_hyp_handle_fpsimd,
1688fb20461SMarc Zyngier [ESR_ELx_EC_FP_ASIMD] = kvm_hyp_handle_fpsimd,
1698fb20461SMarc Zyngier [ESR_ELx_EC_IABT_LOW] = kvm_hyp_handle_iabt_low,
1708fb20461SMarc Zyngier [ESR_ELx_EC_DABT_LOW] = kvm_hyp_handle_dabt_low,
171811154e2SAkihiko Odaki [ESR_ELx_EC_WATCHPT_LOW] = kvm_hyp_handle_watchpt_low,
1728fb20461SMarc Zyngier [ESR_ELx_EC_PAC] = kvm_hyp_handle_ptrauth,
1738fb20461SMarc Zyngier };
1748fb20461SMarc Zyngier
kvm_get_exit_handler_array(struct kvm_vcpu * vcpu)1750c7639ccSMarc Zyngier static const exit_handler_fn *kvm_get_exit_handler_array(struct kvm_vcpu *vcpu)
1768fb20461SMarc Zyngier {
1778fb20461SMarc Zyngier return hyp_exit_handlers;
1788fb20461SMarc Zyngier }
1798fb20461SMarc Zyngier
early_exit_filter(struct kvm_vcpu * vcpu,u64 * exit_code)1807183b2b5SMarc Zyngier static void early_exit_filter(struct kvm_vcpu *vcpu, u64 *exit_code)
1817183b2b5SMarc Zyngier {
182d9552fe1SMarc Zyngier /*
183d9552fe1SMarc Zyngier * If we were in HYP context on entry, adjust the PSTATE view
184d9552fe1SMarc Zyngier * so that the usual helpers work correctly.
185d9552fe1SMarc Zyngier */
186d9552fe1SMarc Zyngier if (unlikely(vcpu_get_flag(vcpu, VCPU_HYP_CONTEXT))) {
187d9552fe1SMarc Zyngier u64 mode = *vcpu_cpsr(vcpu) & (PSR_MODE_MASK | PSR_MODE32_BIT);
188d9552fe1SMarc Zyngier
189d9552fe1SMarc Zyngier switch (mode) {
190d9552fe1SMarc Zyngier case PSR_MODE_EL1t:
191d9552fe1SMarc Zyngier mode = PSR_MODE_EL2t;
192d9552fe1SMarc Zyngier break;
193d9552fe1SMarc Zyngier case PSR_MODE_EL1h:
194d9552fe1SMarc Zyngier mode = PSR_MODE_EL2h;
195d9552fe1SMarc Zyngier break;
196d9552fe1SMarc Zyngier }
197d9552fe1SMarc Zyngier
198d9552fe1SMarc Zyngier *vcpu_cpsr(vcpu) &= ~(PSR_MODE_MASK | PSR_MODE32_BIT);
199d9552fe1SMarc Zyngier *vcpu_cpsr(vcpu) |= mode;
200d9552fe1SMarc Zyngier }
2017183b2b5SMarc Zyngier }
2027183b2b5SMarc Zyngier
20309cf57ebSDavid Brazdil /* Switch to the guest for VHE systems running in EL2 */
__kvm_vcpu_run_vhe(struct kvm_vcpu * vcpu)20409cf57ebSDavid Brazdil static int __kvm_vcpu_run_vhe(struct kvm_vcpu *vcpu)
20509cf57ebSDavid Brazdil {
20609cf57ebSDavid Brazdil struct kvm_cpu_context *host_ctxt;
20709cf57ebSDavid Brazdil struct kvm_cpu_context *guest_ctxt;
20809cf57ebSDavid Brazdil u64 exit_code;
20909cf57ebSDavid Brazdil
210717cf94aSDavid Brazdil host_ctxt = &this_cpu_ptr(&kvm_host_data)->host_ctxt;
21109cf57ebSDavid Brazdil host_ctxt->__hyp_running_vcpu = vcpu;
21209cf57ebSDavid Brazdil guest_ctxt = &vcpu->arch.ctxt;
21309cf57ebSDavid Brazdil
21409cf57ebSDavid Brazdil sysreg_save_host_state_vhe(host_ctxt);
21509cf57ebSDavid Brazdil
21609cf57ebSDavid Brazdil /*
21709cf57ebSDavid Brazdil * ARM erratum 1165522 requires us to configure both stage 1 and
21809cf57ebSDavid Brazdil * stage 2 translation for the guest context before we clear
21909cf57ebSDavid Brazdil * HCR_EL2.TGE.
22009cf57ebSDavid Brazdil *
22109cf57ebSDavid Brazdil * We have already configured the guest's stage 1 translation in
222501a67a2SAndrew Scull * kvm_vcpu_load_sysregs_vhe above. We must now call
2234efc0edeSMarc Zyngier * __load_stage2 before __activate_traps, because
2244efc0edeSMarc Zyngier * __load_stage2 configures stage 2 translation, and
225501a67a2SAndrew Scull * __activate_traps clear HCR_EL2.TGE (among other things).
22609cf57ebSDavid Brazdil */
2274efc0edeSMarc Zyngier __load_stage2(vcpu->arch.hw_mmu, vcpu->arch.hw_mmu->arch);
22809cf57ebSDavid Brazdil __activate_traps(vcpu);
22909cf57ebSDavid Brazdil
230f5e30680SMarc Zyngier __kvm_adjust_pc(vcpu);
231cdb5e02eSMarc Zyngier
23209cf57ebSDavid Brazdil sysreg_restore_guest_state_vhe(guest_ctxt);
23309cf57ebSDavid Brazdil __debug_switch_to_guest(vcpu);
23409cf57ebSDavid Brazdil
235d9552fe1SMarc Zyngier if (is_hyp_ctxt(vcpu))
236d9552fe1SMarc Zyngier vcpu_set_flag(vcpu, VCPU_HYP_CONTEXT);
237d9552fe1SMarc Zyngier else
238d9552fe1SMarc Zyngier vcpu_clear_flag(vcpu, VCPU_HYP_CONTEXT);
239d9552fe1SMarc Zyngier
24009cf57ebSDavid Brazdil do {
24109cf57ebSDavid Brazdil /* Jump in the fire! */
242b619d9aaSAndrew Scull exit_code = __guest_enter(vcpu);
24309cf57ebSDavid Brazdil
24409cf57ebSDavid Brazdil /* And we're baaack! */
24509cf57ebSDavid Brazdil } while (fixup_guest_exit(vcpu, &exit_code));
24609cf57ebSDavid Brazdil
24709cf57ebSDavid Brazdil sysreg_save_guest_state_vhe(guest_ctxt);
24809cf57ebSDavid Brazdil
24909cf57ebSDavid Brazdil __deactivate_traps(vcpu);
25009cf57ebSDavid Brazdil
25109cf57ebSDavid Brazdil sysreg_restore_host_state_vhe(host_ctxt);
25209cf57ebSDavid Brazdil
253f8077b0dSMarc Zyngier if (vcpu->arch.fp_state == FP_STATE_GUEST_OWNED)
25409cf57ebSDavid Brazdil __fpsimd_save_fpexc32(vcpu);
25509cf57ebSDavid Brazdil
25609cf57ebSDavid Brazdil __debug_switch_to_host(vcpu);
25709cf57ebSDavid Brazdil
25809cf57ebSDavid Brazdil return exit_code;
25909cf57ebSDavid Brazdil }
26009cf57ebSDavid Brazdil NOKPROBE_SYMBOL(__kvm_vcpu_run_vhe);
26109cf57ebSDavid Brazdil
__kvm_vcpu_run(struct kvm_vcpu * vcpu)26209cf57ebSDavid Brazdil int __kvm_vcpu_run(struct kvm_vcpu *vcpu)
26309cf57ebSDavid Brazdil {
26409cf57ebSDavid Brazdil int ret;
26509cf57ebSDavid Brazdil
26609cf57ebSDavid Brazdil local_daif_mask();
26709cf57ebSDavid Brazdil
26809cf57ebSDavid Brazdil /*
26909cf57ebSDavid Brazdil * Having IRQs masked via PMR when entering the guest means the GIC
27009cf57ebSDavid Brazdil * will not signal the CPU of interrupts of lower priority, and the
27109cf57ebSDavid Brazdil * only way to get out will be via guest exceptions.
27209cf57ebSDavid Brazdil * Naturally, we want to avoid this.
27309cf57ebSDavid Brazdil *
27409cf57ebSDavid Brazdil * local_daif_mask() already sets GIC_PRIO_PSR_I_SET, we just need a
27509cf57ebSDavid Brazdil * dsb to ensure the redistributor is forwards EL2 IRQs to the CPU.
27609cf57ebSDavid Brazdil */
27709cf57ebSDavid Brazdil pmr_sync();
27809cf57ebSDavid Brazdil
27909cf57ebSDavid Brazdil ret = __kvm_vcpu_run_vhe(vcpu);
28009cf57ebSDavid Brazdil
28109cf57ebSDavid Brazdil /*
28209cf57ebSDavid Brazdil * local_daif_restore() takes care to properly restore PSTATE.DAIF
28309cf57ebSDavid Brazdil * and the GIC PMR if the host is using IRQ priorities.
28409cf57ebSDavid Brazdil */
28509cf57ebSDavid Brazdil local_daif_restore(DAIF_PROCCTX_NOIRQ);
28609cf57ebSDavid Brazdil
28709cf57ebSDavid Brazdil /*
28809cf57ebSDavid Brazdil * When we exit from the guest we change a number of CPU configuration
289bcf3e7daSMarc Zyngier * parameters, such as traps. We rely on the isb() in kvm_call_hyp*()
290bcf3e7daSMarc Zyngier * to make sure these changes take effect before running the host or
291bcf3e7daSMarc Zyngier * additional guests.
29209cf57ebSDavid Brazdil */
29309cf57ebSDavid Brazdil return ret;
29409cf57ebSDavid Brazdil }
29509cf57ebSDavid Brazdil
__hyp_call_panic(u64 spsr,u64 elr,u64 par)2966a0259edSAndrew Scull static void __hyp_call_panic(u64 spsr, u64 elr, u64 par)
29709cf57ebSDavid Brazdil {
2986a0259edSAndrew Scull struct kvm_cpu_context *host_ctxt;
29909cf57ebSDavid Brazdil struct kvm_vcpu *vcpu;
3006a0259edSAndrew Scull
30114ef9d04SMarc Zyngier host_ctxt = &this_cpu_ptr(&kvm_host_data)->host_ctxt;
30209cf57ebSDavid Brazdil vcpu = host_ctxt->__hyp_running_vcpu;
30309cf57ebSDavid Brazdil
30409cf57ebSDavid Brazdil __deactivate_traps(vcpu);
30509cf57ebSDavid Brazdil sysreg_restore_host_state_vhe(host_ctxt);
30609cf57ebSDavid Brazdil
307aec0fae6SAndrew Scull panic("HYP panic:\nPS:%08llx PC:%016llx ESR:%08llx\nFAR:%016llx HPFAR:%016llx PAR:%016llx\nVCPU:%p\n",
30809cf57ebSDavid Brazdil spsr, elr,
30909cf57ebSDavid Brazdil read_sysreg_el2(SYS_ESR), read_sysreg_el2(SYS_FAR),
31009cf57ebSDavid Brazdil read_sysreg(hpfar_el2), par, vcpu);
31109cf57ebSDavid Brazdil }
31209cf57ebSDavid Brazdil NOKPROBE_SYMBOL(__hyp_call_panic);
31309cf57ebSDavid Brazdil
hyp_panic(void)3146a0259edSAndrew Scull void __noreturn hyp_panic(void)
31509cf57ebSDavid Brazdil {
31609cf57ebSDavid Brazdil u64 spsr = read_sysreg_el2(SYS_SPSR);
31709cf57ebSDavid Brazdil u64 elr = read_sysreg_el2(SYS_ELR);
31896d389caSRob Herring u64 par = read_sysreg_par();
31909cf57ebSDavid Brazdil
3206a0259edSAndrew Scull __hyp_call_panic(spsr, elr, par);
32109cf57ebSDavid Brazdil unreachable();
32209cf57ebSDavid Brazdil }
323e9ee186bSJames Morse
kvm_unexpected_el2_exception(void)324e9ee186bSJames Morse asmlinkage void kvm_unexpected_el2_exception(void)
325e9ee186bSJames Morse {
3261c3ace2bSQuentin Perret __kvm_unexpected_el2_exception();
327e9ee186bSJames Morse }
328