113aeb9b4SDavid Brazdil // SPDX-License-Identifier: GPL-2.0-only
213aeb9b4SDavid Brazdil /*
313aeb9b4SDavid Brazdil * Copyright (C) 2012-2015 - ARM Ltd
413aeb9b4SDavid Brazdil * Author: Marc Zyngier <marc.zyngier@arm.com>
513aeb9b4SDavid Brazdil */
613aeb9b4SDavid Brazdil
713aeb9b4SDavid Brazdil #include <hyp/sysreg-sr.h>
813aeb9b4SDavid Brazdil
913aeb9b4SDavid Brazdil #include <linux/compiler.h>
1013aeb9b4SDavid Brazdil #include <linux/kvm_host.h>
1113aeb9b4SDavid Brazdil
1213aeb9b4SDavid Brazdil #include <asm/kprobes.h>
1313aeb9b4SDavid Brazdil #include <asm/kvm_asm.h>
1413aeb9b4SDavid Brazdil #include <asm/kvm_emulate.h>
1513aeb9b4SDavid Brazdil #include <asm/kvm_hyp.h>
16*1ff2755dSMarc Zyngier #include <asm/kvm_nested.h>
1713aeb9b4SDavid Brazdil
1813aeb9b4SDavid Brazdil /*
1913aeb9b4SDavid Brazdil * VHE: Host and guest must save mdscr_el1 and sp_el0 (and the PC and
2013aeb9b4SDavid Brazdil * pstate, which are handled as part of the el2 return state) on every
2113aeb9b4SDavid Brazdil * switch (sp_el0 is being dealt with in the assembly code).
2213aeb9b4SDavid Brazdil * tpidr_el0 and tpidrro_el0 only need to be switched when going
2313aeb9b4SDavid Brazdil * to host userspace or a different VCPU. EL1 registers only need to be
2413aeb9b4SDavid Brazdil * switched when potentially going to run a different VCPU. The latter two
2513aeb9b4SDavid Brazdil * classes are handled as part of kvm_arch_vcpu_load and kvm_arch_vcpu_put.
2613aeb9b4SDavid Brazdil */
2713aeb9b4SDavid Brazdil
sysreg_save_host_state_vhe(struct kvm_cpu_context * ctxt)2813aeb9b4SDavid Brazdil void sysreg_save_host_state_vhe(struct kvm_cpu_context *ctxt)
2913aeb9b4SDavid Brazdil {
3013aeb9b4SDavid Brazdil __sysreg_save_common_state(ctxt);
3113aeb9b4SDavid Brazdil }
3213aeb9b4SDavid Brazdil NOKPROBE_SYMBOL(sysreg_save_host_state_vhe);
3313aeb9b4SDavid Brazdil
sysreg_save_guest_state_vhe(struct kvm_cpu_context * ctxt)3413aeb9b4SDavid Brazdil void sysreg_save_guest_state_vhe(struct kvm_cpu_context *ctxt)
3513aeb9b4SDavid Brazdil {
3613aeb9b4SDavid Brazdil __sysreg_save_common_state(ctxt);
3713aeb9b4SDavid Brazdil __sysreg_save_el2_return_state(ctxt);
3813aeb9b4SDavid Brazdil }
3913aeb9b4SDavid Brazdil NOKPROBE_SYMBOL(sysreg_save_guest_state_vhe);
4013aeb9b4SDavid Brazdil
sysreg_restore_host_state_vhe(struct kvm_cpu_context * ctxt)4113aeb9b4SDavid Brazdil void sysreg_restore_host_state_vhe(struct kvm_cpu_context *ctxt)
4213aeb9b4SDavid Brazdil {
4313aeb9b4SDavid Brazdil __sysreg_restore_common_state(ctxt);
4413aeb9b4SDavid Brazdil }
4513aeb9b4SDavid Brazdil NOKPROBE_SYMBOL(sysreg_restore_host_state_vhe);
4613aeb9b4SDavid Brazdil
sysreg_restore_guest_state_vhe(struct kvm_cpu_context * ctxt)4713aeb9b4SDavid Brazdil void sysreg_restore_guest_state_vhe(struct kvm_cpu_context *ctxt)
4813aeb9b4SDavid Brazdil {
4913aeb9b4SDavid Brazdil __sysreg_restore_common_state(ctxt);
5013aeb9b4SDavid Brazdil __sysreg_restore_el2_return_state(ctxt);
5113aeb9b4SDavid Brazdil }
5213aeb9b4SDavid Brazdil NOKPROBE_SYMBOL(sysreg_restore_guest_state_vhe);
5313aeb9b4SDavid Brazdil
5413aeb9b4SDavid Brazdil /**
5513aeb9b4SDavid Brazdil * kvm_vcpu_load_sysregs_vhe - Load guest system registers to the physical CPU
5613aeb9b4SDavid Brazdil *
5713aeb9b4SDavid Brazdil * @vcpu: The VCPU pointer
5813aeb9b4SDavid Brazdil *
5913aeb9b4SDavid Brazdil * Load system registers that do not affect the host's execution, for
6013aeb9b4SDavid Brazdil * example EL1 system registers on a VHE system where the host kernel
6113aeb9b4SDavid Brazdil * runs at EL2. This function is called from KVM's vcpu_load() function
6213aeb9b4SDavid Brazdil * and loading system register state early avoids having to load them on
6313aeb9b4SDavid Brazdil * every entry to the VM.
6413aeb9b4SDavid Brazdil */
kvm_vcpu_load_sysregs_vhe(struct kvm_vcpu * vcpu)6513aeb9b4SDavid Brazdil void kvm_vcpu_load_sysregs_vhe(struct kvm_vcpu *vcpu)
6613aeb9b4SDavid Brazdil {
6713aeb9b4SDavid Brazdil struct kvm_cpu_context *guest_ctxt = &vcpu->arch.ctxt;
6813aeb9b4SDavid Brazdil struct kvm_cpu_context *host_ctxt;
6913aeb9b4SDavid Brazdil
70717cf94aSDavid Brazdil host_ctxt = &this_cpu_ptr(&kvm_host_data)->host_ctxt;
7113aeb9b4SDavid Brazdil __sysreg_save_user_state(host_ctxt);
7213aeb9b4SDavid Brazdil
7313aeb9b4SDavid Brazdil /*
74*1ff2755dSMarc Zyngier * When running a normal EL1 guest, we only load a new vcpu
75*1ff2755dSMarc Zyngier * after a context switch, which imvolves a DSB, so all
76*1ff2755dSMarc Zyngier * speculative EL1&0 walks will have already completed.
77*1ff2755dSMarc Zyngier * If running NV, the vcpu may transition between vEL1 and
78*1ff2755dSMarc Zyngier * vEL2 without a context switch, so make sure we complete
79*1ff2755dSMarc Zyngier * those walks before loading a new context.
80*1ff2755dSMarc Zyngier */
81*1ff2755dSMarc Zyngier if (vcpu_has_nv(vcpu))
82*1ff2755dSMarc Zyngier dsb(nsh);
83*1ff2755dSMarc Zyngier
84*1ff2755dSMarc Zyngier /*
8513aeb9b4SDavid Brazdil * Load guest EL1 and user state
8613aeb9b4SDavid Brazdil *
8713aeb9b4SDavid Brazdil * We must restore the 32-bit state before the sysregs, thanks
8813aeb9b4SDavid Brazdil * to erratum #852523 (Cortex-A57) or #853709 (Cortex-A72).
8913aeb9b4SDavid Brazdil */
9013aeb9b4SDavid Brazdil __sysreg32_restore_state(vcpu);
9113aeb9b4SDavid Brazdil __sysreg_restore_user_state(guest_ctxt);
9213aeb9b4SDavid Brazdil __sysreg_restore_el1_state(guest_ctxt);
9313aeb9b4SDavid Brazdil
9430b6ab45SMarc Zyngier vcpu_set_flag(vcpu, SYSREGS_ON_CPU);
9513aeb9b4SDavid Brazdil
9613aeb9b4SDavid Brazdil activate_traps_vhe_load(vcpu);
9713aeb9b4SDavid Brazdil }
9813aeb9b4SDavid Brazdil
9913aeb9b4SDavid Brazdil /**
10013aeb9b4SDavid Brazdil * kvm_vcpu_put_sysregs_vhe - Restore host system registers to the physical CPU
10113aeb9b4SDavid Brazdil *
10213aeb9b4SDavid Brazdil * @vcpu: The VCPU pointer
10313aeb9b4SDavid Brazdil *
10413aeb9b4SDavid Brazdil * Save guest system registers that do not affect the host's execution, for
10513aeb9b4SDavid Brazdil * example EL1 system registers on a VHE system where the host kernel
10613aeb9b4SDavid Brazdil * runs at EL2. This function is called from KVM's vcpu_put() function
10713aeb9b4SDavid Brazdil * and deferring saving system register state until we're no longer running the
10813aeb9b4SDavid Brazdil * VCPU avoids having to save them on every exit from the VM.
10913aeb9b4SDavid Brazdil */
kvm_vcpu_put_sysregs_vhe(struct kvm_vcpu * vcpu)11013aeb9b4SDavid Brazdil void kvm_vcpu_put_sysregs_vhe(struct kvm_vcpu *vcpu)
11113aeb9b4SDavid Brazdil {
11213aeb9b4SDavid Brazdil struct kvm_cpu_context *guest_ctxt = &vcpu->arch.ctxt;
11313aeb9b4SDavid Brazdil struct kvm_cpu_context *host_ctxt;
11413aeb9b4SDavid Brazdil
115717cf94aSDavid Brazdil host_ctxt = &this_cpu_ptr(&kvm_host_data)->host_ctxt;
1161460b4b2SFuad Tabba deactivate_traps_vhe_put(vcpu);
11713aeb9b4SDavid Brazdil
11813aeb9b4SDavid Brazdil __sysreg_save_el1_state(guest_ctxt);
11913aeb9b4SDavid Brazdil __sysreg_save_user_state(guest_ctxt);
12013aeb9b4SDavid Brazdil __sysreg32_save_state(vcpu);
12113aeb9b4SDavid Brazdil
12213aeb9b4SDavid Brazdil /* Restore host user state */
12313aeb9b4SDavid Brazdil __sysreg_restore_user_state(host_ctxt);
12413aeb9b4SDavid Brazdil
12530b6ab45SMarc Zyngier vcpu_clear_flag(vcpu, SYSREGS_ON_CPU);
12613aeb9b4SDavid Brazdil }
127