113aeb9b4SDavid Brazdil // SPDX-License-Identifier: GPL-2.0-only 213aeb9b4SDavid Brazdil /* 313aeb9b4SDavid Brazdil * Copyright (C) 2012-2015 - ARM Ltd 413aeb9b4SDavid Brazdil * Author: Marc Zyngier <marc.zyngier@arm.com> 513aeb9b4SDavid Brazdil */ 613aeb9b4SDavid Brazdil 713aeb9b4SDavid Brazdil #include <hyp/sysreg-sr.h> 813aeb9b4SDavid Brazdil 913aeb9b4SDavid Brazdil #include <linux/compiler.h> 1013aeb9b4SDavid Brazdil #include <linux/kvm_host.h> 1113aeb9b4SDavid Brazdil 1213aeb9b4SDavid Brazdil #include <asm/kprobes.h> 1313aeb9b4SDavid Brazdil #include <asm/kvm_asm.h> 1413aeb9b4SDavid Brazdil #include <asm/kvm_emulate.h> 1513aeb9b4SDavid Brazdil #include <asm/kvm_hyp.h> 1613aeb9b4SDavid Brazdil 1713aeb9b4SDavid Brazdil /* 1813aeb9b4SDavid Brazdil * VHE: Host and guest must save mdscr_el1 and sp_el0 (and the PC and 1913aeb9b4SDavid Brazdil * pstate, which are handled as part of the el2 return state) on every 2013aeb9b4SDavid Brazdil * switch (sp_el0 is being dealt with in the assembly code). 2113aeb9b4SDavid Brazdil * tpidr_el0 and tpidrro_el0 only need to be switched when going 2213aeb9b4SDavid Brazdil * to host userspace or a different VCPU. EL1 registers only need to be 2313aeb9b4SDavid Brazdil * switched when potentially going to run a different VCPU. The latter two 2413aeb9b4SDavid Brazdil * classes are handled as part of kvm_arch_vcpu_load and kvm_arch_vcpu_put. 2513aeb9b4SDavid Brazdil */ 2613aeb9b4SDavid Brazdil 2713aeb9b4SDavid Brazdil void sysreg_save_host_state_vhe(struct kvm_cpu_context *ctxt) 2813aeb9b4SDavid Brazdil { 2913aeb9b4SDavid Brazdil __sysreg_save_common_state(ctxt); 3013aeb9b4SDavid Brazdil } 3113aeb9b4SDavid Brazdil NOKPROBE_SYMBOL(sysreg_save_host_state_vhe); 3213aeb9b4SDavid Brazdil 3313aeb9b4SDavid Brazdil void sysreg_save_guest_state_vhe(struct kvm_cpu_context *ctxt) 3413aeb9b4SDavid Brazdil { 3513aeb9b4SDavid Brazdil __sysreg_save_common_state(ctxt); 3613aeb9b4SDavid Brazdil __sysreg_save_el2_return_state(ctxt); 3713aeb9b4SDavid Brazdil } 3813aeb9b4SDavid Brazdil NOKPROBE_SYMBOL(sysreg_save_guest_state_vhe); 3913aeb9b4SDavid Brazdil 4013aeb9b4SDavid Brazdil void sysreg_restore_host_state_vhe(struct kvm_cpu_context *ctxt) 4113aeb9b4SDavid Brazdil { 4213aeb9b4SDavid Brazdil __sysreg_restore_common_state(ctxt); 4313aeb9b4SDavid Brazdil } 4413aeb9b4SDavid Brazdil NOKPROBE_SYMBOL(sysreg_restore_host_state_vhe); 4513aeb9b4SDavid Brazdil 4613aeb9b4SDavid Brazdil void sysreg_restore_guest_state_vhe(struct kvm_cpu_context *ctxt) 4713aeb9b4SDavid Brazdil { 4813aeb9b4SDavid Brazdil __sysreg_restore_common_state(ctxt); 4913aeb9b4SDavid Brazdil __sysreg_restore_el2_return_state(ctxt); 5013aeb9b4SDavid Brazdil } 5113aeb9b4SDavid Brazdil NOKPROBE_SYMBOL(sysreg_restore_guest_state_vhe); 5213aeb9b4SDavid Brazdil 5313aeb9b4SDavid Brazdil /** 5413aeb9b4SDavid Brazdil * kvm_vcpu_load_sysregs_vhe - Load guest system registers to the physical CPU 5513aeb9b4SDavid Brazdil * 5613aeb9b4SDavid Brazdil * @vcpu: The VCPU pointer 5713aeb9b4SDavid Brazdil * 5813aeb9b4SDavid Brazdil * Load system registers that do not affect the host's execution, for 5913aeb9b4SDavid Brazdil * example EL1 system registers on a VHE system where the host kernel 6013aeb9b4SDavid Brazdil * runs at EL2. This function is called from KVM's vcpu_load() function 6113aeb9b4SDavid Brazdil * and loading system register state early avoids having to load them on 6213aeb9b4SDavid Brazdil * every entry to the VM. 6313aeb9b4SDavid Brazdil */ 6413aeb9b4SDavid Brazdil void kvm_vcpu_load_sysregs_vhe(struct kvm_vcpu *vcpu) 6513aeb9b4SDavid Brazdil { 6613aeb9b4SDavid Brazdil struct kvm_cpu_context *guest_ctxt = &vcpu->arch.ctxt; 6713aeb9b4SDavid Brazdil struct kvm_cpu_context *host_ctxt; 6813aeb9b4SDavid Brazdil 6913aeb9b4SDavid Brazdil host_ctxt = &__hyp_this_cpu_ptr(kvm_host_data)->host_ctxt; 7013aeb9b4SDavid Brazdil __sysreg_save_user_state(host_ctxt); 7113aeb9b4SDavid Brazdil 7213aeb9b4SDavid Brazdil /* 7313aeb9b4SDavid Brazdil * Load guest EL1 and user state 7413aeb9b4SDavid Brazdil * 7513aeb9b4SDavid Brazdil * We must restore the 32-bit state before the sysregs, thanks 7613aeb9b4SDavid Brazdil * to erratum #852523 (Cortex-A57) or #853709 (Cortex-A72). 7713aeb9b4SDavid Brazdil */ 7813aeb9b4SDavid Brazdil __sysreg32_restore_state(vcpu); 7913aeb9b4SDavid Brazdil __sysreg_restore_user_state(guest_ctxt); 8013aeb9b4SDavid Brazdil __sysreg_restore_el1_state(guest_ctxt); 8113aeb9b4SDavid Brazdil 8213aeb9b4SDavid Brazdil vcpu->arch.sysregs_loaded_on_cpu = true; 8313aeb9b4SDavid Brazdil 8413aeb9b4SDavid Brazdil activate_traps_vhe_load(vcpu); 8513aeb9b4SDavid Brazdil } 8613aeb9b4SDavid Brazdil 8713aeb9b4SDavid Brazdil /** 8813aeb9b4SDavid Brazdil * kvm_vcpu_put_sysregs_vhe - Restore host system registers to the physical CPU 8913aeb9b4SDavid Brazdil * 9013aeb9b4SDavid Brazdil * @vcpu: The VCPU pointer 9113aeb9b4SDavid Brazdil * 9213aeb9b4SDavid Brazdil * Save guest system registers that do not affect the host's execution, for 9313aeb9b4SDavid Brazdil * example EL1 system registers on a VHE system where the host kernel 9413aeb9b4SDavid Brazdil * runs at EL2. This function is called from KVM's vcpu_put() function 9513aeb9b4SDavid Brazdil * and deferring saving system register state until we're no longer running the 9613aeb9b4SDavid Brazdil * VCPU avoids having to save them on every exit from the VM. 9713aeb9b4SDavid Brazdil */ 9813aeb9b4SDavid Brazdil void kvm_vcpu_put_sysregs_vhe(struct kvm_vcpu *vcpu) 9913aeb9b4SDavid Brazdil { 10013aeb9b4SDavid Brazdil struct kvm_cpu_context *guest_ctxt = &vcpu->arch.ctxt; 10113aeb9b4SDavid Brazdil struct kvm_cpu_context *host_ctxt; 10213aeb9b4SDavid Brazdil 10313aeb9b4SDavid Brazdil host_ctxt = &__hyp_this_cpu_ptr(kvm_host_data)->host_ctxt; 10413aeb9b4SDavid Brazdil deactivate_traps_vhe_put(); 10513aeb9b4SDavid Brazdil 10613aeb9b4SDavid Brazdil __sysreg_save_el1_state(guest_ctxt); 10713aeb9b4SDavid Brazdil __sysreg_save_user_state(guest_ctxt); 10813aeb9b4SDavid Brazdil __sysreg32_save_state(vcpu); 10913aeb9b4SDavid Brazdil 11013aeb9b4SDavid Brazdil /* Restore host user state */ 11113aeb9b4SDavid Brazdil __sysreg_restore_user_state(host_ctxt); 11213aeb9b4SDavid Brazdil 11313aeb9b4SDavid Brazdil vcpu->arch.sysregs_loaded_on_cpu = false; 11413aeb9b4SDavid Brazdil } 115