1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * Copyright (C) 2012-2015 - ARM Ltd 4 * Author: Marc Zyngier <marc.zyngier@arm.com> 5 */ 6 7 #include <hyp/sysreg-sr.h> 8 9 #include <linux/compiler.h> 10 #include <linux/kvm_host.h> 11 12 #include <asm/kprobes.h> 13 #include <asm/kvm_asm.h> 14 #include <asm/kvm_emulate.h> 15 #include <asm/kvm_hyp.h> 16 #include <asm/kvm_nested.h> 17 18 /* 19 * VHE: Host and guest must save mdscr_el1 and sp_el0 (and the PC and 20 * pstate, which are handled as part of the el2 return state) on every 21 * switch (sp_el0 is being dealt with in the assembly code). 22 * tpidr_el0 and tpidrro_el0 only need to be switched when going 23 * to host userspace or a different VCPU. EL1 registers only need to be 24 * switched when potentially going to run a different VCPU. The latter two 25 * classes are handled as part of kvm_arch_vcpu_load and kvm_arch_vcpu_put. 26 */ 27 28 void sysreg_save_host_state_vhe(struct kvm_cpu_context *ctxt) 29 { 30 __sysreg_save_common_state(ctxt); 31 } 32 NOKPROBE_SYMBOL(sysreg_save_host_state_vhe); 33 34 void sysreg_save_guest_state_vhe(struct kvm_cpu_context *ctxt) 35 { 36 __sysreg_save_common_state(ctxt); 37 __sysreg_save_el2_return_state(ctxt); 38 } 39 NOKPROBE_SYMBOL(sysreg_save_guest_state_vhe); 40 41 void sysreg_restore_host_state_vhe(struct kvm_cpu_context *ctxt) 42 { 43 __sysreg_restore_common_state(ctxt); 44 } 45 NOKPROBE_SYMBOL(sysreg_restore_host_state_vhe); 46 47 void sysreg_restore_guest_state_vhe(struct kvm_cpu_context *ctxt) 48 { 49 __sysreg_restore_common_state(ctxt); 50 __sysreg_restore_el2_return_state(ctxt); 51 } 52 NOKPROBE_SYMBOL(sysreg_restore_guest_state_vhe); 53 54 /** 55 * kvm_vcpu_load_sysregs_vhe - Load guest system registers to the physical CPU 56 * 57 * @vcpu: The VCPU pointer 58 * 59 * Load system registers that do not affect the host's execution, for 60 * example EL1 system registers on a VHE system where the host kernel 61 * runs at EL2. This function is called from KVM's vcpu_load() function 62 * and loading system register state early avoids having to load them on 63 * every entry to the VM. 64 */ 65 void kvm_vcpu_load_sysregs_vhe(struct kvm_vcpu *vcpu) 66 { 67 struct kvm_cpu_context *guest_ctxt = &vcpu->arch.ctxt; 68 struct kvm_cpu_context *host_ctxt; 69 70 host_ctxt = &this_cpu_ptr(&kvm_host_data)->host_ctxt; 71 __sysreg_save_user_state(host_ctxt); 72 73 /* 74 * When running a normal EL1 guest, we only load a new vcpu 75 * after a context switch, which imvolves a DSB, so all 76 * speculative EL1&0 walks will have already completed. 77 * If running NV, the vcpu may transition between vEL1 and 78 * vEL2 without a context switch, so make sure we complete 79 * those walks before loading a new context. 80 */ 81 if (vcpu_has_nv(vcpu)) 82 dsb(nsh); 83 84 /* 85 * Load guest EL1 and user state 86 * 87 * We must restore the 32-bit state before the sysregs, thanks 88 * to erratum #852523 (Cortex-A57) or #853709 (Cortex-A72). 89 */ 90 __sysreg32_restore_state(vcpu); 91 __sysreg_restore_user_state(guest_ctxt); 92 __sysreg_restore_el1_state(guest_ctxt); 93 94 vcpu_set_flag(vcpu, SYSREGS_ON_CPU); 95 96 activate_traps_vhe_load(vcpu); 97 } 98 99 /** 100 * kvm_vcpu_put_sysregs_vhe - Restore host system registers to the physical CPU 101 * 102 * @vcpu: The VCPU pointer 103 * 104 * Save guest system registers that do not affect the host's execution, for 105 * example EL1 system registers on a VHE system where the host kernel 106 * runs at EL2. This function is called from KVM's vcpu_put() function 107 * and deferring saving system register state until we're no longer running the 108 * VCPU avoids having to save them on every exit from the VM. 109 */ 110 void kvm_vcpu_put_sysregs_vhe(struct kvm_vcpu *vcpu) 111 { 112 struct kvm_cpu_context *guest_ctxt = &vcpu->arch.ctxt; 113 struct kvm_cpu_context *host_ctxt; 114 115 host_ctxt = &this_cpu_ptr(&kvm_host_data)->host_ctxt; 116 deactivate_traps_vhe_put(vcpu); 117 118 __sysreg_save_el1_state(guest_ctxt); 119 __sysreg_save_user_state(guest_ctxt); 120 __sysreg32_save_state(vcpu); 121 122 /* Restore host user state */ 123 __sysreg_restore_user_state(host_ctxt); 124 125 vcpu_clear_flag(vcpu, SYSREGS_ON_CPU); 126 } 127