1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * Copyright (C) 2016 - Linaro and Columbia University 4 * Author: Jintack Lim <jintack.lim@linaro.org> 5 */ 6 7 #include <linux/kvm.h> 8 #include <linux/kvm_host.h> 9 10 #include <asm/kvm_emulate.h> 11 #include <asm/kvm_nested.h> 12 13 #include "hyp/include/hyp/adjust_pc.h" 14 15 #include "trace.h" 16 17 static u64 kvm_check_illegal_exception_return(struct kvm_vcpu *vcpu, u64 spsr) 18 { 19 u64 mode = spsr & PSR_MODE_MASK; 20 21 /* 22 * Possible causes for an Illegal Exception Return from EL2: 23 * - trying to return to EL3 24 * - trying to return to an illegal M value 25 * - trying to return to a 32bit EL 26 * - trying to return to EL1 with HCR_EL2.TGE set 27 */ 28 if (mode == PSR_MODE_EL3t || mode == PSR_MODE_EL3h || 29 mode == 0b00001 || (mode & BIT(1)) || 30 (spsr & PSR_MODE32_BIT) || 31 (vcpu_el2_tge_is_set(vcpu) && (mode == PSR_MODE_EL1t || 32 mode == PSR_MODE_EL1h))) { 33 /* 34 * The guest is playing with our nerves. Preserve EL, SP, 35 * masks, flags from the existing PSTATE, and set IL. 36 * The HW will then generate an Illegal State Exception 37 * immediately after ERET. 38 */ 39 spsr = *vcpu_cpsr(vcpu); 40 41 spsr &= (PSR_D_BIT | PSR_A_BIT | PSR_I_BIT | PSR_F_BIT | 42 PSR_N_BIT | PSR_Z_BIT | PSR_C_BIT | PSR_V_BIT | 43 PSR_MODE_MASK | PSR_MODE32_BIT); 44 spsr |= PSR_IL_BIT; 45 } 46 47 return spsr; 48 } 49 50 void kvm_emulate_nested_eret(struct kvm_vcpu *vcpu) 51 { 52 u64 spsr, elr, mode; 53 bool direct_eret; 54 55 /* 56 * Going through the whole put/load motions is a waste of time 57 * if this is a VHE guest hypervisor returning to its own 58 * userspace, or the hypervisor performing a local exception 59 * return. No need to save/restore registers, no need to 60 * switch S2 MMU. Just do the canonical ERET. 61 */ 62 spsr = vcpu_read_sys_reg(vcpu, SPSR_EL2); 63 spsr = kvm_check_illegal_exception_return(vcpu, spsr); 64 65 mode = spsr & (PSR_MODE_MASK | PSR_MODE32_BIT); 66 67 direct_eret = (mode == PSR_MODE_EL0t && 68 vcpu_el2_e2h_is_set(vcpu) && 69 vcpu_el2_tge_is_set(vcpu)); 70 direct_eret |= (mode == PSR_MODE_EL2h || mode == PSR_MODE_EL2t); 71 72 if (direct_eret) { 73 *vcpu_pc(vcpu) = vcpu_read_sys_reg(vcpu, ELR_EL2); 74 *vcpu_cpsr(vcpu) = spsr; 75 trace_kvm_nested_eret(vcpu, *vcpu_pc(vcpu), spsr); 76 return; 77 } 78 79 preempt_disable(); 80 kvm_arch_vcpu_put(vcpu); 81 82 elr = __vcpu_sys_reg(vcpu, ELR_EL2); 83 84 trace_kvm_nested_eret(vcpu, elr, spsr); 85 86 /* 87 * Note that the current exception level is always the virtual EL2, 88 * since we set HCR_EL2.NV bit only when entering the virtual EL2. 89 */ 90 *vcpu_pc(vcpu) = elr; 91 *vcpu_cpsr(vcpu) = spsr; 92 93 kvm_arch_vcpu_load(vcpu, smp_processor_id()); 94 preempt_enable(); 95 } 96 97 static void kvm_inject_el2_exception(struct kvm_vcpu *vcpu, u64 esr_el2, 98 enum exception_type type) 99 { 100 trace_kvm_inject_nested_exception(vcpu, esr_el2, type); 101 102 switch (type) { 103 case except_type_sync: 104 kvm_pend_exception(vcpu, EXCEPT_AA64_EL2_SYNC); 105 vcpu_write_sys_reg(vcpu, esr_el2, ESR_EL2); 106 break; 107 case except_type_irq: 108 kvm_pend_exception(vcpu, EXCEPT_AA64_EL2_IRQ); 109 break; 110 default: 111 WARN_ONCE(1, "Unsupported EL2 exception injection %d\n", type); 112 } 113 } 114 115 /* 116 * Emulate taking an exception to EL2. 117 * See ARM ARM J8.1.2 AArch64.TakeException() 118 */ 119 static int kvm_inject_nested(struct kvm_vcpu *vcpu, u64 esr_el2, 120 enum exception_type type) 121 { 122 u64 pstate, mode; 123 bool direct_inject; 124 125 if (!vcpu_has_nv(vcpu)) { 126 kvm_err("Unexpected call to %s for the non-nesting configuration\n", 127 __func__); 128 return -EINVAL; 129 } 130 131 /* 132 * As for ERET, we can avoid doing too much on the injection path by 133 * checking that we either took the exception from a VHE host 134 * userspace or from vEL2. In these cases, there is no change in 135 * translation regime (or anything else), so let's do as little as 136 * possible. 137 */ 138 pstate = *vcpu_cpsr(vcpu); 139 mode = pstate & (PSR_MODE_MASK | PSR_MODE32_BIT); 140 141 direct_inject = (mode == PSR_MODE_EL0t && 142 vcpu_el2_e2h_is_set(vcpu) && 143 vcpu_el2_tge_is_set(vcpu)); 144 direct_inject |= (mode == PSR_MODE_EL2h || mode == PSR_MODE_EL2t); 145 146 if (direct_inject) { 147 kvm_inject_el2_exception(vcpu, esr_el2, type); 148 return 1; 149 } 150 151 preempt_disable(); 152 153 /* 154 * We may have an exception or PC update in the EL0/EL1 context. 155 * Commit it before entering EL2. 156 */ 157 __kvm_adjust_pc(vcpu); 158 159 kvm_arch_vcpu_put(vcpu); 160 161 kvm_inject_el2_exception(vcpu, esr_el2, type); 162 163 /* 164 * A hard requirement is that a switch between EL1 and EL2 165 * contexts has to happen between a put/load, so that we can 166 * pick the correct timer and interrupt configuration, among 167 * other things. 168 * 169 * Make sure the exception actually took place before we load 170 * the new context. 171 */ 172 __kvm_adjust_pc(vcpu); 173 174 kvm_arch_vcpu_load(vcpu, smp_processor_id()); 175 preempt_enable(); 176 177 return 1; 178 } 179 180 int kvm_inject_nested_sync(struct kvm_vcpu *vcpu, u64 esr_el2) 181 { 182 return kvm_inject_nested(vcpu, esr_el2, except_type_sync); 183 } 184 185 int kvm_inject_nested_irq(struct kvm_vcpu *vcpu) 186 { 187 /* 188 * Do not inject an irq if the: 189 * - Current exception level is EL2, and 190 * - virtual HCR_EL2.TGE == 0 191 * - virtual HCR_EL2.IMO == 0 192 * 193 * See Table D1-17 "Physical interrupt target and masking when EL3 is 194 * not implemented and EL2 is implemented" in ARM DDI 0487C.a. 195 */ 196 197 if (vcpu_is_el2(vcpu) && !vcpu_el2_tge_is_set(vcpu) && 198 !(__vcpu_sys_reg(vcpu, HCR_EL2) & HCR_IMO)) 199 return 1; 200 201 /* esr_el2 value doesn't matter for exits due to irqs. */ 202 return kvm_inject_nested(vcpu, 0, except_type_irq); 203 } 204