1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * Fault injection for both 32 and 64bit guests. 4 * 5 * Copyright (C) 2012,2013 - ARM Ltd 6 * Author: Marc Zyngier <marc.zyngier@arm.com> 7 * 8 * Based on arch/arm/kvm/emulate.c 9 * Copyright (C) 2012 - Virtual Open Systems and Columbia University 10 * Author: Christoffer Dall <c.dall@virtualopensystems.com> 11 */ 12 13 #include <hyp/adjust_pc.h> 14 #include <linux/kvm_host.h> 15 #include <asm/kvm_emulate.h> 16 17 #if !defined (__KVM_NVHE_HYPERVISOR__) && !defined (__KVM_VHE_HYPERVISOR__) 18 #error Hypervisor code only! 19 #endif 20 21 static inline u64 __vcpu_read_sys_reg(const struct kvm_vcpu *vcpu, int reg) 22 { 23 u64 val; 24 25 if (__vcpu_read_sys_reg_from_cpu(reg, &val)) 26 return val; 27 28 return __vcpu_sys_reg(vcpu, reg); 29 } 30 31 static inline void __vcpu_write_sys_reg(struct kvm_vcpu *vcpu, u64 val, int reg) 32 { 33 if (__vcpu_write_sys_reg_to_cpu(val, reg)) 34 return; 35 36 __vcpu_sys_reg(vcpu, reg) = val; 37 } 38 39 static void __vcpu_write_spsr(struct kvm_vcpu *vcpu, u64 val) 40 { 41 write_sysreg_el1(val, SYS_SPSR); 42 } 43 44 static void __vcpu_write_spsr_abt(struct kvm_vcpu *vcpu, u64 val) 45 { 46 if (has_vhe()) 47 write_sysreg(val, spsr_abt); 48 else 49 vcpu->arch.ctxt.spsr_abt = val; 50 } 51 52 static void __vcpu_write_spsr_und(struct kvm_vcpu *vcpu, u64 val) 53 { 54 if (has_vhe()) 55 write_sysreg(val, spsr_und); 56 else 57 vcpu->arch.ctxt.spsr_und = val; 58 } 59 60 /* 61 * This performs the exception entry at a given EL (@target_mode), stashing PC 62 * and PSTATE into ELR and SPSR respectively, and compute the new PC/PSTATE. 63 * The EL passed to this function *must* be a non-secure, privileged mode with 64 * bit 0 being set (PSTATE.SP == 1). 65 * 66 * When an exception is taken, most PSTATE fields are left unchanged in the 67 * handler. However, some are explicitly overridden (e.g. M[4:0]). Luckily all 68 * of the inherited bits have the same position in the AArch64/AArch32 SPSR_ELx 69 * layouts, so we don't need to shuffle these for exceptions from AArch32 EL0. 70 * 71 * For the SPSR_ELx layout for AArch64, see ARM DDI 0487E.a page C5-429. 72 * For the SPSR_ELx layout for AArch32, see ARM DDI 0487E.a page C5-426. 73 * 74 * Here we manipulate the fields in order of the AArch64 SPSR_ELx layout, from 75 * MSB to LSB. 76 */ 77 static void enter_exception64(struct kvm_vcpu *vcpu, unsigned long target_mode, 78 enum exception_type type) 79 { 80 unsigned long sctlr, vbar, old, new, mode; 81 u64 exc_offset; 82 83 mode = *vcpu_cpsr(vcpu) & (PSR_MODE_MASK | PSR_MODE32_BIT); 84 85 if (mode == target_mode) 86 exc_offset = CURRENT_EL_SP_ELx_VECTOR; 87 else if ((mode | PSR_MODE_THREAD_BIT) == target_mode) 88 exc_offset = CURRENT_EL_SP_EL0_VECTOR; 89 else if (!(mode & PSR_MODE32_BIT)) 90 exc_offset = LOWER_EL_AArch64_VECTOR; 91 else 92 exc_offset = LOWER_EL_AArch32_VECTOR; 93 94 switch (target_mode) { 95 case PSR_MODE_EL1h: 96 vbar = __vcpu_read_sys_reg(vcpu, VBAR_EL1); 97 sctlr = __vcpu_read_sys_reg(vcpu, SCTLR_EL1); 98 __vcpu_write_sys_reg(vcpu, *vcpu_pc(vcpu), ELR_EL1); 99 break; 100 default: 101 /* Don't do that */ 102 BUG(); 103 } 104 105 *vcpu_pc(vcpu) = vbar + exc_offset + type; 106 107 old = *vcpu_cpsr(vcpu); 108 new = 0; 109 110 new |= (old & PSR_N_BIT); 111 new |= (old & PSR_Z_BIT); 112 new |= (old & PSR_C_BIT); 113 new |= (old & PSR_V_BIT); 114 115 // TODO: TCO (if/when ARMv8.5-MemTag is exposed to guests) 116 117 new |= (old & PSR_DIT_BIT); 118 119 // PSTATE.UAO is set to zero upon any exception to AArch64 120 // See ARM DDI 0487E.a, page D5-2579. 121 122 // PSTATE.PAN is unchanged unless SCTLR_ELx.SPAN == 0b0 123 // SCTLR_ELx.SPAN is RES1 when ARMv8.1-PAN is not implemented 124 // See ARM DDI 0487E.a, page D5-2578. 125 new |= (old & PSR_PAN_BIT); 126 if (!(sctlr & SCTLR_EL1_SPAN)) 127 new |= PSR_PAN_BIT; 128 129 // PSTATE.SS is set to zero upon any exception to AArch64 130 // See ARM DDI 0487E.a, page D2-2452. 131 132 // PSTATE.IL is set to zero upon any exception to AArch64 133 // See ARM DDI 0487E.a, page D1-2306. 134 135 // PSTATE.SSBS is set to SCTLR_ELx.DSSBS upon any exception to AArch64 136 // See ARM DDI 0487E.a, page D13-3258 137 if (sctlr & SCTLR_ELx_DSSBS) 138 new |= PSR_SSBS_BIT; 139 140 // PSTATE.BTYPE is set to zero upon any exception to AArch64 141 // See ARM DDI 0487E.a, pages D1-2293 to D1-2294. 142 143 new |= PSR_D_BIT; 144 new |= PSR_A_BIT; 145 new |= PSR_I_BIT; 146 new |= PSR_F_BIT; 147 148 new |= target_mode; 149 150 *vcpu_cpsr(vcpu) = new; 151 __vcpu_write_spsr(vcpu, old); 152 } 153 154 /* 155 * When an exception is taken, most CPSR fields are left unchanged in the 156 * handler. However, some are explicitly overridden (e.g. M[4:0]). 157 * 158 * The SPSR/SPSR_ELx layouts differ, and the below is intended to work with 159 * either format. Note: SPSR.J bit doesn't exist in SPSR_ELx, but this bit was 160 * obsoleted by the ARMv7 virtualization extensions and is RES0. 161 * 162 * For the SPSR layout seen from AArch32, see: 163 * - ARM DDI 0406C.d, page B1-1148 164 * - ARM DDI 0487E.a, page G8-6264 165 * 166 * For the SPSR_ELx layout for AArch32 seen from AArch64, see: 167 * - ARM DDI 0487E.a, page C5-426 168 * 169 * Here we manipulate the fields in order of the AArch32 SPSR_ELx layout, from 170 * MSB to LSB. 171 */ 172 static unsigned long get_except32_cpsr(struct kvm_vcpu *vcpu, u32 mode) 173 { 174 u32 sctlr = __vcpu_read_sys_reg(vcpu, SCTLR_EL1); 175 unsigned long old, new; 176 177 old = *vcpu_cpsr(vcpu); 178 new = 0; 179 180 new |= (old & PSR_AA32_N_BIT); 181 new |= (old & PSR_AA32_Z_BIT); 182 new |= (old & PSR_AA32_C_BIT); 183 new |= (old & PSR_AA32_V_BIT); 184 new |= (old & PSR_AA32_Q_BIT); 185 186 // CPSR.IT[7:0] are set to zero upon any exception 187 // See ARM DDI 0487E.a, section G1.12.3 188 // See ARM DDI 0406C.d, section B1.8.3 189 190 new |= (old & PSR_AA32_DIT_BIT); 191 192 // CPSR.SSBS is set to SCTLR.DSSBS upon any exception 193 // See ARM DDI 0487E.a, page G8-6244 194 if (sctlr & BIT(31)) 195 new |= PSR_AA32_SSBS_BIT; 196 197 // CPSR.PAN is unchanged unless SCTLR.SPAN == 0b0 198 // SCTLR.SPAN is RES1 when ARMv8.1-PAN is not implemented 199 // See ARM DDI 0487E.a, page G8-6246 200 new |= (old & PSR_AA32_PAN_BIT); 201 if (!(sctlr & BIT(23))) 202 new |= PSR_AA32_PAN_BIT; 203 204 // SS does not exist in AArch32, so ignore 205 206 // CPSR.IL is set to zero upon any exception 207 // See ARM DDI 0487E.a, page G1-5527 208 209 new |= (old & PSR_AA32_GE_MASK); 210 211 // CPSR.IT[7:0] are set to zero upon any exception 212 // See prior comment above 213 214 // CPSR.E is set to SCTLR.EE upon any exception 215 // See ARM DDI 0487E.a, page G8-6245 216 // See ARM DDI 0406C.d, page B4-1701 217 if (sctlr & BIT(25)) 218 new |= PSR_AA32_E_BIT; 219 220 // CPSR.A is unchanged upon an exception to Undefined, Supervisor 221 // CPSR.A is set upon an exception to other modes 222 // See ARM DDI 0487E.a, pages G1-5515 to G1-5516 223 // See ARM DDI 0406C.d, page B1-1182 224 new |= (old & PSR_AA32_A_BIT); 225 if (mode != PSR_AA32_MODE_UND && mode != PSR_AA32_MODE_SVC) 226 new |= PSR_AA32_A_BIT; 227 228 // CPSR.I is set upon any exception 229 // See ARM DDI 0487E.a, pages G1-5515 to G1-5516 230 // See ARM DDI 0406C.d, page B1-1182 231 new |= PSR_AA32_I_BIT; 232 233 // CPSR.F is set upon an exception to FIQ 234 // CPSR.F is unchanged upon an exception to other modes 235 // See ARM DDI 0487E.a, pages G1-5515 to G1-5516 236 // See ARM DDI 0406C.d, page B1-1182 237 new |= (old & PSR_AA32_F_BIT); 238 if (mode == PSR_AA32_MODE_FIQ) 239 new |= PSR_AA32_F_BIT; 240 241 // CPSR.T is set to SCTLR.TE upon any exception 242 // See ARM DDI 0487E.a, page G8-5514 243 // See ARM DDI 0406C.d, page B1-1181 244 if (sctlr & BIT(30)) 245 new |= PSR_AA32_T_BIT; 246 247 new |= mode; 248 249 return new; 250 } 251 252 /* 253 * Table taken from ARMv8 ARM DDI0487B-B, table G1-10. 254 */ 255 static const u8 return_offsets[8][2] = { 256 [0] = { 0, 0 }, /* Reset, unused */ 257 [1] = { 4, 2 }, /* Undefined */ 258 [2] = { 0, 0 }, /* SVC, unused */ 259 [3] = { 4, 4 }, /* Prefetch abort */ 260 [4] = { 8, 8 }, /* Data abort */ 261 [5] = { 0, 0 }, /* HVC, unused */ 262 [6] = { 4, 4 }, /* IRQ, unused */ 263 [7] = { 4, 4 }, /* FIQ, unused */ 264 }; 265 266 static void enter_exception32(struct kvm_vcpu *vcpu, u32 mode, u32 vect_offset) 267 { 268 unsigned long spsr = *vcpu_cpsr(vcpu); 269 bool is_thumb = (spsr & PSR_AA32_T_BIT); 270 u32 sctlr = __vcpu_read_sys_reg(vcpu, SCTLR_EL1); 271 u32 return_address; 272 273 *vcpu_cpsr(vcpu) = get_except32_cpsr(vcpu, mode); 274 return_address = *vcpu_pc(vcpu); 275 return_address += return_offsets[vect_offset >> 2][is_thumb]; 276 277 /* KVM only enters the ABT and UND modes, so only deal with those */ 278 switch(mode) { 279 case PSR_AA32_MODE_ABT: 280 __vcpu_write_spsr_abt(vcpu, host_spsr_to_spsr32(spsr)); 281 vcpu_gp_regs(vcpu)->compat_lr_abt = return_address; 282 break; 283 284 case PSR_AA32_MODE_UND: 285 __vcpu_write_spsr_und(vcpu, host_spsr_to_spsr32(spsr)); 286 vcpu_gp_regs(vcpu)->compat_lr_und = return_address; 287 break; 288 } 289 290 /* Branch to exception vector */ 291 if (sctlr & (1 << 13)) 292 vect_offset += 0xffff0000; 293 else /* always have security exceptions */ 294 vect_offset += __vcpu_read_sys_reg(vcpu, VBAR_EL1); 295 296 *vcpu_pc(vcpu) = vect_offset; 297 } 298 299 void kvm_inject_exception(struct kvm_vcpu *vcpu) 300 { 301 if (vcpu_el1_is_32bit(vcpu)) { 302 switch (vcpu->arch.flags & KVM_ARM64_EXCEPT_MASK) { 303 case KVM_ARM64_EXCEPT_AA32_UND: 304 enter_exception32(vcpu, PSR_AA32_MODE_UND, 4); 305 break; 306 case KVM_ARM64_EXCEPT_AA32_IABT: 307 enter_exception32(vcpu, PSR_AA32_MODE_ABT, 12); 308 break; 309 case KVM_ARM64_EXCEPT_AA32_DABT: 310 enter_exception32(vcpu, PSR_AA32_MODE_ABT, 16); 311 break; 312 default: 313 /* Err... */ 314 break; 315 } 316 } else { 317 switch (vcpu->arch.flags & KVM_ARM64_EXCEPT_MASK) { 318 case (KVM_ARM64_EXCEPT_AA64_ELx_SYNC | 319 KVM_ARM64_EXCEPT_AA64_EL1): 320 enter_exception64(vcpu, PSR_MODE_EL1h, except_type_sync); 321 break; 322 default: 323 /* 324 * Only EL1_SYNC makes sense so far, EL2_{SYNC,IRQ} 325 * will be implemented at some point. Everything 326 * else gets silently ignored. 327 */ 328 break; 329 } 330 } 331 } 332