1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * Fault injection for both 32 and 64bit guests. 4 * 5 * Copyright (C) 2012,2013 - ARM Ltd 6 * Author: Marc Zyngier <marc.zyngier@arm.com> 7 * 8 * Based on arch/arm/kvm/emulate.c 9 * Copyright (C) 2012 - Virtual Open Systems and Columbia University 10 * Author: Christoffer Dall <c.dall@virtualopensystems.com> 11 */ 12 13 #include <hyp/adjust_pc.h> 14 #include <linux/kvm_host.h> 15 #include <asm/kvm_emulate.h> 16 #include <asm/kvm_mmu.h> 17 18 #if !defined (__KVM_NVHE_HYPERVISOR__) && !defined (__KVM_VHE_HYPERVISOR__) 19 #error Hypervisor code only! 20 #endif 21 22 static inline u64 __vcpu_read_sys_reg(const struct kvm_vcpu *vcpu, int reg) 23 { 24 u64 val; 25 26 if (__vcpu_read_sys_reg_from_cpu(reg, &val)) 27 return val; 28 29 return __vcpu_sys_reg(vcpu, reg); 30 } 31 32 static inline void __vcpu_write_sys_reg(struct kvm_vcpu *vcpu, u64 val, int reg) 33 { 34 if (__vcpu_write_sys_reg_to_cpu(val, reg)) 35 return; 36 37 __vcpu_sys_reg(vcpu, reg) = val; 38 } 39 40 static void __vcpu_write_spsr(struct kvm_vcpu *vcpu, u64 val) 41 { 42 if (has_vhe()) 43 write_sysreg_el1(val, SYS_SPSR); 44 else 45 __vcpu_sys_reg(vcpu, SPSR_EL1) = val; 46 } 47 48 static void __vcpu_write_spsr_abt(struct kvm_vcpu *vcpu, u64 val) 49 { 50 if (has_vhe()) 51 write_sysreg(val, spsr_abt); 52 else 53 vcpu->arch.ctxt.spsr_abt = val; 54 } 55 56 static void __vcpu_write_spsr_und(struct kvm_vcpu *vcpu, u64 val) 57 { 58 if (has_vhe()) 59 write_sysreg(val, spsr_und); 60 else 61 vcpu->arch.ctxt.spsr_und = val; 62 } 63 64 /* 65 * This performs the exception entry at a given EL (@target_mode), stashing PC 66 * and PSTATE into ELR and SPSR respectively, and compute the new PC/PSTATE. 67 * The EL passed to this function *must* be a non-secure, privileged mode with 68 * bit 0 being set (PSTATE.SP == 1). 69 * 70 * When an exception is taken, most PSTATE fields are left unchanged in the 71 * handler. However, some are explicitly overridden (e.g. M[4:0]). Luckily all 72 * of the inherited bits have the same position in the AArch64/AArch32 SPSR_ELx 73 * layouts, so we don't need to shuffle these for exceptions from AArch32 EL0. 74 * 75 * For the SPSR_ELx layout for AArch64, see ARM DDI 0487E.a page C5-429. 76 * For the SPSR_ELx layout for AArch32, see ARM DDI 0487E.a page C5-426. 77 * 78 * Here we manipulate the fields in order of the AArch64 SPSR_ELx layout, from 79 * MSB to LSB. 80 */ 81 static void enter_exception64(struct kvm_vcpu *vcpu, unsigned long target_mode, 82 enum exception_type type) 83 { 84 unsigned long sctlr, vbar, old, new, mode; 85 u64 exc_offset; 86 87 mode = *vcpu_cpsr(vcpu) & (PSR_MODE_MASK | PSR_MODE32_BIT); 88 89 if (mode == target_mode) 90 exc_offset = CURRENT_EL_SP_ELx_VECTOR; 91 else if ((mode | PSR_MODE_THREAD_BIT) == target_mode) 92 exc_offset = CURRENT_EL_SP_EL0_VECTOR; 93 else if (!(mode & PSR_MODE32_BIT)) 94 exc_offset = LOWER_EL_AArch64_VECTOR; 95 else 96 exc_offset = LOWER_EL_AArch32_VECTOR; 97 98 switch (target_mode) { 99 case PSR_MODE_EL1h: 100 vbar = __vcpu_read_sys_reg(vcpu, VBAR_EL1); 101 sctlr = __vcpu_read_sys_reg(vcpu, SCTLR_EL1); 102 __vcpu_write_sys_reg(vcpu, *vcpu_pc(vcpu), ELR_EL1); 103 break; 104 default: 105 /* Don't do that */ 106 BUG(); 107 } 108 109 *vcpu_pc(vcpu) = vbar + exc_offset + type; 110 111 old = *vcpu_cpsr(vcpu); 112 new = 0; 113 114 new |= (old & PSR_N_BIT); 115 new |= (old & PSR_Z_BIT); 116 new |= (old & PSR_C_BIT); 117 new |= (old & PSR_V_BIT); 118 119 if (kvm_has_mte(kern_hyp_va(vcpu->kvm))) 120 new |= PSR_TCO_BIT; 121 122 new |= (old & PSR_DIT_BIT); 123 124 // PSTATE.UAO is set to zero upon any exception to AArch64 125 // See ARM DDI 0487E.a, page D5-2579. 126 127 // PSTATE.PAN is unchanged unless SCTLR_ELx.SPAN == 0b0 128 // SCTLR_ELx.SPAN is RES1 when ARMv8.1-PAN is not implemented 129 // See ARM DDI 0487E.a, page D5-2578. 130 new |= (old & PSR_PAN_BIT); 131 if (!(sctlr & SCTLR_EL1_SPAN)) 132 new |= PSR_PAN_BIT; 133 134 // PSTATE.SS is set to zero upon any exception to AArch64 135 // See ARM DDI 0487E.a, page D2-2452. 136 137 // PSTATE.IL is set to zero upon any exception to AArch64 138 // See ARM DDI 0487E.a, page D1-2306. 139 140 // PSTATE.SSBS is set to SCTLR_ELx.DSSBS upon any exception to AArch64 141 // See ARM DDI 0487E.a, page D13-3258 142 if (sctlr & SCTLR_ELx_DSSBS) 143 new |= PSR_SSBS_BIT; 144 145 // PSTATE.BTYPE is set to zero upon any exception to AArch64 146 // See ARM DDI 0487E.a, pages D1-2293 to D1-2294. 147 148 new |= PSR_D_BIT; 149 new |= PSR_A_BIT; 150 new |= PSR_I_BIT; 151 new |= PSR_F_BIT; 152 153 new |= target_mode; 154 155 *vcpu_cpsr(vcpu) = new; 156 __vcpu_write_spsr(vcpu, old); 157 } 158 159 /* 160 * When an exception is taken, most CPSR fields are left unchanged in the 161 * handler. However, some are explicitly overridden (e.g. M[4:0]). 162 * 163 * The SPSR/SPSR_ELx layouts differ, and the below is intended to work with 164 * either format. Note: SPSR.J bit doesn't exist in SPSR_ELx, but this bit was 165 * obsoleted by the ARMv7 virtualization extensions and is RES0. 166 * 167 * For the SPSR layout seen from AArch32, see: 168 * - ARM DDI 0406C.d, page B1-1148 169 * - ARM DDI 0487E.a, page G8-6264 170 * 171 * For the SPSR_ELx layout for AArch32 seen from AArch64, see: 172 * - ARM DDI 0487E.a, page C5-426 173 * 174 * Here we manipulate the fields in order of the AArch32 SPSR_ELx layout, from 175 * MSB to LSB. 176 */ 177 static unsigned long get_except32_cpsr(struct kvm_vcpu *vcpu, u32 mode) 178 { 179 u32 sctlr = __vcpu_read_sys_reg(vcpu, SCTLR_EL1); 180 unsigned long old, new; 181 182 old = *vcpu_cpsr(vcpu); 183 new = 0; 184 185 new |= (old & PSR_AA32_N_BIT); 186 new |= (old & PSR_AA32_Z_BIT); 187 new |= (old & PSR_AA32_C_BIT); 188 new |= (old & PSR_AA32_V_BIT); 189 new |= (old & PSR_AA32_Q_BIT); 190 191 // CPSR.IT[7:0] are set to zero upon any exception 192 // See ARM DDI 0487E.a, section G1.12.3 193 // See ARM DDI 0406C.d, section B1.8.3 194 195 new |= (old & PSR_AA32_DIT_BIT); 196 197 // CPSR.SSBS is set to SCTLR.DSSBS upon any exception 198 // See ARM DDI 0487E.a, page G8-6244 199 if (sctlr & BIT(31)) 200 new |= PSR_AA32_SSBS_BIT; 201 202 // CPSR.PAN is unchanged unless SCTLR.SPAN == 0b0 203 // SCTLR.SPAN is RES1 when ARMv8.1-PAN is not implemented 204 // See ARM DDI 0487E.a, page G8-6246 205 new |= (old & PSR_AA32_PAN_BIT); 206 if (!(sctlr & BIT(23))) 207 new |= PSR_AA32_PAN_BIT; 208 209 // SS does not exist in AArch32, so ignore 210 211 // CPSR.IL is set to zero upon any exception 212 // See ARM DDI 0487E.a, page G1-5527 213 214 new |= (old & PSR_AA32_GE_MASK); 215 216 // CPSR.IT[7:0] are set to zero upon any exception 217 // See prior comment above 218 219 // CPSR.E is set to SCTLR.EE upon any exception 220 // See ARM DDI 0487E.a, page G8-6245 221 // See ARM DDI 0406C.d, page B4-1701 222 if (sctlr & BIT(25)) 223 new |= PSR_AA32_E_BIT; 224 225 // CPSR.A is unchanged upon an exception to Undefined, Supervisor 226 // CPSR.A is set upon an exception to other modes 227 // See ARM DDI 0487E.a, pages G1-5515 to G1-5516 228 // See ARM DDI 0406C.d, page B1-1182 229 new |= (old & PSR_AA32_A_BIT); 230 if (mode != PSR_AA32_MODE_UND && mode != PSR_AA32_MODE_SVC) 231 new |= PSR_AA32_A_BIT; 232 233 // CPSR.I is set upon any exception 234 // See ARM DDI 0487E.a, pages G1-5515 to G1-5516 235 // See ARM DDI 0406C.d, page B1-1182 236 new |= PSR_AA32_I_BIT; 237 238 // CPSR.F is set upon an exception to FIQ 239 // CPSR.F is unchanged upon an exception to other modes 240 // See ARM DDI 0487E.a, pages G1-5515 to G1-5516 241 // See ARM DDI 0406C.d, page B1-1182 242 new |= (old & PSR_AA32_F_BIT); 243 if (mode == PSR_AA32_MODE_FIQ) 244 new |= PSR_AA32_F_BIT; 245 246 // CPSR.T is set to SCTLR.TE upon any exception 247 // See ARM DDI 0487E.a, page G8-5514 248 // See ARM DDI 0406C.d, page B1-1181 249 if (sctlr & BIT(30)) 250 new |= PSR_AA32_T_BIT; 251 252 new |= mode; 253 254 return new; 255 } 256 257 /* 258 * Table taken from ARMv8 ARM DDI0487B-B, table G1-10. 259 */ 260 static const u8 return_offsets[8][2] = { 261 [0] = { 0, 0 }, /* Reset, unused */ 262 [1] = { 4, 2 }, /* Undefined */ 263 [2] = { 0, 0 }, /* SVC, unused */ 264 [3] = { 4, 4 }, /* Prefetch abort */ 265 [4] = { 8, 8 }, /* Data abort */ 266 [5] = { 0, 0 }, /* HVC, unused */ 267 [6] = { 4, 4 }, /* IRQ, unused */ 268 [7] = { 4, 4 }, /* FIQ, unused */ 269 }; 270 271 static void enter_exception32(struct kvm_vcpu *vcpu, u32 mode, u32 vect_offset) 272 { 273 unsigned long spsr = *vcpu_cpsr(vcpu); 274 bool is_thumb = (spsr & PSR_AA32_T_BIT); 275 u32 sctlr = __vcpu_read_sys_reg(vcpu, SCTLR_EL1); 276 u32 return_address; 277 278 *vcpu_cpsr(vcpu) = get_except32_cpsr(vcpu, mode); 279 return_address = *vcpu_pc(vcpu); 280 return_address += return_offsets[vect_offset >> 2][is_thumb]; 281 282 /* KVM only enters the ABT and UND modes, so only deal with those */ 283 switch(mode) { 284 case PSR_AA32_MODE_ABT: 285 __vcpu_write_spsr_abt(vcpu, host_spsr_to_spsr32(spsr)); 286 vcpu_gp_regs(vcpu)->compat_lr_abt = return_address; 287 break; 288 289 case PSR_AA32_MODE_UND: 290 __vcpu_write_spsr_und(vcpu, host_spsr_to_spsr32(spsr)); 291 vcpu_gp_regs(vcpu)->compat_lr_und = return_address; 292 break; 293 } 294 295 /* Branch to exception vector */ 296 if (sctlr & (1 << 13)) 297 vect_offset += 0xffff0000; 298 else /* always have security exceptions */ 299 vect_offset += __vcpu_read_sys_reg(vcpu, VBAR_EL1); 300 301 *vcpu_pc(vcpu) = vect_offset; 302 } 303 304 static void kvm_inject_exception(struct kvm_vcpu *vcpu) 305 { 306 if (vcpu_el1_is_32bit(vcpu)) { 307 switch (vcpu_get_flag(vcpu, EXCEPT_MASK)) { 308 case unpack_vcpu_flag(EXCEPT_AA32_UND): 309 enter_exception32(vcpu, PSR_AA32_MODE_UND, 4); 310 break; 311 case unpack_vcpu_flag(EXCEPT_AA32_IABT): 312 enter_exception32(vcpu, PSR_AA32_MODE_ABT, 12); 313 break; 314 case unpack_vcpu_flag(EXCEPT_AA32_DABT): 315 enter_exception32(vcpu, PSR_AA32_MODE_ABT, 16); 316 break; 317 default: 318 /* Err... */ 319 break; 320 } 321 } else { 322 switch (vcpu_get_flag(vcpu, EXCEPT_MASK)) { 323 case unpack_vcpu_flag(EXCEPT_AA64_EL1_SYNC): 324 enter_exception64(vcpu, PSR_MODE_EL1h, except_type_sync); 325 break; 326 default: 327 /* 328 * Only EL1_SYNC makes sense so far, EL2_{SYNC,IRQ} 329 * will be implemented at some point. Everything 330 * else gets silently ignored. 331 */ 332 break; 333 } 334 } 335 } 336 337 /* 338 * Adjust the guest PC (and potentially exception state) depending on 339 * flags provided by the emulation code. 340 */ 341 void __kvm_adjust_pc(struct kvm_vcpu *vcpu) 342 { 343 if (vcpu_get_flag(vcpu, PENDING_EXCEPTION)) { 344 kvm_inject_exception(vcpu); 345 vcpu_clear_flag(vcpu, PENDING_EXCEPTION); 346 vcpu_clear_flag(vcpu, EXCEPT_MASK); 347 } else if (vcpu_get_flag(vcpu, INCREMENT_PC)) { 348 kvm_skip_instr(vcpu); 349 vcpu_clear_flag(vcpu, INCREMENT_PC); 350 } 351 } 352