1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * Copyright (C) 2021 Google LLC 4 * Author: Fuad Tabba <tabba@google.com> 5 */ 6 7 #include <linux/irqchip/arm-gic-v3.h> 8 9 #include <asm/kvm_asm.h> 10 #include <asm/kvm_mmu.h> 11 12 #include <hyp/adjust_pc.h> 13 14 #include <nvhe/fixed_config.h> 15 16 #include "../../sys_regs.h" 17 18 /* 19 * Copies of the host's CPU features registers holding sanitized values at hyp. 20 */ 21 u64 id_aa64pfr0_el1_sys_val; 22 u64 id_aa64pfr1_el1_sys_val; 23 u64 id_aa64isar0_el1_sys_val; 24 u64 id_aa64isar1_el1_sys_val; 25 u64 id_aa64mmfr0_el1_sys_val; 26 u64 id_aa64mmfr1_el1_sys_val; 27 u64 id_aa64mmfr2_el1_sys_val; 28 29 /* 30 * Inject an unknown/undefined exception to an AArch64 guest while most of its 31 * sysregs are live. 32 */ 33 static void inject_undef64(struct kvm_vcpu *vcpu) 34 { 35 u32 esr = (ESR_ELx_EC_UNKNOWN << ESR_ELx_EC_SHIFT); 36 37 *vcpu_pc(vcpu) = read_sysreg_el2(SYS_ELR); 38 *vcpu_cpsr(vcpu) = read_sysreg_el2(SYS_SPSR); 39 40 vcpu->arch.flags |= (KVM_ARM64_EXCEPT_AA64_EL1 | 41 KVM_ARM64_EXCEPT_AA64_ELx_SYNC | 42 KVM_ARM64_PENDING_EXCEPTION); 43 44 __kvm_adjust_pc(vcpu); 45 46 write_sysreg_el1(esr, SYS_ESR); 47 write_sysreg_el1(read_sysreg_el2(SYS_ELR), SYS_ELR); 48 write_sysreg_el2(*vcpu_pc(vcpu), SYS_ELR); 49 write_sysreg_el2(*vcpu_cpsr(vcpu), SYS_SPSR); 50 } 51 52 /* 53 * Returns the restricted features values of the feature register based on the 54 * limitations in restrict_fields. 55 * A feature id field value of 0b0000 does not impose any restrictions. 56 * Note: Use only for unsigned feature field values. 57 */ 58 static u64 get_restricted_features_unsigned(u64 sys_reg_val, 59 u64 restrict_fields) 60 { 61 u64 value = 0UL; 62 u64 mask = GENMASK_ULL(ARM64_FEATURE_FIELD_BITS - 1, 0); 63 64 /* 65 * According to the Arm Architecture Reference Manual, feature fields 66 * use increasing values to indicate increases in functionality. 67 * Iterate over the restricted feature fields and calculate the minimum 68 * unsigned value between the one supported by the system, and what the 69 * value is being restricted to. 70 */ 71 while (sys_reg_val && restrict_fields) { 72 value |= min(sys_reg_val & mask, restrict_fields & mask); 73 sys_reg_val &= ~mask; 74 restrict_fields &= ~mask; 75 mask <<= ARM64_FEATURE_FIELD_BITS; 76 } 77 78 return value; 79 } 80 81 /* 82 * Functions that return the value of feature id registers for protected VMs 83 * based on allowed features, system features, and KVM support. 84 */ 85 86 static u64 get_pvm_id_aa64pfr0(const struct kvm_vcpu *vcpu) 87 { 88 const struct kvm *kvm = (const struct kvm *)kern_hyp_va(vcpu->kvm); 89 u64 set_mask = 0; 90 u64 allow_mask = PVM_ID_AA64PFR0_ALLOW; 91 92 if (!vcpu_has_sve(vcpu)) 93 allow_mask &= ~ARM64_FEATURE_MASK(ID_AA64PFR0_SVE); 94 95 set_mask |= get_restricted_features_unsigned(id_aa64pfr0_el1_sys_val, 96 PVM_ID_AA64PFR0_RESTRICT_UNSIGNED); 97 98 /* Spectre and Meltdown mitigation in KVM */ 99 set_mask |= FIELD_PREP(ARM64_FEATURE_MASK(ID_AA64PFR0_CSV2), 100 (u64)kvm->arch.pfr0_csv2); 101 set_mask |= FIELD_PREP(ARM64_FEATURE_MASK(ID_AA64PFR0_CSV3), 102 (u64)kvm->arch.pfr0_csv3); 103 104 return (id_aa64pfr0_el1_sys_val & allow_mask) | set_mask; 105 } 106 107 static u64 get_pvm_id_aa64pfr1(const struct kvm_vcpu *vcpu) 108 { 109 const struct kvm *kvm = (const struct kvm *)kern_hyp_va(vcpu->kvm); 110 u64 allow_mask = PVM_ID_AA64PFR1_ALLOW; 111 112 if (!kvm_has_mte(kvm)) 113 allow_mask &= ~ARM64_FEATURE_MASK(ID_AA64PFR1_MTE); 114 115 return id_aa64pfr1_el1_sys_val & allow_mask; 116 } 117 118 static u64 get_pvm_id_aa64zfr0(const struct kvm_vcpu *vcpu) 119 { 120 /* 121 * No support for Scalable Vectors, therefore, hyp has no sanitized 122 * copy of the feature id register. 123 */ 124 BUILD_BUG_ON(PVM_ID_AA64ZFR0_ALLOW != 0ULL); 125 return 0; 126 } 127 128 static u64 get_pvm_id_aa64dfr0(const struct kvm_vcpu *vcpu) 129 { 130 /* 131 * No support for debug, including breakpoints, and watchpoints, 132 * therefore, pKVM has no sanitized copy of the feature id register. 133 */ 134 BUILD_BUG_ON(PVM_ID_AA64DFR0_ALLOW != 0ULL); 135 return 0; 136 } 137 138 static u64 get_pvm_id_aa64dfr1(const struct kvm_vcpu *vcpu) 139 { 140 /* 141 * No support for debug, therefore, hyp has no sanitized copy of the 142 * feature id register. 143 */ 144 BUILD_BUG_ON(PVM_ID_AA64DFR1_ALLOW != 0ULL); 145 return 0; 146 } 147 148 static u64 get_pvm_id_aa64afr0(const struct kvm_vcpu *vcpu) 149 { 150 /* 151 * No support for implementation defined features, therefore, hyp has no 152 * sanitized copy of the feature id register. 153 */ 154 BUILD_BUG_ON(PVM_ID_AA64AFR0_ALLOW != 0ULL); 155 return 0; 156 } 157 158 static u64 get_pvm_id_aa64afr1(const struct kvm_vcpu *vcpu) 159 { 160 /* 161 * No support for implementation defined features, therefore, hyp has no 162 * sanitized copy of the feature id register. 163 */ 164 BUILD_BUG_ON(PVM_ID_AA64AFR1_ALLOW != 0ULL); 165 return 0; 166 } 167 168 static u64 get_pvm_id_aa64isar0(const struct kvm_vcpu *vcpu) 169 { 170 return id_aa64isar0_el1_sys_val & PVM_ID_AA64ISAR0_ALLOW; 171 } 172 173 static u64 get_pvm_id_aa64isar1(const struct kvm_vcpu *vcpu) 174 { 175 u64 allow_mask = PVM_ID_AA64ISAR1_ALLOW; 176 177 if (!vcpu_has_ptrauth(vcpu)) 178 allow_mask &= ~(ARM64_FEATURE_MASK(ID_AA64ISAR1_APA) | 179 ARM64_FEATURE_MASK(ID_AA64ISAR1_API) | 180 ARM64_FEATURE_MASK(ID_AA64ISAR1_GPA) | 181 ARM64_FEATURE_MASK(ID_AA64ISAR1_GPI)); 182 183 return id_aa64isar1_el1_sys_val & allow_mask; 184 } 185 186 static u64 get_pvm_id_aa64mmfr0(const struct kvm_vcpu *vcpu) 187 { 188 u64 set_mask; 189 190 set_mask = get_restricted_features_unsigned(id_aa64mmfr0_el1_sys_val, 191 PVM_ID_AA64MMFR0_RESTRICT_UNSIGNED); 192 193 return (id_aa64mmfr0_el1_sys_val & PVM_ID_AA64MMFR0_ALLOW) | set_mask; 194 } 195 196 static u64 get_pvm_id_aa64mmfr1(const struct kvm_vcpu *vcpu) 197 { 198 return id_aa64mmfr1_el1_sys_val & PVM_ID_AA64MMFR1_ALLOW; 199 } 200 201 static u64 get_pvm_id_aa64mmfr2(const struct kvm_vcpu *vcpu) 202 { 203 return id_aa64mmfr2_el1_sys_val & PVM_ID_AA64MMFR2_ALLOW; 204 } 205 206 /* Read a sanitized cpufeature ID register by its encoding */ 207 u64 pvm_read_id_reg(const struct kvm_vcpu *vcpu, u32 id) 208 { 209 switch (id) { 210 case SYS_ID_AA64PFR0_EL1: 211 return get_pvm_id_aa64pfr0(vcpu); 212 case SYS_ID_AA64PFR1_EL1: 213 return get_pvm_id_aa64pfr1(vcpu); 214 case SYS_ID_AA64ZFR0_EL1: 215 return get_pvm_id_aa64zfr0(vcpu); 216 case SYS_ID_AA64DFR0_EL1: 217 return get_pvm_id_aa64dfr0(vcpu); 218 case SYS_ID_AA64DFR1_EL1: 219 return get_pvm_id_aa64dfr1(vcpu); 220 case SYS_ID_AA64AFR0_EL1: 221 return get_pvm_id_aa64afr0(vcpu); 222 case SYS_ID_AA64AFR1_EL1: 223 return get_pvm_id_aa64afr1(vcpu); 224 case SYS_ID_AA64ISAR0_EL1: 225 return get_pvm_id_aa64isar0(vcpu); 226 case SYS_ID_AA64ISAR1_EL1: 227 return get_pvm_id_aa64isar1(vcpu); 228 case SYS_ID_AA64MMFR0_EL1: 229 return get_pvm_id_aa64mmfr0(vcpu); 230 case SYS_ID_AA64MMFR1_EL1: 231 return get_pvm_id_aa64mmfr1(vcpu); 232 case SYS_ID_AA64MMFR2_EL1: 233 return get_pvm_id_aa64mmfr2(vcpu); 234 default: 235 /* 236 * Should never happen because all cases are covered in 237 * pvm_sys_reg_descs[]. 238 */ 239 WARN_ON(1); 240 break; 241 } 242 243 return 0; 244 } 245 246 static u64 read_id_reg(const struct kvm_vcpu *vcpu, 247 struct sys_reg_desc const *r) 248 { 249 return pvm_read_id_reg(vcpu, reg_to_encoding(r)); 250 } 251 252 /* Handler to RAZ/WI sysregs */ 253 static bool pvm_access_raz_wi(struct kvm_vcpu *vcpu, struct sys_reg_params *p, 254 const struct sys_reg_desc *r) 255 { 256 if (!p->is_write) 257 p->regval = 0; 258 259 return true; 260 } 261 262 /* 263 * Accessor for AArch32 feature id registers. 264 * 265 * The value of these registers is "unknown" according to the spec if AArch32 266 * isn't supported. 267 */ 268 static bool pvm_access_id_aarch32(struct kvm_vcpu *vcpu, 269 struct sys_reg_params *p, 270 const struct sys_reg_desc *r) 271 { 272 if (p->is_write) { 273 inject_undef64(vcpu); 274 return false; 275 } 276 277 /* 278 * No support for AArch32 guests, therefore, pKVM has no sanitized copy 279 * of AArch32 feature id registers. 280 */ 281 BUILD_BUG_ON(FIELD_GET(ARM64_FEATURE_MASK(ID_AA64PFR0_EL1), 282 PVM_ID_AA64PFR0_RESTRICT_UNSIGNED) > ID_AA64PFR0_ELx_64BIT_ONLY); 283 284 return pvm_access_raz_wi(vcpu, p, r); 285 } 286 287 /* 288 * Accessor for AArch64 feature id registers. 289 * 290 * If access is allowed, set the regval to the protected VM's view of the 291 * register and return true. 292 * Otherwise, inject an undefined exception and return false. 293 */ 294 static bool pvm_access_id_aarch64(struct kvm_vcpu *vcpu, 295 struct sys_reg_params *p, 296 const struct sys_reg_desc *r) 297 { 298 if (p->is_write) { 299 inject_undef64(vcpu); 300 return false; 301 } 302 303 p->regval = read_id_reg(vcpu, r); 304 return true; 305 } 306 307 static bool pvm_gic_read_sre(struct kvm_vcpu *vcpu, 308 struct sys_reg_params *p, 309 const struct sys_reg_desc *r) 310 { 311 /* pVMs only support GICv3. 'nuf said. */ 312 if (!p->is_write) 313 p->regval = ICC_SRE_EL1_DIB | ICC_SRE_EL1_DFB | ICC_SRE_EL1_SRE; 314 315 return true; 316 } 317 318 /* Mark the specified system register as an AArch32 feature id register. */ 319 #define AARCH32(REG) { SYS_DESC(REG), .access = pvm_access_id_aarch32 } 320 321 /* Mark the specified system register as an AArch64 feature id register. */ 322 #define AARCH64(REG) { SYS_DESC(REG), .access = pvm_access_id_aarch64 } 323 324 /* Mark the specified system register as Read-As-Zero/Write-Ignored */ 325 #define RAZ_WI(REG) { SYS_DESC(REG), .access = pvm_access_raz_wi } 326 327 /* Mark the specified system register as not being handled in hyp. */ 328 #define HOST_HANDLED(REG) { SYS_DESC(REG), .access = NULL } 329 330 /* 331 * Architected system registers. 332 * Important: Must be sorted ascending by Op0, Op1, CRn, CRm, Op2 333 * 334 * NOTE: Anything not explicitly listed here is *restricted by default*, i.e., 335 * it will lead to injecting an exception into the guest. 336 */ 337 static const struct sys_reg_desc pvm_sys_reg_descs[] = { 338 /* Cache maintenance by set/way operations are restricted. */ 339 340 /* Debug and Trace Registers are restricted. */ 341 342 /* AArch64 mappings of the AArch32 ID registers */ 343 /* CRm=1 */ 344 AARCH32(SYS_ID_PFR0_EL1), 345 AARCH32(SYS_ID_PFR1_EL1), 346 AARCH32(SYS_ID_DFR0_EL1), 347 AARCH32(SYS_ID_AFR0_EL1), 348 AARCH32(SYS_ID_MMFR0_EL1), 349 AARCH32(SYS_ID_MMFR1_EL1), 350 AARCH32(SYS_ID_MMFR2_EL1), 351 AARCH32(SYS_ID_MMFR3_EL1), 352 353 /* CRm=2 */ 354 AARCH32(SYS_ID_ISAR0_EL1), 355 AARCH32(SYS_ID_ISAR1_EL1), 356 AARCH32(SYS_ID_ISAR2_EL1), 357 AARCH32(SYS_ID_ISAR3_EL1), 358 AARCH32(SYS_ID_ISAR4_EL1), 359 AARCH32(SYS_ID_ISAR5_EL1), 360 AARCH32(SYS_ID_MMFR4_EL1), 361 AARCH32(SYS_ID_ISAR6_EL1), 362 363 /* CRm=3 */ 364 AARCH32(SYS_MVFR0_EL1), 365 AARCH32(SYS_MVFR1_EL1), 366 AARCH32(SYS_MVFR2_EL1), 367 AARCH32(SYS_ID_PFR2_EL1), 368 AARCH32(SYS_ID_DFR1_EL1), 369 AARCH32(SYS_ID_MMFR5_EL1), 370 371 /* AArch64 ID registers */ 372 /* CRm=4 */ 373 AARCH64(SYS_ID_AA64PFR0_EL1), 374 AARCH64(SYS_ID_AA64PFR1_EL1), 375 AARCH64(SYS_ID_AA64ZFR0_EL1), 376 AARCH64(SYS_ID_AA64DFR0_EL1), 377 AARCH64(SYS_ID_AA64DFR1_EL1), 378 AARCH64(SYS_ID_AA64AFR0_EL1), 379 AARCH64(SYS_ID_AA64AFR1_EL1), 380 AARCH64(SYS_ID_AA64ISAR0_EL1), 381 AARCH64(SYS_ID_AA64ISAR1_EL1), 382 AARCH64(SYS_ID_AA64MMFR0_EL1), 383 AARCH64(SYS_ID_AA64MMFR1_EL1), 384 AARCH64(SYS_ID_AA64MMFR2_EL1), 385 386 /* Scalable Vector Registers are restricted. */ 387 388 RAZ_WI(SYS_ERRIDR_EL1), 389 RAZ_WI(SYS_ERRSELR_EL1), 390 RAZ_WI(SYS_ERXFR_EL1), 391 RAZ_WI(SYS_ERXCTLR_EL1), 392 RAZ_WI(SYS_ERXSTATUS_EL1), 393 RAZ_WI(SYS_ERXADDR_EL1), 394 RAZ_WI(SYS_ERXMISC0_EL1), 395 RAZ_WI(SYS_ERXMISC1_EL1), 396 397 /* Performance Monitoring Registers are restricted. */ 398 399 /* Limited Ordering Regions Registers are restricted. */ 400 401 HOST_HANDLED(SYS_ICC_SGI1R_EL1), 402 HOST_HANDLED(SYS_ICC_ASGI1R_EL1), 403 HOST_HANDLED(SYS_ICC_SGI0R_EL1), 404 { SYS_DESC(SYS_ICC_SRE_EL1), .access = pvm_gic_read_sre, }, 405 406 HOST_HANDLED(SYS_CCSIDR_EL1), 407 HOST_HANDLED(SYS_CLIDR_EL1), 408 HOST_HANDLED(SYS_CSSELR_EL1), 409 HOST_HANDLED(SYS_CTR_EL0), 410 411 /* Performance Monitoring Registers are restricted. */ 412 413 /* Activity Monitoring Registers are restricted. */ 414 415 HOST_HANDLED(SYS_CNTP_TVAL_EL0), 416 HOST_HANDLED(SYS_CNTP_CTL_EL0), 417 HOST_HANDLED(SYS_CNTP_CVAL_EL0), 418 419 /* Performance Monitoring Registers are restricted. */ 420 }; 421 422 /* 423 * Checks that the sysreg table is unique and in-order. 424 * 425 * Returns 0 if the table is consistent, or 1 otherwise. 426 */ 427 int kvm_check_pvm_sysreg_table(void) 428 { 429 unsigned int i; 430 431 for (i = 1; i < ARRAY_SIZE(pvm_sys_reg_descs); i++) { 432 if (cmp_sys_reg(&pvm_sys_reg_descs[i-1], &pvm_sys_reg_descs[i]) >= 0) 433 return 1; 434 } 435 436 return 0; 437 } 438 439 /* 440 * Handler for protected VM MSR, MRS or System instruction execution. 441 * 442 * Returns true if the hypervisor has handled the exit, and control should go 443 * back to the guest, or false if it hasn't, to be handled by the host. 444 */ 445 bool kvm_handle_pvm_sysreg(struct kvm_vcpu *vcpu, u64 *exit_code) 446 { 447 const struct sys_reg_desc *r; 448 struct sys_reg_params params; 449 unsigned long esr = kvm_vcpu_get_esr(vcpu); 450 int Rt = kvm_vcpu_sys_get_rt(vcpu); 451 452 params = esr_sys64_to_params(esr); 453 params.regval = vcpu_get_reg(vcpu, Rt); 454 455 r = find_reg(¶ms, pvm_sys_reg_descs, ARRAY_SIZE(pvm_sys_reg_descs)); 456 457 /* Undefined (RESTRICTED). */ 458 if (r == NULL) { 459 inject_undef64(vcpu); 460 return true; 461 } 462 463 /* Handled by the host (HOST_HANDLED) */ 464 if (r->access == NULL) 465 return false; 466 467 /* Handled by hyp: skip instruction if instructed to do so. */ 468 if (r->access(vcpu, ¶ms, r)) 469 __kvm_skip_instr(vcpu); 470 471 if (!params.is_write) 472 vcpu_set_reg(vcpu, Rt, params.regval); 473 474 return true; 475 } 476 477 /* 478 * Handler for protected VM restricted exceptions. 479 * 480 * Inject an undefined exception into the guest and return true to indicate that 481 * the hypervisor has handled the exit, and control should go back to the guest. 482 */ 483 bool kvm_handle_pvm_restricted(struct kvm_vcpu *vcpu, u64 *exit_code) 484 { 485 inject_undef64(vcpu); 486 return true; 487 } 488