1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * Copyright (C) 2021 Google LLC 4 * Author: Fuad Tabba <tabba@google.com> 5 */ 6 7 #include <linux/irqchip/arm-gic-v3.h> 8 9 #include <asm/kvm_asm.h> 10 #include <asm/kvm_mmu.h> 11 12 #include <hyp/adjust_pc.h> 13 14 #include <nvhe/fixed_config.h> 15 16 #include "../../sys_regs.h" 17 18 /* 19 * Copies of the host's CPU features registers holding sanitized values at hyp. 20 */ 21 u64 id_aa64pfr0_el1_sys_val; 22 u64 id_aa64pfr1_el1_sys_val; 23 u64 id_aa64isar0_el1_sys_val; 24 u64 id_aa64isar1_el1_sys_val; 25 u64 id_aa64isar2_el1_sys_val; 26 u64 id_aa64mmfr0_el1_sys_val; 27 u64 id_aa64mmfr1_el1_sys_val; 28 u64 id_aa64mmfr2_el1_sys_val; 29 30 /* 31 * Inject an unknown/undefined exception to an AArch64 guest while most of its 32 * sysregs are live. 33 */ 34 static void inject_undef64(struct kvm_vcpu *vcpu) 35 { 36 u32 esr = (ESR_ELx_EC_UNKNOWN << ESR_ELx_EC_SHIFT); 37 38 *vcpu_pc(vcpu) = read_sysreg_el2(SYS_ELR); 39 *vcpu_cpsr(vcpu) = read_sysreg_el2(SYS_SPSR); 40 41 vcpu->arch.flags |= (KVM_ARM64_EXCEPT_AA64_EL1 | 42 KVM_ARM64_EXCEPT_AA64_ELx_SYNC | 43 KVM_ARM64_PENDING_EXCEPTION); 44 45 __kvm_adjust_pc(vcpu); 46 47 write_sysreg_el1(esr, SYS_ESR); 48 write_sysreg_el1(read_sysreg_el2(SYS_ELR), SYS_ELR); 49 write_sysreg_el2(*vcpu_pc(vcpu), SYS_ELR); 50 write_sysreg_el2(*vcpu_cpsr(vcpu), SYS_SPSR); 51 } 52 53 /* 54 * Returns the restricted features values of the feature register based on the 55 * limitations in restrict_fields. 56 * A feature id field value of 0b0000 does not impose any restrictions. 57 * Note: Use only for unsigned feature field values. 58 */ 59 static u64 get_restricted_features_unsigned(u64 sys_reg_val, 60 u64 restrict_fields) 61 { 62 u64 value = 0UL; 63 u64 mask = GENMASK_ULL(ARM64_FEATURE_FIELD_BITS - 1, 0); 64 65 /* 66 * According to the Arm Architecture Reference Manual, feature fields 67 * use increasing values to indicate increases in functionality. 68 * Iterate over the restricted feature fields and calculate the minimum 69 * unsigned value between the one supported by the system, and what the 70 * value is being restricted to. 71 */ 72 while (sys_reg_val && restrict_fields) { 73 value |= min(sys_reg_val & mask, restrict_fields & mask); 74 sys_reg_val &= ~mask; 75 restrict_fields &= ~mask; 76 mask <<= ARM64_FEATURE_FIELD_BITS; 77 } 78 79 return value; 80 } 81 82 /* 83 * Functions that return the value of feature id registers for protected VMs 84 * based on allowed features, system features, and KVM support. 85 */ 86 87 static u64 get_pvm_id_aa64pfr0(const struct kvm_vcpu *vcpu) 88 { 89 const struct kvm *kvm = (const struct kvm *)kern_hyp_va(vcpu->kvm); 90 u64 set_mask = 0; 91 u64 allow_mask = PVM_ID_AA64PFR0_ALLOW; 92 93 if (!vcpu_has_sve(vcpu)) 94 allow_mask &= ~ARM64_FEATURE_MASK(ID_AA64PFR0_SVE); 95 96 set_mask |= get_restricted_features_unsigned(id_aa64pfr0_el1_sys_val, 97 PVM_ID_AA64PFR0_RESTRICT_UNSIGNED); 98 99 /* Spectre and Meltdown mitigation in KVM */ 100 set_mask |= FIELD_PREP(ARM64_FEATURE_MASK(ID_AA64PFR0_CSV2), 101 (u64)kvm->arch.pfr0_csv2); 102 set_mask |= FIELD_PREP(ARM64_FEATURE_MASK(ID_AA64PFR0_CSV3), 103 (u64)kvm->arch.pfr0_csv3); 104 105 return (id_aa64pfr0_el1_sys_val & allow_mask) | set_mask; 106 } 107 108 static u64 get_pvm_id_aa64pfr1(const struct kvm_vcpu *vcpu) 109 { 110 const struct kvm *kvm = (const struct kvm *)kern_hyp_va(vcpu->kvm); 111 u64 allow_mask = PVM_ID_AA64PFR1_ALLOW; 112 113 if (!kvm_has_mte(kvm)) 114 allow_mask &= ~ARM64_FEATURE_MASK(ID_AA64PFR1_MTE); 115 116 return id_aa64pfr1_el1_sys_val & allow_mask; 117 } 118 119 static u64 get_pvm_id_aa64zfr0(const struct kvm_vcpu *vcpu) 120 { 121 /* 122 * No support for Scalable Vectors, therefore, hyp has no sanitized 123 * copy of the feature id register. 124 */ 125 BUILD_BUG_ON(PVM_ID_AA64ZFR0_ALLOW != 0ULL); 126 return 0; 127 } 128 129 static u64 get_pvm_id_aa64dfr0(const struct kvm_vcpu *vcpu) 130 { 131 /* 132 * No support for debug, including breakpoints, and watchpoints, 133 * therefore, pKVM has no sanitized copy of the feature id register. 134 */ 135 BUILD_BUG_ON(PVM_ID_AA64DFR0_ALLOW != 0ULL); 136 return 0; 137 } 138 139 static u64 get_pvm_id_aa64dfr1(const struct kvm_vcpu *vcpu) 140 { 141 /* 142 * No support for debug, therefore, hyp has no sanitized copy of the 143 * feature id register. 144 */ 145 BUILD_BUG_ON(PVM_ID_AA64DFR1_ALLOW != 0ULL); 146 return 0; 147 } 148 149 static u64 get_pvm_id_aa64afr0(const struct kvm_vcpu *vcpu) 150 { 151 /* 152 * No support for implementation defined features, therefore, hyp has no 153 * sanitized copy of the feature id register. 154 */ 155 BUILD_BUG_ON(PVM_ID_AA64AFR0_ALLOW != 0ULL); 156 return 0; 157 } 158 159 static u64 get_pvm_id_aa64afr1(const struct kvm_vcpu *vcpu) 160 { 161 /* 162 * No support for implementation defined features, therefore, hyp has no 163 * sanitized copy of the feature id register. 164 */ 165 BUILD_BUG_ON(PVM_ID_AA64AFR1_ALLOW != 0ULL); 166 return 0; 167 } 168 169 static u64 get_pvm_id_aa64isar0(const struct kvm_vcpu *vcpu) 170 { 171 return id_aa64isar0_el1_sys_val & PVM_ID_AA64ISAR0_ALLOW; 172 } 173 174 static u64 get_pvm_id_aa64isar1(const struct kvm_vcpu *vcpu) 175 { 176 u64 allow_mask = PVM_ID_AA64ISAR1_ALLOW; 177 178 if (!vcpu_has_ptrauth(vcpu)) 179 allow_mask &= ~(ARM64_FEATURE_MASK(ID_AA64ISAR1_APA) | 180 ARM64_FEATURE_MASK(ID_AA64ISAR1_API) | 181 ARM64_FEATURE_MASK(ID_AA64ISAR1_GPA) | 182 ARM64_FEATURE_MASK(ID_AA64ISAR1_GPI)); 183 184 return id_aa64isar1_el1_sys_val & allow_mask; 185 } 186 187 static u64 get_pvm_id_aa64isar2(const struct kvm_vcpu *vcpu) 188 { 189 u64 allow_mask = PVM_ID_AA64ISAR2_ALLOW; 190 191 if (!vcpu_has_ptrauth(vcpu)) 192 allow_mask &= ~(ARM64_FEATURE_MASK(ID_AA64ISAR2_APA3) | 193 ARM64_FEATURE_MASK(ID_AA64ISAR2_GPA3)); 194 195 return id_aa64isar2_el1_sys_val & allow_mask; 196 } 197 198 static u64 get_pvm_id_aa64mmfr0(const struct kvm_vcpu *vcpu) 199 { 200 u64 set_mask; 201 202 set_mask = get_restricted_features_unsigned(id_aa64mmfr0_el1_sys_val, 203 PVM_ID_AA64MMFR0_RESTRICT_UNSIGNED); 204 205 return (id_aa64mmfr0_el1_sys_val & PVM_ID_AA64MMFR0_ALLOW) | set_mask; 206 } 207 208 static u64 get_pvm_id_aa64mmfr1(const struct kvm_vcpu *vcpu) 209 { 210 return id_aa64mmfr1_el1_sys_val & PVM_ID_AA64MMFR1_ALLOW; 211 } 212 213 static u64 get_pvm_id_aa64mmfr2(const struct kvm_vcpu *vcpu) 214 { 215 return id_aa64mmfr2_el1_sys_val & PVM_ID_AA64MMFR2_ALLOW; 216 } 217 218 /* Read a sanitized cpufeature ID register by its encoding */ 219 u64 pvm_read_id_reg(const struct kvm_vcpu *vcpu, u32 id) 220 { 221 switch (id) { 222 case SYS_ID_AA64PFR0_EL1: 223 return get_pvm_id_aa64pfr0(vcpu); 224 case SYS_ID_AA64PFR1_EL1: 225 return get_pvm_id_aa64pfr1(vcpu); 226 case SYS_ID_AA64ZFR0_EL1: 227 return get_pvm_id_aa64zfr0(vcpu); 228 case SYS_ID_AA64DFR0_EL1: 229 return get_pvm_id_aa64dfr0(vcpu); 230 case SYS_ID_AA64DFR1_EL1: 231 return get_pvm_id_aa64dfr1(vcpu); 232 case SYS_ID_AA64AFR0_EL1: 233 return get_pvm_id_aa64afr0(vcpu); 234 case SYS_ID_AA64AFR1_EL1: 235 return get_pvm_id_aa64afr1(vcpu); 236 case SYS_ID_AA64ISAR0_EL1: 237 return get_pvm_id_aa64isar0(vcpu); 238 case SYS_ID_AA64ISAR1_EL1: 239 return get_pvm_id_aa64isar1(vcpu); 240 case SYS_ID_AA64ISAR2_EL1: 241 return get_pvm_id_aa64isar2(vcpu); 242 case SYS_ID_AA64MMFR0_EL1: 243 return get_pvm_id_aa64mmfr0(vcpu); 244 case SYS_ID_AA64MMFR1_EL1: 245 return get_pvm_id_aa64mmfr1(vcpu); 246 case SYS_ID_AA64MMFR2_EL1: 247 return get_pvm_id_aa64mmfr2(vcpu); 248 default: 249 /* 250 * Should never happen because all cases are covered in 251 * pvm_sys_reg_descs[]. 252 */ 253 WARN_ON(1); 254 break; 255 } 256 257 return 0; 258 } 259 260 static u64 read_id_reg(const struct kvm_vcpu *vcpu, 261 struct sys_reg_desc const *r) 262 { 263 return pvm_read_id_reg(vcpu, reg_to_encoding(r)); 264 } 265 266 /* Handler to RAZ/WI sysregs */ 267 static bool pvm_access_raz_wi(struct kvm_vcpu *vcpu, struct sys_reg_params *p, 268 const struct sys_reg_desc *r) 269 { 270 if (!p->is_write) 271 p->regval = 0; 272 273 return true; 274 } 275 276 /* 277 * Accessor for AArch32 feature id registers. 278 * 279 * The value of these registers is "unknown" according to the spec if AArch32 280 * isn't supported. 281 */ 282 static bool pvm_access_id_aarch32(struct kvm_vcpu *vcpu, 283 struct sys_reg_params *p, 284 const struct sys_reg_desc *r) 285 { 286 if (p->is_write) { 287 inject_undef64(vcpu); 288 return false; 289 } 290 291 /* 292 * No support for AArch32 guests, therefore, pKVM has no sanitized copy 293 * of AArch32 feature id registers. 294 */ 295 BUILD_BUG_ON(FIELD_GET(ARM64_FEATURE_MASK(ID_AA64PFR0_EL1), 296 PVM_ID_AA64PFR0_RESTRICT_UNSIGNED) > ID_AA64PFR0_ELx_64BIT_ONLY); 297 298 return pvm_access_raz_wi(vcpu, p, r); 299 } 300 301 /* 302 * Accessor for AArch64 feature id registers. 303 * 304 * If access is allowed, set the regval to the protected VM's view of the 305 * register and return true. 306 * Otherwise, inject an undefined exception and return false. 307 */ 308 static bool pvm_access_id_aarch64(struct kvm_vcpu *vcpu, 309 struct sys_reg_params *p, 310 const struct sys_reg_desc *r) 311 { 312 if (p->is_write) { 313 inject_undef64(vcpu); 314 return false; 315 } 316 317 p->regval = read_id_reg(vcpu, r); 318 return true; 319 } 320 321 static bool pvm_gic_read_sre(struct kvm_vcpu *vcpu, 322 struct sys_reg_params *p, 323 const struct sys_reg_desc *r) 324 { 325 /* pVMs only support GICv3. 'nuf said. */ 326 if (!p->is_write) 327 p->regval = ICC_SRE_EL1_DIB | ICC_SRE_EL1_DFB | ICC_SRE_EL1_SRE; 328 329 return true; 330 } 331 332 /* Mark the specified system register as an AArch32 feature id register. */ 333 #define AARCH32(REG) { SYS_DESC(REG), .access = pvm_access_id_aarch32 } 334 335 /* Mark the specified system register as an AArch64 feature id register. */ 336 #define AARCH64(REG) { SYS_DESC(REG), .access = pvm_access_id_aarch64 } 337 338 /* Mark the specified system register as Read-As-Zero/Write-Ignored */ 339 #define RAZ_WI(REG) { SYS_DESC(REG), .access = pvm_access_raz_wi } 340 341 /* Mark the specified system register as not being handled in hyp. */ 342 #define HOST_HANDLED(REG) { SYS_DESC(REG), .access = NULL } 343 344 /* 345 * Architected system registers. 346 * Important: Must be sorted ascending by Op0, Op1, CRn, CRm, Op2 347 * 348 * NOTE: Anything not explicitly listed here is *restricted by default*, i.e., 349 * it will lead to injecting an exception into the guest. 350 */ 351 static const struct sys_reg_desc pvm_sys_reg_descs[] = { 352 /* Cache maintenance by set/way operations are restricted. */ 353 354 /* Debug and Trace Registers are restricted. */ 355 356 /* AArch64 mappings of the AArch32 ID registers */ 357 /* CRm=1 */ 358 AARCH32(SYS_ID_PFR0_EL1), 359 AARCH32(SYS_ID_PFR1_EL1), 360 AARCH32(SYS_ID_DFR0_EL1), 361 AARCH32(SYS_ID_AFR0_EL1), 362 AARCH32(SYS_ID_MMFR0_EL1), 363 AARCH32(SYS_ID_MMFR1_EL1), 364 AARCH32(SYS_ID_MMFR2_EL1), 365 AARCH32(SYS_ID_MMFR3_EL1), 366 367 /* CRm=2 */ 368 AARCH32(SYS_ID_ISAR0_EL1), 369 AARCH32(SYS_ID_ISAR1_EL1), 370 AARCH32(SYS_ID_ISAR2_EL1), 371 AARCH32(SYS_ID_ISAR3_EL1), 372 AARCH32(SYS_ID_ISAR4_EL1), 373 AARCH32(SYS_ID_ISAR5_EL1), 374 AARCH32(SYS_ID_MMFR4_EL1), 375 AARCH32(SYS_ID_ISAR6_EL1), 376 377 /* CRm=3 */ 378 AARCH32(SYS_MVFR0_EL1), 379 AARCH32(SYS_MVFR1_EL1), 380 AARCH32(SYS_MVFR2_EL1), 381 AARCH32(SYS_ID_PFR2_EL1), 382 AARCH32(SYS_ID_DFR1_EL1), 383 AARCH32(SYS_ID_MMFR5_EL1), 384 385 /* AArch64 ID registers */ 386 /* CRm=4 */ 387 AARCH64(SYS_ID_AA64PFR0_EL1), 388 AARCH64(SYS_ID_AA64PFR1_EL1), 389 AARCH64(SYS_ID_AA64ZFR0_EL1), 390 AARCH64(SYS_ID_AA64DFR0_EL1), 391 AARCH64(SYS_ID_AA64DFR1_EL1), 392 AARCH64(SYS_ID_AA64AFR0_EL1), 393 AARCH64(SYS_ID_AA64AFR1_EL1), 394 AARCH64(SYS_ID_AA64ISAR0_EL1), 395 AARCH64(SYS_ID_AA64ISAR1_EL1), 396 AARCH64(SYS_ID_AA64MMFR0_EL1), 397 AARCH64(SYS_ID_AA64MMFR1_EL1), 398 AARCH64(SYS_ID_AA64MMFR2_EL1), 399 400 /* Scalable Vector Registers are restricted. */ 401 402 RAZ_WI(SYS_ERRIDR_EL1), 403 RAZ_WI(SYS_ERRSELR_EL1), 404 RAZ_WI(SYS_ERXFR_EL1), 405 RAZ_WI(SYS_ERXCTLR_EL1), 406 RAZ_WI(SYS_ERXSTATUS_EL1), 407 RAZ_WI(SYS_ERXADDR_EL1), 408 RAZ_WI(SYS_ERXMISC0_EL1), 409 RAZ_WI(SYS_ERXMISC1_EL1), 410 411 /* Performance Monitoring Registers are restricted. */ 412 413 /* Limited Ordering Regions Registers are restricted. */ 414 415 HOST_HANDLED(SYS_ICC_SGI1R_EL1), 416 HOST_HANDLED(SYS_ICC_ASGI1R_EL1), 417 HOST_HANDLED(SYS_ICC_SGI0R_EL1), 418 { SYS_DESC(SYS_ICC_SRE_EL1), .access = pvm_gic_read_sre, }, 419 420 HOST_HANDLED(SYS_CCSIDR_EL1), 421 HOST_HANDLED(SYS_CLIDR_EL1), 422 HOST_HANDLED(SYS_CSSELR_EL1), 423 HOST_HANDLED(SYS_CTR_EL0), 424 425 /* Performance Monitoring Registers are restricted. */ 426 427 /* Activity Monitoring Registers are restricted. */ 428 429 HOST_HANDLED(SYS_CNTP_TVAL_EL0), 430 HOST_HANDLED(SYS_CNTP_CTL_EL0), 431 HOST_HANDLED(SYS_CNTP_CVAL_EL0), 432 433 /* Performance Monitoring Registers are restricted. */ 434 }; 435 436 /* 437 * Checks that the sysreg table is unique and in-order. 438 * 439 * Returns 0 if the table is consistent, or 1 otherwise. 440 */ 441 int kvm_check_pvm_sysreg_table(void) 442 { 443 unsigned int i; 444 445 for (i = 1; i < ARRAY_SIZE(pvm_sys_reg_descs); i++) { 446 if (cmp_sys_reg(&pvm_sys_reg_descs[i-1], &pvm_sys_reg_descs[i]) >= 0) 447 return 1; 448 } 449 450 return 0; 451 } 452 453 /* 454 * Handler for protected VM MSR, MRS or System instruction execution. 455 * 456 * Returns true if the hypervisor has handled the exit, and control should go 457 * back to the guest, or false if it hasn't, to be handled by the host. 458 */ 459 bool kvm_handle_pvm_sysreg(struct kvm_vcpu *vcpu, u64 *exit_code) 460 { 461 const struct sys_reg_desc *r; 462 struct sys_reg_params params; 463 unsigned long esr = kvm_vcpu_get_esr(vcpu); 464 int Rt = kvm_vcpu_sys_get_rt(vcpu); 465 466 params = esr_sys64_to_params(esr); 467 params.regval = vcpu_get_reg(vcpu, Rt); 468 469 r = find_reg(¶ms, pvm_sys_reg_descs, ARRAY_SIZE(pvm_sys_reg_descs)); 470 471 /* Undefined (RESTRICTED). */ 472 if (r == NULL) { 473 inject_undef64(vcpu); 474 return true; 475 } 476 477 /* Handled by the host (HOST_HANDLED) */ 478 if (r->access == NULL) 479 return false; 480 481 /* Handled by hyp: skip instruction if instructed to do so. */ 482 if (r->access(vcpu, ¶ms, r)) 483 __kvm_skip_instr(vcpu); 484 485 if (!params.is_write) 486 vcpu_set_reg(vcpu, Rt, params.regval); 487 488 return true; 489 } 490 491 /* 492 * Handler for protected VM restricted exceptions. 493 * 494 * Inject an undefined exception into the guest and return true to indicate that 495 * the hypervisor has handled the exit, and control should go back to the guest. 496 */ 497 bool kvm_handle_pvm_restricted(struct kvm_vcpu *vcpu, u64 *exit_code) 498 { 499 inject_undef64(vcpu); 500 return true; 501 } 502