1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * Copyright (C) 2021 Google LLC 4 * Author: Fuad Tabba <tabba@google.com> 5 */ 6 7 #include <linux/irqchip/arm-gic-v3.h> 8 9 #include <asm/kvm_asm.h> 10 #include <asm/kvm_mmu.h> 11 12 #include <hyp/adjust_pc.h> 13 14 #include <nvhe/fixed_config.h> 15 16 #include "../../sys_regs.h" 17 18 /* 19 * Copies of the host's CPU features registers holding sanitized values at hyp. 20 */ 21 u64 id_aa64pfr0_el1_sys_val; 22 u64 id_aa64pfr1_el1_sys_val; 23 u64 id_aa64isar0_el1_sys_val; 24 u64 id_aa64isar1_el1_sys_val; 25 u64 id_aa64isar2_el1_sys_val; 26 u64 id_aa64mmfr0_el1_sys_val; 27 u64 id_aa64mmfr1_el1_sys_val; 28 u64 id_aa64mmfr2_el1_sys_val; 29 u64 id_aa64smfr0_el1_sys_val; 30 31 /* 32 * Inject an unknown/undefined exception to an AArch64 guest while most of its 33 * sysregs are live. 34 */ 35 static void inject_undef64(struct kvm_vcpu *vcpu) 36 { 37 u64 esr = (ESR_ELx_EC_UNKNOWN << ESR_ELx_EC_SHIFT); 38 39 *vcpu_pc(vcpu) = read_sysreg_el2(SYS_ELR); 40 *vcpu_cpsr(vcpu) = read_sysreg_el2(SYS_SPSR); 41 42 kvm_pend_exception(vcpu, EXCEPT_AA64_EL1_SYNC); 43 44 __kvm_adjust_pc(vcpu); 45 46 write_sysreg_el1(esr, SYS_ESR); 47 write_sysreg_el1(read_sysreg_el2(SYS_ELR), SYS_ELR); 48 write_sysreg_el2(*vcpu_pc(vcpu), SYS_ELR); 49 write_sysreg_el2(*vcpu_cpsr(vcpu), SYS_SPSR); 50 } 51 52 /* 53 * Returns the restricted features values of the feature register based on the 54 * limitations in restrict_fields. 55 * A feature id field value of 0b0000 does not impose any restrictions. 56 * Note: Use only for unsigned feature field values. 57 */ 58 static u64 get_restricted_features_unsigned(u64 sys_reg_val, 59 u64 restrict_fields) 60 { 61 u64 value = 0UL; 62 u64 mask = GENMASK_ULL(ARM64_FEATURE_FIELD_BITS - 1, 0); 63 64 /* 65 * According to the Arm Architecture Reference Manual, feature fields 66 * use increasing values to indicate increases in functionality. 67 * Iterate over the restricted feature fields and calculate the minimum 68 * unsigned value between the one supported by the system, and what the 69 * value is being restricted to. 70 */ 71 while (sys_reg_val && restrict_fields) { 72 value |= min(sys_reg_val & mask, restrict_fields & mask); 73 sys_reg_val &= ~mask; 74 restrict_fields &= ~mask; 75 mask <<= ARM64_FEATURE_FIELD_BITS; 76 } 77 78 return value; 79 } 80 81 /* 82 * Functions that return the value of feature id registers for protected VMs 83 * based on allowed features, system features, and KVM support. 84 */ 85 86 static u64 get_pvm_id_aa64pfr0(const struct kvm_vcpu *vcpu) 87 { 88 const struct kvm *kvm = (const struct kvm *)kern_hyp_va(vcpu->kvm); 89 u64 set_mask = 0; 90 u64 allow_mask = PVM_ID_AA64PFR0_ALLOW; 91 92 set_mask |= get_restricted_features_unsigned(id_aa64pfr0_el1_sys_val, 93 PVM_ID_AA64PFR0_RESTRICT_UNSIGNED); 94 95 /* Spectre and Meltdown mitigation in KVM */ 96 set_mask |= FIELD_PREP(ARM64_FEATURE_MASK(ID_AA64PFR0_EL1_CSV2), 97 (u64)kvm->arch.pfr0_csv2); 98 set_mask |= FIELD_PREP(ARM64_FEATURE_MASK(ID_AA64PFR0_EL1_CSV3), 99 (u64)kvm->arch.pfr0_csv3); 100 101 return (id_aa64pfr0_el1_sys_val & allow_mask) | set_mask; 102 } 103 104 static u64 get_pvm_id_aa64pfr1(const struct kvm_vcpu *vcpu) 105 { 106 const struct kvm *kvm = (const struct kvm *)kern_hyp_va(vcpu->kvm); 107 u64 allow_mask = PVM_ID_AA64PFR1_ALLOW; 108 109 if (!kvm_has_mte(kvm)) 110 allow_mask &= ~ARM64_FEATURE_MASK(ID_AA64PFR1_EL1_MTE); 111 112 return id_aa64pfr1_el1_sys_val & allow_mask; 113 } 114 115 static u64 get_pvm_id_aa64zfr0(const struct kvm_vcpu *vcpu) 116 { 117 /* 118 * No support for Scalable Vectors, therefore, hyp has no sanitized 119 * copy of the feature id register. 120 */ 121 BUILD_BUG_ON(PVM_ID_AA64ZFR0_ALLOW != 0ULL); 122 return 0; 123 } 124 125 static u64 get_pvm_id_aa64dfr0(const struct kvm_vcpu *vcpu) 126 { 127 /* 128 * No support for debug, including breakpoints, and watchpoints, 129 * therefore, pKVM has no sanitized copy of the feature id register. 130 */ 131 BUILD_BUG_ON(PVM_ID_AA64DFR0_ALLOW != 0ULL); 132 return 0; 133 } 134 135 static u64 get_pvm_id_aa64dfr1(const struct kvm_vcpu *vcpu) 136 { 137 /* 138 * No support for debug, therefore, hyp has no sanitized copy of the 139 * feature id register. 140 */ 141 BUILD_BUG_ON(PVM_ID_AA64DFR1_ALLOW != 0ULL); 142 return 0; 143 } 144 145 static u64 get_pvm_id_aa64afr0(const struct kvm_vcpu *vcpu) 146 { 147 /* 148 * No support for implementation defined features, therefore, hyp has no 149 * sanitized copy of the feature id register. 150 */ 151 BUILD_BUG_ON(PVM_ID_AA64AFR0_ALLOW != 0ULL); 152 return 0; 153 } 154 155 static u64 get_pvm_id_aa64afr1(const struct kvm_vcpu *vcpu) 156 { 157 /* 158 * No support for implementation defined features, therefore, hyp has no 159 * sanitized copy of the feature id register. 160 */ 161 BUILD_BUG_ON(PVM_ID_AA64AFR1_ALLOW != 0ULL); 162 return 0; 163 } 164 165 static u64 get_pvm_id_aa64isar0(const struct kvm_vcpu *vcpu) 166 { 167 return id_aa64isar0_el1_sys_val & PVM_ID_AA64ISAR0_ALLOW; 168 } 169 170 static u64 get_pvm_id_aa64isar1(const struct kvm_vcpu *vcpu) 171 { 172 u64 allow_mask = PVM_ID_AA64ISAR1_ALLOW; 173 174 if (!vcpu_has_ptrauth(vcpu)) 175 allow_mask &= ~(ARM64_FEATURE_MASK(ID_AA64ISAR1_EL1_APA) | 176 ARM64_FEATURE_MASK(ID_AA64ISAR1_EL1_API) | 177 ARM64_FEATURE_MASK(ID_AA64ISAR1_EL1_GPA) | 178 ARM64_FEATURE_MASK(ID_AA64ISAR1_EL1_GPI)); 179 180 return id_aa64isar1_el1_sys_val & allow_mask; 181 } 182 183 static u64 get_pvm_id_aa64isar2(const struct kvm_vcpu *vcpu) 184 { 185 u64 allow_mask = PVM_ID_AA64ISAR2_ALLOW; 186 187 if (!vcpu_has_ptrauth(vcpu)) 188 allow_mask &= ~(ARM64_FEATURE_MASK(ID_AA64ISAR2_EL1_APA3) | 189 ARM64_FEATURE_MASK(ID_AA64ISAR2_EL1_GPA3)); 190 191 return id_aa64isar2_el1_sys_val & allow_mask; 192 } 193 194 static u64 get_pvm_id_aa64mmfr0(const struct kvm_vcpu *vcpu) 195 { 196 u64 set_mask; 197 198 set_mask = get_restricted_features_unsigned(id_aa64mmfr0_el1_sys_val, 199 PVM_ID_AA64MMFR0_RESTRICT_UNSIGNED); 200 201 return (id_aa64mmfr0_el1_sys_val & PVM_ID_AA64MMFR0_ALLOW) | set_mask; 202 } 203 204 static u64 get_pvm_id_aa64mmfr1(const struct kvm_vcpu *vcpu) 205 { 206 return id_aa64mmfr1_el1_sys_val & PVM_ID_AA64MMFR1_ALLOW; 207 } 208 209 static u64 get_pvm_id_aa64mmfr2(const struct kvm_vcpu *vcpu) 210 { 211 return id_aa64mmfr2_el1_sys_val & PVM_ID_AA64MMFR2_ALLOW; 212 } 213 214 /* Read a sanitized cpufeature ID register by its encoding */ 215 u64 pvm_read_id_reg(const struct kvm_vcpu *vcpu, u32 id) 216 { 217 switch (id) { 218 case SYS_ID_AA64PFR0_EL1: 219 return get_pvm_id_aa64pfr0(vcpu); 220 case SYS_ID_AA64PFR1_EL1: 221 return get_pvm_id_aa64pfr1(vcpu); 222 case SYS_ID_AA64ZFR0_EL1: 223 return get_pvm_id_aa64zfr0(vcpu); 224 case SYS_ID_AA64DFR0_EL1: 225 return get_pvm_id_aa64dfr0(vcpu); 226 case SYS_ID_AA64DFR1_EL1: 227 return get_pvm_id_aa64dfr1(vcpu); 228 case SYS_ID_AA64AFR0_EL1: 229 return get_pvm_id_aa64afr0(vcpu); 230 case SYS_ID_AA64AFR1_EL1: 231 return get_pvm_id_aa64afr1(vcpu); 232 case SYS_ID_AA64ISAR0_EL1: 233 return get_pvm_id_aa64isar0(vcpu); 234 case SYS_ID_AA64ISAR1_EL1: 235 return get_pvm_id_aa64isar1(vcpu); 236 case SYS_ID_AA64ISAR2_EL1: 237 return get_pvm_id_aa64isar2(vcpu); 238 case SYS_ID_AA64MMFR0_EL1: 239 return get_pvm_id_aa64mmfr0(vcpu); 240 case SYS_ID_AA64MMFR1_EL1: 241 return get_pvm_id_aa64mmfr1(vcpu); 242 case SYS_ID_AA64MMFR2_EL1: 243 return get_pvm_id_aa64mmfr2(vcpu); 244 default: 245 /* Unhandled ID register, RAZ */ 246 return 0; 247 } 248 } 249 250 static u64 read_id_reg(const struct kvm_vcpu *vcpu, 251 struct sys_reg_desc const *r) 252 { 253 return pvm_read_id_reg(vcpu, reg_to_encoding(r)); 254 } 255 256 /* Handler to RAZ/WI sysregs */ 257 static bool pvm_access_raz_wi(struct kvm_vcpu *vcpu, struct sys_reg_params *p, 258 const struct sys_reg_desc *r) 259 { 260 if (!p->is_write) 261 p->regval = 0; 262 263 return true; 264 } 265 266 /* 267 * Accessor for AArch32 feature id registers. 268 * 269 * The value of these registers is "unknown" according to the spec if AArch32 270 * isn't supported. 271 */ 272 static bool pvm_access_id_aarch32(struct kvm_vcpu *vcpu, 273 struct sys_reg_params *p, 274 const struct sys_reg_desc *r) 275 { 276 if (p->is_write) { 277 inject_undef64(vcpu); 278 return false; 279 } 280 281 /* 282 * No support for AArch32 guests, therefore, pKVM has no sanitized copy 283 * of AArch32 feature id registers. 284 */ 285 BUILD_BUG_ON(FIELD_GET(ARM64_FEATURE_MASK(ID_AA64PFR0_EL1_EL1), 286 PVM_ID_AA64PFR0_RESTRICT_UNSIGNED) > ID_AA64PFR0_EL1_ELx_64BIT_ONLY); 287 288 return pvm_access_raz_wi(vcpu, p, r); 289 } 290 291 /* 292 * Accessor for AArch64 feature id registers. 293 * 294 * If access is allowed, set the regval to the protected VM's view of the 295 * register and return true. 296 * Otherwise, inject an undefined exception and return false. 297 */ 298 static bool pvm_access_id_aarch64(struct kvm_vcpu *vcpu, 299 struct sys_reg_params *p, 300 const struct sys_reg_desc *r) 301 { 302 if (p->is_write) { 303 inject_undef64(vcpu); 304 return false; 305 } 306 307 p->regval = read_id_reg(vcpu, r); 308 return true; 309 } 310 311 static bool pvm_gic_read_sre(struct kvm_vcpu *vcpu, 312 struct sys_reg_params *p, 313 const struct sys_reg_desc *r) 314 { 315 /* pVMs only support GICv3. 'nuf said. */ 316 if (!p->is_write) 317 p->regval = ICC_SRE_EL1_DIB | ICC_SRE_EL1_DFB | ICC_SRE_EL1_SRE; 318 319 return true; 320 } 321 322 /* Mark the specified system register as an AArch32 feature id register. */ 323 #define AARCH32(REG) { SYS_DESC(REG), .access = pvm_access_id_aarch32 } 324 325 /* Mark the specified system register as an AArch64 feature id register. */ 326 #define AARCH64(REG) { SYS_DESC(REG), .access = pvm_access_id_aarch64 } 327 328 /* 329 * sys_reg_desc initialiser for architecturally unallocated cpufeature ID 330 * register with encoding Op0=3, Op1=0, CRn=0, CRm=crm, Op2=op2 331 * (1 <= crm < 8, 0 <= Op2 < 8). 332 */ 333 #define ID_UNALLOCATED(crm, op2) { \ 334 Op0(3), Op1(0), CRn(0), CRm(crm), Op2(op2), \ 335 .access = pvm_access_id_aarch64, \ 336 } 337 338 /* Mark the specified system register as Read-As-Zero/Write-Ignored */ 339 #define RAZ_WI(REG) { SYS_DESC(REG), .access = pvm_access_raz_wi } 340 341 /* Mark the specified system register as not being handled in hyp. */ 342 #define HOST_HANDLED(REG) { SYS_DESC(REG), .access = NULL } 343 344 /* 345 * Architected system registers. 346 * Important: Must be sorted ascending by Op0, Op1, CRn, CRm, Op2 347 * 348 * NOTE: Anything not explicitly listed here is *restricted by default*, i.e., 349 * it will lead to injecting an exception into the guest. 350 */ 351 static const struct sys_reg_desc pvm_sys_reg_descs[] = { 352 /* Cache maintenance by set/way operations are restricted. */ 353 354 /* Debug and Trace Registers are restricted. */ 355 356 /* AArch64 mappings of the AArch32 ID registers */ 357 /* CRm=1 */ 358 AARCH32(SYS_ID_PFR0_EL1), 359 AARCH32(SYS_ID_PFR1_EL1), 360 AARCH32(SYS_ID_DFR0_EL1), 361 AARCH32(SYS_ID_AFR0_EL1), 362 AARCH32(SYS_ID_MMFR0_EL1), 363 AARCH32(SYS_ID_MMFR1_EL1), 364 AARCH32(SYS_ID_MMFR2_EL1), 365 AARCH32(SYS_ID_MMFR3_EL1), 366 367 /* CRm=2 */ 368 AARCH32(SYS_ID_ISAR0_EL1), 369 AARCH32(SYS_ID_ISAR1_EL1), 370 AARCH32(SYS_ID_ISAR2_EL1), 371 AARCH32(SYS_ID_ISAR3_EL1), 372 AARCH32(SYS_ID_ISAR4_EL1), 373 AARCH32(SYS_ID_ISAR5_EL1), 374 AARCH32(SYS_ID_MMFR4_EL1), 375 AARCH32(SYS_ID_ISAR6_EL1), 376 377 /* CRm=3 */ 378 AARCH32(SYS_MVFR0_EL1), 379 AARCH32(SYS_MVFR1_EL1), 380 AARCH32(SYS_MVFR2_EL1), 381 ID_UNALLOCATED(3,3), 382 AARCH32(SYS_ID_PFR2_EL1), 383 AARCH32(SYS_ID_DFR1_EL1), 384 AARCH32(SYS_ID_MMFR5_EL1), 385 ID_UNALLOCATED(3,7), 386 387 /* AArch64 ID registers */ 388 /* CRm=4 */ 389 AARCH64(SYS_ID_AA64PFR0_EL1), 390 AARCH64(SYS_ID_AA64PFR1_EL1), 391 ID_UNALLOCATED(4,2), 392 ID_UNALLOCATED(4,3), 393 AARCH64(SYS_ID_AA64ZFR0_EL1), 394 ID_UNALLOCATED(4,5), 395 ID_UNALLOCATED(4,6), 396 ID_UNALLOCATED(4,7), 397 AARCH64(SYS_ID_AA64DFR0_EL1), 398 AARCH64(SYS_ID_AA64DFR1_EL1), 399 ID_UNALLOCATED(5,2), 400 ID_UNALLOCATED(5,3), 401 AARCH64(SYS_ID_AA64AFR0_EL1), 402 AARCH64(SYS_ID_AA64AFR1_EL1), 403 ID_UNALLOCATED(5,6), 404 ID_UNALLOCATED(5,7), 405 AARCH64(SYS_ID_AA64ISAR0_EL1), 406 AARCH64(SYS_ID_AA64ISAR1_EL1), 407 AARCH64(SYS_ID_AA64ISAR2_EL1), 408 ID_UNALLOCATED(6,3), 409 ID_UNALLOCATED(6,4), 410 ID_UNALLOCATED(6,5), 411 ID_UNALLOCATED(6,6), 412 ID_UNALLOCATED(6,7), 413 AARCH64(SYS_ID_AA64MMFR0_EL1), 414 AARCH64(SYS_ID_AA64MMFR1_EL1), 415 AARCH64(SYS_ID_AA64MMFR2_EL1), 416 ID_UNALLOCATED(7,3), 417 ID_UNALLOCATED(7,4), 418 ID_UNALLOCATED(7,5), 419 ID_UNALLOCATED(7,6), 420 ID_UNALLOCATED(7,7), 421 422 /* Scalable Vector Registers are restricted. */ 423 424 RAZ_WI(SYS_ERRIDR_EL1), 425 RAZ_WI(SYS_ERRSELR_EL1), 426 RAZ_WI(SYS_ERXFR_EL1), 427 RAZ_WI(SYS_ERXCTLR_EL1), 428 RAZ_WI(SYS_ERXSTATUS_EL1), 429 RAZ_WI(SYS_ERXADDR_EL1), 430 RAZ_WI(SYS_ERXMISC0_EL1), 431 RAZ_WI(SYS_ERXMISC1_EL1), 432 433 /* Performance Monitoring Registers are restricted. */ 434 435 /* Limited Ordering Regions Registers are restricted. */ 436 437 HOST_HANDLED(SYS_ICC_SGI1R_EL1), 438 HOST_HANDLED(SYS_ICC_ASGI1R_EL1), 439 HOST_HANDLED(SYS_ICC_SGI0R_EL1), 440 { SYS_DESC(SYS_ICC_SRE_EL1), .access = pvm_gic_read_sre, }, 441 442 HOST_HANDLED(SYS_CCSIDR_EL1), 443 HOST_HANDLED(SYS_CLIDR_EL1), 444 HOST_HANDLED(SYS_CSSELR_EL1), 445 HOST_HANDLED(SYS_CTR_EL0), 446 447 /* Performance Monitoring Registers are restricted. */ 448 449 /* Activity Monitoring Registers are restricted. */ 450 451 HOST_HANDLED(SYS_CNTP_TVAL_EL0), 452 HOST_HANDLED(SYS_CNTP_CTL_EL0), 453 HOST_HANDLED(SYS_CNTP_CVAL_EL0), 454 455 /* Performance Monitoring Registers are restricted. */ 456 }; 457 458 /* 459 * Checks that the sysreg table is unique and in-order. 460 * 461 * Returns 0 if the table is consistent, or 1 otherwise. 462 */ 463 int kvm_check_pvm_sysreg_table(void) 464 { 465 unsigned int i; 466 467 for (i = 1; i < ARRAY_SIZE(pvm_sys_reg_descs); i++) { 468 if (cmp_sys_reg(&pvm_sys_reg_descs[i-1], &pvm_sys_reg_descs[i]) >= 0) 469 return 1; 470 } 471 472 return 0; 473 } 474 475 /* 476 * Handler for protected VM MSR, MRS or System instruction execution. 477 * 478 * Returns true if the hypervisor has handled the exit, and control should go 479 * back to the guest, or false if it hasn't, to be handled by the host. 480 */ 481 bool kvm_handle_pvm_sysreg(struct kvm_vcpu *vcpu, u64 *exit_code) 482 { 483 const struct sys_reg_desc *r; 484 struct sys_reg_params params; 485 unsigned long esr = kvm_vcpu_get_esr(vcpu); 486 int Rt = kvm_vcpu_sys_get_rt(vcpu); 487 488 params = esr_sys64_to_params(esr); 489 params.regval = vcpu_get_reg(vcpu, Rt); 490 491 r = find_reg(¶ms, pvm_sys_reg_descs, ARRAY_SIZE(pvm_sys_reg_descs)); 492 493 /* Undefined (RESTRICTED). */ 494 if (r == NULL) { 495 inject_undef64(vcpu); 496 return true; 497 } 498 499 /* Handled by the host (HOST_HANDLED) */ 500 if (r->access == NULL) 501 return false; 502 503 /* Handled by hyp: skip instruction if instructed to do so. */ 504 if (r->access(vcpu, ¶ms, r)) 505 __kvm_skip_instr(vcpu); 506 507 if (!params.is_write) 508 vcpu_set_reg(vcpu, Rt, params.regval); 509 510 return true; 511 } 512 513 /* 514 * Handler for protected VM restricted exceptions. 515 * 516 * Inject an undefined exception into the guest and return true to indicate that 517 * the hypervisor has handled the exit, and control should go back to the guest. 518 */ 519 bool kvm_handle_pvm_restricted(struct kvm_vcpu *vcpu, u64 *exit_code) 520 { 521 inject_undef64(vcpu); 522 return true; 523 } 524