1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * AMD Encrypted Register State Support 4 * 5 * Author: Joerg Roedel <jroedel@suse.de> 6 * 7 * This file is not compiled stand-alone. It contains code shared 8 * between the pre-decompression boot code and the running Linux kernel 9 * and is included directly into both code-bases. 10 */ 11 12 #ifndef __BOOT_COMPRESSED 13 #define error(v) pr_err(v) 14 #define has_cpuflag(f) boot_cpu_has(f) 15 #else 16 #undef WARN 17 #define WARN(condition, format...) (!!(condition)) 18 #endif 19 20 /* I/O parameters for CPUID-related helpers */ 21 struct cpuid_leaf { 22 u32 fn; 23 u32 subfn; 24 u32 eax; 25 u32 ebx; 26 u32 ecx; 27 u32 edx; 28 }; 29 30 /* 31 * Individual entries of the SNP CPUID table, as defined by the SNP 32 * Firmware ABI, Revision 0.9, Section 7.1, Table 14. 33 */ 34 struct snp_cpuid_fn { 35 u32 eax_in; 36 u32 ecx_in; 37 u64 xcr0_in; 38 u64 xss_in; 39 u32 eax; 40 u32 ebx; 41 u32 ecx; 42 u32 edx; 43 u64 __reserved; 44 } __packed; 45 46 /* 47 * SNP CPUID table, as defined by the SNP Firmware ABI, Revision 0.9, 48 * Section 8.14.2.6. Also noted there is the SNP firmware-enforced limit 49 * of 64 entries per CPUID table. 50 */ 51 #define SNP_CPUID_COUNT_MAX 64 52 53 struct snp_cpuid_table { 54 u32 count; 55 u32 __reserved1; 56 u64 __reserved2; 57 struct snp_cpuid_fn fn[SNP_CPUID_COUNT_MAX]; 58 } __packed; 59 60 /* 61 * Since feature negotiation related variables are set early in the boot 62 * process they must reside in the .data section so as not to be zeroed 63 * out when the .bss section is later cleared. 64 * 65 * GHCB protocol version negotiated with the hypervisor. 66 */ 67 static u16 ghcb_version __ro_after_init; 68 69 /* Copy of the SNP firmware's CPUID page. */ 70 static struct snp_cpuid_table cpuid_table_copy __ro_after_init; 71 72 /* 73 * These will be initialized based on CPUID table so that non-present 74 * all-zero leaves (for sparse tables) can be differentiated from 75 * invalid/out-of-range leaves. This is needed since all-zero leaves 76 * still need to be post-processed. 77 */ 78 static u32 cpuid_std_range_max __ro_after_init; 79 static u32 cpuid_hyp_range_max __ro_after_init; 80 static u32 cpuid_ext_range_max __ro_after_init; 81 82 static bool __init sev_es_check_cpu_features(void) 83 { 84 if (!has_cpuflag(X86_FEATURE_RDRAND)) { 85 error("RDRAND instruction not supported - no trusted source of randomness available\n"); 86 return false; 87 } 88 89 return true; 90 } 91 92 static void __head __noreturn 93 sev_es_terminate(unsigned int set, unsigned int reason) 94 { 95 u64 val = GHCB_MSR_TERM_REQ; 96 97 /* Tell the hypervisor what went wrong. */ 98 val |= GHCB_SEV_TERM_REASON(set, reason); 99 100 /* Request Guest Termination from Hypvervisor */ 101 sev_es_wr_ghcb_msr(val); 102 VMGEXIT(); 103 104 while (true) 105 asm volatile("hlt\n" : : : "memory"); 106 } 107 108 /* 109 * The hypervisor features are available from GHCB version 2 onward. 110 */ 111 static u64 get_hv_features(void) 112 { 113 u64 val; 114 115 if (ghcb_version < 2) 116 return 0; 117 118 sev_es_wr_ghcb_msr(GHCB_MSR_HV_FT_REQ); 119 VMGEXIT(); 120 121 val = sev_es_rd_ghcb_msr(); 122 if (GHCB_RESP_CODE(val) != GHCB_MSR_HV_FT_RESP) 123 return 0; 124 125 return GHCB_MSR_HV_FT_RESP_VAL(val); 126 } 127 128 static void snp_register_ghcb_early(unsigned long paddr) 129 { 130 unsigned long pfn = paddr >> PAGE_SHIFT; 131 u64 val; 132 133 sev_es_wr_ghcb_msr(GHCB_MSR_REG_GPA_REQ_VAL(pfn)); 134 VMGEXIT(); 135 136 val = sev_es_rd_ghcb_msr(); 137 138 /* If the response GPA is not ours then abort the guest */ 139 if ((GHCB_RESP_CODE(val) != GHCB_MSR_REG_GPA_RESP) || 140 (GHCB_MSR_REG_GPA_RESP_VAL(val) != pfn)) 141 sev_es_terminate(SEV_TERM_SET_LINUX, GHCB_TERM_REGISTER); 142 } 143 144 static bool sev_es_negotiate_protocol(void) 145 { 146 u64 val; 147 148 /* Do the GHCB protocol version negotiation */ 149 sev_es_wr_ghcb_msr(GHCB_MSR_SEV_INFO_REQ); 150 VMGEXIT(); 151 val = sev_es_rd_ghcb_msr(); 152 153 if (GHCB_MSR_INFO(val) != GHCB_MSR_SEV_INFO_RESP) 154 return false; 155 156 if (GHCB_MSR_PROTO_MAX(val) < GHCB_PROTOCOL_MIN || 157 GHCB_MSR_PROTO_MIN(val) > GHCB_PROTOCOL_MAX) 158 return false; 159 160 ghcb_version = min_t(size_t, GHCB_MSR_PROTO_MAX(val), GHCB_PROTOCOL_MAX); 161 162 return true; 163 } 164 165 static __always_inline void vc_ghcb_invalidate(struct ghcb *ghcb) 166 { 167 ghcb->save.sw_exit_code = 0; 168 __builtin_memset(ghcb->save.valid_bitmap, 0, sizeof(ghcb->save.valid_bitmap)); 169 } 170 171 static bool vc_decoding_needed(unsigned long exit_code) 172 { 173 /* Exceptions don't require to decode the instruction */ 174 return !(exit_code >= SVM_EXIT_EXCP_BASE && 175 exit_code <= SVM_EXIT_LAST_EXCP); 176 } 177 178 static enum es_result vc_init_em_ctxt(struct es_em_ctxt *ctxt, 179 struct pt_regs *regs, 180 unsigned long exit_code) 181 { 182 enum es_result ret = ES_OK; 183 184 memset(ctxt, 0, sizeof(*ctxt)); 185 ctxt->regs = regs; 186 187 if (vc_decoding_needed(exit_code)) 188 ret = vc_decode_insn(ctxt); 189 190 return ret; 191 } 192 193 static void vc_finish_insn(struct es_em_ctxt *ctxt) 194 { 195 ctxt->regs->ip += ctxt->insn.length; 196 } 197 198 static enum es_result verify_exception_info(struct ghcb *ghcb, struct es_em_ctxt *ctxt) 199 { 200 u32 ret; 201 202 ret = ghcb->save.sw_exit_info_1 & GENMASK_ULL(31, 0); 203 if (!ret) 204 return ES_OK; 205 206 if (ret == 1) { 207 u64 info = ghcb->save.sw_exit_info_2; 208 unsigned long v = info & SVM_EVTINJ_VEC_MASK; 209 210 /* Check if exception information from hypervisor is sane. */ 211 if ((info & SVM_EVTINJ_VALID) && 212 ((v == X86_TRAP_GP) || (v == X86_TRAP_UD)) && 213 ((info & SVM_EVTINJ_TYPE_MASK) == SVM_EVTINJ_TYPE_EXEPT)) { 214 ctxt->fi.vector = v; 215 216 if (info & SVM_EVTINJ_VALID_ERR) 217 ctxt->fi.error_code = info >> 32; 218 219 return ES_EXCEPTION; 220 } 221 } 222 223 return ES_VMM_ERROR; 224 } 225 226 static enum es_result sev_es_ghcb_hv_call(struct ghcb *ghcb, 227 struct es_em_ctxt *ctxt, 228 u64 exit_code, u64 exit_info_1, 229 u64 exit_info_2) 230 { 231 /* Fill in protocol and format specifiers */ 232 ghcb->protocol_version = ghcb_version; 233 ghcb->ghcb_usage = GHCB_DEFAULT_USAGE; 234 235 ghcb_set_sw_exit_code(ghcb, exit_code); 236 ghcb_set_sw_exit_info_1(ghcb, exit_info_1); 237 ghcb_set_sw_exit_info_2(ghcb, exit_info_2); 238 239 sev_es_wr_ghcb_msr(__pa(ghcb)); 240 VMGEXIT(); 241 242 return verify_exception_info(ghcb, ctxt); 243 } 244 245 static int __sev_cpuid_hv(u32 fn, int reg_idx, u32 *reg) 246 { 247 u64 val; 248 249 sev_es_wr_ghcb_msr(GHCB_CPUID_REQ(fn, reg_idx)); 250 VMGEXIT(); 251 val = sev_es_rd_ghcb_msr(); 252 if (GHCB_RESP_CODE(val) != GHCB_MSR_CPUID_RESP) 253 return -EIO; 254 255 *reg = (val >> 32); 256 257 return 0; 258 } 259 260 static int __sev_cpuid_hv_msr(struct cpuid_leaf *leaf) 261 { 262 int ret; 263 264 /* 265 * MSR protocol does not support fetching non-zero subfunctions, but is 266 * sufficient to handle current early-boot cases. Should that change, 267 * make sure to report an error rather than ignoring the index and 268 * grabbing random values. If this issue arises in the future, handling 269 * can be added here to use GHCB-page protocol for cases that occur late 270 * enough in boot that GHCB page is available. 271 */ 272 if (cpuid_function_is_indexed(leaf->fn) && leaf->subfn) 273 return -EINVAL; 274 275 ret = __sev_cpuid_hv(leaf->fn, GHCB_CPUID_REQ_EAX, &leaf->eax); 276 ret = ret ? : __sev_cpuid_hv(leaf->fn, GHCB_CPUID_REQ_EBX, &leaf->ebx); 277 ret = ret ? : __sev_cpuid_hv(leaf->fn, GHCB_CPUID_REQ_ECX, &leaf->ecx); 278 ret = ret ? : __sev_cpuid_hv(leaf->fn, GHCB_CPUID_REQ_EDX, &leaf->edx); 279 280 return ret; 281 } 282 283 static int __sev_cpuid_hv_ghcb(struct ghcb *ghcb, struct es_em_ctxt *ctxt, struct cpuid_leaf *leaf) 284 { 285 u32 cr4 = native_read_cr4(); 286 int ret; 287 288 ghcb_set_rax(ghcb, leaf->fn); 289 ghcb_set_rcx(ghcb, leaf->subfn); 290 291 if (cr4 & X86_CR4_OSXSAVE) 292 /* Safe to read xcr0 */ 293 ghcb_set_xcr0(ghcb, xgetbv(XCR_XFEATURE_ENABLED_MASK)); 294 else 295 /* xgetbv will cause #UD - use reset value for xcr0 */ 296 ghcb_set_xcr0(ghcb, 1); 297 298 ret = sev_es_ghcb_hv_call(ghcb, ctxt, SVM_EXIT_CPUID, 0, 0); 299 if (ret != ES_OK) 300 return ret; 301 302 if (!(ghcb_rax_is_valid(ghcb) && 303 ghcb_rbx_is_valid(ghcb) && 304 ghcb_rcx_is_valid(ghcb) && 305 ghcb_rdx_is_valid(ghcb))) 306 return ES_VMM_ERROR; 307 308 leaf->eax = ghcb->save.rax; 309 leaf->ebx = ghcb->save.rbx; 310 leaf->ecx = ghcb->save.rcx; 311 leaf->edx = ghcb->save.rdx; 312 313 return ES_OK; 314 } 315 316 static int sev_cpuid_hv(struct ghcb *ghcb, struct es_em_ctxt *ctxt, struct cpuid_leaf *leaf) 317 { 318 return ghcb ? __sev_cpuid_hv_ghcb(ghcb, ctxt, leaf) 319 : __sev_cpuid_hv_msr(leaf); 320 } 321 322 /* 323 * This may be called early while still running on the initial identity 324 * mapping. Use RIP-relative addressing to obtain the correct address 325 * while running with the initial identity mapping as well as the 326 * switch-over to kernel virtual addresses later. 327 */ 328 static const struct snp_cpuid_table *snp_cpuid_get_table(void) 329 { 330 return &RIP_REL_REF(cpuid_table_copy); 331 } 332 333 /* 334 * The SNP Firmware ABI, Revision 0.9, Section 7.1, details the use of 335 * XCR0_IN and XSS_IN to encode multiple versions of 0xD subfunctions 0 336 * and 1 based on the corresponding features enabled by a particular 337 * combination of XCR0 and XSS registers so that a guest can look up the 338 * version corresponding to the features currently enabled in its XCR0/XSS 339 * registers. The only values that differ between these versions/table 340 * entries is the enabled XSAVE area size advertised via EBX. 341 * 342 * While hypervisors may choose to make use of this support, it is more 343 * robust/secure for a guest to simply find the entry corresponding to the 344 * base/legacy XSAVE area size (XCR0=1 or XCR0=3), and then calculate the 345 * XSAVE area size using subfunctions 2 through 64, as documented in APM 346 * Volume 3, Rev 3.31, Appendix E.3.8, which is what is done here. 347 * 348 * Since base/legacy XSAVE area size is documented as 0x240, use that value 349 * directly rather than relying on the base size in the CPUID table. 350 * 351 * Return: XSAVE area size on success, 0 otherwise. 352 */ 353 static u32 snp_cpuid_calc_xsave_size(u64 xfeatures_en, bool compacted) 354 { 355 const struct snp_cpuid_table *cpuid_table = snp_cpuid_get_table(); 356 u64 xfeatures_found = 0; 357 u32 xsave_size = 0x240; 358 int i; 359 360 for (i = 0; i < cpuid_table->count; i++) { 361 const struct snp_cpuid_fn *e = &cpuid_table->fn[i]; 362 363 if (!(e->eax_in == 0xD && e->ecx_in > 1 && e->ecx_in < 64)) 364 continue; 365 if (!(xfeatures_en & (BIT_ULL(e->ecx_in)))) 366 continue; 367 if (xfeatures_found & (BIT_ULL(e->ecx_in))) 368 continue; 369 370 xfeatures_found |= (BIT_ULL(e->ecx_in)); 371 372 if (compacted) 373 xsave_size += e->eax; 374 else 375 xsave_size = max(xsave_size, e->eax + e->ebx); 376 } 377 378 /* 379 * Either the guest set unsupported XCR0/XSS bits, or the corresponding 380 * entries in the CPUID table were not present. This is not a valid 381 * state to be in. 382 */ 383 if (xfeatures_found != (xfeatures_en & GENMASK_ULL(63, 2))) 384 return 0; 385 386 return xsave_size; 387 } 388 389 static bool __head 390 snp_cpuid_get_validated_func(struct cpuid_leaf *leaf) 391 { 392 const struct snp_cpuid_table *cpuid_table = snp_cpuid_get_table(); 393 int i; 394 395 for (i = 0; i < cpuid_table->count; i++) { 396 const struct snp_cpuid_fn *e = &cpuid_table->fn[i]; 397 398 if (e->eax_in != leaf->fn) 399 continue; 400 401 if (cpuid_function_is_indexed(leaf->fn) && e->ecx_in != leaf->subfn) 402 continue; 403 404 /* 405 * For 0xD subfunctions 0 and 1, only use the entry corresponding 406 * to the base/legacy XSAVE area size (XCR0=1 or XCR0=3, XSS=0). 407 * See the comments above snp_cpuid_calc_xsave_size() for more 408 * details. 409 */ 410 if (e->eax_in == 0xD && (e->ecx_in == 0 || e->ecx_in == 1)) 411 if (!(e->xcr0_in == 1 || e->xcr0_in == 3) || e->xss_in) 412 continue; 413 414 leaf->eax = e->eax; 415 leaf->ebx = e->ebx; 416 leaf->ecx = e->ecx; 417 leaf->edx = e->edx; 418 419 return true; 420 } 421 422 return false; 423 } 424 425 static void snp_cpuid_hv(struct ghcb *ghcb, struct es_em_ctxt *ctxt, struct cpuid_leaf *leaf) 426 { 427 if (sev_cpuid_hv(ghcb, ctxt, leaf)) 428 sev_es_terminate(SEV_TERM_SET_LINUX, GHCB_TERM_CPUID_HV); 429 } 430 431 static int snp_cpuid_postprocess(struct ghcb *ghcb, struct es_em_ctxt *ctxt, 432 struct cpuid_leaf *leaf) 433 { 434 struct cpuid_leaf leaf_hv = *leaf; 435 436 switch (leaf->fn) { 437 case 0x1: 438 snp_cpuid_hv(ghcb, ctxt, &leaf_hv); 439 440 /* initial APIC ID */ 441 leaf->ebx = (leaf_hv.ebx & GENMASK(31, 24)) | (leaf->ebx & GENMASK(23, 0)); 442 /* APIC enabled bit */ 443 leaf->edx = (leaf_hv.edx & BIT(9)) | (leaf->edx & ~BIT(9)); 444 445 /* OSXSAVE enabled bit */ 446 if (native_read_cr4() & X86_CR4_OSXSAVE) 447 leaf->ecx |= BIT(27); 448 break; 449 case 0x7: 450 /* OSPKE enabled bit */ 451 leaf->ecx &= ~BIT(4); 452 if (native_read_cr4() & X86_CR4_PKE) 453 leaf->ecx |= BIT(4); 454 break; 455 case 0xB: 456 leaf_hv.subfn = 0; 457 snp_cpuid_hv(ghcb, ctxt, &leaf_hv); 458 459 /* extended APIC ID */ 460 leaf->edx = leaf_hv.edx; 461 break; 462 case 0xD: { 463 bool compacted = false; 464 u64 xcr0 = 1, xss = 0; 465 u32 xsave_size; 466 467 if (leaf->subfn != 0 && leaf->subfn != 1) 468 return 0; 469 470 if (native_read_cr4() & X86_CR4_OSXSAVE) 471 xcr0 = xgetbv(XCR_XFEATURE_ENABLED_MASK); 472 if (leaf->subfn == 1) { 473 /* Get XSS value if XSAVES is enabled. */ 474 if (leaf->eax & BIT(3)) { 475 unsigned long lo, hi; 476 477 asm volatile("rdmsr" : "=a" (lo), "=d" (hi) 478 : "c" (MSR_IA32_XSS)); 479 xss = (hi << 32) | lo; 480 } 481 482 /* 483 * The PPR and APM aren't clear on what size should be 484 * encoded in 0xD:0x1:EBX when compaction is not enabled 485 * by either XSAVEC (feature bit 1) or XSAVES (feature 486 * bit 3) since SNP-capable hardware has these feature 487 * bits fixed as 1. KVM sets it to 0 in this case, but 488 * to avoid this becoming an issue it's safer to simply 489 * treat this as unsupported for SNP guests. 490 */ 491 if (!(leaf->eax & (BIT(1) | BIT(3)))) 492 return -EINVAL; 493 494 compacted = true; 495 } 496 497 xsave_size = snp_cpuid_calc_xsave_size(xcr0 | xss, compacted); 498 if (!xsave_size) 499 return -EINVAL; 500 501 leaf->ebx = xsave_size; 502 } 503 break; 504 case 0x8000001E: 505 snp_cpuid_hv(ghcb, ctxt, &leaf_hv); 506 507 /* extended APIC ID */ 508 leaf->eax = leaf_hv.eax; 509 /* compute ID */ 510 leaf->ebx = (leaf->ebx & GENMASK(31, 8)) | (leaf_hv.ebx & GENMASK(7, 0)); 511 /* node ID */ 512 leaf->ecx = (leaf->ecx & GENMASK(31, 8)) | (leaf_hv.ecx & GENMASK(7, 0)); 513 break; 514 default: 515 /* No fix-ups needed, use values as-is. */ 516 break; 517 } 518 519 return 0; 520 } 521 522 /* 523 * Returns -EOPNOTSUPP if feature not enabled. Any other non-zero return value 524 * should be treated as fatal by caller. 525 */ 526 static int __head 527 snp_cpuid(struct ghcb *ghcb, struct es_em_ctxt *ctxt, struct cpuid_leaf *leaf) 528 { 529 const struct snp_cpuid_table *cpuid_table = snp_cpuid_get_table(); 530 531 if (!cpuid_table->count) 532 return -EOPNOTSUPP; 533 534 if (!snp_cpuid_get_validated_func(leaf)) { 535 /* 536 * Some hypervisors will avoid keeping track of CPUID entries 537 * where all values are zero, since they can be handled the 538 * same as out-of-range values (all-zero). This is useful here 539 * as well as it allows virtually all guest configurations to 540 * work using a single SNP CPUID table. 541 * 542 * To allow for this, there is a need to distinguish between 543 * out-of-range entries and in-range zero entries, since the 544 * CPUID table entries are only a template that may need to be 545 * augmented with additional values for things like 546 * CPU-specific information during post-processing. So if it's 547 * not in the table, set the values to zero. Then, if they are 548 * within a valid CPUID range, proceed with post-processing 549 * using zeros as the initial values. Otherwise, skip 550 * post-processing and just return zeros immediately. 551 */ 552 leaf->eax = leaf->ebx = leaf->ecx = leaf->edx = 0; 553 554 /* Skip post-processing for out-of-range zero leafs. */ 555 if (!(leaf->fn <= RIP_REL_REF(cpuid_std_range_max) || 556 (leaf->fn >= 0x40000000 && leaf->fn <= RIP_REL_REF(cpuid_hyp_range_max)) || 557 (leaf->fn >= 0x80000000 && leaf->fn <= RIP_REL_REF(cpuid_ext_range_max)))) 558 return 0; 559 } 560 561 return snp_cpuid_postprocess(ghcb, ctxt, leaf); 562 } 563 564 /* 565 * Boot VC Handler - This is the first VC handler during boot, there is no GHCB 566 * page yet, so it only supports the MSR based communication with the 567 * hypervisor and only the CPUID exit-code. 568 */ 569 void __head do_vc_no_ghcb(struct pt_regs *regs, unsigned long exit_code) 570 { 571 unsigned int subfn = lower_bits(regs->cx, 32); 572 unsigned int fn = lower_bits(regs->ax, 32); 573 struct cpuid_leaf leaf; 574 int ret; 575 576 /* Only CPUID is supported via MSR protocol */ 577 if (exit_code != SVM_EXIT_CPUID) 578 goto fail; 579 580 leaf.fn = fn; 581 leaf.subfn = subfn; 582 583 ret = snp_cpuid(NULL, NULL, &leaf); 584 if (!ret) 585 goto cpuid_done; 586 587 if (ret != -EOPNOTSUPP) 588 goto fail; 589 590 if (__sev_cpuid_hv_msr(&leaf)) 591 goto fail; 592 593 cpuid_done: 594 regs->ax = leaf.eax; 595 regs->bx = leaf.ebx; 596 regs->cx = leaf.ecx; 597 regs->dx = leaf.edx; 598 599 /* 600 * This is a VC handler and the #VC is only raised when SEV-ES is 601 * active, which means SEV must be active too. Do sanity checks on the 602 * CPUID results to make sure the hypervisor does not trick the kernel 603 * into the no-sev path. This could map sensitive data unencrypted and 604 * make it accessible to the hypervisor. 605 * 606 * In particular, check for: 607 * - Availability of CPUID leaf 0x8000001f 608 * - SEV CPUID bit. 609 * 610 * The hypervisor might still report the wrong C-bit position, but this 611 * can't be checked here. 612 */ 613 614 if (fn == 0x80000000 && (regs->ax < 0x8000001f)) 615 /* SEV leaf check */ 616 goto fail; 617 else if ((fn == 0x8000001f && !(regs->ax & BIT(1)))) 618 /* SEV bit */ 619 goto fail; 620 621 /* Skip over the CPUID two-byte opcode */ 622 regs->ip += 2; 623 624 return; 625 626 fail: 627 /* Terminate the guest */ 628 sev_es_terminate(SEV_TERM_SET_GEN, GHCB_SEV_ES_GEN_REQ); 629 } 630 631 static enum es_result vc_insn_string_check(struct es_em_ctxt *ctxt, 632 unsigned long address, 633 bool write) 634 { 635 if (user_mode(ctxt->regs) && fault_in_kernel_space(address)) { 636 ctxt->fi.vector = X86_TRAP_PF; 637 ctxt->fi.error_code = X86_PF_USER; 638 ctxt->fi.cr2 = address; 639 if (write) 640 ctxt->fi.error_code |= X86_PF_WRITE; 641 642 return ES_EXCEPTION; 643 } 644 645 return ES_OK; 646 } 647 648 static enum es_result vc_insn_string_read(struct es_em_ctxt *ctxt, 649 void *src, char *buf, 650 unsigned int data_size, 651 unsigned int count, 652 bool backwards) 653 { 654 int i, b = backwards ? -1 : 1; 655 unsigned long address = (unsigned long)src; 656 enum es_result ret; 657 658 ret = vc_insn_string_check(ctxt, address, false); 659 if (ret != ES_OK) 660 return ret; 661 662 for (i = 0; i < count; i++) { 663 void *s = src + (i * data_size * b); 664 char *d = buf + (i * data_size); 665 666 ret = vc_read_mem(ctxt, s, d, data_size); 667 if (ret != ES_OK) 668 break; 669 } 670 671 return ret; 672 } 673 674 static enum es_result vc_insn_string_write(struct es_em_ctxt *ctxt, 675 void *dst, char *buf, 676 unsigned int data_size, 677 unsigned int count, 678 bool backwards) 679 { 680 int i, s = backwards ? -1 : 1; 681 unsigned long address = (unsigned long)dst; 682 enum es_result ret; 683 684 ret = vc_insn_string_check(ctxt, address, true); 685 if (ret != ES_OK) 686 return ret; 687 688 for (i = 0; i < count; i++) { 689 void *d = dst + (i * data_size * s); 690 char *b = buf + (i * data_size); 691 692 ret = vc_write_mem(ctxt, d, b, data_size); 693 if (ret != ES_OK) 694 break; 695 } 696 697 return ret; 698 } 699 700 #define IOIO_TYPE_STR BIT(2) 701 #define IOIO_TYPE_IN 1 702 #define IOIO_TYPE_INS (IOIO_TYPE_IN | IOIO_TYPE_STR) 703 #define IOIO_TYPE_OUT 0 704 #define IOIO_TYPE_OUTS (IOIO_TYPE_OUT | IOIO_TYPE_STR) 705 706 #define IOIO_REP BIT(3) 707 708 #define IOIO_ADDR_64 BIT(9) 709 #define IOIO_ADDR_32 BIT(8) 710 #define IOIO_ADDR_16 BIT(7) 711 712 #define IOIO_DATA_32 BIT(6) 713 #define IOIO_DATA_16 BIT(5) 714 #define IOIO_DATA_8 BIT(4) 715 716 #define IOIO_SEG_ES (0 << 10) 717 #define IOIO_SEG_DS (3 << 10) 718 719 static enum es_result vc_ioio_exitinfo(struct es_em_ctxt *ctxt, u64 *exitinfo) 720 { 721 struct insn *insn = &ctxt->insn; 722 size_t size; 723 u64 port; 724 725 *exitinfo = 0; 726 727 switch (insn->opcode.bytes[0]) { 728 /* INS opcodes */ 729 case 0x6c: 730 case 0x6d: 731 *exitinfo |= IOIO_TYPE_INS; 732 *exitinfo |= IOIO_SEG_ES; 733 port = ctxt->regs->dx & 0xffff; 734 break; 735 736 /* OUTS opcodes */ 737 case 0x6e: 738 case 0x6f: 739 *exitinfo |= IOIO_TYPE_OUTS; 740 *exitinfo |= IOIO_SEG_DS; 741 port = ctxt->regs->dx & 0xffff; 742 break; 743 744 /* IN immediate opcodes */ 745 case 0xe4: 746 case 0xe5: 747 *exitinfo |= IOIO_TYPE_IN; 748 port = (u8)insn->immediate.value & 0xffff; 749 break; 750 751 /* OUT immediate opcodes */ 752 case 0xe6: 753 case 0xe7: 754 *exitinfo |= IOIO_TYPE_OUT; 755 port = (u8)insn->immediate.value & 0xffff; 756 break; 757 758 /* IN register opcodes */ 759 case 0xec: 760 case 0xed: 761 *exitinfo |= IOIO_TYPE_IN; 762 port = ctxt->regs->dx & 0xffff; 763 break; 764 765 /* OUT register opcodes */ 766 case 0xee: 767 case 0xef: 768 *exitinfo |= IOIO_TYPE_OUT; 769 port = ctxt->regs->dx & 0xffff; 770 break; 771 772 default: 773 return ES_DECODE_FAILED; 774 } 775 776 *exitinfo |= port << 16; 777 778 switch (insn->opcode.bytes[0]) { 779 case 0x6c: 780 case 0x6e: 781 case 0xe4: 782 case 0xe6: 783 case 0xec: 784 case 0xee: 785 /* Single byte opcodes */ 786 *exitinfo |= IOIO_DATA_8; 787 size = 1; 788 break; 789 default: 790 /* Length determined by instruction parsing */ 791 *exitinfo |= (insn->opnd_bytes == 2) ? IOIO_DATA_16 792 : IOIO_DATA_32; 793 size = (insn->opnd_bytes == 2) ? 2 : 4; 794 } 795 796 switch (insn->addr_bytes) { 797 case 2: 798 *exitinfo |= IOIO_ADDR_16; 799 break; 800 case 4: 801 *exitinfo |= IOIO_ADDR_32; 802 break; 803 case 8: 804 *exitinfo |= IOIO_ADDR_64; 805 break; 806 } 807 808 if (insn_has_rep_prefix(insn)) 809 *exitinfo |= IOIO_REP; 810 811 return vc_ioio_check(ctxt, (u16)port, size); 812 } 813 814 static enum es_result vc_handle_ioio(struct ghcb *ghcb, struct es_em_ctxt *ctxt) 815 { 816 struct pt_regs *regs = ctxt->regs; 817 u64 exit_info_1, exit_info_2; 818 enum es_result ret; 819 820 ret = vc_ioio_exitinfo(ctxt, &exit_info_1); 821 if (ret != ES_OK) 822 return ret; 823 824 if (exit_info_1 & IOIO_TYPE_STR) { 825 826 /* (REP) INS/OUTS */ 827 828 bool df = ((regs->flags & X86_EFLAGS_DF) == X86_EFLAGS_DF); 829 unsigned int io_bytes, exit_bytes; 830 unsigned int ghcb_count, op_count; 831 unsigned long es_base; 832 u64 sw_scratch; 833 834 /* 835 * For the string variants with rep prefix the amount of in/out 836 * operations per #VC exception is limited so that the kernel 837 * has a chance to take interrupts and re-schedule while the 838 * instruction is emulated. 839 */ 840 io_bytes = (exit_info_1 >> 4) & 0x7; 841 ghcb_count = sizeof(ghcb->shared_buffer) / io_bytes; 842 843 op_count = (exit_info_1 & IOIO_REP) ? regs->cx : 1; 844 exit_info_2 = min(op_count, ghcb_count); 845 exit_bytes = exit_info_2 * io_bytes; 846 847 es_base = insn_get_seg_base(ctxt->regs, INAT_SEG_REG_ES); 848 849 /* Read bytes of OUTS into the shared buffer */ 850 if (!(exit_info_1 & IOIO_TYPE_IN)) { 851 ret = vc_insn_string_read(ctxt, 852 (void *)(es_base + regs->si), 853 ghcb->shared_buffer, io_bytes, 854 exit_info_2, df); 855 if (ret) 856 return ret; 857 } 858 859 /* 860 * Issue an VMGEXIT to the HV to consume the bytes from the 861 * shared buffer or to have it write them into the shared buffer 862 * depending on the instruction: OUTS or INS. 863 */ 864 sw_scratch = __pa(ghcb) + offsetof(struct ghcb, shared_buffer); 865 ghcb_set_sw_scratch(ghcb, sw_scratch); 866 ret = sev_es_ghcb_hv_call(ghcb, ctxt, SVM_EXIT_IOIO, 867 exit_info_1, exit_info_2); 868 if (ret != ES_OK) 869 return ret; 870 871 /* Read bytes from shared buffer into the guest's destination. */ 872 if (exit_info_1 & IOIO_TYPE_IN) { 873 ret = vc_insn_string_write(ctxt, 874 (void *)(es_base + regs->di), 875 ghcb->shared_buffer, io_bytes, 876 exit_info_2, df); 877 if (ret) 878 return ret; 879 880 if (df) 881 regs->di -= exit_bytes; 882 else 883 regs->di += exit_bytes; 884 } else { 885 if (df) 886 regs->si -= exit_bytes; 887 else 888 regs->si += exit_bytes; 889 } 890 891 if (exit_info_1 & IOIO_REP) 892 regs->cx -= exit_info_2; 893 894 ret = regs->cx ? ES_RETRY : ES_OK; 895 896 } else { 897 898 /* IN/OUT into/from rAX */ 899 900 int bits = (exit_info_1 & 0x70) >> 1; 901 u64 rax = 0; 902 903 if (!(exit_info_1 & IOIO_TYPE_IN)) 904 rax = lower_bits(regs->ax, bits); 905 906 ghcb_set_rax(ghcb, rax); 907 908 ret = sev_es_ghcb_hv_call(ghcb, ctxt, SVM_EXIT_IOIO, exit_info_1, 0); 909 if (ret != ES_OK) 910 return ret; 911 912 if (exit_info_1 & IOIO_TYPE_IN) { 913 if (!ghcb_rax_is_valid(ghcb)) 914 return ES_VMM_ERROR; 915 regs->ax = lower_bits(ghcb->save.rax, bits); 916 } 917 } 918 919 return ret; 920 } 921 922 static int vc_handle_cpuid_snp(struct ghcb *ghcb, struct es_em_ctxt *ctxt) 923 { 924 struct pt_regs *regs = ctxt->regs; 925 struct cpuid_leaf leaf; 926 int ret; 927 928 leaf.fn = regs->ax; 929 leaf.subfn = regs->cx; 930 ret = snp_cpuid(ghcb, ctxt, &leaf); 931 if (!ret) { 932 regs->ax = leaf.eax; 933 regs->bx = leaf.ebx; 934 regs->cx = leaf.ecx; 935 regs->dx = leaf.edx; 936 } 937 938 return ret; 939 } 940 941 static enum es_result vc_handle_cpuid(struct ghcb *ghcb, 942 struct es_em_ctxt *ctxt) 943 { 944 struct pt_regs *regs = ctxt->regs; 945 u32 cr4 = native_read_cr4(); 946 enum es_result ret; 947 int snp_cpuid_ret; 948 949 snp_cpuid_ret = vc_handle_cpuid_snp(ghcb, ctxt); 950 if (!snp_cpuid_ret) 951 return ES_OK; 952 if (snp_cpuid_ret != -EOPNOTSUPP) 953 return ES_VMM_ERROR; 954 955 ghcb_set_rax(ghcb, regs->ax); 956 ghcb_set_rcx(ghcb, regs->cx); 957 958 if (cr4 & X86_CR4_OSXSAVE) 959 /* Safe to read xcr0 */ 960 ghcb_set_xcr0(ghcb, xgetbv(XCR_XFEATURE_ENABLED_MASK)); 961 else 962 /* xgetbv will cause #GP - use reset value for xcr0 */ 963 ghcb_set_xcr0(ghcb, 1); 964 965 ret = sev_es_ghcb_hv_call(ghcb, ctxt, SVM_EXIT_CPUID, 0, 0); 966 if (ret != ES_OK) 967 return ret; 968 969 if (!(ghcb_rax_is_valid(ghcb) && 970 ghcb_rbx_is_valid(ghcb) && 971 ghcb_rcx_is_valid(ghcb) && 972 ghcb_rdx_is_valid(ghcb))) 973 return ES_VMM_ERROR; 974 975 regs->ax = ghcb->save.rax; 976 regs->bx = ghcb->save.rbx; 977 regs->cx = ghcb->save.rcx; 978 regs->dx = ghcb->save.rdx; 979 980 return ES_OK; 981 } 982 983 static enum es_result vc_handle_rdtsc(struct ghcb *ghcb, 984 struct es_em_ctxt *ctxt, 985 unsigned long exit_code) 986 { 987 bool rdtscp = (exit_code == SVM_EXIT_RDTSCP); 988 enum es_result ret; 989 990 ret = sev_es_ghcb_hv_call(ghcb, ctxt, exit_code, 0, 0); 991 if (ret != ES_OK) 992 return ret; 993 994 if (!(ghcb_rax_is_valid(ghcb) && ghcb_rdx_is_valid(ghcb) && 995 (!rdtscp || ghcb_rcx_is_valid(ghcb)))) 996 return ES_VMM_ERROR; 997 998 ctxt->regs->ax = ghcb->save.rax; 999 ctxt->regs->dx = ghcb->save.rdx; 1000 if (rdtscp) 1001 ctxt->regs->cx = ghcb->save.rcx; 1002 1003 return ES_OK; 1004 } 1005 1006 struct cc_setup_data { 1007 struct setup_data header; 1008 u32 cc_blob_address; 1009 }; 1010 1011 /* 1012 * Search for a Confidential Computing blob passed in as a setup_data entry 1013 * via the Linux Boot Protocol. 1014 */ 1015 static __head 1016 struct cc_blob_sev_info *find_cc_blob_setup_data(struct boot_params *bp) 1017 { 1018 struct cc_setup_data *sd = NULL; 1019 struct setup_data *hdr; 1020 1021 hdr = (struct setup_data *)bp->hdr.setup_data; 1022 1023 while (hdr) { 1024 if (hdr->type == SETUP_CC_BLOB) { 1025 sd = (struct cc_setup_data *)hdr; 1026 return (struct cc_blob_sev_info *)(unsigned long)sd->cc_blob_address; 1027 } 1028 hdr = (struct setup_data *)hdr->next; 1029 } 1030 1031 return NULL; 1032 } 1033 1034 /* 1035 * Initialize the kernel's copy of the SNP CPUID table, and set up the 1036 * pointer that will be used to access it. 1037 * 1038 * Maintaining a direct mapping of the SNP CPUID table used by firmware would 1039 * be possible as an alternative, but the approach is brittle since the 1040 * mapping needs to be updated in sync with all the changes to virtual memory 1041 * layout and related mapping facilities throughout the boot process. 1042 */ 1043 static void __head setup_cpuid_table(const struct cc_blob_sev_info *cc_info) 1044 { 1045 const struct snp_cpuid_table *cpuid_table_fw, *cpuid_table; 1046 int i; 1047 1048 if (!cc_info || !cc_info->cpuid_phys || cc_info->cpuid_len < PAGE_SIZE) 1049 sev_es_terminate(SEV_TERM_SET_LINUX, GHCB_TERM_CPUID); 1050 1051 cpuid_table_fw = (const struct snp_cpuid_table *)cc_info->cpuid_phys; 1052 if (!cpuid_table_fw->count || cpuid_table_fw->count > SNP_CPUID_COUNT_MAX) 1053 sev_es_terminate(SEV_TERM_SET_LINUX, GHCB_TERM_CPUID); 1054 1055 cpuid_table = snp_cpuid_get_table(); 1056 memcpy((void *)cpuid_table, cpuid_table_fw, sizeof(*cpuid_table)); 1057 1058 /* Initialize CPUID ranges for range-checking. */ 1059 for (i = 0; i < cpuid_table->count; i++) { 1060 const struct snp_cpuid_fn *fn = &cpuid_table->fn[i]; 1061 1062 if (fn->eax_in == 0x0) 1063 RIP_REL_REF(cpuid_std_range_max) = fn->eax; 1064 else if (fn->eax_in == 0x40000000) 1065 RIP_REL_REF(cpuid_hyp_range_max) = fn->eax; 1066 else if (fn->eax_in == 0x80000000) 1067 RIP_REL_REF(cpuid_ext_range_max) = fn->eax; 1068 } 1069 } 1070 1071 static void pvalidate_pages(struct snp_psc_desc *desc) 1072 { 1073 struct psc_entry *e; 1074 unsigned long vaddr; 1075 unsigned int size; 1076 unsigned int i; 1077 bool validate; 1078 int rc; 1079 1080 for (i = 0; i <= desc->hdr.end_entry; i++) { 1081 e = &desc->entries[i]; 1082 1083 vaddr = (unsigned long)pfn_to_kaddr(e->gfn); 1084 size = e->pagesize ? RMP_PG_SIZE_2M : RMP_PG_SIZE_4K; 1085 validate = e->operation == SNP_PAGE_STATE_PRIVATE; 1086 1087 rc = pvalidate(vaddr, size, validate); 1088 if (rc == PVALIDATE_FAIL_SIZEMISMATCH && size == RMP_PG_SIZE_2M) { 1089 unsigned long vaddr_end = vaddr + PMD_SIZE; 1090 1091 for (; vaddr < vaddr_end; vaddr += PAGE_SIZE) { 1092 rc = pvalidate(vaddr, RMP_PG_SIZE_4K, validate); 1093 if (rc) 1094 break; 1095 } 1096 } 1097 1098 if (rc) { 1099 WARN(1, "Failed to validate address 0x%lx ret %d", vaddr, rc); 1100 sev_es_terminate(SEV_TERM_SET_LINUX, GHCB_TERM_PVALIDATE); 1101 } 1102 } 1103 } 1104 1105 static int vmgexit_psc(struct ghcb *ghcb, struct snp_psc_desc *desc) 1106 { 1107 int cur_entry, end_entry, ret = 0; 1108 struct snp_psc_desc *data; 1109 struct es_em_ctxt ctxt; 1110 1111 vc_ghcb_invalidate(ghcb); 1112 1113 /* Copy the input desc into GHCB shared buffer */ 1114 data = (struct snp_psc_desc *)ghcb->shared_buffer; 1115 memcpy(ghcb->shared_buffer, desc, min_t(int, GHCB_SHARED_BUF_SIZE, sizeof(*desc))); 1116 1117 /* 1118 * As per the GHCB specification, the hypervisor can resume the guest 1119 * before processing all the entries. Check whether all the entries 1120 * are processed. If not, then keep retrying. Note, the hypervisor 1121 * will update the data memory directly to indicate the status, so 1122 * reference the data->hdr everywhere. 1123 * 1124 * The strategy here is to wait for the hypervisor to change the page 1125 * state in the RMP table before guest accesses the memory pages. If the 1126 * page state change was not successful, then later memory access will 1127 * result in a crash. 1128 */ 1129 cur_entry = data->hdr.cur_entry; 1130 end_entry = data->hdr.end_entry; 1131 1132 while (data->hdr.cur_entry <= data->hdr.end_entry) { 1133 ghcb_set_sw_scratch(ghcb, (u64)__pa(data)); 1134 1135 /* This will advance the shared buffer data points to. */ 1136 ret = sev_es_ghcb_hv_call(ghcb, &ctxt, SVM_VMGEXIT_PSC, 0, 0); 1137 1138 /* 1139 * Page State Change VMGEXIT can pass error code through 1140 * exit_info_2. 1141 */ 1142 if (WARN(ret || ghcb->save.sw_exit_info_2, 1143 "SNP: PSC failed ret=%d exit_info_2=%llx\n", 1144 ret, ghcb->save.sw_exit_info_2)) { 1145 ret = 1; 1146 goto out; 1147 } 1148 1149 /* Verify that reserved bit is not set */ 1150 if (WARN(data->hdr.reserved, "Reserved bit is set in the PSC header\n")) { 1151 ret = 1; 1152 goto out; 1153 } 1154 1155 /* 1156 * Sanity check that entry processing is not going backwards. 1157 * This will happen only if hypervisor is tricking us. 1158 */ 1159 if (WARN(data->hdr.end_entry > end_entry || cur_entry > data->hdr.cur_entry, 1160 "SNP: PSC processing going backward, end_entry %d (got %d) cur_entry %d (got %d)\n", 1161 end_entry, data->hdr.end_entry, cur_entry, data->hdr.cur_entry)) { 1162 ret = 1; 1163 goto out; 1164 } 1165 } 1166 1167 out: 1168 return ret; 1169 } 1170