1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * AMD Encrypted Register State Support 4 * 5 * Author: Joerg Roedel <jroedel@suse.de> 6 * 7 * This file is not compiled stand-alone. It contains code shared 8 * between the pre-decompression boot code and the running Linux kernel 9 * and is included directly into both code-bases. 10 */ 11 12 #ifndef __BOOT_COMPRESSED 13 #define error(v) pr_err(v) 14 #define has_cpuflag(f) boot_cpu_has(f) 15 #endif 16 17 /* I/O parameters for CPUID-related helpers */ 18 struct cpuid_leaf { 19 u32 fn; 20 u32 subfn; 21 u32 eax; 22 u32 ebx; 23 u32 ecx; 24 u32 edx; 25 }; 26 27 /* 28 * Individual entries of the SNP CPUID table, as defined by the SNP 29 * Firmware ABI, Revision 0.9, Section 7.1, Table 14. 30 */ 31 struct snp_cpuid_fn { 32 u32 eax_in; 33 u32 ecx_in; 34 u64 xcr0_in; 35 u64 xss_in; 36 u32 eax; 37 u32 ebx; 38 u32 ecx; 39 u32 edx; 40 u64 __reserved; 41 } __packed; 42 43 /* 44 * SNP CPUID table, as defined by the SNP Firmware ABI, Revision 0.9, 45 * Section 8.14.2.6. Also noted there is the SNP firmware-enforced limit 46 * of 64 entries per CPUID table. 47 */ 48 #define SNP_CPUID_COUNT_MAX 64 49 50 struct snp_cpuid_table { 51 u32 count; 52 u32 __reserved1; 53 u64 __reserved2; 54 struct snp_cpuid_fn fn[SNP_CPUID_COUNT_MAX]; 55 } __packed; 56 57 /* 58 * Since feature negotiation related variables are set early in the boot 59 * process they must reside in the .data section so as not to be zeroed 60 * out when the .bss section is later cleared. 61 * 62 * GHCB protocol version negotiated with the hypervisor. 63 */ 64 static u16 ghcb_version __ro_after_init; 65 66 /* Copy of the SNP firmware's CPUID page. */ 67 static struct snp_cpuid_table cpuid_table_copy __ro_after_init; 68 69 /* 70 * These will be initialized based on CPUID table so that non-present 71 * all-zero leaves (for sparse tables) can be differentiated from 72 * invalid/out-of-range leaves. This is needed since all-zero leaves 73 * still need to be post-processed. 74 */ 75 static u32 cpuid_std_range_max __ro_after_init; 76 static u32 cpuid_hyp_range_max __ro_after_init; 77 static u32 cpuid_ext_range_max __ro_after_init; 78 79 static bool __init sev_es_check_cpu_features(void) 80 { 81 if (!has_cpuflag(X86_FEATURE_RDRAND)) { 82 error("RDRAND instruction not supported - no trusted source of randomness available\n"); 83 return false; 84 } 85 86 return true; 87 } 88 89 static void __noreturn sev_es_terminate(unsigned int set, unsigned int reason) 90 { 91 u64 val = GHCB_MSR_TERM_REQ; 92 93 /* Tell the hypervisor what went wrong. */ 94 val |= GHCB_SEV_TERM_REASON(set, reason); 95 96 /* Request Guest Termination from Hypvervisor */ 97 sev_es_wr_ghcb_msr(val); 98 VMGEXIT(); 99 100 while (true) 101 asm volatile("hlt\n" : : : "memory"); 102 } 103 104 /* 105 * The hypervisor features are available from GHCB version 2 onward. 106 */ 107 static u64 get_hv_features(void) 108 { 109 u64 val; 110 111 if (ghcb_version < 2) 112 return 0; 113 114 sev_es_wr_ghcb_msr(GHCB_MSR_HV_FT_REQ); 115 VMGEXIT(); 116 117 val = sev_es_rd_ghcb_msr(); 118 if (GHCB_RESP_CODE(val) != GHCB_MSR_HV_FT_RESP) 119 return 0; 120 121 return GHCB_MSR_HV_FT_RESP_VAL(val); 122 } 123 124 static void snp_register_ghcb_early(unsigned long paddr) 125 { 126 unsigned long pfn = paddr >> PAGE_SHIFT; 127 u64 val; 128 129 sev_es_wr_ghcb_msr(GHCB_MSR_REG_GPA_REQ_VAL(pfn)); 130 VMGEXIT(); 131 132 val = sev_es_rd_ghcb_msr(); 133 134 /* If the response GPA is not ours then abort the guest */ 135 if ((GHCB_RESP_CODE(val) != GHCB_MSR_REG_GPA_RESP) || 136 (GHCB_MSR_REG_GPA_RESP_VAL(val) != pfn)) 137 sev_es_terminate(SEV_TERM_SET_LINUX, GHCB_TERM_REGISTER); 138 } 139 140 static bool sev_es_negotiate_protocol(void) 141 { 142 u64 val; 143 144 /* Do the GHCB protocol version negotiation */ 145 sev_es_wr_ghcb_msr(GHCB_MSR_SEV_INFO_REQ); 146 VMGEXIT(); 147 val = sev_es_rd_ghcb_msr(); 148 149 if (GHCB_MSR_INFO(val) != GHCB_MSR_SEV_INFO_RESP) 150 return false; 151 152 if (GHCB_MSR_PROTO_MAX(val) < GHCB_PROTOCOL_MIN || 153 GHCB_MSR_PROTO_MIN(val) > GHCB_PROTOCOL_MAX) 154 return false; 155 156 ghcb_version = min_t(size_t, GHCB_MSR_PROTO_MAX(val), GHCB_PROTOCOL_MAX); 157 158 return true; 159 } 160 161 static __always_inline void vc_ghcb_invalidate(struct ghcb *ghcb) 162 { 163 ghcb->save.sw_exit_code = 0; 164 __builtin_memset(ghcb->save.valid_bitmap, 0, sizeof(ghcb->save.valid_bitmap)); 165 } 166 167 static bool vc_decoding_needed(unsigned long exit_code) 168 { 169 /* Exceptions don't require to decode the instruction */ 170 return !(exit_code >= SVM_EXIT_EXCP_BASE && 171 exit_code <= SVM_EXIT_LAST_EXCP); 172 } 173 174 static enum es_result vc_init_em_ctxt(struct es_em_ctxt *ctxt, 175 struct pt_regs *regs, 176 unsigned long exit_code) 177 { 178 enum es_result ret = ES_OK; 179 180 memset(ctxt, 0, sizeof(*ctxt)); 181 ctxt->regs = regs; 182 183 if (vc_decoding_needed(exit_code)) 184 ret = vc_decode_insn(ctxt); 185 186 return ret; 187 } 188 189 static void vc_finish_insn(struct es_em_ctxt *ctxt) 190 { 191 ctxt->regs->ip += ctxt->insn.length; 192 } 193 194 static enum es_result verify_exception_info(struct ghcb *ghcb, struct es_em_ctxt *ctxt) 195 { 196 u32 ret; 197 198 ret = ghcb->save.sw_exit_info_1 & GENMASK_ULL(31, 0); 199 if (!ret) 200 return ES_OK; 201 202 if (ret == 1) { 203 u64 info = ghcb->save.sw_exit_info_2; 204 unsigned long v = info & SVM_EVTINJ_VEC_MASK; 205 206 /* Check if exception information from hypervisor is sane. */ 207 if ((info & SVM_EVTINJ_VALID) && 208 ((v == X86_TRAP_GP) || (v == X86_TRAP_UD)) && 209 ((info & SVM_EVTINJ_TYPE_MASK) == SVM_EVTINJ_TYPE_EXEPT)) { 210 ctxt->fi.vector = v; 211 212 if (info & SVM_EVTINJ_VALID_ERR) 213 ctxt->fi.error_code = info >> 32; 214 215 return ES_EXCEPTION; 216 } 217 } 218 219 return ES_VMM_ERROR; 220 } 221 222 static enum es_result sev_es_ghcb_hv_call(struct ghcb *ghcb, 223 struct es_em_ctxt *ctxt, 224 u64 exit_code, u64 exit_info_1, 225 u64 exit_info_2) 226 { 227 /* Fill in protocol and format specifiers */ 228 ghcb->protocol_version = ghcb_version; 229 ghcb->ghcb_usage = GHCB_DEFAULT_USAGE; 230 231 ghcb_set_sw_exit_code(ghcb, exit_code); 232 ghcb_set_sw_exit_info_1(ghcb, exit_info_1); 233 ghcb_set_sw_exit_info_2(ghcb, exit_info_2); 234 235 sev_es_wr_ghcb_msr(__pa(ghcb)); 236 VMGEXIT(); 237 238 return verify_exception_info(ghcb, ctxt); 239 } 240 241 static int __sev_cpuid_hv(u32 fn, int reg_idx, u32 *reg) 242 { 243 u64 val; 244 245 sev_es_wr_ghcb_msr(GHCB_CPUID_REQ(fn, reg_idx)); 246 VMGEXIT(); 247 val = sev_es_rd_ghcb_msr(); 248 if (GHCB_RESP_CODE(val) != GHCB_MSR_CPUID_RESP) 249 return -EIO; 250 251 *reg = (val >> 32); 252 253 return 0; 254 } 255 256 static int sev_cpuid_hv(struct cpuid_leaf *leaf) 257 { 258 int ret; 259 260 /* 261 * MSR protocol does not support fetching non-zero subfunctions, but is 262 * sufficient to handle current early-boot cases. Should that change, 263 * make sure to report an error rather than ignoring the index and 264 * grabbing random values. If this issue arises in the future, handling 265 * can be added here to use GHCB-page protocol for cases that occur late 266 * enough in boot that GHCB page is available. 267 */ 268 if (cpuid_function_is_indexed(leaf->fn) && leaf->subfn) 269 return -EINVAL; 270 271 ret = __sev_cpuid_hv(leaf->fn, GHCB_CPUID_REQ_EAX, &leaf->eax); 272 ret = ret ? : __sev_cpuid_hv(leaf->fn, GHCB_CPUID_REQ_EBX, &leaf->ebx); 273 ret = ret ? : __sev_cpuid_hv(leaf->fn, GHCB_CPUID_REQ_ECX, &leaf->ecx); 274 ret = ret ? : __sev_cpuid_hv(leaf->fn, GHCB_CPUID_REQ_EDX, &leaf->edx); 275 276 return ret; 277 } 278 279 /* 280 * This may be called early while still running on the initial identity 281 * mapping. Use RIP-relative addressing to obtain the correct address 282 * while running with the initial identity mapping as well as the 283 * switch-over to kernel virtual addresses later. 284 */ 285 static const struct snp_cpuid_table *snp_cpuid_get_table(void) 286 { 287 void *ptr; 288 289 asm ("lea cpuid_table_copy(%%rip), %0" 290 : "=r" (ptr) 291 : "p" (&cpuid_table_copy)); 292 293 return ptr; 294 } 295 296 /* 297 * The SNP Firmware ABI, Revision 0.9, Section 7.1, details the use of 298 * XCR0_IN and XSS_IN to encode multiple versions of 0xD subfunctions 0 299 * and 1 based on the corresponding features enabled by a particular 300 * combination of XCR0 and XSS registers so that a guest can look up the 301 * version corresponding to the features currently enabled in its XCR0/XSS 302 * registers. The only values that differ between these versions/table 303 * entries is the enabled XSAVE area size advertised via EBX. 304 * 305 * While hypervisors may choose to make use of this support, it is more 306 * robust/secure for a guest to simply find the entry corresponding to the 307 * base/legacy XSAVE area size (XCR0=1 or XCR0=3), and then calculate the 308 * XSAVE area size using subfunctions 2 through 64, as documented in APM 309 * Volume 3, Rev 3.31, Appendix E.3.8, which is what is done here. 310 * 311 * Since base/legacy XSAVE area size is documented as 0x240, use that value 312 * directly rather than relying on the base size in the CPUID table. 313 * 314 * Return: XSAVE area size on success, 0 otherwise. 315 */ 316 static u32 snp_cpuid_calc_xsave_size(u64 xfeatures_en, bool compacted) 317 { 318 const struct snp_cpuid_table *cpuid_table = snp_cpuid_get_table(); 319 u64 xfeatures_found = 0; 320 u32 xsave_size = 0x240; 321 int i; 322 323 for (i = 0; i < cpuid_table->count; i++) { 324 const struct snp_cpuid_fn *e = &cpuid_table->fn[i]; 325 326 if (!(e->eax_in == 0xD && e->ecx_in > 1 && e->ecx_in < 64)) 327 continue; 328 if (!(xfeatures_en & (BIT_ULL(e->ecx_in)))) 329 continue; 330 if (xfeatures_found & (BIT_ULL(e->ecx_in))) 331 continue; 332 333 xfeatures_found |= (BIT_ULL(e->ecx_in)); 334 335 if (compacted) 336 xsave_size += e->eax; 337 else 338 xsave_size = max(xsave_size, e->eax + e->ebx); 339 } 340 341 /* 342 * Either the guest set unsupported XCR0/XSS bits, or the corresponding 343 * entries in the CPUID table were not present. This is not a valid 344 * state to be in. 345 */ 346 if (xfeatures_found != (xfeatures_en & GENMASK_ULL(63, 2))) 347 return 0; 348 349 return xsave_size; 350 } 351 352 static bool 353 snp_cpuid_get_validated_func(struct cpuid_leaf *leaf) 354 { 355 const struct snp_cpuid_table *cpuid_table = snp_cpuid_get_table(); 356 int i; 357 358 for (i = 0; i < cpuid_table->count; i++) { 359 const struct snp_cpuid_fn *e = &cpuid_table->fn[i]; 360 361 if (e->eax_in != leaf->fn) 362 continue; 363 364 if (cpuid_function_is_indexed(leaf->fn) && e->ecx_in != leaf->subfn) 365 continue; 366 367 /* 368 * For 0xD subfunctions 0 and 1, only use the entry corresponding 369 * to the base/legacy XSAVE area size (XCR0=1 or XCR0=3, XSS=0). 370 * See the comments above snp_cpuid_calc_xsave_size() for more 371 * details. 372 */ 373 if (e->eax_in == 0xD && (e->ecx_in == 0 || e->ecx_in == 1)) 374 if (!(e->xcr0_in == 1 || e->xcr0_in == 3) || e->xss_in) 375 continue; 376 377 leaf->eax = e->eax; 378 leaf->ebx = e->ebx; 379 leaf->ecx = e->ecx; 380 leaf->edx = e->edx; 381 382 return true; 383 } 384 385 return false; 386 } 387 388 static void snp_cpuid_hv(struct cpuid_leaf *leaf) 389 { 390 if (sev_cpuid_hv(leaf)) 391 sev_es_terminate(SEV_TERM_SET_LINUX, GHCB_TERM_CPUID_HV); 392 } 393 394 static int snp_cpuid_postprocess(struct cpuid_leaf *leaf) 395 { 396 struct cpuid_leaf leaf_hv = *leaf; 397 398 switch (leaf->fn) { 399 case 0x1: 400 snp_cpuid_hv(&leaf_hv); 401 402 /* initial APIC ID */ 403 leaf->ebx = (leaf_hv.ebx & GENMASK(31, 24)) | (leaf->ebx & GENMASK(23, 0)); 404 /* APIC enabled bit */ 405 leaf->edx = (leaf_hv.edx & BIT(9)) | (leaf->edx & ~BIT(9)); 406 407 /* OSXSAVE enabled bit */ 408 if (native_read_cr4() & X86_CR4_OSXSAVE) 409 leaf->ecx |= BIT(27); 410 break; 411 case 0x7: 412 /* OSPKE enabled bit */ 413 leaf->ecx &= ~BIT(4); 414 if (native_read_cr4() & X86_CR4_PKE) 415 leaf->ecx |= BIT(4); 416 break; 417 case 0xB: 418 leaf_hv.subfn = 0; 419 snp_cpuid_hv(&leaf_hv); 420 421 /* extended APIC ID */ 422 leaf->edx = leaf_hv.edx; 423 break; 424 case 0xD: { 425 bool compacted = false; 426 u64 xcr0 = 1, xss = 0; 427 u32 xsave_size; 428 429 if (leaf->subfn != 0 && leaf->subfn != 1) 430 return 0; 431 432 if (native_read_cr4() & X86_CR4_OSXSAVE) 433 xcr0 = xgetbv(XCR_XFEATURE_ENABLED_MASK); 434 if (leaf->subfn == 1) { 435 /* Get XSS value if XSAVES is enabled. */ 436 if (leaf->eax & BIT(3)) { 437 unsigned long lo, hi; 438 439 asm volatile("rdmsr" : "=a" (lo), "=d" (hi) 440 : "c" (MSR_IA32_XSS)); 441 xss = (hi << 32) | lo; 442 } 443 444 /* 445 * The PPR and APM aren't clear on what size should be 446 * encoded in 0xD:0x1:EBX when compaction is not enabled 447 * by either XSAVEC (feature bit 1) or XSAVES (feature 448 * bit 3) since SNP-capable hardware has these feature 449 * bits fixed as 1. KVM sets it to 0 in this case, but 450 * to avoid this becoming an issue it's safer to simply 451 * treat this as unsupported for SNP guests. 452 */ 453 if (!(leaf->eax & (BIT(1) | BIT(3)))) 454 return -EINVAL; 455 456 compacted = true; 457 } 458 459 xsave_size = snp_cpuid_calc_xsave_size(xcr0 | xss, compacted); 460 if (!xsave_size) 461 return -EINVAL; 462 463 leaf->ebx = xsave_size; 464 } 465 break; 466 case 0x8000001E: 467 snp_cpuid_hv(&leaf_hv); 468 469 /* extended APIC ID */ 470 leaf->eax = leaf_hv.eax; 471 /* compute ID */ 472 leaf->ebx = (leaf->ebx & GENMASK(31, 8)) | (leaf_hv.ebx & GENMASK(7, 0)); 473 /* node ID */ 474 leaf->ecx = (leaf->ecx & GENMASK(31, 8)) | (leaf_hv.ecx & GENMASK(7, 0)); 475 break; 476 default: 477 /* No fix-ups needed, use values as-is. */ 478 break; 479 } 480 481 return 0; 482 } 483 484 /* 485 * Returns -EOPNOTSUPP if feature not enabled. Any other non-zero return value 486 * should be treated as fatal by caller. 487 */ 488 static int snp_cpuid(struct cpuid_leaf *leaf) 489 { 490 const struct snp_cpuid_table *cpuid_table = snp_cpuid_get_table(); 491 492 if (!cpuid_table->count) 493 return -EOPNOTSUPP; 494 495 if (!snp_cpuid_get_validated_func(leaf)) { 496 /* 497 * Some hypervisors will avoid keeping track of CPUID entries 498 * where all values are zero, since they can be handled the 499 * same as out-of-range values (all-zero). This is useful here 500 * as well as it allows virtually all guest configurations to 501 * work using a single SNP CPUID table. 502 * 503 * To allow for this, there is a need to distinguish between 504 * out-of-range entries and in-range zero entries, since the 505 * CPUID table entries are only a template that may need to be 506 * augmented with additional values for things like 507 * CPU-specific information during post-processing. So if it's 508 * not in the table, set the values to zero. Then, if they are 509 * within a valid CPUID range, proceed with post-processing 510 * using zeros as the initial values. Otherwise, skip 511 * post-processing and just return zeros immediately. 512 */ 513 leaf->eax = leaf->ebx = leaf->ecx = leaf->edx = 0; 514 515 /* Skip post-processing for out-of-range zero leafs. */ 516 if (!(leaf->fn <= cpuid_std_range_max || 517 (leaf->fn >= 0x40000000 && leaf->fn <= cpuid_hyp_range_max) || 518 (leaf->fn >= 0x80000000 && leaf->fn <= cpuid_ext_range_max))) 519 return 0; 520 } 521 522 return snp_cpuid_postprocess(leaf); 523 } 524 525 /* 526 * Boot VC Handler - This is the first VC handler during boot, there is no GHCB 527 * page yet, so it only supports the MSR based communication with the 528 * hypervisor and only the CPUID exit-code. 529 */ 530 void __init do_vc_no_ghcb(struct pt_regs *regs, unsigned long exit_code) 531 { 532 unsigned int subfn = lower_bits(regs->cx, 32); 533 unsigned int fn = lower_bits(regs->ax, 32); 534 struct cpuid_leaf leaf; 535 int ret; 536 537 /* Only CPUID is supported via MSR protocol */ 538 if (exit_code != SVM_EXIT_CPUID) 539 goto fail; 540 541 leaf.fn = fn; 542 leaf.subfn = subfn; 543 544 ret = snp_cpuid(&leaf); 545 if (!ret) 546 goto cpuid_done; 547 548 if (ret != -EOPNOTSUPP) 549 goto fail; 550 551 if (sev_cpuid_hv(&leaf)) 552 goto fail; 553 554 cpuid_done: 555 regs->ax = leaf.eax; 556 regs->bx = leaf.ebx; 557 regs->cx = leaf.ecx; 558 regs->dx = leaf.edx; 559 560 /* 561 * This is a VC handler and the #VC is only raised when SEV-ES is 562 * active, which means SEV must be active too. Do sanity checks on the 563 * CPUID results to make sure the hypervisor does not trick the kernel 564 * into the no-sev path. This could map sensitive data unencrypted and 565 * make it accessible to the hypervisor. 566 * 567 * In particular, check for: 568 * - Availability of CPUID leaf 0x8000001f 569 * - SEV CPUID bit. 570 * 571 * The hypervisor might still report the wrong C-bit position, but this 572 * can't be checked here. 573 */ 574 575 if (fn == 0x80000000 && (regs->ax < 0x8000001f)) 576 /* SEV leaf check */ 577 goto fail; 578 else if ((fn == 0x8000001f && !(regs->ax & BIT(1)))) 579 /* SEV bit */ 580 goto fail; 581 582 /* Skip over the CPUID two-byte opcode */ 583 regs->ip += 2; 584 585 return; 586 587 fail: 588 /* Terminate the guest */ 589 sev_es_terminate(SEV_TERM_SET_GEN, GHCB_SEV_ES_GEN_REQ); 590 } 591 592 static enum es_result vc_insn_string_read(struct es_em_ctxt *ctxt, 593 void *src, char *buf, 594 unsigned int data_size, 595 unsigned int count, 596 bool backwards) 597 { 598 int i, b = backwards ? -1 : 1; 599 enum es_result ret = ES_OK; 600 601 for (i = 0; i < count; i++) { 602 void *s = src + (i * data_size * b); 603 char *d = buf + (i * data_size); 604 605 ret = vc_read_mem(ctxt, s, d, data_size); 606 if (ret != ES_OK) 607 break; 608 } 609 610 return ret; 611 } 612 613 static enum es_result vc_insn_string_write(struct es_em_ctxt *ctxt, 614 void *dst, char *buf, 615 unsigned int data_size, 616 unsigned int count, 617 bool backwards) 618 { 619 int i, s = backwards ? -1 : 1; 620 enum es_result ret = ES_OK; 621 622 for (i = 0; i < count; i++) { 623 void *d = dst + (i * data_size * s); 624 char *b = buf + (i * data_size); 625 626 ret = vc_write_mem(ctxt, d, b, data_size); 627 if (ret != ES_OK) 628 break; 629 } 630 631 return ret; 632 } 633 634 #define IOIO_TYPE_STR BIT(2) 635 #define IOIO_TYPE_IN 1 636 #define IOIO_TYPE_INS (IOIO_TYPE_IN | IOIO_TYPE_STR) 637 #define IOIO_TYPE_OUT 0 638 #define IOIO_TYPE_OUTS (IOIO_TYPE_OUT | IOIO_TYPE_STR) 639 640 #define IOIO_REP BIT(3) 641 642 #define IOIO_ADDR_64 BIT(9) 643 #define IOIO_ADDR_32 BIT(8) 644 #define IOIO_ADDR_16 BIT(7) 645 646 #define IOIO_DATA_32 BIT(6) 647 #define IOIO_DATA_16 BIT(5) 648 #define IOIO_DATA_8 BIT(4) 649 650 #define IOIO_SEG_ES (0 << 10) 651 #define IOIO_SEG_DS (3 << 10) 652 653 static enum es_result vc_ioio_exitinfo(struct es_em_ctxt *ctxt, u64 *exitinfo) 654 { 655 struct insn *insn = &ctxt->insn; 656 *exitinfo = 0; 657 658 switch (insn->opcode.bytes[0]) { 659 /* INS opcodes */ 660 case 0x6c: 661 case 0x6d: 662 *exitinfo |= IOIO_TYPE_INS; 663 *exitinfo |= IOIO_SEG_ES; 664 *exitinfo |= (ctxt->regs->dx & 0xffff) << 16; 665 break; 666 667 /* OUTS opcodes */ 668 case 0x6e: 669 case 0x6f: 670 *exitinfo |= IOIO_TYPE_OUTS; 671 *exitinfo |= IOIO_SEG_DS; 672 *exitinfo |= (ctxt->regs->dx & 0xffff) << 16; 673 break; 674 675 /* IN immediate opcodes */ 676 case 0xe4: 677 case 0xe5: 678 *exitinfo |= IOIO_TYPE_IN; 679 *exitinfo |= (u8)insn->immediate.value << 16; 680 break; 681 682 /* OUT immediate opcodes */ 683 case 0xe6: 684 case 0xe7: 685 *exitinfo |= IOIO_TYPE_OUT; 686 *exitinfo |= (u8)insn->immediate.value << 16; 687 break; 688 689 /* IN register opcodes */ 690 case 0xec: 691 case 0xed: 692 *exitinfo |= IOIO_TYPE_IN; 693 *exitinfo |= (ctxt->regs->dx & 0xffff) << 16; 694 break; 695 696 /* OUT register opcodes */ 697 case 0xee: 698 case 0xef: 699 *exitinfo |= IOIO_TYPE_OUT; 700 *exitinfo |= (ctxt->regs->dx & 0xffff) << 16; 701 break; 702 703 default: 704 return ES_DECODE_FAILED; 705 } 706 707 switch (insn->opcode.bytes[0]) { 708 case 0x6c: 709 case 0x6e: 710 case 0xe4: 711 case 0xe6: 712 case 0xec: 713 case 0xee: 714 /* Single byte opcodes */ 715 *exitinfo |= IOIO_DATA_8; 716 break; 717 default: 718 /* Length determined by instruction parsing */ 719 *exitinfo |= (insn->opnd_bytes == 2) ? IOIO_DATA_16 720 : IOIO_DATA_32; 721 } 722 switch (insn->addr_bytes) { 723 case 2: 724 *exitinfo |= IOIO_ADDR_16; 725 break; 726 case 4: 727 *exitinfo |= IOIO_ADDR_32; 728 break; 729 case 8: 730 *exitinfo |= IOIO_ADDR_64; 731 break; 732 } 733 734 if (insn_has_rep_prefix(insn)) 735 *exitinfo |= IOIO_REP; 736 737 return ES_OK; 738 } 739 740 static enum es_result vc_handle_ioio(struct ghcb *ghcb, struct es_em_ctxt *ctxt) 741 { 742 struct pt_regs *regs = ctxt->regs; 743 u64 exit_info_1, exit_info_2; 744 enum es_result ret; 745 746 ret = vc_ioio_exitinfo(ctxt, &exit_info_1); 747 if (ret != ES_OK) 748 return ret; 749 750 if (exit_info_1 & IOIO_TYPE_STR) { 751 752 /* (REP) INS/OUTS */ 753 754 bool df = ((regs->flags & X86_EFLAGS_DF) == X86_EFLAGS_DF); 755 unsigned int io_bytes, exit_bytes; 756 unsigned int ghcb_count, op_count; 757 unsigned long es_base; 758 u64 sw_scratch; 759 760 /* 761 * For the string variants with rep prefix the amount of in/out 762 * operations per #VC exception is limited so that the kernel 763 * has a chance to take interrupts and re-schedule while the 764 * instruction is emulated. 765 */ 766 io_bytes = (exit_info_1 >> 4) & 0x7; 767 ghcb_count = sizeof(ghcb->shared_buffer) / io_bytes; 768 769 op_count = (exit_info_1 & IOIO_REP) ? regs->cx : 1; 770 exit_info_2 = min(op_count, ghcb_count); 771 exit_bytes = exit_info_2 * io_bytes; 772 773 es_base = insn_get_seg_base(ctxt->regs, INAT_SEG_REG_ES); 774 775 /* Read bytes of OUTS into the shared buffer */ 776 if (!(exit_info_1 & IOIO_TYPE_IN)) { 777 ret = vc_insn_string_read(ctxt, 778 (void *)(es_base + regs->si), 779 ghcb->shared_buffer, io_bytes, 780 exit_info_2, df); 781 if (ret) 782 return ret; 783 } 784 785 /* 786 * Issue an VMGEXIT to the HV to consume the bytes from the 787 * shared buffer or to have it write them into the shared buffer 788 * depending on the instruction: OUTS or INS. 789 */ 790 sw_scratch = __pa(ghcb) + offsetof(struct ghcb, shared_buffer); 791 ghcb_set_sw_scratch(ghcb, sw_scratch); 792 ret = sev_es_ghcb_hv_call(ghcb, ctxt, SVM_EXIT_IOIO, 793 exit_info_1, exit_info_2); 794 if (ret != ES_OK) 795 return ret; 796 797 /* Read bytes from shared buffer into the guest's destination. */ 798 if (exit_info_1 & IOIO_TYPE_IN) { 799 ret = vc_insn_string_write(ctxt, 800 (void *)(es_base + regs->di), 801 ghcb->shared_buffer, io_bytes, 802 exit_info_2, df); 803 if (ret) 804 return ret; 805 806 if (df) 807 regs->di -= exit_bytes; 808 else 809 regs->di += exit_bytes; 810 } else { 811 if (df) 812 regs->si -= exit_bytes; 813 else 814 regs->si += exit_bytes; 815 } 816 817 if (exit_info_1 & IOIO_REP) 818 regs->cx -= exit_info_2; 819 820 ret = regs->cx ? ES_RETRY : ES_OK; 821 822 } else { 823 824 /* IN/OUT into/from rAX */ 825 826 int bits = (exit_info_1 & 0x70) >> 1; 827 u64 rax = 0; 828 829 if (!(exit_info_1 & IOIO_TYPE_IN)) 830 rax = lower_bits(regs->ax, bits); 831 832 ghcb_set_rax(ghcb, rax); 833 834 ret = sev_es_ghcb_hv_call(ghcb, ctxt, SVM_EXIT_IOIO, exit_info_1, 0); 835 if (ret != ES_OK) 836 return ret; 837 838 if (exit_info_1 & IOIO_TYPE_IN) { 839 if (!ghcb_rax_is_valid(ghcb)) 840 return ES_VMM_ERROR; 841 regs->ax = lower_bits(ghcb->save.rax, bits); 842 } 843 } 844 845 return ret; 846 } 847 848 static int vc_handle_cpuid_snp(struct pt_regs *regs) 849 { 850 struct cpuid_leaf leaf; 851 int ret; 852 853 leaf.fn = regs->ax; 854 leaf.subfn = regs->cx; 855 ret = snp_cpuid(&leaf); 856 if (!ret) { 857 regs->ax = leaf.eax; 858 regs->bx = leaf.ebx; 859 regs->cx = leaf.ecx; 860 regs->dx = leaf.edx; 861 } 862 863 return ret; 864 } 865 866 static enum es_result vc_handle_cpuid(struct ghcb *ghcb, 867 struct es_em_ctxt *ctxt) 868 { 869 struct pt_regs *regs = ctxt->regs; 870 u32 cr4 = native_read_cr4(); 871 enum es_result ret; 872 int snp_cpuid_ret; 873 874 snp_cpuid_ret = vc_handle_cpuid_snp(regs); 875 if (!snp_cpuid_ret) 876 return ES_OK; 877 if (snp_cpuid_ret != -EOPNOTSUPP) 878 return ES_VMM_ERROR; 879 880 ghcb_set_rax(ghcb, regs->ax); 881 ghcb_set_rcx(ghcb, regs->cx); 882 883 if (cr4 & X86_CR4_OSXSAVE) 884 /* Safe to read xcr0 */ 885 ghcb_set_xcr0(ghcb, xgetbv(XCR_XFEATURE_ENABLED_MASK)); 886 else 887 /* xgetbv will cause #GP - use reset value for xcr0 */ 888 ghcb_set_xcr0(ghcb, 1); 889 890 ret = sev_es_ghcb_hv_call(ghcb, ctxt, SVM_EXIT_CPUID, 0, 0); 891 if (ret != ES_OK) 892 return ret; 893 894 if (!(ghcb_rax_is_valid(ghcb) && 895 ghcb_rbx_is_valid(ghcb) && 896 ghcb_rcx_is_valid(ghcb) && 897 ghcb_rdx_is_valid(ghcb))) 898 return ES_VMM_ERROR; 899 900 regs->ax = ghcb->save.rax; 901 regs->bx = ghcb->save.rbx; 902 regs->cx = ghcb->save.rcx; 903 regs->dx = ghcb->save.rdx; 904 905 return ES_OK; 906 } 907 908 static enum es_result vc_handle_rdtsc(struct ghcb *ghcb, 909 struct es_em_ctxt *ctxt, 910 unsigned long exit_code) 911 { 912 bool rdtscp = (exit_code == SVM_EXIT_RDTSCP); 913 enum es_result ret; 914 915 ret = sev_es_ghcb_hv_call(ghcb, ctxt, exit_code, 0, 0); 916 if (ret != ES_OK) 917 return ret; 918 919 if (!(ghcb_rax_is_valid(ghcb) && ghcb_rdx_is_valid(ghcb) && 920 (!rdtscp || ghcb_rcx_is_valid(ghcb)))) 921 return ES_VMM_ERROR; 922 923 ctxt->regs->ax = ghcb->save.rax; 924 ctxt->regs->dx = ghcb->save.rdx; 925 if (rdtscp) 926 ctxt->regs->cx = ghcb->save.rcx; 927 928 return ES_OK; 929 } 930 931 struct cc_setup_data { 932 struct setup_data header; 933 u32 cc_blob_address; 934 }; 935 936 /* 937 * Search for a Confidential Computing blob passed in as a setup_data entry 938 * via the Linux Boot Protocol. 939 */ 940 static struct cc_blob_sev_info *find_cc_blob_setup_data(struct boot_params *bp) 941 { 942 struct cc_setup_data *sd = NULL; 943 struct setup_data *hdr; 944 945 hdr = (struct setup_data *)bp->hdr.setup_data; 946 947 while (hdr) { 948 if (hdr->type == SETUP_CC_BLOB) { 949 sd = (struct cc_setup_data *)hdr; 950 return (struct cc_blob_sev_info *)(unsigned long)sd->cc_blob_address; 951 } 952 hdr = (struct setup_data *)hdr->next; 953 } 954 955 return NULL; 956 } 957 958 /* 959 * Initialize the kernel's copy of the SNP CPUID table, and set up the 960 * pointer that will be used to access it. 961 * 962 * Maintaining a direct mapping of the SNP CPUID table used by firmware would 963 * be possible as an alternative, but the approach is brittle since the 964 * mapping needs to be updated in sync with all the changes to virtual memory 965 * layout and related mapping facilities throughout the boot process. 966 */ 967 static void __init setup_cpuid_table(const struct cc_blob_sev_info *cc_info) 968 { 969 const struct snp_cpuid_table *cpuid_table_fw, *cpuid_table; 970 int i; 971 972 if (!cc_info || !cc_info->cpuid_phys || cc_info->cpuid_len < PAGE_SIZE) 973 sev_es_terminate(SEV_TERM_SET_LINUX, GHCB_TERM_CPUID); 974 975 cpuid_table_fw = (const struct snp_cpuid_table *)cc_info->cpuid_phys; 976 if (!cpuid_table_fw->count || cpuid_table_fw->count > SNP_CPUID_COUNT_MAX) 977 sev_es_terminate(SEV_TERM_SET_LINUX, GHCB_TERM_CPUID); 978 979 cpuid_table = snp_cpuid_get_table(); 980 memcpy((void *)cpuid_table, cpuid_table_fw, sizeof(*cpuid_table)); 981 982 /* Initialize CPUID ranges for range-checking. */ 983 for (i = 0; i < cpuid_table->count; i++) { 984 const struct snp_cpuid_fn *fn = &cpuid_table->fn[i]; 985 986 if (fn->eax_in == 0x0) 987 cpuid_std_range_max = fn->eax; 988 else if (fn->eax_in == 0x40000000) 989 cpuid_hyp_range_max = fn->eax; 990 else if (fn->eax_in == 0x80000000) 991 cpuid_ext_range_max = fn->eax; 992 } 993 } 994