1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * AMD Encrypted Register State Support 4 * 5 * Author: Joerg Roedel <jroedel@suse.de> 6 * 7 * This file is not compiled stand-alone. It contains code shared 8 * between the pre-decompression boot code and the running Linux kernel 9 * and is included directly into both code-bases. 10 */ 11 12 #ifndef __BOOT_COMPRESSED 13 #define error(v) pr_err(v) 14 #define has_cpuflag(f) boot_cpu_has(f) 15 #else 16 #undef WARN 17 #define WARN(condition, format...) (!!(condition)) 18 #endif 19 20 /* I/O parameters for CPUID-related helpers */ 21 struct cpuid_leaf { 22 u32 fn; 23 u32 subfn; 24 u32 eax; 25 u32 ebx; 26 u32 ecx; 27 u32 edx; 28 }; 29 30 /* 31 * Individual entries of the SNP CPUID table, as defined by the SNP 32 * Firmware ABI, Revision 0.9, Section 7.1, Table 14. 33 */ 34 struct snp_cpuid_fn { 35 u32 eax_in; 36 u32 ecx_in; 37 u64 xcr0_in; 38 u64 xss_in; 39 u32 eax; 40 u32 ebx; 41 u32 ecx; 42 u32 edx; 43 u64 __reserved; 44 } __packed; 45 46 /* 47 * SNP CPUID table, as defined by the SNP Firmware ABI, Revision 0.9, 48 * Section 8.14.2.6. Also noted there is the SNP firmware-enforced limit 49 * of 64 entries per CPUID table. 50 */ 51 #define SNP_CPUID_COUNT_MAX 64 52 53 struct snp_cpuid_table { 54 u32 count; 55 u32 __reserved1; 56 u64 __reserved2; 57 struct snp_cpuid_fn fn[SNP_CPUID_COUNT_MAX]; 58 } __packed; 59 60 /* 61 * Since feature negotiation related variables are set early in the boot 62 * process they must reside in the .data section so as not to be zeroed 63 * out when the .bss section is later cleared. 64 * 65 * GHCB protocol version negotiated with the hypervisor. 66 */ 67 static u16 ghcb_version __ro_after_init; 68 69 /* Copy of the SNP firmware's CPUID page. */ 70 static struct snp_cpuid_table cpuid_table_copy __ro_after_init; 71 72 /* 73 * These will be initialized based on CPUID table so that non-present 74 * all-zero leaves (for sparse tables) can be differentiated from 75 * invalid/out-of-range leaves. This is needed since all-zero leaves 76 * still need to be post-processed. 77 */ 78 static u32 cpuid_std_range_max __ro_after_init; 79 static u32 cpuid_hyp_range_max __ro_after_init; 80 static u32 cpuid_ext_range_max __ro_after_init; 81 82 static bool __init sev_es_check_cpu_features(void) 83 { 84 if (!has_cpuflag(X86_FEATURE_RDRAND)) { 85 error("RDRAND instruction not supported - no trusted source of randomness available\n"); 86 return false; 87 } 88 89 return true; 90 } 91 92 static void __noreturn sev_es_terminate(unsigned int set, unsigned int reason) 93 { 94 u64 val = GHCB_MSR_TERM_REQ; 95 96 /* Tell the hypervisor what went wrong. */ 97 val |= GHCB_SEV_TERM_REASON(set, reason); 98 99 /* Request Guest Termination from Hypvervisor */ 100 sev_es_wr_ghcb_msr(val); 101 VMGEXIT(); 102 103 while (true) 104 asm volatile("hlt\n" : : : "memory"); 105 } 106 107 /* 108 * The hypervisor features are available from GHCB version 2 onward. 109 */ 110 static u64 get_hv_features(void) 111 { 112 u64 val; 113 114 if (ghcb_version < 2) 115 return 0; 116 117 sev_es_wr_ghcb_msr(GHCB_MSR_HV_FT_REQ); 118 VMGEXIT(); 119 120 val = sev_es_rd_ghcb_msr(); 121 if (GHCB_RESP_CODE(val) != GHCB_MSR_HV_FT_RESP) 122 return 0; 123 124 return GHCB_MSR_HV_FT_RESP_VAL(val); 125 } 126 127 static void snp_register_ghcb_early(unsigned long paddr) 128 { 129 unsigned long pfn = paddr >> PAGE_SHIFT; 130 u64 val; 131 132 sev_es_wr_ghcb_msr(GHCB_MSR_REG_GPA_REQ_VAL(pfn)); 133 VMGEXIT(); 134 135 val = sev_es_rd_ghcb_msr(); 136 137 /* If the response GPA is not ours then abort the guest */ 138 if ((GHCB_RESP_CODE(val) != GHCB_MSR_REG_GPA_RESP) || 139 (GHCB_MSR_REG_GPA_RESP_VAL(val) != pfn)) 140 sev_es_terminate(SEV_TERM_SET_LINUX, GHCB_TERM_REGISTER); 141 } 142 143 static bool sev_es_negotiate_protocol(void) 144 { 145 u64 val; 146 147 /* Do the GHCB protocol version negotiation */ 148 sev_es_wr_ghcb_msr(GHCB_MSR_SEV_INFO_REQ); 149 VMGEXIT(); 150 val = sev_es_rd_ghcb_msr(); 151 152 if (GHCB_MSR_INFO(val) != GHCB_MSR_SEV_INFO_RESP) 153 return false; 154 155 if (GHCB_MSR_PROTO_MAX(val) < GHCB_PROTOCOL_MIN || 156 GHCB_MSR_PROTO_MIN(val) > GHCB_PROTOCOL_MAX) 157 return false; 158 159 ghcb_version = min_t(size_t, GHCB_MSR_PROTO_MAX(val), GHCB_PROTOCOL_MAX); 160 161 return true; 162 } 163 164 static __always_inline void vc_ghcb_invalidate(struct ghcb *ghcb) 165 { 166 ghcb->save.sw_exit_code = 0; 167 __builtin_memset(ghcb->save.valid_bitmap, 0, sizeof(ghcb->save.valid_bitmap)); 168 } 169 170 static bool vc_decoding_needed(unsigned long exit_code) 171 { 172 /* Exceptions don't require to decode the instruction */ 173 return !(exit_code >= SVM_EXIT_EXCP_BASE && 174 exit_code <= SVM_EXIT_LAST_EXCP); 175 } 176 177 static enum es_result vc_init_em_ctxt(struct es_em_ctxt *ctxt, 178 struct pt_regs *regs, 179 unsigned long exit_code) 180 { 181 enum es_result ret = ES_OK; 182 183 memset(ctxt, 0, sizeof(*ctxt)); 184 ctxt->regs = regs; 185 186 if (vc_decoding_needed(exit_code)) 187 ret = vc_decode_insn(ctxt); 188 189 return ret; 190 } 191 192 static void vc_finish_insn(struct es_em_ctxt *ctxt) 193 { 194 ctxt->regs->ip += ctxt->insn.length; 195 } 196 197 static enum es_result verify_exception_info(struct ghcb *ghcb, struct es_em_ctxt *ctxt) 198 { 199 u32 ret; 200 201 ret = ghcb->save.sw_exit_info_1 & GENMASK_ULL(31, 0); 202 if (!ret) 203 return ES_OK; 204 205 if (ret == 1) { 206 u64 info = ghcb->save.sw_exit_info_2; 207 unsigned long v = info & SVM_EVTINJ_VEC_MASK; 208 209 /* Check if exception information from hypervisor is sane. */ 210 if ((info & SVM_EVTINJ_VALID) && 211 ((v == X86_TRAP_GP) || (v == X86_TRAP_UD)) && 212 ((info & SVM_EVTINJ_TYPE_MASK) == SVM_EVTINJ_TYPE_EXEPT)) { 213 ctxt->fi.vector = v; 214 215 if (info & SVM_EVTINJ_VALID_ERR) 216 ctxt->fi.error_code = info >> 32; 217 218 return ES_EXCEPTION; 219 } 220 } 221 222 return ES_VMM_ERROR; 223 } 224 225 static enum es_result sev_es_ghcb_hv_call(struct ghcb *ghcb, 226 struct es_em_ctxt *ctxt, 227 u64 exit_code, u64 exit_info_1, 228 u64 exit_info_2) 229 { 230 /* Fill in protocol and format specifiers */ 231 ghcb->protocol_version = ghcb_version; 232 ghcb->ghcb_usage = GHCB_DEFAULT_USAGE; 233 234 ghcb_set_sw_exit_code(ghcb, exit_code); 235 ghcb_set_sw_exit_info_1(ghcb, exit_info_1); 236 ghcb_set_sw_exit_info_2(ghcb, exit_info_2); 237 238 sev_es_wr_ghcb_msr(__pa(ghcb)); 239 VMGEXIT(); 240 241 return verify_exception_info(ghcb, ctxt); 242 } 243 244 static int __sev_cpuid_hv(u32 fn, int reg_idx, u32 *reg) 245 { 246 u64 val; 247 248 sev_es_wr_ghcb_msr(GHCB_CPUID_REQ(fn, reg_idx)); 249 VMGEXIT(); 250 val = sev_es_rd_ghcb_msr(); 251 if (GHCB_RESP_CODE(val) != GHCB_MSR_CPUID_RESP) 252 return -EIO; 253 254 *reg = (val >> 32); 255 256 return 0; 257 } 258 259 static int __sev_cpuid_hv_msr(struct cpuid_leaf *leaf) 260 { 261 int ret; 262 263 /* 264 * MSR protocol does not support fetching non-zero subfunctions, but is 265 * sufficient to handle current early-boot cases. Should that change, 266 * make sure to report an error rather than ignoring the index and 267 * grabbing random values. If this issue arises in the future, handling 268 * can be added here to use GHCB-page protocol for cases that occur late 269 * enough in boot that GHCB page is available. 270 */ 271 if (cpuid_function_is_indexed(leaf->fn) && leaf->subfn) 272 return -EINVAL; 273 274 ret = __sev_cpuid_hv(leaf->fn, GHCB_CPUID_REQ_EAX, &leaf->eax); 275 ret = ret ? : __sev_cpuid_hv(leaf->fn, GHCB_CPUID_REQ_EBX, &leaf->ebx); 276 ret = ret ? : __sev_cpuid_hv(leaf->fn, GHCB_CPUID_REQ_ECX, &leaf->ecx); 277 ret = ret ? : __sev_cpuid_hv(leaf->fn, GHCB_CPUID_REQ_EDX, &leaf->edx); 278 279 return ret; 280 } 281 282 static int __sev_cpuid_hv_ghcb(struct ghcb *ghcb, struct es_em_ctxt *ctxt, struct cpuid_leaf *leaf) 283 { 284 u32 cr4 = native_read_cr4(); 285 int ret; 286 287 ghcb_set_rax(ghcb, leaf->fn); 288 ghcb_set_rcx(ghcb, leaf->subfn); 289 290 if (cr4 & X86_CR4_OSXSAVE) 291 /* Safe to read xcr0 */ 292 ghcb_set_xcr0(ghcb, xgetbv(XCR_XFEATURE_ENABLED_MASK)); 293 else 294 /* xgetbv will cause #UD - use reset value for xcr0 */ 295 ghcb_set_xcr0(ghcb, 1); 296 297 ret = sev_es_ghcb_hv_call(ghcb, ctxt, SVM_EXIT_CPUID, 0, 0); 298 if (ret != ES_OK) 299 return ret; 300 301 if (!(ghcb_rax_is_valid(ghcb) && 302 ghcb_rbx_is_valid(ghcb) && 303 ghcb_rcx_is_valid(ghcb) && 304 ghcb_rdx_is_valid(ghcb))) 305 return ES_VMM_ERROR; 306 307 leaf->eax = ghcb->save.rax; 308 leaf->ebx = ghcb->save.rbx; 309 leaf->ecx = ghcb->save.rcx; 310 leaf->edx = ghcb->save.rdx; 311 312 return ES_OK; 313 } 314 315 static int sev_cpuid_hv(struct ghcb *ghcb, struct es_em_ctxt *ctxt, struct cpuid_leaf *leaf) 316 { 317 return ghcb ? __sev_cpuid_hv_ghcb(ghcb, ctxt, leaf) 318 : __sev_cpuid_hv_msr(leaf); 319 } 320 321 /* 322 * This may be called early while still running on the initial identity 323 * mapping. Use RIP-relative addressing to obtain the correct address 324 * while running with the initial identity mapping as well as the 325 * switch-over to kernel virtual addresses later. 326 */ 327 static const struct snp_cpuid_table *snp_cpuid_get_table(void) 328 { 329 void *ptr; 330 331 asm ("lea cpuid_table_copy(%%rip), %0" 332 : "=r" (ptr) 333 : "p" (&cpuid_table_copy)); 334 335 return ptr; 336 } 337 338 /* 339 * The SNP Firmware ABI, Revision 0.9, Section 7.1, details the use of 340 * XCR0_IN and XSS_IN to encode multiple versions of 0xD subfunctions 0 341 * and 1 based on the corresponding features enabled by a particular 342 * combination of XCR0 and XSS registers so that a guest can look up the 343 * version corresponding to the features currently enabled in its XCR0/XSS 344 * registers. The only values that differ between these versions/table 345 * entries is the enabled XSAVE area size advertised via EBX. 346 * 347 * While hypervisors may choose to make use of this support, it is more 348 * robust/secure for a guest to simply find the entry corresponding to the 349 * base/legacy XSAVE area size (XCR0=1 or XCR0=3), and then calculate the 350 * XSAVE area size using subfunctions 2 through 64, as documented in APM 351 * Volume 3, Rev 3.31, Appendix E.3.8, which is what is done here. 352 * 353 * Since base/legacy XSAVE area size is documented as 0x240, use that value 354 * directly rather than relying on the base size in the CPUID table. 355 * 356 * Return: XSAVE area size on success, 0 otherwise. 357 */ 358 static u32 snp_cpuid_calc_xsave_size(u64 xfeatures_en, bool compacted) 359 { 360 const struct snp_cpuid_table *cpuid_table = snp_cpuid_get_table(); 361 u64 xfeatures_found = 0; 362 u32 xsave_size = 0x240; 363 int i; 364 365 for (i = 0; i < cpuid_table->count; i++) { 366 const struct snp_cpuid_fn *e = &cpuid_table->fn[i]; 367 368 if (!(e->eax_in == 0xD && e->ecx_in > 1 && e->ecx_in < 64)) 369 continue; 370 if (!(xfeatures_en & (BIT_ULL(e->ecx_in)))) 371 continue; 372 if (xfeatures_found & (BIT_ULL(e->ecx_in))) 373 continue; 374 375 xfeatures_found |= (BIT_ULL(e->ecx_in)); 376 377 if (compacted) 378 xsave_size += e->eax; 379 else 380 xsave_size = max(xsave_size, e->eax + e->ebx); 381 } 382 383 /* 384 * Either the guest set unsupported XCR0/XSS bits, or the corresponding 385 * entries in the CPUID table were not present. This is not a valid 386 * state to be in. 387 */ 388 if (xfeatures_found != (xfeatures_en & GENMASK_ULL(63, 2))) 389 return 0; 390 391 return xsave_size; 392 } 393 394 static bool 395 snp_cpuid_get_validated_func(struct cpuid_leaf *leaf) 396 { 397 const struct snp_cpuid_table *cpuid_table = snp_cpuid_get_table(); 398 int i; 399 400 for (i = 0; i < cpuid_table->count; i++) { 401 const struct snp_cpuid_fn *e = &cpuid_table->fn[i]; 402 403 if (e->eax_in != leaf->fn) 404 continue; 405 406 if (cpuid_function_is_indexed(leaf->fn) && e->ecx_in != leaf->subfn) 407 continue; 408 409 /* 410 * For 0xD subfunctions 0 and 1, only use the entry corresponding 411 * to the base/legacy XSAVE area size (XCR0=1 or XCR0=3, XSS=0). 412 * See the comments above snp_cpuid_calc_xsave_size() for more 413 * details. 414 */ 415 if (e->eax_in == 0xD && (e->ecx_in == 0 || e->ecx_in == 1)) 416 if (!(e->xcr0_in == 1 || e->xcr0_in == 3) || e->xss_in) 417 continue; 418 419 leaf->eax = e->eax; 420 leaf->ebx = e->ebx; 421 leaf->ecx = e->ecx; 422 leaf->edx = e->edx; 423 424 return true; 425 } 426 427 return false; 428 } 429 430 static void snp_cpuid_hv(struct ghcb *ghcb, struct es_em_ctxt *ctxt, struct cpuid_leaf *leaf) 431 { 432 if (sev_cpuid_hv(ghcb, ctxt, leaf)) 433 sev_es_terminate(SEV_TERM_SET_LINUX, GHCB_TERM_CPUID_HV); 434 } 435 436 static int snp_cpuid_postprocess(struct ghcb *ghcb, struct es_em_ctxt *ctxt, 437 struct cpuid_leaf *leaf) 438 { 439 struct cpuid_leaf leaf_hv = *leaf; 440 441 switch (leaf->fn) { 442 case 0x1: 443 snp_cpuid_hv(ghcb, ctxt, &leaf_hv); 444 445 /* initial APIC ID */ 446 leaf->ebx = (leaf_hv.ebx & GENMASK(31, 24)) | (leaf->ebx & GENMASK(23, 0)); 447 /* APIC enabled bit */ 448 leaf->edx = (leaf_hv.edx & BIT(9)) | (leaf->edx & ~BIT(9)); 449 450 /* OSXSAVE enabled bit */ 451 if (native_read_cr4() & X86_CR4_OSXSAVE) 452 leaf->ecx |= BIT(27); 453 break; 454 case 0x7: 455 /* OSPKE enabled bit */ 456 leaf->ecx &= ~BIT(4); 457 if (native_read_cr4() & X86_CR4_PKE) 458 leaf->ecx |= BIT(4); 459 break; 460 case 0xB: 461 leaf_hv.subfn = 0; 462 snp_cpuid_hv(ghcb, ctxt, &leaf_hv); 463 464 /* extended APIC ID */ 465 leaf->edx = leaf_hv.edx; 466 break; 467 case 0xD: { 468 bool compacted = false; 469 u64 xcr0 = 1, xss = 0; 470 u32 xsave_size; 471 472 if (leaf->subfn != 0 && leaf->subfn != 1) 473 return 0; 474 475 if (native_read_cr4() & X86_CR4_OSXSAVE) 476 xcr0 = xgetbv(XCR_XFEATURE_ENABLED_MASK); 477 if (leaf->subfn == 1) { 478 /* Get XSS value if XSAVES is enabled. */ 479 if (leaf->eax & BIT(3)) { 480 unsigned long lo, hi; 481 482 asm volatile("rdmsr" : "=a" (lo), "=d" (hi) 483 : "c" (MSR_IA32_XSS)); 484 xss = (hi << 32) | lo; 485 } 486 487 /* 488 * The PPR and APM aren't clear on what size should be 489 * encoded in 0xD:0x1:EBX when compaction is not enabled 490 * by either XSAVEC (feature bit 1) or XSAVES (feature 491 * bit 3) since SNP-capable hardware has these feature 492 * bits fixed as 1. KVM sets it to 0 in this case, but 493 * to avoid this becoming an issue it's safer to simply 494 * treat this as unsupported for SNP guests. 495 */ 496 if (!(leaf->eax & (BIT(1) | BIT(3)))) 497 return -EINVAL; 498 499 compacted = true; 500 } 501 502 xsave_size = snp_cpuid_calc_xsave_size(xcr0 | xss, compacted); 503 if (!xsave_size) 504 return -EINVAL; 505 506 leaf->ebx = xsave_size; 507 } 508 break; 509 case 0x8000001E: 510 snp_cpuid_hv(ghcb, ctxt, &leaf_hv); 511 512 /* extended APIC ID */ 513 leaf->eax = leaf_hv.eax; 514 /* compute ID */ 515 leaf->ebx = (leaf->ebx & GENMASK(31, 8)) | (leaf_hv.ebx & GENMASK(7, 0)); 516 /* node ID */ 517 leaf->ecx = (leaf->ecx & GENMASK(31, 8)) | (leaf_hv.ecx & GENMASK(7, 0)); 518 break; 519 default: 520 /* No fix-ups needed, use values as-is. */ 521 break; 522 } 523 524 return 0; 525 } 526 527 /* 528 * Returns -EOPNOTSUPP if feature not enabled. Any other non-zero return value 529 * should be treated as fatal by caller. 530 */ 531 static int snp_cpuid(struct ghcb *ghcb, struct es_em_ctxt *ctxt, struct cpuid_leaf *leaf) 532 { 533 const struct snp_cpuid_table *cpuid_table = snp_cpuid_get_table(); 534 535 if (!cpuid_table->count) 536 return -EOPNOTSUPP; 537 538 if (!snp_cpuid_get_validated_func(leaf)) { 539 /* 540 * Some hypervisors will avoid keeping track of CPUID entries 541 * where all values are zero, since they can be handled the 542 * same as out-of-range values (all-zero). This is useful here 543 * as well as it allows virtually all guest configurations to 544 * work using a single SNP CPUID table. 545 * 546 * To allow for this, there is a need to distinguish between 547 * out-of-range entries and in-range zero entries, since the 548 * CPUID table entries are only a template that may need to be 549 * augmented with additional values for things like 550 * CPU-specific information during post-processing. So if it's 551 * not in the table, set the values to zero. Then, if they are 552 * within a valid CPUID range, proceed with post-processing 553 * using zeros as the initial values. Otherwise, skip 554 * post-processing and just return zeros immediately. 555 */ 556 leaf->eax = leaf->ebx = leaf->ecx = leaf->edx = 0; 557 558 /* Skip post-processing for out-of-range zero leafs. */ 559 if (!(leaf->fn <= cpuid_std_range_max || 560 (leaf->fn >= 0x40000000 && leaf->fn <= cpuid_hyp_range_max) || 561 (leaf->fn >= 0x80000000 && leaf->fn <= cpuid_ext_range_max))) 562 return 0; 563 } 564 565 return snp_cpuid_postprocess(ghcb, ctxt, leaf); 566 } 567 568 /* 569 * Boot VC Handler - This is the first VC handler during boot, there is no GHCB 570 * page yet, so it only supports the MSR based communication with the 571 * hypervisor and only the CPUID exit-code. 572 */ 573 void __init do_vc_no_ghcb(struct pt_regs *regs, unsigned long exit_code) 574 { 575 unsigned int subfn = lower_bits(regs->cx, 32); 576 unsigned int fn = lower_bits(regs->ax, 32); 577 struct cpuid_leaf leaf; 578 int ret; 579 580 /* Only CPUID is supported via MSR protocol */ 581 if (exit_code != SVM_EXIT_CPUID) 582 goto fail; 583 584 leaf.fn = fn; 585 leaf.subfn = subfn; 586 587 ret = snp_cpuid(NULL, NULL, &leaf); 588 if (!ret) 589 goto cpuid_done; 590 591 if (ret != -EOPNOTSUPP) 592 goto fail; 593 594 if (__sev_cpuid_hv_msr(&leaf)) 595 goto fail; 596 597 cpuid_done: 598 regs->ax = leaf.eax; 599 regs->bx = leaf.ebx; 600 regs->cx = leaf.ecx; 601 regs->dx = leaf.edx; 602 603 /* 604 * This is a VC handler and the #VC is only raised when SEV-ES is 605 * active, which means SEV must be active too. Do sanity checks on the 606 * CPUID results to make sure the hypervisor does not trick the kernel 607 * into the no-sev path. This could map sensitive data unencrypted and 608 * make it accessible to the hypervisor. 609 * 610 * In particular, check for: 611 * - Availability of CPUID leaf 0x8000001f 612 * - SEV CPUID bit. 613 * 614 * The hypervisor might still report the wrong C-bit position, but this 615 * can't be checked here. 616 */ 617 618 if (fn == 0x80000000 && (regs->ax < 0x8000001f)) 619 /* SEV leaf check */ 620 goto fail; 621 else if ((fn == 0x8000001f && !(regs->ax & BIT(1)))) 622 /* SEV bit */ 623 goto fail; 624 625 /* Skip over the CPUID two-byte opcode */ 626 regs->ip += 2; 627 628 return; 629 630 fail: 631 /* Terminate the guest */ 632 sev_es_terminate(SEV_TERM_SET_GEN, GHCB_SEV_ES_GEN_REQ); 633 } 634 635 static enum es_result vc_insn_string_check(struct es_em_ctxt *ctxt, 636 unsigned long address, 637 bool write) 638 { 639 if (user_mode(ctxt->regs) && fault_in_kernel_space(address)) { 640 ctxt->fi.vector = X86_TRAP_PF; 641 ctxt->fi.error_code = X86_PF_USER; 642 ctxt->fi.cr2 = address; 643 if (write) 644 ctxt->fi.error_code |= X86_PF_WRITE; 645 646 return ES_EXCEPTION; 647 } 648 649 return ES_OK; 650 } 651 652 static enum es_result vc_insn_string_read(struct es_em_ctxt *ctxt, 653 void *src, char *buf, 654 unsigned int data_size, 655 unsigned int count, 656 bool backwards) 657 { 658 int i, b = backwards ? -1 : 1; 659 unsigned long address = (unsigned long)src; 660 enum es_result ret; 661 662 ret = vc_insn_string_check(ctxt, address, false); 663 if (ret != ES_OK) 664 return ret; 665 666 for (i = 0; i < count; i++) { 667 void *s = src + (i * data_size * b); 668 char *d = buf + (i * data_size); 669 670 ret = vc_read_mem(ctxt, s, d, data_size); 671 if (ret != ES_OK) 672 break; 673 } 674 675 return ret; 676 } 677 678 static enum es_result vc_insn_string_write(struct es_em_ctxt *ctxt, 679 void *dst, char *buf, 680 unsigned int data_size, 681 unsigned int count, 682 bool backwards) 683 { 684 int i, s = backwards ? -1 : 1; 685 unsigned long address = (unsigned long)dst; 686 enum es_result ret; 687 688 ret = vc_insn_string_check(ctxt, address, true); 689 if (ret != ES_OK) 690 return ret; 691 692 for (i = 0; i < count; i++) { 693 void *d = dst + (i * data_size * s); 694 char *b = buf + (i * data_size); 695 696 ret = vc_write_mem(ctxt, d, b, data_size); 697 if (ret != ES_OK) 698 break; 699 } 700 701 return ret; 702 } 703 704 #define IOIO_TYPE_STR BIT(2) 705 #define IOIO_TYPE_IN 1 706 #define IOIO_TYPE_INS (IOIO_TYPE_IN | IOIO_TYPE_STR) 707 #define IOIO_TYPE_OUT 0 708 #define IOIO_TYPE_OUTS (IOIO_TYPE_OUT | IOIO_TYPE_STR) 709 710 #define IOIO_REP BIT(3) 711 712 #define IOIO_ADDR_64 BIT(9) 713 #define IOIO_ADDR_32 BIT(8) 714 #define IOIO_ADDR_16 BIT(7) 715 716 #define IOIO_DATA_32 BIT(6) 717 #define IOIO_DATA_16 BIT(5) 718 #define IOIO_DATA_8 BIT(4) 719 720 #define IOIO_SEG_ES (0 << 10) 721 #define IOIO_SEG_DS (3 << 10) 722 723 static enum es_result vc_ioio_exitinfo(struct es_em_ctxt *ctxt, u64 *exitinfo) 724 { 725 struct insn *insn = &ctxt->insn; 726 size_t size; 727 u64 port; 728 729 *exitinfo = 0; 730 731 switch (insn->opcode.bytes[0]) { 732 /* INS opcodes */ 733 case 0x6c: 734 case 0x6d: 735 *exitinfo |= IOIO_TYPE_INS; 736 *exitinfo |= IOIO_SEG_ES; 737 port = ctxt->regs->dx & 0xffff; 738 break; 739 740 /* OUTS opcodes */ 741 case 0x6e: 742 case 0x6f: 743 *exitinfo |= IOIO_TYPE_OUTS; 744 *exitinfo |= IOIO_SEG_DS; 745 port = ctxt->regs->dx & 0xffff; 746 break; 747 748 /* IN immediate opcodes */ 749 case 0xe4: 750 case 0xe5: 751 *exitinfo |= IOIO_TYPE_IN; 752 port = (u8)insn->immediate.value & 0xffff; 753 break; 754 755 /* OUT immediate opcodes */ 756 case 0xe6: 757 case 0xe7: 758 *exitinfo |= IOIO_TYPE_OUT; 759 port = (u8)insn->immediate.value & 0xffff; 760 break; 761 762 /* IN register opcodes */ 763 case 0xec: 764 case 0xed: 765 *exitinfo |= IOIO_TYPE_IN; 766 port = ctxt->regs->dx & 0xffff; 767 break; 768 769 /* OUT register opcodes */ 770 case 0xee: 771 case 0xef: 772 *exitinfo |= IOIO_TYPE_OUT; 773 port = ctxt->regs->dx & 0xffff; 774 break; 775 776 default: 777 return ES_DECODE_FAILED; 778 } 779 780 *exitinfo |= port << 16; 781 782 switch (insn->opcode.bytes[0]) { 783 case 0x6c: 784 case 0x6e: 785 case 0xe4: 786 case 0xe6: 787 case 0xec: 788 case 0xee: 789 /* Single byte opcodes */ 790 *exitinfo |= IOIO_DATA_8; 791 size = 1; 792 break; 793 default: 794 /* Length determined by instruction parsing */ 795 *exitinfo |= (insn->opnd_bytes == 2) ? IOIO_DATA_16 796 : IOIO_DATA_32; 797 size = (insn->opnd_bytes == 2) ? 2 : 4; 798 } 799 800 switch (insn->addr_bytes) { 801 case 2: 802 *exitinfo |= IOIO_ADDR_16; 803 break; 804 case 4: 805 *exitinfo |= IOIO_ADDR_32; 806 break; 807 case 8: 808 *exitinfo |= IOIO_ADDR_64; 809 break; 810 } 811 812 if (insn_has_rep_prefix(insn)) 813 *exitinfo |= IOIO_REP; 814 815 return vc_ioio_check(ctxt, (u16)port, size); 816 } 817 818 static enum es_result vc_handle_ioio(struct ghcb *ghcb, struct es_em_ctxt *ctxt) 819 { 820 struct pt_regs *regs = ctxt->regs; 821 u64 exit_info_1, exit_info_2; 822 enum es_result ret; 823 824 ret = vc_ioio_exitinfo(ctxt, &exit_info_1); 825 if (ret != ES_OK) 826 return ret; 827 828 if (exit_info_1 & IOIO_TYPE_STR) { 829 830 /* (REP) INS/OUTS */ 831 832 bool df = ((regs->flags & X86_EFLAGS_DF) == X86_EFLAGS_DF); 833 unsigned int io_bytes, exit_bytes; 834 unsigned int ghcb_count, op_count; 835 unsigned long es_base; 836 u64 sw_scratch; 837 838 /* 839 * For the string variants with rep prefix the amount of in/out 840 * operations per #VC exception is limited so that the kernel 841 * has a chance to take interrupts and re-schedule while the 842 * instruction is emulated. 843 */ 844 io_bytes = (exit_info_1 >> 4) & 0x7; 845 ghcb_count = sizeof(ghcb->shared_buffer) / io_bytes; 846 847 op_count = (exit_info_1 & IOIO_REP) ? regs->cx : 1; 848 exit_info_2 = min(op_count, ghcb_count); 849 exit_bytes = exit_info_2 * io_bytes; 850 851 es_base = insn_get_seg_base(ctxt->regs, INAT_SEG_REG_ES); 852 853 /* Read bytes of OUTS into the shared buffer */ 854 if (!(exit_info_1 & IOIO_TYPE_IN)) { 855 ret = vc_insn_string_read(ctxt, 856 (void *)(es_base + regs->si), 857 ghcb->shared_buffer, io_bytes, 858 exit_info_2, df); 859 if (ret) 860 return ret; 861 } 862 863 /* 864 * Issue an VMGEXIT to the HV to consume the bytes from the 865 * shared buffer or to have it write them into the shared buffer 866 * depending on the instruction: OUTS or INS. 867 */ 868 sw_scratch = __pa(ghcb) + offsetof(struct ghcb, shared_buffer); 869 ghcb_set_sw_scratch(ghcb, sw_scratch); 870 ret = sev_es_ghcb_hv_call(ghcb, ctxt, SVM_EXIT_IOIO, 871 exit_info_1, exit_info_2); 872 if (ret != ES_OK) 873 return ret; 874 875 /* Read bytes from shared buffer into the guest's destination. */ 876 if (exit_info_1 & IOIO_TYPE_IN) { 877 ret = vc_insn_string_write(ctxt, 878 (void *)(es_base + regs->di), 879 ghcb->shared_buffer, io_bytes, 880 exit_info_2, df); 881 if (ret) 882 return ret; 883 884 if (df) 885 regs->di -= exit_bytes; 886 else 887 regs->di += exit_bytes; 888 } else { 889 if (df) 890 regs->si -= exit_bytes; 891 else 892 regs->si += exit_bytes; 893 } 894 895 if (exit_info_1 & IOIO_REP) 896 regs->cx -= exit_info_2; 897 898 ret = regs->cx ? ES_RETRY : ES_OK; 899 900 } else { 901 902 /* IN/OUT into/from rAX */ 903 904 int bits = (exit_info_1 & 0x70) >> 1; 905 u64 rax = 0; 906 907 if (!(exit_info_1 & IOIO_TYPE_IN)) 908 rax = lower_bits(regs->ax, bits); 909 910 ghcb_set_rax(ghcb, rax); 911 912 ret = sev_es_ghcb_hv_call(ghcb, ctxt, SVM_EXIT_IOIO, exit_info_1, 0); 913 if (ret != ES_OK) 914 return ret; 915 916 if (exit_info_1 & IOIO_TYPE_IN) { 917 if (!ghcb_rax_is_valid(ghcb)) 918 return ES_VMM_ERROR; 919 regs->ax = lower_bits(ghcb->save.rax, bits); 920 } 921 } 922 923 return ret; 924 } 925 926 static int vc_handle_cpuid_snp(struct ghcb *ghcb, struct es_em_ctxt *ctxt) 927 { 928 struct pt_regs *regs = ctxt->regs; 929 struct cpuid_leaf leaf; 930 int ret; 931 932 leaf.fn = regs->ax; 933 leaf.subfn = regs->cx; 934 ret = snp_cpuid(ghcb, ctxt, &leaf); 935 if (!ret) { 936 regs->ax = leaf.eax; 937 regs->bx = leaf.ebx; 938 regs->cx = leaf.ecx; 939 regs->dx = leaf.edx; 940 } 941 942 return ret; 943 } 944 945 static enum es_result vc_handle_cpuid(struct ghcb *ghcb, 946 struct es_em_ctxt *ctxt) 947 { 948 struct pt_regs *regs = ctxt->regs; 949 u32 cr4 = native_read_cr4(); 950 enum es_result ret; 951 int snp_cpuid_ret; 952 953 snp_cpuid_ret = vc_handle_cpuid_snp(ghcb, ctxt); 954 if (!snp_cpuid_ret) 955 return ES_OK; 956 if (snp_cpuid_ret != -EOPNOTSUPP) 957 return ES_VMM_ERROR; 958 959 ghcb_set_rax(ghcb, regs->ax); 960 ghcb_set_rcx(ghcb, regs->cx); 961 962 if (cr4 & X86_CR4_OSXSAVE) 963 /* Safe to read xcr0 */ 964 ghcb_set_xcr0(ghcb, xgetbv(XCR_XFEATURE_ENABLED_MASK)); 965 else 966 /* xgetbv will cause #GP - use reset value for xcr0 */ 967 ghcb_set_xcr0(ghcb, 1); 968 969 ret = sev_es_ghcb_hv_call(ghcb, ctxt, SVM_EXIT_CPUID, 0, 0); 970 if (ret != ES_OK) 971 return ret; 972 973 if (!(ghcb_rax_is_valid(ghcb) && 974 ghcb_rbx_is_valid(ghcb) && 975 ghcb_rcx_is_valid(ghcb) && 976 ghcb_rdx_is_valid(ghcb))) 977 return ES_VMM_ERROR; 978 979 regs->ax = ghcb->save.rax; 980 regs->bx = ghcb->save.rbx; 981 regs->cx = ghcb->save.rcx; 982 regs->dx = ghcb->save.rdx; 983 984 return ES_OK; 985 } 986 987 static enum es_result vc_handle_rdtsc(struct ghcb *ghcb, 988 struct es_em_ctxt *ctxt, 989 unsigned long exit_code) 990 { 991 bool rdtscp = (exit_code == SVM_EXIT_RDTSCP); 992 enum es_result ret; 993 994 ret = sev_es_ghcb_hv_call(ghcb, ctxt, exit_code, 0, 0); 995 if (ret != ES_OK) 996 return ret; 997 998 if (!(ghcb_rax_is_valid(ghcb) && ghcb_rdx_is_valid(ghcb) && 999 (!rdtscp || ghcb_rcx_is_valid(ghcb)))) 1000 return ES_VMM_ERROR; 1001 1002 ctxt->regs->ax = ghcb->save.rax; 1003 ctxt->regs->dx = ghcb->save.rdx; 1004 if (rdtscp) 1005 ctxt->regs->cx = ghcb->save.rcx; 1006 1007 return ES_OK; 1008 } 1009 1010 struct cc_setup_data { 1011 struct setup_data header; 1012 u32 cc_blob_address; 1013 }; 1014 1015 /* 1016 * Search for a Confidential Computing blob passed in as a setup_data entry 1017 * via the Linux Boot Protocol. 1018 */ 1019 static struct cc_blob_sev_info *find_cc_blob_setup_data(struct boot_params *bp) 1020 { 1021 struct cc_setup_data *sd = NULL; 1022 struct setup_data *hdr; 1023 1024 hdr = (struct setup_data *)bp->hdr.setup_data; 1025 1026 while (hdr) { 1027 if (hdr->type == SETUP_CC_BLOB) { 1028 sd = (struct cc_setup_data *)hdr; 1029 return (struct cc_blob_sev_info *)(unsigned long)sd->cc_blob_address; 1030 } 1031 hdr = (struct setup_data *)hdr->next; 1032 } 1033 1034 return NULL; 1035 } 1036 1037 /* 1038 * Initialize the kernel's copy of the SNP CPUID table, and set up the 1039 * pointer that will be used to access it. 1040 * 1041 * Maintaining a direct mapping of the SNP CPUID table used by firmware would 1042 * be possible as an alternative, but the approach is brittle since the 1043 * mapping needs to be updated in sync with all the changes to virtual memory 1044 * layout and related mapping facilities throughout the boot process. 1045 */ 1046 static void __init setup_cpuid_table(const struct cc_blob_sev_info *cc_info) 1047 { 1048 const struct snp_cpuid_table *cpuid_table_fw, *cpuid_table; 1049 int i; 1050 1051 if (!cc_info || !cc_info->cpuid_phys || cc_info->cpuid_len < PAGE_SIZE) 1052 sev_es_terminate(SEV_TERM_SET_LINUX, GHCB_TERM_CPUID); 1053 1054 cpuid_table_fw = (const struct snp_cpuid_table *)cc_info->cpuid_phys; 1055 if (!cpuid_table_fw->count || cpuid_table_fw->count > SNP_CPUID_COUNT_MAX) 1056 sev_es_terminate(SEV_TERM_SET_LINUX, GHCB_TERM_CPUID); 1057 1058 cpuid_table = snp_cpuid_get_table(); 1059 memcpy((void *)cpuid_table, cpuid_table_fw, sizeof(*cpuid_table)); 1060 1061 /* Initialize CPUID ranges for range-checking. */ 1062 for (i = 0; i < cpuid_table->count; i++) { 1063 const struct snp_cpuid_fn *fn = &cpuid_table->fn[i]; 1064 1065 if (fn->eax_in == 0x0) 1066 cpuid_std_range_max = fn->eax; 1067 else if (fn->eax_in == 0x40000000) 1068 cpuid_hyp_range_max = fn->eax; 1069 else if (fn->eax_in == 0x80000000) 1070 cpuid_ext_range_max = fn->eax; 1071 } 1072 } 1073 1074 static void pvalidate_pages(struct snp_psc_desc *desc) 1075 { 1076 struct psc_entry *e; 1077 unsigned long vaddr; 1078 unsigned int size; 1079 unsigned int i; 1080 bool validate; 1081 int rc; 1082 1083 for (i = 0; i <= desc->hdr.end_entry; i++) { 1084 e = &desc->entries[i]; 1085 1086 vaddr = (unsigned long)pfn_to_kaddr(e->gfn); 1087 size = e->pagesize ? RMP_PG_SIZE_2M : RMP_PG_SIZE_4K; 1088 validate = e->operation == SNP_PAGE_STATE_PRIVATE; 1089 1090 rc = pvalidate(vaddr, size, validate); 1091 if (rc == PVALIDATE_FAIL_SIZEMISMATCH && size == RMP_PG_SIZE_2M) { 1092 unsigned long vaddr_end = vaddr + PMD_SIZE; 1093 1094 for (; vaddr < vaddr_end; vaddr += PAGE_SIZE) { 1095 rc = pvalidate(vaddr, RMP_PG_SIZE_4K, validate); 1096 if (rc) 1097 break; 1098 } 1099 } 1100 1101 if (rc) { 1102 WARN(1, "Failed to validate address 0x%lx ret %d", vaddr, rc); 1103 sev_es_terminate(SEV_TERM_SET_LINUX, GHCB_TERM_PVALIDATE); 1104 } 1105 } 1106 } 1107 1108 static int vmgexit_psc(struct ghcb *ghcb, struct snp_psc_desc *desc) 1109 { 1110 int cur_entry, end_entry, ret = 0; 1111 struct snp_psc_desc *data; 1112 struct es_em_ctxt ctxt; 1113 1114 vc_ghcb_invalidate(ghcb); 1115 1116 /* Copy the input desc into GHCB shared buffer */ 1117 data = (struct snp_psc_desc *)ghcb->shared_buffer; 1118 memcpy(ghcb->shared_buffer, desc, min_t(int, GHCB_SHARED_BUF_SIZE, sizeof(*desc))); 1119 1120 /* 1121 * As per the GHCB specification, the hypervisor can resume the guest 1122 * before processing all the entries. Check whether all the entries 1123 * are processed. If not, then keep retrying. Note, the hypervisor 1124 * will update the data memory directly to indicate the status, so 1125 * reference the data->hdr everywhere. 1126 * 1127 * The strategy here is to wait for the hypervisor to change the page 1128 * state in the RMP table before guest accesses the memory pages. If the 1129 * page state change was not successful, then later memory access will 1130 * result in a crash. 1131 */ 1132 cur_entry = data->hdr.cur_entry; 1133 end_entry = data->hdr.end_entry; 1134 1135 while (data->hdr.cur_entry <= data->hdr.end_entry) { 1136 ghcb_set_sw_scratch(ghcb, (u64)__pa(data)); 1137 1138 /* This will advance the shared buffer data points to. */ 1139 ret = sev_es_ghcb_hv_call(ghcb, &ctxt, SVM_VMGEXIT_PSC, 0, 0); 1140 1141 /* 1142 * Page State Change VMGEXIT can pass error code through 1143 * exit_info_2. 1144 */ 1145 if (WARN(ret || ghcb->save.sw_exit_info_2, 1146 "SNP: PSC failed ret=%d exit_info_2=%llx\n", 1147 ret, ghcb->save.sw_exit_info_2)) { 1148 ret = 1; 1149 goto out; 1150 } 1151 1152 /* Verify that reserved bit is not set */ 1153 if (WARN(data->hdr.reserved, "Reserved bit is set in the PSC header\n")) { 1154 ret = 1; 1155 goto out; 1156 } 1157 1158 /* 1159 * Sanity check that entry processing is not going backwards. 1160 * This will happen only if hypervisor is tricking us. 1161 */ 1162 if (WARN(data->hdr.end_entry > end_entry || cur_entry > data->hdr.cur_entry, 1163 "SNP: PSC processing going backward, end_entry %d (got %d) cur_entry %d (got %d)\n", 1164 end_entry, data->hdr.end_entry, cur_entry, data->hdr.cur_entry)) { 1165 ret = 1; 1166 goto out; 1167 } 1168 } 1169 1170 out: 1171 return ret; 1172 } 1173