1 // SPDX-License-Identifier: GPL-2.0 2 /* Copyright (C) 2021-2022 Intel Corporation */ 3 4 #undef pr_fmt 5 #define pr_fmt(fmt) "tdx: " fmt 6 7 #include <linux/cpufeature.h> 8 #include <linux/export.h> 9 #include <linux/io.h> 10 #include <asm/coco.h> 11 #include <asm/tdx.h> 12 #include <asm/vmx.h> 13 #include <asm/ia32.h> 14 #include <asm/insn.h> 15 #include <asm/insn-eval.h> 16 #include <asm/pgtable.h> 17 18 /* MMIO direction */ 19 #define EPT_READ 0 20 #define EPT_WRITE 1 21 22 /* Port I/O direction */ 23 #define PORT_READ 0 24 #define PORT_WRITE 1 25 26 /* See Exit Qualification for I/O Instructions in VMX documentation */ 27 #define VE_IS_IO_IN(e) ((e) & BIT(3)) 28 #define VE_GET_IO_SIZE(e) (((e) & GENMASK(2, 0)) + 1) 29 #define VE_GET_PORT_NUM(e) ((e) >> 16) 30 #define VE_IS_IO_STRING(e) ((e) & BIT(4)) 31 32 #define ATTR_DEBUG BIT(0) 33 #define ATTR_SEPT_VE_DISABLE BIT(28) 34 35 /* TDX Module call error codes */ 36 #define TDCALL_RETURN_CODE(a) ((a) >> 32) 37 #define TDCALL_INVALID_OPERAND 0xc0000100 38 39 #define TDREPORT_SUBTYPE_0 0 40 41 /* Called from __tdx_hypercall() for unrecoverable failure */ 42 noinstr void __tdx_hypercall_failed(void) 43 { 44 instrumentation_begin(); 45 panic("TDVMCALL failed. TDX module bug?"); 46 } 47 48 #ifdef CONFIG_KVM_GUEST 49 long tdx_kvm_hypercall(unsigned int nr, unsigned long p1, unsigned long p2, 50 unsigned long p3, unsigned long p4) 51 { 52 struct tdx_hypercall_args args = { 53 .r10 = nr, 54 .r11 = p1, 55 .r12 = p2, 56 .r13 = p3, 57 .r14 = p4, 58 }; 59 60 return __tdx_hypercall(&args); 61 } 62 EXPORT_SYMBOL_GPL(tdx_kvm_hypercall); 63 #endif 64 65 /* 66 * Used for TDX guests to make calls directly to the TD module. This 67 * should only be used for calls that have no legitimate reason to fail 68 * or where the kernel can not survive the call failing. 69 */ 70 static inline void tdx_module_call(u64 fn, u64 rcx, u64 rdx, u64 r8, u64 r9, 71 struct tdx_module_output *out) 72 { 73 if (__tdx_module_call(fn, rcx, rdx, r8, r9, out)) 74 panic("TDCALL %lld failed (Buggy TDX module!)\n", fn); 75 } 76 77 /** 78 * tdx_mcall_get_report0() - Wrapper to get TDREPORT0 (a.k.a. TDREPORT 79 * subtype 0) using TDG.MR.REPORT TDCALL. 80 * @reportdata: Address of the input buffer which contains user-defined 81 * REPORTDATA to be included into TDREPORT. 82 * @tdreport: Address of the output buffer to store TDREPORT. 83 * 84 * Refer to section titled "TDG.MR.REPORT leaf" in the TDX Module 85 * v1.0 specification for more information on TDG.MR.REPORT TDCALL. 86 * It is used in the TDX guest driver module to get the TDREPORT0. 87 * 88 * Return 0 on success, -EINVAL for invalid operands, or -EIO on 89 * other TDCALL failures. 90 */ 91 int tdx_mcall_get_report0(u8 *reportdata, u8 *tdreport) 92 { 93 u64 ret; 94 95 ret = __tdx_module_call(TDX_GET_REPORT, virt_to_phys(tdreport), 96 virt_to_phys(reportdata), TDREPORT_SUBTYPE_0, 97 0, NULL); 98 if (ret) { 99 if (TDCALL_RETURN_CODE(ret) == TDCALL_INVALID_OPERAND) 100 return -EINVAL; 101 return -EIO; 102 } 103 104 return 0; 105 } 106 EXPORT_SYMBOL_GPL(tdx_mcall_get_report0); 107 108 static void __noreturn tdx_panic(const char *msg) 109 { 110 struct tdx_hypercall_args args = { 111 .r10 = TDX_HYPERCALL_STANDARD, 112 .r11 = TDVMCALL_REPORT_FATAL_ERROR, 113 .r12 = 0, /* Error code: 0 is Panic */ 114 }; 115 union { 116 /* Define register order according to the GHCI */ 117 struct { u64 r14, r15, rbx, rdi, rsi, r8, r9, rdx; }; 118 119 char str[64]; 120 } message; 121 122 /* VMM assumes '\0' in byte 65, if the message took all 64 bytes */ 123 strncpy(message.str, msg, 64); 124 125 args.r8 = message.r8; 126 args.r9 = message.r9; 127 args.r14 = message.r14; 128 args.r15 = message.r15; 129 args.rdi = message.rdi; 130 args.rsi = message.rsi; 131 args.rbx = message.rbx; 132 args.rdx = message.rdx; 133 134 /* 135 * This hypercall should never return and it is not safe 136 * to keep the guest running. Call it forever if it 137 * happens to return. 138 */ 139 while (1) 140 __tdx_hypercall(&args); 141 } 142 143 static void tdx_parse_tdinfo(u64 *cc_mask) 144 { 145 struct tdx_module_output out; 146 unsigned int gpa_width; 147 u64 td_attr; 148 149 /* 150 * TDINFO TDX module call is used to get the TD execution environment 151 * information like GPA width, number of available vcpus, debug mode 152 * information, etc. More details about the ABI can be found in TDX 153 * Guest-Host-Communication Interface (GHCI), section 2.4.2 TDCALL 154 * [TDG.VP.INFO]. 155 */ 156 tdx_module_call(TDX_GET_INFO, 0, 0, 0, 0, &out); 157 158 /* 159 * The highest bit of a guest physical address is the "sharing" bit. 160 * Set it for shared pages and clear it for private pages. 161 * 162 * The GPA width that comes out of this call is critical. TDX guests 163 * can not meaningfully run without it. 164 */ 165 gpa_width = out.rcx & GENMASK(5, 0); 166 *cc_mask = BIT_ULL(gpa_width - 1); 167 168 /* 169 * The kernel can not handle #VE's when accessing normal kernel 170 * memory. Ensure that no #VE will be delivered for accesses to 171 * TD-private memory. Only VMM-shared memory (MMIO) will #VE. 172 */ 173 td_attr = out.rdx; 174 if (!(td_attr & ATTR_SEPT_VE_DISABLE)) { 175 const char *msg = "TD misconfiguration: SEPT_VE_DISABLE attribute must be set."; 176 177 /* Relax SEPT_VE_DISABLE check for debug TD. */ 178 if (td_attr & ATTR_DEBUG) 179 pr_warn("%s\n", msg); 180 else 181 tdx_panic(msg); 182 } 183 } 184 185 /* 186 * The TDX module spec states that #VE may be injected for a limited set of 187 * reasons: 188 * 189 * - Emulation of the architectural #VE injection on EPT violation; 190 * 191 * - As a result of guest TD execution of a disallowed instruction, 192 * a disallowed MSR access, or CPUID virtualization; 193 * 194 * - A notification to the guest TD about anomalous behavior; 195 * 196 * The last one is opt-in and is not used by the kernel. 197 * 198 * The Intel Software Developer's Manual describes cases when instruction 199 * length field can be used in section "Information for VM Exits Due to 200 * Instruction Execution". 201 * 202 * For TDX, it ultimately means GET_VEINFO provides reliable instruction length 203 * information if #VE occurred due to instruction execution, but not for EPT 204 * violations. 205 */ 206 static int ve_instr_len(struct ve_info *ve) 207 { 208 switch (ve->exit_reason) { 209 case EXIT_REASON_HLT: 210 case EXIT_REASON_MSR_READ: 211 case EXIT_REASON_MSR_WRITE: 212 case EXIT_REASON_CPUID: 213 case EXIT_REASON_IO_INSTRUCTION: 214 /* It is safe to use ve->instr_len for #VE due instructions */ 215 return ve->instr_len; 216 case EXIT_REASON_EPT_VIOLATION: 217 /* 218 * For EPT violations, ve->insn_len is not defined. For those, 219 * the kernel must decode instructions manually and should not 220 * be using this function. 221 */ 222 WARN_ONCE(1, "ve->instr_len is not defined for EPT violations"); 223 return 0; 224 default: 225 WARN_ONCE(1, "Unexpected #VE-type: %lld\n", ve->exit_reason); 226 return ve->instr_len; 227 } 228 } 229 230 static u64 __cpuidle __halt(const bool irq_disabled) 231 { 232 struct tdx_hypercall_args args = { 233 .r10 = TDX_HYPERCALL_STANDARD, 234 .r11 = hcall_func(EXIT_REASON_HLT), 235 .r12 = irq_disabled, 236 }; 237 238 /* 239 * Emulate HLT operation via hypercall. More info about ABI 240 * can be found in TDX Guest-Host-Communication Interface 241 * (GHCI), section 3.8 TDG.VP.VMCALL<Instruction.HLT>. 242 * 243 * The VMM uses the "IRQ disabled" param to understand IRQ 244 * enabled status (RFLAGS.IF) of the TD guest and to determine 245 * whether or not it should schedule the halted vCPU if an 246 * IRQ becomes pending. E.g. if IRQs are disabled, the VMM 247 * can keep the vCPU in virtual HLT, even if an IRQ is 248 * pending, without hanging/breaking the guest. 249 */ 250 return __tdx_hypercall(&args); 251 } 252 253 static int handle_halt(struct ve_info *ve) 254 { 255 const bool irq_disabled = irqs_disabled(); 256 257 if (__halt(irq_disabled)) 258 return -EIO; 259 260 return ve_instr_len(ve); 261 } 262 263 void __cpuidle tdx_safe_halt(void) 264 { 265 const bool irq_disabled = false; 266 267 /* 268 * Use WARN_ONCE() to report the failure. 269 */ 270 if (__halt(irq_disabled)) 271 WARN_ONCE(1, "HLT instruction emulation failed\n"); 272 } 273 274 static int read_msr(struct pt_regs *regs, struct ve_info *ve) 275 { 276 struct tdx_hypercall_args args = { 277 .r10 = TDX_HYPERCALL_STANDARD, 278 .r11 = hcall_func(EXIT_REASON_MSR_READ), 279 .r12 = regs->cx, 280 }; 281 282 /* 283 * Emulate the MSR read via hypercall. More info about ABI 284 * can be found in TDX Guest-Host-Communication Interface 285 * (GHCI), section titled "TDG.VP.VMCALL<Instruction.RDMSR>". 286 */ 287 if (__tdx_hypercall_ret(&args)) 288 return -EIO; 289 290 regs->ax = lower_32_bits(args.r11); 291 regs->dx = upper_32_bits(args.r11); 292 return ve_instr_len(ve); 293 } 294 295 static int write_msr(struct pt_regs *regs, struct ve_info *ve) 296 { 297 struct tdx_hypercall_args args = { 298 .r10 = TDX_HYPERCALL_STANDARD, 299 .r11 = hcall_func(EXIT_REASON_MSR_WRITE), 300 .r12 = regs->cx, 301 .r13 = (u64)regs->dx << 32 | regs->ax, 302 }; 303 304 /* 305 * Emulate the MSR write via hypercall. More info about ABI 306 * can be found in TDX Guest-Host-Communication Interface 307 * (GHCI) section titled "TDG.VP.VMCALL<Instruction.WRMSR>". 308 */ 309 if (__tdx_hypercall(&args)) 310 return -EIO; 311 312 return ve_instr_len(ve); 313 } 314 315 static int handle_cpuid(struct pt_regs *regs, struct ve_info *ve) 316 { 317 struct tdx_hypercall_args args = { 318 .r10 = TDX_HYPERCALL_STANDARD, 319 .r11 = hcall_func(EXIT_REASON_CPUID), 320 .r12 = regs->ax, 321 .r13 = regs->cx, 322 }; 323 324 /* 325 * Only allow VMM to control range reserved for hypervisor 326 * communication. 327 * 328 * Return all-zeros for any CPUID outside the range. It matches CPU 329 * behaviour for non-supported leaf. 330 */ 331 if (regs->ax < 0x40000000 || regs->ax > 0x4FFFFFFF) { 332 regs->ax = regs->bx = regs->cx = regs->dx = 0; 333 return ve_instr_len(ve); 334 } 335 336 /* 337 * Emulate the CPUID instruction via a hypercall. More info about 338 * ABI can be found in TDX Guest-Host-Communication Interface 339 * (GHCI), section titled "VP.VMCALL<Instruction.CPUID>". 340 */ 341 if (__tdx_hypercall_ret(&args)) 342 return -EIO; 343 344 /* 345 * As per TDX GHCI CPUID ABI, r12-r15 registers contain contents of 346 * EAX, EBX, ECX, EDX registers after the CPUID instruction execution. 347 * So copy the register contents back to pt_regs. 348 */ 349 regs->ax = args.r12; 350 regs->bx = args.r13; 351 regs->cx = args.r14; 352 regs->dx = args.r15; 353 354 return ve_instr_len(ve); 355 } 356 357 static bool mmio_read(int size, unsigned long addr, unsigned long *val) 358 { 359 struct tdx_hypercall_args args = { 360 .r10 = TDX_HYPERCALL_STANDARD, 361 .r11 = hcall_func(EXIT_REASON_EPT_VIOLATION), 362 .r12 = size, 363 .r13 = EPT_READ, 364 .r14 = addr, 365 .r15 = *val, 366 }; 367 368 if (__tdx_hypercall_ret(&args)) 369 return false; 370 *val = args.r11; 371 return true; 372 } 373 374 static bool mmio_write(int size, unsigned long addr, unsigned long val) 375 { 376 return !_tdx_hypercall(hcall_func(EXIT_REASON_EPT_VIOLATION), size, 377 EPT_WRITE, addr, val); 378 } 379 380 static int handle_mmio(struct pt_regs *regs, struct ve_info *ve) 381 { 382 unsigned long *reg, val, vaddr; 383 char buffer[MAX_INSN_SIZE]; 384 enum insn_mmio_type mmio; 385 struct insn insn = {}; 386 int size, extend_size; 387 u8 extend_val = 0; 388 389 /* Only in-kernel MMIO is supported */ 390 if (WARN_ON_ONCE(user_mode(regs))) 391 return -EFAULT; 392 393 if (copy_from_kernel_nofault(buffer, (void *)regs->ip, MAX_INSN_SIZE)) 394 return -EFAULT; 395 396 if (insn_decode(&insn, buffer, MAX_INSN_SIZE, INSN_MODE_64)) 397 return -EINVAL; 398 399 mmio = insn_decode_mmio(&insn, &size); 400 if (WARN_ON_ONCE(mmio == INSN_MMIO_DECODE_FAILED)) 401 return -EINVAL; 402 403 if (mmio != INSN_MMIO_WRITE_IMM && mmio != INSN_MMIO_MOVS) { 404 reg = insn_get_modrm_reg_ptr(&insn, regs); 405 if (!reg) 406 return -EINVAL; 407 } 408 409 /* 410 * Reject EPT violation #VEs that split pages. 411 * 412 * MMIO accesses are supposed to be naturally aligned and therefore 413 * never cross page boundaries. Seeing split page accesses indicates 414 * a bug or a load_unaligned_zeropad() that stepped into an MMIO page. 415 * 416 * load_unaligned_zeropad() will recover using exception fixups. 417 */ 418 vaddr = (unsigned long)insn_get_addr_ref(&insn, regs); 419 if (vaddr / PAGE_SIZE != (vaddr + size - 1) / PAGE_SIZE) 420 return -EFAULT; 421 422 /* Handle writes first */ 423 switch (mmio) { 424 case INSN_MMIO_WRITE: 425 memcpy(&val, reg, size); 426 if (!mmio_write(size, ve->gpa, val)) 427 return -EIO; 428 return insn.length; 429 case INSN_MMIO_WRITE_IMM: 430 val = insn.immediate.value; 431 if (!mmio_write(size, ve->gpa, val)) 432 return -EIO; 433 return insn.length; 434 case INSN_MMIO_READ: 435 case INSN_MMIO_READ_ZERO_EXTEND: 436 case INSN_MMIO_READ_SIGN_EXTEND: 437 /* Reads are handled below */ 438 break; 439 case INSN_MMIO_MOVS: 440 case INSN_MMIO_DECODE_FAILED: 441 /* 442 * MMIO was accessed with an instruction that could not be 443 * decoded or handled properly. It was likely not using io.h 444 * helpers or accessed MMIO accidentally. 445 */ 446 return -EINVAL; 447 default: 448 WARN_ONCE(1, "Unknown insn_decode_mmio() decode value?"); 449 return -EINVAL; 450 } 451 452 /* Handle reads */ 453 if (!mmio_read(size, ve->gpa, &val)) 454 return -EIO; 455 456 switch (mmio) { 457 case INSN_MMIO_READ: 458 /* Zero-extend for 32-bit operation */ 459 extend_size = size == 4 ? sizeof(*reg) : 0; 460 break; 461 case INSN_MMIO_READ_ZERO_EXTEND: 462 /* Zero extend based on operand size */ 463 extend_size = insn.opnd_bytes; 464 break; 465 case INSN_MMIO_READ_SIGN_EXTEND: 466 /* Sign extend based on operand size */ 467 extend_size = insn.opnd_bytes; 468 if (size == 1 && val & BIT(7)) 469 extend_val = 0xFF; 470 else if (size > 1 && val & BIT(15)) 471 extend_val = 0xFF; 472 break; 473 default: 474 /* All other cases has to be covered with the first switch() */ 475 WARN_ON_ONCE(1); 476 return -EINVAL; 477 } 478 479 if (extend_size) 480 memset(reg, extend_val, extend_size); 481 memcpy(reg, &val, size); 482 return insn.length; 483 } 484 485 static bool handle_in(struct pt_regs *regs, int size, int port) 486 { 487 struct tdx_hypercall_args args = { 488 .r10 = TDX_HYPERCALL_STANDARD, 489 .r11 = hcall_func(EXIT_REASON_IO_INSTRUCTION), 490 .r12 = size, 491 .r13 = PORT_READ, 492 .r14 = port, 493 }; 494 u64 mask = GENMASK(BITS_PER_BYTE * size, 0); 495 bool success; 496 497 /* 498 * Emulate the I/O read via hypercall. More info about ABI can be found 499 * in TDX Guest-Host-Communication Interface (GHCI) section titled 500 * "TDG.VP.VMCALL<Instruction.IO>". 501 */ 502 success = !__tdx_hypercall_ret(&args); 503 504 /* Update part of the register affected by the emulated instruction */ 505 regs->ax &= ~mask; 506 if (success) 507 regs->ax |= args.r11 & mask; 508 509 return success; 510 } 511 512 static bool handle_out(struct pt_regs *regs, int size, int port) 513 { 514 u64 mask = GENMASK(BITS_PER_BYTE * size, 0); 515 516 /* 517 * Emulate the I/O write via hypercall. More info about ABI can be found 518 * in TDX Guest-Host-Communication Interface (GHCI) section titled 519 * "TDG.VP.VMCALL<Instruction.IO>". 520 */ 521 return !_tdx_hypercall(hcall_func(EXIT_REASON_IO_INSTRUCTION), size, 522 PORT_WRITE, port, regs->ax & mask); 523 } 524 525 /* 526 * Emulate I/O using hypercall. 527 * 528 * Assumes the IO instruction was using ax, which is enforced 529 * by the standard io.h macros. 530 * 531 * Return True on success or False on failure. 532 */ 533 static int handle_io(struct pt_regs *regs, struct ve_info *ve) 534 { 535 u32 exit_qual = ve->exit_qual; 536 int size, port; 537 bool in, ret; 538 539 if (VE_IS_IO_STRING(exit_qual)) 540 return -EIO; 541 542 in = VE_IS_IO_IN(exit_qual); 543 size = VE_GET_IO_SIZE(exit_qual); 544 port = VE_GET_PORT_NUM(exit_qual); 545 546 547 if (in) 548 ret = handle_in(regs, size, port); 549 else 550 ret = handle_out(regs, size, port); 551 if (!ret) 552 return -EIO; 553 554 return ve_instr_len(ve); 555 } 556 557 /* 558 * Early #VE exception handler. Only handles a subset of port I/O. 559 * Intended only for earlyprintk. If failed, return false. 560 */ 561 __init bool tdx_early_handle_ve(struct pt_regs *regs) 562 { 563 struct ve_info ve; 564 int insn_len; 565 566 tdx_get_ve_info(&ve); 567 568 if (ve.exit_reason != EXIT_REASON_IO_INSTRUCTION) 569 return false; 570 571 insn_len = handle_io(regs, &ve); 572 if (insn_len < 0) 573 return false; 574 575 regs->ip += insn_len; 576 return true; 577 } 578 579 void tdx_get_ve_info(struct ve_info *ve) 580 { 581 struct tdx_module_output out; 582 583 /* 584 * Called during #VE handling to retrieve the #VE info from the 585 * TDX module. 586 * 587 * This has to be called early in #VE handling. A "nested" #VE which 588 * occurs before this will raise a #DF and is not recoverable. 589 * 590 * The call retrieves the #VE info from the TDX module, which also 591 * clears the "#VE valid" flag. This must be done before anything else 592 * because any #VE that occurs while the valid flag is set will lead to 593 * #DF. 594 * 595 * Note, the TDX module treats virtual NMIs as inhibited if the #VE 596 * valid flag is set. It means that NMI=>#VE will not result in a #DF. 597 */ 598 tdx_module_call(TDX_GET_VEINFO, 0, 0, 0, 0, &out); 599 600 /* Transfer the output parameters */ 601 ve->exit_reason = out.rcx; 602 ve->exit_qual = out.rdx; 603 ve->gla = out.r8; 604 ve->gpa = out.r9; 605 ve->instr_len = lower_32_bits(out.r10); 606 ve->instr_info = upper_32_bits(out.r10); 607 } 608 609 /* 610 * Handle the user initiated #VE. 611 * 612 * On success, returns the number of bytes RIP should be incremented (>=0) 613 * or -errno on error. 614 */ 615 static int virt_exception_user(struct pt_regs *regs, struct ve_info *ve) 616 { 617 switch (ve->exit_reason) { 618 case EXIT_REASON_CPUID: 619 return handle_cpuid(regs, ve); 620 default: 621 pr_warn("Unexpected #VE: %lld\n", ve->exit_reason); 622 return -EIO; 623 } 624 } 625 626 static inline bool is_private_gpa(u64 gpa) 627 { 628 return gpa == cc_mkenc(gpa); 629 } 630 631 /* 632 * Handle the kernel #VE. 633 * 634 * On success, returns the number of bytes RIP should be incremented (>=0) 635 * or -errno on error. 636 */ 637 static int virt_exception_kernel(struct pt_regs *regs, struct ve_info *ve) 638 { 639 switch (ve->exit_reason) { 640 case EXIT_REASON_HLT: 641 return handle_halt(ve); 642 case EXIT_REASON_MSR_READ: 643 return read_msr(regs, ve); 644 case EXIT_REASON_MSR_WRITE: 645 return write_msr(regs, ve); 646 case EXIT_REASON_CPUID: 647 return handle_cpuid(regs, ve); 648 case EXIT_REASON_EPT_VIOLATION: 649 if (is_private_gpa(ve->gpa)) 650 panic("Unexpected EPT-violation on private memory."); 651 return handle_mmio(regs, ve); 652 case EXIT_REASON_IO_INSTRUCTION: 653 return handle_io(regs, ve); 654 default: 655 pr_warn("Unexpected #VE: %lld\n", ve->exit_reason); 656 return -EIO; 657 } 658 } 659 660 bool tdx_handle_virt_exception(struct pt_regs *regs, struct ve_info *ve) 661 { 662 int insn_len; 663 664 if (user_mode(regs)) 665 insn_len = virt_exception_user(regs, ve); 666 else 667 insn_len = virt_exception_kernel(regs, ve); 668 if (insn_len < 0) 669 return false; 670 671 /* After successful #VE handling, move the IP */ 672 regs->ip += insn_len; 673 674 return true; 675 } 676 677 static bool tdx_tlb_flush_required(bool private) 678 { 679 /* 680 * TDX guest is responsible for flushing TLB on private->shared 681 * transition. VMM is responsible for flushing on shared->private. 682 * 683 * The VMM _can't_ flush private addresses as it can't generate PAs 684 * with the guest's HKID. Shared memory isn't subject to integrity 685 * checking, i.e. the VMM doesn't need to flush for its own protection. 686 * 687 * There's no need to flush when converting from shared to private, 688 * as flushing is the VMM's responsibility in this case, e.g. it must 689 * flush to avoid integrity failures in the face of a buggy or 690 * malicious guest. 691 */ 692 return !private; 693 } 694 695 static bool tdx_cache_flush_required(void) 696 { 697 /* 698 * AMD SME/SEV can avoid cache flushing if HW enforces cache coherence. 699 * TDX doesn't have such capability. 700 * 701 * Flush cache unconditionally. 702 */ 703 return true; 704 } 705 706 /* 707 * Inform the VMM of the guest's intent for this physical page: shared with 708 * the VMM or private to the guest. The VMM is expected to change its mapping 709 * of the page in response. 710 */ 711 static bool tdx_enc_status_changed(unsigned long vaddr, int numpages, bool enc) 712 { 713 phys_addr_t start = __pa(vaddr); 714 phys_addr_t end = __pa(vaddr + numpages * PAGE_SIZE); 715 716 if (!enc) { 717 /* Set the shared (decrypted) bits: */ 718 start |= cc_mkdec(0); 719 end |= cc_mkdec(0); 720 } 721 722 /* 723 * Notify the VMM about page mapping conversion. More info about ABI 724 * can be found in TDX Guest-Host-Communication Interface (GHCI), 725 * section "TDG.VP.VMCALL<MapGPA>" 726 */ 727 if (_tdx_hypercall(TDVMCALL_MAP_GPA, start, end - start, 0, 0)) 728 return false; 729 730 /* shared->private conversion requires memory to be accepted before use */ 731 if (enc) 732 return tdx_accept_memory(start, end); 733 734 return true; 735 } 736 737 static bool tdx_enc_status_change_prepare(unsigned long vaddr, int numpages, 738 bool enc) 739 { 740 /* 741 * Only handle shared->private conversion here. 742 * See the comment in tdx_early_init(). 743 */ 744 if (enc) 745 return tdx_enc_status_changed(vaddr, numpages, enc); 746 return true; 747 } 748 749 static bool tdx_enc_status_change_finish(unsigned long vaddr, int numpages, 750 bool enc) 751 { 752 /* 753 * Only handle private->shared conversion here. 754 * See the comment in tdx_early_init(). 755 */ 756 if (!enc) 757 return tdx_enc_status_changed(vaddr, numpages, enc); 758 return true; 759 } 760 761 void __init tdx_early_init(void) 762 { 763 u64 cc_mask; 764 u32 eax, sig[3]; 765 766 cpuid_count(TDX_CPUID_LEAF_ID, 0, &eax, &sig[0], &sig[2], &sig[1]); 767 768 if (memcmp(TDX_IDENT, sig, sizeof(sig))) 769 return; 770 771 setup_force_cpu_cap(X86_FEATURE_TDX_GUEST); 772 773 cc_vendor = CC_VENDOR_INTEL; 774 tdx_parse_tdinfo(&cc_mask); 775 cc_set_mask(cc_mask); 776 777 /* Kernel does not use NOTIFY_ENABLES and does not need random #VEs */ 778 tdx_module_call(TDX_WR, 0, TDCS_NOTIFY_ENABLES, 0, -1ULL, NULL); 779 780 /* 781 * All bits above GPA width are reserved and kernel treats shared bit 782 * as flag, not as part of physical address. 783 * 784 * Adjust physical mask to only cover valid GPA bits. 785 */ 786 physical_mask &= cc_mask - 1; 787 788 /* 789 * The kernel mapping should match the TDX metadata for the page. 790 * load_unaligned_zeropad() can touch memory *adjacent* to that which is 791 * owned by the caller and can catch even _momentary_ mismatches. Bad 792 * things happen on mismatch: 793 * 794 * - Private mapping => Shared Page == Guest shutdown 795 * - Shared mapping => Private Page == Recoverable #VE 796 * 797 * guest.enc_status_change_prepare() converts the page from 798 * shared=>private before the mapping becomes private. 799 * 800 * guest.enc_status_change_finish() converts the page from 801 * private=>shared after the mapping becomes private. 802 * 803 * In both cases there is a temporary shared mapping to a private page, 804 * which can result in a #VE. But, there is never a private mapping to 805 * a shared page. 806 */ 807 x86_platform.guest.enc_status_change_prepare = tdx_enc_status_change_prepare; 808 x86_platform.guest.enc_status_change_finish = tdx_enc_status_change_finish; 809 810 x86_platform.guest.enc_cache_flush_required = tdx_cache_flush_required; 811 x86_platform.guest.enc_tlb_flush_required = tdx_tlb_flush_required; 812 813 /* 814 * TDX intercepts the RDMSR to read the X2APIC ID in the parallel 815 * bringup low level code. That raises #VE which cannot be handled 816 * there. 817 * 818 * Intel-TDX has a secure RDMSR hypercall, but that needs to be 819 * implemented seperately in the low level startup ASM code. 820 * Until that is in place, disable parallel bringup for TDX. 821 */ 822 x86_cpuinit.parallel_bringup = false; 823 824 pr_info("Guest detected\n"); 825 } 826