1 // SPDX-License-Identifier: GPL-2.0
2 /* Copyright (C) 2021-2022 Intel Corporation */
3
4 #undef pr_fmt
5 #define pr_fmt(fmt) "tdx: " fmt
6
7 #include <linux/cpufeature.h>
8 #include <linux/export.h>
9 #include <linux/io.h>
10 #include <asm/coco.h>
11 #include <asm/tdx.h>
12 #include <asm/vmx.h>
13 #include <asm/ia32.h>
14 #include <asm/insn.h>
15 #include <asm/insn-eval.h>
16 #include <asm/pgtable.h>
17
18 /* MMIO direction */
19 #define EPT_READ 0
20 #define EPT_WRITE 1
21
22 /* Port I/O direction */
23 #define PORT_READ 0
24 #define PORT_WRITE 1
25
26 /* See Exit Qualification for I/O Instructions in VMX documentation */
27 #define VE_IS_IO_IN(e) ((e) & BIT(3))
28 #define VE_GET_IO_SIZE(e) (((e) & GENMASK(2, 0)) + 1)
29 #define VE_GET_PORT_NUM(e) ((e) >> 16)
30 #define VE_IS_IO_STRING(e) ((e) & BIT(4))
31
32 #define ATTR_DEBUG BIT(0)
33 #define ATTR_SEPT_VE_DISABLE BIT(28)
34
35 /* TDX Module call error codes */
36 #define TDCALL_RETURN_CODE(a) ((a) >> 32)
37 #define TDCALL_INVALID_OPERAND 0xc0000100
38
39 #define TDREPORT_SUBTYPE_0 0
40
41 /* Called from __tdx_hypercall() for unrecoverable failure */
__tdx_hypercall_failed(void)42 noinstr void __tdx_hypercall_failed(void)
43 {
44 instrumentation_begin();
45 panic("TDVMCALL failed. TDX module bug?");
46 }
47
48 #ifdef CONFIG_KVM_GUEST
tdx_kvm_hypercall(unsigned int nr,unsigned long p1,unsigned long p2,unsigned long p3,unsigned long p4)49 long tdx_kvm_hypercall(unsigned int nr, unsigned long p1, unsigned long p2,
50 unsigned long p3, unsigned long p4)
51 {
52 struct tdx_hypercall_args args = {
53 .r10 = nr,
54 .r11 = p1,
55 .r12 = p2,
56 .r13 = p3,
57 .r14 = p4,
58 };
59
60 return __tdx_hypercall(&args);
61 }
62 EXPORT_SYMBOL_GPL(tdx_kvm_hypercall);
63 #endif
64
65 /*
66 * Used for TDX guests to make calls directly to the TD module. This
67 * should only be used for calls that have no legitimate reason to fail
68 * or where the kernel can not survive the call failing.
69 */
tdx_module_call(u64 fn,u64 rcx,u64 rdx,u64 r8,u64 r9,struct tdx_module_output * out)70 static inline void tdx_module_call(u64 fn, u64 rcx, u64 rdx, u64 r8, u64 r9,
71 struct tdx_module_output *out)
72 {
73 if (__tdx_module_call(fn, rcx, rdx, r8, r9, out))
74 panic("TDCALL %lld failed (Buggy TDX module!)\n", fn);
75 }
76
77 /**
78 * tdx_mcall_get_report0() - Wrapper to get TDREPORT0 (a.k.a. TDREPORT
79 * subtype 0) using TDG.MR.REPORT TDCALL.
80 * @reportdata: Address of the input buffer which contains user-defined
81 * REPORTDATA to be included into TDREPORT.
82 * @tdreport: Address of the output buffer to store TDREPORT.
83 *
84 * Refer to section titled "TDG.MR.REPORT leaf" in the TDX Module
85 * v1.0 specification for more information on TDG.MR.REPORT TDCALL.
86 * It is used in the TDX guest driver module to get the TDREPORT0.
87 *
88 * Return 0 on success, -EINVAL for invalid operands, or -EIO on
89 * other TDCALL failures.
90 */
tdx_mcall_get_report0(u8 * reportdata,u8 * tdreport)91 int tdx_mcall_get_report0(u8 *reportdata, u8 *tdreport)
92 {
93 u64 ret;
94
95 ret = __tdx_module_call(TDX_GET_REPORT, virt_to_phys(tdreport),
96 virt_to_phys(reportdata), TDREPORT_SUBTYPE_0,
97 0, NULL);
98 if (ret) {
99 if (TDCALL_RETURN_CODE(ret) == TDCALL_INVALID_OPERAND)
100 return -EINVAL;
101 return -EIO;
102 }
103
104 return 0;
105 }
106 EXPORT_SYMBOL_GPL(tdx_mcall_get_report0);
107
tdx_panic(const char * msg)108 static void __noreturn tdx_panic(const char *msg)
109 {
110 struct tdx_hypercall_args args = {
111 .r10 = TDX_HYPERCALL_STANDARD,
112 .r11 = TDVMCALL_REPORT_FATAL_ERROR,
113 .r12 = 0, /* Error code: 0 is Panic */
114 };
115 union {
116 /* Define register order according to the GHCI */
117 struct { u64 r14, r15, rbx, rdi, rsi, r8, r9, rdx; };
118
119 char str[64];
120 } message;
121
122 /* VMM assumes '\0' in byte 65, if the message took all 64 bytes */
123 strncpy(message.str, msg, 64);
124
125 args.r8 = message.r8;
126 args.r9 = message.r9;
127 args.r14 = message.r14;
128 args.r15 = message.r15;
129 args.rdi = message.rdi;
130 args.rsi = message.rsi;
131 args.rbx = message.rbx;
132 args.rdx = message.rdx;
133
134 /*
135 * This hypercall should never return and it is not safe
136 * to keep the guest running. Call it forever if it
137 * happens to return.
138 */
139 while (1)
140 __tdx_hypercall(&args);
141 }
142
tdx_parse_tdinfo(u64 * cc_mask)143 static void tdx_parse_tdinfo(u64 *cc_mask)
144 {
145 struct tdx_module_output out;
146 unsigned int gpa_width;
147 u64 td_attr;
148
149 /*
150 * TDINFO TDX module call is used to get the TD execution environment
151 * information like GPA width, number of available vcpus, debug mode
152 * information, etc. More details about the ABI can be found in TDX
153 * Guest-Host-Communication Interface (GHCI), section 2.4.2 TDCALL
154 * [TDG.VP.INFO].
155 */
156 tdx_module_call(TDX_GET_INFO, 0, 0, 0, 0, &out);
157
158 /*
159 * The highest bit of a guest physical address is the "sharing" bit.
160 * Set it for shared pages and clear it for private pages.
161 *
162 * The GPA width that comes out of this call is critical. TDX guests
163 * can not meaningfully run without it.
164 */
165 gpa_width = out.rcx & GENMASK(5, 0);
166 *cc_mask = BIT_ULL(gpa_width - 1);
167
168 /*
169 * The kernel can not handle #VE's when accessing normal kernel
170 * memory. Ensure that no #VE will be delivered for accesses to
171 * TD-private memory. Only VMM-shared memory (MMIO) will #VE.
172 */
173 td_attr = out.rdx;
174 if (!(td_attr & ATTR_SEPT_VE_DISABLE)) {
175 const char *msg = "TD misconfiguration: SEPT_VE_DISABLE attribute must be set.";
176
177 /* Relax SEPT_VE_DISABLE check for debug TD. */
178 if (td_attr & ATTR_DEBUG)
179 pr_warn("%s\n", msg);
180 else
181 tdx_panic(msg);
182 }
183 }
184
185 /*
186 * The TDX module spec states that #VE may be injected for a limited set of
187 * reasons:
188 *
189 * - Emulation of the architectural #VE injection on EPT violation;
190 *
191 * - As a result of guest TD execution of a disallowed instruction,
192 * a disallowed MSR access, or CPUID virtualization;
193 *
194 * - A notification to the guest TD about anomalous behavior;
195 *
196 * The last one is opt-in and is not used by the kernel.
197 *
198 * The Intel Software Developer's Manual describes cases when instruction
199 * length field can be used in section "Information for VM Exits Due to
200 * Instruction Execution".
201 *
202 * For TDX, it ultimately means GET_VEINFO provides reliable instruction length
203 * information if #VE occurred due to instruction execution, but not for EPT
204 * violations.
205 */
ve_instr_len(struct ve_info * ve)206 static int ve_instr_len(struct ve_info *ve)
207 {
208 switch (ve->exit_reason) {
209 case EXIT_REASON_HLT:
210 case EXIT_REASON_MSR_READ:
211 case EXIT_REASON_MSR_WRITE:
212 case EXIT_REASON_CPUID:
213 case EXIT_REASON_IO_INSTRUCTION:
214 /* It is safe to use ve->instr_len for #VE due instructions */
215 return ve->instr_len;
216 case EXIT_REASON_EPT_VIOLATION:
217 /*
218 * For EPT violations, ve->insn_len is not defined. For those,
219 * the kernel must decode instructions manually and should not
220 * be using this function.
221 */
222 WARN_ONCE(1, "ve->instr_len is not defined for EPT violations");
223 return 0;
224 default:
225 WARN_ONCE(1, "Unexpected #VE-type: %lld\n", ve->exit_reason);
226 return ve->instr_len;
227 }
228 }
229
__halt(const bool irq_disabled)230 static u64 __cpuidle __halt(const bool irq_disabled)
231 {
232 struct tdx_hypercall_args args = {
233 .r10 = TDX_HYPERCALL_STANDARD,
234 .r11 = hcall_func(EXIT_REASON_HLT),
235 .r12 = irq_disabled,
236 };
237
238 /*
239 * Emulate HLT operation via hypercall. More info about ABI
240 * can be found in TDX Guest-Host-Communication Interface
241 * (GHCI), section 3.8 TDG.VP.VMCALL<Instruction.HLT>.
242 *
243 * The VMM uses the "IRQ disabled" param to understand IRQ
244 * enabled status (RFLAGS.IF) of the TD guest and to determine
245 * whether or not it should schedule the halted vCPU if an
246 * IRQ becomes pending. E.g. if IRQs are disabled, the VMM
247 * can keep the vCPU in virtual HLT, even if an IRQ is
248 * pending, without hanging/breaking the guest.
249 */
250 return __tdx_hypercall(&args);
251 }
252
handle_halt(struct ve_info * ve)253 static int handle_halt(struct ve_info *ve)
254 {
255 const bool irq_disabled = irqs_disabled();
256
257 if (__halt(irq_disabled))
258 return -EIO;
259
260 return ve_instr_len(ve);
261 }
262
tdx_safe_halt(void)263 void __cpuidle tdx_safe_halt(void)
264 {
265 const bool irq_disabled = false;
266
267 /*
268 * Use WARN_ONCE() to report the failure.
269 */
270 if (__halt(irq_disabled))
271 WARN_ONCE(1, "HLT instruction emulation failed\n");
272 }
273
read_msr(struct pt_regs * regs,struct ve_info * ve)274 static int read_msr(struct pt_regs *regs, struct ve_info *ve)
275 {
276 struct tdx_hypercall_args args = {
277 .r10 = TDX_HYPERCALL_STANDARD,
278 .r11 = hcall_func(EXIT_REASON_MSR_READ),
279 .r12 = regs->cx,
280 };
281
282 /*
283 * Emulate the MSR read via hypercall. More info about ABI
284 * can be found in TDX Guest-Host-Communication Interface
285 * (GHCI), section titled "TDG.VP.VMCALL<Instruction.RDMSR>".
286 */
287 if (__tdx_hypercall_ret(&args))
288 return -EIO;
289
290 regs->ax = lower_32_bits(args.r11);
291 regs->dx = upper_32_bits(args.r11);
292 return ve_instr_len(ve);
293 }
294
write_msr(struct pt_regs * regs,struct ve_info * ve)295 static int write_msr(struct pt_regs *regs, struct ve_info *ve)
296 {
297 struct tdx_hypercall_args args = {
298 .r10 = TDX_HYPERCALL_STANDARD,
299 .r11 = hcall_func(EXIT_REASON_MSR_WRITE),
300 .r12 = regs->cx,
301 .r13 = (u64)regs->dx << 32 | regs->ax,
302 };
303
304 /*
305 * Emulate the MSR write via hypercall. More info about ABI
306 * can be found in TDX Guest-Host-Communication Interface
307 * (GHCI) section titled "TDG.VP.VMCALL<Instruction.WRMSR>".
308 */
309 if (__tdx_hypercall(&args))
310 return -EIO;
311
312 return ve_instr_len(ve);
313 }
314
handle_cpuid(struct pt_regs * regs,struct ve_info * ve)315 static int handle_cpuid(struct pt_regs *regs, struct ve_info *ve)
316 {
317 struct tdx_hypercall_args args = {
318 .r10 = TDX_HYPERCALL_STANDARD,
319 .r11 = hcall_func(EXIT_REASON_CPUID),
320 .r12 = regs->ax,
321 .r13 = regs->cx,
322 };
323
324 /*
325 * Only allow VMM to control range reserved for hypervisor
326 * communication.
327 *
328 * Return all-zeros for any CPUID outside the range. It matches CPU
329 * behaviour for non-supported leaf.
330 */
331 if (regs->ax < 0x40000000 || regs->ax > 0x4FFFFFFF) {
332 regs->ax = regs->bx = regs->cx = regs->dx = 0;
333 return ve_instr_len(ve);
334 }
335
336 /*
337 * Emulate the CPUID instruction via a hypercall. More info about
338 * ABI can be found in TDX Guest-Host-Communication Interface
339 * (GHCI), section titled "VP.VMCALL<Instruction.CPUID>".
340 */
341 if (__tdx_hypercall_ret(&args))
342 return -EIO;
343
344 /*
345 * As per TDX GHCI CPUID ABI, r12-r15 registers contain contents of
346 * EAX, EBX, ECX, EDX registers after the CPUID instruction execution.
347 * So copy the register contents back to pt_regs.
348 */
349 regs->ax = args.r12;
350 regs->bx = args.r13;
351 regs->cx = args.r14;
352 regs->dx = args.r15;
353
354 return ve_instr_len(ve);
355 }
356
mmio_read(int size,unsigned long addr,unsigned long * val)357 static bool mmio_read(int size, unsigned long addr, unsigned long *val)
358 {
359 struct tdx_hypercall_args args = {
360 .r10 = TDX_HYPERCALL_STANDARD,
361 .r11 = hcall_func(EXIT_REASON_EPT_VIOLATION),
362 .r12 = size,
363 .r13 = EPT_READ,
364 .r14 = addr,
365 .r15 = *val,
366 };
367
368 if (__tdx_hypercall_ret(&args))
369 return false;
370 *val = args.r11;
371 return true;
372 }
373
mmio_write(int size,unsigned long addr,unsigned long val)374 static bool mmio_write(int size, unsigned long addr, unsigned long val)
375 {
376 return !_tdx_hypercall(hcall_func(EXIT_REASON_EPT_VIOLATION), size,
377 EPT_WRITE, addr, val);
378 }
379
handle_mmio(struct pt_regs * regs,struct ve_info * ve)380 static int handle_mmio(struct pt_regs *regs, struct ve_info *ve)
381 {
382 unsigned long *reg, val, vaddr;
383 char buffer[MAX_INSN_SIZE];
384 enum insn_mmio_type mmio;
385 struct insn insn = {};
386 int size, extend_size;
387 u8 extend_val = 0;
388
389 /* Only in-kernel MMIO is supported */
390 if (WARN_ON_ONCE(user_mode(regs)))
391 return -EFAULT;
392
393 if (copy_from_kernel_nofault(buffer, (void *)regs->ip, MAX_INSN_SIZE))
394 return -EFAULT;
395
396 if (insn_decode(&insn, buffer, MAX_INSN_SIZE, INSN_MODE_64))
397 return -EINVAL;
398
399 mmio = insn_decode_mmio(&insn, &size);
400 if (WARN_ON_ONCE(mmio == INSN_MMIO_DECODE_FAILED))
401 return -EINVAL;
402
403 if (mmio != INSN_MMIO_WRITE_IMM && mmio != INSN_MMIO_MOVS) {
404 reg = insn_get_modrm_reg_ptr(&insn, regs);
405 if (!reg)
406 return -EINVAL;
407 }
408
409 /*
410 * Reject EPT violation #VEs that split pages.
411 *
412 * MMIO accesses are supposed to be naturally aligned and therefore
413 * never cross page boundaries. Seeing split page accesses indicates
414 * a bug or a load_unaligned_zeropad() that stepped into an MMIO page.
415 *
416 * load_unaligned_zeropad() will recover using exception fixups.
417 */
418 vaddr = (unsigned long)insn_get_addr_ref(&insn, regs);
419 if (vaddr / PAGE_SIZE != (vaddr + size - 1) / PAGE_SIZE)
420 return -EFAULT;
421
422 /* Handle writes first */
423 switch (mmio) {
424 case INSN_MMIO_WRITE:
425 memcpy(&val, reg, size);
426 if (!mmio_write(size, ve->gpa, val))
427 return -EIO;
428 return insn.length;
429 case INSN_MMIO_WRITE_IMM:
430 val = insn.immediate.value;
431 if (!mmio_write(size, ve->gpa, val))
432 return -EIO;
433 return insn.length;
434 case INSN_MMIO_READ:
435 case INSN_MMIO_READ_ZERO_EXTEND:
436 case INSN_MMIO_READ_SIGN_EXTEND:
437 /* Reads are handled below */
438 break;
439 case INSN_MMIO_MOVS:
440 case INSN_MMIO_DECODE_FAILED:
441 /*
442 * MMIO was accessed with an instruction that could not be
443 * decoded or handled properly. It was likely not using io.h
444 * helpers or accessed MMIO accidentally.
445 */
446 return -EINVAL;
447 default:
448 WARN_ONCE(1, "Unknown insn_decode_mmio() decode value?");
449 return -EINVAL;
450 }
451
452 /* Handle reads */
453 if (!mmio_read(size, ve->gpa, &val))
454 return -EIO;
455
456 switch (mmio) {
457 case INSN_MMIO_READ:
458 /* Zero-extend for 32-bit operation */
459 extend_size = size == 4 ? sizeof(*reg) : 0;
460 break;
461 case INSN_MMIO_READ_ZERO_EXTEND:
462 /* Zero extend based on operand size */
463 extend_size = insn.opnd_bytes;
464 break;
465 case INSN_MMIO_READ_SIGN_EXTEND:
466 /* Sign extend based on operand size */
467 extend_size = insn.opnd_bytes;
468 if (size == 1 && val & BIT(7))
469 extend_val = 0xFF;
470 else if (size > 1 && val & BIT(15))
471 extend_val = 0xFF;
472 break;
473 default:
474 /* All other cases has to be covered with the first switch() */
475 WARN_ON_ONCE(1);
476 return -EINVAL;
477 }
478
479 if (extend_size)
480 memset(reg, extend_val, extend_size);
481 memcpy(reg, &val, size);
482 return insn.length;
483 }
484
handle_in(struct pt_regs * regs,int size,int port)485 static bool handle_in(struct pt_regs *regs, int size, int port)
486 {
487 struct tdx_hypercall_args args = {
488 .r10 = TDX_HYPERCALL_STANDARD,
489 .r11 = hcall_func(EXIT_REASON_IO_INSTRUCTION),
490 .r12 = size,
491 .r13 = PORT_READ,
492 .r14 = port,
493 };
494 u64 mask = GENMASK(BITS_PER_BYTE * size, 0);
495 bool success;
496
497 /*
498 * Emulate the I/O read via hypercall. More info about ABI can be found
499 * in TDX Guest-Host-Communication Interface (GHCI) section titled
500 * "TDG.VP.VMCALL<Instruction.IO>".
501 */
502 success = !__tdx_hypercall_ret(&args);
503
504 /* Update part of the register affected by the emulated instruction */
505 regs->ax &= ~mask;
506 if (success)
507 regs->ax |= args.r11 & mask;
508
509 return success;
510 }
511
handle_out(struct pt_regs * regs,int size,int port)512 static bool handle_out(struct pt_regs *regs, int size, int port)
513 {
514 u64 mask = GENMASK(BITS_PER_BYTE * size, 0);
515
516 /*
517 * Emulate the I/O write via hypercall. More info about ABI can be found
518 * in TDX Guest-Host-Communication Interface (GHCI) section titled
519 * "TDG.VP.VMCALL<Instruction.IO>".
520 */
521 return !_tdx_hypercall(hcall_func(EXIT_REASON_IO_INSTRUCTION), size,
522 PORT_WRITE, port, regs->ax & mask);
523 }
524
525 /*
526 * Emulate I/O using hypercall.
527 *
528 * Assumes the IO instruction was using ax, which is enforced
529 * by the standard io.h macros.
530 *
531 * Return True on success or False on failure.
532 */
handle_io(struct pt_regs * regs,struct ve_info * ve)533 static int handle_io(struct pt_regs *regs, struct ve_info *ve)
534 {
535 u32 exit_qual = ve->exit_qual;
536 int size, port;
537 bool in, ret;
538
539 if (VE_IS_IO_STRING(exit_qual))
540 return -EIO;
541
542 in = VE_IS_IO_IN(exit_qual);
543 size = VE_GET_IO_SIZE(exit_qual);
544 port = VE_GET_PORT_NUM(exit_qual);
545
546
547 if (in)
548 ret = handle_in(regs, size, port);
549 else
550 ret = handle_out(regs, size, port);
551 if (!ret)
552 return -EIO;
553
554 return ve_instr_len(ve);
555 }
556
557 /*
558 * Early #VE exception handler. Only handles a subset of port I/O.
559 * Intended only for earlyprintk. If failed, return false.
560 */
tdx_early_handle_ve(struct pt_regs * regs)561 __init bool tdx_early_handle_ve(struct pt_regs *regs)
562 {
563 struct ve_info ve;
564 int insn_len;
565
566 tdx_get_ve_info(&ve);
567
568 if (ve.exit_reason != EXIT_REASON_IO_INSTRUCTION)
569 return false;
570
571 insn_len = handle_io(regs, &ve);
572 if (insn_len < 0)
573 return false;
574
575 regs->ip += insn_len;
576 return true;
577 }
578
tdx_get_ve_info(struct ve_info * ve)579 void tdx_get_ve_info(struct ve_info *ve)
580 {
581 struct tdx_module_output out;
582
583 /*
584 * Called during #VE handling to retrieve the #VE info from the
585 * TDX module.
586 *
587 * This has to be called early in #VE handling. A "nested" #VE which
588 * occurs before this will raise a #DF and is not recoverable.
589 *
590 * The call retrieves the #VE info from the TDX module, which also
591 * clears the "#VE valid" flag. This must be done before anything else
592 * because any #VE that occurs while the valid flag is set will lead to
593 * #DF.
594 *
595 * Note, the TDX module treats virtual NMIs as inhibited if the #VE
596 * valid flag is set. It means that NMI=>#VE will not result in a #DF.
597 */
598 tdx_module_call(TDX_GET_VEINFO, 0, 0, 0, 0, &out);
599
600 /* Transfer the output parameters */
601 ve->exit_reason = out.rcx;
602 ve->exit_qual = out.rdx;
603 ve->gla = out.r8;
604 ve->gpa = out.r9;
605 ve->instr_len = lower_32_bits(out.r10);
606 ve->instr_info = upper_32_bits(out.r10);
607 }
608
609 /*
610 * Handle the user initiated #VE.
611 *
612 * On success, returns the number of bytes RIP should be incremented (>=0)
613 * or -errno on error.
614 */
virt_exception_user(struct pt_regs * regs,struct ve_info * ve)615 static int virt_exception_user(struct pt_regs *regs, struct ve_info *ve)
616 {
617 switch (ve->exit_reason) {
618 case EXIT_REASON_CPUID:
619 return handle_cpuid(regs, ve);
620 default:
621 pr_warn("Unexpected #VE: %lld\n", ve->exit_reason);
622 return -EIO;
623 }
624 }
625
is_private_gpa(u64 gpa)626 static inline bool is_private_gpa(u64 gpa)
627 {
628 return gpa == cc_mkenc(gpa);
629 }
630
631 /*
632 * Handle the kernel #VE.
633 *
634 * On success, returns the number of bytes RIP should be incremented (>=0)
635 * or -errno on error.
636 */
virt_exception_kernel(struct pt_regs * regs,struct ve_info * ve)637 static int virt_exception_kernel(struct pt_regs *regs, struct ve_info *ve)
638 {
639 switch (ve->exit_reason) {
640 case EXIT_REASON_HLT:
641 return handle_halt(ve);
642 case EXIT_REASON_MSR_READ:
643 return read_msr(regs, ve);
644 case EXIT_REASON_MSR_WRITE:
645 return write_msr(regs, ve);
646 case EXIT_REASON_CPUID:
647 return handle_cpuid(regs, ve);
648 case EXIT_REASON_EPT_VIOLATION:
649 if (is_private_gpa(ve->gpa))
650 panic("Unexpected EPT-violation on private memory.");
651 return handle_mmio(regs, ve);
652 case EXIT_REASON_IO_INSTRUCTION:
653 return handle_io(regs, ve);
654 default:
655 pr_warn("Unexpected #VE: %lld\n", ve->exit_reason);
656 return -EIO;
657 }
658 }
659
tdx_handle_virt_exception(struct pt_regs * regs,struct ve_info * ve)660 bool tdx_handle_virt_exception(struct pt_regs *regs, struct ve_info *ve)
661 {
662 int insn_len;
663
664 if (user_mode(regs))
665 insn_len = virt_exception_user(regs, ve);
666 else
667 insn_len = virt_exception_kernel(regs, ve);
668 if (insn_len < 0)
669 return false;
670
671 /* After successful #VE handling, move the IP */
672 regs->ip += insn_len;
673
674 return true;
675 }
676
tdx_tlb_flush_required(bool private)677 static bool tdx_tlb_flush_required(bool private)
678 {
679 /*
680 * TDX guest is responsible for flushing TLB on private->shared
681 * transition. VMM is responsible for flushing on shared->private.
682 *
683 * The VMM _can't_ flush private addresses as it can't generate PAs
684 * with the guest's HKID. Shared memory isn't subject to integrity
685 * checking, i.e. the VMM doesn't need to flush for its own protection.
686 *
687 * There's no need to flush when converting from shared to private,
688 * as flushing is the VMM's responsibility in this case, e.g. it must
689 * flush to avoid integrity failures in the face of a buggy or
690 * malicious guest.
691 */
692 return !private;
693 }
694
tdx_cache_flush_required(void)695 static bool tdx_cache_flush_required(void)
696 {
697 /*
698 * AMD SME/SEV can avoid cache flushing if HW enforces cache coherence.
699 * TDX doesn't have such capability.
700 *
701 * Flush cache unconditionally.
702 */
703 return true;
704 }
705
706 /*
707 * Inform the VMM of the guest's intent for this physical page: shared with
708 * the VMM or private to the guest. The VMM is expected to change its mapping
709 * of the page in response.
710 */
tdx_enc_status_changed(unsigned long vaddr,int numpages,bool enc)711 static bool tdx_enc_status_changed(unsigned long vaddr, int numpages, bool enc)
712 {
713 phys_addr_t start = __pa(vaddr);
714 phys_addr_t end = __pa(vaddr + numpages * PAGE_SIZE);
715
716 if (!enc) {
717 /* Set the shared (decrypted) bits: */
718 start |= cc_mkdec(0);
719 end |= cc_mkdec(0);
720 }
721
722 /*
723 * Notify the VMM about page mapping conversion. More info about ABI
724 * can be found in TDX Guest-Host-Communication Interface (GHCI),
725 * section "TDG.VP.VMCALL<MapGPA>"
726 */
727 if (_tdx_hypercall(TDVMCALL_MAP_GPA, start, end - start, 0, 0))
728 return false;
729
730 /* shared->private conversion requires memory to be accepted before use */
731 if (enc)
732 return tdx_accept_memory(start, end);
733
734 return true;
735 }
736
tdx_enc_status_change_prepare(unsigned long vaddr,int numpages,bool enc)737 static bool tdx_enc_status_change_prepare(unsigned long vaddr, int numpages,
738 bool enc)
739 {
740 /*
741 * Only handle shared->private conversion here.
742 * See the comment in tdx_early_init().
743 */
744 if (enc)
745 return tdx_enc_status_changed(vaddr, numpages, enc);
746 return true;
747 }
748
tdx_enc_status_change_finish(unsigned long vaddr,int numpages,bool enc)749 static bool tdx_enc_status_change_finish(unsigned long vaddr, int numpages,
750 bool enc)
751 {
752 /*
753 * Only handle private->shared conversion here.
754 * See the comment in tdx_early_init().
755 */
756 if (!enc)
757 return tdx_enc_status_changed(vaddr, numpages, enc);
758 return true;
759 }
760
tdx_early_init(void)761 void __init tdx_early_init(void)
762 {
763 u64 cc_mask;
764 u32 eax, sig[3];
765
766 cpuid_count(TDX_CPUID_LEAF_ID, 0, &eax, &sig[0], &sig[2], &sig[1]);
767
768 if (memcmp(TDX_IDENT, sig, sizeof(sig)))
769 return;
770
771 setup_force_cpu_cap(X86_FEATURE_TDX_GUEST);
772
773 cc_vendor = CC_VENDOR_INTEL;
774 tdx_parse_tdinfo(&cc_mask);
775 cc_set_mask(cc_mask);
776
777 /* Kernel does not use NOTIFY_ENABLES and does not need random #VEs */
778 tdx_module_call(TDX_WR, 0, TDCS_NOTIFY_ENABLES, 0, -1ULL, NULL);
779
780 /*
781 * All bits above GPA width are reserved and kernel treats shared bit
782 * as flag, not as part of physical address.
783 *
784 * Adjust physical mask to only cover valid GPA bits.
785 */
786 physical_mask &= cc_mask - 1;
787
788 /*
789 * The kernel mapping should match the TDX metadata for the page.
790 * load_unaligned_zeropad() can touch memory *adjacent* to that which is
791 * owned by the caller and can catch even _momentary_ mismatches. Bad
792 * things happen on mismatch:
793 *
794 * - Private mapping => Shared Page == Guest shutdown
795 * - Shared mapping => Private Page == Recoverable #VE
796 *
797 * guest.enc_status_change_prepare() converts the page from
798 * shared=>private before the mapping becomes private.
799 *
800 * guest.enc_status_change_finish() converts the page from
801 * private=>shared after the mapping becomes private.
802 *
803 * In both cases there is a temporary shared mapping to a private page,
804 * which can result in a #VE. But, there is never a private mapping to
805 * a shared page.
806 */
807 x86_platform.guest.enc_status_change_prepare = tdx_enc_status_change_prepare;
808 x86_platform.guest.enc_status_change_finish = tdx_enc_status_change_finish;
809
810 x86_platform.guest.enc_cache_flush_required = tdx_cache_flush_required;
811 x86_platform.guest.enc_tlb_flush_required = tdx_tlb_flush_required;
812
813 /*
814 * TDX intercepts the RDMSR to read the X2APIC ID in the parallel
815 * bringup low level code. That raises #VE which cannot be handled
816 * there.
817 *
818 * Intel-TDX has a secure RDMSR hypercall, but that needs to be
819 * implemented seperately in the low level startup ASM code.
820 * Until that is in place, disable parallel bringup for TDX.
821 */
822 x86_cpuinit.parallel_bringup = false;
823
824 pr_info("Guest detected\n");
825 }
826