1 // SPDX-License-Identifier: GPL-2.0
2 /*
3 * AMD Encrypted Register State Support
4 *
5 * Author: Joerg Roedel <jroedel@suse.de>
6 */
7
8 /*
9 * misc.h needs to be first because it knows how to include the other kernel
10 * headers in the pre-decompression code in a way that does not break
11 * compilation.
12 */
13 #include "misc.h"
14
15 #include <asm/pgtable_types.h>
16 #include <asm/sev.h>
17 #include <asm/trapnr.h>
18 #include <asm/trap_pf.h>
19 #include <asm/msr-index.h>
20 #include <asm/fpu/xcr.h>
21 #include <asm/ptrace.h>
22 #include <asm/svm.h>
23 #include <asm/cpuid.h>
24
25 #include "error.h"
26 #include "../msr.h"
27
28 struct ghcb boot_ghcb_page __aligned(PAGE_SIZE);
29 struct ghcb *boot_ghcb;
30
31 /*
32 * Copy a version of this function here - insn-eval.c can't be used in
33 * pre-decompression code.
34 */
insn_has_rep_prefix(struct insn * insn)35 static bool insn_has_rep_prefix(struct insn *insn)
36 {
37 insn_byte_t p;
38 int i;
39
40 insn_get_prefixes(insn);
41
42 for_each_insn_prefix(insn, i, p) {
43 if (p == 0xf2 || p == 0xf3)
44 return true;
45 }
46
47 return false;
48 }
49
50 /*
51 * Only a dummy for insn_get_seg_base() - Early boot-code is 64bit only and
52 * doesn't use segments.
53 */
insn_get_seg_base(struct pt_regs * regs,int seg_reg_idx)54 static unsigned long insn_get_seg_base(struct pt_regs *regs, int seg_reg_idx)
55 {
56 return 0UL;
57 }
58
sev_es_rd_ghcb_msr(void)59 static inline u64 sev_es_rd_ghcb_msr(void)
60 {
61 struct msr m;
62
63 boot_rdmsr(MSR_AMD64_SEV_ES_GHCB, &m);
64
65 return m.q;
66 }
67
sev_es_wr_ghcb_msr(u64 val)68 static inline void sev_es_wr_ghcb_msr(u64 val)
69 {
70 struct msr m;
71
72 m.q = val;
73 boot_wrmsr(MSR_AMD64_SEV_ES_GHCB, &m);
74 }
75
vc_decode_insn(struct es_em_ctxt * ctxt)76 static enum es_result vc_decode_insn(struct es_em_ctxt *ctxt)
77 {
78 char buffer[MAX_INSN_SIZE];
79 int ret;
80
81 memcpy(buffer, (unsigned char *)ctxt->regs->ip, MAX_INSN_SIZE);
82
83 ret = insn_decode(&ctxt->insn, buffer, MAX_INSN_SIZE, INSN_MODE_64);
84 if (ret < 0)
85 return ES_DECODE_FAILED;
86
87 return ES_OK;
88 }
89
vc_write_mem(struct es_em_ctxt * ctxt,void * dst,char * buf,size_t size)90 static enum es_result vc_write_mem(struct es_em_ctxt *ctxt,
91 void *dst, char *buf, size_t size)
92 {
93 memcpy(dst, buf, size);
94
95 return ES_OK;
96 }
97
vc_read_mem(struct es_em_ctxt * ctxt,void * src,char * buf,size_t size)98 static enum es_result vc_read_mem(struct es_em_ctxt *ctxt,
99 void *src, char *buf, size_t size)
100 {
101 memcpy(buf, src, size);
102
103 return ES_OK;
104 }
105
vc_ioio_check(struct es_em_ctxt * ctxt,u16 port,size_t size)106 static enum es_result vc_ioio_check(struct es_em_ctxt *ctxt, u16 port, size_t size)
107 {
108 return ES_OK;
109 }
110
fault_in_kernel_space(unsigned long address)111 static bool fault_in_kernel_space(unsigned long address)
112 {
113 return false;
114 }
115
116 #undef __init
117 #define __init
118
119 #undef __head
120 #define __head
121
122 #define __BOOT_COMPRESSED
123
124 /* Basic instruction decoding support needed */
125 #include "../../lib/inat.c"
126 #include "../../lib/insn.c"
127
128 /* Include code for early handlers */
129 #include "../../kernel/sev-shared.c"
130
sev_snp_enabled(void)131 bool sev_snp_enabled(void)
132 {
133 return sev_status & MSR_AMD64_SEV_SNP_ENABLED;
134 }
135
__page_state_change(unsigned long paddr,enum psc_op op)136 static void __page_state_change(unsigned long paddr, enum psc_op op)
137 {
138 u64 val, msr;
139
140 /*
141 * If private -> shared then invalidate the page before requesting the
142 * state change in the RMP table.
143 */
144 if (op == SNP_PAGE_STATE_SHARED && pvalidate(paddr, RMP_PG_SIZE_4K, 0))
145 sev_es_terminate(SEV_TERM_SET_LINUX, GHCB_TERM_PVALIDATE);
146
147 /* Save the current GHCB MSR value */
148 msr = sev_es_rd_ghcb_msr();
149
150 /* Issue VMGEXIT to change the page state in RMP table. */
151 sev_es_wr_ghcb_msr(GHCB_MSR_PSC_REQ_GFN(paddr >> PAGE_SHIFT, op));
152 VMGEXIT();
153
154 /* Read the response of the VMGEXIT. */
155 val = sev_es_rd_ghcb_msr();
156 if ((GHCB_RESP_CODE(val) != GHCB_MSR_PSC_RESP) || GHCB_MSR_PSC_RESP_VAL(val))
157 sev_es_terminate(SEV_TERM_SET_LINUX, GHCB_TERM_PSC);
158
159 /* Restore the GHCB MSR value */
160 sev_es_wr_ghcb_msr(msr);
161
162 /*
163 * Now that page state is changed in the RMP table, validate it so that it is
164 * consistent with the RMP entry.
165 */
166 if (op == SNP_PAGE_STATE_PRIVATE && pvalidate(paddr, RMP_PG_SIZE_4K, 1))
167 sev_es_terminate(SEV_TERM_SET_LINUX, GHCB_TERM_PVALIDATE);
168 }
169
snp_set_page_private(unsigned long paddr)170 void snp_set_page_private(unsigned long paddr)
171 {
172 if (!sev_snp_enabled())
173 return;
174
175 __page_state_change(paddr, SNP_PAGE_STATE_PRIVATE);
176 }
177
snp_set_page_shared(unsigned long paddr)178 void snp_set_page_shared(unsigned long paddr)
179 {
180 if (!sev_snp_enabled())
181 return;
182
183 __page_state_change(paddr, SNP_PAGE_STATE_SHARED);
184 }
185
early_setup_ghcb(void)186 static bool early_setup_ghcb(void)
187 {
188 if (set_page_decrypted((unsigned long)&boot_ghcb_page))
189 return false;
190
191 /* Page is now mapped decrypted, clear it */
192 memset(&boot_ghcb_page, 0, sizeof(boot_ghcb_page));
193
194 boot_ghcb = &boot_ghcb_page;
195
196 /* Initialize lookup tables for the instruction decoder */
197 inat_init_tables();
198
199 /* SNP guest requires the GHCB GPA must be registered */
200 if (sev_snp_enabled())
201 snp_register_ghcb_early(__pa(&boot_ghcb_page));
202
203 return true;
204 }
205
snp_accept_memory(phys_addr_t start,phys_addr_t end)206 void snp_accept_memory(phys_addr_t start, phys_addr_t end)
207 {
208 for (phys_addr_t pa = start; pa < end; pa += PAGE_SIZE)
209 __page_state_change(pa, SNP_PAGE_STATE_PRIVATE);
210 }
211
sev_es_shutdown_ghcb(void)212 void sev_es_shutdown_ghcb(void)
213 {
214 if (!boot_ghcb)
215 return;
216
217 if (!sev_es_check_cpu_features())
218 error("SEV-ES CPU Features missing.");
219
220 /*
221 * GHCB Page must be flushed from the cache and mapped encrypted again.
222 * Otherwise the running kernel will see strange cache effects when
223 * trying to use that page.
224 */
225 if (set_page_encrypted((unsigned long)&boot_ghcb_page))
226 error("Can't map GHCB page encrypted");
227
228 /*
229 * GHCB page is mapped encrypted again and flushed from the cache.
230 * Mark it non-present now to catch bugs when #VC exceptions trigger
231 * after this point.
232 */
233 if (set_page_non_present((unsigned long)&boot_ghcb_page))
234 error("Can't unmap GHCB page");
235 }
236
sev_es_ghcb_terminate(struct ghcb * ghcb,unsigned int set,unsigned int reason,u64 exit_info_2)237 static void __noreturn sev_es_ghcb_terminate(struct ghcb *ghcb, unsigned int set,
238 unsigned int reason, u64 exit_info_2)
239 {
240 u64 exit_info_1 = SVM_VMGEXIT_TERM_REASON(set, reason);
241
242 vc_ghcb_invalidate(ghcb);
243 ghcb_set_sw_exit_code(ghcb, SVM_VMGEXIT_TERM_REQUEST);
244 ghcb_set_sw_exit_info_1(ghcb, exit_info_1);
245 ghcb_set_sw_exit_info_2(ghcb, exit_info_2);
246
247 sev_es_wr_ghcb_msr(__pa(ghcb));
248 VMGEXIT();
249
250 while (true)
251 asm volatile("hlt\n" : : : "memory");
252 }
253
sev_es_check_ghcb_fault(unsigned long address)254 bool sev_es_check_ghcb_fault(unsigned long address)
255 {
256 /* Check whether the fault was on the GHCB page */
257 return ((address & PAGE_MASK) == (unsigned long)&boot_ghcb_page);
258 }
259
do_boot_stage2_vc(struct pt_regs * regs,unsigned long exit_code)260 void do_boot_stage2_vc(struct pt_regs *regs, unsigned long exit_code)
261 {
262 struct es_em_ctxt ctxt;
263 enum es_result result;
264
265 if (!boot_ghcb && !early_setup_ghcb())
266 sev_es_terminate(SEV_TERM_SET_GEN, GHCB_SEV_ES_GEN_REQ);
267
268 vc_ghcb_invalidate(boot_ghcb);
269 result = vc_init_em_ctxt(&ctxt, regs, exit_code);
270 if (result != ES_OK)
271 goto finish;
272
273 switch (exit_code) {
274 case SVM_EXIT_RDTSC:
275 case SVM_EXIT_RDTSCP:
276 result = vc_handle_rdtsc(boot_ghcb, &ctxt, exit_code);
277 break;
278 case SVM_EXIT_IOIO:
279 result = vc_handle_ioio(boot_ghcb, &ctxt);
280 break;
281 case SVM_EXIT_CPUID:
282 result = vc_handle_cpuid(boot_ghcb, &ctxt);
283 break;
284 default:
285 result = ES_UNSUPPORTED;
286 break;
287 }
288
289 finish:
290 if (result == ES_OK)
291 vc_finish_insn(&ctxt);
292 else if (result != ES_RETRY)
293 sev_es_terminate(SEV_TERM_SET_GEN, GHCB_SEV_ES_GEN_REQ);
294 }
295
enforce_vmpl0(void)296 static void enforce_vmpl0(void)
297 {
298 u64 attrs;
299 int err;
300
301 /*
302 * RMPADJUST modifies RMP permissions of a lesser-privileged (numerically
303 * higher) privilege level. Here, clear the VMPL1 permission mask of the
304 * GHCB page. If the guest is not running at VMPL0, this will fail.
305 *
306 * If the guest is running at VMPL0, it will succeed. Even if that operation
307 * modifies permission bits, it is still ok to do so currently because Linux
308 * SNP guests are supported only on VMPL0 so VMPL1 or higher permission masks
309 * changing is a don't-care.
310 */
311 attrs = 1;
312 if (rmpadjust((unsigned long)&boot_ghcb_page, RMP_PG_SIZE_4K, attrs))
313 sev_es_terminate(SEV_TERM_SET_LINUX, GHCB_TERM_NOT_VMPL0);
314 }
315
316 /*
317 * SNP_FEATURES_IMPL_REQ is the mask of SNP features that will need
318 * guest side implementation for proper functioning of the guest. If any
319 * of these features are enabled in the hypervisor but are lacking guest
320 * side implementation, the behavior of the guest will be undefined. The
321 * guest could fail in non-obvious way making it difficult to debug.
322 *
323 * As the behavior of reserved feature bits is unknown to be on the
324 * safe side add them to the required features mask.
325 */
326 #define SNP_FEATURES_IMPL_REQ (MSR_AMD64_SNP_VTOM | \
327 MSR_AMD64_SNP_REFLECT_VC | \
328 MSR_AMD64_SNP_RESTRICTED_INJ | \
329 MSR_AMD64_SNP_ALT_INJ | \
330 MSR_AMD64_SNP_DEBUG_SWAP | \
331 MSR_AMD64_SNP_VMPL_SSS | \
332 MSR_AMD64_SNP_SECURE_TSC | \
333 MSR_AMD64_SNP_VMGEXIT_PARAM | \
334 MSR_AMD64_SNP_VMSA_REG_PROTECTION | \
335 MSR_AMD64_SNP_RESERVED_BIT13 | \
336 MSR_AMD64_SNP_RESERVED_BIT15 | \
337 MSR_AMD64_SNP_RESERVED_MASK)
338
339 /*
340 * SNP_FEATURES_PRESENT is the mask of SNP features that are implemented
341 * by the guest kernel. As and when a new feature is implemented in the
342 * guest kernel, a corresponding bit should be added to the mask.
343 */
344 #define SNP_FEATURES_PRESENT MSR_AMD64_SNP_DEBUG_SWAP
345
snp_get_unsupported_features(u64 status)346 u64 snp_get_unsupported_features(u64 status)
347 {
348 if (!(status & MSR_AMD64_SEV_SNP_ENABLED))
349 return 0;
350
351 return status & SNP_FEATURES_IMPL_REQ & ~SNP_FEATURES_PRESENT;
352 }
353
snp_check_features(void)354 void snp_check_features(void)
355 {
356 u64 unsupported;
357
358 /*
359 * Terminate the boot if hypervisor has enabled any feature lacking
360 * guest side implementation. Pass on the unsupported features mask through
361 * EXIT_INFO_2 of the GHCB protocol so that those features can be reported
362 * as part of the guest boot failure.
363 */
364 unsupported = snp_get_unsupported_features(sev_status);
365 if (unsupported) {
366 if (ghcb_version < 2 || (!boot_ghcb && !early_setup_ghcb()))
367 sev_es_terminate(SEV_TERM_SET_GEN, GHCB_SNP_UNSUPPORTED);
368
369 sev_es_ghcb_terminate(boot_ghcb, SEV_TERM_SET_GEN,
370 GHCB_SNP_UNSUPPORTED, unsupported);
371 }
372 }
373
374 /*
375 * sev_check_cpu_support - Check for SEV support in the CPU capabilities
376 *
377 * Returns < 0 if SEV is not supported, otherwise the position of the
378 * encryption bit in the page table descriptors.
379 */
sev_check_cpu_support(void)380 static int sev_check_cpu_support(void)
381 {
382 unsigned int eax, ebx, ecx, edx;
383
384 /* Check for the SME/SEV support leaf */
385 eax = 0x80000000;
386 ecx = 0;
387 native_cpuid(&eax, &ebx, &ecx, &edx);
388 if (eax < 0x8000001f)
389 return -ENODEV;
390
391 /*
392 * Check for the SME/SEV feature:
393 * CPUID Fn8000_001F[EAX]
394 * - Bit 0 - Secure Memory Encryption support
395 * - Bit 1 - Secure Encrypted Virtualization support
396 * CPUID Fn8000_001F[EBX]
397 * - Bits 5:0 - Pagetable bit position used to indicate encryption
398 */
399 eax = 0x8000001f;
400 ecx = 0;
401 native_cpuid(&eax, &ebx, &ecx, &edx);
402 /* Check whether SEV is supported */
403 if (!(eax & BIT(1)))
404 return -ENODEV;
405
406 return ebx & 0x3f;
407 }
408
sev_enable(struct boot_params * bp)409 void sev_enable(struct boot_params *bp)
410 {
411 struct msr m;
412 int bitpos;
413 bool snp;
414
415 /*
416 * bp->cc_blob_address should only be set by boot/compressed kernel.
417 * Initialize it to 0 to ensure that uninitialized values from
418 * buggy bootloaders aren't propagated.
419 */
420 if (bp)
421 bp->cc_blob_address = 0;
422
423 /*
424 * Do an initial SEV capability check before snp_init() which
425 * loads the CPUID page and the same checks afterwards are done
426 * without the hypervisor and are trustworthy.
427 *
428 * If the HV fakes SEV support, the guest will crash'n'burn
429 * which is good enough.
430 */
431
432 if (sev_check_cpu_support() < 0)
433 return;
434
435 /*
436 * Setup/preliminary detection of SNP. This will be sanity-checked
437 * against CPUID/MSR values later.
438 */
439 snp = snp_init(bp);
440
441 /* Now repeat the checks with the SNP CPUID table. */
442
443 bitpos = sev_check_cpu_support();
444 if (bitpos < 0) {
445 if (snp)
446 error("SEV-SNP support indicated by CC blob, but not CPUID.");
447 return;
448 }
449
450 /* Set the SME mask if this is an SEV guest. */
451 boot_rdmsr(MSR_AMD64_SEV, &m);
452 sev_status = m.q;
453 if (!(sev_status & MSR_AMD64_SEV_ENABLED))
454 return;
455
456 /* Negotiate the GHCB protocol version. */
457 if (sev_status & MSR_AMD64_SEV_ES_ENABLED) {
458 if (!sev_es_negotiate_protocol())
459 sev_es_terminate(SEV_TERM_SET_GEN, GHCB_SEV_ES_PROT_UNSUPPORTED);
460 }
461
462 /*
463 * SNP is supported in v2 of the GHCB spec which mandates support for HV
464 * features.
465 */
466 if (sev_status & MSR_AMD64_SEV_SNP_ENABLED) {
467 if (!(get_hv_features() & GHCB_HV_FT_SNP))
468 sev_es_terminate(SEV_TERM_SET_GEN, GHCB_SNP_UNSUPPORTED);
469
470 enforce_vmpl0();
471 }
472
473 if (snp && !(sev_status & MSR_AMD64_SEV_SNP_ENABLED))
474 error("SEV-SNP supported indicated by CC blob, but not SEV status MSR.");
475
476 sme_me_mask = BIT_ULL(bitpos);
477 }
478
479 /*
480 * sev_get_status - Retrieve the SEV status mask
481 *
482 * Returns 0 if the CPU is not SEV capable, otherwise the value of the
483 * AMD64_SEV MSR.
484 */
sev_get_status(void)485 u64 sev_get_status(void)
486 {
487 struct msr m;
488
489 if (sev_check_cpu_support() < 0)
490 return 0;
491
492 boot_rdmsr(MSR_AMD64_SEV, &m);
493 return m.q;
494 }
495
496 /* Search for Confidential Computing blob in the EFI config table. */
find_cc_blob_efi(struct boot_params * bp)497 static struct cc_blob_sev_info *find_cc_blob_efi(struct boot_params *bp)
498 {
499 unsigned long cfg_table_pa;
500 unsigned int cfg_table_len;
501 int ret;
502
503 ret = efi_get_conf_table(bp, &cfg_table_pa, &cfg_table_len);
504 if (ret)
505 return NULL;
506
507 return (struct cc_blob_sev_info *)efi_find_vendor_table(bp, cfg_table_pa,
508 cfg_table_len,
509 EFI_CC_BLOB_GUID);
510 }
511
512 /*
513 * Initial set up of SNP relies on information provided by the
514 * Confidential Computing blob, which can be passed to the boot kernel
515 * by firmware/bootloader in the following ways:
516 *
517 * - via an entry in the EFI config table
518 * - via a setup_data structure, as defined by the Linux Boot Protocol
519 *
520 * Scan for the blob in that order.
521 */
find_cc_blob(struct boot_params * bp)522 static struct cc_blob_sev_info *find_cc_blob(struct boot_params *bp)
523 {
524 struct cc_blob_sev_info *cc_info;
525
526 cc_info = find_cc_blob_efi(bp);
527 if (cc_info)
528 goto found_cc_info;
529
530 cc_info = find_cc_blob_setup_data(bp);
531 if (!cc_info)
532 return NULL;
533
534 found_cc_info:
535 if (cc_info->magic != CC_BLOB_SEV_HDR_MAGIC)
536 sev_es_terminate(SEV_TERM_SET_GEN, GHCB_SNP_UNSUPPORTED);
537
538 return cc_info;
539 }
540
541 /*
542 * Indicate SNP based on presence of SNP-specific CC blob. Subsequent checks
543 * will verify the SNP CPUID/MSR bits.
544 */
snp_init(struct boot_params * bp)545 bool snp_init(struct boot_params *bp)
546 {
547 struct cc_blob_sev_info *cc_info;
548
549 if (!bp)
550 return false;
551
552 cc_info = find_cc_blob(bp);
553 if (!cc_info)
554 return false;
555
556 /*
557 * If a SNP-specific Confidential Computing blob is present, then
558 * firmware/bootloader have indicated SNP support. Verifying this
559 * involves CPUID checks which will be more reliable if the SNP
560 * CPUID table is used. See comments over snp_setup_cpuid_table() for
561 * more details.
562 */
563 setup_cpuid_table(cc_info);
564
565 /*
566 * Pass run-time kernel a pointer to CC info via boot_params so EFI
567 * config table doesn't need to be searched again during early startup
568 * phase.
569 */
570 bp->cc_blob_address = (u32)(unsigned long)cc_info;
571
572 return true;
573 }
574
sev_prep_identity_maps(unsigned long top_level_pgt)575 void sev_prep_identity_maps(unsigned long top_level_pgt)
576 {
577 /*
578 * The Confidential Computing blob is used very early in uncompressed
579 * kernel to find the in-memory CPUID table to handle CPUID
580 * instructions. Make sure an identity-mapping exists so it can be
581 * accessed after switchover.
582 */
583 if (sev_snp_enabled()) {
584 unsigned long cc_info_pa = boot_params_ptr->cc_blob_address;
585 struct cc_blob_sev_info *cc_info;
586
587 kernel_add_identity_map(cc_info_pa, cc_info_pa + sizeof(*cc_info));
588
589 cc_info = (struct cc_blob_sev_info *)cc_info_pa;
590 kernel_add_identity_map(cc_info->cpuid_phys, cc_info->cpuid_phys + cc_info->cpuid_len);
591 }
592
593 sev_verify_cbit(top_level_pgt);
594 }
595