1 // SPDX-License-Identifier: GPL-2.0
2 /*
3 * AMD Encrypted Register State Support
4 *
5 * Author: Joerg Roedel <jroedel@suse.de>
6 *
7 * This file is not compiled stand-alone. It contains code shared
8 * between the pre-decompression boot code and the running Linux kernel
9 * and is included directly into both code-bases.
10 */
11
12 #ifndef __BOOT_COMPRESSED
13 #define error(v) pr_err(v)
14 #define has_cpuflag(f) boot_cpu_has(f)
15 #else
16 #undef WARN
17 #define WARN(condition, format...) (!!(condition))
18 #endif
19
20 /* I/O parameters for CPUID-related helpers */
21 struct cpuid_leaf {
22 u32 fn;
23 u32 subfn;
24 u32 eax;
25 u32 ebx;
26 u32 ecx;
27 u32 edx;
28 };
29
30 /*
31 * Individual entries of the SNP CPUID table, as defined by the SNP
32 * Firmware ABI, Revision 0.9, Section 7.1, Table 14.
33 */
34 struct snp_cpuid_fn {
35 u32 eax_in;
36 u32 ecx_in;
37 u64 xcr0_in;
38 u64 xss_in;
39 u32 eax;
40 u32 ebx;
41 u32 ecx;
42 u32 edx;
43 u64 __reserved;
44 } __packed;
45
46 /*
47 * SNP CPUID table, as defined by the SNP Firmware ABI, Revision 0.9,
48 * Section 8.14.2.6. Also noted there is the SNP firmware-enforced limit
49 * of 64 entries per CPUID table.
50 */
51 #define SNP_CPUID_COUNT_MAX 64
52
53 struct snp_cpuid_table {
54 u32 count;
55 u32 __reserved1;
56 u64 __reserved2;
57 struct snp_cpuid_fn fn[SNP_CPUID_COUNT_MAX];
58 } __packed;
59
60 /*
61 * Since feature negotiation related variables are set early in the boot
62 * process they must reside in the .data section so as not to be zeroed
63 * out when the .bss section is later cleared.
64 *
65 * GHCB protocol version negotiated with the hypervisor.
66 */
67 static u16 ghcb_version __ro_after_init;
68
69 /* Copy of the SNP firmware's CPUID page. */
70 static struct snp_cpuid_table cpuid_table_copy __ro_after_init;
71
72 /*
73 * These will be initialized based on CPUID table so that non-present
74 * all-zero leaves (for sparse tables) can be differentiated from
75 * invalid/out-of-range leaves. This is needed since all-zero leaves
76 * still need to be post-processed.
77 */
78 static u32 cpuid_std_range_max __ro_after_init;
79 static u32 cpuid_hyp_range_max __ro_after_init;
80 static u32 cpuid_ext_range_max __ro_after_init;
81
sev_es_check_cpu_features(void)82 static bool __init sev_es_check_cpu_features(void)
83 {
84 if (!has_cpuflag(X86_FEATURE_RDRAND)) {
85 error("RDRAND instruction not supported - no trusted source of randomness available\n");
86 return false;
87 }
88
89 return true;
90 }
91
92 static void __head __noreturn
sev_es_terminate(unsigned int set,unsigned int reason)93 sev_es_terminate(unsigned int set, unsigned int reason)
94 {
95 u64 val = GHCB_MSR_TERM_REQ;
96
97 /* Tell the hypervisor what went wrong. */
98 val |= GHCB_SEV_TERM_REASON(set, reason);
99
100 /* Request Guest Termination from Hypvervisor */
101 sev_es_wr_ghcb_msr(val);
102 VMGEXIT();
103
104 while (true)
105 asm volatile("hlt\n" : : : "memory");
106 }
107
108 /*
109 * The hypervisor features are available from GHCB version 2 onward.
110 */
get_hv_features(void)111 static u64 get_hv_features(void)
112 {
113 u64 val;
114
115 if (ghcb_version < 2)
116 return 0;
117
118 sev_es_wr_ghcb_msr(GHCB_MSR_HV_FT_REQ);
119 VMGEXIT();
120
121 val = sev_es_rd_ghcb_msr();
122 if (GHCB_RESP_CODE(val) != GHCB_MSR_HV_FT_RESP)
123 return 0;
124
125 return GHCB_MSR_HV_FT_RESP_VAL(val);
126 }
127
snp_register_ghcb_early(unsigned long paddr)128 static void snp_register_ghcb_early(unsigned long paddr)
129 {
130 unsigned long pfn = paddr >> PAGE_SHIFT;
131 u64 val;
132
133 sev_es_wr_ghcb_msr(GHCB_MSR_REG_GPA_REQ_VAL(pfn));
134 VMGEXIT();
135
136 val = sev_es_rd_ghcb_msr();
137
138 /* If the response GPA is not ours then abort the guest */
139 if ((GHCB_RESP_CODE(val) != GHCB_MSR_REG_GPA_RESP) ||
140 (GHCB_MSR_REG_GPA_RESP_VAL(val) != pfn))
141 sev_es_terminate(SEV_TERM_SET_LINUX, GHCB_TERM_REGISTER);
142 }
143
sev_es_negotiate_protocol(void)144 static bool sev_es_negotiate_protocol(void)
145 {
146 u64 val;
147
148 /* Do the GHCB protocol version negotiation */
149 sev_es_wr_ghcb_msr(GHCB_MSR_SEV_INFO_REQ);
150 VMGEXIT();
151 val = sev_es_rd_ghcb_msr();
152
153 if (GHCB_MSR_INFO(val) != GHCB_MSR_SEV_INFO_RESP)
154 return false;
155
156 if (GHCB_MSR_PROTO_MAX(val) < GHCB_PROTOCOL_MIN ||
157 GHCB_MSR_PROTO_MIN(val) > GHCB_PROTOCOL_MAX)
158 return false;
159
160 ghcb_version = min_t(size_t, GHCB_MSR_PROTO_MAX(val), GHCB_PROTOCOL_MAX);
161
162 return true;
163 }
164
vc_ghcb_invalidate(struct ghcb * ghcb)165 static __always_inline void vc_ghcb_invalidate(struct ghcb *ghcb)
166 {
167 ghcb->save.sw_exit_code = 0;
168 __builtin_memset(ghcb->save.valid_bitmap, 0, sizeof(ghcb->save.valid_bitmap));
169 }
170
vc_decoding_needed(unsigned long exit_code)171 static bool vc_decoding_needed(unsigned long exit_code)
172 {
173 /* Exceptions don't require to decode the instruction */
174 return !(exit_code >= SVM_EXIT_EXCP_BASE &&
175 exit_code <= SVM_EXIT_LAST_EXCP);
176 }
177
vc_init_em_ctxt(struct es_em_ctxt * ctxt,struct pt_regs * regs,unsigned long exit_code)178 static enum es_result vc_init_em_ctxt(struct es_em_ctxt *ctxt,
179 struct pt_regs *regs,
180 unsigned long exit_code)
181 {
182 enum es_result ret = ES_OK;
183
184 memset(ctxt, 0, sizeof(*ctxt));
185 ctxt->regs = regs;
186
187 if (vc_decoding_needed(exit_code))
188 ret = vc_decode_insn(ctxt);
189
190 return ret;
191 }
192
vc_finish_insn(struct es_em_ctxt * ctxt)193 static void vc_finish_insn(struct es_em_ctxt *ctxt)
194 {
195 ctxt->regs->ip += ctxt->insn.length;
196 }
197
verify_exception_info(struct ghcb * ghcb,struct es_em_ctxt * ctxt)198 static enum es_result verify_exception_info(struct ghcb *ghcb, struct es_em_ctxt *ctxt)
199 {
200 u32 ret;
201
202 ret = ghcb->save.sw_exit_info_1 & GENMASK_ULL(31, 0);
203 if (!ret)
204 return ES_OK;
205
206 if (ret == 1) {
207 u64 info = ghcb->save.sw_exit_info_2;
208 unsigned long v = info & SVM_EVTINJ_VEC_MASK;
209
210 /* Check if exception information from hypervisor is sane. */
211 if ((info & SVM_EVTINJ_VALID) &&
212 ((v == X86_TRAP_GP) || (v == X86_TRAP_UD)) &&
213 ((info & SVM_EVTINJ_TYPE_MASK) == SVM_EVTINJ_TYPE_EXEPT)) {
214 ctxt->fi.vector = v;
215
216 if (info & SVM_EVTINJ_VALID_ERR)
217 ctxt->fi.error_code = info >> 32;
218
219 return ES_EXCEPTION;
220 }
221 }
222
223 return ES_VMM_ERROR;
224 }
225
sev_es_ghcb_hv_call(struct ghcb * ghcb,struct es_em_ctxt * ctxt,u64 exit_code,u64 exit_info_1,u64 exit_info_2)226 static enum es_result sev_es_ghcb_hv_call(struct ghcb *ghcb,
227 struct es_em_ctxt *ctxt,
228 u64 exit_code, u64 exit_info_1,
229 u64 exit_info_2)
230 {
231 /* Fill in protocol and format specifiers */
232 ghcb->protocol_version = ghcb_version;
233 ghcb->ghcb_usage = GHCB_DEFAULT_USAGE;
234
235 ghcb_set_sw_exit_code(ghcb, exit_code);
236 ghcb_set_sw_exit_info_1(ghcb, exit_info_1);
237 ghcb_set_sw_exit_info_2(ghcb, exit_info_2);
238
239 sev_es_wr_ghcb_msr(__pa(ghcb));
240 VMGEXIT();
241
242 return verify_exception_info(ghcb, ctxt);
243 }
244
__sev_cpuid_hv(u32 fn,int reg_idx,u32 * reg)245 static int __sev_cpuid_hv(u32 fn, int reg_idx, u32 *reg)
246 {
247 u64 val;
248
249 sev_es_wr_ghcb_msr(GHCB_CPUID_REQ(fn, reg_idx));
250 VMGEXIT();
251 val = sev_es_rd_ghcb_msr();
252 if (GHCB_RESP_CODE(val) != GHCB_MSR_CPUID_RESP)
253 return -EIO;
254
255 *reg = (val >> 32);
256
257 return 0;
258 }
259
__sev_cpuid_hv_msr(struct cpuid_leaf * leaf)260 static int __sev_cpuid_hv_msr(struct cpuid_leaf *leaf)
261 {
262 int ret;
263
264 /*
265 * MSR protocol does not support fetching non-zero subfunctions, but is
266 * sufficient to handle current early-boot cases. Should that change,
267 * make sure to report an error rather than ignoring the index and
268 * grabbing random values. If this issue arises in the future, handling
269 * can be added here to use GHCB-page protocol for cases that occur late
270 * enough in boot that GHCB page is available.
271 */
272 if (cpuid_function_is_indexed(leaf->fn) && leaf->subfn)
273 return -EINVAL;
274
275 ret = __sev_cpuid_hv(leaf->fn, GHCB_CPUID_REQ_EAX, &leaf->eax);
276 ret = ret ? : __sev_cpuid_hv(leaf->fn, GHCB_CPUID_REQ_EBX, &leaf->ebx);
277 ret = ret ? : __sev_cpuid_hv(leaf->fn, GHCB_CPUID_REQ_ECX, &leaf->ecx);
278 ret = ret ? : __sev_cpuid_hv(leaf->fn, GHCB_CPUID_REQ_EDX, &leaf->edx);
279
280 return ret;
281 }
282
__sev_cpuid_hv_ghcb(struct ghcb * ghcb,struct es_em_ctxt * ctxt,struct cpuid_leaf * leaf)283 static int __sev_cpuid_hv_ghcb(struct ghcb *ghcb, struct es_em_ctxt *ctxt, struct cpuid_leaf *leaf)
284 {
285 u32 cr4 = native_read_cr4();
286 int ret;
287
288 ghcb_set_rax(ghcb, leaf->fn);
289 ghcb_set_rcx(ghcb, leaf->subfn);
290
291 if (cr4 & X86_CR4_OSXSAVE)
292 /* Safe to read xcr0 */
293 ghcb_set_xcr0(ghcb, xgetbv(XCR_XFEATURE_ENABLED_MASK));
294 else
295 /* xgetbv will cause #UD - use reset value for xcr0 */
296 ghcb_set_xcr0(ghcb, 1);
297
298 ret = sev_es_ghcb_hv_call(ghcb, ctxt, SVM_EXIT_CPUID, 0, 0);
299 if (ret != ES_OK)
300 return ret;
301
302 if (!(ghcb_rax_is_valid(ghcb) &&
303 ghcb_rbx_is_valid(ghcb) &&
304 ghcb_rcx_is_valid(ghcb) &&
305 ghcb_rdx_is_valid(ghcb)))
306 return ES_VMM_ERROR;
307
308 leaf->eax = ghcb->save.rax;
309 leaf->ebx = ghcb->save.rbx;
310 leaf->ecx = ghcb->save.rcx;
311 leaf->edx = ghcb->save.rdx;
312
313 return ES_OK;
314 }
315
sev_cpuid_hv(struct ghcb * ghcb,struct es_em_ctxt * ctxt,struct cpuid_leaf * leaf)316 static int sev_cpuid_hv(struct ghcb *ghcb, struct es_em_ctxt *ctxt, struct cpuid_leaf *leaf)
317 {
318 return ghcb ? __sev_cpuid_hv_ghcb(ghcb, ctxt, leaf)
319 : __sev_cpuid_hv_msr(leaf);
320 }
321
322 /*
323 * This may be called early while still running on the initial identity
324 * mapping. Use RIP-relative addressing to obtain the correct address
325 * while running with the initial identity mapping as well as the
326 * switch-over to kernel virtual addresses later.
327 */
snp_cpuid_get_table(void)328 static const struct snp_cpuid_table *snp_cpuid_get_table(void)
329 {
330 return &RIP_REL_REF(cpuid_table_copy);
331 }
332
333 /*
334 * The SNP Firmware ABI, Revision 0.9, Section 7.1, details the use of
335 * XCR0_IN and XSS_IN to encode multiple versions of 0xD subfunctions 0
336 * and 1 based on the corresponding features enabled by a particular
337 * combination of XCR0 and XSS registers so that a guest can look up the
338 * version corresponding to the features currently enabled in its XCR0/XSS
339 * registers. The only values that differ between these versions/table
340 * entries is the enabled XSAVE area size advertised via EBX.
341 *
342 * While hypervisors may choose to make use of this support, it is more
343 * robust/secure for a guest to simply find the entry corresponding to the
344 * base/legacy XSAVE area size (XCR0=1 or XCR0=3), and then calculate the
345 * XSAVE area size using subfunctions 2 through 64, as documented in APM
346 * Volume 3, Rev 3.31, Appendix E.3.8, which is what is done here.
347 *
348 * Since base/legacy XSAVE area size is documented as 0x240, use that value
349 * directly rather than relying on the base size in the CPUID table.
350 *
351 * Return: XSAVE area size on success, 0 otherwise.
352 */
snp_cpuid_calc_xsave_size(u64 xfeatures_en,bool compacted)353 static u32 snp_cpuid_calc_xsave_size(u64 xfeatures_en, bool compacted)
354 {
355 const struct snp_cpuid_table *cpuid_table = snp_cpuid_get_table();
356 u64 xfeatures_found = 0;
357 u32 xsave_size = 0x240;
358 int i;
359
360 for (i = 0; i < cpuid_table->count; i++) {
361 const struct snp_cpuid_fn *e = &cpuid_table->fn[i];
362
363 if (!(e->eax_in == 0xD && e->ecx_in > 1 && e->ecx_in < 64))
364 continue;
365 if (!(xfeatures_en & (BIT_ULL(e->ecx_in))))
366 continue;
367 if (xfeatures_found & (BIT_ULL(e->ecx_in)))
368 continue;
369
370 xfeatures_found |= (BIT_ULL(e->ecx_in));
371
372 if (compacted)
373 xsave_size += e->eax;
374 else
375 xsave_size = max(xsave_size, e->eax + e->ebx);
376 }
377
378 /*
379 * Either the guest set unsupported XCR0/XSS bits, or the corresponding
380 * entries in the CPUID table were not present. This is not a valid
381 * state to be in.
382 */
383 if (xfeatures_found != (xfeatures_en & GENMASK_ULL(63, 2)))
384 return 0;
385
386 return xsave_size;
387 }
388
389 static bool __head
snp_cpuid_get_validated_func(struct cpuid_leaf * leaf)390 snp_cpuid_get_validated_func(struct cpuid_leaf *leaf)
391 {
392 const struct snp_cpuid_table *cpuid_table = snp_cpuid_get_table();
393 int i;
394
395 for (i = 0; i < cpuid_table->count; i++) {
396 const struct snp_cpuid_fn *e = &cpuid_table->fn[i];
397
398 if (e->eax_in != leaf->fn)
399 continue;
400
401 if (cpuid_function_is_indexed(leaf->fn) && e->ecx_in != leaf->subfn)
402 continue;
403
404 /*
405 * For 0xD subfunctions 0 and 1, only use the entry corresponding
406 * to the base/legacy XSAVE area size (XCR0=1 or XCR0=3, XSS=0).
407 * See the comments above snp_cpuid_calc_xsave_size() for more
408 * details.
409 */
410 if (e->eax_in == 0xD && (e->ecx_in == 0 || e->ecx_in == 1))
411 if (!(e->xcr0_in == 1 || e->xcr0_in == 3) || e->xss_in)
412 continue;
413
414 leaf->eax = e->eax;
415 leaf->ebx = e->ebx;
416 leaf->ecx = e->ecx;
417 leaf->edx = e->edx;
418
419 return true;
420 }
421
422 return false;
423 }
424
snp_cpuid_hv(struct ghcb * ghcb,struct es_em_ctxt * ctxt,struct cpuid_leaf * leaf)425 static void snp_cpuid_hv(struct ghcb *ghcb, struct es_em_ctxt *ctxt, struct cpuid_leaf *leaf)
426 {
427 if (sev_cpuid_hv(ghcb, ctxt, leaf))
428 sev_es_terminate(SEV_TERM_SET_LINUX, GHCB_TERM_CPUID_HV);
429 }
430
snp_cpuid_postprocess(struct ghcb * ghcb,struct es_em_ctxt * ctxt,struct cpuid_leaf * leaf)431 static int snp_cpuid_postprocess(struct ghcb *ghcb, struct es_em_ctxt *ctxt,
432 struct cpuid_leaf *leaf)
433 {
434 struct cpuid_leaf leaf_hv = *leaf;
435
436 switch (leaf->fn) {
437 case 0x1:
438 snp_cpuid_hv(ghcb, ctxt, &leaf_hv);
439
440 /* initial APIC ID */
441 leaf->ebx = (leaf_hv.ebx & GENMASK(31, 24)) | (leaf->ebx & GENMASK(23, 0));
442 /* APIC enabled bit */
443 leaf->edx = (leaf_hv.edx & BIT(9)) | (leaf->edx & ~BIT(9));
444
445 /* OSXSAVE enabled bit */
446 if (native_read_cr4() & X86_CR4_OSXSAVE)
447 leaf->ecx |= BIT(27);
448 break;
449 case 0x7:
450 /* OSPKE enabled bit */
451 leaf->ecx &= ~BIT(4);
452 if (native_read_cr4() & X86_CR4_PKE)
453 leaf->ecx |= BIT(4);
454 break;
455 case 0xB:
456 leaf_hv.subfn = 0;
457 snp_cpuid_hv(ghcb, ctxt, &leaf_hv);
458
459 /* extended APIC ID */
460 leaf->edx = leaf_hv.edx;
461 break;
462 case 0xD: {
463 bool compacted = false;
464 u64 xcr0 = 1, xss = 0;
465 u32 xsave_size;
466
467 if (leaf->subfn != 0 && leaf->subfn != 1)
468 return 0;
469
470 if (native_read_cr4() & X86_CR4_OSXSAVE)
471 xcr0 = xgetbv(XCR_XFEATURE_ENABLED_MASK);
472 if (leaf->subfn == 1) {
473 /* Get XSS value if XSAVES is enabled. */
474 if (leaf->eax & BIT(3)) {
475 unsigned long lo, hi;
476
477 asm volatile("rdmsr" : "=a" (lo), "=d" (hi)
478 : "c" (MSR_IA32_XSS));
479 xss = (hi << 32) | lo;
480 }
481
482 /*
483 * The PPR and APM aren't clear on what size should be
484 * encoded in 0xD:0x1:EBX when compaction is not enabled
485 * by either XSAVEC (feature bit 1) or XSAVES (feature
486 * bit 3) since SNP-capable hardware has these feature
487 * bits fixed as 1. KVM sets it to 0 in this case, but
488 * to avoid this becoming an issue it's safer to simply
489 * treat this as unsupported for SNP guests.
490 */
491 if (!(leaf->eax & (BIT(1) | BIT(3))))
492 return -EINVAL;
493
494 compacted = true;
495 }
496
497 xsave_size = snp_cpuid_calc_xsave_size(xcr0 | xss, compacted);
498 if (!xsave_size)
499 return -EINVAL;
500
501 leaf->ebx = xsave_size;
502 }
503 break;
504 case 0x8000001E:
505 snp_cpuid_hv(ghcb, ctxt, &leaf_hv);
506
507 /* extended APIC ID */
508 leaf->eax = leaf_hv.eax;
509 /* compute ID */
510 leaf->ebx = (leaf->ebx & GENMASK(31, 8)) | (leaf_hv.ebx & GENMASK(7, 0));
511 /* node ID */
512 leaf->ecx = (leaf->ecx & GENMASK(31, 8)) | (leaf_hv.ecx & GENMASK(7, 0));
513 break;
514 default:
515 /* No fix-ups needed, use values as-is. */
516 break;
517 }
518
519 return 0;
520 }
521
522 /*
523 * Returns -EOPNOTSUPP if feature not enabled. Any other non-zero return value
524 * should be treated as fatal by caller.
525 */
526 static int __head
snp_cpuid(struct ghcb * ghcb,struct es_em_ctxt * ctxt,struct cpuid_leaf * leaf)527 snp_cpuid(struct ghcb *ghcb, struct es_em_ctxt *ctxt, struct cpuid_leaf *leaf)
528 {
529 const struct snp_cpuid_table *cpuid_table = snp_cpuid_get_table();
530
531 if (!cpuid_table->count)
532 return -EOPNOTSUPP;
533
534 if (!snp_cpuid_get_validated_func(leaf)) {
535 /*
536 * Some hypervisors will avoid keeping track of CPUID entries
537 * where all values are zero, since they can be handled the
538 * same as out-of-range values (all-zero). This is useful here
539 * as well as it allows virtually all guest configurations to
540 * work using a single SNP CPUID table.
541 *
542 * To allow for this, there is a need to distinguish between
543 * out-of-range entries and in-range zero entries, since the
544 * CPUID table entries are only a template that may need to be
545 * augmented with additional values for things like
546 * CPU-specific information during post-processing. So if it's
547 * not in the table, set the values to zero. Then, if they are
548 * within a valid CPUID range, proceed with post-processing
549 * using zeros as the initial values. Otherwise, skip
550 * post-processing and just return zeros immediately.
551 */
552 leaf->eax = leaf->ebx = leaf->ecx = leaf->edx = 0;
553
554 /* Skip post-processing for out-of-range zero leafs. */
555 if (!(leaf->fn <= RIP_REL_REF(cpuid_std_range_max) ||
556 (leaf->fn >= 0x40000000 && leaf->fn <= RIP_REL_REF(cpuid_hyp_range_max)) ||
557 (leaf->fn >= 0x80000000 && leaf->fn <= RIP_REL_REF(cpuid_ext_range_max))))
558 return 0;
559 }
560
561 return snp_cpuid_postprocess(ghcb, ctxt, leaf);
562 }
563
564 /*
565 * Boot VC Handler - This is the first VC handler during boot, there is no GHCB
566 * page yet, so it only supports the MSR based communication with the
567 * hypervisor and only the CPUID exit-code.
568 */
do_vc_no_ghcb(struct pt_regs * regs,unsigned long exit_code)569 void __head do_vc_no_ghcb(struct pt_regs *regs, unsigned long exit_code)
570 {
571 unsigned int subfn = lower_bits(regs->cx, 32);
572 unsigned int fn = lower_bits(regs->ax, 32);
573 struct cpuid_leaf leaf;
574 int ret;
575
576 /* Only CPUID is supported via MSR protocol */
577 if (exit_code != SVM_EXIT_CPUID)
578 goto fail;
579
580 leaf.fn = fn;
581 leaf.subfn = subfn;
582
583 ret = snp_cpuid(NULL, NULL, &leaf);
584 if (!ret)
585 goto cpuid_done;
586
587 if (ret != -EOPNOTSUPP)
588 goto fail;
589
590 if (__sev_cpuid_hv_msr(&leaf))
591 goto fail;
592
593 cpuid_done:
594 regs->ax = leaf.eax;
595 regs->bx = leaf.ebx;
596 regs->cx = leaf.ecx;
597 regs->dx = leaf.edx;
598
599 /*
600 * This is a VC handler and the #VC is only raised when SEV-ES is
601 * active, which means SEV must be active too. Do sanity checks on the
602 * CPUID results to make sure the hypervisor does not trick the kernel
603 * into the no-sev path. This could map sensitive data unencrypted and
604 * make it accessible to the hypervisor.
605 *
606 * In particular, check for:
607 * - Availability of CPUID leaf 0x8000001f
608 * - SEV CPUID bit.
609 *
610 * The hypervisor might still report the wrong C-bit position, but this
611 * can't be checked here.
612 */
613
614 if (fn == 0x80000000 && (regs->ax < 0x8000001f))
615 /* SEV leaf check */
616 goto fail;
617 else if ((fn == 0x8000001f && !(regs->ax & BIT(1))))
618 /* SEV bit */
619 goto fail;
620
621 /* Skip over the CPUID two-byte opcode */
622 regs->ip += 2;
623
624 return;
625
626 fail:
627 /* Terminate the guest */
628 sev_es_terminate(SEV_TERM_SET_GEN, GHCB_SEV_ES_GEN_REQ);
629 }
630
vc_insn_string_check(struct es_em_ctxt * ctxt,unsigned long address,bool write)631 static enum es_result vc_insn_string_check(struct es_em_ctxt *ctxt,
632 unsigned long address,
633 bool write)
634 {
635 if (user_mode(ctxt->regs) && fault_in_kernel_space(address)) {
636 ctxt->fi.vector = X86_TRAP_PF;
637 ctxt->fi.error_code = X86_PF_USER;
638 ctxt->fi.cr2 = address;
639 if (write)
640 ctxt->fi.error_code |= X86_PF_WRITE;
641
642 return ES_EXCEPTION;
643 }
644
645 return ES_OK;
646 }
647
vc_insn_string_read(struct es_em_ctxt * ctxt,void * src,char * buf,unsigned int data_size,unsigned int count,bool backwards)648 static enum es_result vc_insn_string_read(struct es_em_ctxt *ctxt,
649 void *src, char *buf,
650 unsigned int data_size,
651 unsigned int count,
652 bool backwards)
653 {
654 int i, b = backwards ? -1 : 1;
655 unsigned long address = (unsigned long)src;
656 enum es_result ret;
657
658 ret = vc_insn_string_check(ctxt, address, false);
659 if (ret != ES_OK)
660 return ret;
661
662 for (i = 0; i < count; i++) {
663 void *s = src + (i * data_size * b);
664 char *d = buf + (i * data_size);
665
666 ret = vc_read_mem(ctxt, s, d, data_size);
667 if (ret != ES_OK)
668 break;
669 }
670
671 return ret;
672 }
673
vc_insn_string_write(struct es_em_ctxt * ctxt,void * dst,char * buf,unsigned int data_size,unsigned int count,bool backwards)674 static enum es_result vc_insn_string_write(struct es_em_ctxt *ctxt,
675 void *dst, char *buf,
676 unsigned int data_size,
677 unsigned int count,
678 bool backwards)
679 {
680 int i, s = backwards ? -1 : 1;
681 unsigned long address = (unsigned long)dst;
682 enum es_result ret;
683
684 ret = vc_insn_string_check(ctxt, address, true);
685 if (ret != ES_OK)
686 return ret;
687
688 for (i = 0; i < count; i++) {
689 void *d = dst + (i * data_size * s);
690 char *b = buf + (i * data_size);
691
692 ret = vc_write_mem(ctxt, d, b, data_size);
693 if (ret != ES_OK)
694 break;
695 }
696
697 return ret;
698 }
699
700 #define IOIO_TYPE_STR BIT(2)
701 #define IOIO_TYPE_IN 1
702 #define IOIO_TYPE_INS (IOIO_TYPE_IN | IOIO_TYPE_STR)
703 #define IOIO_TYPE_OUT 0
704 #define IOIO_TYPE_OUTS (IOIO_TYPE_OUT | IOIO_TYPE_STR)
705
706 #define IOIO_REP BIT(3)
707
708 #define IOIO_ADDR_64 BIT(9)
709 #define IOIO_ADDR_32 BIT(8)
710 #define IOIO_ADDR_16 BIT(7)
711
712 #define IOIO_DATA_32 BIT(6)
713 #define IOIO_DATA_16 BIT(5)
714 #define IOIO_DATA_8 BIT(4)
715
716 #define IOIO_SEG_ES (0 << 10)
717 #define IOIO_SEG_DS (3 << 10)
718
vc_ioio_exitinfo(struct es_em_ctxt * ctxt,u64 * exitinfo)719 static enum es_result vc_ioio_exitinfo(struct es_em_ctxt *ctxt, u64 *exitinfo)
720 {
721 struct insn *insn = &ctxt->insn;
722 size_t size;
723 u64 port;
724
725 *exitinfo = 0;
726
727 switch (insn->opcode.bytes[0]) {
728 /* INS opcodes */
729 case 0x6c:
730 case 0x6d:
731 *exitinfo |= IOIO_TYPE_INS;
732 *exitinfo |= IOIO_SEG_ES;
733 port = ctxt->regs->dx & 0xffff;
734 break;
735
736 /* OUTS opcodes */
737 case 0x6e:
738 case 0x6f:
739 *exitinfo |= IOIO_TYPE_OUTS;
740 *exitinfo |= IOIO_SEG_DS;
741 port = ctxt->regs->dx & 0xffff;
742 break;
743
744 /* IN immediate opcodes */
745 case 0xe4:
746 case 0xe5:
747 *exitinfo |= IOIO_TYPE_IN;
748 port = (u8)insn->immediate.value & 0xffff;
749 break;
750
751 /* OUT immediate opcodes */
752 case 0xe6:
753 case 0xe7:
754 *exitinfo |= IOIO_TYPE_OUT;
755 port = (u8)insn->immediate.value & 0xffff;
756 break;
757
758 /* IN register opcodes */
759 case 0xec:
760 case 0xed:
761 *exitinfo |= IOIO_TYPE_IN;
762 port = ctxt->regs->dx & 0xffff;
763 break;
764
765 /* OUT register opcodes */
766 case 0xee:
767 case 0xef:
768 *exitinfo |= IOIO_TYPE_OUT;
769 port = ctxt->regs->dx & 0xffff;
770 break;
771
772 default:
773 return ES_DECODE_FAILED;
774 }
775
776 *exitinfo |= port << 16;
777
778 switch (insn->opcode.bytes[0]) {
779 case 0x6c:
780 case 0x6e:
781 case 0xe4:
782 case 0xe6:
783 case 0xec:
784 case 0xee:
785 /* Single byte opcodes */
786 *exitinfo |= IOIO_DATA_8;
787 size = 1;
788 break;
789 default:
790 /* Length determined by instruction parsing */
791 *exitinfo |= (insn->opnd_bytes == 2) ? IOIO_DATA_16
792 : IOIO_DATA_32;
793 size = (insn->opnd_bytes == 2) ? 2 : 4;
794 }
795
796 switch (insn->addr_bytes) {
797 case 2:
798 *exitinfo |= IOIO_ADDR_16;
799 break;
800 case 4:
801 *exitinfo |= IOIO_ADDR_32;
802 break;
803 case 8:
804 *exitinfo |= IOIO_ADDR_64;
805 break;
806 }
807
808 if (insn_has_rep_prefix(insn))
809 *exitinfo |= IOIO_REP;
810
811 return vc_ioio_check(ctxt, (u16)port, size);
812 }
813
vc_handle_ioio(struct ghcb * ghcb,struct es_em_ctxt * ctxt)814 static enum es_result vc_handle_ioio(struct ghcb *ghcb, struct es_em_ctxt *ctxt)
815 {
816 struct pt_regs *regs = ctxt->regs;
817 u64 exit_info_1, exit_info_2;
818 enum es_result ret;
819
820 ret = vc_ioio_exitinfo(ctxt, &exit_info_1);
821 if (ret != ES_OK)
822 return ret;
823
824 if (exit_info_1 & IOIO_TYPE_STR) {
825
826 /* (REP) INS/OUTS */
827
828 bool df = ((regs->flags & X86_EFLAGS_DF) == X86_EFLAGS_DF);
829 unsigned int io_bytes, exit_bytes;
830 unsigned int ghcb_count, op_count;
831 unsigned long es_base;
832 u64 sw_scratch;
833
834 /*
835 * For the string variants with rep prefix the amount of in/out
836 * operations per #VC exception is limited so that the kernel
837 * has a chance to take interrupts and re-schedule while the
838 * instruction is emulated.
839 */
840 io_bytes = (exit_info_1 >> 4) & 0x7;
841 ghcb_count = sizeof(ghcb->shared_buffer) / io_bytes;
842
843 op_count = (exit_info_1 & IOIO_REP) ? regs->cx : 1;
844 exit_info_2 = min(op_count, ghcb_count);
845 exit_bytes = exit_info_2 * io_bytes;
846
847 es_base = insn_get_seg_base(ctxt->regs, INAT_SEG_REG_ES);
848
849 /* Read bytes of OUTS into the shared buffer */
850 if (!(exit_info_1 & IOIO_TYPE_IN)) {
851 ret = vc_insn_string_read(ctxt,
852 (void *)(es_base + regs->si),
853 ghcb->shared_buffer, io_bytes,
854 exit_info_2, df);
855 if (ret)
856 return ret;
857 }
858
859 /*
860 * Issue an VMGEXIT to the HV to consume the bytes from the
861 * shared buffer or to have it write them into the shared buffer
862 * depending on the instruction: OUTS or INS.
863 */
864 sw_scratch = __pa(ghcb) + offsetof(struct ghcb, shared_buffer);
865 ghcb_set_sw_scratch(ghcb, sw_scratch);
866 ret = sev_es_ghcb_hv_call(ghcb, ctxt, SVM_EXIT_IOIO,
867 exit_info_1, exit_info_2);
868 if (ret != ES_OK)
869 return ret;
870
871 /* Read bytes from shared buffer into the guest's destination. */
872 if (exit_info_1 & IOIO_TYPE_IN) {
873 ret = vc_insn_string_write(ctxt,
874 (void *)(es_base + regs->di),
875 ghcb->shared_buffer, io_bytes,
876 exit_info_2, df);
877 if (ret)
878 return ret;
879
880 if (df)
881 regs->di -= exit_bytes;
882 else
883 regs->di += exit_bytes;
884 } else {
885 if (df)
886 regs->si -= exit_bytes;
887 else
888 regs->si += exit_bytes;
889 }
890
891 if (exit_info_1 & IOIO_REP)
892 regs->cx -= exit_info_2;
893
894 ret = regs->cx ? ES_RETRY : ES_OK;
895
896 } else {
897
898 /* IN/OUT into/from rAX */
899
900 int bits = (exit_info_1 & 0x70) >> 1;
901 u64 rax = 0;
902
903 if (!(exit_info_1 & IOIO_TYPE_IN))
904 rax = lower_bits(regs->ax, bits);
905
906 ghcb_set_rax(ghcb, rax);
907
908 ret = sev_es_ghcb_hv_call(ghcb, ctxt, SVM_EXIT_IOIO, exit_info_1, 0);
909 if (ret != ES_OK)
910 return ret;
911
912 if (exit_info_1 & IOIO_TYPE_IN) {
913 if (!ghcb_rax_is_valid(ghcb))
914 return ES_VMM_ERROR;
915 regs->ax = lower_bits(ghcb->save.rax, bits);
916 }
917 }
918
919 return ret;
920 }
921
vc_handle_cpuid_snp(struct ghcb * ghcb,struct es_em_ctxt * ctxt)922 static int vc_handle_cpuid_snp(struct ghcb *ghcb, struct es_em_ctxt *ctxt)
923 {
924 struct pt_regs *regs = ctxt->regs;
925 struct cpuid_leaf leaf;
926 int ret;
927
928 leaf.fn = regs->ax;
929 leaf.subfn = regs->cx;
930 ret = snp_cpuid(ghcb, ctxt, &leaf);
931 if (!ret) {
932 regs->ax = leaf.eax;
933 regs->bx = leaf.ebx;
934 regs->cx = leaf.ecx;
935 regs->dx = leaf.edx;
936 }
937
938 return ret;
939 }
940
vc_handle_cpuid(struct ghcb * ghcb,struct es_em_ctxt * ctxt)941 static enum es_result vc_handle_cpuid(struct ghcb *ghcb,
942 struct es_em_ctxt *ctxt)
943 {
944 struct pt_regs *regs = ctxt->regs;
945 u32 cr4 = native_read_cr4();
946 enum es_result ret;
947 int snp_cpuid_ret;
948
949 snp_cpuid_ret = vc_handle_cpuid_snp(ghcb, ctxt);
950 if (!snp_cpuid_ret)
951 return ES_OK;
952 if (snp_cpuid_ret != -EOPNOTSUPP)
953 return ES_VMM_ERROR;
954
955 ghcb_set_rax(ghcb, regs->ax);
956 ghcb_set_rcx(ghcb, regs->cx);
957
958 if (cr4 & X86_CR4_OSXSAVE)
959 /* Safe to read xcr0 */
960 ghcb_set_xcr0(ghcb, xgetbv(XCR_XFEATURE_ENABLED_MASK));
961 else
962 /* xgetbv will cause #GP - use reset value for xcr0 */
963 ghcb_set_xcr0(ghcb, 1);
964
965 ret = sev_es_ghcb_hv_call(ghcb, ctxt, SVM_EXIT_CPUID, 0, 0);
966 if (ret != ES_OK)
967 return ret;
968
969 if (!(ghcb_rax_is_valid(ghcb) &&
970 ghcb_rbx_is_valid(ghcb) &&
971 ghcb_rcx_is_valid(ghcb) &&
972 ghcb_rdx_is_valid(ghcb)))
973 return ES_VMM_ERROR;
974
975 regs->ax = ghcb->save.rax;
976 regs->bx = ghcb->save.rbx;
977 regs->cx = ghcb->save.rcx;
978 regs->dx = ghcb->save.rdx;
979
980 return ES_OK;
981 }
982
vc_handle_rdtsc(struct ghcb * ghcb,struct es_em_ctxt * ctxt,unsigned long exit_code)983 static enum es_result vc_handle_rdtsc(struct ghcb *ghcb,
984 struct es_em_ctxt *ctxt,
985 unsigned long exit_code)
986 {
987 bool rdtscp = (exit_code == SVM_EXIT_RDTSCP);
988 enum es_result ret;
989
990 ret = sev_es_ghcb_hv_call(ghcb, ctxt, exit_code, 0, 0);
991 if (ret != ES_OK)
992 return ret;
993
994 if (!(ghcb_rax_is_valid(ghcb) && ghcb_rdx_is_valid(ghcb) &&
995 (!rdtscp || ghcb_rcx_is_valid(ghcb))))
996 return ES_VMM_ERROR;
997
998 ctxt->regs->ax = ghcb->save.rax;
999 ctxt->regs->dx = ghcb->save.rdx;
1000 if (rdtscp)
1001 ctxt->regs->cx = ghcb->save.rcx;
1002
1003 return ES_OK;
1004 }
1005
1006 struct cc_setup_data {
1007 struct setup_data header;
1008 u32 cc_blob_address;
1009 };
1010
1011 /*
1012 * Search for a Confidential Computing blob passed in as a setup_data entry
1013 * via the Linux Boot Protocol.
1014 */
1015 static __head
find_cc_blob_setup_data(struct boot_params * bp)1016 struct cc_blob_sev_info *find_cc_blob_setup_data(struct boot_params *bp)
1017 {
1018 struct cc_setup_data *sd = NULL;
1019 struct setup_data *hdr;
1020
1021 hdr = (struct setup_data *)bp->hdr.setup_data;
1022
1023 while (hdr) {
1024 if (hdr->type == SETUP_CC_BLOB) {
1025 sd = (struct cc_setup_data *)hdr;
1026 return (struct cc_blob_sev_info *)(unsigned long)sd->cc_blob_address;
1027 }
1028 hdr = (struct setup_data *)hdr->next;
1029 }
1030
1031 return NULL;
1032 }
1033
1034 /*
1035 * Initialize the kernel's copy of the SNP CPUID table, and set up the
1036 * pointer that will be used to access it.
1037 *
1038 * Maintaining a direct mapping of the SNP CPUID table used by firmware would
1039 * be possible as an alternative, but the approach is brittle since the
1040 * mapping needs to be updated in sync with all the changes to virtual memory
1041 * layout and related mapping facilities throughout the boot process.
1042 */
setup_cpuid_table(const struct cc_blob_sev_info * cc_info)1043 static void __head setup_cpuid_table(const struct cc_blob_sev_info *cc_info)
1044 {
1045 const struct snp_cpuid_table *cpuid_table_fw, *cpuid_table;
1046 int i;
1047
1048 if (!cc_info || !cc_info->cpuid_phys || cc_info->cpuid_len < PAGE_SIZE)
1049 sev_es_terminate(SEV_TERM_SET_LINUX, GHCB_TERM_CPUID);
1050
1051 cpuid_table_fw = (const struct snp_cpuid_table *)cc_info->cpuid_phys;
1052 if (!cpuid_table_fw->count || cpuid_table_fw->count > SNP_CPUID_COUNT_MAX)
1053 sev_es_terminate(SEV_TERM_SET_LINUX, GHCB_TERM_CPUID);
1054
1055 cpuid_table = snp_cpuid_get_table();
1056 memcpy((void *)cpuid_table, cpuid_table_fw, sizeof(*cpuid_table));
1057
1058 /* Initialize CPUID ranges for range-checking. */
1059 for (i = 0; i < cpuid_table->count; i++) {
1060 const struct snp_cpuid_fn *fn = &cpuid_table->fn[i];
1061
1062 if (fn->eax_in == 0x0)
1063 RIP_REL_REF(cpuid_std_range_max) = fn->eax;
1064 else if (fn->eax_in == 0x40000000)
1065 RIP_REL_REF(cpuid_hyp_range_max) = fn->eax;
1066 else if (fn->eax_in == 0x80000000)
1067 RIP_REL_REF(cpuid_ext_range_max) = fn->eax;
1068 }
1069 }
1070
pvalidate_pages(struct snp_psc_desc * desc)1071 static void pvalidate_pages(struct snp_psc_desc *desc)
1072 {
1073 struct psc_entry *e;
1074 unsigned long vaddr;
1075 unsigned int size;
1076 unsigned int i;
1077 bool validate;
1078 int rc;
1079
1080 for (i = 0; i <= desc->hdr.end_entry; i++) {
1081 e = &desc->entries[i];
1082
1083 vaddr = (unsigned long)pfn_to_kaddr(e->gfn);
1084 size = e->pagesize ? RMP_PG_SIZE_2M : RMP_PG_SIZE_4K;
1085 validate = e->operation == SNP_PAGE_STATE_PRIVATE;
1086
1087 rc = pvalidate(vaddr, size, validate);
1088 if (rc == PVALIDATE_FAIL_SIZEMISMATCH && size == RMP_PG_SIZE_2M) {
1089 unsigned long vaddr_end = vaddr + PMD_SIZE;
1090
1091 for (; vaddr < vaddr_end; vaddr += PAGE_SIZE) {
1092 rc = pvalidate(vaddr, RMP_PG_SIZE_4K, validate);
1093 if (rc)
1094 break;
1095 }
1096 }
1097
1098 if (rc) {
1099 WARN(1, "Failed to validate address 0x%lx ret %d", vaddr, rc);
1100 sev_es_terminate(SEV_TERM_SET_LINUX, GHCB_TERM_PVALIDATE);
1101 }
1102 }
1103 }
1104
vmgexit_psc(struct ghcb * ghcb,struct snp_psc_desc * desc)1105 static int vmgexit_psc(struct ghcb *ghcb, struct snp_psc_desc *desc)
1106 {
1107 int cur_entry, end_entry, ret = 0;
1108 struct snp_psc_desc *data;
1109 struct es_em_ctxt ctxt;
1110
1111 vc_ghcb_invalidate(ghcb);
1112
1113 /* Copy the input desc into GHCB shared buffer */
1114 data = (struct snp_psc_desc *)ghcb->shared_buffer;
1115 memcpy(ghcb->shared_buffer, desc, min_t(int, GHCB_SHARED_BUF_SIZE, sizeof(*desc)));
1116
1117 /*
1118 * As per the GHCB specification, the hypervisor can resume the guest
1119 * before processing all the entries. Check whether all the entries
1120 * are processed. If not, then keep retrying. Note, the hypervisor
1121 * will update the data memory directly to indicate the status, so
1122 * reference the data->hdr everywhere.
1123 *
1124 * The strategy here is to wait for the hypervisor to change the page
1125 * state in the RMP table before guest accesses the memory pages. If the
1126 * page state change was not successful, then later memory access will
1127 * result in a crash.
1128 */
1129 cur_entry = data->hdr.cur_entry;
1130 end_entry = data->hdr.end_entry;
1131
1132 while (data->hdr.cur_entry <= data->hdr.end_entry) {
1133 ghcb_set_sw_scratch(ghcb, (u64)__pa(data));
1134
1135 /* This will advance the shared buffer data points to. */
1136 ret = sev_es_ghcb_hv_call(ghcb, &ctxt, SVM_VMGEXIT_PSC, 0, 0);
1137
1138 /*
1139 * Page State Change VMGEXIT can pass error code through
1140 * exit_info_2.
1141 */
1142 if (WARN(ret || ghcb->save.sw_exit_info_2,
1143 "SNP: PSC failed ret=%d exit_info_2=%llx\n",
1144 ret, ghcb->save.sw_exit_info_2)) {
1145 ret = 1;
1146 goto out;
1147 }
1148
1149 /* Verify that reserved bit is not set */
1150 if (WARN(data->hdr.reserved, "Reserved bit is set in the PSC header\n")) {
1151 ret = 1;
1152 goto out;
1153 }
1154
1155 /*
1156 * Sanity check that entry processing is not going backwards.
1157 * This will happen only if hypervisor is tricking us.
1158 */
1159 if (WARN(data->hdr.end_entry > end_entry || cur_entry > data->hdr.cur_entry,
1160 "SNP: PSC processing going backward, end_entry %d (got %d) cur_entry %d (got %d)\n",
1161 end_entry, data->hdr.end_entry, cur_entry, data->hdr.cur_entry)) {
1162 ret = 1;
1163 goto out;
1164 }
1165 }
1166
1167 out:
1168 return ret;
1169 }
1170