1e759959fSBrijesh Singh // SPDX-License-Identifier: GPL-2.0
2e759959fSBrijesh Singh /*
3e759959fSBrijesh Singh * AMD Encrypted Register State Support
4e759959fSBrijesh Singh *
5e759959fSBrijesh Singh * Author: Joerg Roedel <jroedel@suse.de>
6e759959fSBrijesh Singh *
7e759959fSBrijesh Singh * This file is not compiled stand-alone. It contains code shared
8e759959fSBrijesh Singh * between the pre-decompression boot code and the running Linux kernel
9e759959fSBrijesh Singh * and is included directly into both code-bases.
10e759959fSBrijesh Singh */
11e759959fSBrijesh Singh
12e759959fSBrijesh Singh #ifndef __BOOT_COMPRESSED
13e759959fSBrijesh Singh #define error(v) pr_err(v)
14e759959fSBrijesh Singh #define has_cpuflag(f) boot_cpu_has(f)
156c321179STom Lendacky #else
166c321179STom Lendacky #undef WARN
176c321179STom Lendacky #define WARN(condition, format...) (!!(condition))
18e759959fSBrijesh Singh #endif
19e759959fSBrijesh Singh
20801baa69SMichael Roth /* I/O parameters for CPUID-related helpers */
21801baa69SMichael Roth struct cpuid_leaf {
22801baa69SMichael Roth u32 fn;
23801baa69SMichael Roth u32 subfn;
24801baa69SMichael Roth u32 eax;
25801baa69SMichael Roth u32 ebx;
26801baa69SMichael Roth u32 ecx;
27801baa69SMichael Roth u32 edx;
28801baa69SMichael Roth };
29801baa69SMichael Roth
302ea29c5aSBrijesh Singh /*
31ee0bfa08SMichael Roth * Individual entries of the SNP CPUID table, as defined by the SNP
32ee0bfa08SMichael Roth * Firmware ABI, Revision 0.9, Section 7.1, Table 14.
33ee0bfa08SMichael Roth */
34ee0bfa08SMichael Roth struct snp_cpuid_fn {
35ee0bfa08SMichael Roth u32 eax_in;
36ee0bfa08SMichael Roth u32 ecx_in;
37ee0bfa08SMichael Roth u64 xcr0_in;
38ee0bfa08SMichael Roth u64 xss_in;
39ee0bfa08SMichael Roth u32 eax;
40ee0bfa08SMichael Roth u32 ebx;
41ee0bfa08SMichael Roth u32 ecx;
42ee0bfa08SMichael Roth u32 edx;
43ee0bfa08SMichael Roth u64 __reserved;
44ee0bfa08SMichael Roth } __packed;
45ee0bfa08SMichael Roth
46ee0bfa08SMichael Roth /*
47ee0bfa08SMichael Roth * SNP CPUID table, as defined by the SNP Firmware ABI, Revision 0.9,
48ee0bfa08SMichael Roth * Section 8.14.2.6. Also noted there is the SNP firmware-enforced limit
49ee0bfa08SMichael Roth * of 64 entries per CPUID table.
50ee0bfa08SMichael Roth */
51ee0bfa08SMichael Roth #define SNP_CPUID_COUNT_MAX 64
52ee0bfa08SMichael Roth
53ee0bfa08SMichael Roth struct snp_cpuid_table {
54ee0bfa08SMichael Roth u32 count;
55ee0bfa08SMichael Roth u32 __reserved1;
56ee0bfa08SMichael Roth u64 __reserved2;
57ee0bfa08SMichael Roth struct snp_cpuid_fn fn[SNP_CPUID_COUNT_MAX];
58ee0bfa08SMichael Roth } __packed;
59ee0bfa08SMichael Roth
60ee0bfa08SMichael Roth /*
612ea29c5aSBrijesh Singh * Since feature negotiation related variables are set early in the boot
622ea29c5aSBrijesh Singh * process they must reside in the .data section so as not to be zeroed
632ea29c5aSBrijesh Singh * out when the .bss section is later cleared.
642ea29c5aSBrijesh Singh *
652ea29c5aSBrijesh Singh * GHCB protocol version negotiated with the hypervisor.
662ea29c5aSBrijesh Singh */
672ea29c5aSBrijesh Singh static u16 ghcb_version __ro_after_init;
682ea29c5aSBrijesh Singh
69ee0bfa08SMichael Roth /* Copy of the SNP firmware's CPUID page. */
70ee0bfa08SMichael Roth static struct snp_cpuid_table cpuid_table_copy __ro_after_init;
71ee0bfa08SMichael Roth
72ee0bfa08SMichael Roth /*
73ee0bfa08SMichael Roth * These will be initialized based on CPUID table so that non-present
74ee0bfa08SMichael Roth * all-zero leaves (for sparse tables) can be differentiated from
75ee0bfa08SMichael Roth * invalid/out-of-range leaves. This is needed since all-zero leaves
76ee0bfa08SMichael Roth * still need to be post-processed.
77ee0bfa08SMichael Roth */
78ee0bfa08SMichael Roth static u32 cpuid_std_range_max __ro_after_init;
79ee0bfa08SMichael Roth static u32 cpuid_hyp_range_max __ro_after_init;
80ee0bfa08SMichael Roth static u32 cpuid_ext_range_max __ro_after_init;
81ee0bfa08SMichael Roth
sev_es_check_cpu_features(void)82e759959fSBrijesh Singh static bool __init sev_es_check_cpu_features(void)
83e759959fSBrijesh Singh {
84e759959fSBrijesh Singh if (!has_cpuflag(X86_FEATURE_RDRAND)) {
85e759959fSBrijesh Singh error("RDRAND instruction not supported - no trusted source of randomness available\n");
86e759959fSBrijesh Singh return false;
87e759959fSBrijesh Singh }
88e759959fSBrijesh Singh
89e759959fSBrijesh Singh return true;
90e759959fSBrijesh Singh }
91e759959fSBrijesh Singh
92*56408ed9SArd Biesheuvel static void __head __noreturn
sev_es_terminate(unsigned int set,unsigned int reason)93*56408ed9SArd Biesheuvel sev_es_terminate(unsigned int set, unsigned int reason)
94e759959fSBrijesh Singh {
95b81fc74dSBrijesh Singh u64 val = GHCB_MSR_TERM_REQ;
96e759959fSBrijesh Singh
976c0f74d6SBrijesh Singh /* Tell the hypervisor what went wrong. */
986c0f74d6SBrijesh Singh val |= GHCB_SEV_TERM_REASON(set, reason);
99e759959fSBrijesh Singh
100e759959fSBrijesh Singh /* Request Guest Termination from Hypvervisor */
101e759959fSBrijesh Singh sev_es_wr_ghcb_msr(val);
102e759959fSBrijesh Singh VMGEXIT();
103e759959fSBrijesh Singh
104e759959fSBrijesh Singh while (true)
105e759959fSBrijesh Singh asm volatile("hlt\n" : : : "memory");
106e759959fSBrijesh Singh }
107e759959fSBrijesh Singh
108cbd3d4f7SBrijesh Singh /*
109cbd3d4f7SBrijesh Singh * The hypervisor features are available from GHCB version 2 onward.
110cbd3d4f7SBrijesh Singh */
get_hv_features(void)111cbd3d4f7SBrijesh Singh static u64 get_hv_features(void)
112cbd3d4f7SBrijesh Singh {
113cbd3d4f7SBrijesh Singh u64 val;
114cbd3d4f7SBrijesh Singh
115cbd3d4f7SBrijesh Singh if (ghcb_version < 2)
116cbd3d4f7SBrijesh Singh return 0;
117cbd3d4f7SBrijesh Singh
118cbd3d4f7SBrijesh Singh sev_es_wr_ghcb_msr(GHCB_MSR_HV_FT_REQ);
119cbd3d4f7SBrijesh Singh VMGEXIT();
120cbd3d4f7SBrijesh Singh
121cbd3d4f7SBrijesh Singh val = sev_es_rd_ghcb_msr();
122cbd3d4f7SBrijesh Singh if (GHCB_RESP_CODE(val) != GHCB_MSR_HV_FT_RESP)
123cbd3d4f7SBrijesh Singh return 0;
124cbd3d4f7SBrijesh Singh
125cbd3d4f7SBrijesh Singh return GHCB_MSR_HV_FT_RESP_VAL(val);
126cbd3d4f7SBrijesh Singh }
127cbd3d4f7SBrijesh Singh
snp_register_ghcb_early(unsigned long paddr)12895d33bfaSBrijesh Singh static void snp_register_ghcb_early(unsigned long paddr)
12987294bdbSBrijesh Singh {
13087294bdbSBrijesh Singh unsigned long pfn = paddr >> PAGE_SHIFT;
13187294bdbSBrijesh Singh u64 val;
13287294bdbSBrijesh Singh
13387294bdbSBrijesh Singh sev_es_wr_ghcb_msr(GHCB_MSR_REG_GPA_REQ_VAL(pfn));
13487294bdbSBrijesh Singh VMGEXIT();
13587294bdbSBrijesh Singh
13687294bdbSBrijesh Singh val = sev_es_rd_ghcb_msr();
13787294bdbSBrijesh Singh
13887294bdbSBrijesh Singh /* If the response GPA is not ours then abort the guest */
13987294bdbSBrijesh Singh if ((GHCB_RESP_CODE(val) != GHCB_MSR_REG_GPA_RESP) ||
14087294bdbSBrijesh Singh (GHCB_MSR_REG_GPA_RESP_VAL(val) != pfn))
14187294bdbSBrijesh Singh sev_es_terminate(SEV_TERM_SET_LINUX, GHCB_TERM_REGISTER);
14287294bdbSBrijesh Singh }
14387294bdbSBrijesh Singh
sev_es_negotiate_protocol(void)144e759959fSBrijesh Singh static bool sev_es_negotiate_protocol(void)
145e759959fSBrijesh Singh {
146e759959fSBrijesh Singh u64 val;
147e759959fSBrijesh Singh
148e759959fSBrijesh Singh /* Do the GHCB protocol version negotiation */
149b81fc74dSBrijesh Singh sev_es_wr_ghcb_msr(GHCB_MSR_SEV_INFO_REQ);
150e759959fSBrijesh Singh VMGEXIT();
151e759959fSBrijesh Singh val = sev_es_rd_ghcb_msr();
152e759959fSBrijesh Singh
153b81fc74dSBrijesh Singh if (GHCB_MSR_INFO(val) != GHCB_MSR_SEV_INFO_RESP)
154e759959fSBrijesh Singh return false;
155e759959fSBrijesh Singh
1562ea29c5aSBrijesh Singh if (GHCB_MSR_PROTO_MAX(val) < GHCB_PROTOCOL_MIN ||
1572ea29c5aSBrijesh Singh GHCB_MSR_PROTO_MIN(val) > GHCB_PROTOCOL_MAX)
158e759959fSBrijesh Singh return false;
159e759959fSBrijesh Singh
1602ea29c5aSBrijesh Singh ghcb_version = min_t(size_t, GHCB_MSR_PROTO_MAX(val), GHCB_PROTOCOL_MAX);
1612ea29c5aSBrijesh Singh
162e759959fSBrijesh Singh return true;
163e759959fSBrijesh Singh }
164e759959fSBrijesh Singh
vc_ghcb_invalidate(struct ghcb * ghcb)165e759959fSBrijesh Singh static __always_inline void vc_ghcb_invalidate(struct ghcb *ghcb)
166e759959fSBrijesh Singh {
167a50c5bebSTom Lendacky ghcb->save.sw_exit_code = 0;
1682c36d87bSPeter Zijlstra __builtin_memset(ghcb->save.valid_bitmap, 0, sizeof(ghcb->save.valid_bitmap));
169e759959fSBrijesh Singh }
170e759959fSBrijesh Singh
vc_decoding_needed(unsigned long exit_code)171e759959fSBrijesh Singh static bool vc_decoding_needed(unsigned long exit_code)
172e759959fSBrijesh Singh {
173e759959fSBrijesh Singh /* Exceptions don't require to decode the instruction */
174e759959fSBrijesh Singh return !(exit_code >= SVM_EXIT_EXCP_BASE &&
175e759959fSBrijesh Singh exit_code <= SVM_EXIT_LAST_EXCP);
176e759959fSBrijesh Singh }
177e759959fSBrijesh Singh
vc_init_em_ctxt(struct es_em_ctxt * ctxt,struct pt_regs * regs,unsigned long exit_code)178e759959fSBrijesh Singh static enum es_result vc_init_em_ctxt(struct es_em_ctxt *ctxt,
179e759959fSBrijesh Singh struct pt_regs *regs,
180e759959fSBrijesh Singh unsigned long exit_code)
181e759959fSBrijesh Singh {
182e759959fSBrijesh Singh enum es_result ret = ES_OK;
183e759959fSBrijesh Singh
184e759959fSBrijesh Singh memset(ctxt, 0, sizeof(*ctxt));
185e759959fSBrijesh Singh ctxt->regs = regs;
186e759959fSBrijesh Singh
187e759959fSBrijesh Singh if (vc_decoding_needed(exit_code))
188e759959fSBrijesh Singh ret = vc_decode_insn(ctxt);
189e759959fSBrijesh Singh
190e759959fSBrijesh Singh return ret;
191e759959fSBrijesh Singh }
192e759959fSBrijesh Singh
vc_finish_insn(struct es_em_ctxt * ctxt)193e759959fSBrijesh Singh static void vc_finish_insn(struct es_em_ctxt *ctxt)
194e759959fSBrijesh Singh {
195e759959fSBrijesh Singh ctxt->regs->ip += ctxt->insn.length;
196e759959fSBrijesh Singh }
197e759959fSBrijesh Singh
verify_exception_info(struct ghcb * ghcb,struct es_em_ctxt * ctxt)198c688bd5dSBorislav Petkov static enum es_result verify_exception_info(struct ghcb *ghcb, struct es_em_ctxt *ctxt)
199e759959fSBrijesh Singh {
200c688bd5dSBorislav Petkov u32 ret;
201e759959fSBrijesh Singh
202c688bd5dSBorislav Petkov ret = ghcb->save.sw_exit_info_1 & GENMASK_ULL(31, 0);
203c688bd5dSBorislav Petkov if (!ret)
204c688bd5dSBorislav Petkov return ES_OK;
205e759959fSBrijesh Singh
206c688bd5dSBorislav Petkov if (ret == 1) {
207e759959fSBrijesh Singh u64 info = ghcb->save.sw_exit_info_2;
2080621210aSColin Ian King unsigned long v = info & SVM_EVTINJ_VEC_MASK;
209e759959fSBrijesh Singh
210e759959fSBrijesh Singh /* Check if exception information from hypervisor is sane. */
211e759959fSBrijesh Singh if ((info & SVM_EVTINJ_VALID) &&
212e759959fSBrijesh Singh ((v == X86_TRAP_GP) || (v == X86_TRAP_UD)) &&
213e759959fSBrijesh Singh ((info & SVM_EVTINJ_TYPE_MASK) == SVM_EVTINJ_TYPE_EXEPT)) {
214e759959fSBrijesh Singh ctxt->fi.vector = v;
215c688bd5dSBorislav Petkov
216e759959fSBrijesh Singh if (info & SVM_EVTINJ_VALID_ERR)
217e759959fSBrijesh Singh ctxt->fi.error_code = info >> 32;
218c688bd5dSBorislav Petkov
219c688bd5dSBorislav Petkov return ES_EXCEPTION;
220e759959fSBrijesh Singh }
221e759959fSBrijesh Singh }
222e759959fSBrijesh Singh
223c688bd5dSBorislav Petkov return ES_VMM_ERROR;
224c688bd5dSBorislav Petkov }
225c688bd5dSBorislav Petkov
sev_es_ghcb_hv_call(struct ghcb * ghcb,struct es_em_ctxt * ctxt,u64 exit_code,u64 exit_info_1,u64 exit_info_2)2265bb6c1d1SBorislav Petkov static enum es_result sev_es_ghcb_hv_call(struct ghcb *ghcb,
2275bb6c1d1SBorislav Petkov struct es_em_ctxt *ctxt,
2285bb6c1d1SBorislav Petkov u64 exit_code, u64 exit_info_1,
2295bb6c1d1SBorislav Petkov u64 exit_info_2)
230c688bd5dSBorislav Petkov {
231c688bd5dSBorislav Petkov /* Fill in protocol and format specifiers */
2322ea29c5aSBrijesh Singh ghcb->protocol_version = ghcb_version;
233c688bd5dSBorislav Petkov ghcb->ghcb_usage = GHCB_DEFAULT_USAGE;
234c688bd5dSBorislav Petkov
235c688bd5dSBorislav Petkov ghcb_set_sw_exit_code(ghcb, exit_code);
236c688bd5dSBorislav Petkov ghcb_set_sw_exit_info_1(ghcb, exit_info_1);
237c688bd5dSBorislav Petkov ghcb_set_sw_exit_info_2(ghcb, exit_info_2);
238c688bd5dSBorislav Petkov
239c688bd5dSBorislav Petkov sev_es_wr_ghcb_msr(__pa(ghcb));
240c688bd5dSBorislav Petkov VMGEXIT();
241c688bd5dSBorislav Petkov
242c688bd5dSBorislav Petkov return verify_exception_info(ghcb, ctxt);
243e759959fSBrijesh Singh }
244e759959fSBrijesh Singh
__sev_cpuid_hv(u32 fn,int reg_idx,u32 * reg)245801baa69SMichael Roth static int __sev_cpuid_hv(u32 fn, int reg_idx, u32 *reg)
246801baa69SMichael Roth {
247801baa69SMichael Roth u64 val;
248801baa69SMichael Roth
249801baa69SMichael Roth sev_es_wr_ghcb_msr(GHCB_CPUID_REQ(fn, reg_idx));
250801baa69SMichael Roth VMGEXIT();
251801baa69SMichael Roth val = sev_es_rd_ghcb_msr();
252801baa69SMichael Roth if (GHCB_RESP_CODE(val) != GHCB_MSR_CPUID_RESP)
253801baa69SMichael Roth return -EIO;
254801baa69SMichael Roth
255801baa69SMichael Roth *reg = (val >> 32);
256801baa69SMichael Roth
257801baa69SMichael Roth return 0;
258801baa69SMichael Roth }
259801baa69SMichael Roth
__sev_cpuid_hv_msr(struct cpuid_leaf * leaf)2606bc6f7d9STom Lendacky static int __sev_cpuid_hv_msr(struct cpuid_leaf *leaf)
261801baa69SMichael Roth {
262801baa69SMichael Roth int ret;
263801baa69SMichael Roth
264801baa69SMichael Roth /*
265801baa69SMichael Roth * MSR protocol does not support fetching non-zero subfunctions, but is
266801baa69SMichael Roth * sufficient to handle current early-boot cases. Should that change,
267801baa69SMichael Roth * make sure to report an error rather than ignoring the index and
268801baa69SMichael Roth * grabbing random values. If this issue arises in the future, handling
269801baa69SMichael Roth * can be added here to use GHCB-page protocol for cases that occur late
270801baa69SMichael Roth * enough in boot that GHCB page is available.
271801baa69SMichael Roth */
272801baa69SMichael Roth if (cpuid_function_is_indexed(leaf->fn) && leaf->subfn)
273801baa69SMichael Roth return -EINVAL;
274801baa69SMichael Roth
275801baa69SMichael Roth ret = __sev_cpuid_hv(leaf->fn, GHCB_CPUID_REQ_EAX, &leaf->eax);
276801baa69SMichael Roth ret = ret ? : __sev_cpuid_hv(leaf->fn, GHCB_CPUID_REQ_EBX, &leaf->ebx);
277801baa69SMichael Roth ret = ret ? : __sev_cpuid_hv(leaf->fn, GHCB_CPUID_REQ_ECX, &leaf->ecx);
278801baa69SMichael Roth ret = ret ? : __sev_cpuid_hv(leaf->fn, GHCB_CPUID_REQ_EDX, &leaf->edx);
279801baa69SMichael Roth
280801baa69SMichael Roth return ret;
281801baa69SMichael Roth }
282801baa69SMichael Roth
__sev_cpuid_hv_ghcb(struct ghcb * ghcb,struct es_em_ctxt * ctxt,struct cpuid_leaf * leaf)2836bc6f7d9STom Lendacky static int __sev_cpuid_hv_ghcb(struct ghcb *ghcb, struct es_em_ctxt *ctxt, struct cpuid_leaf *leaf)
2846bc6f7d9STom Lendacky {
2856bc6f7d9STom Lendacky u32 cr4 = native_read_cr4();
2866bc6f7d9STom Lendacky int ret;
2876bc6f7d9STom Lendacky
2886bc6f7d9STom Lendacky ghcb_set_rax(ghcb, leaf->fn);
2896bc6f7d9STom Lendacky ghcb_set_rcx(ghcb, leaf->subfn);
2906bc6f7d9STom Lendacky
2916bc6f7d9STom Lendacky if (cr4 & X86_CR4_OSXSAVE)
2926bc6f7d9STom Lendacky /* Safe to read xcr0 */
2936bc6f7d9STom Lendacky ghcb_set_xcr0(ghcb, xgetbv(XCR_XFEATURE_ENABLED_MASK));
2946bc6f7d9STom Lendacky else
2956bc6f7d9STom Lendacky /* xgetbv will cause #UD - use reset value for xcr0 */
2966bc6f7d9STom Lendacky ghcb_set_xcr0(ghcb, 1);
2976bc6f7d9STom Lendacky
2986bc6f7d9STom Lendacky ret = sev_es_ghcb_hv_call(ghcb, ctxt, SVM_EXIT_CPUID, 0, 0);
2996bc6f7d9STom Lendacky if (ret != ES_OK)
3006bc6f7d9STom Lendacky return ret;
3016bc6f7d9STom Lendacky
3026bc6f7d9STom Lendacky if (!(ghcb_rax_is_valid(ghcb) &&
3036bc6f7d9STom Lendacky ghcb_rbx_is_valid(ghcb) &&
3046bc6f7d9STom Lendacky ghcb_rcx_is_valid(ghcb) &&
3056bc6f7d9STom Lendacky ghcb_rdx_is_valid(ghcb)))
3066bc6f7d9STom Lendacky return ES_VMM_ERROR;
3076bc6f7d9STom Lendacky
3086bc6f7d9STom Lendacky leaf->eax = ghcb->save.rax;
3096bc6f7d9STom Lendacky leaf->ebx = ghcb->save.rbx;
3106bc6f7d9STom Lendacky leaf->ecx = ghcb->save.rcx;
3116bc6f7d9STom Lendacky leaf->edx = ghcb->save.rdx;
3126bc6f7d9STom Lendacky
3136bc6f7d9STom Lendacky return ES_OK;
3146bc6f7d9STom Lendacky }
3156bc6f7d9STom Lendacky
sev_cpuid_hv(struct ghcb * ghcb,struct es_em_ctxt * ctxt,struct cpuid_leaf * leaf)3166bc6f7d9STom Lendacky static int sev_cpuid_hv(struct ghcb *ghcb, struct es_em_ctxt *ctxt, struct cpuid_leaf *leaf)
3176bc6f7d9STom Lendacky {
3186bc6f7d9STom Lendacky return ghcb ? __sev_cpuid_hv_ghcb(ghcb, ctxt, leaf)
3196bc6f7d9STom Lendacky : __sev_cpuid_hv_msr(leaf);
3206bc6f7d9STom Lendacky }
3216bc6f7d9STom Lendacky
322e759959fSBrijesh Singh /*
323ee0bfa08SMichael Roth * This may be called early while still running on the initial identity
324ee0bfa08SMichael Roth * mapping. Use RIP-relative addressing to obtain the correct address
325ee0bfa08SMichael Roth * while running with the initial identity mapping as well as the
326ee0bfa08SMichael Roth * switch-over to kernel virtual addresses later.
327ee0bfa08SMichael Roth */
snp_cpuid_get_table(void)328ee0bfa08SMichael Roth static const struct snp_cpuid_table *snp_cpuid_get_table(void)
329ee0bfa08SMichael Roth {
330*56408ed9SArd Biesheuvel return &RIP_REL_REF(cpuid_table_copy);
331ee0bfa08SMichael Roth }
332ee0bfa08SMichael Roth
333ee0bfa08SMichael Roth /*
334ee0bfa08SMichael Roth * The SNP Firmware ABI, Revision 0.9, Section 7.1, details the use of
335ee0bfa08SMichael Roth * XCR0_IN and XSS_IN to encode multiple versions of 0xD subfunctions 0
336ee0bfa08SMichael Roth * and 1 based on the corresponding features enabled by a particular
337ee0bfa08SMichael Roth * combination of XCR0 and XSS registers so that a guest can look up the
338ee0bfa08SMichael Roth * version corresponding to the features currently enabled in its XCR0/XSS
339ee0bfa08SMichael Roth * registers. The only values that differ between these versions/table
340ee0bfa08SMichael Roth * entries is the enabled XSAVE area size advertised via EBX.
341ee0bfa08SMichael Roth *
342ee0bfa08SMichael Roth * While hypervisors may choose to make use of this support, it is more
343ee0bfa08SMichael Roth * robust/secure for a guest to simply find the entry corresponding to the
344ee0bfa08SMichael Roth * base/legacy XSAVE area size (XCR0=1 or XCR0=3), and then calculate the
345ee0bfa08SMichael Roth * XSAVE area size using subfunctions 2 through 64, as documented in APM
346ee0bfa08SMichael Roth * Volume 3, Rev 3.31, Appendix E.3.8, which is what is done here.
347ee0bfa08SMichael Roth *
348ee0bfa08SMichael Roth * Since base/legacy XSAVE area size is documented as 0x240, use that value
349ee0bfa08SMichael Roth * directly rather than relying on the base size in the CPUID table.
350ee0bfa08SMichael Roth *
351ee0bfa08SMichael Roth * Return: XSAVE area size on success, 0 otherwise.
352ee0bfa08SMichael Roth */
snp_cpuid_calc_xsave_size(u64 xfeatures_en,bool compacted)353ee0bfa08SMichael Roth static u32 snp_cpuid_calc_xsave_size(u64 xfeatures_en, bool compacted)
354ee0bfa08SMichael Roth {
355ee0bfa08SMichael Roth const struct snp_cpuid_table *cpuid_table = snp_cpuid_get_table();
356ee0bfa08SMichael Roth u64 xfeatures_found = 0;
357ee0bfa08SMichael Roth u32 xsave_size = 0x240;
358ee0bfa08SMichael Roth int i;
359ee0bfa08SMichael Roth
360ee0bfa08SMichael Roth for (i = 0; i < cpuid_table->count; i++) {
361ee0bfa08SMichael Roth const struct snp_cpuid_fn *e = &cpuid_table->fn[i];
362ee0bfa08SMichael Roth
363ee0bfa08SMichael Roth if (!(e->eax_in == 0xD && e->ecx_in > 1 && e->ecx_in < 64))
364ee0bfa08SMichael Roth continue;
365ee0bfa08SMichael Roth if (!(xfeatures_en & (BIT_ULL(e->ecx_in))))
366ee0bfa08SMichael Roth continue;
367ee0bfa08SMichael Roth if (xfeatures_found & (BIT_ULL(e->ecx_in)))
368ee0bfa08SMichael Roth continue;
369ee0bfa08SMichael Roth
370ee0bfa08SMichael Roth xfeatures_found |= (BIT_ULL(e->ecx_in));
371ee0bfa08SMichael Roth
372ee0bfa08SMichael Roth if (compacted)
373ee0bfa08SMichael Roth xsave_size += e->eax;
374ee0bfa08SMichael Roth else
375ee0bfa08SMichael Roth xsave_size = max(xsave_size, e->eax + e->ebx);
376ee0bfa08SMichael Roth }
377ee0bfa08SMichael Roth
378ee0bfa08SMichael Roth /*
379ee0bfa08SMichael Roth * Either the guest set unsupported XCR0/XSS bits, or the corresponding
380ee0bfa08SMichael Roth * entries in the CPUID table were not present. This is not a valid
381ee0bfa08SMichael Roth * state to be in.
382ee0bfa08SMichael Roth */
383ee0bfa08SMichael Roth if (xfeatures_found != (xfeatures_en & GENMASK_ULL(63, 2)))
384ee0bfa08SMichael Roth return 0;
385ee0bfa08SMichael Roth
386ee0bfa08SMichael Roth return xsave_size;
387ee0bfa08SMichael Roth }
388ee0bfa08SMichael Roth
389*56408ed9SArd Biesheuvel static bool __head
snp_cpuid_get_validated_func(struct cpuid_leaf * leaf)390ee0bfa08SMichael Roth snp_cpuid_get_validated_func(struct cpuid_leaf *leaf)
391ee0bfa08SMichael Roth {
392ee0bfa08SMichael Roth const struct snp_cpuid_table *cpuid_table = snp_cpuid_get_table();
393ee0bfa08SMichael Roth int i;
394ee0bfa08SMichael Roth
395ee0bfa08SMichael Roth for (i = 0; i < cpuid_table->count; i++) {
396ee0bfa08SMichael Roth const struct snp_cpuid_fn *e = &cpuid_table->fn[i];
397ee0bfa08SMichael Roth
398ee0bfa08SMichael Roth if (e->eax_in != leaf->fn)
399ee0bfa08SMichael Roth continue;
400ee0bfa08SMichael Roth
401ee0bfa08SMichael Roth if (cpuid_function_is_indexed(leaf->fn) && e->ecx_in != leaf->subfn)
402ee0bfa08SMichael Roth continue;
403ee0bfa08SMichael Roth
404ee0bfa08SMichael Roth /*
405ee0bfa08SMichael Roth * For 0xD subfunctions 0 and 1, only use the entry corresponding
406ee0bfa08SMichael Roth * to the base/legacy XSAVE area size (XCR0=1 or XCR0=3, XSS=0).
407ee0bfa08SMichael Roth * See the comments above snp_cpuid_calc_xsave_size() for more
408ee0bfa08SMichael Roth * details.
409ee0bfa08SMichael Roth */
410ee0bfa08SMichael Roth if (e->eax_in == 0xD && (e->ecx_in == 0 || e->ecx_in == 1))
411ee0bfa08SMichael Roth if (!(e->xcr0_in == 1 || e->xcr0_in == 3) || e->xss_in)
412ee0bfa08SMichael Roth continue;
413ee0bfa08SMichael Roth
414ee0bfa08SMichael Roth leaf->eax = e->eax;
415ee0bfa08SMichael Roth leaf->ebx = e->ebx;
416ee0bfa08SMichael Roth leaf->ecx = e->ecx;
417ee0bfa08SMichael Roth leaf->edx = e->edx;
418ee0bfa08SMichael Roth
419ee0bfa08SMichael Roth return true;
420ee0bfa08SMichael Roth }
421ee0bfa08SMichael Roth
422ee0bfa08SMichael Roth return false;
423ee0bfa08SMichael Roth }
424ee0bfa08SMichael Roth
snp_cpuid_hv(struct ghcb * ghcb,struct es_em_ctxt * ctxt,struct cpuid_leaf * leaf)4256bc6f7d9STom Lendacky static void snp_cpuid_hv(struct ghcb *ghcb, struct es_em_ctxt *ctxt, struct cpuid_leaf *leaf)
426ee0bfa08SMichael Roth {
4276bc6f7d9STom Lendacky if (sev_cpuid_hv(ghcb, ctxt, leaf))
428ee0bfa08SMichael Roth sev_es_terminate(SEV_TERM_SET_LINUX, GHCB_TERM_CPUID_HV);
429ee0bfa08SMichael Roth }
430ee0bfa08SMichael Roth
snp_cpuid_postprocess(struct ghcb * ghcb,struct es_em_ctxt * ctxt,struct cpuid_leaf * leaf)4316bc6f7d9STom Lendacky static int snp_cpuid_postprocess(struct ghcb *ghcb, struct es_em_ctxt *ctxt,
4326bc6f7d9STom Lendacky struct cpuid_leaf *leaf)
433ee0bfa08SMichael Roth {
434ee0bfa08SMichael Roth struct cpuid_leaf leaf_hv = *leaf;
435ee0bfa08SMichael Roth
436ee0bfa08SMichael Roth switch (leaf->fn) {
437ee0bfa08SMichael Roth case 0x1:
4386bc6f7d9STom Lendacky snp_cpuid_hv(ghcb, ctxt, &leaf_hv);
439ee0bfa08SMichael Roth
440ee0bfa08SMichael Roth /* initial APIC ID */
441ee0bfa08SMichael Roth leaf->ebx = (leaf_hv.ebx & GENMASK(31, 24)) | (leaf->ebx & GENMASK(23, 0));
442ee0bfa08SMichael Roth /* APIC enabled bit */
443ee0bfa08SMichael Roth leaf->edx = (leaf_hv.edx & BIT(9)) | (leaf->edx & ~BIT(9));
444ee0bfa08SMichael Roth
445ee0bfa08SMichael Roth /* OSXSAVE enabled bit */
446ee0bfa08SMichael Roth if (native_read_cr4() & X86_CR4_OSXSAVE)
447ee0bfa08SMichael Roth leaf->ecx |= BIT(27);
448ee0bfa08SMichael Roth break;
449ee0bfa08SMichael Roth case 0x7:
450ee0bfa08SMichael Roth /* OSPKE enabled bit */
451ee0bfa08SMichael Roth leaf->ecx &= ~BIT(4);
452ee0bfa08SMichael Roth if (native_read_cr4() & X86_CR4_PKE)
453ee0bfa08SMichael Roth leaf->ecx |= BIT(4);
454ee0bfa08SMichael Roth break;
455ee0bfa08SMichael Roth case 0xB:
456ee0bfa08SMichael Roth leaf_hv.subfn = 0;
4576bc6f7d9STom Lendacky snp_cpuid_hv(ghcb, ctxt, &leaf_hv);
458ee0bfa08SMichael Roth
459ee0bfa08SMichael Roth /* extended APIC ID */
460ee0bfa08SMichael Roth leaf->edx = leaf_hv.edx;
461ee0bfa08SMichael Roth break;
462ee0bfa08SMichael Roth case 0xD: {
463ee0bfa08SMichael Roth bool compacted = false;
464ee0bfa08SMichael Roth u64 xcr0 = 1, xss = 0;
465ee0bfa08SMichael Roth u32 xsave_size;
466ee0bfa08SMichael Roth
467ee0bfa08SMichael Roth if (leaf->subfn != 0 && leaf->subfn != 1)
468ee0bfa08SMichael Roth return 0;
469ee0bfa08SMichael Roth
470ee0bfa08SMichael Roth if (native_read_cr4() & X86_CR4_OSXSAVE)
471ee0bfa08SMichael Roth xcr0 = xgetbv(XCR_XFEATURE_ENABLED_MASK);
472ee0bfa08SMichael Roth if (leaf->subfn == 1) {
473ee0bfa08SMichael Roth /* Get XSS value if XSAVES is enabled. */
474ee0bfa08SMichael Roth if (leaf->eax & BIT(3)) {
475ee0bfa08SMichael Roth unsigned long lo, hi;
476ee0bfa08SMichael Roth
477ee0bfa08SMichael Roth asm volatile("rdmsr" : "=a" (lo), "=d" (hi)
478ee0bfa08SMichael Roth : "c" (MSR_IA32_XSS));
479ee0bfa08SMichael Roth xss = (hi << 32) | lo;
480ee0bfa08SMichael Roth }
481ee0bfa08SMichael Roth
482ee0bfa08SMichael Roth /*
483ee0bfa08SMichael Roth * The PPR and APM aren't clear on what size should be
484ee0bfa08SMichael Roth * encoded in 0xD:0x1:EBX when compaction is not enabled
485ee0bfa08SMichael Roth * by either XSAVEC (feature bit 1) or XSAVES (feature
486ee0bfa08SMichael Roth * bit 3) since SNP-capable hardware has these feature
487ee0bfa08SMichael Roth * bits fixed as 1. KVM sets it to 0 in this case, but
488ee0bfa08SMichael Roth * to avoid this becoming an issue it's safer to simply
489ee0bfa08SMichael Roth * treat this as unsupported for SNP guests.
490ee0bfa08SMichael Roth */
491ee0bfa08SMichael Roth if (!(leaf->eax & (BIT(1) | BIT(3))))
492ee0bfa08SMichael Roth return -EINVAL;
493ee0bfa08SMichael Roth
494ee0bfa08SMichael Roth compacted = true;
495ee0bfa08SMichael Roth }
496ee0bfa08SMichael Roth
497ee0bfa08SMichael Roth xsave_size = snp_cpuid_calc_xsave_size(xcr0 | xss, compacted);
498ee0bfa08SMichael Roth if (!xsave_size)
499ee0bfa08SMichael Roth return -EINVAL;
500ee0bfa08SMichael Roth
501ee0bfa08SMichael Roth leaf->ebx = xsave_size;
502ee0bfa08SMichael Roth }
503ee0bfa08SMichael Roth break;
504ee0bfa08SMichael Roth case 0x8000001E:
5056bc6f7d9STom Lendacky snp_cpuid_hv(ghcb, ctxt, &leaf_hv);
506ee0bfa08SMichael Roth
507ee0bfa08SMichael Roth /* extended APIC ID */
508ee0bfa08SMichael Roth leaf->eax = leaf_hv.eax;
509ee0bfa08SMichael Roth /* compute ID */
510ee0bfa08SMichael Roth leaf->ebx = (leaf->ebx & GENMASK(31, 8)) | (leaf_hv.ebx & GENMASK(7, 0));
511ee0bfa08SMichael Roth /* node ID */
512ee0bfa08SMichael Roth leaf->ecx = (leaf->ecx & GENMASK(31, 8)) | (leaf_hv.ecx & GENMASK(7, 0));
513ee0bfa08SMichael Roth break;
514ee0bfa08SMichael Roth default:
515ee0bfa08SMichael Roth /* No fix-ups needed, use values as-is. */
516ee0bfa08SMichael Roth break;
517ee0bfa08SMichael Roth }
518ee0bfa08SMichael Roth
519ee0bfa08SMichael Roth return 0;
520ee0bfa08SMichael Roth }
521ee0bfa08SMichael Roth
522ee0bfa08SMichael Roth /*
523ee0bfa08SMichael Roth * Returns -EOPNOTSUPP if feature not enabled. Any other non-zero return value
524ee0bfa08SMichael Roth * should be treated as fatal by caller.
525ee0bfa08SMichael Roth */
526*56408ed9SArd Biesheuvel static int __head
snp_cpuid(struct ghcb * ghcb,struct es_em_ctxt * ctxt,struct cpuid_leaf * leaf)527*56408ed9SArd Biesheuvel snp_cpuid(struct ghcb *ghcb, struct es_em_ctxt *ctxt, struct cpuid_leaf *leaf)
528ee0bfa08SMichael Roth {
529ee0bfa08SMichael Roth const struct snp_cpuid_table *cpuid_table = snp_cpuid_get_table();
530ee0bfa08SMichael Roth
531ee0bfa08SMichael Roth if (!cpuid_table->count)
532ee0bfa08SMichael Roth return -EOPNOTSUPP;
533ee0bfa08SMichael Roth
534ee0bfa08SMichael Roth if (!snp_cpuid_get_validated_func(leaf)) {
535ee0bfa08SMichael Roth /*
536ee0bfa08SMichael Roth * Some hypervisors will avoid keeping track of CPUID entries
537ee0bfa08SMichael Roth * where all values are zero, since they can be handled the
538ee0bfa08SMichael Roth * same as out-of-range values (all-zero). This is useful here
539ee0bfa08SMichael Roth * as well as it allows virtually all guest configurations to
540ee0bfa08SMichael Roth * work using a single SNP CPUID table.
541ee0bfa08SMichael Roth *
542ee0bfa08SMichael Roth * To allow for this, there is a need to distinguish between
543ee0bfa08SMichael Roth * out-of-range entries and in-range zero entries, since the
544ee0bfa08SMichael Roth * CPUID table entries are only a template that may need to be
545ee0bfa08SMichael Roth * augmented with additional values for things like
546ee0bfa08SMichael Roth * CPU-specific information during post-processing. So if it's
547ee0bfa08SMichael Roth * not in the table, set the values to zero. Then, if they are
548ee0bfa08SMichael Roth * within a valid CPUID range, proceed with post-processing
549ee0bfa08SMichael Roth * using zeros as the initial values. Otherwise, skip
550ee0bfa08SMichael Roth * post-processing and just return zeros immediately.
551ee0bfa08SMichael Roth */
552ee0bfa08SMichael Roth leaf->eax = leaf->ebx = leaf->ecx = leaf->edx = 0;
553ee0bfa08SMichael Roth
554ee0bfa08SMichael Roth /* Skip post-processing for out-of-range zero leafs. */
5550982fd6bSArd Biesheuvel if (!(leaf->fn <= RIP_REL_REF(cpuid_std_range_max) ||
5560982fd6bSArd Biesheuvel (leaf->fn >= 0x40000000 && leaf->fn <= RIP_REL_REF(cpuid_hyp_range_max)) ||
5570982fd6bSArd Biesheuvel (leaf->fn >= 0x80000000 && leaf->fn <= RIP_REL_REF(cpuid_ext_range_max))))
558ee0bfa08SMichael Roth return 0;
559ee0bfa08SMichael Roth }
560ee0bfa08SMichael Roth
5616bc6f7d9STom Lendacky return snp_cpuid_postprocess(ghcb, ctxt, leaf);
562ee0bfa08SMichael Roth }
563ee0bfa08SMichael Roth
564ee0bfa08SMichael Roth /*
565e759959fSBrijesh Singh * Boot VC Handler - This is the first VC handler during boot, there is no GHCB
566e759959fSBrijesh Singh * page yet, so it only supports the MSR based communication with the
567e759959fSBrijesh Singh * hypervisor and only the CPUID exit-code.
568e759959fSBrijesh Singh */
do_vc_no_ghcb(struct pt_regs * regs,unsigned long exit_code)569*56408ed9SArd Biesheuvel void __head do_vc_no_ghcb(struct pt_regs *regs, unsigned long exit_code)
570e759959fSBrijesh Singh {
571801baa69SMichael Roth unsigned int subfn = lower_bits(regs->cx, 32);
572e759959fSBrijesh Singh unsigned int fn = lower_bits(regs->ax, 32);
573801baa69SMichael Roth struct cpuid_leaf leaf;
574ee0bfa08SMichael Roth int ret;
575e759959fSBrijesh Singh
576e759959fSBrijesh Singh /* Only CPUID is supported via MSR protocol */
577e759959fSBrijesh Singh if (exit_code != SVM_EXIT_CPUID)
578e759959fSBrijesh Singh goto fail;
579e759959fSBrijesh Singh
580801baa69SMichael Roth leaf.fn = fn;
581801baa69SMichael Roth leaf.subfn = subfn;
582ee0bfa08SMichael Roth
5836bc6f7d9STom Lendacky ret = snp_cpuid(NULL, NULL, &leaf);
584ee0bfa08SMichael Roth if (!ret)
585ee0bfa08SMichael Roth goto cpuid_done;
586ee0bfa08SMichael Roth
587ee0bfa08SMichael Roth if (ret != -EOPNOTSUPP)
588ee0bfa08SMichael Roth goto fail;
589ee0bfa08SMichael Roth
5906bc6f7d9STom Lendacky if (__sev_cpuid_hv_msr(&leaf))
591e759959fSBrijesh Singh goto fail;
592e759959fSBrijesh Singh
593ee0bfa08SMichael Roth cpuid_done:
594801baa69SMichael Roth regs->ax = leaf.eax;
595801baa69SMichael Roth regs->bx = leaf.ebx;
596801baa69SMichael Roth regs->cx = leaf.ecx;
597801baa69SMichael Roth regs->dx = leaf.edx;
598e759959fSBrijesh Singh
599e759959fSBrijesh Singh /*
600e759959fSBrijesh Singh * This is a VC handler and the #VC is only raised when SEV-ES is
601e759959fSBrijesh Singh * active, which means SEV must be active too. Do sanity checks on the
602e759959fSBrijesh Singh * CPUID results to make sure the hypervisor does not trick the kernel
603e759959fSBrijesh Singh * into the no-sev path. This could map sensitive data unencrypted and
604e759959fSBrijesh Singh * make it accessible to the hypervisor.
605e759959fSBrijesh Singh *
606e759959fSBrijesh Singh * In particular, check for:
607e759959fSBrijesh Singh * - Availability of CPUID leaf 0x8000001f
608e759959fSBrijesh Singh * - SEV CPUID bit.
609e759959fSBrijesh Singh *
610e759959fSBrijesh Singh * The hypervisor might still report the wrong C-bit position, but this
611e759959fSBrijesh Singh * can't be checked here.
612e759959fSBrijesh Singh */
613e759959fSBrijesh Singh
614e759959fSBrijesh Singh if (fn == 0x80000000 && (regs->ax < 0x8000001f))
615e759959fSBrijesh Singh /* SEV leaf check */
616e759959fSBrijesh Singh goto fail;
617e759959fSBrijesh Singh else if ((fn == 0x8000001f && !(regs->ax & BIT(1))))
618e759959fSBrijesh Singh /* SEV bit */
619e759959fSBrijesh Singh goto fail;
620e759959fSBrijesh Singh
621e759959fSBrijesh Singh /* Skip over the CPUID two-byte opcode */
622e759959fSBrijesh Singh regs->ip += 2;
623e759959fSBrijesh Singh
624e759959fSBrijesh Singh return;
625e759959fSBrijesh Singh
626e759959fSBrijesh Singh fail:
627e759959fSBrijesh Singh /* Terminate the guest */
6286c0f74d6SBrijesh Singh sev_es_terminate(SEV_TERM_SET_GEN, GHCB_SEV_ES_GEN_REQ);
629e759959fSBrijesh Singh }
630e759959fSBrijesh Singh
vc_insn_string_check(struct es_em_ctxt * ctxt,unsigned long address,bool write)63163e44bc5SJoerg Roedel static enum es_result vc_insn_string_check(struct es_em_ctxt *ctxt,
63263e44bc5SJoerg Roedel unsigned long address,
63363e44bc5SJoerg Roedel bool write)
63463e44bc5SJoerg Roedel {
63563e44bc5SJoerg Roedel if (user_mode(ctxt->regs) && fault_in_kernel_space(address)) {
63663e44bc5SJoerg Roedel ctxt->fi.vector = X86_TRAP_PF;
63763e44bc5SJoerg Roedel ctxt->fi.error_code = X86_PF_USER;
63863e44bc5SJoerg Roedel ctxt->fi.cr2 = address;
63963e44bc5SJoerg Roedel if (write)
64063e44bc5SJoerg Roedel ctxt->fi.error_code |= X86_PF_WRITE;
64163e44bc5SJoerg Roedel
64263e44bc5SJoerg Roedel return ES_EXCEPTION;
64363e44bc5SJoerg Roedel }
64463e44bc5SJoerg Roedel
64563e44bc5SJoerg Roedel return ES_OK;
64663e44bc5SJoerg Roedel }
64763e44bc5SJoerg Roedel
vc_insn_string_read(struct es_em_ctxt * ctxt,void * src,char * buf,unsigned int data_size,unsigned int count,bool backwards)648e759959fSBrijesh Singh static enum es_result vc_insn_string_read(struct es_em_ctxt *ctxt,
649e759959fSBrijesh Singh void *src, char *buf,
650e759959fSBrijesh Singh unsigned int data_size,
651e759959fSBrijesh Singh unsigned int count,
652e759959fSBrijesh Singh bool backwards)
653e759959fSBrijesh Singh {
654e759959fSBrijesh Singh int i, b = backwards ? -1 : 1;
65563e44bc5SJoerg Roedel unsigned long address = (unsigned long)src;
65663e44bc5SJoerg Roedel enum es_result ret;
65763e44bc5SJoerg Roedel
65863e44bc5SJoerg Roedel ret = vc_insn_string_check(ctxt, address, false);
65963e44bc5SJoerg Roedel if (ret != ES_OK)
66063e44bc5SJoerg Roedel return ret;
661e759959fSBrijesh Singh
662e759959fSBrijesh Singh for (i = 0; i < count; i++) {
663e759959fSBrijesh Singh void *s = src + (i * data_size * b);
664e759959fSBrijesh Singh char *d = buf + (i * data_size);
665e759959fSBrijesh Singh
666e759959fSBrijesh Singh ret = vc_read_mem(ctxt, s, d, data_size);
667e759959fSBrijesh Singh if (ret != ES_OK)
668e759959fSBrijesh Singh break;
669e759959fSBrijesh Singh }
670e759959fSBrijesh Singh
671e759959fSBrijesh Singh return ret;
672e759959fSBrijesh Singh }
673e759959fSBrijesh Singh
vc_insn_string_write(struct es_em_ctxt * ctxt,void * dst,char * buf,unsigned int data_size,unsigned int count,bool backwards)674e759959fSBrijesh Singh static enum es_result vc_insn_string_write(struct es_em_ctxt *ctxt,
675e759959fSBrijesh Singh void *dst, char *buf,
676e759959fSBrijesh Singh unsigned int data_size,
677e759959fSBrijesh Singh unsigned int count,
678e759959fSBrijesh Singh bool backwards)
679e759959fSBrijesh Singh {
680e759959fSBrijesh Singh int i, s = backwards ? -1 : 1;
68163e44bc5SJoerg Roedel unsigned long address = (unsigned long)dst;
68263e44bc5SJoerg Roedel enum es_result ret;
68363e44bc5SJoerg Roedel
68463e44bc5SJoerg Roedel ret = vc_insn_string_check(ctxt, address, true);
68563e44bc5SJoerg Roedel if (ret != ES_OK)
68663e44bc5SJoerg Roedel return ret;
687e759959fSBrijesh Singh
688e759959fSBrijesh Singh for (i = 0; i < count; i++) {
689e759959fSBrijesh Singh void *d = dst + (i * data_size * s);
690e759959fSBrijesh Singh char *b = buf + (i * data_size);
691e759959fSBrijesh Singh
692e759959fSBrijesh Singh ret = vc_write_mem(ctxt, d, b, data_size);
693e759959fSBrijesh Singh if (ret != ES_OK)
694e759959fSBrijesh Singh break;
695e759959fSBrijesh Singh }
696e759959fSBrijesh Singh
697e759959fSBrijesh Singh return ret;
698e759959fSBrijesh Singh }
699e759959fSBrijesh Singh
700e759959fSBrijesh Singh #define IOIO_TYPE_STR BIT(2)
701e759959fSBrijesh Singh #define IOIO_TYPE_IN 1
702e759959fSBrijesh Singh #define IOIO_TYPE_INS (IOIO_TYPE_IN | IOIO_TYPE_STR)
703e759959fSBrijesh Singh #define IOIO_TYPE_OUT 0
704e759959fSBrijesh Singh #define IOIO_TYPE_OUTS (IOIO_TYPE_OUT | IOIO_TYPE_STR)
705e759959fSBrijesh Singh
706e759959fSBrijesh Singh #define IOIO_REP BIT(3)
707e759959fSBrijesh Singh
708e759959fSBrijesh Singh #define IOIO_ADDR_64 BIT(9)
709e759959fSBrijesh Singh #define IOIO_ADDR_32 BIT(8)
710e759959fSBrijesh Singh #define IOIO_ADDR_16 BIT(7)
711e759959fSBrijesh Singh
712e759959fSBrijesh Singh #define IOIO_DATA_32 BIT(6)
713e759959fSBrijesh Singh #define IOIO_DATA_16 BIT(5)
714e759959fSBrijesh Singh #define IOIO_DATA_8 BIT(4)
715e759959fSBrijesh Singh
716e759959fSBrijesh Singh #define IOIO_SEG_ES (0 << 10)
717e759959fSBrijesh Singh #define IOIO_SEG_DS (3 << 10)
718e759959fSBrijesh Singh
vc_ioio_exitinfo(struct es_em_ctxt * ctxt,u64 * exitinfo)719e759959fSBrijesh Singh static enum es_result vc_ioio_exitinfo(struct es_em_ctxt *ctxt, u64 *exitinfo)
720e759959fSBrijesh Singh {
721e759959fSBrijesh Singh struct insn *insn = &ctxt->insn;
722b9cb9c45SJoerg Roedel size_t size;
723b9cb9c45SJoerg Roedel u64 port;
724b9cb9c45SJoerg Roedel
725e759959fSBrijesh Singh *exitinfo = 0;
726e759959fSBrijesh Singh
727e759959fSBrijesh Singh switch (insn->opcode.bytes[0]) {
728e759959fSBrijesh Singh /* INS opcodes */
729e759959fSBrijesh Singh case 0x6c:
730e759959fSBrijesh Singh case 0x6d:
731e759959fSBrijesh Singh *exitinfo |= IOIO_TYPE_INS;
732e759959fSBrijesh Singh *exitinfo |= IOIO_SEG_ES;
733b9cb9c45SJoerg Roedel port = ctxt->regs->dx & 0xffff;
734e759959fSBrijesh Singh break;
735e759959fSBrijesh Singh
736e759959fSBrijesh Singh /* OUTS opcodes */
737e759959fSBrijesh Singh case 0x6e:
738e759959fSBrijesh Singh case 0x6f:
739e759959fSBrijesh Singh *exitinfo |= IOIO_TYPE_OUTS;
740e759959fSBrijesh Singh *exitinfo |= IOIO_SEG_DS;
741b9cb9c45SJoerg Roedel port = ctxt->regs->dx & 0xffff;
742e759959fSBrijesh Singh break;
743e759959fSBrijesh Singh
744e759959fSBrijesh Singh /* IN immediate opcodes */
745e759959fSBrijesh Singh case 0xe4:
746e759959fSBrijesh Singh case 0xe5:
747e759959fSBrijesh Singh *exitinfo |= IOIO_TYPE_IN;
748b9cb9c45SJoerg Roedel port = (u8)insn->immediate.value & 0xffff;
749e759959fSBrijesh Singh break;
750e759959fSBrijesh Singh
751e759959fSBrijesh Singh /* OUT immediate opcodes */
752e759959fSBrijesh Singh case 0xe6:
753e759959fSBrijesh Singh case 0xe7:
754e759959fSBrijesh Singh *exitinfo |= IOIO_TYPE_OUT;
755b9cb9c45SJoerg Roedel port = (u8)insn->immediate.value & 0xffff;
756e759959fSBrijesh Singh break;
757e759959fSBrijesh Singh
758e759959fSBrijesh Singh /* IN register opcodes */
759e759959fSBrijesh Singh case 0xec:
760e759959fSBrijesh Singh case 0xed:
761e759959fSBrijesh Singh *exitinfo |= IOIO_TYPE_IN;
762b9cb9c45SJoerg Roedel port = ctxt->regs->dx & 0xffff;
763e759959fSBrijesh Singh break;
764e759959fSBrijesh Singh
765e759959fSBrijesh Singh /* OUT register opcodes */
766e759959fSBrijesh Singh case 0xee:
767e759959fSBrijesh Singh case 0xef:
768e759959fSBrijesh Singh *exitinfo |= IOIO_TYPE_OUT;
769b9cb9c45SJoerg Roedel port = ctxt->regs->dx & 0xffff;
770e759959fSBrijesh Singh break;
771e759959fSBrijesh Singh
772e759959fSBrijesh Singh default:
773e759959fSBrijesh Singh return ES_DECODE_FAILED;
774e759959fSBrijesh Singh }
775e759959fSBrijesh Singh
776b9cb9c45SJoerg Roedel *exitinfo |= port << 16;
777b9cb9c45SJoerg Roedel
778e759959fSBrijesh Singh switch (insn->opcode.bytes[0]) {
779e759959fSBrijesh Singh case 0x6c:
780e759959fSBrijesh Singh case 0x6e:
781e759959fSBrijesh Singh case 0xe4:
782e759959fSBrijesh Singh case 0xe6:
783e759959fSBrijesh Singh case 0xec:
784e759959fSBrijesh Singh case 0xee:
785e759959fSBrijesh Singh /* Single byte opcodes */
786e759959fSBrijesh Singh *exitinfo |= IOIO_DATA_8;
787b9cb9c45SJoerg Roedel size = 1;
788e759959fSBrijesh Singh break;
789e759959fSBrijesh Singh default:
790e759959fSBrijesh Singh /* Length determined by instruction parsing */
791e759959fSBrijesh Singh *exitinfo |= (insn->opnd_bytes == 2) ? IOIO_DATA_16
792e759959fSBrijesh Singh : IOIO_DATA_32;
793b9cb9c45SJoerg Roedel size = (insn->opnd_bytes == 2) ? 2 : 4;
794e759959fSBrijesh Singh }
795b9cb9c45SJoerg Roedel
796e759959fSBrijesh Singh switch (insn->addr_bytes) {
797e759959fSBrijesh Singh case 2:
798e759959fSBrijesh Singh *exitinfo |= IOIO_ADDR_16;
799e759959fSBrijesh Singh break;
800e759959fSBrijesh Singh case 4:
801e759959fSBrijesh Singh *exitinfo |= IOIO_ADDR_32;
802e759959fSBrijesh Singh break;
803e759959fSBrijesh Singh case 8:
804e759959fSBrijesh Singh *exitinfo |= IOIO_ADDR_64;
805e759959fSBrijesh Singh break;
806e759959fSBrijesh Singh }
807e759959fSBrijesh Singh
808e759959fSBrijesh Singh if (insn_has_rep_prefix(insn))
809e759959fSBrijesh Singh *exitinfo |= IOIO_REP;
810e759959fSBrijesh Singh
811b9cb9c45SJoerg Roedel return vc_ioio_check(ctxt, (u16)port, size);
812e759959fSBrijesh Singh }
813e759959fSBrijesh Singh
vc_handle_ioio(struct ghcb * ghcb,struct es_em_ctxt * ctxt)814e759959fSBrijesh Singh static enum es_result vc_handle_ioio(struct ghcb *ghcb, struct es_em_ctxt *ctxt)
815e759959fSBrijesh Singh {
816e759959fSBrijesh Singh struct pt_regs *regs = ctxt->regs;
817e759959fSBrijesh Singh u64 exit_info_1, exit_info_2;
818e759959fSBrijesh Singh enum es_result ret;
819e759959fSBrijesh Singh
820e759959fSBrijesh Singh ret = vc_ioio_exitinfo(ctxt, &exit_info_1);
821e759959fSBrijesh Singh if (ret != ES_OK)
822e759959fSBrijesh Singh return ret;
823e759959fSBrijesh Singh
824e759959fSBrijesh Singh if (exit_info_1 & IOIO_TYPE_STR) {
825e759959fSBrijesh Singh
826e759959fSBrijesh Singh /* (REP) INS/OUTS */
827e759959fSBrijesh Singh
828e759959fSBrijesh Singh bool df = ((regs->flags & X86_EFLAGS_DF) == X86_EFLAGS_DF);
829e759959fSBrijesh Singh unsigned int io_bytes, exit_bytes;
830e759959fSBrijesh Singh unsigned int ghcb_count, op_count;
831e759959fSBrijesh Singh unsigned long es_base;
832e759959fSBrijesh Singh u64 sw_scratch;
833e759959fSBrijesh Singh
834e759959fSBrijesh Singh /*
835e759959fSBrijesh Singh * For the string variants with rep prefix the amount of in/out
836e759959fSBrijesh Singh * operations per #VC exception is limited so that the kernel
837e759959fSBrijesh Singh * has a chance to take interrupts and re-schedule while the
838e759959fSBrijesh Singh * instruction is emulated.
839e759959fSBrijesh Singh */
840e759959fSBrijesh Singh io_bytes = (exit_info_1 >> 4) & 0x7;
841e759959fSBrijesh Singh ghcb_count = sizeof(ghcb->shared_buffer) / io_bytes;
842e759959fSBrijesh Singh
843e759959fSBrijesh Singh op_count = (exit_info_1 & IOIO_REP) ? regs->cx : 1;
844e759959fSBrijesh Singh exit_info_2 = min(op_count, ghcb_count);
845e759959fSBrijesh Singh exit_bytes = exit_info_2 * io_bytes;
846e759959fSBrijesh Singh
847e759959fSBrijesh Singh es_base = insn_get_seg_base(ctxt->regs, INAT_SEG_REG_ES);
848e759959fSBrijesh Singh
849e759959fSBrijesh Singh /* Read bytes of OUTS into the shared buffer */
850e759959fSBrijesh Singh if (!(exit_info_1 & IOIO_TYPE_IN)) {
851e759959fSBrijesh Singh ret = vc_insn_string_read(ctxt,
852e759959fSBrijesh Singh (void *)(es_base + regs->si),
853e759959fSBrijesh Singh ghcb->shared_buffer, io_bytes,
854e759959fSBrijesh Singh exit_info_2, df);
855e759959fSBrijesh Singh if (ret)
856e759959fSBrijesh Singh return ret;
857e759959fSBrijesh Singh }
858e759959fSBrijesh Singh
859e759959fSBrijesh Singh /*
860e759959fSBrijesh Singh * Issue an VMGEXIT to the HV to consume the bytes from the
861e759959fSBrijesh Singh * shared buffer or to have it write them into the shared buffer
862e759959fSBrijesh Singh * depending on the instruction: OUTS or INS.
863e759959fSBrijesh Singh */
864e759959fSBrijesh Singh sw_scratch = __pa(ghcb) + offsetof(struct ghcb, shared_buffer);
865e759959fSBrijesh Singh ghcb_set_sw_scratch(ghcb, sw_scratch);
8665bb6c1d1SBorislav Petkov ret = sev_es_ghcb_hv_call(ghcb, ctxt, SVM_EXIT_IOIO,
867e759959fSBrijesh Singh exit_info_1, exit_info_2);
868e759959fSBrijesh Singh if (ret != ES_OK)
869e759959fSBrijesh Singh return ret;
870e759959fSBrijesh Singh
871e759959fSBrijesh Singh /* Read bytes from shared buffer into the guest's destination. */
872e759959fSBrijesh Singh if (exit_info_1 & IOIO_TYPE_IN) {
873e759959fSBrijesh Singh ret = vc_insn_string_write(ctxt,
874e759959fSBrijesh Singh (void *)(es_base + regs->di),
875e759959fSBrijesh Singh ghcb->shared_buffer, io_bytes,
876e759959fSBrijesh Singh exit_info_2, df);
877e759959fSBrijesh Singh if (ret)
878e759959fSBrijesh Singh return ret;
879e759959fSBrijesh Singh
880e759959fSBrijesh Singh if (df)
881e759959fSBrijesh Singh regs->di -= exit_bytes;
882e759959fSBrijesh Singh else
883e759959fSBrijesh Singh regs->di += exit_bytes;
884e759959fSBrijesh Singh } else {
885e759959fSBrijesh Singh if (df)
886e759959fSBrijesh Singh regs->si -= exit_bytes;
887e759959fSBrijesh Singh else
888e759959fSBrijesh Singh regs->si += exit_bytes;
889e759959fSBrijesh Singh }
890e759959fSBrijesh Singh
891e759959fSBrijesh Singh if (exit_info_1 & IOIO_REP)
892e759959fSBrijesh Singh regs->cx -= exit_info_2;
893e759959fSBrijesh Singh
894e759959fSBrijesh Singh ret = regs->cx ? ES_RETRY : ES_OK;
895e759959fSBrijesh Singh
896e759959fSBrijesh Singh } else {
897e759959fSBrijesh Singh
898e759959fSBrijesh Singh /* IN/OUT into/from rAX */
899e759959fSBrijesh Singh
900e759959fSBrijesh Singh int bits = (exit_info_1 & 0x70) >> 1;
901e759959fSBrijesh Singh u64 rax = 0;
902e759959fSBrijesh Singh
903e759959fSBrijesh Singh if (!(exit_info_1 & IOIO_TYPE_IN))
904e759959fSBrijesh Singh rax = lower_bits(regs->ax, bits);
905e759959fSBrijesh Singh
906e759959fSBrijesh Singh ghcb_set_rax(ghcb, rax);
907e759959fSBrijesh Singh
9085bb6c1d1SBorislav Petkov ret = sev_es_ghcb_hv_call(ghcb, ctxt, SVM_EXIT_IOIO, exit_info_1, 0);
909e759959fSBrijesh Singh if (ret != ES_OK)
910e759959fSBrijesh Singh return ret;
911e759959fSBrijesh Singh
912e759959fSBrijesh Singh if (exit_info_1 & IOIO_TYPE_IN) {
913e759959fSBrijesh Singh if (!ghcb_rax_is_valid(ghcb))
914e759959fSBrijesh Singh return ES_VMM_ERROR;
915e759959fSBrijesh Singh regs->ax = lower_bits(ghcb->save.rax, bits);
916e759959fSBrijesh Singh }
917e759959fSBrijesh Singh }
918e759959fSBrijesh Singh
919e759959fSBrijesh Singh return ret;
920e759959fSBrijesh Singh }
921e759959fSBrijesh Singh
vc_handle_cpuid_snp(struct ghcb * ghcb,struct es_em_ctxt * ctxt)9226bc6f7d9STom Lendacky static int vc_handle_cpuid_snp(struct ghcb *ghcb, struct es_em_ctxt *ctxt)
923ee0bfa08SMichael Roth {
9246bc6f7d9STom Lendacky struct pt_regs *regs = ctxt->regs;
925ee0bfa08SMichael Roth struct cpuid_leaf leaf;
926ee0bfa08SMichael Roth int ret;
927ee0bfa08SMichael Roth
928ee0bfa08SMichael Roth leaf.fn = regs->ax;
929ee0bfa08SMichael Roth leaf.subfn = regs->cx;
9306bc6f7d9STom Lendacky ret = snp_cpuid(ghcb, ctxt, &leaf);
931ee0bfa08SMichael Roth if (!ret) {
932ee0bfa08SMichael Roth regs->ax = leaf.eax;
933ee0bfa08SMichael Roth regs->bx = leaf.ebx;
934ee0bfa08SMichael Roth regs->cx = leaf.ecx;
935ee0bfa08SMichael Roth regs->dx = leaf.edx;
936ee0bfa08SMichael Roth }
937ee0bfa08SMichael Roth
938ee0bfa08SMichael Roth return ret;
939ee0bfa08SMichael Roth }
940ee0bfa08SMichael Roth
vc_handle_cpuid(struct ghcb * ghcb,struct es_em_ctxt * ctxt)941e759959fSBrijesh Singh static enum es_result vc_handle_cpuid(struct ghcb *ghcb,
942e759959fSBrijesh Singh struct es_em_ctxt *ctxt)
943e759959fSBrijesh Singh {
944e759959fSBrijesh Singh struct pt_regs *regs = ctxt->regs;
945e759959fSBrijesh Singh u32 cr4 = native_read_cr4();
946e759959fSBrijesh Singh enum es_result ret;
947ee0bfa08SMichael Roth int snp_cpuid_ret;
948ee0bfa08SMichael Roth
9496bc6f7d9STom Lendacky snp_cpuid_ret = vc_handle_cpuid_snp(ghcb, ctxt);
950ee0bfa08SMichael Roth if (!snp_cpuid_ret)
951ee0bfa08SMichael Roth return ES_OK;
952ee0bfa08SMichael Roth if (snp_cpuid_ret != -EOPNOTSUPP)
953ee0bfa08SMichael Roth return ES_VMM_ERROR;
954e759959fSBrijesh Singh
955e759959fSBrijesh Singh ghcb_set_rax(ghcb, regs->ax);
956e759959fSBrijesh Singh ghcb_set_rcx(ghcb, regs->cx);
957e759959fSBrijesh Singh
958e759959fSBrijesh Singh if (cr4 & X86_CR4_OSXSAVE)
959e759959fSBrijesh Singh /* Safe to read xcr0 */
960e759959fSBrijesh Singh ghcb_set_xcr0(ghcb, xgetbv(XCR_XFEATURE_ENABLED_MASK));
961e759959fSBrijesh Singh else
962e759959fSBrijesh Singh /* xgetbv will cause #GP - use reset value for xcr0 */
963e759959fSBrijesh Singh ghcb_set_xcr0(ghcb, 1);
964e759959fSBrijesh Singh
9655bb6c1d1SBorislav Petkov ret = sev_es_ghcb_hv_call(ghcb, ctxt, SVM_EXIT_CPUID, 0, 0);
966e759959fSBrijesh Singh if (ret != ES_OK)
967e759959fSBrijesh Singh return ret;
968e759959fSBrijesh Singh
969e759959fSBrijesh Singh if (!(ghcb_rax_is_valid(ghcb) &&
970e759959fSBrijesh Singh ghcb_rbx_is_valid(ghcb) &&
971e759959fSBrijesh Singh ghcb_rcx_is_valid(ghcb) &&
972e759959fSBrijesh Singh ghcb_rdx_is_valid(ghcb)))
973e759959fSBrijesh Singh return ES_VMM_ERROR;
974e759959fSBrijesh Singh
975e759959fSBrijesh Singh regs->ax = ghcb->save.rax;
976e759959fSBrijesh Singh regs->bx = ghcb->save.rbx;
977e759959fSBrijesh Singh regs->cx = ghcb->save.rcx;
978e759959fSBrijesh Singh regs->dx = ghcb->save.rdx;
979e759959fSBrijesh Singh
980e759959fSBrijesh Singh return ES_OK;
981e759959fSBrijesh Singh }
982e759959fSBrijesh Singh
vc_handle_rdtsc(struct ghcb * ghcb,struct es_em_ctxt * ctxt,unsigned long exit_code)983e759959fSBrijesh Singh static enum es_result vc_handle_rdtsc(struct ghcb *ghcb,
984e759959fSBrijesh Singh struct es_em_ctxt *ctxt,
985e759959fSBrijesh Singh unsigned long exit_code)
986e759959fSBrijesh Singh {
987e759959fSBrijesh Singh bool rdtscp = (exit_code == SVM_EXIT_RDTSCP);
988e759959fSBrijesh Singh enum es_result ret;
989e759959fSBrijesh Singh
9905bb6c1d1SBorislav Petkov ret = sev_es_ghcb_hv_call(ghcb, ctxt, exit_code, 0, 0);
991e759959fSBrijesh Singh if (ret != ES_OK)
992e759959fSBrijesh Singh return ret;
993e759959fSBrijesh Singh
994e759959fSBrijesh Singh if (!(ghcb_rax_is_valid(ghcb) && ghcb_rdx_is_valid(ghcb) &&
995e759959fSBrijesh Singh (!rdtscp || ghcb_rcx_is_valid(ghcb))))
996e759959fSBrijesh Singh return ES_VMM_ERROR;
997e759959fSBrijesh Singh
998e759959fSBrijesh Singh ctxt->regs->ax = ghcb->save.rax;
999e759959fSBrijesh Singh ctxt->regs->dx = ghcb->save.rdx;
1000e759959fSBrijesh Singh if (rdtscp)
1001e759959fSBrijesh Singh ctxt->regs->cx = ghcb->save.rcx;
1002e759959fSBrijesh Singh
1003e759959fSBrijesh Singh return ES_OK;
1004e759959fSBrijesh Singh }
1005b190a043SMichael Roth
1006b190a043SMichael Roth struct cc_setup_data {
1007b190a043SMichael Roth struct setup_data header;
1008b190a043SMichael Roth u32 cc_blob_address;
1009b190a043SMichael Roth };
1010b190a043SMichael Roth
1011b190a043SMichael Roth /*
1012b190a043SMichael Roth * Search for a Confidential Computing blob passed in as a setup_data entry
1013b190a043SMichael Roth * via the Linux Boot Protocol.
1014b190a043SMichael Roth */
1015*56408ed9SArd Biesheuvel static __head
find_cc_blob_setup_data(struct boot_params * bp)1016*56408ed9SArd Biesheuvel struct cc_blob_sev_info *find_cc_blob_setup_data(struct boot_params *bp)
1017b190a043SMichael Roth {
1018b190a043SMichael Roth struct cc_setup_data *sd = NULL;
1019b190a043SMichael Roth struct setup_data *hdr;
1020b190a043SMichael Roth
1021b190a043SMichael Roth hdr = (struct setup_data *)bp->hdr.setup_data;
1022b190a043SMichael Roth
1023b190a043SMichael Roth while (hdr) {
1024b190a043SMichael Roth if (hdr->type == SETUP_CC_BLOB) {
1025b190a043SMichael Roth sd = (struct cc_setup_data *)hdr;
1026b190a043SMichael Roth return (struct cc_blob_sev_info *)(unsigned long)sd->cc_blob_address;
1027b190a043SMichael Roth }
1028b190a043SMichael Roth hdr = (struct setup_data *)hdr->next;
1029b190a043SMichael Roth }
1030b190a043SMichael Roth
1031b190a043SMichael Roth return NULL;
1032b190a043SMichael Roth }
103330612045SMichael Roth
103430612045SMichael Roth /*
103530612045SMichael Roth * Initialize the kernel's copy of the SNP CPUID table, and set up the
103630612045SMichael Roth * pointer that will be used to access it.
103730612045SMichael Roth *
103830612045SMichael Roth * Maintaining a direct mapping of the SNP CPUID table used by firmware would
103930612045SMichael Roth * be possible as an alternative, but the approach is brittle since the
104030612045SMichael Roth * mapping needs to be updated in sync with all the changes to virtual memory
104130612045SMichael Roth * layout and related mapping facilities throughout the boot process.
104230612045SMichael Roth */
setup_cpuid_table(const struct cc_blob_sev_info * cc_info)1043*56408ed9SArd Biesheuvel static void __head setup_cpuid_table(const struct cc_blob_sev_info *cc_info)
104430612045SMichael Roth {
104530612045SMichael Roth const struct snp_cpuid_table *cpuid_table_fw, *cpuid_table;
104630612045SMichael Roth int i;
104730612045SMichael Roth
104830612045SMichael Roth if (!cc_info || !cc_info->cpuid_phys || cc_info->cpuid_len < PAGE_SIZE)
104930612045SMichael Roth sev_es_terminate(SEV_TERM_SET_LINUX, GHCB_TERM_CPUID);
105030612045SMichael Roth
105130612045SMichael Roth cpuid_table_fw = (const struct snp_cpuid_table *)cc_info->cpuid_phys;
105230612045SMichael Roth if (!cpuid_table_fw->count || cpuid_table_fw->count > SNP_CPUID_COUNT_MAX)
105330612045SMichael Roth sev_es_terminate(SEV_TERM_SET_LINUX, GHCB_TERM_CPUID);
105430612045SMichael Roth
105530612045SMichael Roth cpuid_table = snp_cpuid_get_table();
105630612045SMichael Roth memcpy((void *)cpuid_table, cpuid_table_fw, sizeof(*cpuid_table));
105730612045SMichael Roth
105830612045SMichael Roth /* Initialize CPUID ranges for range-checking. */
105930612045SMichael Roth for (i = 0; i < cpuid_table->count; i++) {
106030612045SMichael Roth const struct snp_cpuid_fn *fn = &cpuid_table->fn[i];
106130612045SMichael Roth
106230612045SMichael Roth if (fn->eax_in == 0x0)
10630982fd6bSArd Biesheuvel RIP_REL_REF(cpuid_std_range_max) = fn->eax;
106430612045SMichael Roth else if (fn->eax_in == 0x40000000)
10650982fd6bSArd Biesheuvel RIP_REL_REF(cpuid_hyp_range_max) = fn->eax;
106630612045SMichael Roth else if (fn->eax_in == 0x80000000)
10670982fd6bSArd Biesheuvel RIP_REL_REF(cpuid_ext_range_max) = fn->eax;
106830612045SMichael Roth }
106930612045SMichael Roth }
10706c321179STom Lendacky
pvalidate_pages(struct snp_psc_desc * desc)10716c321179STom Lendacky static void pvalidate_pages(struct snp_psc_desc *desc)
10726c321179STom Lendacky {
10736c321179STom Lendacky struct psc_entry *e;
10746c321179STom Lendacky unsigned long vaddr;
10756c321179STom Lendacky unsigned int size;
10766c321179STom Lendacky unsigned int i;
10776c321179STom Lendacky bool validate;
10786c321179STom Lendacky int rc;
10796c321179STom Lendacky
10806c321179STom Lendacky for (i = 0; i <= desc->hdr.end_entry; i++) {
10816c321179STom Lendacky e = &desc->entries[i];
10826c321179STom Lendacky
10836c321179STom Lendacky vaddr = (unsigned long)pfn_to_kaddr(e->gfn);
10846c321179STom Lendacky size = e->pagesize ? RMP_PG_SIZE_2M : RMP_PG_SIZE_4K;
10856c321179STom Lendacky validate = e->operation == SNP_PAGE_STATE_PRIVATE;
10866c321179STom Lendacky
10876c321179STom Lendacky rc = pvalidate(vaddr, size, validate);
10886c321179STom Lendacky if (rc == PVALIDATE_FAIL_SIZEMISMATCH && size == RMP_PG_SIZE_2M) {
10896c321179STom Lendacky unsigned long vaddr_end = vaddr + PMD_SIZE;
10906c321179STom Lendacky
10916c321179STom Lendacky for (; vaddr < vaddr_end; vaddr += PAGE_SIZE) {
10926c321179STom Lendacky rc = pvalidate(vaddr, RMP_PG_SIZE_4K, validate);
10936c321179STom Lendacky if (rc)
10946c321179STom Lendacky break;
10956c321179STom Lendacky }
10966c321179STom Lendacky }
10976c321179STom Lendacky
10986c321179STom Lendacky if (rc) {
10996c321179STom Lendacky WARN(1, "Failed to validate address 0x%lx ret %d", vaddr, rc);
11006c321179STom Lendacky sev_es_terminate(SEV_TERM_SET_LINUX, GHCB_TERM_PVALIDATE);
11016c321179STom Lendacky }
11026c321179STom Lendacky }
11036c321179STom Lendacky }
11046c321179STom Lendacky
vmgexit_psc(struct ghcb * ghcb,struct snp_psc_desc * desc)11056c321179STom Lendacky static int vmgexit_psc(struct ghcb *ghcb, struct snp_psc_desc *desc)
11066c321179STom Lendacky {
11076c321179STom Lendacky int cur_entry, end_entry, ret = 0;
11086c321179STom Lendacky struct snp_psc_desc *data;
11096c321179STom Lendacky struct es_em_ctxt ctxt;
11106c321179STom Lendacky
11116c321179STom Lendacky vc_ghcb_invalidate(ghcb);
11126c321179STom Lendacky
11136c321179STom Lendacky /* Copy the input desc into GHCB shared buffer */
11146c321179STom Lendacky data = (struct snp_psc_desc *)ghcb->shared_buffer;
11156c321179STom Lendacky memcpy(ghcb->shared_buffer, desc, min_t(int, GHCB_SHARED_BUF_SIZE, sizeof(*desc)));
11166c321179STom Lendacky
11176c321179STom Lendacky /*
11186c321179STom Lendacky * As per the GHCB specification, the hypervisor can resume the guest
11196c321179STom Lendacky * before processing all the entries. Check whether all the entries
11206c321179STom Lendacky * are processed. If not, then keep retrying. Note, the hypervisor
11216c321179STom Lendacky * will update the data memory directly to indicate the status, so
11226c321179STom Lendacky * reference the data->hdr everywhere.
11236c321179STom Lendacky *
11246c321179STom Lendacky * The strategy here is to wait for the hypervisor to change the page
11256c321179STom Lendacky * state in the RMP table before guest accesses the memory pages. If the
11266c321179STom Lendacky * page state change was not successful, then later memory access will
11276c321179STom Lendacky * result in a crash.
11286c321179STom Lendacky */
11296c321179STom Lendacky cur_entry = data->hdr.cur_entry;
11306c321179STom Lendacky end_entry = data->hdr.end_entry;
11316c321179STom Lendacky
11326c321179STom Lendacky while (data->hdr.cur_entry <= data->hdr.end_entry) {
11336c321179STom Lendacky ghcb_set_sw_scratch(ghcb, (u64)__pa(data));
11346c321179STom Lendacky
11356c321179STom Lendacky /* This will advance the shared buffer data points to. */
11366c321179STom Lendacky ret = sev_es_ghcb_hv_call(ghcb, &ctxt, SVM_VMGEXIT_PSC, 0, 0);
11376c321179STom Lendacky
11386c321179STom Lendacky /*
11396c321179STom Lendacky * Page State Change VMGEXIT can pass error code through
11406c321179STom Lendacky * exit_info_2.
11416c321179STom Lendacky */
11426c321179STom Lendacky if (WARN(ret || ghcb->save.sw_exit_info_2,
11436c321179STom Lendacky "SNP: PSC failed ret=%d exit_info_2=%llx\n",
11446c321179STom Lendacky ret, ghcb->save.sw_exit_info_2)) {
11456c321179STom Lendacky ret = 1;
11466c321179STom Lendacky goto out;
11476c321179STom Lendacky }
11486c321179STom Lendacky
11496c321179STom Lendacky /* Verify that reserved bit is not set */
11506c321179STom Lendacky if (WARN(data->hdr.reserved, "Reserved bit is set in the PSC header\n")) {
11516c321179STom Lendacky ret = 1;
11526c321179STom Lendacky goto out;
11536c321179STom Lendacky }
11546c321179STom Lendacky
11556c321179STom Lendacky /*
11566c321179STom Lendacky * Sanity check that entry processing is not going backwards.
11576c321179STom Lendacky * This will happen only if hypervisor is tricking us.
11586c321179STom Lendacky */
11596c321179STom Lendacky if (WARN(data->hdr.end_entry > end_entry || cur_entry > data->hdr.cur_entry,
11606c321179STom Lendacky "SNP: PSC processing going backward, end_entry %d (got %d) cur_entry %d (got %d)\n",
11616c321179STom Lendacky end_entry, data->hdr.end_entry, cur_entry, data->hdr.cur_entry)) {
11626c321179STom Lendacky ret = 1;
11636c321179STom Lendacky goto out;
11646c321179STom Lendacky }
11656c321179STom Lendacky }
11666c321179STom Lendacky
11676c321179STom Lendacky out:
11686c321179STom Lendacky return ret;
11696c321179STom Lendacky }
1170