1e759959fSBrijesh Singh // SPDX-License-Identifier: GPL-2.0 2e759959fSBrijesh Singh /* 3e759959fSBrijesh Singh * AMD Encrypted Register State Support 4e759959fSBrijesh Singh * 5e759959fSBrijesh Singh * Author: Joerg Roedel <jroedel@suse.de> 6e759959fSBrijesh Singh * 7e759959fSBrijesh Singh * This file is not compiled stand-alone. It contains code shared 8e759959fSBrijesh Singh * between the pre-decompression boot code and the running Linux kernel 9e759959fSBrijesh Singh * and is included directly into both code-bases. 10e759959fSBrijesh Singh */ 11e759959fSBrijesh Singh 12e759959fSBrijesh Singh #ifndef __BOOT_COMPRESSED 13e759959fSBrijesh Singh #define error(v) pr_err(v) 14e759959fSBrijesh Singh #define has_cpuflag(f) boot_cpu_has(f) 15e759959fSBrijesh Singh #endif 16e759959fSBrijesh Singh 172ea29c5aSBrijesh Singh /* 182ea29c5aSBrijesh Singh * Since feature negotiation related variables are set early in the boot 192ea29c5aSBrijesh Singh * process they must reside in the .data section so as not to be zeroed 202ea29c5aSBrijesh Singh * out when the .bss section is later cleared. 212ea29c5aSBrijesh Singh * 222ea29c5aSBrijesh Singh * GHCB protocol version negotiated with the hypervisor. 232ea29c5aSBrijesh Singh */ 242ea29c5aSBrijesh Singh static u16 ghcb_version __ro_after_init; 252ea29c5aSBrijesh Singh 26e759959fSBrijesh Singh static bool __init sev_es_check_cpu_features(void) 27e759959fSBrijesh Singh { 28e759959fSBrijesh Singh if (!has_cpuflag(X86_FEATURE_RDRAND)) { 29e759959fSBrijesh Singh error("RDRAND instruction not supported - no trusted source of randomness available\n"); 30e759959fSBrijesh Singh return false; 31e759959fSBrijesh Singh } 32e759959fSBrijesh Singh 33e759959fSBrijesh Singh return true; 34e759959fSBrijesh Singh } 35e759959fSBrijesh Singh 366c0f74d6SBrijesh Singh static void __noreturn sev_es_terminate(unsigned int set, unsigned int reason) 37e759959fSBrijesh Singh { 38b81fc74dSBrijesh Singh u64 val = GHCB_MSR_TERM_REQ; 39e759959fSBrijesh Singh 406c0f74d6SBrijesh Singh /* Tell the hypervisor what went wrong. */ 416c0f74d6SBrijesh Singh val |= GHCB_SEV_TERM_REASON(set, reason); 42e759959fSBrijesh Singh 43e759959fSBrijesh Singh /* Request Guest Termination from Hypvervisor */ 44e759959fSBrijesh Singh sev_es_wr_ghcb_msr(val); 45e759959fSBrijesh Singh VMGEXIT(); 46e759959fSBrijesh Singh 47e759959fSBrijesh Singh while (true) 48e759959fSBrijesh Singh asm volatile("hlt\n" : : : "memory"); 49e759959fSBrijesh Singh } 50e759959fSBrijesh Singh 51cbd3d4f7SBrijesh Singh /* 52cbd3d4f7SBrijesh Singh * The hypervisor features are available from GHCB version 2 onward. 53cbd3d4f7SBrijesh Singh */ 54cbd3d4f7SBrijesh Singh static u64 get_hv_features(void) 55cbd3d4f7SBrijesh Singh { 56cbd3d4f7SBrijesh Singh u64 val; 57cbd3d4f7SBrijesh Singh 58cbd3d4f7SBrijesh Singh if (ghcb_version < 2) 59cbd3d4f7SBrijesh Singh return 0; 60cbd3d4f7SBrijesh Singh 61cbd3d4f7SBrijesh Singh sev_es_wr_ghcb_msr(GHCB_MSR_HV_FT_REQ); 62cbd3d4f7SBrijesh Singh VMGEXIT(); 63cbd3d4f7SBrijesh Singh 64cbd3d4f7SBrijesh Singh val = sev_es_rd_ghcb_msr(); 65cbd3d4f7SBrijesh Singh if (GHCB_RESP_CODE(val) != GHCB_MSR_HV_FT_RESP) 66cbd3d4f7SBrijesh Singh return 0; 67cbd3d4f7SBrijesh Singh 68cbd3d4f7SBrijesh Singh return GHCB_MSR_HV_FT_RESP_VAL(val); 69cbd3d4f7SBrijesh Singh } 70cbd3d4f7SBrijesh Singh 71*87294bdbSBrijesh Singh static void __maybe_unused snp_register_ghcb_early(unsigned long paddr) 72*87294bdbSBrijesh Singh { 73*87294bdbSBrijesh Singh unsigned long pfn = paddr >> PAGE_SHIFT; 74*87294bdbSBrijesh Singh u64 val; 75*87294bdbSBrijesh Singh 76*87294bdbSBrijesh Singh sev_es_wr_ghcb_msr(GHCB_MSR_REG_GPA_REQ_VAL(pfn)); 77*87294bdbSBrijesh Singh VMGEXIT(); 78*87294bdbSBrijesh Singh 79*87294bdbSBrijesh Singh val = sev_es_rd_ghcb_msr(); 80*87294bdbSBrijesh Singh 81*87294bdbSBrijesh Singh /* If the response GPA is not ours then abort the guest */ 82*87294bdbSBrijesh Singh if ((GHCB_RESP_CODE(val) != GHCB_MSR_REG_GPA_RESP) || 83*87294bdbSBrijesh Singh (GHCB_MSR_REG_GPA_RESP_VAL(val) != pfn)) 84*87294bdbSBrijesh Singh sev_es_terminate(SEV_TERM_SET_LINUX, GHCB_TERM_REGISTER); 85*87294bdbSBrijesh Singh } 86*87294bdbSBrijesh Singh 87e759959fSBrijesh Singh static bool sev_es_negotiate_protocol(void) 88e759959fSBrijesh Singh { 89e759959fSBrijesh Singh u64 val; 90e759959fSBrijesh Singh 91e759959fSBrijesh Singh /* Do the GHCB protocol version negotiation */ 92b81fc74dSBrijesh Singh sev_es_wr_ghcb_msr(GHCB_MSR_SEV_INFO_REQ); 93e759959fSBrijesh Singh VMGEXIT(); 94e759959fSBrijesh Singh val = sev_es_rd_ghcb_msr(); 95e759959fSBrijesh Singh 96b81fc74dSBrijesh Singh if (GHCB_MSR_INFO(val) != GHCB_MSR_SEV_INFO_RESP) 97e759959fSBrijesh Singh return false; 98e759959fSBrijesh Singh 992ea29c5aSBrijesh Singh if (GHCB_MSR_PROTO_MAX(val) < GHCB_PROTOCOL_MIN || 1002ea29c5aSBrijesh Singh GHCB_MSR_PROTO_MIN(val) > GHCB_PROTOCOL_MAX) 101e759959fSBrijesh Singh return false; 102e759959fSBrijesh Singh 1032ea29c5aSBrijesh Singh ghcb_version = min_t(size_t, GHCB_MSR_PROTO_MAX(val), GHCB_PROTOCOL_MAX); 1042ea29c5aSBrijesh Singh 105e759959fSBrijesh Singh return true; 106e759959fSBrijesh Singh } 107e759959fSBrijesh Singh 108e759959fSBrijesh Singh static __always_inline void vc_ghcb_invalidate(struct ghcb *ghcb) 109e759959fSBrijesh Singh { 110a50c5bebSTom Lendacky ghcb->save.sw_exit_code = 0; 1112c36d87bSPeter Zijlstra __builtin_memset(ghcb->save.valid_bitmap, 0, sizeof(ghcb->save.valid_bitmap)); 112e759959fSBrijesh Singh } 113e759959fSBrijesh Singh 114e759959fSBrijesh Singh static bool vc_decoding_needed(unsigned long exit_code) 115e759959fSBrijesh Singh { 116e759959fSBrijesh Singh /* Exceptions don't require to decode the instruction */ 117e759959fSBrijesh Singh return !(exit_code >= SVM_EXIT_EXCP_BASE && 118e759959fSBrijesh Singh exit_code <= SVM_EXIT_LAST_EXCP); 119e759959fSBrijesh Singh } 120e759959fSBrijesh Singh 121e759959fSBrijesh Singh static enum es_result vc_init_em_ctxt(struct es_em_ctxt *ctxt, 122e759959fSBrijesh Singh struct pt_regs *regs, 123e759959fSBrijesh Singh unsigned long exit_code) 124e759959fSBrijesh Singh { 125e759959fSBrijesh Singh enum es_result ret = ES_OK; 126e759959fSBrijesh Singh 127e759959fSBrijesh Singh memset(ctxt, 0, sizeof(*ctxt)); 128e759959fSBrijesh Singh ctxt->regs = regs; 129e759959fSBrijesh Singh 130e759959fSBrijesh Singh if (vc_decoding_needed(exit_code)) 131e759959fSBrijesh Singh ret = vc_decode_insn(ctxt); 132e759959fSBrijesh Singh 133e759959fSBrijesh Singh return ret; 134e759959fSBrijesh Singh } 135e759959fSBrijesh Singh 136e759959fSBrijesh Singh static void vc_finish_insn(struct es_em_ctxt *ctxt) 137e759959fSBrijesh Singh { 138e759959fSBrijesh Singh ctxt->regs->ip += ctxt->insn.length; 139e759959fSBrijesh Singh } 140e759959fSBrijesh Singh 141c688bd5dSBorislav Petkov static enum es_result verify_exception_info(struct ghcb *ghcb, struct es_em_ctxt *ctxt) 142e759959fSBrijesh Singh { 143c688bd5dSBorislav Petkov u32 ret; 144e759959fSBrijesh Singh 145c688bd5dSBorislav Petkov ret = ghcb->save.sw_exit_info_1 & GENMASK_ULL(31, 0); 146c688bd5dSBorislav Petkov if (!ret) 147c688bd5dSBorislav Petkov return ES_OK; 148e759959fSBrijesh Singh 149c688bd5dSBorislav Petkov if (ret == 1) { 150e759959fSBrijesh Singh u64 info = ghcb->save.sw_exit_info_2; 151e759959fSBrijesh Singh unsigned long v; 152e759959fSBrijesh Singh 153e759959fSBrijesh Singh info = ghcb->save.sw_exit_info_2; 154e759959fSBrijesh Singh v = info & SVM_EVTINJ_VEC_MASK; 155e759959fSBrijesh Singh 156e759959fSBrijesh Singh /* Check if exception information from hypervisor is sane. */ 157e759959fSBrijesh Singh if ((info & SVM_EVTINJ_VALID) && 158e759959fSBrijesh Singh ((v == X86_TRAP_GP) || (v == X86_TRAP_UD)) && 159e759959fSBrijesh Singh ((info & SVM_EVTINJ_TYPE_MASK) == SVM_EVTINJ_TYPE_EXEPT)) { 160e759959fSBrijesh Singh ctxt->fi.vector = v; 161c688bd5dSBorislav Petkov 162e759959fSBrijesh Singh if (info & SVM_EVTINJ_VALID_ERR) 163e759959fSBrijesh Singh ctxt->fi.error_code = info >> 32; 164c688bd5dSBorislav Petkov 165c688bd5dSBorislav Petkov return ES_EXCEPTION; 166e759959fSBrijesh Singh } 167e759959fSBrijesh Singh } 168e759959fSBrijesh Singh 169c688bd5dSBorislav Petkov return ES_VMM_ERROR; 170c688bd5dSBorislav Petkov } 171c688bd5dSBorislav Petkov 172007faec0STianyu Lan enum es_result sev_es_ghcb_hv_call(struct ghcb *ghcb, bool set_ghcb_msr, 173007faec0STianyu Lan struct es_em_ctxt *ctxt, u64 exit_code, 174007faec0STianyu Lan u64 exit_info_1, u64 exit_info_2) 175c688bd5dSBorislav Petkov { 176c688bd5dSBorislav Petkov /* Fill in protocol and format specifiers */ 1772ea29c5aSBrijesh Singh ghcb->protocol_version = ghcb_version; 178c688bd5dSBorislav Petkov ghcb->ghcb_usage = GHCB_DEFAULT_USAGE; 179c688bd5dSBorislav Petkov 180c688bd5dSBorislav Petkov ghcb_set_sw_exit_code(ghcb, exit_code); 181c688bd5dSBorislav Petkov ghcb_set_sw_exit_info_1(ghcb, exit_info_1); 182c688bd5dSBorislav Petkov ghcb_set_sw_exit_info_2(ghcb, exit_info_2); 183c688bd5dSBorislav Petkov 184007faec0STianyu Lan /* 185007faec0STianyu Lan * Hyper-V unenlightened guests use a paravisor for communicating and 186007faec0STianyu Lan * GHCB pages are being allocated and set up by that paravisor. Linux 187007faec0STianyu Lan * should not change the GHCB page's physical address. 188007faec0STianyu Lan */ 189007faec0STianyu Lan if (set_ghcb_msr) 190c688bd5dSBorislav Petkov sev_es_wr_ghcb_msr(__pa(ghcb)); 191007faec0STianyu Lan 192c688bd5dSBorislav Petkov VMGEXIT(); 193c688bd5dSBorislav Petkov 194c688bd5dSBorislav Petkov return verify_exception_info(ghcb, ctxt); 195e759959fSBrijesh Singh } 196e759959fSBrijesh Singh 197e759959fSBrijesh Singh /* 198e759959fSBrijesh Singh * Boot VC Handler - This is the first VC handler during boot, there is no GHCB 199e759959fSBrijesh Singh * page yet, so it only supports the MSR based communication with the 200e759959fSBrijesh Singh * hypervisor and only the CPUID exit-code. 201e759959fSBrijesh Singh */ 202e759959fSBrijesh Singh void __init do_vc_no_ghcb(struct pt_regs *regs, unsigned long exit_code) 203e759959fSBrijesh Singh { 204e759959fSBrijesh Singh unsigned int fn = lower_bits(regs->ax, 32); 205e759959fSBrijesh Singh unsigned long val; 206e759959fSBrijesh Singh 207e759959fSBrijesh Singh /* Only CPUID is supported via MSR protocol */ 208e759959fSBrijesh Singh if (exit_code != SVM_EXIT_CPUID) 209e759959fSBrijesh Singh goto fail; 210e759959fSBrijesh Singh 211e759959fSBrijesh Singh sev_es_wr_ghcb_msr(GHCB_CPUID_REQ(fn, GHCB_CPUID_REQ_EAX)); 212e759959fSBrijesh Singh VMGEXIT(); 213e759959fSBrijesh Singh val = sev_es_rd_ghcb_msr(); 214b81fc74dSBrijesh Singh if (GHCB_RESP_CODE(val) != GHCB_MSR_CPUID_RESP) 215e759959fSBrijesh Singh goto fail; 216e759959fSBrijesh Singh regs->ax = val >> 32; 217e759959fSBrijesh Singh 218e759959fSBrijesh Singh sev_es_wr_ghcb_msr(GHCB_CPUID_REQ(fn, GHCB_CPUID_REQ_EBX)); 219e759959fSBrijesh Singh VMGEXIT(); 220e759959fSBrijesh Singh val = sev_es_rd_ghcb_msr(); 221b81fc74dSBrijesh Singh if (GHCB_RESP_CODE(val) != GHCB_MSR_CPUID_RESP) 222e759959fSBrijesh Singh goto fail; 223e759959fSBrijesh Singh regs->bx = val >> 32; 224e759959fSBrijesh Singh 225e759959fSBrijesh Singh sev_es_wr_ghcb_msr(GHCB_CPUID_REQ(fn, GHCB_CPUID_REQ_ECX)); 226e759959fSBrijesh Singh VMGEXIT(); 227e759959fSBrijesh Singh val = sev_es_rd_ghcb_msr(); 228b81fc74dSBrijesh Singh if (GHCB_RESP_CODE(val) != GHCB_MSR_CPUID_RESP) 229e759959fSBrijesh Singh goto fail; 230e759959fSBrijesh Singh regs->cx = val >> 32; 231e759959fSBrijesh Singh 232e759959fSBrijesh Singh sev_es_wr_ghcb_msr(GHCB_CPUID_REQ(fn, GHCB_CPUID_REQ_EDX)); 233e759959fSBrijesh Singh VMGEXIT(); 234e759959fSBrijesh Singh val = sev_es_rd_ghcb_msr(); 235b81fc74dSBrijesh Singh if (GHCB_RESP_CODE(val) != GHCB_MSR_CPUID_RESP) 236e759959fSBrijesh Singh goto fail; 237e759959fSBrijesh Singh regs->dx = val >> 32; 238e759959fSBrijesh Singh 239e759959fSBrijesh Singh /* 240e759959fSBrijesh Singh * This is a VC handler and the #VC is only raised when SEV-ES is 241e759959fSBrijesh Singh * active, which means SEV must be active too. Do sanity checks on the 242e759959fSBrijesh Singh * CPUID results to make sure the hypervisor does not trick the kernel 243e759959fSBrijesh Singh * into the no-sev path. This could map sensitive data unencrypted and 244e759959fSBrijesh Singh * make it accessible to the hypervisor. 245e759959fSBrijesh Singh * 246e759959fSBrijesh Singh * In particular, check for: 247e759959fSBrijesh Singh * - Availability of CPUID leaf 0x8000001f 248e759959fSBrijesh Singh * - SEV CPUID bit. 249e759959fSBrijesh Singh * 250e759959fSBrijesh Singh * The hypervisor might still report the wrong C-bit position, but this 251e759959fSBrijesh Singh * can't be checked here. 252e759959fSBrijesh Singh */ 253e759959fSBrijesh Singh 254e759959fSBrijesh Singh if (fn == 0x80000000 && (regs->ax < 0x8000001f)) 255e759959fSBrijesh Singh /* SEV leaf check */ 256e759959fSBrijesh Singh goto fail; 257e759959fSBrijesh Singh else if ((fn == 0x8000001f && !(regs->ax & BIT(1)))) 258e759959fSBrijesh Singh /* SEV bit */ 259e759959fSBrijesh Singh goto fail; 260e759959fSBrijesh Singh 261e759959fSBrijesh Singh /* Skip over the CPUID two-byte opcode */ 262e759959fSBrijesh Singh regs->ip += 2; 263e759959fSBrijesh Singh 264e759959fSBrijesh Singh return; 265e759959fSBrijesh Singh 266e759959fSBrijesh Singh fail: 267e759959fSBrijesh Singh /* Terminate the guest */ 2686c0f74d6SBrijesh Singh sev_es_terminate(SEV_TERM_SET_GEN, GHCB_SEV_ES_GEN_REQ); 269e759959fSBrijesh Singh } 270e759959fSBrijesh Singh 271e759959fSBrijesh Singh static enum es_result vc_insn_string_read(struct es_em_ctxt *ctxt, 272e759959fSBrijesh Singh void *src, char *buf, 273e759959fSBrijesh Singh unsigned int data_size, 274e759959fSBrijesh Singh unsigned int count, 275e759959fSBrijesh Singh bool backwards) 276e759959fSBrijesh Singh { 277e759959fSBrijesh Singh int i, b = backwards ? -1 : 1; 278e759959fSBrijesh Singh enum es_result ret = ES_OK; 279e759959fSBrijesh Singh 280e759959fSBrijesh Singh for (i = 0; i < count; i++) { 281e759959fSBrijesh Singh void *s = src + (i * data_size * b); 282e759959fSBrijesh Singh char *d = buf + (i * data_size); 283e759959fSBrijesh Singh 284e759959fSBrijesh Singh ret = vc_read_mem(ctxt, s, d, data_size); 285e759959fSBrijesh Singh if (ret != ES_OK) 286e759959fSBrijesh Singh break; 287e759959fSBrijesh Singh } 288e759959fSBrijesh Singh 289e759959fSBrijesh Singh return ret; 290e759959fSBrijesh Singh } 291e759959fSBrijesh Singh 292e759959fSBrijesh Singh static enum es_result vc_insn_string_write(struct es_em_ctxt *ctxt, 293e759959fSBrijesh Singh void *dst, char *buf, 294e759959fSBrijesh Singh unsigned int data_size, 295e759959fSBrijesh Singh unsigned int count, 296e759959fSBrijesh Singh bool backwards) 297e759959fSBrijesh Singh { 298e759959fSBrijesh Singh int i, s = backwards ? -1 : 1; 299e759959fSBrijesh Singh enum es_result ret = ES_OK; 300e759959fSBrijesh Singh 301e759959fSBrijesh Singh for (i = 0; i < count; i++) { 302e759959fSBrijesh Singh void *d = dst + (i * data_size * s); 303e759959fSBrijesh Singh char *b = buf + (i * data_size); 304e759959fSBrijesh Singh 305e759959fSBrijesh Singh ret = vc_write_mem(ctxt, d, b, data_size); 306e759959fSBrijesh Singh if (ret != ES_OK) 307e759959fSBrijesh Singh break; 308e759959fSBrijesh Singh } 309e759959fSBrijesh Singh 310e759959fSBrijesh Singh return ret; 311e759959fSBrijesh Singh } 312e759959fSBrijesh Singh 313e759959fSBrijesh Singh #define IOIO_TYPE_STR BIT(2) 314e759959fSBrijesh Singh #define IOIO_TYPE_IN 1 315e759959fSBrijesh Singh #define IOIO_TYPE_INS (IOIO_TYPE_IN | IOIO_TYPE_STR) 316e759959fSBrijesh Singh #define IOIO_TYPE_OUT 0 317e759959fSBrijesh Singh #define IOIO_TYPE_OUTS (IOIO_TYPE_OUT | IOIO_TYPE_STR) 318e759959fSBrijesh Singh 319e759959fSBrijesh Singh #define IOIO_REP BIT(3) 320e759959fSBrijesh Singh 321e759959fSBrijesh Singh #define IOIO_ADDR_64 BIT(9) 322e759959fSBrijesh Singh #define IOIO_ADDR_32 BIT(8) 323e759959fSBrijesh Singh #define IOIO_ADDR_16 BIT(7) 324e759959fSBrijesh Singh 325e759959fSBrijesh Singh #define IOIO_DATA_32 BIT(6) 326e759959fSBrijesh Singh #define IOIO_DATA_16 BIT(5) 327e759959fSBrijesh Singh #define IOIO_DATA_8 BIT(4) 328e759959fSBrijesh Singh 329e759959fSBrijesh Singh #define IOIO_SEG_ES (0 << 10) 330e759959fSBrijesh Singh #define IOIO_SEG_DS (3 << 10) 331e759959fSBrijesh Singh 332e759959fSBrijesh Singh static enum es_result vc_ioio_exitinfo(struct es_em_ctxt *ctxt, u64 *exitinfo) 333e759959fSBrijesh Singh { 334e759959fSBrijesh Singh struct insn *insn = &ctxt->insn; 335e759959fSBrijesh Singh *exitinfo = 0; 336e759959fSBrijesh Singh 337e759959fSBrijesh Singh switch (insn->opcode.bytes[0]) { 338e759959fSBrijesh Singh /* INS opcodes */ 339e759959fSBrijesh Singh case 0x6c: 340e759959fSBrijesh Singh case 0x6d: 341e759959fSBrijesh Singh *exitinfo |= IOIO_TYPE_INS; 342e759959fSBrijesh Singh *exitinfo |= IOIO_SEG_ES; 343e759959fSBrijesh Singh *exitinfo |= (ctxt->regs->dx & 0xffff) << 16; 344e759959fSBrijesh Singh break; 345e759959fSBrijesh Singh 346e759959fSBrijesh Singh /* OUTS opcodes */ 347e759959fSBrijesh Singh case 0x6e: 348e759959fSBrijesh Singh case 0x6f: 349e759959fSBrijesh Singh *exitinfo |= IOIO_TYPE_OUTS; 350e759959fSBrijesh Singh *exitinfo |= IOIO_SEG_DS; 351e759959fSBrijesh Singh *exitinfo |= (ctxt->regs->dx & 0xffff) << 16; 352e759959fSBrijesh Singh break; 353e759959fSBrijesh Singh 354e759959fSBrijesh Singh /* IN immediate opcodes */ 355e759959fSBrijesh Singh case 0xe4: 356e759959fSBrijesh Singh case 0xe5: 357e759959fSBrijesh Singh *exitinfo |= IOIO_TYPE_IN; 358e759959fSBrijesh Singh *exitinfo |= (u8)insn->immediate.value << 16; 359e759959fSBrijesh Singh break; 360e759959fSBrijesh Singh 361e759959fSBrijesh Singh /* OUT immediate opcodes */ 362e759959fSBrijesh Singh case 0xe6: 363e759959fSBrijesh Singh case 0xe7: 364e759959fSBrijesh Singh *exitinfo |= IOIO_TYPE_OUT; 365e759959fSBrijesh Singh *exitinfo |= (u8)insn->immediate.value << 16; 366e759959fSBrijesh Singh break; 367e759959fSBrijesh Singh 368e759959fSBrijesh Singh /* IN register opcodes */ 369e759959fSBrijesh Singh case 0xec: 370e759959fSBrijesh Singh case 0xed: 371e759959fSBrijesh Singh *exitinfo |= IOIO_TYPE_IN; 372e759959fSBrijesh Singh *exitinfo |= (ctxt->regs->dx & 0xffff) << 16; 373e759959fSBrijesh Singh break; 374e759959fSBrijesh Singh 375e759959fSBrijesh Singh /* OUT register opcodes */ 376e759959fSBrijesh Singh case 0xee: 377e759959fSBrijesh Singh case 0xef: 378e759959fSBrijesh Singh *exitinfo |= IOIO_TYPE_OUT; 379e759959fSBrijesh Singh *exitinfo |= (ctxt->regs->dx & 0xffff) << 16; 380e759959fSBrijesh Singh break; 381e759959fSBrijesh Singh 382e759959fSBrijesh Singh default: 383e759959fSBrijesh Singh return ES_DECODE_FAILED; 384e759959fSBrijesh Singh } 385e759959fSBrijesh Singh 386e759959fSBrijesh Singh switch (insn->opcode.bytes[0]) { 387e759959fSBrijesh Singh case 0x6c: 388e759959fSBrijesh Singh case 0x6e: 389e759959fSBrijesh Singh case 0xe4: 390e759959fSBrijesh Singh case 0xe6: 391e759959fSBrijesh Singh case 0xec: 392e759959fSBrijesh Singh case 0xee: 393e759959fSBrijesh Singh /* Single byte opcodes */ 394e759959fSBrijesh Singh *exitinfo |= IOIO_DATA_8; 395e759959fSBrijesh Singh break; 396e759959fSBrijesh Singh default: 397e759959fSBrijesh Singh /* Length determined by instruction parsing */ 398e759959fSBrijesh Singh *exitinfo |= (insn->opnd_bytes == 2) ? IOIO_DATA_16 399e759959fSBrijesh Singh : IOIO_DATA_32; 400e759959fSBrijesh Singh } 401e759959fSBrijesh Singh switch (insn->addr_bytes) { 402e759959fSBrijesh Singh case 2: 403e759959fSBrijesh Singh *exitinfo |= IOIO_ADDR_16; 404e759959fSBrijesh Singh break; 405e759959fSBrijesh Singh case 4: 406e759959fSBrijesh Singh *exitinfo |= IOIO_ADDR_32; 407e759959fSBrijesh Singh break; 408e759959fSBrijesh Singh case 8: 409e759959fSBrijesh Singh *exitinfo |= IOIO_ADDR_64; 410e759959fSBrijesh Singh break; 411e759959fSBrijesh Singh } 412e759959fSBrijesh Singh 413e759959fSBrijesh Singh if (insn_has_rep_prefix(insn)) 414e759959fSBrijesh Singh *exitinfo |= IOIO_REP; 415e759959fSBrijesh Singh 416e759959fSBrijesh Singh return ES_OK; 417e759959fSBrijesh Singh } 418e759959fSBrijesh Singh 419e759959fSBrijesh Singh static enum es_result vc_handle_ioio(struct ghcb *ghcb, struct es_em_ctxt *ctxt) 420e759959fSBrijesh Singh { 421e759959fSBrijesh Singh struct pt_regs *regs = ctxt->regs; 422e759959fSBrijesh Singh u64 exit_info_1, exit_info_2; 423e759959fSBrijesh Singh enum es_result ret; 424e759959fSBrijesh Singh 425e759959fSBrijesh Singh ret = vc_ioio_exitinfo(ctxt, &exit_info_1); 426e759959fSBrijesh Singh if (ret != ES_OK) 427e759959fSBrijesh Singh return ret; 428e759959fSBrijesh Singh 429e759959fSBrijesh Singh if (exit_info_1 & IOIO_TYPE_STR) { 430e759959fSBrijesh Singh 431e759959fSBrijesh Singh /* (REP) INS/OUTS */ 432e759959fSBrijesh Singh 433e759959fSBrijesh Singh bool df = ((regs->flags & X86_EFLAGS_DF) == X86_EFLAGS_DF); 434e759959fSBrijesh Singh unsigned int io_bytes, exit_bytes; 435e759959fSBrijesh Singh unsigned int ghcb_count, op_count; 436e759959fSBrijesh Singh unsigned long es_base; 437e759959fSBrijesh Singh u64 sw_scratch; 438e759959fSBrijesh Singh 439e759959fSBrijesh Singh /* 440e759959fSBrijesh Singh * For the string variants with rep prefix the amount of in/out 441e759959fSBrijesh Singh * operations per #VC exception is limited so that the kernel 442e759959fSBrijesh Singh * has a chance to take interrupts and re-schedule while the 443e759959fSBrijesh Singh * instruction is emulated. 444e759959fSBrijesh Singh */ 445e759959fSBrijesh Singh io_bytes = (exit_info_1 >> 4) & 0x7; 446e759959fSBrijesh Singh ghcb_count = sizeof(ghcb->shared_buffer) / io_bytes; 447e759959fSBrijesh Singh 448e759959fSBrijesh Singh op_count = (exit_info_1 & IOIO_REP) ? regs->cx : 1; 449e759959fSBrijesh Singh exit_info_2 = min(op_count, ghcb_count); 450e759959fSBrijesh Singh exit_bytes = exit_info_2 * io_bytes; 451e759959fSBrijesh Singh 452e759959fSBrijesh Singh es_base = insn_get_seg_base(ctxt->regs, INAT_SEG_REG_ES); 453e759959fSBrijesh Singh 454e759959fSBrijesh Singh /* Read bytes of OUTS into the shared buffer */ 455e759959fSBrijesh Singh if (!(exit_info_1 & IOIO_TYPE_IN)) { 456e759959fSBrijesh Singh ret = vc_insn_string_read(ctxt, 457e759959fSBrijesh Singh (void *)(es_base + regs->si), 458e759959fSBrijesh Singh ghcb->shared_buffer, io_bytes, 459e759959fSBrijesh Singh exit_info_2, df); 460e759959fSBrijesh Singh if (ret) 461e759959fSBrijesh Singh return ret; 462e759959fSBrijesh Singh } 463e759959fSBrijesh Singh 464e759959fSBrijesh Singh /* 465e759959fSBrijesh Singh * Issue an VMGEXIT to the HV to consume the bytes from the 466e759959fSBrijesh Singh * shared buffer or to have it write them into the shared buffer 467e759959fSBrijesh Singh * depending on the instruction: OUTS or INS. 468e759959fSBrijesh Singh */ 469e759959fSBrijesh Singh sw_scratch = __pa(ghcb) + offsetof(struct ghcb, shared_buffer); 470e759959fSBrijesh Singh ghcb_set_sw_scratch(ghcb, sw_scratch); 471007faec0STianyu Lan ret = sev_es_ghcb_hv_call(ghcb, true, ctxt, SVM_EXIT_IOIO, 472e759959fSBrijesh Singh exit_info_1, exit_info_2); 473e759959fSBrijesh Singh if (ret != ES_OK) 474e759959fSBrijesh Singh return ret; 475e759959fSBrijesh Singh 476e759959fSBrijesh Singh /* Read bytes from shared buffer into the guest's destination. */ 477e759959fSBrijesh Singh if (exit_info_1 & IOIO_TYPE_IN) { 478e759959fSBrijesh Singh ret = vc_insn_string_write(ctxt, 479e759959fSBrijesh Singh (void *)(es_base + regs->di), 480e759959fSBrijesh Singh ghcb->shared_buffer, io_bytes, 481e759959fSBrijesh Singh exit_info_2, df); 482e759959fSBrijesh Singh if (ret) 483e759959fSBrijesh Singh return ret; 484e759959fSBrijesh Singh 485e759959fSBrijesh Singh if (df) 486e759959fSBrijesh Singh regs->di -= exit_bytes; 487e759959fSBrijesh Singh else 488e759959fSBrijesh Singh regs->di += exit_bytes; 489e759959fSBrijesh Singh } else { 490e759959fSBrijesh Singh if (df) 491e759959fSBrijesh Singh regs->si -= exit_bytes; 492e759959fSBrijesh Singh else 493e759959fSBrijesh Singh regs->si += exit_bytes; 494e759959fSBrijesh Singh } 495e759959fSBrijesh Singh 496e759959fSBrijesh Singh if (exit_info_1 & IOIO_REP) 497e759959fSBrijesh Singh regs->cx -= exit_info_2; 498e759959fSBrijesh Singh 499e759959fSBrijesh Singh ret = regs->cx ? ES_RETRY : ES_OK; 500e759959fSBrijesh Singh 501e759959fSBrijesh Singh } else { 502e759959fSBrijesh Singh 503e759959fSBrijesh Singh /* IN/OUT into/from rAX */ 504e759959fSBrijesh Singh 505e759959fSBrijesh Singh int bits = (exit_info_1 & 0x70) >> 1; 506e759959fSBrijesh Singh u64 rax = 0; 507e759959fSBrijesh Singh 508e759959fSBrijesh Singh if (!(exit_info_1 & IOIO_TYPE_IN)) 509e759959fSBrijesh Singh rax = lower_bits(regs->ax, bits); 510e759959fSBrijesh Singh 511e759959fSBrijesh Singh ghcb_set_rax(ghcb, rax); 512e759959fSBrijesh Singh 513007faec0STianyu Lan ret = sev_es_ghcb_hv_call(ghcb, true, ctxt, 514007faec0STianyu Lan SVM_EXIT_IOIO, exit_info_1, 0); 515e759959fSBrijesh Singh if (ret != ES_OK) 516e759959fSBrijesh Singh return ret; 517e759959fSBrijesh Singh 518e759959fSBrijesh Singh if (exit_info_1 & IOIO_TYPE_IN) { 519e759959fSBrijesh Singh if (!ghcb_rax_is_valid(ghcb)) 520e759959fSBrijesh Singh return ES_VMM_ERROR; 521e759959fSBrijesh Singh regs->ax = lower_bits(ghcb->save.rax, bits); 522e759959fSBrijesh Singh } 523e759959fSBrijesh Singh } 524e759959fSBrijesh Singh 525e759959fSBrijesh Singh return ret; 526e759959fSBrijesh Singh } 527e759959fSBrijesh Singh 528e759959fSBrijesh Singh static enum es_result vc_handle_cpuid(struct ghcb *ghcb, 529e759959fSBrijesh Singh struct es_em_ctxt *ctxt) 530e759959fSBrijesh Singh { 531e759959fSBrijesh Singh struct pt_regs *regs = ctxt->regs; 532e759959fSBrijesh Singh u32 cr4 = native_read_cr4(); 533e759959fSBrijesh Singh enum es_result ret; 534e759959fSBrijesh Singh 535e759959fSBrijesh Singh ghcb_set_rax(ghcb, regs->ax); 536e759959fSBrijesh Singh ghcb_set_rcx(ghcb, regs->cx); 537e759959fSBrijesh Singh 538e759959fSBrijesh Singh if (cr4 & X86_CR4_OSXSAVE) 539e759959fSBrijesh Singh /* Safe to read xcr0 */ 540e759959fSBrijesh Singh ghcb_set_xcr0(ghcb, xgetbv(XCR_XFEATURE_ENABLED_MASK)); 541e759959fSBrijesh Singh else 542e759959fSBrijesh Singh /* xgetbv will cause #GP - use reset value for xcr0 */ 543e759959fSBrijesh Singh ghcb_set_xcr0(ghcb, 1); 544e759959fSBrijesh Singh 545007faec0STianyu Lan ret = sev_es_ghcb_hv_call(ghcb, true, ctxt, SVM_EXIT_CPUID, 0, 0); 546e759959fSBrijesh Singh if (ret != ES_OK) 547e759959fSBrijesh Singh return ret; 548e759959fSBrijesh Singh 549e759959fSBrijesh Singh if (!(ghcb_rax_is_valid(ghcb) && 550e759959fSBrijesh Singh ghcb_rbx_is_valid(ghcb) && 551e759959fSBrijesh Singh ghcb_rcx_is_valid(ghcb) && 552e759959fSBrijesh Singh ghcb_rdx_is_valid(ghcb))) 553e759959fSBrijesh Singh return ES_VMM_ERROR; 554e759959fSBrijesh Singh 555e759959fSBrijesh Singh regs->ax = ghcb->save.rax; 556e759959fSBrijesh Singh regs->bx = ghcb->save.rbx; 557e759959fSBrijesh Singh regs->cx = ghcb->save.rcx; 558e759959fSBrijesh Singh regs->dx = ghcb->save.rdx; 559e759959fSBrijesh Singh 560e759959fSBrijesh Singh return ES_OK; 561e759959fSBrijesh Singh } 562e759959fSBrijesh Singh 563e759959fSBrijesh Singh static enum es_result vc_handle_rdtsc(struct ghcb *ghcb, 564e759959fSBrijesh Singh struct es_em_ctxt *ctxt, 565e759959fSBrijesh Singh unsigned long exit_code) 566e759959fSBrijesh Singh { 567e759959fSBrijesh Singh bool rdtscp = (exit_code == SVM_EXIT_RDTSCP); 568e759959fSBrijesh Singh enum es_result ret; 569e759959fSBrijesh Singh 570007faec0STianyu Lan ret = sev_es_ghcb_hv_call(ghcb, true, ctxt, exit_code, 0, 0); 571e759959fSBrijesh Singh if (ret != ES_OK) 572e759959fSBrijesh Singh return ret; 573e759959fSBrijesh Singh 574e759959fSBrijesh Singh if (!(ghcb_rax_is_valid(ghcb) && ghcb_rdx_is_valid(ghcb) && 575e759959fSBrijesh Singh (!rdtscp || ghcb_rcx_is_valid(ghcb)))) 576e759959fSBrijesh Singh return ES_VMM_ERROR; 577e759959fSBrijesh Singh 578e759959fSBrijesh Singh ctxt->regs->ax = ghcb->save.rax; 579e759959fSBrijesh Singh ctxt->regs->dx = ghcb->save.rdx; 580e759959fSBrijesh Singh if (rdtscp) 581e759959fSBrijesh Singh ctxt->regs->cx = ghcb->save.rcx; 582e759959fSBrijesh Singh 583e759959fSBrijesh Singh return ES_OK; 584e759959fSBrijesh Singh } 585