1e759959fSBrijesh Singh // SPDX-License-Identifier: GPL-2.0 2e759959fSBrijesh Singh /* 3e759959fSBrijesh Singh * AMD Encrypted Register State Support 4e759959fSBrijesh Singh * 5e759959fSBrijesh Singh * Author: Joerg Roedel <jroedel@suse.de> 6e759959fSBrijesh Singh * 7e759959fSBrijesh Singh * This file is not compiled stand-alone. It contains code shared 8e759959fSBrijesh Singh * between the pre-decompression boot code and the running Linux kernel 9e759959fSBrijesh Singh * and is included directly into both code-bases. 10e759959fSBrijesh Singh */ 11e759959fSBrijesh Singh 12e759959fSBrijesh Singh #ifndef __BOOT_COMPRESSED 13e759959fSBrijesh Singh #define error(v) pr_err(v) 14e759959fSBrijesh Singh #define has_cpuflag(f) boot_cpu_has(f) 15e759959fSBrijesh Singh #endif 16e759959fSBrijesh Singh 17e759959fSBrijesh Singh static bool __init sev_es_check_cpu_features(void) 18e759959fSBrijesh Singh { 19e759959fSBrijesh Singh if (!has_cpuflag(X86_FEATURE_RDRAND)) { 20e759959fSBrijesh Singh error("RDRAND instruction not supported - no trusted source of randomness available\n"); 21e759959fSBrijesh Singh return false; 22e759959fSBrijesh Singh } 23e759959fSBrijesh Singh 24e759959fSBrijesh Singh return true; 25e759959fSBrijesh Singh } 26e759959fSBrijesh Singh 27e759959fSBrijesh Singh static void __noreturn sev_es_terminate(unsigned int reason) 28e759959fSBrijesh Singh { 29b81fc74dSBrijesh Singh u64 val = GHCB_MSR_TERM_REQ; 30e759959fSBrijesh Singh 31e759959fSBrijesh Singh /* 32e759959fSBrijesh Singh * Tell the hypervisor what went wrong - only reason-set 0 is 33e759959fSBrijesh Singh * currently supported. 34e759959fSBrijesh Singh */ 35b81fc74dSBrijesh Singh val |= GHCB_SEV_TERM_REASON(0, reason); 36e759959fSBrijesh Singh 37e759959fSBrijesh Singh /* Request Guest Termination from Hypvervisor */ 38e759959fSBrijesh Singh sev_es_wr_ghcb_msr(val); 39e759959fSBrijesh Singh VMGEXIT(); 40e759959fSBrijesh Singh 41e759959fSBrijesh Singh while (true) 42e759959fSBrijesh Singh asm volatile("hlt\n" : : : "memory"); 43e759959fSBrijesh Singh } 44e759959fSBrijesh Singh 45e759959fSBrijesh Singh static bool sev_es_negotiate_protocol(void) 46e759959fSBrijesh Singh { 47e759959fSBrijesh Singh u64 val; 48e759959fSBrijesh Singh 49e759959fSBrijesh Singh /* Do the GHCB protocol version negotiation */ 50b81fc74dSBrijesh Singh sev_es_wr_ghcb_msr(GHCB_MSR_SEV_INFO_REQ); 51e759959fSBrijesh Singh VMGEXIT(); 52e759959fSBrijesh Singh val = sev_es_rd_ghcb_msr(); 53e759959fSBrijesh Singh 54b81fc74dSBrijesh Singh if (GHCB_MSR_INFO(val) != GHCB_MSR_SEV_INFO_RESP) 55e759959fSBrijesh Singh return false; 56e759959fSBrijesh Singh 57b81fc74dSBrijesh Singh if (GHCB_MSR_PROTO_MAX(val) < GHCB_PROTO_OUR || 58b81fc74dSBrijesh Singh GHCB_MSR_PROTO_MIN(val) > GHCB_PROTO_OUR) 59e759959fSBrijesh Singh return false; 60e759959fSBrijesh Singh 61e759959fSBrijesh Singh return true; 62e759959fSBrijesh Singh } 63e759959fSBrijesh Singh 64e759959fSBrijesh Singh static __always_inline void vc_ghcb_invalidate(struct ghcb *ghcb) 65e759959fSBrijesh Singh { 66a50c5bebSTom Lendacky ghcb->save.sw_exit_code = 0; 67e759959fSBrijesh Singh memset(ghcb->save.valid_bitmap, 0, sizeof(ghcb->save.valid_bitmap)); 68e759959fSBrijesh Singh } 69e759959fSBrijesh Singh 70e759959fSBrijesh Singh static bool vc_decoding_needed(unsigned long exit_code) 71e759959fSBrijesh Singh { 72e759959fSBrijesh Singh /* Exceptions don't require to decode the instruction */ 73e759959fSBrijesh Singh return !(exit_code >= SVM_EXIT_EXCP_BASE && 74e759959fSBrijesh Singh exit_code <= SVM_EXIT_LAST_EXCP); 75e759959fSBrijesh Singh } 76e759959fSBrijesh Singh 77e759959fSBrijesh Singh static enum es_result vc_init_em_ctxt(struct es_em_ctxt *ctxt, 78e759959fSBrijesh Singh struct pt_regs *regs, 79e759959fSBrijesh Singh unsigned long exit_code) 80e759959fSBrijesh Singh { 81e759959fSBrijesh Singh enum es_result ret = ES_OK; 82e759959fSBrijesh Singh 83e759959fSBrijesh Singh memset(ctxt, 0, sizeof(*ctxt)); 84e759959fSBrijesh Singh ctxt->regs = regs; 85e759959fSBrijesh Singh 86e759959fSBrijesh Singh if (vc_decoding_needed(exit_code)) 87e759959fSBrijesh Singh ret = vc_decode_insn(ctxt); 88e759959fSBrijesh Singh 89e759959fSBrijesh Singh return ret; 90e759959fSBrijesh Singh } 91e759959fSBrijesh Singh 92e759959fSBrijesh Singh static void vc_finish_insn(struct es_em_ctxt *ctxt) 93e759959fSBrijesh Singh { 94e759959fSBrijesh Singh ctxt->regs->ip += ctxt->insn.length; 95e759959fSBrijesh Singh } 96e759959fSBrijesh Singh 97*c688bd5dSBorislav Petkov static enum es_result verify_exception_info(struct ghcb *ghcb, struct es_em_ctxt *ctxt) 98e759959fSBrijesh Singh { 99*c688bd5dSBorislav Petkov u32 ret; 100e759959fSBrijesh Singh 101*c688bd5dSBorislav Petkov ret = ghcb->save.sw_exit_info_1 & GENMASK_ULL(31, 0); 102*c688bd5dSBorislav Petkov if (!ret) 103*c688bd5dSBorislav Petkov return ES_OK; 104e759959fSBrijesh Singh 105*c688bd5dSBorislav Petkov if (ret == 1) { 106e759959fSBrijesh Singh u64 info = ghcb->save.sw_exit_info_2; 107e759959fSBrijesh Singh unsigned long v; 108e759959fSBrijesh Singh 109e759959fSBrijesh Singh info = ghcb->save.sw_exit_info_2; 110e759959fSBrijesh Singh v = info & SVM_EVTINJ_VEC_MASK; 111e759959fSBrijesh Singh 112e759959fSBrijesh Singh /* Check if exception information from hypervisor is sane. */ 113e759959fSBrijesh Singh if ((info & SVM_EVTINJ_VALID) && 114e759959fSBrijesh Singh ((v == X86_TRAP_GP) || (v == X86_TRAP_UD)) && 115e759959fSBrijesh Singh ((info & SVM_EVTINJ_TYPE_MASK) == SVM_EVTINJ_TYPE_EXEPT)) { 116e759959fSBrijesh Singh ctxt->fi.vector = v; 117*c688bd5dSBorislav Petkov 118e759959fSBrijesh Singh if (info & SVM_EVTINJ_VALID_ERR) 119e759959fSBrijesh Singh ctxt->fi.error_code = info >> 32; 120*c688bd5dSBorislav Petkov 121*c688bd5dSBorislav Petkov return ES_EXCEPTION; 122e759959fSBrijesh Singh } 123e759959fSBrijesh Singh } 124e759959fSBrijesh Singh 125*c688bd5dSBorislav Petkov return ES_VMM_ERROR; 126*c688bd5dSBorislav Petkov } 127*c688bd5dSBorislav Petkov 128*c688bd5dSBorislav Petkov static enum es_result sev_es_ghcb_hv_call(struct ghcb *ghcb, 129*c688bd5dSBorislav Petkov struct es_em_ctxt *ctxt, 130*c688bd5dSBorislav Petkov u64 exit_code, u64 exit_info_1, 131*c688bd5dSBorislav Petkov u64 exit_info_2) 132*c688bd5dSBorislav Petkov { 133*c688bd5dSBorislav Petkov /* Fill in protocol and format specifiers */ 134*c688bd5dSBorislav Petkov ghcb->protocol_version = GHCB_PROTOCOL_MAX; 135*c688bd5dSBorislav Petkov ghcb->ghcb_usage = GHCB_DEFAULT_USAGE; 136*c688bd5dSBorislav Petkov 137*c688bd5dSBorislav Petkov ghcb_set_sw_exit_code(ghcb, exit_code); 138*c688bd5dSBorislav Petkov ghcb_set_sw_exit_info_1(ghcb, exit_info_1); 139*c688bd5dSBorislav Petkov ghcb_set_sw_exit_info_2(ghcb, exit_info_2); 140*c688bd5dSBorislav Petkov 141*c688bd5dSBorislav Petkov sev_es_wr_ghcb_msr(__pa(ghcb)); 142*c688bd5dSBorislav Petkov VMGEXIT(); 143*c688bd5dSBorislav Petkov 144*c688bd5dSBorislav Petkov return verify_exception_info(ghcb, ctxt); 145e759959fSBrijesh Singh } 146e759959fSBrijesh Singh 147e759959fSBrijesh Singh /* 148e759959fSBrijesh Singh * Boot VC Handler - This is the first VC handler during boot, there is no GHCB 149e759959fSBrijesh Singh * page yet, so it only supports the MSR based communication with the 150e759959fSBrijesh Singh * hypervisor and only the CPUID exit-code. 151e759959fSBrijesh Singh */ 152e759959fSBrijesh Singh void __init do_vc_no_ghcb(struct pt_regs *regs, unsigned long exit_code) 153e759959fSBrijesh Singh { 154e759959fSBrijesh Singh unsigned int fn = lower_bits(regs->ax, 32); 155e759959fSBrijesh Singh unsigned long val; 156e759959fSBrijesh Singh 157e759959fSBrijesh Singh /* Only CPUID is supported via MSR protocol */ 158e759959fSBrijesh Singh if (exit_code != SVM_EXIT_CPUID) 159e759959fSBrijesh Singh goto fail; 160e759959fSBrijesh Singh 161e759959fSBrijesh Singh sev_es_wr_ghcb_msr(GHCB_CPUID_REQ(fn, GHCB_CPUID_REQ_EAX)); 162e759959fSBrijesh Singh VMGEXIT(); 163e759959fSBrijesh Singh val = sev_es_rd_ghcb_msr(); 164b81fc74dSBrijesh Singh if (GHCB_RESP_CODE(val) != GHCB_MSR_CPUID_RESP) 165e759959fSBrijesh Singh goto fail; 166e759959fSBrijesh Singh regs->ax = val >> 32; 167e759959fSBrijesh Singh 168e759959fSBrijesh Singh sev_es_wr_ghcb_msr(GHCB_CPUID_REQ(fn, GHCB_CPUID_REQ_EBX)); 169e759959fSBrijesh Singh VMGEXIT(); 170e759959fSBrijesh Singh val = sev_es_rd_ghcb_msr(); 171b81fc74dSBrijesh Singh if (GHCB_RESP_CODE(val) != GHCB_MSR_CPUID_RESP) 172e759959fSBrijesh Singh goto fail; 173e759959fSBrijesh Singh regs->bx = val >> 32; 174e759959fSBrijesh Singh 175e759959fSBrijesh Singh sev_es_wr_ghcb_msr(GHCB_CPUID_REQ(fn, GHCB_CPUID_REQ_ECX)); 176e759959fSBrijesh Singh VMGEXIT(); 177e759959fSBrijesh Singh val = sev_es_rd_ghcb_msr(); 178b81fc74dSBrijesh Singh if (GHCB_RESP_CODE(val) != GHCB_MSR_CPUID_RESP) 179e759959fSBrijesh Singh goto fail; 180e759959fSBrijesh Singh regs->cx = val >> 32; 181e759959fSBrijesh Singh 182e759959fSBrijesh Singh sev_es_wr_ghcb_msr(GHCB_CPUID_REQ(fn, GHCB_CPUID_REQ_EDX)); 183e759959fSBrijesh Singh VMGEXIT(); 184e759959fSBrijesh Singh val = sev_es_rd_ghcb_msr(); 185b81fc74dSBrijesh Singh if (GHCB_RESP_CODE(val) != GHCB_MSR_CPUID_RESP) 186e759959fSBrijesh Singh goto fail; 187e759959fSBrijesh Singh regs->dx = val >> 32; 188e759959fSBrijesh Singh 189e759959fSBrijesh Singh /* 190e759959fSBrijesh Singh * This is a VC handler and the #VC is only raised when SEV-ES is 191e759959fSBrijesh Singh * active, which means SEV must be active too. Do sanity checks on the 192e759959fSBrijesh Singh * CPUID results to make sure the hypervisor does not trick the kernel 193e759959fSBrijesh Singh * into the no-sev path. This could map sensitive data unencrypted and 194e759959fSBrijesh Singh * make it accessible to the hypervisor. 195e759959fSBrijesh Singh * 196e759959fSBrijesh Singh * In particular, check for: 197e759959fSBrijesh Singh * - Availability of CPUID leaf 0x8000001f 198e759959fSBrijesh Singh * - SEV CPUID bit. 199e759959fSBrijesh Singh * 200e759959fSBrijesh Singh * The hypervisor might still report the wrong C-bit position, but this 201e759959fSBrijesh Singh * can't be checked here. 202e759959fSBrijesh Singh */ 203e759959fSBrijesh Singh 204e759959fSBrijesh Singh if (fn == 0x80000000 && (regs->ax < 0x8000001f)) 205e759959fSBrijesh Singh /* SEV leaf check */ 206e759959fSBrijesh Singh goto fail; 207e759959fSBrijesh Singh else if ((fn == 0x8000001f && !(regs->ax & BIT(1)))) 208e759959fSBrijesh Singh /* SEV bit */ 209e759959fSBrijesh Singh goto fail; 210e759959fSBrijesh Singh 211e759959fSBrijesh Singh /* Skip over the CPUID two-byte opcode */ 212e759959fSBrijesh Singh regs->ip += 2; 213e759959fSBrijesh Singh 214e759959fSBrijesh Singh return; 215e759959fSBrijesh Singh 216e759959fSBrijesh Singh fail: 217e759959fSBrijesh Singh /* Terminate the guest */ 218e759959fSBrijesh Singh sev_es_terminate(GHCB_SEV_ES_REASON_GENERAL_REQUEST); 219e759959fSBrijesh Singh } 220e759959fSBrijesh Singh 221e759959fSBrijesh Singh static enum es_result vc_insn_string_read(struct es_em_ctxt *ctxt, 222e759959fSBrijesh Singh void *src, char *buf, 223e759959fSBrijesh Singh unsigned int data_size, 224e759959fSBrijesh Singh unsigned int count, 225e759959fSBrijesh Singh bool backwards) 226e759959fSBrijesh Singh { 227e759959fSBrijesh Singh int i, b = backwards ? -1 : 1; 228e759959fSBrijesh Singh enum es_result ret = ES_OK; 229e759959fSBrijesh Singh 230e759959fSBrijesh Singh for (i = 0; i < count; i++) { 231e759959fSBrijesh Singh void *s = src + (i * data_size * b); 232e759959fSBrijesh Singh char *d = buf + (i * data_size); 233e759959fSBrijesh Singh 234e759959fSBrijesh Singh ret = vc_read_mem(ctxt, s, d, data_size); 235e759959fSBrijesh Singh if (ret != ES_OK) 236e759959fSBrijesh Singh break; 237e759959fSBrijesh Singh } 238e759959fSBrijesh Singh 239e759959fSBrijesh Singh return ret; 240e759959fSBrijesh Singh } 241e759959fSBrijesh Singh 242e759959fSBrijesh Singh static enum es_result vc_insn_string_write(struct es_em_ctxt *ctxt, 243e759959fSBrijesh Singh void *dst, char *buf, 244e759959fSBrijesh Singh unsigned int data_size, 245e759959fSBrijesh Singh unsigned int count, 246e759959fSBrijesh Singh bool backwards) 247e759959fSBrijesh Singh { 248e759959fSBrijesh Singh int i, s = backwards ? -1 : 1; 249e759959fSBrijesh Singh enum es_result ret = ES_OK; 250e759959fSBrijesh Singh 251e759959fSBrijesh Singh for (i = 0; i < count; i++) { 252e759959fSBrijesh Singh void *d = dst + (i * data_size * s); 253e759959fSBrijesh Singh char *b = buf + (i * data_size); 254e759959fSBrijesh Singh 255e759959fSBrijesh Singh ret = vc_write_mem(ctxt, d, b, data_size); 256e759959fSBrijesh Singh if (ret != ES_OK) 257e759959fSBrijesh Singh break; 258e759959fSBrijesh Singh } 259e759959fSBrijesh Singh 260e759959fSBrijesh Singh return ret; 261e759959fSBrijesh Singh } 262e759959fSBrijesh Singh 263e759959fSBrijesh Singh #define IOIO_TYPE_STR BIT(2) 264e759959fSBrijesh Singh #define IOIO_TYPE_IN 1 265e759959fSBrijesh Singh #define IOIO_TYPE_INS (IOIO_TYPE_IN | IOIO_TYPE_STR) 266e759959fSBrijesh Singh #define IOIO_TYPE_OUT 0 267e759959fSBrijesh Singh #define IOIO_TYPE_OUTS (IOIO_TYPE_OUT | IOIO_TYPE_STR) 268e759959fSBrijesh Singh 269e759959fSBrijesh Singh #define IOIO_REP BIT(3) 270e759959fSBrijesh Singh 271e759959fSBrijesh Singh #define IOIO_ADDR_64 BIT(9) 272e759959fSBrijesh Singh #define IOIO_ADDR_32 BIT(8) 273e759959fSBrijesh Singh #define IOIO_ADDR_16 BIT(7) 274e759959fSBrijesh Singh 275e759959fSBrijesh Singh #define IOIO_DATA_32 BIT(6) 276e759959fSBrijesh Singh #define IOIO_DATA_16 BIT(5) 277e759959fSBrijesh Singh #define IOIO_DATA_8 BIT(4) 278e759959fSBrijesh Singh 279e759959fSBrijesh Singh #define IOIO_SEG_ES (0 << 10) 280e759959fSBrijesh Singh #define IOIO_SEG_DS (3 << 10) 281e759959fSBrijesh Singh 282e759959fSBrijesh Singh static enum es_result vc_ioio_exitinfo(struct es_em_ctxt *ctxt, u64 *exitinfo) 283e759959fSBrijesh Singh { 284e759959fSBrijesh Singh struct insn *insn = &ctxt->insn; 285e759959fSBrijesh Singh *exitinfo = 0; 286e759959fSBrijesh Singh 287e759959fSBrijesh Singh switch (insn->opcode.bytes[0]) { 288e759959fSBrijesh Singh /* INS opcodes */ 289e759959fSBrijesh Singh case 0x6c: 290e759959fSBrijesh Singh case 0x6d: 291e759959fSBrijesh Singh *exitinfo |= IOIO_TYPE_INS; 292e759959fSBrijesh Singh *exitinfo |= IOIO_SEG_ES; 293e759959fSBrijesh Singh *exitinfo |= (ctxt->regs->dx & 0xffff) << 16; 294e759959fSBrijesh Singh break; 295e759959fSBrijesh Singh 296e759959fSBrijesh Singh /* OUTS opcodes */ 297e759959fSBrijesh Singh case 0x6e: 298e759959fSBrijesh Singh case 0x6f: 299e759959fSBrijesh Singh *exitinfo |= IOIO_TYPE_OUTS; 300e759959fSBrijesh Singh *exitinfo |= IOIO_SEG_DS; 301e759959fSBrijesh Singh *exitinfo |= (ctxt->regs->dx & 0xffff) << 16; 302e759959fSBrijesh Singh break; 303e759959fSBrijesh Singh 304e759959fSBrijesh Singh /* IN immediate opcodes */ 305e759959fSBrijesh Singh case 0xe4: 306e759959fSBrijesh Singh case 0xe5: 307e759959fSBrijesh Singh *exitinfo |= IOIO_TYPE_IN; 308e759959fSBrijesh Singh *exitinfo |= (u8)insn->immediate.value << 16; 309e759959fSBrijesh Singh break; 310e759959fSBrijesh Singh 311e759959fSBrijesh Singh /* OUT immediate opcodes */ 312e759959fSBrijesh Singh case 0xe6: 313e759959fSBrijesh Singh case 0xe7: 314e759959fSBrijesh Singh *exitinfo |= IOIO_TYPE_OUT; 315e759959fSBrijesh Singh *exitinfo |= (u8)insn->immediate.value << 16; 316e759959fSBrijesh Singh break; 317e759959fSBrijesh Singh 318e759959fSBrijesh Singh /* IN register opcodes */ 319e759959fSBrijesh Singh case 0xec: 320e759959fSBrijesh Singh case 0xed: 321e759959fSBrijesh Singh *exitinfo |= IOIO_TYPE_IN; 322e759959fSBrijesh Singh *exitinfo |= (ctxt->regs->dx & 0xffff) << 16; 323e759959fSBrijesh Singh break; 324e759959fSBrijesh Singh 325e759959fSBrijesh Singh /* OUT register opcodes */ 326e759959fSBrijesh Singh case 0xee: 327e759959fSBrijesh Singh case 0xef: 328e759959fSBrijesh Singh *exitinfo |= IOIO_TYPE_OUT; 329e759959fSBrijesh Singh *exitinfo |= (ctxt->regs->dx & 0xffff) << 16; 330e759959fSBrijesh Singh break; 331e759959fSBrijesh Singh 332e759959fSBrijesh Singh default: 333e759959fSBrijesh Singh return ES_DECODE_FAILED; 334e759959fSBrijesh Singh } 335e759959fSBrijesh Singh 336e759959fSBrijesh Singh switch (insn->opcode.bytes[0]) { 337e759959fSBrijesh Singh case 0x6c: 338e759959fSBrijesh Singh case 0x6e: 339e759959fSBrijesh Singh case 0xe4: 340e759959fSBrijesh Singh case 0xe6: 341e759959fSBrijesh Singh case 0xec: 342e759959fSBrijesh Singh case 0xee: 343e759959fSBrijesh Singh /* Single byte opcodes */ 344e759959fSBrijesh Singh *exitinfo |= IOIO_DATA_8; 345e759959fSBrijesh Singh break; 346e759959fSBrijesh Singh default: 347e759959fSBrijesh Singh /* Length determined by instruction parsing */ 348e759959fSBrijesh Singh *exitinfo |= (insn->opnd_bytes == 2) ? IOIO_DATA_16 349e759959fSBrijesh Singh : IOIO_DATA_32; 350e759959fSBrijesh Singh } 351e759959fSBrijesh Singh switch (insn->addr_bytes) { 352e759959fSBrijesh Singh case 2: 353e759959fSBrijesh Singh *exitinfo |= IOIO_ADDR_16; 354e759959fSBrijesh Singh break; 355e759959fSBrijesh Singh case 4: 356e759959fSBrijesh Singh *exitinfo |= IOIO_ADDR_32; 357e759959fSBrijesh Singh break; 358e759959fSBrijesh Singh case 8: 359e759959fSBrijesh Singh *exitinfo |= IOIO_ADDR_64; 360e759959fSBrijesh Singh break; 361e759959fSBrijesh Singh } 362e759959fSBrijesh Singh 363e759959fSBrijesh Singh if (insn_has_rep_prefix(insn)) 364e759959fSBrijesh Singh *exitinfo |= IOIO_REP; 365e759959fSBrijesh Singh 366e759959fSBrijesh Singh return ES_OK; 367e759959fSBrijesh Singh } 368e759959fSBrijesh Singh 369e759959fSBrijesh Singh static enum es_result vc_handle_ioio(struct ghcb *ghcb, struct es_em_ctxt *ctxt) 370e759959fSBrijesh Singh { 371e759959fSBrijesh Singh struct pt_regs *regs = ctxt->regs; 372e759959fSBrijesh Singh u64 exit_info_1, exit_info_2; 373e759959fSBrijesh Singh enum es_result ret; 374e759959fSBrijesh Singh 375e759959fSBrijesh Singh ret = vc_ioio_exitinfo(ctxt, &exit_info_1); 376e759959fSBrijesh Singh if (ret != ES_OK) 377e759959fSBrijesh Singh return ret; 378e759959fSBrijesh Singh 379e759959fSBrijesh Singh if (exit_info_1 & IOIO_TYPE_STR) { 380e759959fSBrijesh Singh 381e759959fSBrijesh Singh /* (REP) INS/OUTS */ 382e759959fSBrijesh Singh 383e759959fSBrijesh Singh bool df = ((regs->flags & X86_EFLAGS_DF) == X86_EFLAGS_DF); 384e759959fSBrijesh Singh unsigned int io_bytes, exit_bytes; 385e759959fSBrijesh Singh unsigned int ghcb_count, op_count; 386e759959fSBrijesh Singh unsigned long es_base; 387e759959fSBrijesh Singh u64 sw_scratch; 388e759959fSBrijesh Singh 389e759959fSBrijesh Singh /* 390e759959fSBrijesh Singh * For the string variants with rep prefix the amount of in/out 391e759959fSBrijesh Singh * operations per #VC exception is limited so that the kernel 392e759959fSBrijesh Singh * has a chance to take interrupts and re-schedule while the 393e759959fSBrijesh Singh * instruction is emulated. 394e759959fSBrijesh Singh */ 395e759959fSBrijesh Singh io_bytes = (exit_info_1 >> 4) & 0x7; 396e759959fSBrijesh Singh ghcb_count = sizeof(ghcb->shared_buffer) / io_bytes; 397e759959fSBrijesh Singh 398e759959fSBrijesh Singh op_count = (exit_info_1 & IOIO_REP) ? regs->cx : 1; 399e759959fSBrijesh Singh exit_info_2 = min(op_count, ghcb_count); 400e759959fSBrijesh Singh exit_bytes = exit_info_2 * io_bytes; 401e759959fSBrijesh Singh 402e759959fSBrijesh Singh es_base = insn_get_seg_base(ctxt->regs, INAT_SEG_REG_ES); 403e759959fSBrijesh Singh 404e759959fSBrijesh Singh /* Read bytes of OUTS into the shared buffer */ 405e759959fSBrijesh Singh if (!(exit_info_1 & IOIO_TYPE_IN)) { 406e759959fSBrijesh Singh ret = vc_insn_string_read(ctxt, 407e759959fSBrijesh Singh (void *)(es_base + regs->si), 408e759959fSBrijesh Singh ghcb->shared_buffer, io_bytes, 409e759959fSBrijesh Singh exit_info_2, df); 410e759959fSBrijesh Singh if (ret) 411e759959fSBrijesh Singh return ret; 412e759959fSBrijesh Singh } 413e759959fSBrijesh Singh 414e759959fSBrijesh Singh /* 415e759959fSBrijesh Singh * Issue an VMGEXIT to the HV to consume the bytes from the 416e759959fSBrijesh Singh * shared buffer or to have it write them into the shared buffer 417e759959fSBrijesh Singh * depending on the instruction: OUTS or INS. 418e759959fSBrijesh Singh */ 419e759959fSBrijesh Singh sw_scratch = __pa(ghcb) + offsetof(struct ghcb, shared_buffer); 420e759959fSBrijesh Singh ghcb_set_sw_scratch(ghcb, sw_scratch); 421e759959fSBrijesh Singh ret = sev_es_ghcb_hv_call(ghcb, ctxt, SVM_EXIT_IOIO, 422e759959fSBrijesh Singh exit_info_1, exit_info_2); 423e759959fSBrijesh Singh if (ret != ES_OK) 424e759959fSBrijesh Singh return ret; 425e759959fSBrijesh Singh 426e759959fSBrijesh Singh /* Read bytes from shared buffer into the guest's destination. */ 427e759959fSBrijesh Singh if (exit_info_1 & IOIO_TYPE_IN) { 428e759959fSBrijesh Singh ret = vc_insn_string_write(ctxt, 429e759959fSBrijesh Singh (void *)(es_base + regs->di), 430e759959fSBrijesh Singh ghcb->shared_buffer, io_bytes, 431e759959fSBrijesh Singh exit_info_2, df); 432e759959fSBrijesh Singh if (ret) 433e759959fSBrijesh Singh return ret; 434e759959fSBrijesh Singh 435e759959fSBrijesh Singh if (df) 436e759959fSBrijesh Singh regs->di -= exit_bytes; 437e759959fSBrijesh Singh else 438e759959fSBrijesh Singh regs->di += exit_bytes; 439e759959fSBrijesh Singh } else { 440e759959fSBrijesh Singh if (df) 441e759959fSBrijesh Singh regs->si -= exit_bytes; 442e759959fSBrijesh Singh else 443e759959fSBrijesh Singh regs->si += exit_bytes; 444e759959fSBrijesh Singh } 445e759959fSBrijesh Singh 446e759959fSBrijesh Singh if (exit_info_1 & IOIO_REP) 447e759959fSBrijesh Singh regs->cx -= exit_info_2; 448e759959fSBrijesh Singh 449e759959fSBrijesh Singh ret = regs->cx ? ES_RETRY : ES_OK; 450e759959fSBrijesh Singh 451e759959fSBrijesh Singh } else { 452e759959fSBrijesh Singh 453e759959fSBrijesh Singh /* IN/OUT into/from rAX */ 454e759959fSBrijesh Singh 455e759959fSBrijesh Singh int bits = (exit_info_1 & 0x70) >> 1; 456e759959fSBrijesh Singh u64 rax = 0; 457e759959fSBrijesh Singh 458e759959fSBrijesh Singh if (!(exit_info_1 & IOIO_TYPE_IN)) 459e759959fSBrijesh Singh rax = lower_bits(regs->ax, bits); 460e759959fSBrijesh Singh 461e759959fSBrijesh Singh ghcb_set_rax(ghcb, rax); 462e759959fSBrijesh Singh 463e759959fSBrijesh Singh ret = sev_es_ghcb_hv_call(ghcb, ctxt, SVM_EXIT_IOIO, exit_info_1, 0); 464e759959fSBrijesh Singh if (ret != ES_OK) 465e759959fSBrijesh Singh return ret; 466e759959fSBrijesh Singh 467e759959fSBrijesh Singh if (exit_info_1 & IOIO_TYPE_IN) { 468e759959fSBrijesh Singh if (!ghcb_rax_is_valid(ghcb)) 469e759959fSBrijesh Singh return ES_VMM_ERROR; 470e759959fSBrijesh Singh regs->ax = lower_bits(ghcb->save.rax, bits); 471e759959fSBrijesh Singh } 472e759959fSBrijesh Singh } 473e759959fSBrijesh Singh 474e759959fSBrijesh Singh return ret; 475e759959fSBrijesh Singh } 476e759959fSBrijesh Singh 477e759959fSBrijesh Singh static enum es_result vc_handle_cpuid(struct ghcb *ghcb, 478e759959fSBrijesh Singh struct es_em_ctxt *ctxt) 479e759959fSBrijesh Singh { 480e759959fSBrijesh Singh struct pt_regs *regs = ctxt->regs; 481e759959fSBrijesh Singh u32 cr4 = native_read_cr4(); 482e759959fSBrijesh Singh enum es_result ret; 483e759959fSBrijesh Singh 484e759959fSBrijesh Singh ghcb_set_rax(ghcb, regs->ax); 485e759959fSBrijesh Singh ghcb_set_rcx(ghcb, regs->cx); 486e759959fSBrijesh Singh 487e759959fSBrijesh Singh if (cr4 & X86_CR4_OSXSAVE) 488e759959fSBrijesh Singh /* Safe to read xcr0 */ 489e759959fSBrijesh Singh ghcb_set_xcr0(ghcb, xgetbv(XCR_XFEATURE_ENABLED_MASK)); 490e759959fSBrijesh Singh else 491e759959fSBrijesh Singh /* xgetbv will cause #GP - use reset value for xcr0 */ 492e759959fSBrijesh Singh ghcb_set_xcr0(ghcb, 1); 493e759959fSBrijesh Singh 494e759959fSBrijesh Singh ret = sev_es_ghcb_hv_call(ghcb, ctxt, SVM_EXIT_CPUID, 0, 0); 495e759959fSBrijesh Singh if (ret != ES_OK) 496e759959fSBrijesh Singh return ret; 497e759959fSBrijesh Singh 498e759959fSBrijesh Singh if (!(ghcb_rax_is_valid(ghcb) && 499e759959fSBrijesh Singh ghcb_rbx_is_valid(ghcb) && 500e759959fSBrijesh Singh ghcb_rcx_is_valid(ghcb) && 501e759959fSBrijesh Singh ghcb_rdx_is_valid(ghcb))) 502e759959fSBrijesh Singh return ES_VMM_ERROR; 503e759959fSBrijesh Singh 504e759959fSBrijesh Singh regs->ax = ghcb->save.rax; 505e759959fSBrijesh Singh regs->bx = ghcb->save.rbx; 506e759959fSBrijesh Singh regs->cx = ghcb->save.rcx; 507e759959fSBrijesh Singh regs->dx = ghcb->save.rdx; 508e759959fSBrijesh Singh 509e759959fSBrijesh Singh return ES_OK; 510e759959fSBrijesh Singh } 511e759959fSBrijesh Singh 512e759959fSBrijesh Singh static enum es_result vc_handle_rdtsc(struct ghcb *ghcb, 513e759959fSBrijesh Singh struct es_em_ctxt *ctxt, 514e759959fSBrijesh Singh unsigned long exit_code) 515e759959fSBrijesh Singh { 516e759959fSBrijesh Singh bool rdtscp = (exit_code == SVM_EXIT_RDTSCP); 517e759959fSBrijesh Singh enum es_result ret; 518e759959fSBrijesh Singh 519e759959fSBrijesh Singh ret = sev_es_ghcb_hv_call(ghcb, ctxt, exit_code, 0, 0); 520e759959fSBrijesh Singh if (ret != ES_OK) 521e759959fSBrijesh Singh return ret; 522e759959fSBrijesh Singh 523e759959fSBrijesh Singh if (!(ghcb_rax_is_valid(ghcb) && ghcb_rdx_is_valid(ghcb) && 524e759959fSBrijesh Singh (!rdtscp || ghcb_rcx_is_valid(ghcb)))) 525e759959fSBrijesh Singh return ES_VMM_ERROR; 526e759959fSBrijesh Singh 527e759959fSBrijesh Singh ctxt->regs->ax = ghcb->save.rax; 528e759959fSBrijesh Singh ctxt->regs->dx = ghcb->save.rdx; 529e759959fSBrijesh Singh if (rdtscp) 530e759959fSBrijesh Singh ctxt->regs->cx = ghcb->save.rcx; 531e759959fSBrijesh Singh 532e759959fSBrijesh Singh return ES_OK; 533e759959fSBrijesh Singh } 534