1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * AMD Encrypted Register State Support 4 * 5 * Author: Joerg Roedel <jroedel@suse.de> 6 * 7 * This file is not compiled stand-alone. It contains code shared 8 * between the pre-decompression boot code and the running Linux kernel 9 * and is included directly into both code-bases. 10 */ 11 12 #ifndef __BOOT_COMPRESSED 13 #define error(v) pr_err(v) 14 #define has_cpuflag(f) boot_cpu_has(f) 15 #endif 16 17 static bool __init sev_es_check_cpu_features(void) 18 { 19 if (!has_cpuflag(X86_FEATURE_RDRAND)) { 20 error("RDRAND instruction not supported - no trusted source of randomness available\n"); 21 return false; 22 } 23 24 return true; 25 } 26 27 static void __noreturn sev_es_terminate(unsigned int reason) 28 { 29 u64 val = GHCB_MSR_TERM_REQ; 30 31 /* 32 * Tell the hypervisor what went wrong - only reason-set 0 is 33 * currently supported. 34 */ 35 val |= GHCB_SEV_TERM_REASON(0, reason); 36 37 /* Request Guest Termination from Hypvervisor */ 38 sev_es_wr_ghcb_msr(val); 39 VMGEXIT(); 40 41 while (true) 42 asm volatile("hlt\n" : : : "memory"); 43 } 44 45 static bool sev_es_negotiate_protocol(void) 46 { 47 u64 val; 48 49 /* Do the GHCB protocol version negotiation */ 50 sev_es_wr_ghcb_msr(GHCB_MSR_SEV_INFO_REQ); 51 VMGEXIT(); 52 val = sev_es_rd_ghcb_msr(); 53 54 if (GHCB_MSR_INFO(val) != GHCB_MSR_SEV_INFO_RESP) 55 return false; 56 57 if (GHCB_MSR_PROTO_MAX(val) < GHCB_PROTO_OUR || 58 GHCB_MSR_PROTO_MIN(val) > GHCB_PROTO_OUR) 59 return false; 60 61 return true; 62 } 63 64 static __always_inline void vc_ghcb_invalidate(struct ghcb *ghcb) 65 { 66 memset(ghcb->save.valid_bitmap, 0, sizeof(ghcb->save.valid_bitmap)); 67 } 68 69 static bool vc_decoding_needed(unsigned long exit_code) 70 { 71 /* Exceptions don't require to decode the instruction */ 72 return !(exit_code >= SVM_EXIT_EXCP_BASE && 73 exit_code <= SVM_EXIT_LAST_EXCP); 74 } 75 76 static enum es_result vc_init_em_ctxt(struct es_em_ctxt *ctxt, 77 struct pt_regs *regs, 78 unsigned long exit_code) 79 { 80 enum es_result ret = ES_OK; 81 82 memset(ctxt, 0, sizeof(*ctxt)); 83 ctxt->regs = regs; 84 85 if (vc_decoding_needed(exit_code)) 86 ret = vc_decode_insn(ctxt); 87 88 return ret; 89 } 90 91 static void vc_finish_insn(struct es_em_ctxt *ctxt) 92 { 93 ctxt->regs->ip += ctxt->insn.length; 94 } 95 96 static enum es_result sev_es_ghcb_hv_call(struct ghcb *ghcb, 97 struct es_em_ctxt *ctxt, 98 u64 exit_code, u64 exit_info_1, 99 u64 exit_info_2) 100 { 101 enum es_result ret; 102 103 /* Fill in protocol and format specifiers */ 104 ghcb->protocol_version = GHCB_PROTOCOL_MAX; 105 ghcb->ghcb_usage = GHCB_DEFAULT_USAGE; 106 107 ghcb_set_sw_exit_code(ghcb, exit_code); 108 ghcb_set_sw_exit_info_1(ghcb, exit_info_1); 109 ghcb_set_sw_exit_info_2(ghcb, exit_info_2); 110 111 sev_es_wr_ghcb_msr(__pa(ghcb)); 112 VMGEXIT(); 113 114 if ((ghcb->save.sw_exit_info_1 & 0xffffffff) == 1) { 115 u64 info = ghcb->save.sw_exit_info_2; 116 unsigned long v; 117 118 info = ghcb->save.sw_exit_info_2; 119 v = info & SVM_EVTINJ_VEC_MASK; 120 121 /* Check if exception information from hypervisor is sane. */ 122 if ((info & SVM_EVTINJ_VALID) && 123 ((v == X86_TRAP_GP) || (v == X86_TRAP_UD)) && 124 ((info & SVM_EVTINJ_TYPE_MASK) == SVM_EVTINJ_TYPE_EXEPT)) { 125 ctxt->fi.vector = v; 126 if (info & SVM_EVTINJ_VALID_ERR) 127 ctxt->fi.error_code = info >> 32; 128 ret = ES_EXCEPTION; 129 } else { 130 ret = ES_VMM_ERROR; 131 } 132 } else { 133 ret = ES_OK; 134 } 135 136 return ret; 137 } 138 139 /* 140 * Boot VC Handler - This is the first VC handler during boot, there is no GHCB 141 * page yet, so it only supports the MSR based communication with the 142 * hypervisor and only the CPUID exit-code. 143 */ 144 void __init do_vc_no_ghcb(struct pt_regs *regs, unsigned long exit_code) 145 { 146 unsigned int fn = lower_bits(regs->ax, 32); 147 unsigned long val; 148 149 /* Only CPUID is supported via MSR protocol */ 150 if (exit_code != SVM_EXIT_CPUID) 151 goto fail; 152 153 sev_es_wr_ghcb_msr(GHCB_CPUID_REQ(fn, GHCB_CPUID_REQ_EAX)); 154 VMGEXIT(); 155 val = sev_es_rd_ghcb_msr(); 156 if (GHCB_RESP_CODE(val) != GHCB_MSR_CPUID_RESP) 157 goto fail; 158 regs->ax = val >> 32; 159 160 sev_es_wr_ghcb_msr(GHCB_CPUID_REQ(fn, GHCB_CPUID_REQ_EBX)); 161 VMGEXIT(); 162 val = sev_es_rd_ghcb_msr(); 163 if (GHCB_RESP_CODE(val) != GHCB_MSR_CPUID_RESP) 164 goto fail; 165 regs->bx = val >> 32; 166 167 sev_es_wr_ghcb_msr(GHCB_CPUID_REQ(fn, GHCB_CPUID_REQ_ECX)); 168 VMGEXIT(); 169 val = sev_es_rd_ghcb_msr(); 170 if (GHCB_RESP_CODE(val) != GHCB_MSR_CPUID_RESP) 171 goto fail; 172 regs->cx = val >> 32; 173 174 sev_es_wr_ghcb_msr(GHCB_CPUID_REQ(fn, GHCB_CPUID_REQ_EDX)); 175 VMGEXIT(); 176 val = sev_es_rd_ghcb_msr(); 177 if (GHCB_RESP_CODE(val) != GHCB_MSR_CPUID_RESP) 178 goto fail; 179 regs->dx = val >> 32; 180 181 /* 182 * This is a VC handler and the #VC is only raised when SEV-ES is 183 * active, which means SEV must be active too. Do sanity checks on the 184 * CPUID results to make sure the hypervisor does not trick the kernel 185 * into the no-sev path. This could map sensitive data unencrypted and 186 * make it accessible to the hypervisor. 187 * 188 * In particular, check for: 189 * - Availability of CPUID leaf 0x8000001f 190 * - SEV CPUID bit. 191 * 192 * The hypervisor might still report the wrong C-bit position, but this 193 * can't be checked here. 194 */ 195 196 if (fn == 0x80000000 && (regs->ax < 0x8000001f)) 197 /* SEV leaf check */ 198 goto fail; 199 else if ((fn == 0x8000001f && !(regs->ax & BIT(1)))) 200 /* SEV bit */ 201 goto fail; 202 203 /* Skip over the CPUID two-byte opcode */ 204 regs->ip += 2; 205 206 return; 207 208 fail: 209 /* Terminate the guest */ 210 sev_es_terminate(GHCB_SEV_ES_REASON_GENERAL_REQUEST); 211 } 212 213 static enum es_result vc_insn_string_read(struct es_em_ctxt *ctxt, 214 void *src, char *buf, 215 unsigned int data_size, 216 unsigned int count, 217 bool backwards) 218 { 219 int i, b = backwards ? -1 : 1; 220 enum es_result ret = ES_OK; 221 222 for (i = 0; i < count; i++) { 223 void *s = src + (i * data_size * b); 224 char *d = buf + (i * data_size); 225 226 ret = vc_read_mem(ctxt, s, d, data_size); 227 if (ret != ES_OK) 228 break; 229 } 230 231 return ret; 232 } 233 234 static enum es_result vc_insn_string_write(struct es_em_ctxt *ctxt, 235 void *dst, char *buf, 236 unsigned int data_size, 237 unsigned int count, 238 bool backwards) 239 { 240 int i, s = backwards ? -1 : 1; 241 enum es_result ret = ES_OK; 242 243 for (i = 0; i < count; i++) { 244 void *d = dst + (i * data_size * s); 245 char *b = buf + (i * data_size); 246 247 ret = vc_write_mem(ctxt, d, b, data_size); 248 if (ret != ES_OK) 249 break; 250 } 251 252 return ret; 253 } 254 255 #define IOIO_TYPE_STR BIT(2) 256 #define IOIO_TYPE_IN 1 257 #define IOIO_TYPE_INS (IOIO_TYPE_IN | IOIO_TYPE_STR) 258 #define IOIO_TYPE_OUT 0 259 #define IOIO_TYPE_OUTS (IOIO_TYPE_OUT | IOIO_TYPE_STR) 260 261 #define IOIO_REP BIT(3) 262 263 #define IOIO_ADDR_64 BIT(9) 264 #define IOIO_ADDR_32 BIT(8) 265 #define IOIO_ADDR_16 BIT(7) 266 267 #define IOIO_DATA_32 BIT(6) 268 #define IOIO_DATA_16 BIT(5) 269 #define IOIO_DATA_8 BIT(4) 270 271 #define IOIO_SEG_ES (0 << 10) 272 #define IOIO_SEG_DS (3 << 10) 273 274 static enum es_result vc_ioio_exitinfo(struct es_em_ctxt *ctxt, u64 *exitinfo) 275 { 276 struct insn *insn = &ctxt->insn; 277 *exitinfo = 0; 278 279 switch (insn->opcode.bytes[0]) { 280 /* INS opcodes */ 281 case 0x6c: 282 case 0x6d: 283 *exitinfo |= IOIO_TYPE_INS; 284 *exitinfo |= IOIO_SEG_ES; 285 *exitinfo |= (ctxt->regs->dx & 0xffff) << 16; 286 break; 287 288 /* OUTS opcodes */ 289 case 0x6e: 290 case 0x6f: 291 *exitinfo |= IOIO_TYPE_OUTS; 292 *exitinfo |= IOIO_SEG_DS; 293 *exitinfo |= (ctxt->regs->dx & 0xffff) << 16; 294 break; 295 296 /* IN immediate opcodes */ 297 case 0xe4: 298 case 0xe5: 299 *exitinfo |= IOIO_TYPE_IN; 300 *exitinfo |= (u8)insn->immediate.value << 16; 301 break; 302 303 /* OUT immediate opcodes */ 304 case 0xe6: 305 case 0xe7: 306 *exitinfo |= IOIO_TYPE_OUT; 307 *exitinfo |= (u8)insn->immediate.value << 16; 308 break; 309 310 /* IN register opcodes */ 311 case 0xec: 312 case 0xed: 313 *exitinfo |= IOIO_TYPE_IN; 314 *exitinfo |= (ctxt->regs->dx & 0xffff) << 16; 315 break; 316 317 /* OUT register opcodes */ 318 case 0xee: 319 case 0xef: 320 *exitinfo |= IOIO_TYPE_OUT; 321 *exitinfo |= (ctxt->regs->dx & 0xffff) << 16; 322 break; 323 324 default: 325 return ES_DECODE_FAILED; 326 } 327 328 switch (insn->opcode.bytes[0]) { 329 case 0x6c: 330 case 0x6e: 331 case 0xe4: 332 case 0xe6: 333 case 0xec: 334 case 0xee: 335 /* Single byte opcodes */ 336 *exitinfo |= IOIO_DATA_8; 337 break; 338 default: 339 /* Length determined by instruction parsing */ 340 *exitinfo |= (insn->opnd_bytes == 2) ? IOIO_DATA_16 341 : IOIO_DATA_32; 342 } 343 switch (insn->addr_bytes) { 344 case 2: 345 *exitinfo |= IOIO_ADDR_16; 346 break; 347 case 4: 348 *exitinfo |= IOIO_ADDR_32; 349 break; 350 case 8: 351 *exitinfo |= IOIO_ADDR_64; 352 break; 353 } 354 355 if (insn_has_rep_prefix(insn)) 356 *exitinfo |= IOIO_REP; 357 358 return ES_OK; 359 } 360 361 static enum es_result vc_handle_ioio(struct ghcb *ghcb, struct es_em_ctxt *ctxt) 362 { 363 struct pt_regs *regs = ctxt->regs; 364 u64 exit_info_1, exit_info_2; 365 enum es_result ret; 366 367 ret = vc_ioio_exitinfo(ctxt, &exit_info_1); 368 if (ret != ES_OK) 369 return ret; 370 371 if (exit_info_1 & IOIO_TYPE_STR) { 372 373 /* (REP) INS/OUTS */ 374 375 bool df = ((regs->flags & X86_EFLAGS_DF) == X86_EFLAGS_DF); 376 unsigned int io_bytes, exit_bytes; 377 unsigned int ghcb_count, op_count; 378 unsigned long es_base; 379 u64 sw_scratch; 380 381 /* 382 * For the string variants with rep prefix the amount of in/out 383 * operations per #VC exception is limited so that the kernel 384 * has a chance to take interrupts and re-schedule while the 385 * instruction is emulated. 386 */ 387 io_bytes = (exit_info_1 >> 4) & 0x7; 388 ghcb_count = sizeof(ghcb->shared_buffer) / io_bytes; 389 390 op_count = (exit_info_1 & IOIO_REP) ? regs->cx : 1; 391 exit_info_2 = min(op_count, ghcb_count); 392 exit_bytes = exit_info_2 * io_bytes; 393 394 es_base = insn_get_seg_base(ctxt->regs, INAT_SEG_REG_ES); 395 396 /* Read bytes of OUTS into the shared buffer */ 397 if (!(exit_info_1 & IOIO_TYPE_IN)) { 398 ret = vc_insn_string_read(ctxt, 399 (void *)(es_base + regs->si), 400 ghcb->shared_buffer, io_bytes, 401 exit_info_2, df); 402 if (ret) 403 return ret; 404 } 405 406 /* 407 * Issue an VMGEXIT to the HV to consume the bytes from the 408 * shared buffer or to have it write them into the shared buffer 409 * depending on the instruction: OUTS or INS. 410 */ 411 sw_scratch = __pa(ghcb) + offsetof(struct ghcb, shared_buffer); 412 ghcb_set_sw_scratch(ghcb, sw_scratch); 413 ret = sev_es_ghcb_hv_call(ghcb, ctxt, SVM_EXIT_IOIO, 414 exit_info_1, exit_info_2); 415 if (ret != ES_OK) 416 return ret; 417 418 /* Read bytes from shared buffer into the guest's destination. */ 419 if (exit_info_1 & IOIO_TYPE_IN) { 420 ret = vc_insn_string_write(ctxt, 421 (void *)(es_base + regs->di), 422 ghcb->shared_buffer, io_bytes, 423 exit_info_2, df); 424 if (ret) 425 return ret; 426 427 if (df) 428 regs->di -= exit_bytes; 429 else 430 regs->di += exit_bytes; 431 } else { 432 if (df) 433 regs->si -= exit_bytes; 434 else 435 regs->si += exit_bytes; 436 } 437 438 if (exit_info_1 & IOIO_REP) 439 regs->cx -= exit_info_2; 440 441 ret = regs->cx ? ES_RETRY : ES_OK; 442 443 } else { 444 445 /* IN/OUT into/from rAX */ 446 447 int bits = (exit_info_1 & 0x70) >> 1; 448 u64 rax = 0; 449 450 if (!(exit_info_1 & IOIO_TYPE_IN)) 451 rax = lower_bits(regs->ax, bits); 452 453 ghcb_set_rax(ghcb, rax); 454 455 ret = sev_es_ghcb_hv_call(ghcb, ctxt, SVM_EXIT_IOIO, exit_info_1, 0); 456 if (ret != ES_OK) 457 return ret; 458 459 if (exit_info_1 & IOIO_TYPE_IN) { 460 if (!ghcb_rax_is_valid(ghcb)) 461 return ES_VMM_ERROR; 462 regs->ax = lower_bits(ghcb->save.rax, bits); 463 } 464 } 465 466 return ret; 467 } 468 469 static enum es_result vc_handle_cpuid(struct ghcb *ghcb, 470 struct es_em_ctxt *ctxt) 471 { 472 struct pt_regs *regs = ctxt->regs; 473 u32 cr4 = native_read_cr4(); 474 enum es_result ret; 475 476 ghcb_set_rax(ghcb, regs->ax); 477 ghcb_set_rcx(ghcb, regs->cx); 478 479 if (cr4 & X86_CR4_OSXSAVE) 480 /* Safe to read xcr0 */ 481 ghcb_set_xcr0(ghcb, xgetbv(XCR_XFEATURE_ENABLED_MASK)); 482 else 483 /* xgetbv will cause #GP - use reset value for xcr0 */ 484 ghcb_set_xcr0(ghcb, 1); 485 486 ret = sev_es_ghcb_hv_call(ghcb, ctxt, SVM_EXIT_CPUID, 0, 0); 487 if (ret != ES_OK) 488 return ret; 489 490 if (!(ghcb_rax_is_valid(ghcb) && 491 ghcb_rbx_is_valid(ghcb) && 492 ghcb_rcx_is_valid(ghcb) && 493 ghcb_rdx_is_valid(ghcb))) 494 return ES_VMM_ERROR; 495 496 regs->ax = ghcb->save.rax; 497 regs->bx = ghcb->save.rbx; 498 regs->cx = ghcb->save.rcx; 499 regs->dx = ghcb->save.rdx; 500 501 return ES_OK; 502 } 503 504 static enum es_result vc_handle_rdtsc(struct ghcb *ghcb, 505 struct es_em_ctxt *ctxt, 506 unsigned long exit_code) 507 { 508 bool rdtscp = (exit_code == SVM_EXIT_RDTSCP); 509 enum es_result ret; 510 511 ret = sev_es_ghcb_hv_call(ghcb, ctxt, exit_code, 0, 0); 512 if (ret != ES_OK) 513 return ret; 514 515 if (!(ghcb_rax_is_valid(ghcb) && ghcb_rdx_is_valid(ghcb) && 516 (!rdtscp || ghcb_rcx_is_valid(ghcb)))) 517 return ES_VMM_ERROR; 518 519 ctxt->regs->ax = ghcb->save.rax; 520 ctxt->regs->dx = ghcb->save.rdx; 521 if (rdtscp) 522 ctxt->regs->cx = ghcb->save.rcx; 523 524 return ES_OK; 525 } 526