1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * AMD Encrypted Register State Support 4 * 5 * Author: Joerg Roedel <jroedel@suse.de> 6 * 7 * This file is not compiled stand-alone. It contains code shared 8 * between the pre-decompression boot code and the running Linux kernel 9 * and is included directly into both code-bases. 10 */ 11 12 #ifndef __BOOT_COMPRESSED 13 #define error(v) pr_err(v) 14 #define has_cpuflag(f) boot_cpu_has(f) 15 #endif 16 17 /* 18 * Since feature negotiation related variables are set early in the boot 19 * process they must reside in the .data section so as not to be zeroed 20 * out when the .bss section is later cleared. 21 * 22 * GHCB protocol version negotiated with the hypervisor. 23 */ 24 static u16 ghcb_version __ro_after_init; 25 26 static bool __init sev_es_check_cpu_features(void) 27 { 28 if (!has_cpuflag(X86_FEATURE_RDRAND)) { 29 error("RDRAND instruction not supported - no trusted source of randomness available\n"); 30 return false; 31 } 32 33 return true; 34 } 35 36 static void __noreturn sev_es_terminate(unsigned int set, unsigned int reason) 37 { 38 u64 val = GHCB_MSR_TERM_REQ; 39 40 /* Tell the hypervisor what went wrong. */ 41 val |= GHCB_SEV_TERM_REASON(set, reason); 42 43 /* Request Guest Termination from Hypvervisor */ 44 sev_es_wr_ghcb_msr(val); 45 VMGEXIT(); 46 47 while (true) 48 asm volatile("hlt\n" : : : "memory"); 49 } 50 51 /* 52 * The hypervisor features are available from GHCB version 2 onward. 53 */ 54 static u64 get_hv_features(void) 55 { 56 u64 val; 57 58 if (ghcb_version < 2) 59 return 0; 60 61 sev_es_wr_ghcb_msr(GHCB_MSR_HV_FT_REQ); 62 VMGEXIT(); 63 64 val = sev_es_rd_ghcb_msr(); 65 if (GHCB_RESP_CODE(val) != GHCB_MSR_HV_FT_RESP) 66 return 0; 67 68 return GHCB_MSR_HV_FT_RESP_VAL(val); 69 } 70 71 static bool sev_es_negotiate_protocol(void) 72 { 73 u64 val; 74 75 /* Do the GHCB protocol version negotiation */ 76 sev_es_wr_ghcb_msr(GHCB_MSR_SEV_INFO_REQ); 77 VMGEXIT(); 78 val = sev_es_rd_ghcb_msr(); 79 80 if (GHCB_MSR_INFO(val) != GHCB_MSR_SEV_INFO_RESP) 81 return false; 82 83 if (GHCB_MSR_PROTO_MAX(val) < GHCB_PROTOCOL_MIN || 84 GHCB_MSR_PROTO_MIN(val) > GHCB_PROTOCOL_MAX) 85 return false; 86 87 ghcb_version = min_t(size_t, GHCB_MSR_PROTO_MAX(val), GHCB_PROTOCOL_MAX); 88 89 return true; 90 } 91 92 static __always_inline void vc_ghcb_invalidate(struct ghcb *ghcb) 93 { 94 ghcb->save.sw_exit_code = 0; 95 __builtin_memset(ghcb->save.valid_bitmap, 0, sizeof(ghcb->save.valid_bitmap)); 96 } 97 98 static bool vc_decoding_needed(unsigned long exit_code) 99 { 100 /* Exceptions don't require to decode the instruction */ 101 return !(exit_code >= SVM_EXIT_EXCP_BASE && 102 exit_code <= SVM_EXIT_LAST_EXCP); 103 } 104 105 static enum es_result vc_init_em_ctxt(struct es_em_ctxt *ctxt, 106 struct pt_regs *regs, 107 unsigned long exit_code) 108 { 109 enum es_result ret = ES_OK; 110 111 memset(ctxt, 0, sizeof(*ctxt)); 112 ctxt->regs = regs; 113 114 if (vc_decoding_needed(exit_code)) 115 ret = vc_decode_insn(ctxt); 116 117 return ret; 118 } 119 120 static void vc_finish_insn(struct es_em_ctxt *ctxt) 121 { 122 ctxt->regs->ip += ctxt->insn.length; 123 } 124 125 static enum es_result verify_exception_info(struct ghcb *ghcb, struct es_em_ctxt *ctxt) 126 { 127 u32 ret; 128 129 ret = ghcb->save.sw_exit_info_1 & GENMASK_ULL(31, 0); 130 if (!ret) 131 return ES_OK; 132 133 if (ret == 1) { 134 u64 info = ghcb->save.sw_exit_info_2; 135 unsigned long v; 136 137 info = ghcb->save.sw_exit_info_2; 138 v = info & SVM_EVTINJ_VEC_MASK; 139 140 /* Check if exception information from hypervisor is sane. */ 141 if ((info & SVM_EVTINJ_VALID) && 142 ((v == X86_TRAP_GP) || (v == X86_TRAP_UD)) && 143 ((info & SVM_EVTINJ_TYPE_MASK) == SVM_EVTINJ_TYPE_EXEPT)) { 144 ctxt->fi.vector = v; 145 146 if (info & SVM_EVTINJ_VALID_ERR) 147 ctxt->fi.error_code = info >> 32; 148 149 return ES_EXCEPTION; 150 } 151 } 152 153 return ES_VMM_ERROR; 154 } 155 156 enum es_result sev_es_ghcb_hv_call(struct ghcb *ghcb, bool set_ghcb_msr, 157 struct es_em_ctxt *ctxt, u64 exit_code, 158 u64 exit_info_1, u64 exit_info_2) 159 { 160 /* Fill in protocol and format specifiers */ 161 ghcb->protocol_version = ghcb_version; 162 ghcb->ghcb_usage = GHCB_DEFAULT_USAGE; 163 164 ghcb_set_sw_exit_code(ghcb, exit_code); 165 ghcb_set_sw_exit_info_1(ghcb, exit_info_1); 166 ghcb_set_sw_exit_info_2(ghcb, exit_info_2); 167 168 /* 169 * Hyper-V unenlightened guests use a paravisor for communicating and 170 * GHCB pages are being allocated and set up by that paravisor. Linux 171 * should not change the GHCB page's physical address. 172 */ 173 if (set_ghcb_msr) 174 sev_es_wr_ghcb_msr(__pa(ghcb)); 175 176 VMGEXIT(); 177 178 return verify_exception_info(ghcb, ctxt); 179 } 180 181 /* 182 * Boot VC Handler - This is the first VC handler during boot, there is no GHCB 183 * page yet, so it only supports the MSR based communication with the 184 * hypervisor and only the CPUID exit-code. 185 */ 186 void __init do_vc_no_ghcb(struct pt_regs *regs, unsigned long exit_code) 187 { 188 unsigned int fn = lower_bits(regs->ax, 32); 189 unsigned long val; 190 191 /* Only CPUID is supported via MSR protocol */ 192 if (exit_code != SVM_EXIT_CPUID) 193 goto fail; 194 195 sev_es_wr_ghcb_msr(GHCB_CPUID_REQ(fn, GHCB_CPUID_REQ_EAX)); 196 VMGEXIT(); 197 val = sev_es_rd_ghcb_msr(); 198 if (GHCB_RESP_CODE(val) != GHCB_MSR_CPUID_RESP) 199 goto fail; 200 regs->ax = val >> 32; 201 202 sev_es_wr_ghcb_msr(GHCB_CPUID_REQ(fn, GHCB_CPUID_REQ_EBX)); 203 VMGEXIT(); 204 val = sev_es_rd_ghcb_msr(); 205 if (GHCB_RESP_CODE(val) != GHCB_MSR_CPUID_RESP) 206 goto fail; 207 regs->bx = val >> 32; 208 209 sev_es_wr_ghcb_msr(GHCB_CPUID_REQ(fn, GHCB_CPUID_REQ_ECX)); 210 VMGEXIT(); 211 val = sev_es_rd_ghcb_msr(); 212 if (GHCB_RESP_CODE(val) != GHCB_MSR_CPUID_RESP) 213 goto fail; 214 regs->cx = val >> 32; 215 216 sev_es_wr_ghcb_msr(GHCB_CPUID_REQ(fn, GHCB_CPUID_REQ_EDX)); 217 VMGEXIT(); 218 val = sev_es_rd_ghcb_msr(); 219 if (GHCB_RESP_CODE(val) != GHCB_MSR_CPUID_RESP) 220 goto fail; 221 regs->dx = val >> 32; 222 223 /* 224 * This is a VC handler and the #VC is only raised when SEV-ES is 225 * active, which means SEV must be active too. Do sanity checks on the 226 * CPUID results to make sure the hypervisor does not trick the kernel 227 * into the no-sev path. This could map sensitive data unencrypted and 228 * make it accessible to the hypervisor. 229 * 230 * In particular, check for: 231 * - Availability of CPUID leaf 0x8000001f 232 * - SEV CPUID bit. 233 * 234 * The hypervisor might still report the wrong C-bit position, but this 235 * can't be checked here. 236 */ 237 238 if (fn == 0x80000000 && (regs->ax < 0x8000001f)) 239 /* SEV leaf check */ 240 goto fail; 241 else if ((fn == 0x8000001f && !(regs->ax & BIT(1)))) 242 /* SEV bit */ 243 goto fail; 244 245 /* Skip over the CPUID two-byte opcode */ 246 regs->ip += 2; 247 248 return; 249 250 fail: 251 /* Terminate the guest */ 252 sev_es_terminate(SEV_TERM_SET_GEN, GHCB_SEV_ES_GEN_REQ); 253 } 254 255 static enum es_result vc_insn_string_read(struct es_em_ctxt *ctxt, 256 void *src, char *buf, 257 unsigned int data_size, 258 unsigned int count, 259 bool backwards) 260 { 261 int i, b = backwards ? -1 : 1; 262 enum es_result ret = ES_OK; 263 264 for (i = 0; i < count; i++) { 265 void *s = src + (i * data_size * b); 266 char *d = buf + (i * data_size); 267 268 ret = vc_read_mem(ctxt, s, d, data_size); 269 if (ret != ES_OK) 270 break; 271 } 272 273 return ret; 274 } 275 276 static enum es_result vc_insn_string_write(struct es_em_ctxt *ctxt, 277 void *dst, char *buf, 278 unsigned int data_size, 279 unsigned int count, 280 bool backwards) 281 { 282 int i, s = backwards ? -1 : 1; 283 enum es_result ret = ES_OK; 284 285 for (i = 0; i < count; i++) { 286 void *d = dst + (i * data_size * s); 287 char *b = buf + (i * data_size); 288 289 ret = vc_write_mem(ctxt, d, b, data_size); 290 if (ret != ES_OK) 291 break; 292 } 293 294 return ret; 295 } 296 297 #define IOIO_TYPE_STR BIT(2) 298 #define IOIO_TYPE_IN 1 299 #define IOIO_TYPE_INS (IOIO_TYPE_IN | IOIO_TYPE_STR) 300 #define IOIO_TYPE_OUT 0 301 #define IOIO_TYPE_OUTS (IOIO_TYPE_OUT | IOIO_TYPE_STR) 302 303 #define IOIO_REP BIT(3) 304 305 #define IOIO_ADDR_64 BIT(9) 306 #define IOIO_ADDR_32 BIT(8) 307 #define IOIO_ADDR_16 BIT(7) 308 309 #define IOIO_DATA_32 BIT(6) 310 #define IOIO_DATA_16 BIT(5) 311 #define IOIO_DATA_8 BIT(4) 312 313 #define IOIO_SEG_ES (0 << 10) 314 #define IOIO_SEG_DS (3 << 10) 315 316 static enum es_result vc_ioio_exitinfo(struct es_em_ctxt *ctxt, u64 *exitinfo) 317 { 318 struct insn *insn = &ctxt->insn; 319 *exitinfo = 0; 320 321 switch (insn->opcode.bytes[0]) { 322 /* INS opcodes */ 323 case 0x6c: 324 case 0x6d: 325 *exitinfo |= IOIO_TYPE_INS; 326 *exitinfo |= IOIO_SEG_ES; 327 *exitinfo |= (ctxt->regs->dx & 0xffff) << 16; 328 break; 329 330 /* OUTS opcodes */ 331 case 0x6e: 332 case 0x6f: 333 *exitinfo |= IOIO_TYPE_OUTS; 334 *exitinfo |= IOIO_SEG_DS; 335 *exitinfo |= (ctxt->regs->dx & 0xffff) << 16; 336 break; 337 338 /* IN immediate opcodes */ 339 case 0xe4: 340 case 0xe5: 341 *exitinfo |= IOIO_TYPE_IN; 342 *exitinfo |= (u8)insn->immediate.value << 16; 343 break; 344 345 /* OUT immediate opcodes */ 346 case 0xe6: 347 case 0xe7: 348 *exitinfo |= IOIO_TYPE_OUT; 349 *exitinfo |= (u8)insn->immediate.value << 16; 350 break; 351 352 /* IN register opcodes */ 353 case 0xec: 354 case 0xed: 355 *exitinfo |= IOIO_TYPE_IN; 356 *exitinfo |= (ctxt->regs->dx & 0xffff) << 16; 357 break; 358 359 /* OUT register opcodes */ 360 case 0xee: 361 case 0xef: 362 *exitinfo |= IOIO_TYPE_OUT; 363 *exitinfo |= (ctxt->regs->dx & 0xffff) << 16; 364 break; 365 366 default: 367 return ES_DECODE_FAILED; 368 } 369 370 switch (insn->opcode.bytes[0]) { 371 case 0x6c: 372 case 0x6e: 373 case 0xe4: 374 case 0xe6: 375 case 0xec: 376 case 0xee: 377 /* Single byte opcodes */ 378 *exitinfo |= IOIO_DATA_8; 379 break; 380 default: 381 /* Length determined by instruction parsing */ 382 *exitinfo |= (insn->opnd_bytes == 2) ? IOIO_DATA_16 383 : IOIO_DATA_32; 384 } 385 switch (insn->addr_bytes) { 386 case 2: 387 *exitinfo |= IOIO_ADDR_16; 388 break; 389 case 4: 390 *exitinfo |= IOIO_ADDR_32; 391 break; 392 case 8: 393 *exitinfo |= IOIO_ADDR_64; 394 break; 395 } 396 397 if (insn_has_rep_prefix(insn)) 398 *exitinfo |= IOIO_REP; 399 400 return ES_OK; 401 } 402 403 static enum es_result vc_handle_ioio(struct ghcb *ghcb, struct es_em_ctxt *ctxt) 404 { 405 struct pt_regs *regs = ctxt->regs; 406 u64 exit_info_1, exit_info_2; 407 enum es_result ret; 408 409 ret = vc_ioio_exitinfo(ctxt, &exit_info_1); 410 if (ret != ES_OK) 411 return ret; 412 413 if (exit_info_1 & IOIO_TYPE_STR) { 414 415 /* (REP) INS/OUTS */ 416 417 bool df = ((regs->flags & X86_EFLAGS_DF) == X86_EFLAGS_DF); 418 unsigned int io_bytes, exit_bytes; 419 unsigned int ghcb_count, op_count; 420 unsigned long es_base; 421 u64 sw_scratch; 422 423 /* 424 * For the string variants with rep prefix the amount of in/out 425 * operations per #VC exception is limited so that the kernel 426 * has a chance to take interrupts and re-schedule while the 427 * instruction is emulated. 428 */ 429 io_bytes = (exit_info_1 >> 4) & 0x7; 430 ghcb_count = sizeof(ghcb->shared_buffer) / io_bytes; 431 432 op_count = (exit_info_1 & IOIO_REP) ? regs->cx : 1; 433 exit_info_2 = min(op_count, ghcb_count); 434 exit_bytes = exit_info_2 * io_bytes; 435 436 es_base = insn_get_seg_base(ctxt->regs, INAT_SEG_REG_ES); 437 438 /* Read bytes of OUTS into the shared buffer */ 439 if (!(exit_info_1 & IOIO_TYPE_IN)) { 440 ret = vc_insn_string_read(ctxt, 441 (void *)(es_base + regs->si), 442 ghcb->shared_buffer, io_bytes, 443 exit_info_2, df); 444 if (ret) 445 return ret; 446 } 447 448 /* 449 * Issue an VMGEXIT to the HV to consume the bytes from the 450 * shared buffer or to have it write them into the shared buffer 451 * depending on the instruction: OUTS or INS. 452 */ 453 sw_scratch = __pa(ghcb) + offsetof(struct ghcb, shared_buffer); 454 ghcb_set_sw_scratch(ghcb, sw_scratch); 455 ret = sev_es_ghcb_hv_call(ghcb, true, ctxt, SVM_EXIT_IOIO, 456 exit_info_1, exit_info_2); 457 if (ret != ES_OK) 458 return ret; 459 460 /* Read bytes from shared buffer into the guest's destination. */ 461 if (exit_info_1 & IOIO_TYPE_IN) { 462 ret = vc_insn_string_write(ctxt, 463 (void *)(es_base + regs->di), 464 ghcb->shared_buffer, io_bytes, 465 exit_info_2, df); 466 if (ret) 467 return ret; 468 469 if (df) 470 regs->di -= exit_bytes; 471 else 472 regs->di += exit_bytes; 473 } else { 474 if (df) 475 regs->si -= exit_bytes; 476 else 477 regs->si += exit_bytes; 478 } 479 480 if (exit_info_1 & IOIO_REP) 481 regs->cx -= exit_info_2; 482 483 ret = regs->cx ? ES_RETRY : ES_OK; 484 485 } else { 486 487 /* IN/OUT into/from rAX */ 488 489 int bits = (exit_info_1 & 0x70) >> 1; 490 u64 rax = 0; 491 492 if (!(exit_info_1 & IOIO_TYPE_IN)) 493 rax = lower_bits(regs->ax, bits); 494 495 ghcb_set_rax(ghcb, rax); 496 497 ret = sev_es_ghcb_hv_call(ghcb, true, ctxt, 498 SVM_EXIT_IOIO, exit_info_1, 0); 499 if (ret != ES_OK) 500 return ret; 501 502 if (exit_info_1 & IOIO_TYPE_IN) { 503 if (!ghcb_rax_is_valid(ghcb)) 504 return ES_VMM_ERROR; 505 regs->ax = lower_bits(ghcb->save.rax, bits); 506 } 507 } 508 509 return ret; 510 } 511 512 static enum es_result vc_handle_cpuid(struct ghcb *ghcb, 513 struct es_em_ctxt *ctxt) 514 { 515 struct pt_regs *regs = ctxt->regs; 516 u32 cr4 = native_read_cr4(); 517 enum es_result ret; 518 519 ghcb_set_rax(ghcb, regs->ax); 520 ghcb_set_rcx(ghcb, regs->cx); 521 522 if (cr4 & X86_CR4_OSXSAVE) 523 /* Safe to read xcr0 */ 524 ghcb_set_xcr0(ghcb, xgetbv(XCR_XFEATURE_ENABLED_MASK)); 525 else 526 /* xgetbv will cause #GP - use reset value for xcr0 */ 527 ghcb_set_xcr0(ghcb, 1); 528 529 ret = sev_es_ghcb_hv_call(ghcb, true, ctxt, SVM_EXIT_CPUID, 0, 0); 530 if (ret != ES_OK) 531 return ret; 532 533 if (!(ghcb_rax_is_valid(ghcb) && 534 ghcb_rbx_is_valid(ghcb) && 535 ghcb_rcx_is_valid(ghcb) && 536 ghcb_rdx_is_valid(ghcb))) 537 return ES_VMM_ERROR; 538 539 regs->ax = ghcb->save.rax; 540 regs->bx = ghcb->save.rbx; 541 regs->cx = ghcb->save.rcx; 542 regs->dx = ghcb->save.rdx; 543 544 return ES_OK; 545 } 546 547 static enum es_result vc_handle_rdtsc(struct ghcb *ghcb, 548 struct es_em_ctxt *ctxt, 549 unsigned long exit_code) 550 { 551 bool rdtscp = (exit_code == SVM_EXIT_RDTSCP); 552 enum es_result ret; 553 554 ret = sev_es_ghcb_hv_call(ghcb, true, ctxt, exit_code, 0, 0); 555 if (ret != ES_OK) 556 return ret; 557 558 if (!(ghcb_rax_is_valid(ghcb) && ghcb_rdx_is_valid(ghcb) && 559 (!rdtscp || ghcb_rcx_is_valid(ghcb)))) 560 return ES_VMM_ERROR; 561 562 ctxt->regs->ax = ghcb->save.rax; 563 ctxt->regs->dx = ghcb->save.rdx; 564 if (rdtscp) 565 ctxt->regs->cx = ghcb->save.rcx; 566 567 return ES_OK; 568 } 569