1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * AMD Encrypted Register State Support 4 * 5 * Author: Joerg Roedel <jroedel@suse.de> 6 * 7 * This file is not compiled stand-alone. It contains code shared 8 * between the pre-decompression boot code and the running Linux kernel 9 * and is included directly into both code-bases. 10 */ 11 12 #ifndef __BOOT_COMPRESSED 13 #define error(v) pr_err(v) 14 #define has_cpuflag(f) boot_cpu_has(f) 15 #endif 16 17 static bool __init sev_es_check_cpu_features(void) 18 { 19 if (!has_cpuflag(X86_FEATURE_RDRAND)) { 20 error("RDRAND instruction not supported - no trusted source of randomness available\n"); 21 return false; 22 } 23 24 return true; 25 } 26 27 static void __noreturn sev_es_terminate(unsigned int reason) 28 { 29 u64 val = GHCB_MSR_TERM_REQ; 30 31 /* 32 * Tell the hypervisor what went wrong - only reason-set 0 is 33 * currently supported. 34 */ 35 val |= GHCB_SEV_TERM_REASON(0, reason); 36 37 /* Request Guest Termination from Hypvervisor */ 38 sev_es_wr_ghcb_msr(val); 39 VMGEXIT(); 40 41 while (true) 42 asm volatile("hlt\n" : : : "memory"); 43 } 44 45 static bool sev_es_negotiate_protocol(void) 46 { 47 u64 val; 48 49 /* Do the GHCB protocol version negotiation */ 50 sev_es_wr_ghcb_msr(GHCB_MSR_SEV_INFO_REQ); 51 VMGEXIT(); 52 val = sev_es_rd_ghcb_msr(); 53 54 if (GHCB_MSR_INFO(val) != GHCB_MSR_SEV_INFO_RESP) 55 return false; 56 57 if (GHCB_MSR_PROTO_MAX(val) < GHCB_PROTO_OUR || 58 GHCB_MSR_PROTO_MIN(val) > GHCB_PROTO_OUR) 59 return false; 60 61 return true; 62 } 63 64 static __always_inline void vc_ghcb_invalidate(struct ghcb *ghcb) 65 { 66 ghcb->save.sw_exit_code = 0; 67 memset(ghcb->save.valid_bitmap, 0, sizeof(ghcb->save.valid_bitmap)); 68 } 69 70 static bool vc_decoding_needed(unsigned long exit_code) 71 { 72 /* Exceptions don't require to decode the instruction */ 73 return !(exit_code >= SVM_EXIT_EXCP_BASE && 74 exit_code <= SVM_EXIT_LAST_EXCP); 75 } 76 77 static enum es_result vc_init_em_ctxt(struct es_em_ctxt *ctxt, 78 struct pt_regs *regs, 79 unsigned long exit_code) 80 { 81 enum es_result ret = ES_OK; 82 83 memset(ctxt, 0, sizeof(*ctxt)); 84 ctxt->regs = regs; 85 86 if (vc_decoding_needed(exit_code)) 87 ret = vc_decode_insn(ctxt); 88 89 return ret; 90 } 91 92 static void vc_finish_insn(struct es_em_ctxt *ctxt) 93 { 94 ctxt->regs->ip += ctxt->insn.length; 95 } 96 97 static enum es_result sev_es_ghcb_hv_call(struct ghcb *ghcb, 98 struct es_em_ctxt *ctxt, 99 u64 exit_code, u64 exit_info_1, 100 u64 exit_info_2) 101 { 102 enum es_result ret; 103 104 /* Fill in protocol and format specifiers */ 105 ghcb->protocol_version = GHCB_PROTOCOL_MAX; 106 ghcb->ghcb_usage = GHCB_DEFAULT_USAGE; 107 108 ghcb_set_sw_exit_code(ghcb, exit_code); 109 ghcb_set_sw_exit_info_1(ghcb, exit_info_1); 110 ghcb_set_sw_exit_info_2(ghcb, exit_info_2); 111 112 sev_es_wr_ghcb_msr(__pa(ghcb)); 113 VMGEXIT(); 114 115 if ((ghcb->save.sw_exit_info_1 & 0xffffffff) == 1) { 116 u64 info = ghcb->save.sw_exit_info_2; 117 unsigned long v; 118 119 info = ghcb->save.sw_exit_info_2; 120 v = info & SVM_EVTINJ_VEC_MASK; 121 122 /* Check if exception information from hypervisor is sane. */ 123 if ((info & SVM_EVTINJ_VALID) && 124 ((v == X86_TRAP_GP) || (v == X86_TRAP_UD)) && 125 ((info & SVM_EVTINJ_TYPE_MASK) == SVM_EVTINJ_TYPE_EXEPT)) { 126 ctxt->fi.vector = v; 127 if (info & SVM_EVTINJ_VALID_ERR) 128 ctxt->fi.error_code = info >> 32; 129 ret = ES_EXCEPTION; 130 } else { 131 ret = ES_VMM_ERROR; 132 } 133 } else if (ghcb->save.sw_exit_info_1 & 0xffffffff) { 134 ret = ES_VMM_ERROR; 135 } else { 136 ret = ES_OK; 137 } 138 139 return ret; 140 } 141 142 /* 143 * Boot VC Handler - This is the first VC handler during boot, there is no GHCB 144 * page yet, so it only supports the MSR based communication with the 145 * hypervisor and only the CPUID exit-code. 146 */ 147 void __init do_vc_no_ghcb(struct pt_regs *regs, unsigned long exit_code) 148 { 149 unsigned int fn = lower_bits(regs->ax, 32); 150 unsigned long val; 151 152 /* Only CPUID is supported via MSR protocol */ 153 if (exit_code != SVM_EXIT_CPUID) 154 goto fail; 155 156 sev_es_wr_ghcb_msr(GHCB_CPUID_REQ(fn, GHCB_CPUID_REQ_EAX)); 157 VMGEXIT(); 158 val = sev_es_rd_ghcb_msr(); 159 if (GHCB_RESP_CODE(val) != GHCB_MSR_CPUID_RESP) 160 goto fail; 161 regs->ax = val >> 32; 162 163 sev_es_wr_ghcb_msr(GHCB_CPUID_REQ(fn, GHCB_CPUID_REQ_EBX)); 164 VMGEXIT(); 165 val = sev_es_rd_ghcb_msr(); 166 if (GHCB_RESP_CODE(val) != GHCB_MSR_CPUID_RESP) 167 goto fail; 168 regs->bx = val >> 32; 169 170 sev_es_wr_ghcb_msr(GHCB_CPUID_REQ(fn, GHCB_CPUID_REQ_ECX)); 171 VMGEXIT(); 172 val = sev_es_rd_ghcb_msr(); 173 if (GHCB_RESP_CODE(val) != GHCB_MSR_CPUID_RESP) 174 goto fail; 175 regs->cx = val >> 32; 176 177 sev_es_wr_ghcb_msr(GHCB_CPUID_REQ(fn, GHCB_CPUID_REQ_EDX)); 178 VMGEXIT(); 179 val = sev_es_rd_ghcb_msr(); 180 if (GHCB_RESP_CODE(val) != GHCB_MSR_CPUID_RESP) 181 goto fail; 182 regs->dx = val >> 32; 183 184 /* 185 * This is a VC handler and the #VC is only raised when SEV-ES is 186 * active, which means SEV must be active too. Do sanity checks on the 187 * CPUID results to make sure the hypervisor does not trick the kernel 188 * into the no-sev path. This could map sensitive data unencrypted and 189 * make it accessible to the hypervisor. 190 * 191 * In particular, check for: 192 * - Availability of CPUID leaf 0x8000001f 193 * - SEV CPUID bit. 194 * 195 * The hypervisor might still report the wrong C-bit position, but this 196 * can't be checked here. 197 */ 198 199 if (fn == 0x80000000 && (regs->ax < 0x8000001f)) 200 /* SEV leaf check */ 201 goto fail; 202 else if ((fn == 0x8000001f && !(regs->ax & BIT(1)))) 203 /* SEV bit */ 204 goto fail; 205 206 /* Skip over the CPUID two-byte opcode */ 207 regs->ip += 2; 208 209 return; 210 211 fail: 212 /* Terminate the guest */ 213 sev_es_terminate(GHCB_SEV_ES_REASON_GENERAL_REQUEST); 214 } 215 216 static enum es_result vc_insn_string_read(struct es_em_ctxt *ctxt, 217 void *src, char *buf, 218 unsigned int data_size, 219 unsigned int count, 220 bool backwards) 221 { 222 int i, b = backwards ? -1 : 1; 223 enum es_result ret = ES_OK; 224 225 for (i = 0; i < count; i++) { 226 void *s = src + (i * data_size * b); 227 char *d = buf + (i * data_size); 228 229 ret = vc_read_mem(ctxt, s, d, data_size); 230 if (ret != ES_OK) 231 break; 232 } 233 234 return ret; 235 } 236 237 static enum es_result vc_insn_string_write(struct es_em_ctxt *ctxt, 238 void *dst, char *buf, 239 unsigned int data_size, 240 unsigned int count, 241 bool backwards) 242 { 243 int i, s = backwards ? -1 : 1; 244 enum es_result ret = ES_OK; 245 246 for (i = 0; i < count; i++) { 247 void *d = dst + (i * data_size * s); 248 char *b = buf + (i * data_size); 249 250 ret = vc_write_mem(ctxt, d, b, data_size); 251 if (ret != ES_OK) 252 break; 253 } 254 255 return ret; 256 } 257 258 #define IOIO_TYPE_STR BIT(2) 259 #define IOIO_TYPE_IN 1 260 #define IOIO_TYPE_INS (IOIO_TYPE_IN | IOIO_TYPE_STR) 261 #define IOIO_TYPE_OUT 0 262 #define IOIO_TYPE_OUTS (IOIO_TYPE_OUT | IOIO_TYPE_STR) 263 264 #define IOIO_REP BIT(3) 265 266 #define IOIO_ADDR_64 BIT(9) 267 #define IOIO_ADDR_32 BIT(8) 268 #define IOIO_ADDR_16 BIT(7) 269 270 #define IOIO_DATA_32 BIT(6) 271 #define IOIO_DATA_16 BIT(5) 272 #define IOIO_DATA_8 BIT(4) 273 274 #define IOIO_SEG_ES (0 << 10) 275 #define IOIO_SEG_DS (3 << 10) 276 277 static enum es_result vc_ioio_exitinfo(struct es_em_ctxt *ctxt, u64 *exitinfo) 278 { 279 struct insn *insn = &ctxt->insn; 280 *exitinfo = 0; 281 282 switch (insn->opcode.bytes[0]) { 283 /* INS opcodes */ 284 case 0x6c: 285 case 0x6d: 286 *exitinfo |= IOIO_TYPE_INS; 287 *exitinfo |= IOIO_SEG_ES; 288 *exitinfo |= (ctxt->regs->dx & 0xffff) << 16; 289 break; 290 291 /* OUTS opcodes */ 292 case 0x6e: 293 case 0x6f: 294 *exitinfo |= IOIO_TYPE_OUTS; 295 *exitinfo |= IOIO_SEG_DS; 296 *exitinfo |= (ctxt->regs->dx & 0xffff) << 16; 297 break; 298 299 /* IN immediate opcodes */ 300 case 0xe4: 301 case 0xe5: 302 *exitinfo |= IOIO_TYPE_IN; 303 *exitinfo |= (u8)insn->immediate.value << 16; 304 break; 305 306 /* OUT immediate opcodes */ 307 case 0xe6: 308 case 0xe7: 309 *exitinfo |= IOIO_TYPE_OUT; 310 *exitinfo |= (u8)insn->immediate.value << 16; 311 break; 312 313 /* IN register opcodes */ 314 case 0xec: 315 case 0xed: 316 *exitinfo |= IOIO_TYPE_IN; 317 *exitinfo |= (ctxt->regs->dx & 0xffff) << 16; 318 break; 319 320 /* OUT register opcodes */ 321 case 0xee: 322 case 0xef: 323 *exitinfo |= IOIO_TYPE_OUT; 324 *exitinfo |= (ctxt->regs->dx & 0xffff) << 16; 325 break; 326 327 default: 328 return ES_DECODE_FAILED; 329 } 330 331 switch (insn->opcode.bytes[0]) { 332 case 0x6c: 333 case 0x6e: 334 case 0xe4: 335 case 0xe6: 336 case 0xec: 337 case 0xee: 338 /* Single byte opcodes */ 339 *exitinfo |= IOIO_DATA_8; 340 break; 341 default: 342 /* Length determined by instruction parsing */ 343 *exitinfo |= (insn->opnd_bytes == 2) ? IOIO_DATA_16 344 : IOIO_DATA_32; 345 } 346 switch (insn->addr_bytes) { 347 case 2: 348 *exitinfo |= IOIO_ADDR_16; 349 break; 350 case 4: 351 *exitinfo |= IOIO_ADDR_32; 352 break; 353 case 8: 354 *exitinfo |= IOIO_ADDR_64; 355 break; 356 } 357 358 if (insn_has_rep_prefix(insn)) 359 *exitinfo |= IOIO_REP; 360 361 return ES_OK; 362 } 363 364 static enum es_result vc_handle_ioio(struct ghcb *ghcb, struct es_em_ctxt *ctxt) 365 { 366 struct pt_regs *regs = ctxt->regs; 367 u64 exit_info_1, exit_info_2; 368 enum es_result ret; 369 370 ret = vc_ioio_exitinfo(ctxt, &exit_info_1); 371 if (ret != ES_OK) 372 return ret; 373 374 if (exit_info_1 & IOIO_TYPE_STR) { 375 376 /* (REP) INS/OUTS */ 377 378 bool df = ((regs->flags & X86_EFLAGS_DF) == X86_EFLAGS_DF); 379 unsigned int io_bytes, exit_bytes; 380 unsigned int ghcb_count, op_count; 381 unsigned long es_base; 382 u64 sw_scratch; 383 384 /* 385 * For the string variants with rep prefix the amount of in/out 386 * operations per #VC exception is limited so that the kernel 387 * has a chance to take interrupts and re-schedule while the 388 * instruction is emulated. 389 */ 390 io_bytes = (exit_info_1 >> 4) & 0x7; 391 ghcb_count = sizeof(ghcb->shared_buffer) / io_bytes; 392 393 op_count = (exit_info_1 & IOIO_REP) ? regs->cx : 1; 394 exit_info_2 = min(op_count, ghcb_count); 395 exit_bytes = exit_info_2 * io_bytes; 396 397 es_base = insn_get_seg_base(ctxt->regs, INAT_SEG_REG_ES); 398 399 /* Read bytes of OUTS into the shared buffer */ 400 if (!(exit_info_1 & IOIO_TYPE_IN)) { 401 ret = vc_insn_string_read(ctxt, 402 (void *)(es_base + regs->si), 403 ghcb->shared_buffer, io_bytes, 404 exit_info_2, df); 405 if (ret) 406 return ret; 407 } 408 409 /* 410 * Issue an VMGEXIT to the HV to consume the bytes from the 411 * shared buffer or to have it write them into the shared buffer 412 * depending on the instruction: OUTS or INS. 413 */ 414 sw_scratch = __pa(ghcb) + offsetof(struct ghcb, shared_buffer); 415 ghcb_set_sw_scratch(ghcb, sw_scratch); 416 ret = sev_es_ghcb_hv_call(ghcb, ctxt, SVM_EXIT_IOIO, 417 exit_info_1, exit_info_2); 418 if (ret != ES_OK) 419 return ret; 420 421 /* Read bytes from shared buffer into the guest's destination. */ 422 if (exit_info_1 & IOIO_TYPE_IN) { 423 ret = vc_insn_string_write(ctxt, 424 (void *)(es_base + regs->di), 425 ghcb->shared_buffer, io_bytes, 426 exit_info_2, df); 427 if (ret) 428 return ret; 429 430 if (df) 431 regs->di -= exit_bytes; 432 else 433 regs->di += exit_bytes; 434 } else { 435 if (df) 436 regs->si -= exit_bytes; 437 else 438 regs->si += exit_bytes; 439 } 440 441 if (exit_info_1 & IOIO_REP) 442 regs->cx -= exit_info_2; 443 444 ret = regs->cx ? ES_RETRY : ES_OK; 445 446 } else { 447 448 /* IN/OUT into/from rAX */ 449 450 int bits = (exit_info_1 & 0x70) >> 1; 451 u64 rax = 0; 452 453 if (!(exit_info_1 & IOIO_TYPE_IN)) 454 rax = lower_bits(regs->ax, bits); 455 456 ghcb_set_rax(ghcb, rax); 457 458 ret = sev_es_ghcb_hv_call(ghcb, ctxt, SVM_EXIT_IOIO, exit_info_1, 0); 459 if (ret != ES_OK) 460 return ret; 461 462 if (exit_info_1 & IOIO_TYPE_IN) { 463 if (!ghcb_rax_is_valid(ghcb)) 464 return ES_VMM_ERROR; 465 regs->ax = lower_bits(ghcb->save.rax, bits); 466 } 467 } 468 469 return ret; 470 } 471 472 static enum es_result vc_handle_cpuid(struct ghcb *ghcb, 473 struct es_em_ctxt *ctxt) 474 { 475 struct pt_regs *regs = ctxt->regs; 476 u32 cr4 = native_read_cr4(); 477 enum es_result ret; 478 479 ghcb_set_rax(ghcb, regs->ax); 480 ghcb_set_rcx(ghcb, regs->cx); 481 482 if (cr4 & X86_CR4_OSXSAVE) 483 /* Safe to read xcr0 */ 484 ghcb_set_xcr0(ghcb, xgetbv(XCR_XFEATURE_ENABLED_MASK)); 485 else 486 /* xgetbv will cause #GP - use reset value for xcr0 */ 487 ghcb_set_xcr0(ghcb, 1); 488 489 ret = sev_es_ghcb_hv_call(ghcb, ctxt, SVM_EXIT_CPUID, 0, 0); 490 if (ret != ES_OK) 491 return ret; 492 493 if (!(ghcb_rax_is_valid(ghcb) && 494 ghcb_rbx_is_valid(ghcb) && 495 ghcb_rcx_is_valid(ghcb) && 496 ghcb_rdx_is_valid(ghcb))) 497 return ES_VMM_ERROR; 498 499 regs->ax = ghcb->save.rax; 500 regs->bx = ghcb->save.rbx; 501 regs->cx = ghcb->save.rcx; 502 regs->dx = ghcb->save.rdx; 503 504 return ES_OK; 505 } 506 507 static enum es_result vc_handle_rdtsc(struct ghcb *ghcb, 508 struct es_em_ctxt *ctxt, 509 unsigned long exit_code) 510 { 511 bool rdtscp = (exit_code == SVM_EXIT_RDTSCP); 512 enum es_result ret; 513 514 ret = sev_es_ghcb_hv_call(ghcb, ctxt, exit_code, 0, 0); 515 if (ret != ES_OK) 516 return ret; 517 518 if (!(ghcb_rax_is_valid(ghcb) && ghcb_rdx_is_valid(ghcb) && 519 (!rdtscp || ghcb_rcx_is_valid(ghcb)))) 520 return ES_VMM_ERROR; 521 522 ctxt->regs->ax = ghcb->save.rax; 523 ctxt->regs->dx = ghcb->save.rdx; 524 if (rdtscp) 525 ctxt->regs->cx = ghcb->save.rcx; 526 527 return ES_OK; 528 } 529