1 #define pr_fmt(fmt) "SVM: " fmt 2 3 #include <linux/kvm_host.h> 4 5 #include "irq.h" 6 #include "mmu.h" 7 #include "kvm_cache_regs.h" 8 #include "x86.h" 9 #include "cpuid.h" 10 #include "pmu.h" 11 12 #include <linux/module.h> 13 #include <linux/mod_devicetable.h> 14 #include <linux/kernel.h> 15 #include <linux/vmalloc.h> 16 #include <linux/highmem.h> 17 #include <linux/amd-iommu.h> 18 #include <linux/sched.h> 19 #include <linux/trace_events.h> 20 #include <linux/slab.h> 21 #include <linux/hashtable.h> 22 #include <linux/objtool.h> 23 #include <linux/psp-sev.h> 24 #include <linux/file.h> 25 #include <linux/pagemap.h> 26 #include <linux/swap.h> 27 #include <linux/rwsem.h> 28 29 #include <asm/apic.h> 30 #include <asm/perf_event.h> 31 #include <asm/tlbflush.h> 32 #include <asm/desc.h> 33 #include <asm/debugreg.h> 34 #include <asm/kvm_para.h> 35 #include <asm/irq_remapping.h> 36 #include <asm/spec-ctrl.h> 37 #include <asm/cpu_device_id.h> 38 #include <asm/traps.h> 39 40 #include <asm/virtext.h> 41 #include "trace.h" 42 43 #include "svm.h" 44 #include "svm_ops.h" 45 46 #define __ex(x) __kvm_handle_fault_on_reboot(x) 47 48 MODULE_AUTHOR("Qumranet"); 49 MODULE_LICENSE("GPL"); 50 51 #ifdef MODULE 52 static const struct x86_cpu_id svm_cpu_id[] = { 53 X86_MATCH_FEATURE(X86_FEATURE_SVM, NULL), 54 {} 55 }; 56 MODULE_DEVICE_TABLE(x86cpu, svm_cpu_id); 57 #endif 58 59 #define SEG_TYPE_LDT 2 60 #define SEG_TYPE_BUSY_TSS16 3 61 62 #define SVM_FEATURE_LBRV (1 << 1) 63 #define SVM_FEATURE_SVML (1 << 2) 64 #define SVM_FEATURE_TSC_RATE (1 << 4) 65 #define SVM_FEATURE_VMCB_CLEAN (1 << 5) 66 #define SVM_FEATURE_FLUSH_ASID (1 << 6) 67 #define SVM_FEATURE_DECODE_ASSIST (1 << 7) 68 #define SVM_FEATURE_PAUSE_FILTER (1 << 10) 69 70 #define DEBUGCTL_RESERVED_BITS (~(0x3fULL)) 71 72 #define TSC_RATIO_RSVD 0xffffff0000000000ULL 73 #define TSC_RATIO_MIN 0x0000000000000001ULL 74 #define TSC_RATIO_MAX 0x000000ffffffffffULL 75 76 static bool erratum_383_found __read_mostly; 77 78 u32 msrpm_offsets[MSRPM_OFFSETS] __read_mostly; 79 80 /* 81 * Set osvw_len to higher value when updated Revision Guides 82 * are published and we know what the new status bits are 83 */ 84 static uint64_t osvw_len = 4, osvw_status; 85 86 static DEFINE_PER_CPU(u64, current_tsc_ratio); 87 #define TSC_RATIO_DEFAULT 0x0100000000ULL 88 89 static const struct svm_direct_access_msrs { 90 u32 index; /* Index of the MSR */ 91 bool always; /* True if intercept is initially cleared */ 92 } direct_access_msrs[MAX_DIRECT_ACCESS_MSRS] = { 93 { .index = MSR_STAR, .always = true }, 94 { .index = MSR_IA32_SYSENTER_CS, .always = true }, 95 { .index = MSR_IA32_SYSENTER_EIP, .always = false }, 96 { .index = MSR_IA32_SYSENTER_ESP, .always = false }, 97 #ifdef CONFIG_X86_64 98 { .index = MSR_GS_BASE, .always = true }, 99 { .index = MSR_FS_BASE, .always = true }, 100 { .index = MSR_KERNEL_GS_BASE, .always = true }, 101 { .index = MSR_LSTAR, .always = true }, 102 { .index = MSR_CSTAR, .always = true }, 103 { .index = MSR_SYSCALL_MASK, .always = true }, 104 #endif 105 { .index = MSR_IA32_SPEC_CTRL, .always = false }, 106 { .index = MSR_IA32_PRED_CMD, .always = false }, 107 { .index = MSR_IA32_LASTBRANCHFROMIP, .always = false }, 108 { .index = MSR_IA32_LASTBRANCHTOIP, .always = false }, 109 { .index = MSR_IA32_LASTINTFROMIP, .always = false }, 110 { .index = MSR_IA32_LASTINTTOIP, .always = false }, 111 { .index = MSR_EFER, .always = false }, 112 { .index = MSR_IA32_CR_PAT, .always = false }, 113 { .index = MSR_AMD64_SEV_ES_GHCB, .always = true }, 114 { .index = MSR_INVALID, .always = false }, 115 }; 116 117 /* 118 * These 2 parameters are used to config the controls for Pause-Loop Exiting: 119 * pause_filter_count: On processors that support Pause filtering(indicated 120 * by CPUID Fn8000_000A_EDX), the VMCB provides a 16 bit pause filter 121 * count value. On VMRUN this value is loaded into an internal counter. 122 * Each time a pause instruction is executed, this counter is decremented 123 * until it reaches zero at which time a #VMEXIT is generated if pause 124 * intercept is enabled. Refer to AMD APM Vol 2 Section 15.14.4 Pause 125 * Intercept Filtering for more details. 126 * This also indicate if ple logic enabled. 127 * 128 * pause_filter_thresh: In addition, some processor families support advanced 129 * pause filtering (indicated by CPUID Fn8000_000A_EDX) upper bound on 130 * the amount of time a guest is allowed to execute in a pause loop. 131 * In this mode, a 16-bit pause filter threshold field is added in the 132 * VMCB. The threshold value is a cycle count that is used to reset the 133 * pause counter. As with simple pause filtering, VMRUN loads the pause 134 * count value from VMCB into an internal counter. Then, on each pause 135 * instruction the hardware checks the elapsed number of cycles since 136 * the most recent pause instruction against the pause filter threshold. 137 * If the elapsed cycle count is greater than the pause filter threshold, 138 * then the internal pause count is reloaded from the VMCB and execution 139 * continues. If the elapsed cycle count is less than the pause filter 140 * threshold, then the internal pause count is decremented. If the count 141 * value is less than zero and PAUSE intercept is enabled, a #VMEXIT is 142 * triggered. If advanced pause filtering is supported and pause filter 143 * threshold field is set to zero, the filter will operate in the simpler, 144 * count only mode. 145 */ 146 147 static unsigned short pause_filter_thresh = KVM_DEFAULT_PLE_GAP; 148 module_param(pause_filter_thresh, ushort, 0444); 149 150 static unsigned short pause_filter_count = KVM_SVM_DEFAULT_PLE_WINDOW; 151 module_param(pause_filter_count, ushort, 0444); 152 153 /* Default doubles per-vcpu window every exit. */ 154 static unsigned short pause_filter_count_grow = KVM_DEFAULT_PLE_WINDOW_GROW; 155 module_param(pause_filter_count_grow, ushort, 0444); 156 157 /* Default resets per-vcpu window every exit to pause_filter_count. */ 158 static unsigned short pause_filter_count_shrink = KVM_DEFAULT_PLE_WINDOW_SHRINK; 159 module_param(pause_filter_count_shrink, ushort, 0444); 160 161 /* Default is to compute the maximum so we can never overflow. */ 162 static unsigned short pause_filter_count_max = KVM_SVM_DEFAULT_PLE_WINDOW_MAX; 163 module_param(pause_filter_count_max, ushort, 0444); 164 165 /* 166 * Use nested page tables by default. Note, NPT may get forced off by 167 * svm_hardware_setup() if it's unsupported by hardware or the host kernel. 168 */ 169 bool npt_enabled = true; 170 module_param_named(npt, npt_enabled, bool, 0444); 171 172 /* allow nested virtualization in KVM/SVM */ 173 static int nested = true; 174 module_param(nested, int, S_IRUGO); 175 176 /* enable/disable Next RIP Save */ 177 static int nrips = true; 178 module_param(nrips, int, 0444); 179 180 /* enable/disable Virtual VMLOAD VMSAVE */ 181 static int vls = true; 182 module_param(vls, int, 0444); 183 184 /* enable/disable Virtual GIF */ 185 static int vgif = true; 186 module_param(vgif, int, 0444); 187 188 bool __read_mostly dump_invalid_vmcb; 189 module_param(dump_invalid_vmcb, bool, 0644); 190 191 static bool svm_gp_erratum_intercept = true; 192 193 static u8 rsm_ins_bytes[] = "\x0f\xaa"; 194 195 static unsigned long iopm_base; 196 197 struct kvm_ldttss_desc { 198 u16 limit0; 199 u16 base0; 200 unsigned base1:8, type:5, dpl:2, p:1; 201 unsigned limit1:4, zero0:3, g:1, base2:8; 202 u32 base3; 203 u32 zero1; 204 } __attribute__((packed)); 205 206 DEFINE_PER_CPU(struct svm_cpu_data *, svm_data); 207 208 /* 209 * Only MSR_TSC_AUX is switched via the user return hook. EFER is switched via 210 * the VMCB, and the SYSCALL/SYSENTER MSRs are handled by VMLOAD/VMSAVE. 211 * 212 * RDTSCP and RDPID are not used in the kernel, specifically to allow KVM to 213 * defer the restoration of TSC_AUX until the CPU returns to userspace. 214 */ 215 #define TSC_AUX_URET_SLOT 0 216 217 static const u32 msrpm_ranges[] = {0, 0xc0000000, 0xc0010000}; 218 219 #define NUM_MSR_MAPS ARRAY_SIZE(msrpm_ranges) 220 #define MSRS_RANGE_SIZE 2048 221 #define MSRS_IN_RANGE (MSRS_RANGE_SIZE * 8 / 2) 222 223 u32 svm_msrpm_offset(u32 msr) 224 { 225 u32 offset; 226 int i; 227 228 for (i = 0; i < NUM_MSR_MAPS; i++) { 229 if (msr < msrpm_ranges[i] || 230 msr >= msrpm_ranges[i] + MSRS_IN_RANGE) 231 continue; 232 233 offset = (msr - msrpm_ranges[i]) / 4; /* 4 msrs per u8 */ 234 offset += (i * MSRS_RANGE_SIZE); /* add range offset */ 235 236 /* Now we have the u8 offset - but need the u32 offset */ 237 return offset / 4; 238 } 239 240 /* MSR not in any range */ 241 return MSR_INVALID; 242 } 243 244 #define MAX_INST_SIZE 15 245 246 static int get_max_npt_level(void) 247 { 248 #ifdef CONFIG_X86_64 249 return PT64_ROOT_4LEVEL; 250 #else 251 return PT32E_ROOT_LEVEL; 252 #endif 253 } 254 255 int svm_set_efer(struct kvm_vcpu *vcpu, u64 efer) 256 { 257 struct vcpu_svm *svm = to_svm(vcpu); 258 u64 old_efer = vcpu->arch.efer; 259 vcpu->arch.efer = efer; 260 261 if (!npt_enabled) { 262 /* Shadow paging assumes NX to be available. */ 263 efer |= EFER_NX; 264 265 if (!(efer & EFER_LMA)) 266 efer &= ~EFER_LME; 267 } 268 269 if ((old_efer & EFER_SVME) != (efer & EFER_SVME)) { 270 if (!(efer & EFER_SVME)) { 271 svm_leave_nested(svm); 272 svm_set_gif(svm, true); 273 /* #GP intercept is still needed for vmware backdoor */ 274 if (!enable_vmware_backdoor) 275 clr_exception_intercept(svm, GP_VECTOR); 276 277 /* 278 * Free the nested guest state, unless we are in SMM. 279 * In this case we will return to the nested guest 280 * as soon as we leave SMM. 281 */ 282 if (!is_smm(vcpu)) 283 svm_free_nested(svm); 284 285 } else { 286 int ret = svm_allocate_nested(svm); 287 288 if (ret) { 289 vcpu->arch.efer = old_efer; 290 return ret; 291 } 292 293 if (svm_gp_erratum_intercept) 294 set_exception_intercept(svm, GP_VECTOR); 295 } 296 } 297 298 svm->vmcb->save.efer = efer | EFER_SVME; 299 vmcb_mark_dirty(svm->vmcb, VMCB_CR); 300 return 0; 301 } 302 303 static int is_external_interrupt(u32 info) 304 { 305 info &= SVM_EVTINJ_TYPE_MASK | SVM_EVTINJ_VALID; 306 return info == (SVM_EVTINJ_VALID | SVM_EVTINJ_TYPE_INTR); 307 } 308 309 static u32 svm_get_interrupt_shadow(struct kvm_vcpu *vcpu) 310 { 311 struct vcpu_svm *svm = to_svm(vcpu); 312 u32 ret = 0; 313 314 if (svm->vmcb->control.int_state & SVM_INTERRUPT_SHADOW_MASK) 315 ret = KVM_X86_SHADOW_INT_STI | KVM_X86_SHADOW_INT_MOV_SS; 316 return ret; 317 } 318 319 static void svm_set_interrupt_shadow(struct kvm_vcpu *vcpu, int mask) 320 { 321 struct vcpu_svm *svm = to_svm(vcpu); 322 323 if (mask == 0) 324 svm->vmcb->control.int_state &= ~SVM_INTERRUPT_SHADOW_MASK; 325 else 326 svm->vmcb->control.int_state |= SVM_INTERRUPT_SHADOW_MASK; 327 328 } 329 330 static int skip_emulated_instruction(struct kvm_vcpu *vcpu) 331 { 332 struct vcpu_svm *svm = to_svm(vcpu); 333 334 /* 335 * SEV-ES does not expose the next RIP. The RIP update is controlled by 336 * the type of exit and the #VC handler in the guest. 337 */ 338 if (sev_es_guest(vcpu->kvm)) 339 goto done; 340 341 if (nrips && svm->vmcb->control.next_rip != 0) { 342 WARN_ON_ONCE(!static_cpu_has(X86_FEATURE_NRIPS)); 343 svm->next_rip = svm->vmcb->control.next_rip; 344 } 345 346 if (!svm->next_rip) { 347 if (!kvm_emulate_instruction(vcpu, EMULTYPE_SKIP)) 348 return 0; 349 } else { 350 kvm_rip_write(vcpu, svm->next_rip); 351 } 352 353 done: 354 svm_set_interrupt_shadow(vcpu, 0); 355 356 return 1; 357 } 358 359 static void svm_queue_exception(struct kvm_vcpu *vcpu) 360 { 361 struct vcpu_svm *svm = to_svm(vcpu); 362 unsigned nr = vcpu->arch.exception.nr; 363 bool has_error_code = vcpu->arch.exception.has_error_code; 364 u32 error_code = vcpu->arch.exception.error_code; 365 366 kvm_deliver_exception_payload(vcpu); 367 368 if (nr == BP_VECTOR && !nrips) { 369 unsigned long rip, old_rip = kvm_rip_read(vcpu); 370 371 /* 372 * For guest debugging where we have to reinject #BP if some 373 * INT3 is guest-owned: 374 * Emulate nRIP by moving RIP forward. Will fail if injection 375 * raises a fault that is not intercepted. Still better than 376 * failing in all cases. 377 */ 378 (void)skip_emulated_instruction(vcpu); 379 rip = kvm_rip_read(vcpu); 380 svm->int3_rip = rip + svm->vmcb->save.cs.base; 381 svm->int3_injected = rip - old_rip; 382 } 383 384 svm->vmcb->control.event_inj = nr 385 | SVM_EVTINJ_VALID 386 | (has_error_code ? SVM_EVTINJ_VALID_ERR : 0) 387 | SVM_EVTINJ_TYPE_EXEPT; 388 svm->vmcb->control.event_inj_err = error_code; 389 } 390 391 static void svm_init_erratum_383(void) 392 { 393 u32 low, high; 394 int err; 395 u64 val; 396 397 if (!static_cpu_has_bug(X86_BUG_AMD_TLB_MMATCH)) 398 return; 399 400 /* Use _safe variants to not break nested virtualization */ 401 val = native_read_msr_safe(MSR_AMD64_DC_CFG, &err); 402 if (err) 403 return; 404 405 val |= (1ULL << 47); 406 407 low = lower_32_bits(val); 408 high = upper_32_bits(val); 409 410 native_write_msr_safe(MSR_AMD64_DC_CFG, low, high); 411 412 erratum_383_found = true; 413 } 414 415 static void svm_init_osvw(struct kvm_vcpu *vcpu) 416 { 417 /* 418 * Guests should see errata 400 and 415 as fixed (assuming that 419 * HLT and IO instructions are intercepted). 420 */ 421 vcpu->arch.osvw.length = (osvw_len >= 3) ? (osvw_len) : 3; 422 vcpu->arch.osvw.status = osvw_status & ~(6ULL); 423 424 /* 425 * By increasing VCPU's osvw.length to 3 we are telling the guest that 426 * all osvw.status bits inside that length, including bit 0 (which is 427 * reserved for erratum 298), are valid. However, if host processor's 428 * osvw_len is 0 then osvw_status[0] carries no information. We need to 429 * be conservative here and therefore we tell the guest that erratum 298 430 * is present (because we really don't know). 431 */ 432 if (osvw_len == 0 && boot_cpu_data.x86 == 0x10) 433 vcpu->arch.osvw.status |= 1; 434 } 435 436 static int has_svm(void) 437 { 438 const char *msg; 439 440 if (!cpu_has_svm(&msg)) { 441 printk(KERN_INFO "has_svm: %s\n", msg); 442 return 0; 443 } 444 445 if (sev_active()) { 446 pr_info("KVM is unsupported when running as an SEV guest\n"); 447 return 0; 448 } 449 450 return 1; 451 } 452 453 static void svm_hardware_disable(void) 454 { 455 /* Make sure we clean up behind us */ 456 if (static_cpu_has(X86_FEATURE_TSCRATEMSR)) 457 wrmsrl(MSR_AMD64_TSC_RATIO, TSC_RATIO_DEFAULT); 458 459 cpu_svm_disable(); 460 461 amd_pmu_disable_virt(); 462 } 463 464 static int svm_hardware_enable(void) 465 { 466 467 struct svm_cpu_data *sd; 468 uint64_t efer; 469 struct desc_struct *gdt; 470 int me = raw_smp_processor_id(); 471 472 rdmsrl(MSR_EFER, efer); 473 if (efer & EFER_SVME) 474 return -EBUSY; 475 476 if (!has_svm()) { 477 pr_err("%s: err EOPNOTSUPP on %d\n", __func__, me); 478 return -EINVAL; 479 } 480 sd = per_cpu(svm_data, me); 481 if (!sd) { 482 pr_err("%s: svm_data is NULL on %d\n", __func__, me); 483 return -EINVAL; 484 } 485 486 sd->asid_generation = 1; 487 sd->max_asid = cpuid_ebx(SVM_CPUID_FUNC) - 1; 488 sd->next_asid = sd->max_asid + 1; 489 sd->min_asid = max_sev_asid + 1; 490 491 gdt = get_current_gdt_rw(); 492 sd->tss_desc = (struct kvm_ldttss_desc *)(gdt + GDT_ENTRY_TSS); 493 494 wrmsrl(MSR_EFER, efer | EFER_SVME); 495 496 wrmsrl(MSR_VM_HSAVE_PA, __sme_page_pa(sd->save_area)); 497 498 if (static_cpu_has(X86_FEATURE_TSCRATEMSR)) { 499 wrmsrl(MSR_AMD64_TSC_RATIO, TSC_RATIO_DEFAULT); 500 __this_cpu_write(current_tsc_ratio, TSC_RATIO_DEFAULT); 501 } 502 503 504 /* 505 * Get OSVW bits. 506 * 507 * Note that it is possible to have a system with mixed processor 508 * revisions and therefore different OSVW bits. If bits are not the same 509 * on different processors then choose the worst case (i.e. if erratum 510 * is present on one processor and not on another then assume that the 511 * erratum is present everywhere). 512 */ 513 if (cpu_has(&boot_cpu_data, X86_FEATURE_OSVW)) { 514 uint64_t len, status = 0; 515 int err; 516 517 len = native_read_msr_safe(MSR_AMD64_OSVW_ID_LENGTH, &err); 518 if (!err) 519 status = native_read_msr_safe(MSR_AMD64_OSVW_STATUS, 520 &err); 521 522 if (err) 523 osvw_status = osvw_len = 0; 524 else { 525 if (len < osvw_len) 526 osvw_len = len; 527 osvw_status |= status; 528 osvw_status &= (1ULL << osvw_len) - 1; 529 } 530 } else 531 osvw_status = osvw_len = 0; 532 533 svm_init_erratum_383(); 534 535 amd_pmu_enable_virt(); 536 537 return 0; 538 } 539 540 static void svm_cpu_uninit(int cpu) 541 { 542 struct svm_cpu_data *sd = per_cpu(svm_data, cpu); 543 544 if (!sd) 545 return; 546 547 per_cpu(svm_data, cpu) = NULL; 548 kfree(sd->sev_vmcbs); 549 __free_page(sd->save_area); 550 kfree(sd); 551 } 552 553 static int svm_cpu_init(int cpu) 554 { 555 struct svm_cpu_data *sd; 556 int ret = -ENOMEM; 557 558 sd = kzalloc(sizeof(struct svm_cpu_data), GFP_KERNEL); 559 if (!sd) 560 return ret; 561 sd->cpu = cpu; 562 sd->save_area = alloc_page(GFP_KERNEL); 563 if (!sd->save_area) 564 goto free_cpu_data; 565 566 clear_page(page_address(sd->save_area)); 567 568 ret = sev_cpu_init(sd); 569 if (ret) 570 goto free_save_area; 571 572 per_cpu(svm_data, cpu) = sd; 573 574 return 0; 575 576 free_save_area: 577 __free_page(sd->save_area); 578 free_cpu_data: 579 kfree(sd); 580 return ret; 581 582 } 583 584 static int direct_access_msr_slot(u32 msr) 585 { 586 u32 i; 587 588 for (i = 0; direct_access_msrs[i].index != MSR_INVALID; i++) 589 if (direct_access_msrs[i].index == msr) 590 return i; 591 592 return -ENOENT; 593 } 594 595 static void set_shadow_msr_intercept(struct kvm_vcpu *vcpu, u32 msr, int read, 596 int write) 597 { 598 struct vcpu_svm *svm = to_svm(vcpu); 599 int slot = direct_access_msr_slot(msr); 600 601 if (slot == -ENOENT) 602 return; 603 604 /* Set the shadow bitmaps to the desired intercept states */ 605 if (read) 606 set_bit(slot, svm->shadow_msr_intercept.read); 607 else 608 clear_bit(slot, svm->shadow_msr_intercept.read); 609 610 if (write) 611 set_bit(slot, svm->shadow_msr_intercept.write); 612 else 613 clear_bit(slot, svm->shadow_msr_intercept.write); 614 } 615 616 static bool valid_msr_intercept(u32 index) 617 { 618 return direct_access_msr_slot(index) != -ENOENT; 619 } 620 621 static bool msr_write_intercepted(struct kvm_vcpu *vcpu, u32 msr) 622 { 623 u8 bit_write; 624 unsigned long tmp; 625 u32 offset; 626 u32 *msrpm; 627 628 msrpm = is_guest_mode(vcpu) ? to_svm(vcpu)->nested.msrpm: 629 to_svm(vcpu)->msrpm; 630 631 offset = svm_msrpm_offset(msr); 632 bit_write = 2 * (msr & 0x0f) + 1; 633 tmp = msrpm[offset]; 634 635 BUG_ON(offset == MSR_INVALID); 636 637 return !!test_bit(bit_write, &tmp); 638 } 639 640 static void set_msr_interception_bitmap(struct kvm_vcpu *vcpu, u32 *msrpm, 641 u32 msr, int read, int write) 642 { 643 u8 bit_read, bit_write; 644 unsigned long tmp; 645 u32 offset; 646 647 /* 648 * If this warning triggers extend the direct_access_msrs list at the 649 * beginning of the file 650 */ 651 WARN_ON(!valid_msr_intercept(msr)); 652 653 /* Enforce non allowed MSRs to trap */ 654 if (read && !kvm_msr_allowed(vcpu, msr, KVM_MSR_FILTER_READ)) 655 read = 0; 656 657 if (write && !kvm_msr_allowed(vcpu, msr, KVM_MSR_FILTER_WRITE)) 658 write = 0; 659 660 offset = svm_msrpm_offset(msr); 661 bit_read = 2 * (msr & 0x0f); 662 bit_write = 2 * (msr & 0x0f) + 1; 663 tmp = msrpm[offset]; 664 665 BUG_ON(offset == MSR_INVALID); 666 667 read ? clear_bit(bit_read, &tmp) : set_bit(bit_read, &tmp); 668 write ? clear_bit(bit_write, &tmp) : set_bit(bit_write, &tmp); 669 670 msrpm[offset] = tmp; 671 } 672 673 void set_msr_interception(struct kvm_vcpu *vcpu, u32 *msrpm, u32 msr, 674 int read, int write) 675 { 676 set_shadow_msr_intercept(vcpu, msr, read, write); 677 set_msr_interception_bitmap(vcpu, msrpm, msr, read, write); 678 } 679 680 u32 *svm_vcpu_alloc_msrpm(void) 681 { 682 unsigned int order = get_order(MSRPM_SIZE); 683 struct page *pages = alloc_pages(GFP_KERNEL_ACCOUNT, order); 684 u32 *msrpm; 685 686 if (!pages) 687 return NULL; 688 689 msrpm = page_address(pages); 690 memset(msrpm, 0xff, PAGE_SIZE * (1 << order)); 691 692 return msrpm; 693 } 694 695 void svm_vcpu_init_msrpm(struct kvm_vcpu *vcpu, u32 *msrpm) 696 { 697 int i; 698 699 for (i = 0; direct_access_msrs[i].index != MSR_INVALID; i++) { 700 if (!direct_access_msrs[i].always) 701 continue; 702 set_msr_interception(vcpu, msrpm, direct_access_msrs[i].index, 1, 1); 703 } 704 } 705 706 707 void svm_vcpu_free_msrpm(u32 *msrpm) 708 { 709 __free_pages(virt_to_page(msrpm), get_order(MSRPM_SIZE)); 710 } 711 712 static void svm_msr_filter_changed(struct kvm_vcpu *vcpu) 713 { 714 struct vcpu_svm *svm = to_svm(vcpu); 715 u32 i; 716 717 /* 718 * Set intercept permissions for all direct access MSRs again. They 719 * will automatically get filtered through the MSR filter, so we are 720 * back in sync after this. 721 */ 722 for (i = 0; direct_access_msrs[i].index != MSR_INVALID; i++) { 723 u32 msr = direct_access_msrs[i].index; 724 u32 read = test_bit(i, svm->shadow_msr_intercept.read); 725 u32 write = test_bit(i, svm->shadow_msr_intercept.write); 726 727 set_msr_interception_bitmap(vcpu, svm->msrpm, msr, read, write); 728 } 729 } 730 731 static void add_msr_offset(u32 offset) 732 { 733 int i; 734 735 for (i = 0; i < MSRPM_OFFSETS; ++i) { 736 737 /* Offset already in list? */ 738 if (msrpm_offsets[i] == offset) 739 return; 740 741 /* Slot used by another offset? */ 742 if (msrpm_offsets[i] != MSR_INVALID) 743 continue; 744 745 /* Add offset to list */ 746 msrpm_offsets[i] = offset; 747 748 return; 749 } 750 751 /* 752 * If this BUG triggers the msrpm_offsets table has an overflow. Just 753 * increase MSRPM_OFFSETS in this case. 754 */ 755 BUG(); 756 } 757 758 static void init_msrpm_offsets(void) 759 { 760 int i; 761 762 memset(msrpm_offsets, 0xff, sizeof(msrpm_offsets)); 763 764 for (i = 0; direct_access_msrs[i].index != MSR_INVALID; i++) { 765 u32 offset; 766 767 offset = svm_msrpm_offset(direct_access_msrs[i].index); 768 BUG_ON(offset == MSR_INVALID); 769 770 add_msr_offset(offset); 771 } 772 } 773 774 static void svm_enable_lbrv(struct kvm_vcpu *vcpu) 775 { 776 struct vcpu_svm *svm = to_svm(vcpu); 777 778 svm->vmcb->control.virt_ext |= LBR_CTL_ENABLE_MASK; 779 set_msr_interception(vcpu, svm->msrpm, MSR_IA32_LASTBRANCHFROMIP, 1, 1); 780 set_msr_interception(vcpu, svm->msrpm, MSR_IA32_LASTBRANCHTOIP, 1, 1); 781 set_msr_interception(vcpu, svm->msrpm, MSR_IA32_LASTINTFROMIP, 1, 1); 782 set_msr_interception(vcpu, svm->msrpm, MSR_IA32_LASTINTTOIP, 1, 1); 783 } 784 785 static void svm_disable_lbrv(struct kvm_vcpu *vcpu) 786 { 787 struct vcpu_svm *svm = to_svm(vcpu); 788 789 svm->vmcb->control.virt_ext &= ~LBR_CTL_ENABLE_MASK; 790 set_msr_interception(vcpu, svm->msrpm, MSR_IA32_LASTBRANCHFROMIP, 0, 0); 791 set_msr_interception(vcpu, svm->msrpm, MSR_IA32_LASTBRANCHTOIP, 0, 0); 792 set_msr_interception(vcpu, svm->msrpm, MSR_IA32_LASTINTFROMIP, 0, 0); 793 set_msr_interception(vcpu, svm->msrpm, MSR_IA32_LASTINTTOIP, 0, 0); 794 } 795 796 void disable_nmi_singlestep(struct vcpu_svm *svm) 797 { 798 svm->nmi_singlestep = false; 799 800 if (!(svm->vcpu.guest_debug & KVM_GUESTDBG_SINGLESTEP)) { 801 /* Clear our flags if they were not set by the guest */ 802 if (!(svm->nmi_singlestep_guest_rflags & X86_EFLAGS_TF)) 803 svm->vmcb->save.rflags &= ~X86_EFLAGS_TF; 804 if (!(svm->nmi_singlestep_guest_rflags & X86_EFLAGS_RF)) 805 svm->vmcb->save.rflags &= ~X86_EFLAGS_RF; 806 } 807 } 808 809 static void grow_ple_window(struct kvm_vcpu *vcpu) 810 { 811 struct vcpu_svm *svm = to_svm(vcpu); 812 struct vmcb_control_area *control = &svm->vmcb->control; 813 int old = control->pause_filter_count; 814 815 control->pause_filter_count = __grow_ple_window(old, 816 pause_filter_count, 817 pause_filter_count_grow, 818 pause_filter_count_max); 819 820 if (control->pause_filter_count != old) { 821 vmcb_mark_dirty(svm->vmcb, VMCB_INTERCEPTS); 822 trace_kvm_ple_window_update(vcpu->vcpu_id, 823 control->pause_filter_count, old); 824 } 825 } 826 827 static void shrink_ple_window(struct kvm_vcpu *vcpu) 828 { 829 struct vcpu_svm *svm = to_svm(vcpu); 830 struct vmcb_control_area *control = &svm->vmcb->control; 831 int old = control->pause_filter_count; 832 833 control->pause_filter_count = 834 __shrink_ple_window(old, 835 pause_filter_count, 836 pause_filter_count_shrink, 837 pause_filter_count); 838 if (control->pause_filter_count != old) { 839 vmcb_mark_dirty(svm->vmcb, VMCB_INTERCEPTS); 840 trace_kvm_ple_window_update(vcpu->vcpu_id, 841 control->pause_filter_count, old); 842 } 843 } 844 845 /* 846 * The default MMIO mask is a single bit (excluding the present bit), 847 * which could conflict with the memory encryption bit. Check for 848 * memory encryption support and override the default MMIO mask if 849 * memory encryption is enabled. 850 */ 851 static __init void svm_adjust_mmio_mask(void) 852 { 853 unsigned int enc_bit, mask_bit; 854 u64 msr, mask; 855 856 /* If there is no memory encryption support, use existing mask */ 857 if (cpuid_eax(0x80000000) < 0x8000001f) 858 return; 859 860 /* If memory encryption is not enabled, use existing mask */ 861 rdmsrl(MSR_K8_SYSCFG, msr); 862 if (!(msr & MSR_K8_SYSCFG_MEM_ENCRYPT)) 863 return; 864 865 enc_bit = cpuid_ebx(0x8000001f) & 0x3f; 866 mask_bit = boot_cpu_data.x86_phys_bits; 867 868 /* Increment the mask bit if it is the same as the encryption bit */ 869 if (enc_bit == mask_bit) 870 mask_bit++; 871 872 /* 873 * If the mask bit location is below 52, then some bits above the 874 * physical addressing limit will always be reserved, so use the 875 * rsvd_bits() function to generate the mask. This mask, along with 876 * the present bit, will be used to generate a page fault with 877 * PFER.RSV = 1. 878 * 879 * If the mask bit location is 52 (or above), then clear the mask. 880 */ 881 mask = (mask_bit < 52) ? rsvd_bits(mask_bit, 51) | PT_PRESENT_MASK : 0; 882 883 kvm_mmu_set_mmio_spte_mask(mask, mask, PT_WRITABLE_MASK | PT_USER_MASK); 884 } 885 886 static void svm_hardware_teardown(void) 887 { 888 int cpu; 889 890 sev_hardware_teardown(); 891 892 for_each_possible_cpu(cpu) 893 svm_cpu_uninit(cpu); 894 895 __free_pages(pfn_to_page(iopm_base >> PAGE_SHIFT), 896 get_order(IOPM_SIZE)); 897 iopm_base = 0; 898 } 899 900 static __init void svm_set_cpu_caps(void) 901 { 902 kvm_set_cpu_caps(); 903 904 supported_xss = 0; 905 906 /* CPUID 0x80000001 and 0x8000000A (SVM features) */ 907 if (nested) { 908 kvm_cpu_cap_set(X86_FEATURE_SVM); 909 910 if (nrips) 911 kvm_cpu_cap_set(X86_FEATURE_NRIPS); 912 913 if (npt_enabled) 914 kvm_cpu_cap_set(X86_FEATURE_NPT); 915 916 /* Nested VM can receive #VMEXIT instead of triggering #GP */ 917 kvm_cpu_cap_set(X86_FEATURE_SVME_ADDR_CHK); 918 } 919 920 /* CPUID 0x80000008 */ 921 if (boot_cpu_has(X86_FEATURE_LS_CFG_SSBD) || 922 boot_cpu_has(X86_FEATURE_AMD_SSBD)) 923 kvm_cpu_cap_set(X86_FEATURE_VIRT_SSBD); 924 925 /* CPUID 0x8000001F (SME/SEV features) */ 926 sev_set_cpu_caps(); 927 } 928 929 static __init int svm_hardware_setup(void) 930 { 931 int cpu; 932 struct page *iopm_pages; 933 void *iopm_va; 934 int r; 935 unsigned int order = get_order(IOPM_SIZE); 936 937 iopm_pages = alloc_pages(GFP_KERNEL, order); 938 939 if (!iopm_pages) 940 return -ENOMEM; 941 942 iopm_va = page_address(iopm_pages); 943 memset(iopm_va, 0xff, PAGE_SIZE * (1 << order)); 944 iopm_base = page_to_pfn(iopm_pages) << PAGE_SHIFT; 945 946 init_msrpm_offsets(); 947 948 supported_xcr0 &= ~(XFEATURE_MASK_BNDREGS | XFEATURE_MASK_BNDCSR); 949 950 if (boot_cpu_has(X86_FEATURE_NX)) 951 kvm_enable_efer_bits(EFER_NX); 952 953 if (boot_cpu_has(X86_FEATURE_FXSR_OPT)) 954 kvm_enable_efer_bits(EFER_FFXSR); 955 956 if (boot_cpu_has(X86_FEATURE_TSCRATEMSR)) { 957 kvm_has_tsc_control = true; 958 kvm_max_tsc_scaling_ratio = TSC_RATIO_MAX; 959 kvm_tsc_scaling_ratio_frac_bits = 32; 960 } 961 962 if (boot_cpu_has(X86_FEATURE_RDTSCP)) 963 kvm_define_user_return_msr(TSC_AUX_URET_SLOT, MSR_TSC_AUX); 964 965 /* Check for pause filtering support */ 966 if (!boot_cpu_has(X86_FEATURE_PAUSEFILTER)) { 967 pause_filter_count = 0; 968 pause_filter_thresh = 0; 969 } else if (!boot_cpu_has(X86_FEATURE_PFTHRESHOLD)) { 970 pause_filter_thresh = 0; 971 } 972 973 if (nested) { 974 printk(KERN_INFO "kvm: Nested Virtualization enabled\n"); 975 kvm_enable_efer_bits(EFER_SVME | EFER_LMSLE); 976 } 977 978 /* 979 * KVM's MMU doesn't support using 2-level paging for itself, and thus 980 * NPT isn't supported if the host is using 2-level paging since host 981 * CR4 is unchanged on VMRUN. 982 */ 983 if (!IS_ENABLED(CONFIG_X86_64) && !IS_ENABLED(CONFIG_X86_PAE)) 984 npt_enabled = false; 985 986 if (!boot_cpu_has(X86_FEATURE_NPT)) 987 npt_enabled = false; 988 989 kvm_configure_mmu(npt_enabled, get_max_npt_level(), PG_LEVEL_1G); 990 pr_info("kvm: Nested Paging %sabled\n", npt_enabled ? "en" : "dis"); 991 992 /* Note, SEV setup consumes npt_enabled. */ 993 sev_hardware_setup(); 994 995 svm_adjust_mmio_mask(); 996 997 for_each_possible_cpu(cpu) { 998 r = svm_cpu_init(cpu); 999 if (r) 1000 goto err; 1001 } 1002 1003 if (nrips) { 1004 if (!boot_cpu_has(X86_FEATURE_NRIPS)) 1005 nrips = false; 1006 } 1007 1008 if (avic) { 1009 if (!npt_enabled || 1010 !boot_cpu_has(X86_FEATURE_AVIC) || 1011 !IS_ENABLED(CONFIG_X86_LOCAL_APIC)) { 1012 avic = false; 1013 } else { 1014 pr_info("AVIC enabled\n"); 1015 1016 amd_iommu_register_ga_log_notifier(&avic_ga_log_notifier); 1017 } 1018 } 1019 1020 if (vls) { 1021 if (!npt_enabled || 1022 !boot_cpu_has(X86_FEATURE_V_VMSAVE_VMLOAD) || 1023 !IS_ENABLED(CONFIG_X86_64)) { 1024 vls = false; 1025 } else { 1026 pr_info("Virtual VMLOAD VMSAVE supported\n"); 1027 } 1028 } 1029 1030 if (boot_cpu_has(X86_FEATURE_SVME_ADDR_CHK)) 1031 svm_gp_erratum_intercept = false; 1032 1033 if (vgif) { 1034 if (!boot_cpu_has(X86_FEATURE_VGIF)) 1035 vgif = false; 1036 else 1037 pr_info("Virtual GIF supported\n"); 1038 } 1039 1040 svm_set_cpu_caps(); 1041 1042 /* 1043 * It seems that on AMD processors PTE's accessed bit is 1044 * being set by the CPU hardware before the NPF vmexit. 1045 * This is not expected behaviour and our tests fail because 1046 * of it. 1047 * A workaround here is to disable support for 1048 * GUEST_MAXPHYADDR < HOST_MAXPHYADDR if NPT is enabled. 1049 * In this case userspace can know if there is support using 1050 * KVM_CAP_SMALLER_MAXPHYADDR extension and decide how to handle 1051 * it 1052 * If future AMD CPU models change the behaviour described above, 1053 * this variable can be changed accordingly 1054 */ 1055 allow_smaller_maxphyaddr = !npt_enabled; 1056 1057 return 0; 1058 1059 err: 1060 svm_hardware_teardown(); 1061 return r; 1062 } 1063 1064 static void init_seg(struct vmcb_seg *seg) 1065 { 1066 seg->selector = 0; 1067 seg->attrib = SVM_SELECTOR_P_MASK | SVM_SELECTOR_S_MASK | 1068 SVM_SELECTOR_WRITE_MASK; /* Read/Write Data Segment */ 1069 seg->limit = 0xffff; 1070 seg->base = 0; 1071 } 1072 1073 static void init_sys_seg(struct vmcb_seg *seg, uint32_t type) 1074 { 1075 seg->selector = 0; 1076 seg->attrib = SVM_SELECTOR_P_MASK | type; 1077 seg->limit = 0xffff; 1078 seg->base = 0; 1079 } 1080 1081 static u64 svm_write_l1_tsc_offset(struct kvm_vcpu *vcpu, u64 offset) 1082 { 1083 struct vcpu_svm *svm = to_svm(vcpu); 1084 u64 g_tsc_offset = 0; 1085 1086 if (is_guest_mode(vcpu)) { 1087 /* Write L1's TSC offset. */ 1088 g_tsc_offset = svm->vmcb->control.tsc_offset - 1089 svm->vmcb01.ptr->control.tsc_offset; 1090 svm->vmcb01.ptr->control.tsc_offset = offset; 1091 } 1092 1093 trace_kvm_write_tsc_offset(vcpu->vcpu_id, 1094 svm->vmcb->control.tsc_offset - g_tsc_offset, 1095 offset); 1096 1097 svm->vmcb->control.tsc_offset = offset + g_tsc_offset; 1098 1099 vmcb_mark_dirty(svm->vmcb, VMCB_INTERCEPTS); 1100 return svm->vmcb->control.tsc_offset; 1101 } 1102 1103 static void svm_check_invpcid(struct vcpu_svm *svm) 1104 { 1105 /* 1106 * Intercept INVPCID if shadow paging is enabled to sync/free shadow 1107 * roots, or if INVPCID is disabled in the guest to inject #UD. 1108 */ 1109 if (kvm_cpu_cap_has(X86_FEATURE_INVPCID)) { 1110 if (!npt_enabled || 1111 !guest_cpuid_has(&svm->vcpu, X86_FEATURE_INVPCID)) 1112 svm_set_intercept(svm, INTERCEPT_INVPCID); 1113 else 1114 svm_clr_intercept(svm, INTERCEPT_INVPCID); 1115 } 1116 } 1117 1118 static void init_vmcb(struct kvm_vcpu *vcpu) 1119 { 1120 struct vcpu_svm *svm = to_svm(vcpu); 1121 struct vmcb_control_area *control = &svm->vmcb->control; 1122 struct vmcb_save_area *save = &svm->vmcb->save; 1123 1124 vcpu->arch.hflags = 0; 1125 1126 svm_set_intercept(svm, INTERCEPT_CR0_READ); 1127 svm_set_intercept(svm, INTERCEPT_CR3_READ); 1128 svm_set_intercept(svm, INTERCEPT_CR4_READ); 1129 svm_set_intercept(svm, INTERCEPT_CR0_WRITE); 1130 svm_set_intercept(svm, INTERCEPT_CR3_WRITE); 1131 svm_set_intercept(svm, INTERCEPT_CR4_WRITE); 1132 if (!kvm_vcpu_apicv_active(vcpu)) 1133 svm_set_intercept(svm, INTERCEPT_CR8_WRITE); 1134 1135 set_dr_intercepts(svm); 1136 1137 set_exception_intercept(svm, PF_VECTOR); 1138 set_exception_intercept(svm, UD_VECTOR); 1139 set_exception_intercept(svm, MC_VECTOR); 1140 set_exception_intercept(svm, AC_VECTOR); 1141 set_exception_intercept(svm, DB_VECTOR); 1142 /* 1143 * Guest access to VMware backdoor ports could legitimately 1144 * trigger #GP because of TSS I/O permission bitmap. 1145 * We intercept those #GP and allow access to them anyway 1146 * as VMware does. 1147 */ 1148 if (enable_vmware_backdoor) 1149 set_exception_intercept(svm, GP_VECTOR); 1150 1151 svm_set_intercept(svm, INTERCEPT_INTR); 1152 svm_set_intercept(svm, INTERCEPT_NMI); 1153 svm_set_intercept(svm, INTERCEPT_SMI); 1154 svm_set_intercept(svm, INTERCEPT_SELECTIVE_CR0); 1155 svm_set_intercept(svm, INTERCEPT_RDPMC); 1156 svm_set_intercept(svm, INTERCEPT_CPUID); 1157 svm_set_intercept(svm, INTERCEPT_INVD); 1158 svm_set_intercept(svm, INTERCEPT_INVLPG); 1159 svm_set_intercept(svm, INTERCEPT_INVLPGA); 1160 svm_set_intercept(svm, INTERCEPT_IOIO_PROT); 1161 svm_set_intercept(svm, INTERCEPT_MSR_PROT); 1162 svm_set_intercept(svm, INTERCEPT_TASK_SWITCH); 1163 svm_set_intercept(svm, INTERCEPT_SHUTDOWN); 1164 svm_set_intercept(svm, INTERCEPT_VMRUN); 1165 svm_set_intercept(svm, INTERCEPT_VMMCALL); 1166 svm_set_intercept(svm, INTERCEPT_VMLOAD); 1167 svm_set_intercept(svm, INTERCEPT_VMSAVE); 1168 svm_set_intercept(svm, INTERCEPT_STGI); 1169 svm_set_intercept(svm, INTERCEPT_CLGI); 1170 svm_set_intercept(svm, INTERCEPT_SKINIT); 1171 svm_set_intercept(svm, INTERCEPT_WBINVD); 1172 svm_set_intercept(svm, INTERCEPT_XSETBV); 1173 svm_set_intercept(svm, INTERCEPT_RDPRU); 1174 svm_set_intercept(svm, INTERCEPT_RSM); 1175 1176 if (!kvm_mwait_in_guest(vcpu->kvm)) { 1177 svm_set_intercept(svm, INTERCEPT_MONITOR); 1178 svm_set_intercept(svm, INTERCEPT_MWAIT); 1179 } 1180 1181 if (!kvm_hlt_in_guest(vcpu->kvm)) 1182 svm_set_intercept(svm, INTERCEPT_HLT); 1183 1184 control->iopm_base_pa = __sme_set(iopm_base); 1185 control->msrpm_base_pa = __sme_set(__pa(svm->msrpm)); 1186 control->int_ctl = V_INTR_MASKING_MASK; 1187 1188 init_seg(&save->es); 1189 init_seg(&save->ss); 1190 init_seg(&save->ds); 1191 init_seg(&save->fs); 1192 init_seg(&save->gs); 1193 1194 save->cs.selector = 0xf000; 1195 save->cs.base = 0xffff0000; 1196 /* Executable/Readable Code Segment */ 1197 save->cs.attrib = SVM_SELECTOR_READ_MASK | SVM_SELECTOR_P_MASK | 1198 SVM_SELECTOR_S_MASK | SVM_SELECTOR_CODE_MASK; 1199 save->cs.limit = 0xffff; 1200 1201 save->gdtr.limit = 0xffff; 1202 save->idtr.limit = 0xffff; 1203 1204 init_sys_seg(&save->ldtr, SEG_TYPE_LDT); 1205 init_sys_seg(&save->tr, SEG_TYPE_BUSY_TSS16); 1206 1207 svm_set_cr4(vcpu, 0); 1208 svm_set_efer(vcpu, 0); 1209 save->dr6 = 0xffff0ff0; 1210 kvm_set_rflags(vcpu, X86_EFLAGS_FIXED); 1211 save->rip = 0x0000fff0; 1212 vcpu->arch.regs[VCPU_REGS_RIP] = save->rip; 1213 1214 /* 1215 * svm_set_cr0() sets PG and WP and clears NW and CD on save->cr0. 1216 * It also updates the guest-visible cr0 value. 1217 */ 1218 svm_set_cr0(vcpu, X86_CR0_NW | X86_CR0_CD | X86_CR0_ET); 1219 kvm_mmu_reset_context(vcpu); 1220 1221 save->cr4 = X86_CR4_PAE; 1222 /* rdx = ?? */ 1223 1224 if (npt_enabled) { 1225 /* Setup VMCB for Nested Paging */ 1226 control->nested_ctl |= SVM_NESTED_CTL_NP_ENABLE; 1227 svm_clr_intercept(svm, INTERCEPT_INVLPG); 1228 clr_exception_intercept(svm, PF_VECTOR); 1229 svm_clr_intercept(svm, INTERCEPT_CR3_READ); 1230 svm_clr_intercept(svm, INTERCEPT_CR3_WRITE); 1231 save->g_pat = vcpu->arch.pat; 1232 save->cr3 = 0; 1233 save->cr4 = 0; 1234 } 1235 svm->current_vmcb->asid_generation = 0; 1236 svm->asid = 0; 1237 1238 svm->nested.vmcb12_gpa = 0; 1239 svm->nested.last_vmcb12_gpa = 0; 1240 vcpu->arch.hflags = 0; 1241 1242 if (!kvm_pause_in_guest(vcpu->kvm)) { 1243 control->pause_filter_count = pause_filter_count; 1244 if (pause_filter_thresh) 1245 control->pause_filter_thresh = pause_filter_thresh; 1246 svm_set_intercept(svm, INTERCEPT_PAUSE); 1247 } else { 1248 svm_clr_intercept(svm, INTERCEPT_PAUSE); 1249 } 1250 1251 svm_check_invpcid(svm); 1252 1253 /* 1254 * If the host supports V_SPEC_CTRL then disable the interception 1255 * of MSR_IA32_SPEC_CTRL. 1256 */ 1257 if (boot_cpu_has(X86_FEATURE_V_SPEC_CTRL)) 1258 set_msr_interception(vcpu, svm->msrpm, MSR_IA32_SPEC_CTRL, 1, 1); 1259 1260 if (kvm_vcpu_apicv_active(vcpu)) 1261 avic_init_vmcb(svm); 1262 1263 if (vgif) { 1264 svm_clr_intercept(svm, INTERCEPT_STGI); 1265 svm_clr_intercept(svm, INTERCEPT_CLGI); 1266 svm->vmcb->control.int_ctl |= V_GIF_ENABLE_MASK; 1267 } 1268 1269 if (sev_guest(vcpu->kvm)) { 1270 svm->vmcb->control.nested_ctl |= SVM_NESTED_CTL_SEV_ENABLE; 1271 clr_exception_intercept(svm, UD_VECTOR); 1272 1273 if (sev_es_guest(vcpu->kvm)) { 1274 /* Perform SEV-ES specific VMCB updates */ 1275 sev_es_init_vmcb(svm); 1276 } 1277 } 1278 1279 vmcb_mark_all_dirty(svm->vmcb); 1280 1281 enable_gif(svm); 1282 1283 } 1284 1285 static void svm_vcpu_reset(struct kvm_vcpu *vcpu, bool init_event) 1286 { 1287 struct vcpu_svm *svm = to_svm(vcpu); 1288 u32 dummy; 1289 u32 eax = 1; 1290 1291 svm->spec_ctrl = 0; 1292 svm->virt_spec_ctrl = 0; 1293 1294 if (!init_event) { 1295 vcpu->arch.apic_base = APIC_DEFAULT_PHYS_BASE | 1296 MSR_IA32_APICBASE_ENABLE; 1297 if (kvm_vcpu_is_reset_bsp(vcpu)) 1298 vcpu->arch.apic_base |= MSR_IA32_APICBASE_BSP; 1299 } 1300 init_vmcb(vcpu); 1301 1302 kvm_cpuid(vcpu, &eax, &dummy, &dummy, &dummy, false); 1303 kvm_rdx_write(vcpu, eax); 1304 1305 if (kvm_vcpu_apicv_active(vcpu) && !init_event) 1306 avic_update_vapic_bar(svm, APIC_DEFAULT_PHYS_BASE); 1307 } 1308 1309 void svm_switch_vmcb(struct vcpu_svm *svm, struct kvm_vmcb_info *target_vmcb) 1310 { 1311 svm->current_vmcb = target_vmcb; 1312 svm->vmcb = target_vmcb->ptr; 1313 } 1314 1315 static int svm_create_vcpu(struct kvm_vcpu *vcpu) 1316 { 1317 struct vcpu_svm *svm; 1318 struct page *vmcb01_page; 1319 struct page *vmsa_page = NULL; 1320 int err; 1321 1322 BUILD_BUG_ON(offsetof(struct vcpu_svm, vcpu) != 0); 1323 svm = to_svm(vcpu); 1324 1325 err = -ENOMEM; 1326 vmcb01_page = alloc_page(GFP_KERNEL_ACCOUNT | __GFP_ZERO); 1327 if (!vmcb01_page) 1328 goto out; 1329 1330 if (sev_es_guest(vcpu->kvm)) { 1331 /* 1332 * SEV-ES guests require a separate VMSA page used to contain 1333 * the encrypted register state of the guest. 1334 */ 1335 vmsa_page = alloc_page(GFP_KERNEL_ACCOUNT | __GFP_ZERO); 1336 if (!vmsa_page) 1337 goto error_free_vmcb_page; 1338 1339 /* 1340 * SEV-ES guests maintain an encrypted version of their FPU 1341 * state which is restored and saved on VMRUN and VMEXIT. 1342 * Free the fpu structure to prevent KVM from attempting to 1343 * access the FPU state. 1344 */ 1345 kvm_free_guest_fpu(vcpu); 1346 } 1347 1348 err = avic_init_vcpu(svm); 1349 if (err) 1350 goto error_free_vmsa_page; 1351 1352 /* We initialize this flag to true to make sure that the is_running 1353 * bit would be set the first time the vcpu is loaded. 1354 */ 1355 if (irqchip_in_kernel(vcpu->kvm) && kvm_apicv_activated(vcpu->kvm)) 1356 svm->avic_is_running = true; 1357 1358 svm->msrpm = svm_vcpu_alloc_msrpm(); 1359 if (!svm->msrpm) { 1360 err = -ENOMEM; 1361 goto error_free_vmsa_page; 1362 } 1363 1364 svm_vcpu_init_msrpm(vcpu, svm->msrpm); 1365 1366 svm->vmcb01.ptr = page_address(vmcb01_page); 1367 svm->vmcb01.pa = __sme_set(page_to_pfn(vmcb01_page) << PAGE_SHIFT); 1368 1369 if (vmsa_page) 1370 svm->vmsa = page_address(vmsa_page); 1371 1372 svm->guest_state_loaded = false; 1373 1374 svm_switch_vmcb(svm, &svm->vmcb01); 1375 init_vmcb(vcpu); 1376 1377 svm_init_osvw(vcpu); 1378 vcpu->arch.microcode_version = 0x01000065; 1379 1380 if (sev_es_guest(vcpu->kvm)) 1381 /* Perform SEV-ES specific VMCB creation updates */ 1382 sev_es_create_vcpu(svm); 1383 1384 return 0; 1385 1386 error_free_vmsa_page: 1387 if (vmsa_page) 1388 __free_page(vmsa_page); 1389 error_free_vmcb_page: 1390 __free_page(vmcb01_page); 1391 out: 1392 return err; 1393 } 1394 1395 static void svm_clear_current_vmcb(struct vmcb *vmcb) 1396 { 1397 int i; 1398 1399 for_each_online_cpu(i) 1400 cmpxchg(&per_cpu(svm_data, i)->current_vmcb, vmcb, NULL); 1401 } 1402 1403 static void svm_free_vcpu(struct kvm_vcpu *vcpu) 1404 { 1405 struct vcpu_svm *svm = to_svm(vcpu); 1406 1407 /* 1408 * The vmcb page can be recycled, causing a false negative in 1409 * svm_vcpu_load(). So, ensure that no logical CPU has this 1410 * vmcb page recorded as its current vmcb. 1411 */ 1412 svm_clear_current_vmcb(svm->vmcb); 1413 1414 svm_free_nested(svm); 1415 1416 sev_free_vcpu(vcpu); 1417 1418 __free_page(pfn_to_page(__sme_clr(svm->vmcb01.pa) >> PAGE_SHIFT)); 1419 __free_pages(virt_to_page(svm->msrpm), get_order(MSRPM_SIZE)); 1420 } 1421 1422 static void svm_prepare_guest_switch(struct kvm_vcpu *vcpu) 1423 { 1424 struct vcpu_svm *svm = to_svm(vcpu); 1425 struct svm_cpu_data *sd = per_cpu(svm_data, vcpu->cpu); 1426 1427 if (svm->guest_state_loaded) 1428 return; 1429 1430 /* 1431 * Save additional host state that will be restored on VMEXIT (sev-es) 1432 * or subsequent vmload of host save area. 1433 */ 1434 if (sev_es_guest(vcpu->kvm)) { 1435 sev_es_prepare_guest_switch(svm, vcpu->cpu); 1436 } else { 1437 vmsave(__sme_page_pa(sd->save_area)); 1438 } 1439 1440 if (static_cpu_has(X86_FEATURE_TSCRATEMSR)) { 1441 u64 tsc_ratio = vcpu->arch.tsc_scaling_ratio; 1442 if (tsc_ratio != __this_cpu_read(current_tsc_ratio)) { 1443 __this_cpu_write(current_tsc_ratio, tsc_ratio); 1444 wrmsrl(MSR_AMD64_TSC_RATIO, tsc_ratio); 1445 } 1446 } 1447 1448 if (static_cpu_has(X86_FEATURE_RDTSCP)) 1449 kvm_set_user_return_msr(TSC_AUX_URET_SLOT, svm->tsc_aux, -1ull); 1450 1451 svm->guest_state_loaded = true; 1452 } 1453 1454 static void svm_prepare_host_switch(struct kvm_vcpu *vcpu) 1455 { 1456 to_svm(vcpu)->guest_state_loaded = false; 1457 } 1458 1459 static void svm_vcpu_load(struct kvm_vcpu *vcpu, int cpu) 1460 { 1461 struct vcpu_svm *svm = to_svm(vcpu); 1462 struct svm_cpu_data *sd = per_cpu(svm_data, cpu); 1463 1464 if (sd->current_vmcb != svm->vmcb) { 1465 sd->current_vmcb = svm->vmcb; 1466 indirect_branch_prediction_barrier(); 1467 } 1468 avic_vcpu_load(vcpu, cpu); 1469 } 1470 1471 static void svm_vcpu_put(struct kvm_vcpu *vcpu) 1472 { 1473 avic_vcpu_put(vcpu); 1474 svm_prepare_host_switch(vcpu); 1475 1476 ++vcpu->stat.host_state_reload; 1477 } 1478 1479 static unsigned long svm_get_rflags(struct kvm_vcpu *vcpu) 1480 { 1481 struct vcpu_svm *svm = to_svm(vcpu); 1482 unsigned long rflags = svm->vmcb->save.rflags; 1483 1484 if (svm->nmi_singlestep) { 1485 /* Hide our flags if they were not set by the guest */ 1486 if (!(svm->nmi_singlestep_guest_rflags & X86_EFLAGS_TF)) 1487 rflags &= ~X86_EFLAGS_TF; 1488 if (!(svm->nmi_singlestep_guest_rflags & X86_EFLAGS_RF)) 1489 rflags &= ~X86_EFLAGS_RF; 1490 } 1491 return rflags; 1492 } 1493 1494 static void svm_set_rflags(struct kvm_vcpu *vcpu, unsigned long rflags) 1495 { 1496 if (to_svm(vcpu)->nmi_singlestep) 1497 rflags |= (X86_EFLAGS_TF | X86_EFLAGS_RF); 1498 1499 /* 1500 * Any change of EFLAGS.VM is accompanied by a reload of SS 1501 * (caused by either a task switch or an inter-privilege IRET), 1502 * so we do not need to update the CPL here. 1503 */ 1504 to_svm(vcpu)->vmcb->save.rflags = rflags; 1505 } 1506 1507 static void svm_cache_reg(struct kvm_vcpu *vcpu, enum kvm_reg reg) 1508 { 1509 switch (reg) { 1510 case VCPU_EXREG_PDPTR: 1511 BUG_ON(!npt_enabled); 1512 load_pdptrs(vcpu, vcpu->arch.walk_mmu, kvm_read_cr3(vcpu)); 1513 break; 1514 default: 1515 WARN_ON_ONCE(1); 1516 } 1517 } 1518 1519 static void svm_set_vintr(struct vcpu_svm *svm) 1520 { 1521 struct vmcb_control_area *control; 1522 1523 /* The following fields are ignored when AVIC is enabled */ 1524 WARN_ON(kvm_vcpu_apicv_active(&svm->vcpu)); 1525 svm_set_intercept(svm, INTERCEPT_VINTR); 1526 1527 /* 1528 * This is just a dummy VINTR to actually cause a vmexit to happen. 1529 * Actual injection of virtual interrupts happens through EVENTINJ. 1530 */ 1531 control = &svm->vmcb->control; 1532 control->int_vector = 0x0; 1533 control->int_ctl &= ~V_INTR_PRIO_MASK; 1534 control->int_ctl |= V_IRQ_MASK | 1535 ((/*control->int_vector >> 4*/ 0xf) << V_INTR_PRIO_SHIFT); 1536 vmcb_mark_dirty(svm->vmcb, VMCB_INTR); 1537 } 1538 1539 static void svm_clear_vintr(struct vcpu_svm *svm) 1540 { 1541 const u32 mask = V_TPR_MASK | V_GIF_ENABLE_MASK | V_GIF_MASK | V_INTR_MASKING_MASK; 1542 svm_clr_intercept(svm, INTERCEPT_VINTR); 1543 1544 /* Drop int_ctl fields related to VINTR injection. */ 1545 svm->vmcb->control.int_ctl &= mask; 1546 if (is_guest_mode(&svm->vcpu)) { 1547 svm->vmcb01.ptr->control.int_ctl &= mask; 1548 1549 WARN_ON((svm->vmcb->control.int_ctl & V_TPR_MASK) != 1550 (svm->nested.ctl.int_ctl & V_TPR_MASK)); 1551 svm->vmcb->control.int_ctl |= svm->nested.ctl.int_ctl & ~mask; 1552 } 1553 1554 vmcb_mark_dirty(svm->vmcb, VMCB_INTR); 1555 } 1556 1557 static struct vmcb_seg *svm_seg(struct kvm_vcpu *vcpu, int seg) 1558 { 1559 struct vmcb_save_area *save = &to_svm(vcpu)->vmcb->save; 1560 struct vmcb_save_area *save01 = &to_svm(vcpu)->vmcb01.ptr->save; 1561 1562 switch (seg) { 1563 case VCPU_SREG_CS: return &save->cs; 1564 case VCPU_SREG_DS: return &save->ds; 1565 case VCPU_SREG_ES: return &save->es; 1566 case VCPU_SREG_FS: return &save01->fs; 1567 case VCPU_SREG_GS: return &save01->gs; 1568 case VCPU_SREG_SS: return &save->ss; 1569 case VCPU_SREG_TR: return &save01->tr; 1570 case VCPU_SREG_LDTR: return &save01->ldtr; 1571 } 1572 BUG(); 1573 return NULL; 1574 } 1575 1576 static u64 svm_get_segment_base(struct kvm_vcpu *vcpu, int seg) 1577 { 1578 struct vmcb_seg *s = svm_seg(vcpu, seg); 1579 1580 return s->base; 1581 } 1582 1583 static void svm_get_segment(struct kvm_vcpu *vcpu, 1584 struct kvm_segment *var, int seg) 1585 { 1586 struct vmcb_seg *s = svm_seg(vcpu, seg); 1587 1588 var->base = s->base; 1589 var->limit = s->limit; 1590 var->selector = s->selector; 1591 var->type = s->attrib & SVM_SELECTOR_TYPE_MASK; 1592 var->s = (s->attrib >> SVM_SELECTOR_S_SHIFT) & 1; 1593 var->dpl = (s->attrib >> SVM_SELECTOR_DPL_SHIFT) & 3; 1594 var->present = (s->attrib >> SVM_SELECTOR_P_SHIFT) & 1; 1595 var->avl = (s->attrib >> SVM_SELECTOR_AVL_SHIFT) & 1; 1596 var->l = (s->attrib >> SVM_SELECTOR_L_SHIFT) & 1; 1597 var->db = (s->attrib >> SVM_SELECTOR_DB_SHIFT) & 1; 1598 1599 /* 1600 * AMD CPUs circa 2014 track the G bit for all segments except CS. 1601 * However, the SVM spec states that the G bit is not observed by the 1602 * CPU, and some VMware virtual CPUs drop the G bit for all segments. 1603 * So let's synthesize a legal G bit for all segments, this helps 1604 * running KVM nested. It also helps cross-vendor migration, because 1605 * Intel's vmentry has a check on the 'G' bit. 1606 */ 1607 var->g = s->limit > 0xfffff; 1608 1609 /* 1610 * AMD's VMCB does not have an explicit unusable field, so emulate it 1611 * for cross vendor migration purposes by "not present" 1612 */ 1613 var->unusable = !var->present; 1614 1615 switch (seg) { 1616 case VCPU_SREG_TR: 1617 /* 1618 * Work around a bug where the busy flag in the tr selector 1619 * isn't exposed 1620 */ 1621 var->type |= 0x2; 1622 break; 1623 case VCPU_SREG_DS: 1624 case VCPU_SREG_ES: 1625 case VCPU_SREG_FS: 1626 case VCPU_SREG_GS: 1627 /* 1628 * The accessed bit must always be set in the segment 1629 * descriptor cache, although it can be cleared in the 1630 * descriptor, the cached bit always remains at 1. Since 1631 * Intel has a check on this, set it here to support 1632 * cross-vendor migration. 1633 */ 1634 if (!var->unusable) 1635 var->type |= 0x1; 1636 break; 1637 case VCPU_SREG_SS: 1638 /* 1639 * On AMD CPUs sometimes the DB bit in the segment 1640 * descriptor is left as 1, although the whole segment has 1641 * been made unusable. Clear it here to pass an Intel VMX 1642 * entry check when cross vendor migrating. 1643 */ 1644 if (var->unusable) 1645 var->db = 0; 1646 /* This is symmetric with svm_set_segment() */ 1647 var->dpl = to_svm(vcpu)->vmcb->save.cpl; 1648 break; 1649 } 1650 } 1651 1652 static int svm_get_cpl(struct kvm_vcpu *vcpu) 1653 { 1654 struct vmcb_save_area *save = &to_svm(vcpu)->vmcb->save; 1655 1656 return save->cpl; 1657 } 1658 1659 static void svm_get_idt(struct kvm_vcpu *vcpu, struct desc_ptr *dt) 1660 { 1661 struct vcpu_svm *svm = to_svm(vcpu); 1662 1663 dt->size = svm->vmcb->save.idtr.limit; 1664 dt->address = svm->vmcb->save.idtr.base; 1665 } 1666 1667 static void svm_set_idt(struct kvm_vcpu *vcpu, struct desc_ptr *dt) 1668 { 1669 struct vcpu_svm *svm = to_svm(vcpu); 1670 1671 svm->vmcb->save.idtr.limit = dt->size; 1672 svm->vmcb->save.idtr.base = dt->address ; 1673 vmcb_mark_dirty(svm->vmcb, VMCB_DT); 1674 } 1675 1676 static void svm_get_gdt(struct kvm_vcpu *vcpu, struct desc_ptr *dt) 1677 { 1678 struct vcpu_svm *svm = to_svm(vcpu); 1679 1680 dt->size = svm->vmcb->save.gdtr.limit; 1681 dt->address = svm->vmcb->save.gdtr.base; 1682 } 1683 1684 static void svm_set_gdt(struct kvm_vcpu *vcpu, struct desc_ptr *dt) 1685 { 1686 struct vcpu_svm *svm = to_svm(vcpu); 1687 1688 svm->vmcb->save.gdtr.limit = dt->size; 1689 svm->vmcb->save.gdtr.base = dt->address ; 1690 vmcb_mark_dirty(svm->vmcb, VMCB_DT); 1691 } 1692 1693 void svm_set_cr0(struct kvm_vcpu *vcpu, unsigned long cr0) 1694 { 1695 struct vcpu_svm *svm = to_svm(vcpu); 1696 u64 hcr0 = cr0; 1697 1698 #ifdef CONFIG_X86_64 1699 if (vcpu->arch.efer & EFER_LME && !vcpu->arch.guest_state_protected) { 1700 if (!is_paging(vcpu) && (cr0 & X86_CR0_PG)) { 1701 vcpu->arch.efer |= EFER_LMA; 1702 svm->vmcb->save.efer |= EFER_LMA | EFER_LME; 1703 } 1704 1705 if (is_paging(vcpu) && !(cr0 & X86_CR0_PG)) { 1706 vcpu->arch.efer &= ~EFER_LMA; 1707 svm->vmcb->save.efer &= ~(EFER_LMA | EFER_LME); 1708 } 1709 } 1710 #endif 1711 vcpu->arch.cr0 = cr0; 1712 1713 if (!npt_enabled) 1714 hcr0 |= X86_CR0_PG | X86_CR0_WP; 1715 1716 /* 1717 * re-enable caching here because the QEMU bios 1718 * does not do it - this results in some delay at 1719 * reboot 1720 */ 1721 if (kvm_check_has_quirk(vcpu->kvm, KVM_X86_QUIRK_CD_NW_CLEARED)) 1722 hcr0 &= ~(X86_CR0_CD | X86_CR0_NW); 1723 1724 svm->vmcb->save.cr0 = hcr0; 1725 vmcb_mark_dirty(svm->vmcb, VMCB_CR); 1726 1727 /* 1728 * SEV-ES guests must always keep the CR intercepts cleared. CR 1729 * tracking is done using the CR write traps. 1730 */ 1731 if (sev_es_guest(vcpu->kvm)) 1732 return; 1733 1734 if (hcr0 == cr0) { 1735 /* Selective CR0 write remains on. */ 1736 svm_clr_intercept(svm, INTERCEPT_CR0_READ); 1737 svm_clr_intercept(svm, INTERCEPT_CR0_WRITE); 1738 } else { 1739 svm_set_intercept(svm, INTERCEPT_CR0_READ); 1740 svm_set_intercept(svm, INTERCEPT_CR0_WRITE); 1741 } 1742 } 1743 1744 static bool svm_is_valid_cr4(struct kvm_vcpu *vcpu, unsigned long cr4) 1745 { 1746 return true; 1747 } 1748 1749 void svm_set_cr4(struct kvm_vcpu *vcpu, unsigned long cr4) 1750 { 1751 unsigned long host_cr4_mce = cr4_read_shadow() & X86_CR4_MCE; 1752 unsigned long old_cr4 = vcpu->arch.cr4; 1753 1754 if (npt_enabled && ((old_cr4 ^ cr4) & X86_CR4_PGE)) 1755 svm_flush_tlb(vcpu); 1756 1757 vcpu->arch.cr4 = cr4; 1758 if (!npt_enabled) 1759 cr4 |= X86_CR4_PAE; 1760 cr4 |= host_cr4_mce; 1761 to_svm(vcpu)->vmcb->save.cr4 = cr4; 1762 vmcb_mark_dirty(to_svm(vcpu)->vmcb, VMCB_CR); 1763 1764 if ((cr4 ^ old_cr4) & (X86_CR4_OSXSAVE | X86_CR4_PKE)) 1765 kvm_update_cpuid_runtime(vcpu); 1766 } 1767 1768 static void svm_set_segment(struct kvm_vcpu *vcpu, 1769 struct kvm_segment *var, int seg) 1770 { 1771 struct vcpu_svm *svm = to_svm(vcpu); 1772 struct vmcb_seg *s = svm_seg(vcpu, seg); 1773 1774 s->base = var->base; 1775 s->limit = var->limit; 1776 s->selector = var->selector; 1777 s->attrib = (var->type & SVM_SELECTOR_TYPE_MASK); 1778 s->attrib |= (var->s & 1) << SVM_SELECTOR_S_SHIFT; 1779 s->attrib |= (var->dpl & 3) << SVM_SELECTOR_DPL_SHIFT; 1780 s->attrib |= ((var->present & 1) && !var->unusable) << SVM_SELECTOR_P_SHIFT; 1781 s->attrib |= (var->avl & 1) << SVM_SELECTOR_AVL_SHIFT; 1782 s->attrib |= (var->l & 1) << SVM_SELECTOR_L_SHIFT; 1783 s->attrib |= (var->db & 1) << SVM_SELECTOR_DB_SHIFT; 1784 s->attrib |= (var->g & 1) << SVM_SELECTOR_G_SHIFT; 1785 1786 /* 1787 * This is always accurate, except if SYSRET returned to a segment 1788 * with SS.DPL != 3. Intel does not have this quirk, and always 1789 * forces SS.DPL to 3 on sysret, so we ignore that case; fixing it 1790 * would entail passing the CPL to userspace and back. 1791 */ 1792 if (seg == VCPU_SREG_SS) 1793 /* This is symmetric with svm_get_segment() */ 1794 svm->vmcb->save.cpl = (var->dpl & 3); 1795 1796 vmcb_mark_dirty(svm->vmcb, VMCB_SEG); 1797 } 1798 1799 static void svm_update_exception_bitmap(struct kvm_vcpu *vcpu) 1800 { 1801 struct vcpu_svm *svm = to_svm(vcpu); 1802 1803 clr_exception_intercept(svm, BP_VECTOR); 1804 1805 if (vcpu->guest_debug & KVM_GUESTDBG_ENABLE) { 1806 if (vcpu->guest_debug & KVM_GUESTDBG_USE_SW_BP) 1807 set_exception_intercept(svm, BP_VECTOR); 1808 } 1809 } 1810 1811 static void new_asid(struct vcpu_svm *svm, struct svm_cpu_data *sd) 1812 { 1813 if (sd->next_asid > sd->max_asid) { 1814 ++sd->asid_generation; 1815 sd->next_asid = sd->min_asid; 1816 svm->vmcb->control.tlb_ctl = TLB_CONTROL_FLUSH_ALL_ASID; 1817 vmcb_mark_dirty(svm->vmcb, VMCB_ASID); 1818 } 1819 1820 svm->current_vmcb->asid_generation = sd->asid_generation; 1821 svm->asid = sd->next_asid++; 1822 } 1823 1824 static void svm_set_dr6(struct vcpu_svm *svm, unsigned long value) 1825 { 1826 struct vmcb *vmcb = svm->vmcb; 1827 1828 if (svm->vcpu.arch.guest_state_protected) 1829 return; 1830 1831 if (unlikely(value != vmcb->save.dr6)) { 1832 vmcb->save.dr6 = value; 1833 vmcb_mark_dirty(vmcb, VMCB_DR); 1834 } 1835 } 1836 1837 static void svm_sync_dirty_debug_regs(struct kvm_vcpu *vcpu) 1838 { 1839 struct vcpu_svm *svm = to_svm(vcpu); 1840 1841 if (vcpu->arch.guest_state_protected) 1842 return; 1843 1844 get_debugreg(vcpu->arch.db[0], 0); 1845 get_debugreg(vcpu->arch.db[1], 1); 1846 get_debugreg(vcpu->arch.db[2], 2); 1847 get_debugreg(vcpu->arch.db[3], 3); 1848 /* 1849 * We cannot reset svm->vmcb->save.dr6 to DR6_ACTIVE_LOW here, 1850 * because db_interception might need it. We can do it before vmentry. 1851 */ 1852 vcpu->arch.dr6 = svm->vmcb->save.dr6; 1853 vcpu->arch.dr7 = svm->vmcb->save.dr7; 1854 vcpu->arch.switch_db_regs &= ~KVM_DEBUGREG_WONT_EXIT; 1855 set_dr_intercepts(svm); 1856 } 1857 1858 static void svm_set_dr7(struct kvm_vcpu *vcpu, unsigned long value) 1859 { 1860 struct vcpu_svm *svm = to_svm(vcpu); 1861 1862 if (vcpu->arch.guest_state_protected) 1863 return; 1864 1865 svm->vmcb->save.dr7 = value; 1866 vmcb_mark_dirty(svm->vmcb, VMCB_DR); 1867 } 1868 1869 static int pf_interception(struct kvm_vcpu *vcpu) 1870 { 1871 struct vcpu_svm *svm = to_svm(vcpu); 1872 1873 u64 fault_address = svm->vmcb->control.exit_info_2; 1874 u64 error_code = svm->vmcb->control.exit_info_1; 1875 1876 return kvm_handle_page_fault(vcpu, error_code, fault_address, 1877 static_cpu_has(X86_FEATURE_DECODEASSISTS) ? 1878 svm->vmcb->control.insn_bytes : NULL, 1879 svm->vmcb->control.insn_len); 1880 } 1881 1882 static int npf_interception(struct kvm_vcpu *vcpu) 1883 { 1884 struct vcpu_svm *svm = to_svm(vcpu); 1885 1886 u64 fault_address = __sme_clr(svm->vmcb->control.exit_info_2); 1887 u64 error_code = svm->vmcb->control.exit_info_1; 1888 1889 trace_kvm_page_fault(fault_address, error_code); 1890 return kvm_mmu_page_fault(vcpu, fault_address, error_code, 1891 static_cpu_has(X86_FEATURE_DECODEASSISTS) ? 1892 svm->vmcb->control.insn_bytes : NULL, 1893 svm->vmcb->control.insn_len); 1894 } 1895 1896 static int db_interception(struct kvm_vcpu *vcpu) 1897 { 1898 struct kvm_run *kvm_run = vcpu->run; 1899 struct vcpu_svm *svm = to_svm(vcpu); 1900 1901 if (!(vcpu->guest_debug & 1902 (KVM_GUESTDBG_SINGLESTEP | KVM_GUESTDBG_USE_HW_BP)) && 1903 !svm->nmi_singlestep) { 1904 u32 payload = svm->vmcb->save.dr6 ^ DR6_ACTIVE_LOW; 1905 kvm_queue_exception_p(vcpu, DB_VECTOR, payload); 1906 return 1; 1907 } 1908 1909 if (svm->nmi_singlestep) { 1910 disable_nmi_singlestep(svm); 1911 /* Make sure we check for pending NMIs upon entry */ 1912 kvm_make_request(KVM_REQ_EVENT, vcpu); 1913 } 1914 1915 if (vcpu->guest_debug & 1916 (KVM_GUESTDBG_SINGLESTEP | KVM_GUESTDBG_USE_HW_BP)) { 1917 kvm_run->exit_reason = KVM_EXIT_DEBUG; 1918 kvm_run->debug.arch.dr6 = svm->vmcb->save.dr6; 1919 kvm_run->debug.arch.dr7 = svm->vmcb->save.dr7; 1920 kvm_run->debug.arch.pc = 1921 svm->vmcb->save.cs.base + svm->vmcb->save.rip; 1922 kvm_run->debug.arch.exception = DB_VECTOR; 1923 return 0; 1924 } 1925 1926 return 1; 1927 } 1928 1929 static int bp_interception(struct kvm_vcpu *vcpu) 1930 { 1931 struct vcpu_svm *svm = to_svm(vcpu); 1932 struct kvm_run *kvm_run = vcpu->run; 1933 1934 kvm_run->exit_reason = KVM_EXIT_DEBUG; 1935 kvm_run->debug.arch.pc = svm->vmcb->save.cs.base + svm->vmcb->save.rip; 1936 kvm_run->debug.arch.exception = BP_VECTOR; 1937 return 0; 1938 } 1939 1940 static int ud_interception(struct kvm_vcpu *vcpu) 1941 { 1942 return handle_ud(vcpu); 1943 } 1944 1945 static int ac_interception(struct kvm_vcpu *vcpu) 1946 { 1947 kvm_queue_exception_e(vcpu, AC_VECTOR, 0); 1948 return 1; 1949 } 1950 1951 static bool is_erratum_383(void) 1952 { 1953 int err, i; 1954 u64 value; 1955 1956 if (!erratum_383_found) 1957 return false; 1958 1959 value = native_read_msr_safe(MSR_IA32_MC0_STATUS, &err); 1960 if (err) 1961 return false; 1962 1963 /* Bit 62 may or may not be set for this mce */ 1964 value &= ~(1ULL << 62); 1965 1966 if (value != 0xb600000000010015ULL) 1967 return false; 1968 1969 /* Clear MCi_STATUS registers */ 1970 for (i = 0; i < 6; ++i) 1971 native_write_msr_safe(MSR_IA32_MCx_STATUS(i), 0, 0); 1972 1973 value = native_read_msr_safe(MSR_IA32_MCG_STATUS, &err); 1974 if (!err) { 1975 u32 low, high; 1976 1977 value &= ~(1ULL << 2); 1978 low = lower_32_bits(value); 1979 high = upper_32_bits(value); 1980 1981 native_write_msr_safe(MSR_IA32_MCG_STATUS, low, high); 1982 } 1983 1984 /* Flush tlb to evict multi-match entries */ 1985 __flush_tlb_all(); 1986 1987 return true; 1988 } 1989 1990 static void svm_handle_mce(struct kvm_vcpu *vcpu) 1991 { 1992 if (is_erratum_383()) { 1993 /* 1994 * Erratum 383 triggered. Guest state is corrupt so kill the 1995 * guest. 1996 */ 1997 pr_err("KVM: Guest triggered AMD Erratum 383\n"); 1998 1999 kvm_make_request(KVM_REQ_TRIPLE_FAULT, vcpu); 2000 2001 return; 2002 } 2003 2004 /* 2005 * On an #MC intercept the MCE handler is not called automatically in 2006 * the host. So do it by hand here. 2007 */ 2008 kvm_machine_check(); 2009 } 2010 2011 static int mc_interception(struct kvm_vcpu *vcpu) 2012 { 2013 return 1; 2014 } 2015 2016 static int shutdown_interception(struct kvm_vcpu *vcpu) 2017 { 2018 struct kvm_run *kvm_run = vcpu->run; 2019 struct vcpu_svm *svm = to_svm(vcpu); 2020 2021 /* 2022 * The VM save area has already been encrypted so it 2023 * cannot be reinitialized - just terminate. 2024 */ 2025 if (sev_es_guest(vcpu->kvm)) 2026 return -EINVAL; 2027 2028 /* 2029 * VMCB is undefined after a SHUTDOWN intercept 2030 * so reinitialize it. 2031 */ 2032 clear_page(svm->vmcb); 2033 init_vmcb(vcpu); 2034 2035 kvm_run->exit_reason = KVM_EXIT_SHUTDOWN; 2036 return 0; 2037 } 2038 2039 static int io_interception(struct kvm_vcpu *vcpu) 2040 { 2041 struct vcpu_svm *svm = to_svm(vcpu); 2042 u32 io_info = svm->vmcb->control.exit_info_1; /* address size bug? */ 2043 int size, in, string; 2044 unsigned port; 2045 2046 ++vcpu->stat.io_exits; 2047 string = (io_info & SVM_IOIO_STR_MASK) != 0; 2048 in = (io_info & SVM_IOIO_TYPE_MASK) != 0; 2049 port = io_info >> 16; 2050 size = (io_info & SVM_IOIO_SIZE_MASK) >> SVM_IOIO_SIZE_SHIFT; 2051 2052 if (string) { 2053 if (sev_es_guest(vcpu->kvm)) 2054 return sev_es_string_io(svm, size, port, in); 2055 else 2056 return kvm_emulate_instruction(vcpu, 0); 2057 } 2058 2059 svm->next_rip = svm->vmcb->control.exit_info_2; 2060 2061 return kvm_fast_pio(vcpu, size, port, in); 2062 } 2063 2064 static int nmi_interception(struct kvm_vcpu *vcpu) 2065 { 2066 return 1; 2067 } 2068 2069 static int intr_interception(struct kvm_vcpu *vcpu) 2070 { 2071 ++vcpu->stat.irq_exits; 2072 return 1; 2073 } 2074 2075 static int vmload_vmsave_interception(struct kvm_vcpu *vcpu, bool vmload) 2076 { 2077 struct vcpu_svm *svm = to_svm(vcpu); 2078 struct vmcb *vmcb12; 2079 struct kvm_host_map map; 2080 int ret; 2081 2082 if (nested_svm_check_permissions(vcpu)) 2083 return 1; 2084 2085 ret = kvm_vcpu_map(vcpu, gpa_to_gfn(svm->vmcb->save.rax), &map); 2086 if (ret) { 2087 if (ret == -EINVAL) 2088 kvm_inject_gp(vcpu, 0); 2089 return 1; 2090 } 2091 2092 vmcb12 = map.hva; 2093 2094 ret = kvm_skip_emulated_instruction(vcpu); 2095 2096 if (vmload) { 2097 nested_svm_vmloadsave(vmcb12, svm->vmcb); 2098 svm->sysenter_eip_hi = 0; 2099 svm->sysenter_esp_hi = 0; 2100 } else 2101 nested_svm_vmloadsave(svm->vmcb, vmcb12); 2102 2103 kvm_vcpu_unmap(vcpu, &map, true); 2104 2105 return ret; 2106 } 2107 2108 static int vmload_interception(struct kvm_vcpu *vcpu) 2109 { 2110 return vmload_vmsave_interception(vcpu, true); 2111 } 2112 2113 static int vmsave_interception(struct kvm_vcpu *vcpu) 2114 { 2115 return vmload_vmsave_interception(vcpu, false); 2116 } 2117 2118 static int vmrun_interception(struct kvm_vcpu *vcpu) 2119 { 2120 if (nested_svm_check_permissions(vcpu)) 2121 return 1; 2122 2123 return nested_svm_vmrun(vcpu); 2124 } 2125 2126 enum { 2127 NONE_SVM_INSTR, 2128 SVM_INSTR_VMRUN, 2129 SVM_INSTR_VMLOAD, 2130 SVM_INSTR_VMSAVE, 2131 }; 2132 2133 /* Return NONE_SVM_INSTR if not SVM instrs, otherwise return decode result */ 2134 static int svm_instr_opcode(struct kvm_vcpu *vcpu) 2135 { 2136 struct x86_emulate_ctxt *ctxt = vcpu->arch.emulate_ctxt; 2137 2138 if (ctxt->b != 0x1 || ctxt->opcode_len != 2) 2139 return NONE_SVM_INSTR; 2140 2141 switch (ctxt->modrm) { 2142 case 0xd8: /* VMRUN */ 2143 return SVM_INSTR_VMRUN; 2144 case 0xda: /* VMLOAD */ 2145 return SVM_INSTR_VMLOAD; 2146 case 0xdb: /* VMSAVE */ 2147 return SVM_INSTR_VMSAVE; 2148 default: 2149 break; 2150 } 2151 2152 return NONE_SVM_INSTR; 2153 } 2154 2155 static int emulate_svm_instr(struct kvm_vcpu *vcpu, int opcode) 2156 { 2157 const int guest_mode_exit_codes[] = { 2158 [SVM_INSTR_VMRUN] = SVM_EXIT_VMRUN, 2159 [SVM_INSTR_VMLOAD] = SVM_EXIT_VMLOAD, 2160 [SVM_INSTR_VMSAVE] = SVM_EXIT_VMSAVE, 2161 }; 2162 int (*const svm_instr_handlers[])(struct kvm_vcpu *vcpu) = { 2163 [SVM_INSTR_VMRUN] = vmrun_interception, 2164 [SVM_INSTR_VMLOAD] = vmload_interception, 2165 [SVM_INSTR_VMSAVE] = vmsave_interception, 2166 }; 2167 struct vcpu_svm *svm = to_svm(vcpu); 2168 int ret; 2169 2170 if (is_guest_mode(vcpu)) { 2171 /* Returns '1' or -errno on failure, '0' on success. */ 2172 ret = nested_svm_simple_vmexit(svm, guest_mode_exit_codes[opcode]); 2173 if (ret) 2174 return ret; 2175 return 1; 2176 } 2177 return svm_instr_handlers[opcode](vcpu); 2178 } 2179 2180 /* 2181 * #GP handling code. Note that #GP can be triggered under the following two 2182 * cases: 2183 * 1) SVM VM-related instructions (VMRUN/VMSAVE/VMLOAD) that trigger #GP on 2184 * some AMD CPUs when EAX of these instructions are in the reserved memory 2185 * regions (e.g. SMM memory on host). 2186 * 2) VMware backdoor 2187 */ 2188 static int gp_interception(struct kvm_vcpu *vcpu) 2189 { 2190 struct vcpu_svm *svm = to_svm(vcpu); 2191 u32 error_code = svm->vmcb->control.exit_info_1; 2192 int opcode; 2193 2194 /* Both #GP cases have zero error_code */ 2195 if (error_code) 2196 goto reinject; 2197 2198 /* Decode the instruction for usage later */ 2199 if (x86_decode_emulated_instruction(vcpu, 0, NULL, 0) != EMULATION_OK) 2200 goto reinject; 2201 2202 opcode = svm_instr_opcode(vcpu); 2203 2204 if (opcode == NONE_SVM_INSTR) { 2205 if (!enable_vmware_backdoor) 2206 goto reinject; 2207 2208 /* 2209 * VMware backdoor emulation on #GP interception only handles 2210 * IN{S}, OUT{S}, and RDPMC. 2211 */ 2212 if (!is_guest_mode(vcpu)) 2213 return kvm_emulate_instruction(vcpu, 2214 EMULTYPE_VMWARE_GP | EMULTYPE_NO_DECODE); 2215 } else 2216 return emulate_svm_instr(vcpu, opcode); 2217 2218 reinject: 2219 kvm_queue_exception_e(vcpu, GP_VECTOR, error_code); 2220 return 1; 2221 } 2222 2223 void svm_set_gif(struct vcpu_svm *svm, bool value) 2224 { 2225 if (value) { 2226 /* 2227 * If VGIF is enabled, the STGI intercept is only added to 2228 * detect the opening of the SMI/NMI window; remove it now. 2229 * Likewise, clear the VINTR intercept, we will set it 2230 * again while processing KVM_REQ_EVENT if needed. 2231 */ 2232 if (vgif_enabled(svm)) 2233 svm_clr_intercept(svm, INTERCEPT_STGI); 2234 if (svm_is_intercept(svm, INTERCEPT_VINTR)) 2235 svm_clear_vintr(svm); 2236 2237 enable_gif(svm); 2238 if (svm->vcpu.arch.smi_pending || 2239 svm->vcpu.arch.nmi_pending || 2240 kvm_cpu_has_injectable_intr(&svm->vcpu)) 2241 kvm_make_request(KVM_REQ_EVENT, &svm->vcpu); 2242 } else { 2243 disable_gif(svm); 2244 2245 /* 2246 * After a CLGI no interrupts should come. But if vGIF is 2247 * in use, we still rely on the VINTR intercept (rather than 2248 * STGI) to detect an open interrupt window. 2249 */ 2250 if (!vgif_enabled(svm)) 2251 svm_clear_vintr(svm); 2252 } 2253 } 2254 2255 static int stgi_interception(struct kvm_vcpu *vcpu) 2256 { 2257 int ret; 2258 2259 if (nested_svm_check_permissions(vcpu)) 2260 return 1; 2261 2262 ret = kvm_skip_emulated_instruction(vcpu); 2263 svm_set_gif(to_svm(vcpu), true); 2264 return ret; 2265 } 2266 2267 static int clgi_interception(struct kvm_vcpu *vcpu) 2268 { 2269 int ret; 2270 2271 if (nested_svm_check_permissions(vcpu)) 2272 return 1; 2273 2274 ret = kvm_skip_emulated_instruction(vcpu); 2275 svm_set_gif(to_svm(vcpu), false); 2276 return ret; 2277 } 2278 2279 static int invlpga_interception(struct kvm_vcpu *vcpu) 2280 { 2281 gva_t gva = kvm_rax_read(vcpu); 2282 u32 asid = kvm_rcx_read(vcpu); 2283 2284 /* FIXME: Handle an address size prefix. */ 2285 if (!is_long_mode(vcpu)) 2286 gva = (u32)gva; 2287 2288 trace_kvm_invlpga(to_svm(vcpu)->vmcb->save.rip, asid, gva); 2289 2290 /* Let's treat INVLPGA the same as INVLPG (can be optimized!) */ 2291 kvm_mmu_invlpg(vcpu, gva); 2292 2293 return kvm_skip_emulated_instruction(vcpu); 2294 } 2295 2296 static int skinit_interception(struct kvm_vcpu *vcpu) 2297 { 2298 trace_kvm_skinit(to_svm(vcpu)->vmcb->save.rip, kvm_rax_read(vcpu)); 2299 2300 kvm_queue_exception(vcpu, UD_VECTOR); 2301 return 1; 2302 } 2303 2304 static int task_switch_interception(struct kvm_vcpu *vcpu) 2305 { 2306 struct vcpu_svm *svm = to_svm(vcpu); 2307 u16 tss_selector; 2308 int reason; 2309 int int_type = svm->vmcb->control.exit_int_info & 2310 SVM_EXITINTINFO_TYPE_MASK; 2311 int int_vec = svm->vmcb->control.exit_int_info & SVM_EVTINJ_VEC_MASK; 2312 uint32_t type = 2313 svm->vmcb->control.exit_int_info & SVM_EXITINTINFO_TYPE_MASK; 2314 uint32_t idt_v = 2315 svm->vmcb->control.exit_int_info & SVM_EXITINTINFO_VALID; 2316 bool has_error_code = false; 2317 u32 error_code = 0; 2318 2319 tss_selector = (u16)svm->vmcb->control.exit_info_1; 2320 2321 if (svm->vmcb->control.exit_info_2 & 2322 (1ULL << SVM_EXITINFOSHIFT_TS_REASON_IRET)) 2323 reason = TASK_SWITCH_IRET; 2324 else if (svm->vmcb->control.exit_info_2 & 2325 (1ULL << SVM_EXITINFOSHIFT_TS_REASON_JMP)) 2326 reason = TASK_SWITCH_JMP; 2327 else if (idt_v) 2328 reason = TASK_SWITCH_GATE; 2329 else 2330 reason = TASK_SWITCH_CALL; 2331 2332 if (reason == TASK_SWITCH_GATE) { 2333 switch (type) { 2334 case SVM_EXITINTINFO_TYPE_NMI: 2335 vcpu->arch.nmi_injected = false; 2336 break; 2337 case SVM_EXITINTINFO_TYPE_EXEPT: 2338 if (svm->vmcb->control.exit_info_2 & 2339 (1ULL << SVM_EXITINFOSHIFT_TS_HAS_ERROR_CODE)) { 2340 has_error_code = true; 2341 error_code = 2342 (u32)svm->vmcb->control.exit_info_2; 2343 } 2344 kvm_clear_exception_queue(vcpu); 2345 break; 2346 case SVM_EXITINTINFO_TYPE_INTR: 2347 kvm_clear_interrupt_queue(vcpu); 2348 break; 2349 default: 2350 break; 2351 } 2352 } 2353 2354 if (reason != TASK_SWITCH_GATE || 2355 int_type == SVM_EXITINTINFO_TYPE_SOFT || 2356 (int_type == SVM_EXITINTINFO_TYPE_EXEPT && 2357 (int_vec == OF_VECTOR || int_vec == BP_VECTOR))) { 2358 if (!skip_emulated_instruction(vcpu)) 2359 return 0; 2360 } 2361 2362 if (int_type != SVM_EXITINTINFO_TYPE_SOFT) 2363 int_vec = -1; 2364 2365 return kvm_task_switch(vcpu, tss_selector, int_vec, reason, 2366 has_error_code, error_code); 2367 } 2368 2369 static int iret_interception(struct kvm_vcpu *vcpu) 2370 { 2371 struct vcpu_svm *svm = to_svm(vcpu); 2372 2373 ++vcpu->stat.nmi_window_exits; 2374 vcpu->arch.hflags |= HF_IRET_MASK; 2375 if (!sev_es_guest(vcpu->kvm)) { 2376 svm_clr_intercept(svm, INTERCEPT_IRET); 2377 svm->nmi_iret_rip = kvm_rip_read(vcpu); 2378 } 2379 kvm_make_request(KVM_REQ_EVENT, vcpu); 2380 return 1; 2381 } 2382 2383 static int invlpg_interception(struct kvm_vcpu *vcpu) 2384 { 2385 if (!static_cpu_has(X86_FEATURE_DECODEASSISTS)) 2386 return kvm_emulate_instruction(vcpu, 0); 2387 2388 kvm_mmu_invlpg(vcpu, to_svm(vcpu)->vmcb->control.exit_info_1); 2389 return kvm_skip_emulated_instruction(vcpu); 2390 } 2391 2392 static int emulate_on_interception(struct kvm_vcpu *vcpu) 2393 { 2394 return kvm_emulate_instruction(vcpu, 0); 2395 } 2396 2397 static int rsm_interception(struct kvm_vcpu *vcpu) 2398 { 2399 return kvm_emulate_instruction_from_buffer(vcpu, rsm_ins_bytes, 2); 2400 } 2401 2402 static bool check_selective_cr0_intercepted(struct kvm_vcpu *vcpu, 2403 unsigned long val) 2404 { 2405 struct vcpu_svm *svm = to_svm(vcpu); 2406 unsigned long cr0 = vcpu->arch.cr0; 2407 bool ret = false; 2408 2409 if (!is_guest_mode(vcpu) || 2410 (!(vmcb_is_intercept(&svm->nested.ctl, INTERCEPT_SELECTIVE_CR0)))) 2411 return false; 2412 2413 cr0 &= ~SVM_CR0_SELECTIVE_MASK; 2414 val &= ~SVM_CR0_SELECTIVE_MASK; 2415 2416 if (cr0 ^ val) { 2417 svm->vmcb->control.exit_code = SVM_EXIT_CR0_SEL_WRITE; 2418 ret = (nested_svm_exit_handled(svm) == NESTED_EXIT_DONE); 2419 } 2420 2421 return ret; 2422 } 2423 2424 #define CR_VALID (1ULL << 63) 2425 2426 static int cr_interception(struct kvm_vcpu *vcpu) 2427 { 2428 struct vcpu_svm *svm = to_svm(vcpu); 2429 int reg, cr; 2430 unsigned long val; 2431 int err; 2432 2433 if (!static_cpu_has(X86_FEATURE_DECODEASSISTS)) 2434 return emulate_on_interception(vcpu); 2435 2436 if (unlikely((svm->vmcb->control.exit_info_1 & CR_VALID) == 0)) 2437 return emulate_on_interception(vcpu); 2438 2439 reg = svm->vmcb->control.exit_info_1 & SVM_EXITINFO_REG_MASK; 2440 if (svm->vmcb->control.exit_code == SVM_EXIT_CR0_SEL_WRITE) 2441 cr = SVM_EXIT_WRITE_CR0 - SVM_EXIT_READ_CR0; 2442 else 2443 cr = svm->vmcb->control.exit_code - SVM_EXIT_READ_CR0; 2444 2445 err = 0; 2446 if (cr >= 16) { /* mov to cr */ 2447 cr -= 16; 2448 val = kvm_register_read(vcpu, reg); 2449 trace_kvm_cr_write(cr, val); 2450 switch (cr) { 2451 case 0: 2452 if (!check_selective_cr0_intercepted(vcpu, val)) 2453 err = kvm_set_cr0(vcpu, val); 2454 else 2455 return 1; 2456 2457 break; 2458 case 3: 2459 err = kvm_set_cr3(vcpu, val); 2460 break; 2461 case 4: 2462 err = kvm_set_cr4(vcpu, val); 2463 break; 2464 case 8: 2465 err = kvm_set_cr8(vcpu, val); 2466 break; 2467 default: 2468 WARN(1, "unhandled write to CR%d", cr); 2469 kvm_queue_exception(vcpu, UD_VECTOR); 2470 return 1; 2471 } 2472 } else { /* mov from cr */ 2473 switch (cr) { 2474 case 0: 2475 val = kvm_read_cr0(vcpu); 2476 break; 2477 case 2: 2478 val = vcpu->arch.cr2; 2479 break; 2480 case 3: 2481 val = kvm_read_cr3(vcpu); 2482 break; 2483 case 4: 2484 val = kvm_read_cr4(vcpu); 2485 break; 2486 case 8: 2487 val = kvm_get_cr8(vcpu); 2488 break; 2489 default: 2490 WARN(1, "unhandled read from CR%d", cr); 2491 kvm_queue_exception(vcpu, UD_VECTOR); 2492 return 1; 2493 } 2494 kvm_register_write(vcpu, reg, val); 2495 trace_kvm_cr_read(cr, val); 2496 } 2497 return kvm_complete_insn_gp(vcpu, err); 2498 } 2499 2500 static int cr_trap(struct kvm_vcpu *vcpu) 2501 { 2502 struct vcpu_svm *svm = to_svm(vcpu); 2503 unsigned long old_value, new_value; 2504 unsigned int cr; 2505 int ret = 0; 2506 2507 new_value = (unsigned long)svm->vmcb->control.exit_info_1; 2508 2509 cr = svm->vmcb->control.exit_code - SVM_EXIT_CR0_WRITE_TRAP; 2510 switch (cr) { 2511 case 0: 2512 old_value = kvm_read_cr0(vcpu); 2513 svm_set_cr0(vcpu, new_value); 2514 2515 kvm_post_set_cr0(vcpu, old_value, new_value); 2516 break; 2517 case 4: 2518 old_value = kvm_read_cr4(vcpu); 2519 svm_set_cr4(vcpu, new_value); 2520 2521 kvm_post_set_cr4(vcpu, old_value, new_value); 2522 break; 2523 case 8: 2524 ret = kvm_set_cr8(vcpu, new_value); 2525 break; 2526 default: 2527 WARN(1, "unhandled CR%d write trap", cr); 2528 kvm_queue_exception(vcpu, UD_VECTOR); 2529 return 1; 2530 } 2531 2532 return kvm_complete_insn_gp(vcpu, ret); 2533 } 2534 2535 static int dr_interception(struct kvm_vcpu *vcpu) 2536 { 2537 struct vcpu_svm *svm = to_svm(vcpu); 2538 int reg, dr; 2539 unsigned long val; 2540 int err = 0; 2541 2542 if (vcpu->guest_debug == 0) { 2543 /* 2544 * No more DR vmexits; force a reload of the debug registers 2545 * and reenter on this instruction. The next vmexit will 2546 * retrieve the full state of the debug registers. 2547 */ 2548 clr_dr_intercepts(svm); 2549 vcpu->arch.switch_db_regs |= KVM_DEBUGREG_WONT_EXIT; 2550 return 1; 2551 } 2552 2553 if (!boot_cpu_has(X86_FEATURE_DECODEASSISTS)) 2554 return emulate_on_interception(vcpu); 2555 2556 reg = svm->vmcb->control.exit_info_1 & SVM_EXITINFO_REG_MASK; 2557 dr = svm->vmcb->control.exit_code - SVM_EXIT_READ_DR0; 2558 if (dr >= 16) { /* mov to DRn */ 2559 dr -= 16; 2560 val = kvm_register_read(vcpu, reg); 2561 err = kvm_set_dr(vcpu, dr, val); 2562 } else { 2563 kvm_get_dr(vcpu, dr, &val); 2564 kvm_register_write(vcpu, reg, val); 2565 } 2566 2567 return kvm_complete_insn_gp(vcpu, err); 2568 } 2569 2570 static int cr8_write_interception(struct kvm_vcpu *vcpu) 2571 { 2572 int r; 2573 2574 u8 cr8_prev = kvm_get_cr8(vcpu); 2575 /* instruction emulation calls kvm_set_cr8() */ 2576 r = cr_interception(vcpu); 2577 if (lapic_in_kernel(vcpu)) 2578 return r; 2579 if (cr8_prev <= kvm_get_cr8(vcpu)) 2580 return r; 2581 vcpu->run->exit_reason = KVM_EXIT_SET_TPR; 2582 return 0; 2583 } 2584 2585 static int efer_trap(struct kvm_vcpu *vcpu) 2586 { 2587 struct msr_data msr_info; 2588 int ret; 2589 2590 /* 2591 * Clear the EFER_SVME bit from EFER. The SVM code always sets this 2592 * bit in svm_set_efer(), but __kvm_valid_efer() checks it against 2593 * whether the guest has X86_FEATURE_SVM - this avoids a failure if 2594 * the guest doesn't have X86_FEATURE_SVM. 2595 */ 2596 msr_info.host_initiated = false; 2597 msr_info.index = MSR_EFER; 2598 msr_info.data = to_svm(vcpu)->vmcb->control.exit_info_1 & ~EFER_SVME; 2599 ret = kvm_set_msr_common(vcpu, &msr_info); 2600 2601 return kvm_complete_insn_gp(vcpu, ret); 2602 } 2603 2604 static int svm_get_msr_feature(struct kvm_msr_entry *msr) 2605 { 2606 msr->data = 0; 2607 2608 switch (msr->index) { 2609 case MSR_F10H_DECFG: 2610 if (boot_cpu_has(X86_FEATURE_LFENCE_RDTSC)) 2611 msr->data |= MSR_F10H_DECFG_LFENCE_SERIALIZE; 2612 break; 2613 case MSR_IA32_PERF_CAPABILITIES: 2614 return 0; 2615 default: 2616 return KVM_MSR_RET_INVALID; 2617 } 2618 2619 return 0; 2620 } 2621 2622 static int svm_get_msr(struct kvm_vcpu *vcpu, struct msr_data *msr_info) 2623 { 2624 struct vcpu_svm *svm = to_svm(vcpu); 2625 2626 switch (msr_info->index) { 2627 case MSR_STAR: 2628 msr_info->data = svm->vmcb01.ptr->save.star; 2629 break; 2630 #ifdef CONFIG_X86_64 2631 case MSR_LSTAR: 2632 msr_info->data = svm->vmcb01.ptr->save.lstar; 2633 break; 2634 case MSR_CSTAR: 2635 msr_info->data = svm->vmcb01.ptr->save.cstar; 2636 break; 2637 case MSR_KERNEL_GS_BASE: 2638 msr_info->data = svm->vmcb01.ptr->save.kernel_gs_base; 2639 break; 2640 case MSR_SYSCALL_MASK: 2641 msr_info->data = svm->vmcb01.ptr->save.sfmask; 2642 break; 2643 #endif 2644 case MSR_IA32_SYSENTER_CS: 2645 msr_info->data = svm->vmcb01.ptr->save.sysenter_cs; 2646 break; 2647 case MSR_IA32_SYSENTER_EIP: 2648 msr_info->data = (u32)svm->vmcb01.ptr->save.sysenter_eip; 2649 if (guest_cpuid_is_intel(vcpu)) 2650 msr_info->data |= (u64)svm->sysenter_eip_hi << 32; 2651 break; 2652 case MSR_IA32_SYSENTER_ESP: 2653 msr_info->data = svm->vmcb01.ptr->save.sysenter_esp; 2654 if (guest_cpuid_is_intel(vcpu)) 2655 msr_info->data |= (u64)svm->sysenter_esp_hi << 32; 2656 break; 2657 case MSR_TSC_AUX: 2658 if (!boot_cpu_has(X86_FEATURE_RDTSCP)) 2659 return 1; 2660 if (!msr_info->host_initiated && 2661 !guest_cpuid_has(vcpu, X86_FEATURE_RDTSCP)) 2662 return 1; 2663 msr_info->data = svm->tsc_aux; 2664 break; 2665 /* 2666 * Nobody will change the following 5 values in the VMCB so we can 2667 * safely return them on rdmsr. They will always be 0 until LBRV is 2668 * implemented. 2669 */ 2670 case MSR_IA32_DEBUGCTLMSR: 2671 msr_info->data = svm->vmcb->save.dbgctl; 2672 break; 2673 case MSR_IA32_LASTBRANCHFROMIP: 2674 msr_info->data = svm->vmcb->save.br_from; 2675 break; 2676 case MSR_IA32_LASTBRANCHTOIP: 2677 msr_info->data = svm->vmcb->save.br_to; 2678 break; 2679 case MSR_IA32_LASTINTFROMIP: 2680 msr_info->data = svm->vmcb->save.last_excp_from; 2681 break; 2682 case MSR_IA32_LASTINTTOIP: 2683 msr_info->data = svm->vmcb->save.last_excp_to; 2684 break; 2685 case MSR_VM_HSAVE_PA: 2686 msr_info->data = svm->nested.hsave_msr; 2687 break; 2688 case MSR_VM_CR: 2689 msr_info->data = svm->nested.vm_cr_msr; 2690 break; 2691 case MSR_IA32_SPEC_CTRL: 2692 if (!msr_info->host_initiated && 2693 !guest_has_spec_ctrl_msr(vcpu)) 2694 return 1; 2695 2696 if (boot_cpu_has(X86_FEATURE_V_SPEC_CTRL)) 2697 msr_info->data = svm->vmcb->save.spec_ctrl; 2698 else 2699 msr_info->data = svm->spec_ctrl; 2700 break; 2701 case MSR_AMD64_VIRT_SPEC_CTRL: 2702 if (!msr_info->host_initiated && 2703 !guest_cpuid_has(vcpu, X86_FEATURE_VIRT_SSBD)) 2704 return 1; 2705 2706 msr_info->data = svm->virt_spec_ctrl; 2707 break; 2708 case MSR_F15H_IC_CFG: { 2709 2710 int family, model; 2711 2712 family = guest_cpuid_family(vcpu); 2713 model = guest_cpuid_model(vcpu); 2714 2715 if (family < 0 || model < 0) 2716 return kvm_get_msr_common(vcpu, msr_info); 2717 2718 msr_info->data = 0; 2719 2720 if (family == 0x15 && 2721 (model >= 0x2 && model < 0x20)) 2722 msr_info->data = 0x1E; 2723 } 2724 break; 2725 case MSR_F10H_DECFG: 2726 msr_info->data = svm->msr_decfg; 2727 break; 2728 default: 2729 return kvm_get_msr_common(vcpu, msr_info); 2730 } 2731 return 0; 2732 } 2733 2734 static int svm_complete_emulated_msr(struct kvm_vcpu *vcpu, int err) 2735 { 2736 struct vcpu_svm *svm = to_svm(vcpu); 2737 if (!err || !sev_es_guest(vcpu->kvm) || WARN_ON_ONCE(!svm->ghcb)) 2738 return kvm_complete_insn_gp(vcpu, err); 2739 2740 ghcb_set_sw_exit_info_1(svm->ghcb, 1); 2741 ghcb_set_sw_exit_info_2(svm->ghcb, 2742 X86_TRAP_GP | 2743 SVM_EVTINJ_TYPE_EXEPT | 2744 SVM_EVTINJ_VALID); 2745 return 1; 2746 } 2747 2748 static int svm_set_vm_cr(struct kvm_vcpu *vcpu, u64 data) 2749 { 2750 struct vcpu_svm *svm = to_svm(vcpu); 2751 int svm_dis, chg_mask; 2752 2753 if (data & ~SVM_VM_CR_VALID_MASK) 2754 return 1; 2755 2756 chg_mask = SVM_VM_CR_VALID_MASK; 2757 2758 if (svm->nested.vm_cr_msr & SVM_VM_CR_SVM_DIS_MASK) 2759 chg_mask &= ~(SVM_VM_CR_SVM_LOCK_MASK | SVM_VM_CR_SVM_DIS_MASK); 2760 2761 svm->nested.vm_cr_msr &= ~chg_mask; 2762 svm->nested.vm_cr_msr |= (data & chg_mask); 2763 2764 svm_dis = svm->nested.vm_cr_msr & SVM_VM_CR_SVM_DIS_MASK; 2765 2766 /* check for svm_disable while efer.svme is set */ 2767 if (svm_dis && (vcpu->arch.efer & EFER_SVME)) 2768 return 1; 2769 2770 return 0; 2771 } 2772 2773 static int svm_set_msr(struct kvm_vcpu *vcpu, struct msr_data *msr) 2774 { 2775 struct vcpu_svm *svm = to_svm(vcpu); 2776 int r; 2777 2778 u32 ecx = msr->index; 2779 u64 data = msr->data; 2780 switch (ecx) { 2781 case MSR_IA32_CR_PAT: 2782 if (!kvm_mtrr_valid(vcpu, MSR_IA32_CR_PAT, data)) 2783 return 1; 2784 vcpu->arch.pat = data; 2785 svm->vmcb01.ptr->save.g_pat = data; 2786 if (is_guest_mode(vcpu)) 2787 nested_vmcb02_compute_g_pat(svm); 2788 vmcb_mark_dirty(svm->vmcb, VMCB_NPT); 2789 break; 2790 case MSR_IA32_SPEC_CTRL: 2791 if (!msr->host_initiated && 2792 !guest_has_spec_ctrl_msr(vcpu)) 2793 return 1; 2794 2795 if (kvm_spec_ctrl_test_value(data)) 2796 return 1; 2797 2798 if (boot_cpu_has(X86_FEATURE_V_SPEC_CTRL)) 2799 svm->vmcb->save.spec_ctrl = data; 2800 else 2801 svm->spec_ctrl = data; 2802 if (!data) 2803 break; 2804 2805 /* 2806 * For non-nested: 2807 * When it's written (to non-zero) for the first time, pass 2808 * it through. 2809 * 2810 * For nested: 2811 * The handling of the MSR bitmap for L2 guests is done in 2812 * nested_svm_vmrun_msrpm. 2813 * We update the L1 MSR bit as well since it will end up 2814 * touching the MSR anyway now. 2815 */ 2816 set_msr_interception(vcpu, svm->msrpm, MSR_IA32_SPEC_CTRL, 1, 1); 2817 break; 2818 case MSR_IA32_PRED_CMD: 2819 if (!msr->host_initiated && 2820 !guest_has_pred_cmd_msr(vcpu)) 2821 return 1; 2822 2823 if (data & ~PRED_CMD_IBPB) 2824 return 1; 2825 if (!boot_cpu_has(X86_FEATURE_IBPB)) 2826 return 1; 2827 if (!data) 2828 break; 2829 2830 wrmsrl(MSR_IA32_PRED_CMD, PRED_CMD_IBPB); 2831 set_msr_interception(vcpu, svm->msrpm, MSR_IA32_PRED_CMD, 0, 1); 2832 break; 2833 case MSR_AMD64_VIRT_SPEC_CTRL: 2834 if (!msr->host_initiated && 2835 !guest_cpuid_has(vcpu, X86_FEATURE_VIRT_SSBD)) 2836 return 1; 2837 2838 if (data & ~SPEC_CTRL_SSBD) 2839 return 1; 2840 2841 svm->virt_spec_ctrl = data; 2842 break; 2843 case MSR_STAR: 2844 svm->vmcb01.ptr->save.star = data; 2845 break; 2846 #ifdef CONFIG_X86_64 2847 case MSR_LSTAR: 2848 svm->vmcb01.ptr->save.lstar = data; 2849 break; 2850 case MSR_CSTAR: 2851 svm->vmcb01.ptr->save.cstar = data; 2852 break; 2853 case MSR_KERNEL_GS_BASE: 2854 svm->vmcb01.ptr->save.kernel_gs_base = data; 2855 break; 2856 case MSR_SYSCALL_MASK: 2857 svm->vmcb01.ptr->save.sfmask = data; 2858 break; 2859 #endif 2860 case MSR_IA32_SYSENTER_CS: 2861 svm->vmcb01.ptr->save.sysenter_cs = data; 2862 break; 2863 case MSR_IA32_SYSENTER_EIP: 2864 svm->vmcb01.ptr->save.sysenter_eip = (u32)data; 2865 /* 2866 * We only intercept the MSR_IA32_SYSENTER_{EIP|ESP} msrs 2867 * when we spoof an Intel vendor ID (for cross vendor migration). 2868 * In this case we use this intercept to track the high 2869 * 32 bit part of these msrs to support Intel's 2870 * implementation of SYSENTER/SYSEXIT. 2871 */ 2872 svm->sysenter_eip_hi = guest_cpuid_is_intel(vcpu) ? (data >> 32) : 0; 2873 break; 2874 case MSR_IA32_SYSENTER_ESP: 2875 svm->vmcb01.ptr->save.sysenter_esp = (u32)data; 2876 svm->sysenter_esp_hi = guest_cpuid_is_intel(vcpu) ? (data >> 32) : 0; 2877 break; 2878 case MSR_TSC_AUX: 2879 if (!boot_cpu_has(X86_FEATURE_RDTSCP)) 2880 return 1; 2881 2882 if (!msr->host_initiated && 2883 !guest_cpuid_has(vcpu, X86_FEATURE_RDTSCP)) 2884 return 1; 2885 2886 /* 2887 * Per Intel's SDM, bits 63:32 are reserved, but AMD's APM has 2888 * incomplete and conflicting architectural behavior. Current 2889 * AMD CPUs completely ignore bits 63:32, i.e. they aren't 2890 * reserved and always read as zeros. Emulate AMD CPU behavior 2891 * to avoid explosions if the vCPU is migrated from an AMD host 2892 * to an Intel host. 2893 */ 2894 data = (u32)data; 2895 2896 /* 2897 * TSC_AUX is usually changed only during boot and never read 2898 * directly. Intercept TSC_AUX instead of exposing it to the 2899 * guest via direct_access_msrs, and switch it via user return. 2900 */ 2901 preempt_disable(); 2902 r = kvm_set_user_return_msr(TSC_AUX_URET_SLOT, data, -1ull); 2903 preempt_enable(); 2904 if (r) 2905 return 1; 2906 2907 svm->tsc_aux = data; 2908 break; 2909 case MSR_IA32_DEBUGCTLMSR: 2910 if (!boot_cpu_has(X86_FEATURE_LBRV)) { 2911 vcpu_unimpl(vcpu, "%s: MSR_IA32_DEBUGCTL 0x%llx, nop\n", 2912 __func__, data); 2913 break; 2914 } 2915 if (data & DEBUGCTL_RESERVED_BITS) 2916 return 1; 2917 2918 svm->vmcb->save.dbgctl = data; 2919 vmcb_mark_dirty(svm->vmcb, VMCB_LBR); 2920 if (data & (1ULL<<0)) 2921 svm_enable_lbrv(vcpu); 2922 else 2923 svm_disable_lbrv(vcpu); 2924 break; 2925 case MSR_VM_HSAVE_PA: 2926 svm->nested.hsave_msr = data; 2927 break; 2928 case MSR_VM_CR: 2929 return svm_set_vm_cr(vcpu, data); 2930 case MSR_VM_IGNNE: 2931 vcpu_unimpl(vcpu, "unimplemented wrmsr: 0x%x data 0x%llx\n", ecx, data); 2932 break; 2933 case MSR_F10H_DECFG: { 2934 struct kvm_msr_entry msr_entry; 2935 2936 msr_entry.index = msr->index; 2937 if (svm_get_msr_feature(&msr_entry)) 2938 return 1; 2939 2940 /* Check the supported bits */ 2941 if (data & ~msr_entry.data) 2942 return 1; 2943 2944 /* Don't allow the guest to change a bit, #GP */ 2945 if (!msr->host_initiated && (data ^ msr_entry.data)) 2946 return 1; 2947 2948 svm->msr_decfg = data; 2949 break; 2950 } 2951 case MSR_IA32_APICBASE: 2952 if (kvm_vcpu_apicv_active(vcpu)) 2953 avic_update_vapic_bar(to_svm(vcpu), data); 2954 fallthrough; 2955 default: 2956 return kvm_set_msr_common(vcpu, msr); 2957 } 2958 return 0; 2959 } 2960 2961 static int msr_interception(struct kvm_vcpu *vcpu) 2962 { 2963 if (to_svm(vcpu)->vmcb->control.exit_info_1) 2964 return kvm_emulate_wrmsr(vcpu); 2965 else 2966 return kvm_emulate_rdmsr(vcpu); 2967 } 2968 2969 static int interrupt_window_interception(struct kvm_vcpu *vcpu) 2970 { 2971 kvm_make_request(KVM_REQ_EVENT, vcpu); 2972 svm_clear_vintr(to_svm(vcpu)); 2973 2974 /* 2975 * For AVIC, the only reason to end up here is ExtINTs. 2976 * In this case AVIC was temporarily disabled for 2977 * requesting the IRQ window and we have to re-enable it. 2978 */ 2979 svm_toggle_avic_for_irq_window(vcpu, true); 2980 2981 ++vcpu->stat.irq_window_exits; 2982 return 1; 2983 } 2984 2985 static int pause_interception(struct kvm_vcpu *vcpu) 2986 { 2987 bool in_kernel; 2988 2989 /* 2990 * CPL is not made available for an SEV-ES guest, therefore 2991 * vcpu->arch.preempted_in_kernel can never be true. Just 2992 * set in_kernel to false as well. 2993 */ 2994 in_kernel = !sev_es_guest(vcpu->kvm) && svm_get_cpl(vcpu) == 0; 2995 2996 if (!kvm_pause_in_guest(vcpu->kvm)) 2997 grow_ple_window(vcpu); 2998 2999 kvm_vcpu_on_spin(vcpu, in_kernel); 3000 return kvm_skip_emulated_instruction(vcpu); 3001 } 3002 3003 static int invpcid_interception(struct kvm_vcpu *vcpu) 3004 { 3005 struct vcpu_svm *svm = to_svm(vcpu); 3006 unsigned long type; 3007 gva_t gva; 3008 3009 if (!guest_cpuid_has(vcpu, X86_FEATURE_INVPCID)) { 3010 kvm_queue_exception(vcpu, UD_VECTOR); 3011 return 1; 3012 } 3013 3014 /* 3015 * For an INVPCID intercept: 3016 * EXITINFO1 provides the linear address of the memory operand. 3017 * EXITINFO2 provides the contents of the register operand. 3018 */ 3019 type = svm->vmcb->control.exit_info_2; 3020 gva = svm->vmcb->control.exit_info_1; 3021 3022 if (type > 3) { 3023 kvm_inject_gp(vcpu, 0); 3024 return 1; 3025 } 3026 3027 return kvm_handle_invpcid(vcpu, type, gva); 3028 } 3029 3030 static int (*const svm_exit_handlers[])(struct kvm_vcpu *vcpu) = { 3031 [SVM_EXIT_READ_CR0] = cr_interception, 3032 [SVM_EXIT_READ_CR3] = cr_interception, 3033 [SVM_EXIT_READ_CR4] = cr_interception, 3034 [SVM_EXIT_READ_CR8] = cr_interception, 3035 [SVM_EXIT_CR0_SEL_WRITE] = cr_interception, 3036 [SVM_EXIT_WRITE_CR0] = cr_interception, 3037 [SVM_EXIT_WRITE_CR3] = cr_interception, 3038 [SVM_EXIT_WRITE_CR4] = cr_interception, 3039 [SVM_EXIT_WRITE_CR8] = cr8_write_interception, 3040 [SVM_EXIT_READ_DR0] = dr_interception, 3041 [SVM_EXIT_READ_DR1] = dr_interception, 3042 [SVM_EXIT_READ_DR2] = dr_interception, 3043 [SVM_EXIT_READ_DR3] = dr_interception, 3044 [SVM_EXIT_READ_DR4] = dr_interception, 3045 [SVM_EXIT_READ_DR5] = dr_interception, 3046 [SVM_EXIT_READ_DR6] = dr_interception, 3047 [SVM_EXIT_READ_DR7] = dr_interception, 3048 [SVM_EXIT_WRITE_DR0] = dr_interception, 3049 [SVM_EXIT_WRITE_DR1] = dr_interception, 3050 [SVM_EXIT_WRITE_DR2] = dr_interception, 3051 [SVM_EXIT_WRITE_DR3] = dr_interception, 3052 [SVM_EXIT_WRITE_DR4] = dr_interception, 3053 [SVM_EXIT_WRITE_DR5] = dr_interception, 3054 [SVM_EXIT_WRITE_DR6] = dr_interception, 3055 [SVM_EXIT_WRITE_DR7] = dr_interception, 3056 [SVM_EXIT_EXCP_BASE + DB_VECTOR] = db_interception, 3057 [SVM_EXIT_EXCP_BASE + BP_VECTOR] = bp_interception, 3058 [SVM_EXIT_EXCP_BASE + UD_VECTOR] = ud_interception, 3059 [SVM_EXIT_EXCP_BASE + PF_VECTOR] = pf_interception, 3060 [SVM_EXIT_EXCP_BASE + MC_VECTOR] = mc_interception, 3061 [SVM_EXIT_EXCP_BASE + AC_VECTOR] = ac_interception, 3062 [SVM_EXIT_EXCP_BASE + GP_VECTOR] = gp_interception, 3063 [SVM_EXIT_INTR] = intr_interception, 3064 [SVM_EXIT_NMI] = nmi_interception, 3065 [SVM_EXIT_SMI] = kvm_emulate_as_nop, 3066 [SVM_EXIT_INIT] = kvm_emulate_as_nop, 3067 [SVM_EXIT_VINTR] = interrupt_window_interception, 3068 [SVM_EXIT_RDPMC] = kvm_emulate_rdpmc, 3069 [SVM_EXIT_CPUID] = kvm_emulate_cpuid, 3070 [SVM_EXIT_IRET] = iret_interception, 3071 [SVM_EXIT_INVD] = kvm_emulate_invd, 3072 [SVM_EXIT_PAUSE] = pause_interception, 3073 [SVM_EXIT_HLT] = kvm_emulate_halt, 3074 [SVM_EXIT_INVLPG] = invlpg_interception, 3075 [SVM_EXIT_INVLPGA] = invlpga_interception, 3076 [SVM_EXIT_IOIO] = io_interception, 3077 [SVM_EXIT_MSR] = msr_interception, 3078 [SVM_EXIT_TASK_SWITCH] = task_switch_interception, 3079 [SVM_EXIT_SHUTDOWN] = shutdown_interception, 3080 [SVM_EXIT_VMRUN] = vmrun_interception, 3081 [SVM_EXIT_VMMCALL] = kvm_emulate_hypercall, 3082 [SVM_EXIT_VMLOAD] = vmload_interception, 3083 [SVM_EXIT_VMSAVE] = vmsave_interception, 3084 [SVM_EXIT_STGI] = stgi_interception, 3085 [SVM_EXIT_CLGI] = clgi_interception, 3086 [SVM_EXIT_SKINIT] = skinit_interception, 3087 [SVM_EXIT_WBINVD] = kvm_emulate_wbinvd, 3088 [SVM_EXIT_MONITOR] = kvm_emulate_monitor, 3089 [SVM_EXIT_MWAIT] = kvm_emulate_mwait, 3090 [SVM_EXIT_XSETBV] = kvm_emulate_xsetbv, 3091 [SVM_EXIT_RDPRU] = kvm_handle_invalid_op, 3092 [SVM_EXIT_EFER_WRITE_TRAP] = efer_trap, 3093 [SVM_EXIT_CR0_WRITE_TRAP] = cr_trap, 3094 [SVM_EXIT_CR4_WRITE_TRAP] = cr_trap, 3095 [SVM_EXIT_CR8_WRITE_TRAP] = cr_trap, 3096 [SVM_EXIT_INVPCID] = invpcid_interception, 3097 [SVM_EXIT_NPF] = npf_interception, 3098 [SVM_EXIT_RSM] = rsm_interception, 3099 [SVM_EXIT_AVIC_INCOMPLETE_IPI] = avic_incomplete_ipi_interception, 3100 [SVM_EXIT_AVIC_UNACCELERATED_ACCESS] = avic_unaccelerated_access_interception, 3101 [SVM_EXIT_VMGEXIT] = sev_handle_vmgexit, 3102 }; 3103 3104 static void dump_vmcb(struct kvm_vcpu *vcpu) 3105 { 3106 struct vcpu_svm *svm = to_svm(vcpu); 3107 struct vmcb_control_area *control = &svm->vmcb->control; 3108 struct vmcb_save_area *save = &svm->vmcb->save; 3109 struct vmcb_save_area *save01 = &svm->vmcb01.ptr->save; 3110 3111 if (!dump_invalid_vmcb) { 3112 pr_warn_ratelimited("set kvm_amd.dump_invalid_vmcb=1 to dump internal KVM state.\n"); 3113 return; 3114 } 3115 3116 pr_err("VMCB Control Area:\n"); 3117 pr_err("%-20s%04x\n", "cr_read:", control->intercepts[INTERCEPT_CR] & 0xffff); 3118 pr_err("%-20s%04x\n", "cr_write:", control->intercepts[INTERCEPT_CR] >> 16); 3119 pr_err("%-20s%04x\n", "dr_read:", control->intercepts[INTERCEPT_DR] & 0xffff); 3120 pr_err("%-20s%04x\n", "dr_write:", control->intercepts[INTERCEPT_DR] >> 16); 3121 pr_err("%-20s%08x\n", "exceptions:", control->intercepts[INTERCEPT_EXCEPTION]); 3122 pr_err("%-20s%08x %08x\n", "intercepts:", 3123 control->intercepts[INTERCEPT_WORD3], 3124 control->intercepts[INTERCEPT_WORD4]); 3125 pr_err("%-20s%d\n", "pause filter count:", control->pause_filter_count); 3126 pr_err("%-20s%d\n", "pause filter threshold:", 3127 control->pause_filter_thresh); 3128 pr_err("%-20s%016llx\n", "iopm_base_pa:", control->iopm_base_pa); 3129 pr_err("%-20s%016llx\n", "msrpm_base_pa:", control->msrpm_base_pa); 3130 pr_err("%-20s%016llx\n", "tsc_offset:", control->tsc_offset); 3131 pr_err("%-20s%d\n", "asid:", control->asid); 3132 pr_err("%-20s%d\n", "tlb_ctl:", control->tlb_ctl); 3133 pr_err("%-20s%08x\n", "int_ctl:", control->int_ctl); 3134 pr_err("%-20s%08x\n", "int_vector:", control->int_vector); 3135 pr_err("%-20s%08x\n", "int_state:", control->int_state); 3136 pr_err("%-20s%08x\n", "exit_code:", control->exit_code); 3137 pr_err("%-20s%016llx\n", "exit_info1:", control->exit_info_1); 3138 pr_err("%-20s%016llx\n", "exit_info2:", control->exit_info_2); 3139 pr_err("%-20s%08x\n", "exit_int_info:", control->exit_int_info); 3140 pr_err("%-20s%08x\n", "exit_int_info_err:", control->exit_int_info_err); 3141 pr_err("%-20s%lld\n", "nested_ctl:", control->nested_ctl); 3142 pr_err("%-20s%016llx\n", "nested_cr3:", control->nested_cr3); 3143 pr_err("%-20s%016llx\n", "avic_vapic_bar:", control->avic_vapic_bar); 3144 pr_err("%-20s%016llx\n", "ghcb:", control->ghcb_gpa); 3145 pr_err("%-20s%08x\n", "event_inj:", control->event_inj); 3146 pr_err("%-20s%08x\n", "event_inj_err:", control->event_inj_err); 3147 pr_err("%-20s%lld\n", "virt_ext:", control->virt_ext); 3148 pr_err("%-20s%016llx\n", "next_rip:", control->next_rip); 3149 pr_err("%-20s%016llx\n", "avic_backing_page:", control->avic_backing_page); 3150 pr_err("%-20s%016llx\n", "avic_logical_id:", control->avic_logical_id); 3151 pr_err("%-20s%016llx\n", "avic_physical_id:", control->avic_physical_id); 3152 pr_err("%-20s%016llx\n", "vmsa_pa:", control->vmsa_pa); 3153 pr_err("VMCB State Save Area:\n"); 3154 pr_err("%-5s s: %04x a: %04x l: %08x b: %016llx\n", 3155 "es:", 3156 save->es.selector, save->es.attrib, 3157 save->es.limit, save->es.base); 3158 pr_err("%-5s s: %04x a: %04x l: %08x b: %016llx\n", 3159 "cs:", 3160 save->cs.selector, save->cs.attrib, 3161 save->cs.limit, save->cs.base); 3162 pr_err("%-5s s: %04x a: %04x l: %08x b: %016llx\n", 3163 "ss:", 3164 save->ss.selector, save->ss.attrib, 3165 save->ss.limit, save->ss.base); 3166 pr_err("%-5s s: %04x a: %04x l: %08x b: %016llx\n", 3167 "ds:", 3168 save->ds.selector, save->ds.attrib, 3169 save->ds.limit, save->ds.base); 3170 pr_err("%-5s s: %04x a: %04x l: %08x b: %016llx\n", 3171 "fs:", 3172 save01->fs.selector, save01->fs.attrib, 3173 save01->fs.limit, save01->fs.base); 3174 pr_err("%-5s s: %04x a: %04x l: %08x b: %016llx\n", 3175 "gs:", 3176 save01->gs.selector, save01->gs.attrib, 3177 save01->gs.limit, save01->gs.base); 3178 pr_err("%-5s s: %04x a: %04x l: %08x b: %016llx\n", 3179 "gdtr:", 3180 save->gdtr.selector, save->gdtr.attrib, 3181 save->gdtr.limit, save->gdtr.base); 3182 pr_err("%-5s s: %04x a: %04x l: %08x b: %016llx\n", 3183 "ldtr:", 3184 save01->ldtr.selector, save01->ldtr.attrib, 3185 save01->ldtr.limit, save01->ldtr.base); 3186 pr_err("%-5s s: %04x a: %04x l: %08x b: %016llx\n", 3187 "idtr:", 3188 save->idtr.selector, save->idtr.attrib, 3189 save->idtr.limit, save->idtr.base); 3190 pr_err("%-5s s: %04x a: %04x l: %08x b: %016llx\n", 3191 "tr:", 3192 save01->tr.selector, save01->tr.attrib, 3193 save01->tr.limit, save01->tr.base); 3194 pr_err("cpl: %d efer: %016llx\n", 3195 save->cpl, save->efer); 3196 pr_err("%-15s %016llx %-13s %016llx\n", 3197 "cr0:", save->cr0, "cr2:", save->cr2); 3198 pr_err("%-15s %016llx %-13s %016llx\n", 3199 "cr3:", save->cr3, "cr4:", save->cr4); 3200 pr_err("%-15s %016llx %-13s %016llx\n", 3201 "dr6:", save->dr6, "dr7:", save->dr7); 3202 pr_err("%-15s %016llx %-13s %016llx\n", 3203 "rip:", save->rip, "rflags:", save->rflags); 3204 pr_err("%-15s %016llx %-13s %016llx\n", 3205 "rsp:", save->rsp, "rax:", save->rax); 3206 pr_err("%-15s %016llx %-13s %016llx\n", 3207 "star:", save01->star, "lstar:", save01->lstar); 3208 pr_err("%-15s %016llx %-13s %016llx\n", 3209 "cstar:", save01->cstar, "sfmask:", save01->sfmask); 3210 pr_err("%-15s %016llx %-13s %016llx\n", 3211 "kernel_gs_base:", save01->kernel_gs_base, 3212 "sysenter_cs:", save01->sysenter_cs); 3213 pr_err("%-15s %016llx %-13s %016llx\n", 3214 "sysenter_esp:", save01->sysenter_esp, 3215 "sysenter_eip:", save01->sysenter_eip); 3216 pr_err("%-15s %016llx %-13s %016llx\n", 3217 "gpat:", save->g_pat, "dbgctl:", save->dbgctl); 3218 pr_err("%-15s %016llx %-13s %016llx\n", 3219 "br_from:", save->br_from, "br_to:", save->br_to); 3220 pr_err("%-15s %016llx %-13s %016llx\n", 3221 "excp_from:", save->last_excp_from, 3222 "excp_to:", save->last_excp_to); 3223 } 3224 3225 static int svm_handle_invalid_exit(struct kvm_vcpu *vcpu, u64 exit_code) 3226 { 3227 if (exit_code < ARRAY_SIZE(svm_exit_handlers) && 3228 svm_exit_handlers[exit_code]) 3229 return 0; 3230 3231 vcpu_unimpl(vcpu, "svm: unexpected exit reason 0x%llx\n", exit_code); 3232 dump_vmcb(vcpu); 3233 vcpu->run->exit_reason = KVM_EXIT_INTERNAL_ERROR; 3234 vcpu->run->internal.suberror = KVM_INTERNAL_ERROR_UNEXPECTED_EXIT_REASON; 3235 vcpu->run->internal.ndata = 2; 3236 vcpu->run->internal.data[0] = exit_code; 3237 vcpu->run->internal.data[1] = vcpu->arch.last_vmentry_cpu; 3238 3239 return -EINVAL; 3240 } 3241 3242 int svm_invoke_exit_handler(struct kvm_vcpu *vcpu, u64 exit_code) 3243 { 3244 if (svm_handle_invalid_exit(vcpu, exit_code)) 3245 return 0; 3246 3247 #ifdef CONFIG_RETPOLINE 3248 if (exit_code == SVM_EXIT_MSR) 3249 return msr_interception(vcpu); 3250 else if (exit_code == SVM_EXIT_VINTR) 3251 return interrupt_window_interception(vcpu); 3252 else if (exit_code == SVM_EXIT_INTR) 3253 return intr_interception(vcpu); 3254 else if (exit_code == SVM_EXIT_HLT) 3255 return kvm_emulate_halt(vcpu); 3256 else if (exit_code == SVM_EXIT_NPF) 3257 return npf_interception(vcpu); 3258 #endif 3259 return svm_exit_handlers[exit_code](vcpu); 3260 } 3261 3262 static void svm_get_exit_info(struct kvm_vcpu *vcpu, u64 *info1, u64 *info2, 3263 u32 *intr_info, u32 *error_code) 3264 { 3265 struct vmcb_control_area *control = &to_svm(vcpu)->vmcb->control; 3266 3267 *info1 = control->exit_info_1; 3268 *info2 = control->exit_info_2; 3269 *intr_info = control->exit_int_info; 3270 if ((*intr_info & SVM_EXITINTINFO_VALID) && 3271 (*intr_info & SVM_EXITINTINFO_VALID_ERR)) 3272 *error_code = control->exit_int_info_err; 3273 else 3274 *error_code = 0; 3275 } 3276 3277 static int handle_exit(struct kvm_vcpu *vcpu, fastpath_t exit_fastpath) 3278 { 3279 struct vcpu_svm *svm = to_svm(vcpu); 3280 struct kvm_run *kvm_run = vcpu->run; 3281 u32 exit_code = svm->vmcb->control.exit_code; 3282 3283 trace_kvm_exit(exit_code, vcpu, KVM_ISA_SVM); 3284 3285 /* SEV-ES guests must use the CR write traps to track CR registers. */ 3286 if (!sev_es_guest(vcpu->kvm)) { 3287 if (!svm_is_intercept(svm, INTERCEPT_CR0_WRITE)) 3288 vcpu->arch.cr0 = svm->vmcb->save.cr0; 3289 if (npt_enabled) 3290 vcpu->arch.cr3 = svm->vmcb->save.cr3; 3291 } 3292 3293 if (is_guest_mode(vcpu)) { 3294 int vmexit; 3295 3296 trace_kvm_nested_vmexit(exit_code, vcpu, KVM_ISA_SVM); 3297 3298 vmexit = nested_svm_exit_special(svm); 3299 3300 if (vmexit == NESTED_EXIT_CONTINUE) 3301 vmexit = nested_svm_exit_handled(svm); 3302 3303 if (vmexit == NESTED_EXIT_DONE) 3304 return 1; 3305 } 3306 3307 if (svm->vmcb->control.exit_code == SVM_EXIT_ERR) { 3308 kvm_run->exit_reason = KVM_EXIT_FAIL_ENTRY; 3309 kvm_run->fail_entry.hardware_entry_failure_reason 3310 = svm->vmcb->control.exit_code; 3311 kvm_run->fail_entry.cpu = vcpu->arch.last_vmentry_cpu; 3312 dump_vmcb(vcpu); 3313 return 0; 3314 } 3315 3316 if (is_external_interrupt(svm->vmcb->control.exit_int_info) && 3317 exit_code != SVM_EXIT_EXCP_BASE + PF_VECTOR && 3318 exit_code != SVM_EXIT_NPF && exit_code != SVM_EXIT_TASK_SWITCH && 3319 exit_code != SVM_EXIT_INTR && exit_code != SVM_EXIT_NMI) 3320 printk(KERN_ERR "%s: unexpected exit_int_info 0x%x " 3321 "exit_code 0x%x\n", 3322 __func__, svm->vmcb->control.exit_int_info, 3323 exit_code); 3324 3325 if (exit_fastpath != EXIT_FASTPATH_NONE) 3326 return 1; 3327 3328 return svm_invoke_exit_handler(vcpu, exit_code); 3329 } 3330 3331 static void reload_tss(struct kvm_vcpu *vcpu) 3332 { 3333 struct svm_cpu_data *sd = per_cpu(svm_data, vcpu->cpu); 3334 3335 sd->tss_desc->type = 9; /* available 32/64-bit TSS */ 3336 load_TR_desc(); 3337 } 3338 3339 static void pre_svm_run(struct kvm_vcpu *vcpu) 3340 { 3341 struct svm_cpu_data *sd = per_cpu(svm_data, vcpu->cpu); 3342 struct vcpu_svm *svm = to_svm(vcpu); 3343 3344 /* 3345 * If the previous vmrun of the vmcb occurred on a different physical 3346 * cpu, then mark the vmcb dirty and assign a new asid. Hardware's 3347 * vmcb clean bits are per logical CPU, as are KVM's asid assignments. 3348 */ 3349 if (unlikely(svm->current_vmcb->cpu != vcpu->cpu)) { 3350 svm->current_vmcb->asid_generation = 0; 3351 vmcb_mark_all_dirty(svm->vmcb); 3352 svm->current_vmcb->cpu = vcpu->cpu; 3353 } 3354 3355 if (sev_guest(vcpu->kvm)) 3356 return pre_sev_run(svm, vcpu->cpu); 3357 3358 /* FIXME: handle wraparound of asid_generation */ 3359 if (svm->current_vmcb->asid_generation != sd->asid_generation) 3360 new_asid(svm, sd); 3361 } 3362 3363 static void svm_inject_nmi(struct kvm_vcpu *vcpu) 3364 { 3365 struct vcpu_svm *svm = to_svm(vcpu); 3366 3367 svm->vmcb->control.event_inj = SVM_EVTINJ_VALID | SVM_EVTINJ_TYPE_NMI; 3368 vcpu->arch.hflags |= HF_NMI_MASK; 3369 if (!sev_es_guest(vcpu->kvm)) 3370 svm_set_intercept(svm, INTERCEPT_IRET); 3371 ++vcpu->stat.nmi_injections; 3372 } 3373 3374 static void svm_set_irq(struct kvm_vcpu *vcpu) 3375 { 3376 struct vcpu_svm *svm = to_svm(vcpu); 3377 3378 BUG_ON(!(gif_set(svm))); 3379 3380 trace_kvm_inj_virq(vcpu->arch.interrupt.nr); 3381 ++vcpu->stat.irq_injections; 3382 3383 svm->vmcb->control.event_inj = vcpu->arch.interrupt.nr | 3384 SVM_EVTINJ_VALID | SVM_EVTINJ_TYPE_INTR; 3385 } 3386 3387 static void svm_update_cr8_intercept(struct kvm_vcpu *vcpu, int tpr, int irr) 3388 { 3389 struct vcpu_svm *svm = to_svm(vcpu); 3390 3391 /* 3392 * SEV-ES guests must always keep the CR intercepts cleared. CR 3393 * tracking is done using the CR write traps. 3394 */ 3395 if (sev_es_guest(vcpu->kvm)) 3396 return; 3397 3398 if (nested_svm_virtualize_tpr(vcpu)) 3399 return; 3400 3401 svm_clr_intercept(svm, INTERCEPT_CR8_WRITE); 3402 3403 if (irr == -1) 3404 return; 3405 3406 if (tpr >= irr) 3407 svm_set_intercept(svm, INTERCEPT_CR8_WRITE); 3408 } 3409 3410 bool svm_nmi_blocked(struct kvm_vcpu *vcpu) 3411 { 3412 struct vcpu_svm *svm = to_svm(vcpu); 3413 struct vmcb *vmcb = svm->vmcb; 3414 bool ret; 3415 3416 if (!gif_set(svm)) 3417 return true; 3418 3419 if (is_guest_mode(vcpu) && nested_exit_on_nmi(svm)) 3420 return false; 3421 3422 ret = (vmcb->control.int_state & SVM_INTERRUPT_SHADOW_MASK) || 3423 (vcpu->arch.hflags & HF_NMI_MASK); 3424 3425 return ret; 3426 } 3427 3428 static int svm_nmi_allowed(struct kvm_vcpu *vcpu, bool for_injection) 3429 { 3430 struct vcpu_svm *svm = to_svm(vcpu); 3431 if (svm->nested.nested_run_pending) 3432 return -EBUSY; 3433 3434 /* An NMI must not be injected into L2 if it's supposed to VM-Exit. */ 3435 if (for_injection && is_guest_mode(vcpu) && nested_exit_on_nmi(svm)) 3436 return -EBUSY; 3437 3438 return !svm_nmi_blocked(vcpu); 3439 } 3440 3441 static bool svm_get_nmi_mask(struct kvm_vcpu *vcpu) 3442 { 3443 return !!(vcpu->arch.hflags & HF_NMI_MASK); 3444 } 3445 3446 static void svm_set_nmi_mask(struct kvm_vcpu *vcpu, bool masked) 3447 { 3448 struct vcpu_svm *svm = to_svm(vcpu); 3449 3450 if (masked) { 3451 vcpu->arch.hflags |= HF_NMI_MASK; 3452 if (!sev_es_guest(vcpu->kvm)) 3453 svm_set_intercept(svm, INTERCEPT_IRET); 3454 } else { 3455 vcpu->arch.hflags &= ~HF_NMI_MASK; 3456 if (!sev_es_guest(vcpu->kvm)) 3457 svm_clr_intercept(svm, INTERCEPT_IRET); 3458 } 3459 } 3460 3461 bool svm_interrupt_blocked(struct kvm_vcpu *vcpu) 3462 { 3463 struct vcpu_svm *svm = to_svm(vcpu); 3464 struct vmcb *vmcb = svm->vmcb; 3465 3466 if (!gif_set(svm)) 3467 return true; 3468 3469 if (sev_es_guest(vcpu->kvm)) { 3470 /* 3471 * SEV-ES guests to not expose RFLAGS. Use the VMCB interrupt mask 3472 * bit to determine the state of the IF flag. 3473 */ 3474 if (!(vmcb->control.int_state & SVM_GUEST_INTERRUPT_MASK)) 3475 return true; 3476 } else if (is_guest_mode(vcpu)) { 3477 /* As long as interrupts are being delivered... */ 3478 if ((svm->nested.ctl.int_ctl & V_INTR_MASKING_MASK) 3479 ? !(svm->vmcb01.ptr->save.rflags & X86_EFLAGS_IF) 3480 : !(kvm_get_rflags(vcpu) & X86_EFLAGS_IF)) 3481 return true; 3482 3483 /* ... vmexits aren't blocked by the interrupt shadow */ 3484 if (nested_exit_on_intr(svm)) 3485 return false; 3486 } else { 3487 if (!(kvm_get_rflags(vcpu) & X86_EFLAGS_IF)) 3488 return true; 3489 } 3490 3491 return (vmcb->control.int_state & SVM_INTERRUPT_SHADOW_MASK); 3492 } 3493 3494 static int svm_interrupt_allowed(struct kvm_vcpu *vcpu, bool for_injection) 3495 { 3496 struct vcpu_svm *svm = to_svm(vcpu); 3497 if (svm->nested.nested_run_pending) 3498 return -EBUSY; 3499 3500 /* 3501 * An IRQ must not be injected into L2 if it's supposed to VM-Exit, 3502 * e.g. if the IRQ arrived asynchronously after checking nested events. 3503 */ 3504 if (for_injection && is_guest_mode(vcpu) && nested_exit_on_intr(svm)) 3505 return -EBUSY; 3506 3507 return !svm_interrupt_blocked(vcpu); 3508 } 3509 3510 static void svm_enable_irq_window(struct kvm_vcpu *vcpu) 3511 { 3512 struct vcpu_svm *svm = to_svm(vcpu); 3513 3514 /* 3515 * In case GIF=0 we can't rely on the CPU to tell us when GIF becomes 3516 * 1, because that's a separate STGI/VMRUN intercept. The next time we 3517 * get that intercept, this function will be called again though and 3518 * we'll get the vintr intercept. However, if the vGIF feature is 3519 * enabled, the STGI interception will not occur. Enable the irq 3520 * window under the assumption that the hardware will set the GIF. 3521 */ 3522 if (vgif_enabled(svm) || gif_set(svm)) { 3523 /* 3524 * IRQ window is not needed when AVIC is enabled, 3525 * unless we have pending ExtINT since it cannot be injected 3526 * via AVIC. In such case, we need to temporarily disable AVIC, 3527 * and fallback to injecting IRQ via V_IRQ. 3528 */ 3529 svm_toggle_avic_for_irq_window(vcpu, false); 3530 svm_set_vintr(svm); 3531 } 3532 } 3533 3534 static void svm_enable_nmi_window(struct kvm_vcpu *vcpu) 3535 { 3536 struct vcpu_svm *svm = to_svm(vcpu); 3537 3538 if ((vcpu->arch.hflags & (HF_NMI_MASK | HF_IRET_MASK)) == HF_NMI_MASK) 3539 return; /* IRET will cause a vm exit */ 3540 3541 if (!gif_set(svm)) { 3542 if (vgif_enabled(svm)) 3543 svm_set_intercept(svm, INTERCEPT_STGI); 3544 return; /* STGI will cause a vm exit */ 3545 } 3546 3547 /* 3548 * Something prevents NMI from been injected. Single step over possible 3549 * problem (IRET or exception injection or interrupt shadow) 3550 */ 3551 svm->nmi_singlestep_guest_rflags = svm_get_rflags(vcpu); 3552 svm->nmi_singlestep = true; 3553 svm->vmcb->save.rflags |= (X86_EFLAGS_TF | X86_EFLAGS_RF); 3554 } 3555 3556 static int svm_set_tss_addr(struct kvm *kvm, unsigned int addr) 3557 { 3558 return 0; 3559 } 3560 3561 static int svm_set_identity_map_addr(struct kvm *kvm, u64 ident_addr) 3562 { 3563 return 0; 3564 } 3565 3566 void svm_flush_tlb(struct kvm_vcpu *vcpu) 3567 { 3568 struct vcpu_svm *svm = to_svm(vcpu); 3569 3570 /* 3571 * Flush only the current ASID even if the TLB flush was invoked via 3572 * kvm_flush_remote_tlbs(). Although flushing remote TLBs requires all 3573 * ASIDs to be flushed, KVM uses a single ASID for L1 and L2, and 3574 * unconditionally does a TLB flush on both nested VM-Enter and nested 3575 * VM-Exit (via kvm_mmu_reset_context()). 3576 */ 3577 if (static_cpu_has(X86_FEATURE_FLUSHBYASID)) 3578 svm->vmcb->control.tlb_ctl = TLB_CONTROL_FLUSH_ASID; 3579 else 3580 svm->current_vmcb->asid_generation--; 3581 } 3582 3583 static void svm_flush_tlb_gva(struct kvm_vcpu *vcpu, gva_t gva) 3584 { 3585 struct vcpu_svm *svm = to_svm(vcpu); 3586 3587 invlpga(gva, svm->vmcb->control.asid); 3588 } 3589 3590 static inline void sync_cr8_to_lapic(struct kvm_vcpu *vcpu) 3591 { 3592 struct vcpu_svm *svm = to_svm(vcpu); 3593 3594 if (nested_svm_virtualize_tpr(vcpu)) 3595 return; 3596 3597 if (!svm_is_intercept(svm, INTERCEPT_CR8_WRITE)) { 3598 int cr8 = svm->vmcb->control.int_ctl & V_TPR_MASK; 3599 kvm_set_cr8(vcpu, cr8); 3600 } 3601 } 3602 3603 static inline void sync_lapic_to_cr8(struct kvm_vcpu *vcpu) 3604 { 3605 struct vcpu_svm *svm = to_svm(vcpu); 3606 u64 cr8; 3607 3608 if (nested_svm_virtualize_tpr(vcpu) || 3609 kvm_vcpu_apicv_active(vcpu)) 3610 return; 3611 3612 cr8 = kvm_get_cr8(vcpu); 3613 svm->vmcb->control.int_ctl &= ~V_TPR_MASK; 3614 svm->vmcb->control.int_ctl |= cr8 & V_TPR_MASK; 3615 } 3616 3617 static void svm_complete_interrupts(struct kvm_vcpu *vcpu) 3618 { 3619 struct vcpu_svm *svm = to_svm(vcpu); 3620 u8 vector; 3621 int type; 3622 u32 exitintinfo = svm->vmcb->control.exit_int_info; 3623 unsigned int3_injected = svm->int3_injected; 3624 3625 svm->int3_injected = 0; 3626 3627 /* 3628 * If we've made progress since setting HF_IRET_MASK, we've 3629 * executed an IRET and can allow NMI injection. 3630 */ 3631 if ((vcpu->arch.hflags & HF_IRET_MASK) && 3632 (sev_es_guest(vcpu->kvm) || 3633 kvm_rip_read(vcpu) != svm->nmi_iret_rip)) { 3634 vcpu->arch.hflags &= ~(HF_NMI_MASK | HF_IRET_MASK); 3635 kvm_make_request(KVM_REQ_EVENT, vcpu); 3636 } 3637 3638 vcpu->arch.nmi_injected = false; 3639 kvm_clear_exception_queue(vcpu); 3640 kvm_clear_interrupt_queue(vcpu); 3641 3642 if (!(exitintinfo & SVM_EXITINTINFO_VALID)) 3643 return; 3644 3645 kvm_make_request(KVM_REQ_EVENT, vcpu); 3646 3647 vector = exitintinfo & SVM_EXITINTINFO_VEC_MASK; 3648 type = exitintinfo & SVM_EXITINTINFO_TYPE_MASK; 3649 3650 switch (type) { 3651 case SVM_EXITINTINFO_TYPE_NMI: 3652 vcpu->arch.nmi_injected = true; 3653 break; 3654 case SVM_EXITINTINFO_TYPE_EXEPT: 3655 /* 3656 * Never re-inject a #VC exception. 3657 */ 3658 if (vector == X86_TRAP_VC) 3659 break; 3660 3661 /* 3662 * In case of software exceptions, do not reinject the vector, 3663 * but re-execute the instruction instead. Rewind RIP first 3664 * if we emulated INT3 before. 3665 */ 3666 if (kvm_exception_is_soft(vector)) { 3667 if (vector == BP_VECTOR && int3_injected && 3668 kvm_is_linear_rip(vcpu, svm->int3_rip)) 3669 kvm_rip_write(vcpu, 3670 kvm_rip_read(vcpu) - int3_injected); 3671 break; 3672 } 3673 if (exitintinfo & SVM_EXITINTINFO_VALID_ERR) { 3674 u32 err = svm->vmcb->control.exit_int_info_err; 3675 kvm_requeue_exception_e(vcpu, vector, err); 3676 3677 } else 3678 kvm_requeue_exception(vcpu, vector); 3679 break; 3680 case SVM_EXITINTINFO_TYPE_INTR: 3681 kvm_queue_interrupt(vcpu, vector, false); 3682 break; 3683 default: 3684 break; 3685 } 3686 } 3687 3688 static void svm_cancel_injection(struct kvm_vcpu *vcpu) 3689 { 3690 struct vcpu_svm *svm = to_svm(vcpu); 3691 struct vmcb_control_area *control = &svm->vmcb->control; 3692 3693 control->exit_int_info = control->event_inj; 3694 control->exit_int_info_err = control->event_inj_err; 3695 control->event_inj = 0; 3696 svm_complete_interrupts(vcpu); 3697 } 3698 3699 static fastpath_t svm_exit_handlers_fastpath(struct kvm_vcpu *vcpu) 3700 { 3701 if (to_svm(vcpu)->vmcb->control.exit_code == SVM_EXIT_MSR && 3702 to_svm(vcpu)->vmcb->control.exit_info_1) 3703 return handle_fastpath_set_msr_irqoff(vcpu); 3704 3705 return EXIT_FASTPATH_NONE; 3706 } 3707 3708 static noinstr void svm_vcpu_enter_exit(struct kvm_vcpu *vcpu) 3709 { 3710 struct vcpu_svm *svm = to_svm(vcpu); 3711 unsigned long vmcb_pa = svm->current_vmcb->pa; 3712 3713 kvm_guest_enter_irqoff(); 3714 3715 if (sev_es_guest(vcpu->kvm)) { 3716 __svm_sev_es_vcpu_run(vmcb_pa); 3717 } else { 3718 struct svm_cpu_data *sd = per_cpu(svm_data, vcpu->cpu); 3719 3720 /* 3721 * Use a single vmcb (vmcb01 because it's always valid) for 3722 * context switching guest state via VMLOAD/VMSAVE, that way 3723 * the state doesn't need to be copied between vmcb01 and 3724 * vmcb02 when switching vmcbs for nested virtualization. 3725 */ 3726 vmload(svm->vmcb01.pa); 3727 __svm_vcpu_run(vmcb_pa, (unsigned long *)&vcpu->arch.regs); 3728 vmsave(svm->vmcb01.pa); 3729 3730 vmload(__sme_page_pa(sd->save_area)); 3731 } 3732 3733 kvm_guest_exit_irqoff(); 3734 } 3735 3736 static __no_kcsan fastpath_t svm_vcpu_run(struct kvm_vcpu *vcpu) 3737 { 3738 struct vcpu_svm *svm = to_svm(vcpu); 3739 3740 trace_kvm_entry(vcpu); 3741 3742 svm->vmcb->save.rax = vcpu->arch.regs[VCPU_REGS_RAX]; 3743 svm->vmcb->save.rsp = vcpu->arch.regs[VCPU_REGS_RSP]; 3744 svm->vmcb->save.rip = vcpu->arch.regs[VCPU_REGS_RIP]; 3745 3746 /* 3747 * Disable singlestep if we're injecting an interrupt/exception. 3748 * We don't want our modified rflags to be pushed on the stack where 3749 * we might not be able to easily reset them if we disabled NMI 3750 * singlestep later. 3751 */ 3752 if (svm->nmi_singlestep && svm->vmcb->control.event_inj) { 3753 /* 3754 * Event injection happens before external interrupts cause a 3755 * vmexit and interrupts are disabled here, so smp_send_reschedule 3756 * is enough to force an immediate vmexit. 3757 */ 3758 disable_nmi_singlestep(svm); 3759 smp_send_reschedule(vcpu->cpu); 3760 } 3761 3762 pre_svm_run(vcpu); 3763 3764 sync_lapic_to_cr8(vcpu); 3765 3766 if (unlikely(svm->asid != svm->vmcb->control.asid)) { 3767 svm->vmcb->control.asid = svm->asid; 3768 vmcb_mark_dirty(svm->vmcb, VMCB_ASID); 3769 } 3770 svm->vmcb->save.cr2 = vcpu->arch.cr2; 3771 3772 /* 3773 * Run with all-zero DR6 unless needed, so that we can get the exact cause 3774 * of a #DB. 3775 */ 3776 if (unlikely(vcpu->arch.switch_db_regs & KVM_DEBUGREG_WONT_EXIT)) 3777 svm_set_dr6(svm, vcpu->arch.dr6); 3778 else 3779 svm_set_dr6(svm, DR6_ACTIVE_LOW); 3780 3781 clgi(); 3782 kvm_load_guest_xsave_state(vcpu); 3783 3784 kvm_wait_lapic_expire(vcpu); 3785 3786 /* 3787 * If this vCPU has touched SPEC_CTRL, restore the guest's value if 3788 * it's non-zero. Since vmentry is serialising on affected CPUs, there 3789 * is no need to worry about the conditional branch over the wrmsr 3790 * being speculatively taken. 3791 */ 3792 if (!static_cpu_has(X86_FEATURE_V_SPEC_CTRL)) 3793 x86_spec_ctrl_set_guest(svm->spec_ctrl, svm->virt_spec_ctrl); 3794 3795 svm_vcpu_enter_exit(vcpu); 3796 3797 /* 3798 * We do not use IBRS in the kernel. If this vCPU has used the 3799 * SPEC_CTRL MSR it may have left it on; save the value and 3800 * turn it off. This is much more efficient than blindly adding 3801 * it to the atomic save/restore list. Especially as the former 3802 * (Saving guest MSRs on vmexit) doesn't even exist in KVM. 3803 * 3804 * For non-nested case: 3805 * If the L01 MSR bitmap does not intercept the MSR, then we need to 3806 * save it. 3807 * 3808 * For nested case: 3809 * If the L02 MSR bitmap does not intercept the MSR, then we need to 3810 * save it. 3811 */ 3812 if (!static_cpu_has(X86_FEATURE_V_SPEC_CTRL) && 3813 unlikely(!msr_write_intercepted(vcpu, MSR_IA32_SPEC_CTRL))) 3814 svm->spec_ctrl = native_read_msr(MSR_IA32_SPEC_CTRL); 3815 3816 if (!sev_es_guest(vcpu->kvm)) 3817 reload_tss(vcpu); 3818 3819 if (!static_cpu_has(X86_FEATURE_V_SPEC_CTRL)) 3820 x86_spec_ctrl_restore_host(svm->spec_ctrl, svm->virt_spec_ctrl); 3821 3822 if (!sev_es_guest(vcpu->kvm)) { 3823 vcpu->arch.cr2 = svm->vmcb->save.cr2; 3824 vcpu->arch.regs[VCPU_REGS_RAX] = svm->vmcb->save.rax; 3825 vcpu->arch.regs[VCPU_REGS_RSP] = svm->vmcb->save.rsp; 3826 vcpu->arch.regs[VCPU_REGS_RIP] = svm->vmcb->save.rip; 3827 } 3828 3829 if (unlikely(svm->vmcb->control.exit_code == SVM_EXIT_NMI)) 3830 kvm_before_interrupt(vcpu); 3831 3832 kvm_load_host_xsave_state(vcpu); 3833 stgi(); 3834 3835 /* Any pending NMI will happen here */ 3836 3837 if (unlikely(svm->vmcb->control.exit_code == SVM_EXIT_NMI)) 3838 kvm_after_interrupt(vcpu); 3839 3840 sync_cr8_to_lapic(vcpu); 3841 3842 svm->next_rip = 0; 3843 if (is_guest_mode(vcpu)) { 3844 nested_sync_control_from_vmcb02(svm); 3845 svm->nested.nested_run_pending = 0; 3846 } 3847 3848 svm->vmcb->control.tlb_ctl = TLB_CONTROL_DO_NOTHING; 3849 vmcb_mark_all_clean(svm->vmcb); 3850 3851 /* if exit due to PF check for async PF */ 3852 if (svm->vmcb->control.exit_code == SVM_EXIT_EXCP_BASE + PF_VECTOR) 3853 vcpu->arch.apf.host_apf_flags = 3854 kvm_read_and_reset_apf_flags(); 3855 3856 if (npt_enabled) { 3857 vcpu->arch.regs_avail &= ~(1 << VCPU_EXREG_PDPTR); 3858 vcpu->arch.regs_dirty &= ~(1 << VCPU_EXREG_PDPTR); 3859 } 3860 3861 /* 3862 * We need to handle MC intercepts here before the vcpu has a chance to 3863 * change the physical cpu 3864 */ 3865 if (unlikely(svm->vmcb->control.exit_code == 3866 SVM_EXIT_EXCP_BASE + MC_VECTOR)) 3867 svm_handle_mce(vcpu); 3868 3869 svm_complete_interrupts(vcpu); 3870 3871 if (is_guest_mode(vcpu)) 3872 return EXIT_FASTPATH_NONE; 3873 3874 return svm_exit_handlers_fastpath(vcpu); 3875 } 3876 3877 static void svm_load_mmu_pgd(struct kvm_vcpu *vcpu, hpa_t root_hpa, 3878 int root_level) 3879 { 3880 struct vcpu_svm *svm = to_svm(vcpu); 3881 unsigned long cr3; 3882 3883 if (npt_enabled) { 3884 svm->vmcb->control.nested_cr3 = __sme_set(root_hpa); 3885 vmcb_mark_dirty(svm->vmcb, VMCB_NPT); 3886 3887 /* Loading L2's CR3 is handled by enter_svm_guest_mode. */ 3888 if (!test_bit(VCPU_EXREG_CR3, (ulong *)&vcpu->arch.regs_avail)) 3889 return; 3890 cr3 = vcpu->arch.cr3; 3891 } else if (vcpu->arch.mmu->shadow_root_level >= PT64_ROOT_4LEVEL) { 3892 cr3 = __sme_set(root_hpa) | kvm_get_active_pcid(vcpu); 3893 } else { 3894 /* PCID in the guest should be impossible with a 32-bit MMU. */ 3895 WARN_ON_ONCE(kvm_get_active_pcid(vcpu)); 3896 cr3 = root_hpa; 3897 } 3898 3899 svm->vmcb->save.cr3 = cr3; 3900 vmcb_mark_dirty(svm->vmcb, VMCB_CR); 3901 } 3902 3903 static int is_disabled(void) 3904 { 3905 u64 vm_cr; 3906 3907 rdmsrl(MSR_VM_CR, vm_cr); 3908 if (vm_cr & (1 << SVM_VM_CR_SVM_DISABLE)) 3909 return 1; 3910 3911 return 0; 3912 } 3913 3914 static void 3915 svm_patch_hypercall(struct kvm_vcpu *vcpu, unsigned char *hypercall) 3916 { 3917 /* 3918 * Patch in the VMMCALL instruction: 3919 */ 3920 hypercall[0] = 0x0f; 3921 hypercall[1] = 0x01; 3922 hypercall[2] = 0xd9; 3923 } 3924 3925 static int __init svm_check_processor_compat(void) 3926 { 3927 return 0; 3928 } 3929 3930 static bool svm_cpu_has_accelerated_tpr(void) 3931 { 3932 return false; 3933 } 3934 3935 /* 3936 * The kvm parameter can be NULL (module initialization, or invocation before 3937 * VM creation). Be sure to check the kvm parameter before using it. 3938 */ 3939 static bool svm_has_emulated_msr(struct kvm *kvm, u32 index) 3940 { 3941 switch (index) { 3942 case MSR_IA32_MCG_EXT_CTL: 3943 case MSR_IA32_VMX_BASIC ... MSR_IA32_VMX_VMFUNC: 3944 return false; 3945 case MSR_IA32_SMBASE: 3946 /* SEV-ES guests do not support SMM, so report false */ 3947 if (kvm && sev_es_guest(kvm)) 3948 return false; 3949 break; 3950 default: 3951 break; 3952 } 3953 3954 return true; 3955 } 3956 3957 static u64 svm_get_mt_mask(struct kvm_vcpu *vcpu, gfn_t gfn, bool is_mmio) 3958 { 3959 return 0; 3960 } 3961 3962 static void svm_vcpu_after_set_cpuid(struct kvm_vcpu *vcpu) 3963 { 3964 struct vcpu_svm *svm = to_svm(vcpu); 3965 struct kvm_cpuid_entry2 *best; 3966 3967 vcpu->arch.xsaves_enabled = guest_cpuid_has(vcpu, X86_FEATURE_XSAVE) && 3968 boot_cpu_has(X86_FEATURE_XSAVE) && 3969 boot_cpu_has(X86_FEATURE_XSAVES); 3970 3971 /* Update nrips enabled cache */ 3972 svm->nrips_enabled = kvm_cpu_cap_has(X86_FEATURE_NRIPS) && 3973 guest_cpuid_has(vcpu, X86_FEATURE_NRIPS); 3974 3975 /* Check again if INVPCID interception if required */ 3976 svm_check_invpcid(svm); 3977 3978 /* For sev guests, the memory encryption bit is not reserved in CR3. */ 3979 if (sev_guest(vcpu->kvm)) { 3980 best = kvm_find_cpuid_entry(vcpu, 0x8000001F, 0); 3981 if (best) 3982 vcpu->arch.reserved_gpa_bits &= ~(1UL << (best->ebx & 0x3f)); 3983 } 3984 3985 if (kvm_vcpu_apicv_active(vcpu)) { 3986 /* 3987 * AVIC does not work with an x2APIC mode guest. If the X2APIC feature 3988 * is exposed to the guest, disable AVIC. 3989 */ 3990 if (guest_cpuid_has(vcpu, X86_FEATURE_X2APIC)) 3991 kvm_request_apicv_update(vcpu->kvm, false, 3992 APICV_INHIBIT_REASON_X2APIC); 3993 3994 /* 3995 * Currently, AVIC does not work with nested virtualization. 3996 * So, we disable AVIC when cpuid for SVM is set in the L1 guest. 3997 */ 3998 if (nested && guest_cpuid_has(vcpu, X86_FEATURE_SVM)) 3999 kvm_request_apicv_update(vcpu->kvm, false, 4000 APICV_INHIBIT_REASON_NESTED); 4001 } 4002 4003 if (guest_cpuid_is_intel(vcpu)) { 4004 /* 4005 * We must intercept SYSENTER_EIP and SYSENTER_ESP 4006 * accesses because the processor only stores 32 bits. 4007 * For the same reason we cannot use virtual VMLOAD/VMSAVE. 4008 */ 4009 svm_set_intercept(svm, INTERCEPT_VMLOAD); 4010 svm_set_intercept(svm, INTERCEPT_VMSAVE); 4011 svm->vmcb->control.virt_ext &= ~VIRTUAL_VMLOAD_VMSAVE_ENABLE_MASK; 4012 4013 set_msr_interception(vcpu, svm->msrpm, MSR_IA32_SYSENTER_EIP, 0, 0); 4014 set_msr_interception(vcpu, svm->msrpm, MSR_IA32_SYSENTER_ESP, 0, 0); 4015 } else { 4016 /* 4017 * If hardware supports Virtual VMLOAD VMSAVE then enable it 4018 * in VMCB and clear intercepts to avoid #VMEXIT. 4019 */ 4020 if (vls) { 4021 svm_clr_intercept(svm, INTERCEPT_VMLOAD); 4022 svm_clr_intercept(svm, INTERCEPT_VMSAVE); 4023 svm->vmcb->control.virt_ext |= VIRTUAL_VMLOAD_VMSAVE_ENABLE_MASK; 4024 } 4025 /* No need to intercept these MSRs */ 4026 set_msr_interception(vcpu, svm->msrpm, MSR_IA32_SYSENTER_EIP, 1, 1); 4027 set_msr_interception(vcpu, svm->msrpm, MSR_IA32_SYSENTER_ESP, 1, 1); 4028 } 4029 } 4030 4031 static bool svm_has_wbinvd_exit(void) 4032 { 4033 return true; 4034 } 4035 4036 #define PRE_EX(exit) { .exit_code = (exit), \ 4037 .stage = X86_ICPT_PRE_EXCEPT, } 4038 #define POST_EX(exit) { .exit_code = (exit), \ 4039 .stage = X86_ICPT_POST_EXCEPT, } 4040 #define POST_MEM(exit) { .exit_code = (exit), \ 4041 .stage = X86_ICPT_POST_MEMACCESS, } 4042 4043 static const struct __x86_intercept { 4044 u32 exit_code; 4045 enum x86_intercept_stage stage; 4046 } x86_intercept_map[] = { 4047 [x86_intercept_cr_read] = POST_EX(SVM_EXIT_READ_CR0), 4048 [x86_intercept_cr_write] = POST_EX(SVM_EXIT_WRITE_CR0), 4049 [x86_intercept_clts] = POST_EX(SVM_EXIT_WRITE_CR0), 4050 [x86_intercept_lmsw] = POST_EX(SVM_EXIT_WRITE_CR0), 4051 [x86_intercept_smsw] = POST_EX(SVM_EXIT_READ_CR0), 4052 [x86_intercept_dr_read] = POST_EX(SVM_EXIT_READ_DR0), 4053 [x86_intercept_dr_write] = POST_EX(SVM_EXIT_WRITE_DR0), 4054 [x86_intercept_sldt] = POST_EX(SVM_EXIT_LDTR_READ), 4055 [x86_intercept_str] = POST_EX(SVM_EXIT_TR_READ), 4056 [x86_intercept_lldt] = POST_EX(SVM_EXIT_LDTR_WRITE), 4057 [x86_intercept_ltr] = POST_EX(SVM_EXIT_TR_WRITE), 4058 [x86_intercept_sgdt] = POST_EX(SVM_EXIT_GDTR_READ), 4059 [x86_intercept_sidt] = POST_EX(SVM_EXIT_IDTR_READ), 4060 [x86_intercept_lgdt] = POST_EX(SVM_EXIT_GDTR_WRITE), 4061 [x86_intercept_lidt] = POST_EX(SVM_EXIT_IDTR_WRITE), 4062 [x86_intercept_vmrun] = POST_EX(SVM_EXIT_VMRUN), 4063 [x86_intercept_vmmcall] = POST_EX(SVM_EXIT_VMMCALL), 4064 [x86_intercept_vmload] = POST_EX(SVM_EXIT_VMLOAD), 4065 [x86_intercept_vmsave] = POST_EX(SVM_EXIT_VMSAVE), 4066 [x86_intercept_stgi] = POST_EX(SVM_EXIT_STGI), 4067 [x86_intercept_clgi] = POST_EX(SVM_EXIT_CLGI), 4068 [x86_intercept_skinit] = POST_EX(SVM_EXIT_SKINIT), 4069 [x86_intercept_invlpga] = POST_EX(SVM_EXIT_INVLPGA), 4070 [x86_intercept_rdtscp] = POST_EX(SVM_EXIT_RDTSCP), 4071 [x86_intercept_monitor] = POST_MEM(SVM_EXIT_MONITOR), 4072 [x86_intercept_mwait] = POST_EX(SVM_EXIT_MWAIT), 4073 [x86_intercept_invlpg] = POST_EX(SVM_EXIT_INVLPG), 4074 [x86_intercept_invd] = POST_EX(SVM_EXIT_INVD), 4075 [x86_intercept_wbinvd] = POST_EX(SVM_EXIT_WBINVD), 4076 [x86_intercept_wrmsr] = POST_EX(SVM_EXIT_MSR), 4077 [x86_intercept_rdtsc] = POST_EX(SVM_EXIT_RDTSC), 4078 [x86_intercept_rdmsr] = POST_EX(SVM_EXIT_MSR), 4079 [x86_intercept_rdpmc] = POST_EX(SVM_EXIT_RDPMC), 4080 [x86_intercept_cpuid] = PRE_EX(SVM_EXIT_CPUID), 4081 [x86_intercept_rsm] = PRE_EX(SVM_EXIT_RSM), 4082 [x86_intercept_pause] = PRE_EX(SVM_EXIT_PAUSE), 4083 [x86_intercept_pushf] = PRE_EX(SVM_EXIT_PUSHF), 4084 [x86_intercept_popf] = PRE_EX(SVM_EXIT_POPF), 4085 [x86_intercept_intn] = PRE_EX(SVM_EXIT_SWINT), 4086 [x86_intercept_iret] = PRE_EX(SVM_EXIT_IRET), 4087 [x86_intercept_icebp] = PRE_EX(SVM_EXIT_ICEBP), 4088 [x86_intercept_hlt] = POST_EX(SVM_EXIT_HLT), 4089 [x86_intercept_in] = POST_EX(SVM_EXIT_IOIO), 4090 [x86_intercept_ins] = POST_EX(SVM_EXIT_IOIO), 4091 [x86_intercept_out] = POST_EX(SVM_EXIT_IOIO), 4092 [x86_intercept_outs] = POST_EX(SVM_EXIT_IOIO), 4093 [x86_intercept_xsetbv] = PRE_EX(SVM_EXIT_XSETBV), 4094 }; 4095 4096 #undef PRE_EX 4097 #undef POST_EX 4098 #undef POST_MEM 4099 4100 static int svm_check_intercept(struct kvm_vcpu *vcpu, 4101 struct x86_instruction_info *info, 4102 enum x86_intercept_stage stage, 4103 struct x86_exception *exception) 4104 { 4105 struct vcpu_svm *svm = to_svm(vcpu); 4106 int vmexit, ret = X86EMUL_CONTINUE; 4107 struct __x86_intercept icpt_info; 4108 struct vmcb *vmcb = svm->vmcb; 4109 4110 if (info->intercept >= ARRAY_SIZE(x86_intercept_map)) 4111 goto out; 4112 4113 icpt_info = x86_intercept_map[info->intercept]; 4114 4115 if (stage != icpt_info.stage) 4116 goto out; 4117 4118 switch (icpt_info.exit_code) { 4119 case SVM_EXIT_READ_CR0: 4120 if (info->intercept == x86_intercept_cr_read) 4121 icpt_info.exit_code += info->modrm_reg; 4122 break; 4123 case SVM_EXIT_WRITE_CR0: { 4124 unsigned long cr0, val; 4125 4126 if (info->intercept == x86_intercept_cr_write) 4127 icpt_info.exit_code += info->modrm_reg; 4128 4129 if (icpt_info.exit_code != SVM_EXIT_WRITE_CR0 || 4130 info->intercept == x86_intercept_clts) 4131 break; 4132 4133 if (!(vmcb_is_intercept(&svm->nested.ctl, 4134 INTERCEPT_SELECTIVE_CR0))) 4135 break; 4136 4137 cr0 = vcpu->arch.cr0 & ~SVM_CR0_SELECTIVE_MASK; 4138 val = info->src_val & ~SVM_CR0_SELECTIVE_MASK; 4139 4140 if (info->intercept == x86_intercept_lmsw) { 4141 cr0 &= 0xfUL; 4142 val &= 0xfUL; 4143 /* lmsw can't clear PE - catch this here */ 4144 if (cr0 & X86_CR0_PE) 4145 val |= X86_CR0_PE; 4146 } 4147 4148 if (cr0 ^ val) 4149 icpt_info.exit_code = SVM_EXIT_CR0_SEL_WRITE; 4150 4151 break; 4152 } 4153 case SVM_EXIT_READ_DR0: 4154 case SVM_EXIT_WRITE_DR0: 4155 icpt_info.exit_code += info->modrm_reg; 4156 break; 4157 case SVM_EXIT_MSR: 4158 if (info->intercept == x86_intercept_wrmsr) 4159 vmcb->control.exit_info_1 = 1; 4160 else 4161 vmcb->control.exit_info_1 = 0; 4162 break; 4163 case SVM_EXIT_PAUSE: 4164 /* 4165 * We get this for NOP only, but pause 4166 * is rep not, check this here 4167 */ 4168 if (info->rep_prefix != REPE_PREFIX) 4169 goto out; 4170 break; 4171 case SVM_EXIT_IOIO: { 4172 u64 exit_info; 4173 u32 bytes; 4174 4175 if (info->intercept == x86_intercept_in || 4176 info->intercept == x86_intercept_ins) { 4177 exit_info = ((info->src_val & 0xffff) << 16) | 4178 SVM_IOIO_TYPE_MASK; 4179 bytes = info->dst_bytes; 4180 } else { 4181 exit_info = (info->dst_val & 0xffff) << 16; 4182 bytes = info->src_bytes; 4183 } 4184 4185 if (info->intercept == x86_intercept_outs || 4186 info->intercept == x86_intercept_ins) 4187 exit_info |= SVM_IOIO_STR_MASK; 4188 4189 if (info->rep_prefix) 4190 exit_info |= SVM_IOIO_REP_MASK; 4191 4192 bytes = min(bytes, 4u); 4193 4194 exit_info |= bytes << SVM_IOIO_SIZE_SHIFT; 4195 4196 exit_info |= (u32)info->ad_bytes << (SVM_IOIO_ASIZE_SHIFT - 1); 4197 4198 vmcb->control.exit_info_1 = exit_info; 4199 vmcb->control.exit_info_2 = info->next_rip; 4200 4201 break; 4202 } 4203 default: 4204 break; 4205 } 4206 4207 /* TODO: Advertise NRIPS to guest hypervisor unconditionally */ 4208 if (static_cpu_has(X86_FEATURE_NRIPS)) 4209 vmcb->control.next_rip = info->next_rip; 4210 vmcb->control.exit_code = icpt_info.exit_code; 4211 vmexit = nested_svm_exit_handled(svm); 4212 4213 ret = (vmexit == NESTED_EXIT_DONE) ? X86EMUL_INTERCEPTED 4214 : X86EMUL_CONTINUE; 4215 4216 out: 4217 return ret; 4218 } 4219 4220 static void svm_handle_exit_irqoff(struct kvm_vcpu *vcpu) 4221 { 4222 } 4223 4224 static void svm_sched_in(struct kvm_vcpu *vcpu, int cpu) 4225 { 4226 if (!kvm_pause_in_guest(vcpu->kvm)) 4227 shrink_ple_window(vcpu); 4228 } 4229 4230 static void svm_setup_mce(struct kvm_vcpu *vcpu) 4231 { 4232 /* [63:9] are reserved. */ 4233 vcpu->arch.mcg_cap &= 0x1ff; 4234 } 4235 4236 bool svm_smi_blocked(struct kvm_vcpu *vcpu) 4237 { 4238 struct vcpu_svm *svm = to_svm(vcpu); 4239 4240 /* Per APM Vol.2 15.22.2 "Response to SMI" */ 4241 if (!gif_set(svm)) 4242 return true; 4243 4244 return is_smm(vcpu); 4245 } 4246 4247 static int svm_smi_allowed(struct kvm_vcpu *vcpu, bool for_injection) 4248 { 4249 struct vcpu_svm *svm = to_svm(vcpu); 4250 if (svm->nested.nested_run_pending) 4251 return -EBUSY; 4252 4253 /* An SMI must not be injected into L2 if it's supposed to VM-Exit. */ 4254 if (for_injection && is_guest_mode(vcpu) && nested_exit_on_smi(svm)) 4255 return -EBUSY; 4256 4257 return !svm_smi_blocked(vcpu); 4258 } 4259 4260 static int svm_pre_enter_smm(struct kvm_vcpu *vcpu, char *smstate) 4261 { 4262 struct vcpu_svm *svm = to_svm(vcpu); 4263 int ret; 4264 4265 if (is_guest_mode(vcpu)) { 4266 /* FED8h - SVM Guest */ 4267 put_smstate(u64, smstate, 0x7ed8, 1); 4268 /* FEE0h - SVM Guest VMCB Physical Address */ 4269 put_smstate(u64, smstate, 0x7ee0, svm->nested.vmcb12_gpa); 4270 4271 svm->vmcb->save.rax = vcpu->arch.regs[VCPU_REGS_RAX]; 4272 svm->vmcb->save.rsp = vcpu->arch.regs[VCPU_REGS_RSP]; 4273 svm->vmcb->save.rip = vcpu->arch.regs[VCPU_REGS_RIP]; 4274 4275 ret = nested_svm_vmexit(svm); 4276 if (ret) 4277 return ret; 4278 } 4279 return 0; 4280 } 4281 4282 static int svm_pre_leave_smm(struct kvm_vcpu *vcpu, const char *smstate) 4283 { 4284 struct vcpu_svm *svm = to_svm(vcpu); 4285 struct kvm_host_map map; 4286 int ret = 0; 4287 4288 if (guest_cpuid_has(vcpu, X86_FEATURE_LM)) { 4289 u64 saved_efer = GET_SMSTATE(u64, smstate, 0x7ed0); 4290 u64 guest = GET_SMSTATE(u64, smstate, 0x7ed8); 4291 u64 vmcb12_gpa = GET_SMSTATE(u64, smstate, 0x7ee0); 4292 4293 if (guest) { 4294 if (!guest_cpuid_has(vcpu, X86_FEATURE_SVM)) 4295 return 1; 4296 4297 if (!(saved_efer & EFER_SVME)) 4298 return 1; 4299 4300 if (kvm_vcpu_map(vcpu, 4301 gpa_to_gfn(vmcb12_gpa), &map) == -EINVAL) 4302 return 1; 4303 4304 if (svm_allocate_nested(svm)) 4305 return 1; 4306 4307 ret = enter_svm_guest_mode(vcpu, vmcb12_gpa, map.hva); 4308 kvm_vcpu_unmap(vcpu, &map, true); 4309 } 4310 } 4311 4312 return ret; 4313 } 4314 4315 static void svm_enable_smi_window(struct kvm_vcpu *vcpu) 4316 { 4317 struct vcpu_svm *svm = to_svm(vcpu); 4318 4319 if (!gif_set(svm)) { 4320 if (vgif_enabled(svm)) 4321 svm_set_intercept(svm, INTERCEPT_STGI); 4322 /* STGI will cause a vm exit */ 4323 } else { 4324 /* We must be in SMM; RSM will cause a vmexit anyway. */ 4325 } 4326 } 4327 4328 static bool svm_can_emulate_instruction(struct kvm_vcpu *vcpu, void *insn, int insn_len) 4329 { 4330 bool smep, smap, is_user; 4331 unsigned long cr4; 4332 4333 /* 4334 * When the guest is an SEV-ES guest, emulation is not possible. 4335 */ 4336 if (sev_es_guest(vcpu->kvm)) 4337 return false; 4338 4339 /* 4340 * Detect and workaround Errata 1096 Fam_17h_00_0Fh. 4341 * 4342 * Errata: 4343 * When CPU raise #NPF on guest data access and vCPU CR4.SMAP=1, it is 4344 * possible that CPU microcode implementing DecodeAssist will fail 4345 * to read bytes of instruction which caused #NPF. In this case, 4346 * GuestIntrBytes field of the VMCB on a VMEXIT will incorrectly 4347 * return 0 instead of the correct guest instruction bytes. 4348 * 4349 * This happens because CPU microcode reading instruction bytes 4350 * uses a special opcode which attempts to read data using CPL=0 4351 * privileges. The microcode reads CS:RIP and if it hits a SMAP 4352 * fault, it gives up and returns no instruction bytes. 4353 * 4354 * Detection: 4355 * We reach here in case CPU supports DecodeAssist, raised #NPF and 4356 * returned 0 in GuestIntrBytes field of the VMCB. 4357 * First, errata can only be triggered in case vCPU CR4.SMAP=1. 4358 * Second, if vCPU CR4.SMEP=1, errata could only be triggered 4359 * in case vCPU CPL==3 (Because otherwise guest would have triggered 4360 * a SMEP fault instead of #NPF). 4361 * Otherwise, vCPU CR4.SMEP=0, errata could be triggered by any vCPU CPL. 4362 * As most guests enable SMAP if they have also enabled SMEP, use above 4363 * logic in order to attempt minimize false-positive of detecting errata 4364 * while still preserving all cases semantic correctness. 4365 * 4366 * Workaround: 4367 * To determine what instruction the guest was executing, the hypervisor 4368 * will have to decode the instruction at the instruction pointer. 4369 * 4370 * In non SEV guest, hypervisor will be able to read the guest 4371 * memory to decode the instruction pointer when insn_len is zero 4372 * so we return true to indicate that decoding is possible. 4373 * 4374 * But in the SEV guest, the guest memory is encrypted with the 4375 * guest specific key and hypervisor will not be able to decode the 4376 * instruction pointer so we will not able to workaround it. Lets 4377 * print the error and request to kill the guest. 4378 */ 4379 if (likely(!insn || insn_len)) 4380 return true; 4381 4382 /* 4383 * If RIP is invalid, go ahead with emulation which will cause an 4384 * internal error exit. 4385 */ 4386 if (!kvm_vcpu_gfn_to_memslot(vcpu, kvm_rip_read(vcpu) >> PAGE_SHIFT)) 4387 return true; 4388 4389 cr4 = kvm_read_cr4(vcpu); 4390 smep = cr4 & X86_CR4_SMEP; 4391 smap = cr4 & X86_CR4_SMAP; 4392 is_user = svm_get_cpl(vcpu) == 3; 4393 if (smap && (!smep || is_user)) { 4394 if (!sev_guest(vcpu->kvm)) 4395 return true; 4396 4397 pr_err_ratelimited("KVM: SEV Guest triggered AMD Erratum 1096\n"); 4398 kvm_make_request(KVM_REQ_TRIPLE_FAULT, vcpu); 4399 } 4400 4401 return false; 4402 } 4403 4404 static bool svm_apic_init_signal_blocked(struct kvm_vcpu *vcpu) 4405 { 4406 struct vcpu_svm *svm = to_svm(vcpu); 4407 4408 /* 4409 * TODO: Last condition latch INIT signals on vCPU when 4410 * vCPU is in guest-mode and vmcb12 defines intercept on INIT. 4411 * To properly emulate the INIT intercept, 4412 * svm_check_nested_events() should call nested_svm_vmexit() 4413 * if an INIT signal is pending. 4414 */ 4415 return !gif_set(svm) || 4416 (vmcb_is_intercept(&svm->vmcb->control, INTERCEPT_INIT)); 4417 } 4418 4419 static void svm_vcpu_deliver_sipi_vector(struct kvm_vcpu *vcpu, u8 vector) 4420 { 4421 if (!sev_es_guest(vcpu->kvm)) 4422 return kvm_vcpu_deliver_sipi_vector(vcpu, vector); 4423 4424 sev_vcpu_deliver_sipi_vector(vcpu, vector); 4425 } 4426 4427 static void svm_vm_destroy(struct kvm *kvm) 4428 { 4429 avic_vm_destroy(kvm); 4430 sev_vm_destroy(kvm); 4431 } 4432 4433 static int svm_vm_init(struct kvm *kvm) 4434 { 4435 if (!pause_filter_count || !pause_filter_thresh) 4436 kvm->arch.pause_in_guest = true; 4437 4438 if (avic) { 4439 int ret = avic_vm_init(kvm); 4440 if (ret) 4441 return ret; 4442 } 4443 4444 kvm_apicv_init(kvm, avic); 4445 return 0; 4446 } 4447 4448 static struct kvm_x86_ops svm_x86_ops __initdata = { 4449 .hardware_unsetup = svm_hardware_teardown, 4450 .hardware_enable = svm_hardware_enable, 4451 .hardware_disable = svm_hardware_disable, 4452 .cpu_has_accelerated_tpr = svm_cpu_has_accelerated_tpr, 4453 .has_emulated_msr = svm_has_emulated_msr, 4454 4455 .vcpu_create = svm_create_vcpu, 4456 .vcpu_free = svm_free_vcpu, 4457 .vcpu_reset = svm_vcpu_reset, 4458 4459 .vm_size = sizeof(struct kvm_svm), 4460 .vm_init = svm_vm_init, 4461 .vm_destroy = svm_vm_destroy, 4462 4463 .prepare_guest_switch = svm_prepare_guest_switch, 4464 .vcpu_load = svm_vcpu_load, 4465 .vcpu_put = svm_vcpu_put, 4466 .vcpu_blocking = svm_vcpu_blocking, 4467 .vcpu_unblocking = svm_vcpu_unblocking, 4468 4469 .update_exception_bitmap = svm_update_exception_bitmap, 4470 .get_msr_feature = svm_get_msr_feature, 4471 .get_msr = svm_get_msr, 4472 .set_msr = svm_set_msr, 4473 .get_segment_base = svm_get_segment_base, 4474 .get_segment = svm_get_segment, 4475 .set_segment = svm_set_segment, 4476 .get_cpl = svm_get_cpl, 4477 .get_cs_db_l_bits = kvm_get_cs_db_l_bits, 4478 .set_cr0 = svm_set_cr0, 4479 .is_valid_cr4 = svm_is_valid_cr4, 4480 .set_cr4 = svm_set_cr4, 4481 .set_efer = svm_set_efer, 4482 .get_idt = svm_get_idt, 4483 .set_idt = svm_set_idt, 4484 .get_gdt = svm_get_gdt, 4485 .set_gdt = svm_set_gdt, 4486 .set_dr7 = svm_set_dr7, 4487 .sync_dirty_debug_regs = svm_sync_dirty_debug_regs, 4488 .cache_reg = svm_cache_reg, 4489 .get_rflags = svm_get_rflags, 4490 .set_rflags = svm_set_rflags, 4491 4492 .tlb_flush_all = svm_flush_tlb, 4493 .tlb_flush_current = svm_flush_tlb, 4494 .tlb_flush_gva = svm_flush_tlb_gva, 4495 .tlb_flush_guest = svm_flush_tlb, 4496 4497 .run = svm_vcpu_run, 4498 .handle_exit = handle_exit, 4499 .skip_emulated_instruction = skip_emulated_instruction, 4500 .update_emulated_instruction = NULL, 4501 .set_interrupt_shadow = svm_set_interrupt_shadow, 4502 .get_interrupt_shadow = svm_get_interrupt_shadow, 4503 .patch_hypercall = svm_patch_hypercall, 4504 .set_irq = svm_set_irq, 4505 .set_nmi = svm_inject_nmi, 4506 .queue_exception = svm_queue_exception, 4507 .cancel_injection = svm_cancel_injection, 4508 .interrupt_allowed = svm_interrupt_allowed, 4509 .nmi_allowed = svm_nmi_allowed, 4510 .get_nmi_mask = svm_get_nmi_mask, 4511 .set_nmi_mask = svm_set_nmi_mask, 4512 .enable_nmi_window = svm_enable_nmi_window, 4513 .enable_irq_window = svm_enable_irq_window, 4514 .update_cr8_intercept = svm_update_cr8_intercept, 4515 .set_virtual_apic_mode = svm_set_virtual_apic_mode, 4516 .refresh_apicv_exec_ctrl = svm_refresh_apicv_exec_ctrl, 4517 .check_apicv_inhibit_reasons = svm_check_apicv_inhibit_reasons, 4518 .pre_update_apicv_exec_ctrl = svm_pre_update_apicv_exec_ctrl, 4519 .load_eoi_exitmap = svm_load_eoi_exitmap, 4520 .hwapic_irr_update = svm_hwapic_irr_update, 4521 .hwapic_isr_update = svm_hwapic_isr_update, 4522 .sync_pir_to_irr = kvm_lapic_find_highest_irr, 4523 .apicv_post_state_restore = avic_post_state_restore, 4524 4525 .set_tss_addr = svm_set_tss_addr, 4526 .set_identity_map_addr = svm_set_identity_map_addr, 4527 .get_mt_mask = svm_get_mt_mask, 4528 4529 .get_exit_info = svm_get_exit_info, 4530 4531 .vcpu_after_set_cpuid = svm_vcpu_after_set_cpuid, 4532 4533 .has_wbinvd_exit = svm_has_wbinvd_exit, 4534 4535 .write_l1_tsc_offset = svm_write_l1_tsc_offset, 4536 4537 .load_mmu_pgd = svm_load_mmu_pgd, 4538 4539 .check_intercept = svm_check_intercept, 4540 .handle_exit_irqoff = svm_handle_exit_irqoff, 4541 4542 .request_immediate_exit = __kvm_request_immediate_exit, 4543 4544 .sched_in = svm_sched_in, 4545 4546 .pmu_ops = &amd_pmu_ops, 4547 .nested_ops = &svm_nested_ops, 4548 4549 .deliver_posted_interrupt = svm_deliver_avic_intr, 4550 .dy_apicv_has_pending_interrupt = svm_dy_apicv_has_pending_interrupt, 4551 .update_pi_irte = svm_update_pi_irte, 4552 .setup_mce = svm_setup_mce, 4553 4554 .smi_allowed = svm_smi_allowed, 4555 .pre_enter_smm = svm_pre_enter_smm, 4556 .pre_leave_smm = svm_pre_leave_smm, 4557 .enable_smi_window = svm_enable_smi_window, 4558 4559 .mem_enc_op = svm_mem_enc_op, 4560 .mem_enc_reg_region = svm_register_enc_region, 4561 .mem_enc_unreg_region = svm_unregister_enc_region, 4562 4563 .vm_copy_enc_context_from = svm_vm_copy_asid_from, 4564 4565 .can_emulate_instruction = svm_can_emulate_instruction, 4566 4567 .apic_init_signal_blocked = svm_apic_init_signal_blocked, 4568 4569 .msr_filter_changed = svm_msr_filter_changed, 4570 .complete_emulated_msr = svm_complete_emulated_msr, 4571 4572 .vcpu_deliver_sipi_vector = svm_vcpu_deliver_sipi_vector, 4573 }; 4574 4575 static struct kvm_x86_init_ops svm_init_ops __initdata = { 4576 .cpu_has_kvm_support = has_svm, 4577 .disabled_by_bios = is_disabled, 4578 .hardware_setup = svm_hardware_setup, 4579 .check_processor_compatibility = svm_check_processor_compat, 4580 4581 .runtime_ops = &svm_x86_ops, 4582 }; 4583 4584 static int __init svm_init(void) 4585 { 4586 __unused_size_checks(); 4587 4588 return kvm_init(&svm_init_ops, sizeof(struct vcpu_svm), 4589 __alignof__(struct vcpu_svm), THIS_MODULE); 4590 } 4591 4592 static void __exit svm_exit(void) 4593 { 4594 kvm_exit(); 4595 } 4596 4597 module_init(svm_init) 4598 module_exit(svm_exit) 4599