1 #define pr_fmt(fmt) "SVM: " fmt 2 3 #include <linux/kvm_host.h> 4 5 #include "irq.h" 6 #include "mmu.h" 7 #include "kvm_cache_regs.h" 8 #include "x86.h" 9 #include "cpuid.h" 10 #include "pmu.h" 11 12 #include <linux/module.h> 13 #include <linux/mod_devicetable.h> 14 #include <linux/kernel.h> 15 #include <linux/vmalloc.h> 16 #include <linux/highmem.h> 17 #include <linux/amd-iommu.h> 18 #include <linux/sched.h> 19 #include <linux/trace_events.h> 20 #include <linux/slab.h> 21 #include <linux/hashtable.h> 22 #include <linux/objtool.h> 23 #include <linux/psp-sev.h> 24 #include <linux/file.h> 25 #include <linux/pagemap.h> 26 #include <linux/swap.h> 27 #include <linux/rwsem.h> 28 #include <linux/cc_platform.h> 29 30 #include <asm/apic.h> 31 #include <asm/perf_event.h> 32 #include <asm/tlbflush.h> 33 #include <asm/desc.h> 34 #include <asm/debugreg.h> 35 #include <asm/kvm_para.h> 36 #include <asm/irq_remapping.h> 37 #include <asm/spec-ctrl.h> 38 #include <asm/cpu_device_id.h> 39 #include <asm/traps.h> 40 #include <asm/fpu/api.h> 41 42 #include <asm/virtext.h> 43 #include "trace.h" 44 45 #include "svm.h" 46 #include "svm_ops.h" 47 48 #include "kvm_onhyperv.h" 49 #include "svm_onhyperv.h" 50 51 MODULE_AUTHOR("Qumranet"); 52 MODULE_LICENSE("GPL"); 53 54 #ifdef MODULE 55 static const struct x86_cpu_id svm_cpu_id[] = { 56 X86_MATCH_FEATURE(X86_FEATURE_SVM, NULL), 57 {} 58 }; 59 MODULE_DEVICE_TABLE(x86cpu, svm_cpu_id); 60 #endif 61 62 #define SEG_TYPE_LDT 2 63 #define SEG_TYPE_BUSY_TSS16 3 64 65 #define SVM_FEATURE_LBRV (1 << 1) 66 #define SVM_FEATURE_SVML (1 << 2) 67 #define SVM_FEATURE_TSC_RATE (1 << 4) 68 #define SVM_FEATURE_VMCB_CLEAN (1 << 5) 69 #define SVM_FEATURE_FLUSH_ASID (1 << 6) 70 #define SVM_FEATURE_DECODE_ASSIST (1 << 7) 71 #define SVM_FEATURE_PAUSE_FILTER (1 << 10) 72 73 #define DEBUGCTL_RESERVED_BITS (~(0x3fULL)) 74 75 #define TSC_RATIO_RSVD 0xffffff0000000000ULL 76 #define TSC_RATIO_MIN 0x0000000000000001ULL 77 #define TSC_RATIO_MAX 0x000000ffffffffffULL 78 79 static bool erratum_383_found __read_mostly; 80 81 u32 msrpm_offsets[MSRPM_OFFSETS] __read_mostly; 82 83 /* 84 * Set osvw_len to higher value when updated Revision Guides 85 * are published and we know what the new status bits are 86 */ 87 static uint64_t osvw_len = 4, osvw_status; 88 89 static DEFINE_PER_CPU(u64, current_tsc_ratio); 90 #define TSC_RATIO_DEFAULT 0x0100000000ULL 91 92 static const struct svm_direct_access_msrs { 93 u32 index; /* Index of the MSR */ 94 bool always; /* True if intercept is initially cleared */ 95 } direct_access_msrs[MAX_DIRECT_ACCESS_MSRS] = { 96 { .index = MSR_STAR, .always = true }, 97 { .index = MSR_IA32_SYSENTER_CS, .always = true }, 98 { .index = MSR_IA32_SYSENTER_EIP, .always = false }, 99 { .index = MSR_IA32_SYSENTER_ESP, .always = false }, 100 #ifdef CONFIG_X86_64 101 { .index = MSR_GS_BASE, .always = true }, 102 { .index = MSR_FS_BASE, .always = true }, 103 { .index = MSR_KERNEL_GS_BASE, .always = true }, 104 { .index = MSR_LSTAR, .always = true }, 105 { .index = MSR_CSTAR, .always = true }, 106 { .index = MSR_SYSCALL_MASK, .always = true }, 107 #endif 108 { .index = MSR_IA32_SPEC_CTRL, .always = false }, 109 { .index = MSR_IA32_PRED_CMD, .always = false }, 110 { .index = MSR_IA32_LASTBRANCHFROMIP, .always = false }, 111 { .index = MSR_IA32_LASTBRANCHTOIP, .always = false }, 112 { .index = MSR_IA32_LASTINTFROMIP, .always = false }, 113 { .index = MSR_IA32_LASTINTTOIP, .always = false }, 114 { .index = MSR_EFER, .always = false }, 115 { .index = MSR_IA32_CR_PAT, .always = false }, 116 { .index = MSR_AMD64_SEV_ES_GHCB, .always = true }, 117 { .index = MSR_INVALID, .always = false }, 118 }; 119 120 /* 121 * These 2 parameters are used to config the controls for Pause-Loop Exiting: 122 * pause_filter_count: On processors that support Pause filtering(indicated 123 * by CPUID Fn8000_000A_EDX), the VMCB provides a 16 bit pause filter 124 * count value. On VMRUN this value is loaded into an internal counter. 125 * Each time a pause instruction is executed, this counter is decremented 126 * until it reaches zero at which time a #VMEXIT is generated if pause 127 * intercept is enabled. Refer to AMD APM Vol 2 Section 15.14.4 Pause 128 * Intercept Filtering for more details. 129 * This also indicate if ple logic enabled. 130 * 131 * pause_filter_thresh: In addition, some processor families support advanced 132 * pause filtering (indicated by CPUID Fn8000_000A_EDX) upper bound on 133 * the amount of time a guest is allowed to execute in a pause loop. 134 * In this mode, a 16-bit pause filter threshold field is added in the 135 * VMCB. The threshold value is a cycle count that is used to reset the 136 * pause counter. As with simple pause filtering, VMRUN loads the pause 137 * count value from VMCB into an internal counter. Then, on each pause 138 * instruction the hardware checks the elapsed number of cycles since 139 * the most recent pause instruction against the pause filter threshold. 140 * If the elapsed cycle count is greater than the pause filter threshold, 141 * then the internal pause count is reloaded from the VMCB and execution 142 * continues. If the elapsed cycle count is less than the pause filter 143 * threshold, then the internal pause count is decremented. If the count 144 * value is less than zero and PAUSE intercept is enabled, a #VMEXIT is 145 * triggered. If advanced pause filtering is supported and pause filter 146 * threshold field is set to zero, the filter will operate in the simpler, 147 * count only mode. 148 */ 149 150 static unsigned short pause_filter_thresh = KVM_DEFAULT_PLE_GAP; 151 module_param(pause_filter_thresh, ushort, 0444); 152 153 static unsigned short pause_filter_count = KVM_SVM_DEFAULT_PLE_WINDOW; 154 module_param(pause_filter_count, ushort, 0444); 155 156 /* Default doubles per-vcpu window every exit. */ 157 static unsigned short pause_filter_count_grow = KVM_DEFAULT_PLE_WINDOW_GROW; 158 module_param(pause_filter_count_grow, ushort, 0444); 159 160 /* Default resets per-vcpu window every exit to pause_filter_count. */ 161 static unsigned short pause_filter_count_shrink = KVM_DEFAULT_PLE_WINDOW_SHRINK; 162 module_param(pause_filter_count_shrink, ushort, 0444); 163 164 /* Default is to compute the maximum so we can never overflow. */ 165 static unsigned short pause_filter_count_max = KVM_SVM_DEFAULT_PLE_WINDOW_MAX; 166 module_param(pause_filter_count_max, ushort, 0444); 167 168 /* 169 * Use nested page tables by default. Note, NPT may get forced off by 170 * svm_hardware_setup() if it's unsupported by hardware or the host kernel. 171 */ 172 bool npt_enabled = true; 173 module_param_named(npt, npt_enabled, bool, 0444); 174 175 /* allow nested virtualization in KVM/SVM */ 176 static int nested = true; 177 module_param(nested, int, S_IRUGO); 178 179 /* enable/disable Next RIP Save */ 180 static int nrips = true; 181 module_param(nrips, int, 0444); 182 183 /* enable/disable Virtual VMLOAD VMSAVE */ 184 static int vls = true; 185 module_param(vls, int, 0444); 186 187 /* enable/disable Virtual GIF */ 188 static int vgif = true; 189 module_param(vgif, int, 0444); 190 191 /* enable/disable LBR virtualization */ 192 static int lbrv = true; 193 module_param(lbrv, int, 0444); 194 195 static int tsc_scaling = true; 196 module_param(tsc_scaling, int, 0444); 197 198 /* 199 * enable / disable AVIC. Because the defaults differ for APICv 200 * support between VMX and SVM we cannot use module_param_named. 201 */ 202 static bool avic; 203 module_param(avic, bool, 0444); 204 205 bool __read_mostly dump_invalid_vmcb; 206 module_param(dump_invalid_vmcb, bool, 0644); 207 208 209 bool intercept_smi = true; 210 module_param(intercept_smi, bool, 0444); 211 212 213 static bool svm_gp_erratum_intercept = true; 214 215 static u8 rsm_ins_bytes[] = "\x0f\xaa"; 216 217 static unsigned long iopm_base; 218 219 struct kvm_ldttss_desc { 220 u16 limit0; 221 u16 base0; 222 unsigned base1:8, type:5, dpl:2, p:1; 223 unsigned limit1:4, zero0:3, g:1, base2:8; 224 u32 base3; 225 u32 zero1; 226 } __attribute__((packed)); 227 228 DEFINE_PER_CPU(struct svm_cpu_data *, svm_data); 229 230 /* 231 * Only MSR_TSC_AUX is switched via the user return hook. EFER is switched via 232 * the VMCB, and the SYSCALL/SYSENTER MSRs are handled by VMLOAD/VMSAVE. 233 * 234 * RDTSCP and RDPID are not used in the kernel, specifically to allow KVM to 235 * defer the restoration of TSC_AUX until the CPU returns to userspace. 236 */ 237 static int tsc_aux_uret_slot __read_mostly = -1; 238 239 static const u32 msrpm_ranges[] = {0, 0xc0000000, 0xc0010000}; 240 241 #define NUM_MSR_MAPS ARRAY_SIZE(msrpm_ranges) 242 #define MSRS_RANGE_SIZE 2048 243 #define MSRS_IN_RANGE (MSRS_RANGE_SIZE * 8 / 2) 244 245 u32 svm_msrpm_offset(u32 msr) 246 { 247 u32 offset; 248 int i; 249 250 for (i = 0; i < NUM_MSR_MAPS; i++) { 251 if (msr < msrpm_ranges[i] || 252 msr >= msrpm_ranges[i] + MSRS_IN_RANGE) 253 continue; 254 255 offset = (msr - msrpm_ranges[i]) / 4; /* 4 msrs per u8 */ 256 offset += (i * MSRS_RANGE_SIZE); /* add range offset */ 257 258 /* Now we have the u8 offset - but need the u32 offset */ 259 return offset / 4; 260 } 261 262 /* MSR not in any range */ 263 return MSR_INVALID; 264 } 265 266 #define MAX_INST_SIZE 15 267 268 static int get_npt_level(void) 269 { 270 #ifdef CONFIG_X86_64 271 return pgtable_l5_enabled() ? PT64_ROOT_5LEVEL : PT64_ROOT_4LEVEL; 272 #else 273 return PT32E_ROOT_LEVEL; 274 #endif 275 } 276 277 int svm_set_efer(struct kvm_vcpu *vcpu, u64 efer) 278 { 279 struct vcpu_svm *svm = to_svm(vcpu); 280 u64 old_efer = vcpu->arch.efer; 281 vcpu->arch.efer = efer; 282 283 if (!npt_enabled) { 284 /* Shadow paging assumes NX to be available. */ 285 efer |= EFER_NX; 286 287 if (!(efer & EFER_LMA)) 288 efer &= ~EFER_LME; 289 } 290 291 if ((old_efer & EFER_SVME) != (efer & EFER_SVME)) { 292 if (!(efer & EFER_SVME)) { 293 svm_leave_nested(svm); 294 svm_set_gif(svm, true); 295 /* #GP intercept is still needed for vmware backdoor */ 296 if (!enable_vmware_backdoor) 297 clr_exception_intercept(svm, GP_VECTOR); 298 299 /* 300 * Free the nested guest state, unless we are in SMM. 301 * In this case we will return to the nested guest 302 * as soon as we leave SMM. 303 */ 304 if (!is_smm(vcpu)) 305 svm_free_nested(svm); 306 307 } else { 308 int ret = svm_allocate_nested(svm); 309 310 if (ret) { 311 vcpu->arch.efer = old_efer; 312 return ret; 313 } 314 315 if (svm_gp_erratum_intercept) 316 set_exception_intercept(svm, GP_VECTOR); 317 } 318 } 319 320 svm->vmcb->save.efer = efer | EFER_SVME; 321 vmcb_mark_dirty(svm->vmcb, VMCB_CR); 322 return 0; 323 } 324 325 static int is_external_interrupt(u32 info) 326 { 327 info &= SVM_EVTINJ_TYPE_MASK | SVM_EVTINJ_VALID; 328 return info == (SVM_EVTINJ_VALID | SVM_EVTINJ_TYPE_INTR); 329 } 330 331 static u32 svm_get_interrupt_shadow(struct kvm_vcpu *vcpu) 332 { 333 struct vcpu_svm *svm = to_svm(vcpu); 334 u32 ret = 0; 335 336 if (svm->vmcb->control.int_state & SVM_INTERRUPT_SHADOW_MASK) 337 ret = KVM_X86_SHADOW_INT_STI | KVM_X86_SHADOW_INT_MOV_SS; 338 return ret; 339 } 340 341 static void svm_set_interrupt_shadow(struct kvm_vcpu *vcpu, int mask) 342 { 343 struct vcpu_svm *svm = to_svm(vcpu); 344 345 if (mask == 0) 346 svm->vmcb->control.int_state &= ~SVM_INTERRUPT_SHADOW_MASK; 347 else 348 svm->vmcb->control.int_state |= SVM_INTERRUPT_SHADOW_MASK; 349 350 } 351 352 static int skip_emulated_instruction(struct kvm_vcpu *vcpu) 353 { 354 struct vcpu_svm *svm = to_svm(vcpu); 355 356 /* 357 * SEV-ES does not expose the next RIP. The RIP update is controlled by 358 * the type of exit and the #VC handler in the guest. 359 */ 360 if (sev_es_guest(vcpu->kvm)) 361 goto done; 362 363 if (nrips && svm->vmcb->control.next_rip != 0) { 364 WARN_ON_ONCE(!static_cpu_has(X86_FEATURE_NRIPS)); 365 svm->next_rip = svm->vmcb->control.next_rip; 366 } 367 368 if (!svm->next_rip) { 369 if (!kvm_emulate_instruction(vcpu, EMULTYPE_SKIP)) 370 return 0; 371 } else { 372 kvm_rip_write(vcpu, svm->next_rip); 373 } 374 375 done: 376 svm_set_interrupt_shadow(vcpu, 0); 377 378 return 1; 379 } 380 381 static void svm_queue_exception(struct kvm_vcpu *vcpu) 382 { 383 struct vcpu_svm *svm = to_svm(vcpu); 384 unsigned nr = vcpu->arch.exception.nr; 385 bool has_error_code = vcpu->arch.exception.has_error_code; 386 u32 error_code = vcpu->arch.exception.error_code; 387 388 kvm_deliver_exception_payload(vcpu); 389 390 if (nr == BP_VECTOR && !nrips) { 391 unsigned long rip, old_rip = kvm_rip_read(vcpu); 392 393 /* 394 * For guest debugging where we have to reinject #BP if some 395 * INT3 is guest-owned: 396 * Emulate nRIP by moving RIP forward. Will fail if injection 397 * raises a fault that is not intercepted. Still better than 398 * failing in all cases. 399 */ 400 (void)skip_emulated_instruction(vcpu); 401 rip = kvm_rip_read(vcpu); 402 svm->int3_rip = rip + svm->vmcb->save.cs.base; 403 svm->int3_injected = rip - old_rip; 404 } 405 406 svm->vmcb->control.event_inj = nr 407 | SVM_EVTINJ_VALID 408 | (has_error_code ? SVM_EVTINJ_VALID_ERR : 0) 409 | SVM_EVTINJ_TYPE_EXEPT; 410 svm->vmcb->control.event_inj_err = error_code; 411 } 412 413 static void svm_init_erratum_383(void) 414 { 415 u32 low, high; 416 int err; 417 u64 val; 418 419 if (!static_cpu_has_bug(X86_BUG_AMD_TLB_MMATCH)) 420 return; 421 422 /* Use _safe variants to not break nested virtualization */ 423 val = native_read_msr_safe(MSR_AMD64_DC_CFG, &err); 424 if (err) 425 return; 426 427 val |= (1ULL << 47); 428 429 low = lower_32_bits(val); 430 high = upper_32_bits(val); 431 432 native_write_msr_safe(MSR_AMD64_DC_CFG, low, high); 433 434 erratum_383_found = true; 435 } 436 437 static void svm_init_osvw(struct kvm_vcpu *vcpu) 438 { 439 /* 440 * Guests should see errata 400 and 415 as fixed (assuming that 441 * HLT and IO instructions are intercepted). 442 */ 443 vcpu->arch.osvw.length = (osvw_len >= 3) ? (osvw_len) : 3; 444 vcpu->arch.osvw.status = osvw_status & ~(6ULL); 445 446 /* 447 * By increasing VCPU's osvw.length to 3 we are telling the guest that 448 * all osvw.status bits inside that length, including bit 0 (which is 449 * reserved for erratum 298), are valid. However, if host processor's 450 * osvw_len is 0 then osvw_status[0] carries no information. We need to 451 * be conservative here and therefore we tell the guest that erratum 298 452 * is present (because we really don't know). 453 */ 454 if (osvw_len == 0 && boot_cpu_data.x86 == 0x10) 455 vcpu->arch.osvw.status |= 1; 456 } 457 458 static int has_svm(void) 459 { 460 const char *msg; 461 462 if (!cpu_has_svm(&msg)) { 463 printk(KERN_INFO "has_svm: %s\n", msg); 464 return 0; 465 } 466 467 if (cc_platform_has(CC_ATTR_GUEST_MEM_ENCRYPT)) { 468 pr_info("KVM is unsupported when running as an SEV guest\n"); 469 return 0; 470 } 471 472 return 1; 473 } 474 475 static void svm_hardware_disable(void) 476 { 477 /* Make sure we clean up behind us */ 478 if (tsc_scaling) 479 wrmsrl(MSR_AMD64_TSC_RATIO, TSC_RATIO_DEFAULT); 480 481 cpu_svm_disable(); 482 483 amd_pmu_disable_virt(); 484 } 485 486 static int svm_hardware_enable(void) 487 { 488 489 struct svm_cpu_data *sd; 490 uint64_t efer; 491 struct desc_struct *gdt; 492 int me = raw_smp_processor_id(); 493 494 rdmsrl(MSR_EFER, efer); 495 if (efer & EFER_SVME) 496 return -EBUSY; 497 498 if (!has_svm()) { 499 pr_err("%s: err EOPNOTSUPP on %d\n", __func__, me); 500 return -EINVAL; 501 } 502 sd = per_cpu(svm_data, me); 503 if (!sd) { 504 pr_err("%s: svm_data is NULL on %d\n", __func__, me); 505 return -EINVAL; 506 } 507 508 sd->asid_generation = 1; 509 sd->max_asid = cpuid_ebx(SVM_CPUID_FUNC) - 1; 510 sd->next_asid = sd->max_asid + 1; 511 sd->min_asid = max_sev_asid + 1; 512 513 gdt = get_current_gdt_rw(); 514 sd->tss_desc = (struct kvm_ldttss_desc *)(gdt + GDT_ENTRY_TSS); 515 516 wrmsrl(MSR_EFER, efer | EFER_SVME); 517 518 wrmsrl(MSR_VM_HSAVE_PA, __sme_page_pa(sd->save_area)); 519 520 if (static_cpu_has(X86_FEATURE_TSCRATEMSR)) { 521 /* 522 * Set the default value, even if we don't use TSC scaling 523 * to avoid having stale value in the msr 524 */ 525 wrmsrl(MSR_AMD64_TSC_RATIO, TSC_RATIO_DEFAULT); 526 __this_cpu_write(current_tsc_ratio, TSC_RATIO_DEFAULT); 527 } 528 529 530 /* 531 * Get OSVW bits. 532 * 533 * Note that it is possible to have a system with mixed processor 534 * revisions and therefore different OSVW bits. If bits are not the same 535 * on different processors then choose the worst case (i.e. if erratum 536 * is present on one processor and not on another then assume that the 537 * erratum is present everywhere). 538 */ 539 if (cpu_has(&boot_cpu_data, X86_FEATURE_OSVW)) { 540 uint64_t len, status = 0; 541 int err; 542 543 len = native_read_msr_safe(MSR_AMD64_OSVW_ID_LENGTH, &err); 544 if (!err) 545 status = native_read_msr_safe(MSR_AMD64_OSVW_STATUS, 546 &err); 547 548 if (err) 549 osvw_status = osvw_len = 0; 550 else { 551 if (len < osvw_len) 552 osvw_len = len; 553 osvw_status |= status; 554 osvw_status &= (1ULL << osvw_len) - 1; 555 } 556 } else 557 osvw_status = osvw_len = 0; 558 559 svm_init_erratum_383(); 560 561 amd_pmu_enable_virt(); 562 563 return 0; 564 } 565 566 static void svm_cpu_uninit(int cpu) 567 { 568 struct svm_cpu_data *sd = per_cpu(svm_data, cpu); 569 570 if (!sd) 571 return; 572 573 per_cpu(svm_data, cpu) = NULL; 574 kfree(sd->sev_vmcbs); 575 __free_page(sd->save_area); 576 kfree(sd); 577 } 578 579 static int svm_cpu_init(int cpu) 580 { 581 struct svm_cpu_data *sd; 582 int ret = -ENOMEM; 583 584 sd = kzalloc(sizeof(struct svm_cpu_data), GFP_KERNEL); 585 if (!sd) 586 return ret; 587 sd->cpu = cpu; 588 sd->save_area = alloc_page(GFP_KERNEL | __GFP_ZERO); 589 if (!sd->save_area) 590 goto free_cpu_data; 591 592 ret = sev_cpu_init(sd); 593 if (ret) 594 goto free_save_area; 595 596 per_cpu(svm_data, cpu) = sd; 597 598 return 0; 599 600 free_save_area: 601 __free_page(sd->save_area); 602 free_cpu_data: 603 kfree(sd); 604 return ret; 605 606 } 607 608 static int direct_access_msr_slot(u32 msr) 609 { 610 u32 i; 611 612 for (i = 0; direct_access_msrs[i].index != MSR_INVALID; i++) 613 if (direct_access_msrs[i].index == msr) 614 return i; 615 616 return -ENOENT; 617 } 618 619 static void set_shadow_msr_intercept(struct kvm_vcpu *vcpu, u32 msr, int read, 620 int write) 621 { 622 struct vcpu_svm *svm = to_svm(vcpu); 623 int slot = direct_access_msr_slot(msr); 624 625 if (slot == -ENOENT) 626 return; 627 628 /* Set the shadow bitmaps to the desired intercept states */ 629 if (read) 630 set_bit(slot, svm->shadow_msr_intercept.read); 631 else 632 clear_bit(slot, svm->shadow_msr_intercept.read); 633 634 if (write) 635 set_bit(slot, svm->shadow_msr_intercept.write); 636 else 637 clear_bit(slot, svm->shadow_msr_intercept.write); 638 } 639 640 static bool valid_msr_intercept(u32 index) 641 { 642 return direct_access_msr_slot(index) != -ENOENT; 643 } 644 645 static bool msr_write_intercepted(struct kvm_vcpu *vcpu, u32 msr) 646 { 647 u8 bit_write; 648 unsigned long tmp; 649 u32 offset; 650 u32 *msrpm; 651 652 msrpm = is_guest_mode(vcpu) ? to_svm(vcpu)->nested.msrpm: 653 to_svm(vcpu)->msrpm; 654 655 offset = svm_msrpm_offset(msr); 656 bit_write = 2 * (msr & 0x0f) + 1; 657 tmp = msrpm[offset]; 658 659 BUG_ON(offset == MSR_INVALID); 660 661 return !!test_bit(bit_write, &tmp); 662 } 663 664 static void set_msr_interception_bitmap(struct kvm_vcpu *vcpu, u32 *msrpm, 665 u32 msr, int read, int write) 666 { 667 u8 bit_read, bit_write; 668 unsigned long tmp; 669 u32 offset; 670 671 /* 672 * If this warning triggers extend the direct_access_msrs list at the 673 * beginning of the file 674 */ 675 WARN_ON(!valid_msr_intercept(msr)); 676 677 /* Enforce non allowed MSRs to trap */ 678 if (read && !kvm_msr_allowed(vcpu, msr, KVM_MSR_FILTER_READ)) 679 read = 0; 680 681 if (write && !kvm_msr_allowed(vcpu, msr, KVM_MSR_FILTER_WRITE)) 682 write = 0; 683 684 offset = svm_msrpm_offset(msr); 685 bit_read = 2 * (msr & 0x0f); 686 bit_write = 2 * (msr & 0x0f) + 1; 687 tmp = msrpm[offset]; 688 689 BUG_ON(offset == MSR_INVALID); 690 691 read ? clear_bit(bit_read, &tmp) : set_bit(bit_read, &tmp); 692 write ? clear_bit(bit_write, &tmp) : set_bit(bit_write, &tmp); 693 694 msrpm[offset] = tmp; 695 696 svm_hv_vmcb_dirty_nested_enlightenments(vcpu); 697 698 } 699 700 void set_msr_interception(struct kvm_vcpu *vcpu, u32 *msrpm, u32 msr, 701 int read, int write) 702 { 703 set_shadow_msr_intercept(vcpu, msr, read, write); 704 set_msr_interception_bitmap(vcpu, msrpm, msr, read, write); 705 } 706 707 u32 *svm_vcpu_alloc_msrpm(void) 708 { 709 unsigned int order = get_order(MSRPM_SIZE); 710 struct page *pages = alloc_pages(GFP_KERNEL_ACCOUNT, order); 711 u32 *msrpm; 712 713 if (!pages) 714 return NULL; 715 716 msrpm = page_address(pages); 717 memset(msrpm, 0xff, PAGE_SIZE * (1 << order)); 718 719 return msrpm; 720 } 721 722 void svm_vcpu_init_msrpm(struct kvm_vcpu *vcpu, u32 *msrpm) 723 { 724 int i; 725 726 for (i = 0; direct_access_msrs[i].index != MSR_INVALID; i++) { 727 if (!direct_access_msrs[i].always) 728 continue; 729 set_msr_interception(vcpu, msrpm, direct_access_msrs[i].index, 1, 1); 730 } 731 } 732 733 734 void svm_vcpu_free_msrpm(u32 *msrpm) 735 { 736 __free_pages(virt_to_page(msrpm), get_order(MSRPM_SIZE)); 737 } 738 739 static void svm_msr_filter_changed(struct kvm_vcpu *vcpu) 740 { 741 struct vcpu_svm *svm = to_svm(vcpu); 742 u32 i; 743 744 /* 745 * Set intercept permissions for all direct access MSRs again. They 746 * will automatically get filtered through the MSR filter, so we are 747 * back in sync after this. 748 */ 749 for (i = 0; direct_access_msrs[i].index != MSR_INVALID; i++) { 750 u32 msr = direct_access_msrs[i].index; 751 u32 read = test_bit(i, svm->shadow_msr_intercept.read); 752 u32 write = test_bit(i, svm->shadow_msr_intercept.write); 753 754 set_msr_interception_bitmap(vcpu, svm->msrpm, msr, read, write); 755 } 756 } 757 758 static void add_msr_offset(u32 offset) 759 { 760 int i; 761 762 for (i = 0; i < MSRPM_OFFSETS; ++i) { 763 764 /* Offset already in list? */ 765 if (msrpm_offsets[i] == offset) 766 return; 767 768 /* Slot used by another offset? */ 769 if (msrpm_offsets[i] != MSR_INVALID) 770 continue; 771 772 /* Add offset to list */ 773 msrpm_offsets[i] = offset; 774 775 return; 776 } 777 778 /* 779 * If this BUG triggers the msrpm_offsets table has an overflow. Just 780 * increase MSRPM_OFFSETS in this case. 781 */ 782 BUG(); 783 } 784 785 static void init_msrpm_offsets(void) 786 { 787 int i; 788 789 memset(msrpm_offsets, 0xff, sizeof(msrpm_offsets)); 790 791 for (i = 0; direct_access_msrs[i].index != MSR_INVALID; i++) { 792 u32 offset; 793 794 offset = svm_msrpm_offset(direct_access_msrs[i].index); 795 BUG_ON(offset == MSR_INVALID); 796 797 add_msr_offset(offset); 798 } 799 } 800 801 static void svm_enable_lbrv(struct kvm_vcpu *vcpu) 802 { 803 struct vcpu_svm *svm = to_svm(vcpu); 804 805 svm->vmcb->control.virt_ext |= LBR_CTL_ENABLE_MASK; 806 set_msr_interception(vcpu, svm->msrpm, MSR_IA32_LASTBRANCHFROMIP, 1, 1); 807 set_msr_interception(vcpu, svm->msrpm, MSR_IA32_LASTBRANCHTOIP, 1, 1); 808 set_msr_interception(vcpu, svm->msrpm, MSR_IA32_LASTINTFROMIP, 1, 1); 809 set_msr_interception(vcpu, svm->msrpm, MSR_IA32_LASTINTTOIP, 1, 1); 810 } 811 812 static void svm_disable_lbrv(struct kvm_vcpu *vcpu) 813 { 814 struct vcpu_svm *svm = to_svm(vcpu); 815 816 svm->vmcb->control.virt_ext &= ~LBR_CTL_ENABLE_MASK; 817 set_msr_interception(vcpu, svm->msrpm, MSR_IA32_LASTBRANCHFROMIP, 0, 0); 818 set_msr_interception(vcpu, svm->msrpm, MSR_IA32_LASTBRANCHTOIP, 0, 0); 819 set_msr_interception(vcpu, svm->msrpm, MSR_IA32_LASTINTFROMIP, 0, 0); 820 set_msr_interception(vcpu, svm->msrpm, MSR_IA32_LASTINTTOIP, 0, 0); 821 } 822 823 void disable_nmi_singlestep(struct vcpu_svm *svm) 824 { 825 svm->nmi_singlestep = false; 826 827 if (!(svm->vcpu.guest_debug & KVM_GUESTDBG_SINGLESTEP)) { 828 /* Clear our flags if they were not set by the guest */ 829 if (!(svm->nmi_singlestep_guest_rflags & X86_EFLAGS_TF)) 830 svm->vmcb->save.rflags &= ~X86_EFLAGS_TF; 831 if (!(svm->nmi_singlestep_guest_rflags & X86_EFLAGS_RF)) 832 svm->vmcb->save.rflags &= ~X86_EFLAGS_RF; 833 } 834 } 835 836 static void grow_ple_window(struct kvm_vcpu *vcpu) 837 { 838 struct vcpu_svm *svm = to_svm(vcpu); 839 struct vmcb_control_area *control = &svm->vmcb->control; 840 int old = control->pause_filter_count; 841 842 control->pause_filter_count = __grow_ple_window(old, 843 pause_filter_count, 844 pause_filter_count_grow, 845 pause_filter_count_max); 846 847 if (control->pause_filter_count != old) { 848 vmcb_mark_dirty(svm->vmcb, VMCB_INTERCEPTS); 849 trace_kvm_ple_window_update(vcpu->vcpu_id, 850 control->pause_filter_count, old); 851 } 852 } 853 854 static void shrink_ple_window(struct kvm_vcpu *vcpu) 855 { 856 struct vcpu_svm *svm = to_svm(vcpu); 857 struct vmcb_control_area *control = &svm->vmcb->control; 858 int old = control->pause_filter_count; 859 860 control->pause_filter_count = 861 __shrink_ple_window(old, 862 pause_filter_count, 863 pause_filter_count_shrink, 864 pause_filter_count); 865 if (control->pause_filter_count != old) { 866 vmcb_mark_dirty(svm->vmcb, VMCB_INTERCEPTS); 867 trace_kvm_ple_window_update(vcpu->vcpu_id, 868 control->pause_filter_count, old); 869 } 870 } 871 872 static void svm_hardware_teardown(void) 873 { 874 int cpu; 875 876 sev_hardware_teardown(); 877 878 for_each_possible_cpu(cpu) 879 svm_cpu_uninit(cpu); 880 881 __free_pages(pfn_to_page(iopm_base >> PAGE_SHIFT), 882 get_order(IOPM_SIZE)); 883 iopm_base = 0; 884 } 885 886 static void init_seg(struct vmcb_seg *seg) 887 { 888 seg->selector = 0; 889 seg->attrib = SVM_SELECTOR_P_MASK | SVM_SELECTOR_S_MASK | 890 SVM_SELECTOR_WRITE_MASK; /* Read/Write Data Segment */ 891 seg->limit = 0xffff; 892 seg->base = 0; 893 } 894 895 static void init_sys_seg(struct vmcb_seg *seg, uint32_t type) 896 { 897 seg->selector = 0; 898 seg->attrib = SVM_SELECTOR_P_MASK | type; 899 seg->limit = 0xffff; 900 seg->base = 0; 901 } 902 903 static u64 svm_get_l2_tsc_offset(struct kvm_vcpu *vcpu) 904 { 905 struct vcpu_svm *svm = to_svm(vcpu); 906 907 return svm->nested.ctl.tsc_offset; 908 } 909 910 static u64 svm_get_l2_tsc_multiplier(struct kvm_vcpu *vcpu) 911 { 912 struct vcpu_svm *svm = to_svm(vcpu); 913 914 return svm->tsc_ratio_msr; 915 } 916 917 static void svm_write_tsc_offset(struct kvm_vcpu *vcpu, u64 offset) 918 { 919 struct vcpu_svm *svm = to_svm(vcpu); 920 921 svm->vmcb01.ptr->control.tsc_offset = vcpu->arch.l1_tsc_offset; 922 svm->vmcb->control.tsc_offset = offset; 923 vmcb_mark_dirty(svm->vmcb, VMCB_INTERCEPTS); 924 } 925 926 void svm_write_tsc_multiplier(struct kvm_vcpu *vcpu, u64 multiplier) 927 { 928 wrmsrl(MSR_AMD64_TSC_RATIO, multiplier); 929 } 930 931 /* Evaluate instruction intercepts that depend on guest CPUID features. */ 932 static void svm_recalc_instruction_intercepts(struct kvm_vcpu *vcpu, 933 struct vcpu_svm *svm) 934 { 935 /* 936 * Intercept INVPCID if shadow paging is enabled to sync/free shadow 937 * roots, or if INVPCID is disabled in the guest to inject #UD. 938 */ 939 if (kvm_cpu_cap_has(X86_FEATURE_INVPCID)) { 940 if (!npt_enabled || 941 !guest_cpuid_has(&svm->vcpu, X86_FEATURE_INVPCID)) 942 svm_set_intercept(svm, INTERCEPT_INVPCID); 943 else 944 svm_clr_intercept(svm, INTERCEPT_INVPCID); 945 } 946 947 if (kvm_cpu_cap_has(X86_FEATURE_RDTSCP)) { 948 if (guest_cpuid_has(vcpu, X86_FEATURE_RDTSCP)) 949 svm_clr_intercept(svm, INTERCEPT_RDTSCP); 950 else 951 svm_set_intercept(svm, INTERCEPT_RDTSCP); 952 } 953 } 954 955 static inline void init_vmcb_after_set_cpuid(struct kvm_vcpu *vcpu) 956 { 957 struct vcpu_svm *svm = to_svm(vcpu); 958 959 if (guest_cpuid_is_intel(vcpu)) { 960 /* 961 * We must intercept SYSENTER_EIP and SYSENTER_ESP 962 * accesses because the processor only stores 32 bits. 963 * For the same reason we cannot use virtual VMLOAD/VMSAVE. 964 */ 965 svm_set_intercept(svm, INTERCEPT_VMLOAD); 966 svm_set_intercept(svm, INTERCEPT_VMSAVE); 967 svm->vmcb->control.virt_ext &= ~VIRTUAL_VMLOAD_VMSAVE_ENABLE_MASK; 968 969 set_msr_interception(vcpu, svm->msrpm, MSR_IA32_SYSENTER_EIP, 0, 0); 970 set_msr_interception(vcpu, svm->msrpm, MSR_IA32_SYSENTER_ESP, 0, 0); 971 } else { 972 /* 973 * If hardware supports Virtual VMLOAD VMSAVE then enable it 974 * in VMCB and clear intercepts to avoid #VMEXIT. 975 */ 976 if (vls) { 977 svm_clr_intercept(svm, INTERCEPT_VMLOAD); 978 svm_clr_intercept(svm, INTERCEPT_VMSAVE); 979 svm->vmcb->control.virt_ext |= VIRTUAL_VMLOAD_VMSAVE_ENABLE_MASK; 980 } 981 /* No need to intercept these MSRs */ 982 set_msr_interception(vcpu, svm->msrpm, MSR_IA32_SYSENTER_EIP, 1, 1); 983 set_msr_interception(vcpu, svm->msrpm, MSR_IA32_SYSENTER_ESP, 1, 1); 984 } 985 } 986 987 static void init_vmcb(struct kvm_vcpu *vcpu) 988 { 989 struct vcpu_svm *svm = to_svm(vcpu); 990 struct vmcb_control_area *control = &svm->vmcb->control; 991 struct vmcb_save_area *save = &svm->vmcb->save; 992 993 svm_set_intercept(svm, INTERCEPT_CR0_READ); 994 svm_set_intercept(svm, INTERCEPT_CR3_READ); 995 svm_set_intercept(svm, INTERCEPT_CR4_READ); 996 svm_set_intercept(svm, INTERCEPT_CR0_WRITE); 997 svm_set_intercept(svm, INTERCEPT_CR3_WRITE); 998 svm_set_intercept(svm, INTERCEPT_CR4_WRITE); 999 if (!kvm_vcpu_apicv_active(vcpu)) 1000 svm_set_intercept(svm, INTERCEPT_CR8_WRITE); 1001 1002 set_dr_intercepts(svm); 1003 1004 set_exception_intercept(svm, PF_VECTOR); 1005 set_exception_intercept(svm, UD_VECTOR); 1006 set_exception_intercept(svm, MC_VECTOR); 1007 set_exception_intercept(svm, AC_VECTOR); 1008 set_exception_intercept(svm, DB_VECTOR); 1009 /* 1010 * Guest access to VMware backdoor ports could legitimately 1011 * trigger #GP because of TSS I/O permission bitmap. 1012 * We intercept those #GP and allow access to them anyway 1013 * as VMware does. 1014 */ 1015 if (enable_vmware_backdoor) 1016 set_exception_intercept(svm, GP_VECTOR); 1017 1018 svm_set_intercept(svm, INTERCEPT_INTR); 1019 svm_set_intercept(svm, INTERCEPT_NMI); 1020 1021 if (intercept_smi) 1022 svm_set_intercept(svm, INTERCEPT_SMI); 1023 1024 svm_set_intercept(svm, INTERCEPT_SELECTIVE_CR0); 1025 svm_set_intercept(svm, INTERCEPT_RDPMC); 1026 svm_set_intercept(svm, INTERCEPT_CPUID); 1027 svm_set_intercept(svm, INTERCEPT_INVD); 1028 svm_set_intercept(svm, INTERCEPT_INVLPG); 1029 svm_set_intercept(svm, INTERCEPT_INVLPGA); 1030 svm_set_intercept(svm, INTERCEPT_IOIO_PROT); 1031 svm_set_intercept(svm, INTERCEPT_MSR_PROT); 1032 svm_set_intercept(svm, INTERCEPT_TASK_SWITCH); 1033 svm_set_intercept(svm, INTERCEPT_SHUTDOWN); 1034 svm_set_intercept(svm, INTERCEPT_VMRUN); 1035 svm_set_intercept(svm, INTERCEPT_VMMCALL); 1036 svm_set_intercept(svm, INTERCEPT_VMLOAD); 1037 svm_set_intercept(svm, INTERCEPT_VMSAVE); 1038 svm_set_intercept(svm, INTERCEPT_STGI); 1039 svm_set_intercept(svm, INTERCEPT_CLGI); 1040 svm_set_intercept(svm, INTERCEPT_SKINIT); 1041 svm_set_intercept(svm, INTERCEPT_WBINVD); 1042 svm_set_intercept(svm, INTERCEPT_XSETBV); 1043 svm_set_intercept(svm, INTERCEPT_RDPRU); 1044 svm_set_intercept(svm, INTERCEPT_RSM); 1045 1046 if (!kvm_mwait_in_guest(vcpu->kvm)) { 1047 svm_set_intercept(svm, INTERCEPT_MONITOR); 1048 svm_set_intercept(svm, INTERCEPT_MWAIT); 1049 } 1050 1051 if (!kvm_hlt_in_guest(vcpu->kvm)) 1052 svm_set_intercept(svm, INTERCEPT_HLT); 1053 1054 control->iopm_base_pa = __sme_set(iopm_base); 1055 control->msrpm_base_pa = __sme_set(__pa(svm->msrpm)); 1056 control->int_ctl = V_INTR_MASKING_MASK; 1057 1058 init_seg(&save->es); 1059 init_seg(&save->ss); 1060 init_seg(&save->ds); 1061 init_seg(&save->fs); 1062 init_seg(&save->gs); 1063 1064 save->cs.selector = 0xf000; 1065 save->cs.base = 0xffff0000; 1066 /* Executable/Readable Code Segment */ 1067 save->cs.attrib = SVM_SELECTOR_READ_MASK | SVM_SELECTOR_P_MASK | 1068 SVM_SELECTOR_S_MASK | SVM_SELECTOR_CODE_MASK; 1069 save->cs.limit = 0xffff; 1070 1071 save->gdtr.base = 0; 1072 save->gdtr.limit = 0xffff; 1073 save->idtr.base = 0; 1074 save->idtr.limit = 0xffff; 1075 1076 init_sys_seg(&save->ldtr, SEG_TYPE_LDT); 1077 init_sys_seg(&save->tr, SEG_TYPE_BUSY_TSS16); 1078 1079 if (npt_enabled) { 1080 /* Setup VMCB for Nested Paging */ 1081 control->nested_ctl |= SVM_NESTED_CTL_NP_ENABLE; 1082 svm_clr_intercept(svm, INTERCEPT_INVLPG); 1083 clr_exception_intercept(svm, PF_VECTOR); 1084 svm_clr_intercept(svm, INTERCEPT_CR3_READ); 1085 svm_clr_intercept(svm, INTERCEPT_CR3_WRITE); 1086 save->g_pat = vcpu->arch.pat; 1087 save->cr3 = 0; 1088 } 1089 svm->current_vmcb->asid_generation = 0; 1090 svm->asid = 0; 1091 1092 svm->nested.vmcb12_gpa = INVALID_GPA; 1093 svm->nested.last_vmcb12_gpa = INVALID_GPA; 1094 1095 if (!kvm_pause_in_guest(vcpu->kvm)) { 1096 control->pause_filter_count = pause_filter_count; 1097 if (pause_filter_thresh) 1098 control->pause_filter_thresh = pause_filter_thresh; 1099 svm_set_intercept(svm, INTERCEPT_PAUSE); 1100 } else { 1101 svm_clr_intercept(svm, INTERCEPT_PAUSE); 1102 } 1103 1104 svm_recalc_instruction_intercepts(vcpu, svm); 1105 1106 /* 1107 * If the host supports V_SPEC_CTRL then disable the interception 1108 * of MSR_IA32_SPEC_CTRL. 1109 */ 1110 if (boot_cpu_has(X86_FEATURE_V_SPEC_CTRL)) 1111 set_msr_interception(vcpu, svm->msrpm, MSR_IA32_SPEC_CTRL, 1, 1); 1112 1113 if (kvm_vcpu_apicv_active(vcpu)) 1114 avic_init_vmcb(svm); 1115 1116 if (vgif) { 1117 svm_clr_intercept(svm, INTERCEPT_STGI); 1118 svm_clr_intercept(svm, INTERCEPT_CLGI); 1119 svm->vmcb->control.int_ctl |= V_GIF_ENABLE_MASK; 1120 } 1121 1122 if (sev_guest(vcpu->kvm)) { 1123 svm->vmcb->control.nested_ctl |= SVM_NESTED_CTL_SEV_ENABLE; 1124 clr_exception_intercept(svm, UD_VECTOR); 1125 1126 if (sev_es_guest(vcpu->kvm)) { 1127 /* Perform SEV-ES specific VMCB updates */ 1128 sev_es_init_vmcb(svm); 1129 } 1130 } 1131 1132 svm_hv_init_vmcb(svm->vmcb); 1133 init_vmcb_after_set_cpuid(vcpu); 1134 1135 vmcb_mark_all_dirty(svm->vmcb); 1136 1137 enable_gif(svm); 1138 } 1139 1140 static void __svm_vcpu_reset(struct kvm_vcpu *vcpu) 1141 { 1142 struct vcpu_svm *svm = to_svm(vcpu); 1143 1144 svm_vcpu_init_msrpm(vcpu, svm->msrpm); 1145 1146 svm_init_osvw(vcpu); 1147 vcpu->arch.microcode_version = 0x01000065; 1148 svm->tsc_ratio_msr = kvm_default_tsc_scaling_ratio; 1149 1150 if (sev_es_guest(vcpu->kvm)) 1151 sev_es_vcpu_reset(svm); 1152 } 1153 1154 static void svm_vcpu_reset(struct kvm_vcpu *vcpu, bool init_event) 1155 { 1156 struct vcpu_svm *svm = to_svm(vcpu); 1157 1158 svm->spec_ctrl = 0; 1159 svm->virt_spec_ctrl = 0; 1160 1161 init_vmcb(vcpu); 1162 1163 if (!init_event) 1164 __svm_vcpu_reset(vcpu); 1165 } 1166 1167 void svm_switch_vmcb(struct vcpu_svm *svm, struct kvm_vmcb_info *target_vmcb) 1168 { 1169 svm->current_vmcb = target_vmcb; 1170 svm->vmcb = target_vmcb->ptr; 1171 } 1172 1173 static int svm_create_vcpu(struct kvm_vcpu *vcpu) 1174 { 1175 struct vcpu_svm *svm; 1176 struct page *vmcb01_page; 1177 struct page *vmsa_page = NULL; 1178 int err; 1179 1180 BUILD_BUG_ON(offsetof(struct vcpu_svm, vcpu) != 0); 1181 svm = to_svm(vcpu); 1182 1183 err = -ENOMEM; 1184 vmcb01_page = alloc_page(GFP_KERNEL_ACCOUNT | __GFP_ZERO); 1185 if (!vmcb01_page) 1186 goto out; 1187 1188 if (sev_es_guest(vcpu->kvm)) { 1189 /* 1190 * SEV-ES guests require a separate VMSA page used to contain 1191 * the encrypted register state of the guest. 1192 */ 1193 vmsa_page = alloc_page(GFP_KERNEL_ACCOUNT | __GFP_ZERO); 1194 if (!vmsa_page) 1195 goto error_free_vmcb_page; 1196 1197 /* 1198 * SEV-ES guests maintain an encrypted version of their FPU 1199 * state which is restored and saved on VMRUN and VMEXIT. 1200 * Mark vcpu->arch.guest_fpu->fpstate as scratch so it won't 1201 * do xsave/xrstor on it. 1202 */ 1203 fpstate_set_confidential(&vcpu->arch.guest_fpu); 1204 } 1205 1206 err = avic_init_vcpu(svm); 1207 if (err) 1208 goto error_free_vmsa_page; 1209 1210 svm->msrpm = svm_vcpu_alloc_msrpm(); 1211 if (!svm->msrpm) { 1212 err = -ENOMEM; 1213 goto error_free_vmsa_page; 1214 } 1215 1216 svm->vmcb01.ptr = page_address(vmcb01_page); 1217 svm->vmcb01.pa = __sme_set(page_to_pfn(vmcb01_page) << PAGE_SHIFT); 1218 svm_switch_vmcb(svm, &svm->vmcb01); 1219 1220 if (vmsa_page) 1221 svm->sev_es.vmsa = page_address(vmsa_page); 1222 1223 svm->guest_state_loaded = false; 1224 1225 return 0; 1226 1227 error_free_vmsa_page: 1228 if (vmsa_page) 1229 __free_page(vmsa_page); 1230 error_free_vmcb_page: 1231 __free_page(vmcb01_page); 1232 out: 1233 return err; 1234 } 1235 1236 static void svm_clear_current_vmcb(struct vmcb *vmcb) 1237 { 1238 int i; 1239 1240 for_each_online_cpu(i) 1241 cmpxchg(&per_cpu(svm_data, i)->current_vmcb, vmcb, NULL); 1242 } 1243 1244 static void svm_free_vcpu(struct kvm_vcpu *vcpu) 1245 { 1246 struct vcpu_svm *svm = to_svm(vcpu); 1247 1248 /* 1249 * The vmcb page can be recycled, causing a false negative in 1250 * svm_vcpu_load(). So, ensure that no logical CPU has this 1251 * vmcb page recorded as its current vmcb. 1252 */ 1253 svm_clear_current_vmcb(svm->vmcb); 1254 1255 svm_free_nested(svm); 1256 1257 sev_free_vcpu(vcpu); 1258 1259 __free_page(pfn_to_page(__sme_clr(svm->vmcb01.pa) >> PAGE_SHIFT)); 1260 __free_pages(virt_to_page(svm->msrpm), get_order(MSRPM_SIZE)); 1261 } 1262 1263 static void svm_prepare_guest_switch(struct kvm_vcpu *vcpu) 1264 { 1265 struct vcpu_svm *svm = to_svm(vcpu); 1266 struct svm_cpu_data *sd = per_cpu(svm_data, vcpu->cpu); 1267 1268 if (sev_es_guest(vcpu->kvm)) 1269 sev_es_unmap_ghcb(svm); 1270 1271 if (svm->guest_state_loaded) 1272 return; 1273 1274 /* 1275 * Save additional host state that will be restored on VMEXIT (sev-es) 1276 * or subsequent vmload of host save area. 1277 */ 1278 if (sev_es_guest(vcpu->kvm)) { 1279 sev_es_prepare_guest_switch(svm, vcpu->cpu); 1280 } else { 1281 vmsave(__sme_page_pa(sd->save_area)); 1282 } 1283 1284 if (tsc_scaling) { 1285 u64 tsc_ratio = vcpu->arch.tsc_scaling_ratio; 1286 if (tsc_ratio != __this_cpu_read(current_tsc_ratio)) { 1287 __this_cpu_write(current_tsc_ratio, tsc_ratio); 1288 wrmsrl(MSR_AMD64_TSC_RATIO, tsc_ratio); 1289 } 1290 } 1291 1292 if (likely(tsc_aux_uret_slot >= 0)) 1293 kvm_set_user_return_msr(tsc_aux_uret_slot, svm->tsc_aux, -1ull); 1294 1295 svm->guest_state_loaded = true; 1296 } 1297 1298 static void svm_prepare_host_switch(struct kvm_vcpu *vcpu) 1299 { 1300 to_svm(vcpu)->guest_state_loaded = false; 1301 } 1302 1303 static void svm_vcpu_load(struct kvm_vcpu *vcpu, int cpu) 1304 { 1305 struct vcpu_svm *svm = to_svm(vcpu); 1306 struct svm_cpu_data *sd = per_cpu(svm_data, cpu); 1307 1308 if (sd->current_vmcb != svm->vmcb) { 1309 sd->current_vmcb = svm->vmcb; 1310 indirect_branch_prediction_barrier(); 1311 } 1312 if (kvm_vcpu_apicv_active(vcpu)) 1313 avic_vcpu_load(vcpu, cpu); 1314 } 1315 1316 static void svm_vcpu_put(struct kvm_vcpu *vcpu) 1317 { 1318 if (kvm_vcpu_apicv_active(vcpu)) 1319 avic_vcpu_put(vcpu); 1320 1321 svm_prepare_host_switch(vcpu); 1322 1323 ++vcpu->stat.host_state_reload; 1324 } 1325 1326 static unsigned long svm_get_rflags(struct kvm_vcpu *vcpu) 1327 { 1328 struct vcpu_svm *svm = to_svm(vcpu); 1329 unsigned long rflags = svm->vmcb->save.rflags; 1330 1331 if (svm->nmi_singlestep) { 1332 /* Hide our flags if they were not set by the guest */ 1333 if (!(svm->nmi_singlestep_guest_rflags & X86_EFLAGS_TF)) 1334 rflags &= ~X86_EFLAGS_TF; 1335 if (!(svm->nmi_singlestep_guest_rflags & X86_EFLAGS_RF)) 1336 rflags &= ~X86_EFLAGS_RF; 1337 } 1338 return rflags; 1339 } 1340 1341 static void svm_set_rflags(struct kvm_vcpu *vcpu, unsigned long rflags) 1342 { 1343 if (to_svm(vcpu)->nmi_singlestep) 1344 rflags |= (X86_EFLAGS_TF | X86_EFLAGS_RF); 1345 1346 /* 1347 * Any change of EFLAGS.VM is accompanied by a reload of SS 1348 * (caused by either a task switch or an inter-privilege IRET), 1349 * so we do not need to update the CPL here. 1350 */ 1351 to_svm(vcpu)->vmcb->save.rflags = rflags; 1352 } 1353 1354 static bool svm_get_if_flag(struct kvm_vcpu *vcpu) 1355 { 1356 struct vmcb *vmcb = to_svm(vcpu)->vmcb; 1357 1358 return sev_es_guest(vcpu->kvm) 1359 ? vmcb->control.int_state & SVM_GUEST_INTERRUPT_MASK 1360 : kvm_get_rflags(vcpu) & X86_EFLAGS_IF; 1361 } 1362 1363 static void svm_cache_reg(struct kvm_vcpu *vcpu, enum kvm_reg reg) 1364 { 1365 kvm_register_mark_available(vcpu, reg); 1366 1367 switch (reg) { 1368 case VCPU_EXREG_PDPTR: 1369 /* 1370 * When !npt_enabled, mmu->pdptrs[] is already available since 1371 * it is always updated per SDM when moving to CRs. 1372 */ 1373 if (npt_enabled) 1374 load_pdptrs(vcpu, kvm_read_cr3(vcpu)); 1375 break; 1376 default: 1377 KVM_BUG_ON(1, vcpu->kvm); 1378 } 1379 } 1380 1381 static void svm_set_vintr(struct vcpu_svm *svm) 1382 { 1383 struct vmcb_control_area *control; 1384 1385 /* 1386 * The following fields are ignored when AVIC is enabled 1387 */ 1388 WARN_ON(kvm_apicv_activated(svm->vcpu.kvm)); 1389 1390 svm_set_intercept(svm, INTERCEPT_VINTR); 1391 1392 /* 1393 * This is just a dummy VINTR to actually cause a vmexit to happen. 1394 * Actual injection of virtual interrupts happens through EVENTINJ. 1395 */ 1396 control = &svm->vmcb->control; 1397 control->int_vector = 0x0; 1398 control->int_ctl &= ~V_INTR_PRIO_MASK; 1399 control->int_ctl |= V_IRQ_MASK | 1400 ((/*control->int_vector >> 4*/ 0xf) << V_INTR_PRIO_SHIFT); 1401 vmcb_mark_dirty(svm->vmcb, VMCB_INTR); 1402 } 1403 1404 static void svm_clear_vintr(struct vcpu_svm *svm) 1405 { 1406 svm_clr_intercept(svm, INTERCEPT_VINTR); 1407 1408 /* Drop int_ctl fields related to VINTR injection. */ 1409 svm->vmcb->control.int_ctl &= ~V_IRQ_INJECTION_BITS_MASK; 1410 if (is_guest_mode(&svm->vcpu)) { 1411 svm->vmcb01.ptr->control.int_ctl &= ~V_IRQ_INJECTION_BITS_MASK; 1412 1413 WARN_ON((svm->vmcb->control.int_ctl & V_TPR_MASK) != 1414 (svm->nested.ctl.int_ctl & V_TPR_MASK)); 1415 1416 svm->vmcb->control.int_ctl |= svm->nested.ctl.int_ctl & 1417 V_IRQ_INJECTION_BITS_MASK; 1418 1419 svm->vmcb->control.int_vector = svm->nested.ctl.int_vector; 1420 } 1421 1422 vmcb_mark_dirty(svm->vmcb, VMCB_INTR); 1423 } 1424 1425 static struct vmcb_seg *svm_seg(struct kvm_vcpu *vcpu, int seg) 1426 { 1427 struct vmcb_save_area *save = &to_svm(vcpu)->vmcb->save; 1428 struct vmcb_save_area *save01 = &to_svm(vcpu)->vmcb01.ptr->save; 1429 1430 switch (seg) { 1431 case VCPU_SREG_CS: return &save->cs; 1432 case VCPU_SREG_DS: return &save->ds; 1433 case VCPU_SREG_ES: return &save->es; 1434 case VCPU_SREG_FS: return &save01->fs; 1435 case VCPU_SREG_GS: return &save01->gs; 1436 case VCPU_SREG_SS: return &save->ss; 1437 case VCPU_SREG_TR: return &save01->tr; 1438 case VCPU_SREG_LDTR: return &save01->ldtr; 1439 } 1440 BUG(); 1441 return NULL; 1442 } 1443 1444 static u64 svm_get_segment_base(struct kvm_vcpu *vcpu, int seg) 1445 { 1446 struct vmcb_seg *s = svm_seg(vcpu, seg); 1447 1448 return s->base; 1449 } 1450 1451 static void svm_get_segment(struct kvm_vcpu *vcpu, 1452 struct kvm_segment *var, int seg) 1453 { 1454 struct vmcb_seg *s = svm_seg(vcpu, seg); 1455 1456 var->base = s->base; 1457 var->limit = s->limit; 1458 var->selector = s->selector; 1459 var->type = s->attrib & SVM_SELECTOR_TYPE_MASK; 1460 var->s = (s->attrib >> SVM_SELECTOR_S_SHIFT) & 1; 1461 var->dpl = (s->attrib >> SVM_SELECTOR_DPL_SHIFT) & 3; 1462 var->present = (s->attrib >> SVM_SELECTOR_P_SHIFT) & 1; 1463 var->avl = (s->attrib >> SVM_SELECTOR_AVL_SHIFT) & 1; 1464 var->l = (s->attrib >> SVM_SELECTOR_L_SHIFT) & 1; 1465 var->db = (s->attrib >> SVM_SELECTOR_DB_SHIFT) & 1; 1466 1467 /* 1468 * AMD CPUs circa 2014 track the G bit for all segments except CS. 1469 * However, the SVM spec states that the G bit is not observed by the 1470 * CPU, and some VMware virtual CPUs drop the G bit for all segments. 1471 * So let's synthesize a legal G bit for all segments, this helps 1472 * running KVM nested. It also helps cross-vendor migration, because 1473 * Intel's vmentry has a check on the 'G' bit. 1474 */ 1475 var->g = s->limit > 0xfffff; 1476 1477 /* 1478 * AMD's VMCB does not have an explicit unusable field, so emulate it 1479 * for cross vendor migration purposes by "not present" 1480 */ 1481 var->unusable = !var->present; 1482 1483 switch (seg) { 1484 case VCPU_SREG_TR: 1485 /* 1486 * Work around a bug where the busy flag in the tr selector 1487 * isn't exposed 1488 */ 1489 var->type |= 0x2; 1490 break; 1491 case VCPU_SREG_DS: 1492 case VCPU_SREG_ES: 1493 case VCPU_SREG_FS: 1494 case VCPU_SREG_GS: 1495 /* 1496 * The accessed bit must always be set in the segment 1497 * descriptor cache, although it can be cleared in the 1498 * descriptor, the cached bit always remains at 1. Since 1499 * Intel has a check on this, set it here to support 1500 * cross-vendor migration. 1501 */ 1502 if (!var->unusable) 1503 var->type |= 0x1; 1504 break; 1505 case VCPU_SREG_SS: 1506 /* 1507 * On AMD CPUs sometimes the DB bit in the segment 1508 * descriptor is left as 1, although the whole segment has 1509 * been made unusable. Clear it here to pass an Intel VMX 1510 * entry check when cross vendor migrating. 1511 */ 1512 if (var->unusable) 1513 var->db = 0; 1514 /* This is symmetric with svm_set_segment() */ 1515 var->dpl = to_svm(vcpu)->vmcb->save.cpl; 1516 break; 1517 } 1518 } 1519 1520 static int svm_get_cpl(struct kvm_vcpu *vcpu) 1521 { 1522 struct vmcb_save_area *save = &to_svm(vcpu)->vmcb->save; 1523 1524 return save->cpl; 1525 } 1526 1527 static void svm_get_idt(struct kvm_vcpu *vcpu, struct desc_ptr *dt) 1528 { 1529 struct vcpu_svm *svm = to_svm(vcpu); 1530 1531 dt->size = svm->vmcb->save.idtr.limit; 1532 dt->address = svm->vmcb->save.idtr.base; 1533 } 1534 1535 static void svm_set_idt(struct kvm_vcpu *vcpu, struct desc_ptr *dt) 1536 { 1537 struct vcpu_svm *svm = to_svm(vcpu); 1538 1539 svm->vmcb->save.idtr.limit = dt->size; 1540 svm->vmcb->save.idtr.base = dt->address ; 1541 vmcb_mark_dirty(svm->vmcb, VMCB_DT); 1542 } 1543 1544 static void svm_get_gdt(struct kvm_vcpu *vcpu, struct desc_ptr *dt) 1545 { 1546 struct vcpu_svm *svm = to_svm(vcpu); 1547 1548 dt->size = svm->vmcb->save.gdtr.limit; 1549 dt->address = svm->vmcb->save.gdtr.base; 1550 } 1551 1552 static void svm_set_gdt(struct kvm_vcpu *vcpu, struct desc_ptr *dt) 1553 { 1554 struct vcpu_svm *svm = to_svm(vcpu); 1555 1556 svm->vmcb->save.gdtr.limit = dt->size; 1557 svm->vmcb->save.gdtr.base = dt->address ; 1558 vmcb_mark_dirty(svm->vmcb, VMCB_DT); 1559 } 1560 1561 static void svm_post_set_cr3(struct kvm_vcpu *vcpu, unsigned long cr3) 1562 { 1563 struct vcpu_svm *svm = to_svm(vcpu); 1564 1565 /* 1566 * For guests that don't set guest_state_protected, the cr3 update is 1567 * handled via kvm_mmu_load() while entering the guest. For guests 1568 * that do (SEV-ES/SEV-SNP), the cr3 update needs to be written to 1569 * VMCB save area now, since the save area will become the initial 1570 * contents of the VMSA, and future VMCB save area updates won't be 1571 * seen. 1572 */ 1573 if (sev_es_guest(vcpu->kvm)) { 1574 svm->vmcb->save.cr3 = cr3; 1575 vmcb_mark_dirty(svm->vmcb, VMCB_CR); 1576 } 1577 } 1578 1579 void svm_set_cr0(struct kvm_vcpu *vcpu, unsigned long cr0) 1580 { 1581 struct vcpu_svm *svm = to_svm(vcpu); 1582 u64 hcr0 = cr0; 1583 1584 #ifdef CONFIG_X86_64 1585 if (vcpu->arch.efer & EFER_LME && !vcpu->arch.guest_state_protected) { 1586 if (!is_paging(vcpu) && (cr0 & X86_CR0_PG)) { 1587 vcpu->arch.efer |= EFER_LMA; 1588 svm->vmcb->save.efer |= EFER_LMA | EFER_LME; 1589 } 1590 1591 if (is_paging(vcpu) && !(cr0 & X86_CR0_PG)) { 1592 vcpu->arch.efer &= ~EFER_LMA; 1593 svm->vmcb->save.efer &= ~(EFER_LMA | EFER_LME); 1594 } 1595 } 1596 #endif 1597 vcpu->arch.cr0 = cr0; 1598 1599 if (!npt_enabled) 1600 hcr0 |= X86_CR0_PG | X86_CR0_WP; 1601 1602 /* 1603 * re-enable caching here because the QEMU bios 1604 * does not do it - this results in some delay at 1605 * reboot 1606 */ 1607 if (kvm_check_has_quirk(vcpu->kvm, KVM_X86_QUIRK_CD_NW_CLEARED)) 1608 hcr0 &= ~(X86_CR0_CD | X86_CR0_NW); 1609 1610 svm->vmcb->save.cr0 = hcr0; 1611 vmcb_mark_dirty(svm->vmcb, VMCB_CR); 1612 1613 /* 1614 * SEV-ES guests must always keep the CR intercepts cleared. CR 1615 * tracking is done using the CR write traps. 1616 */ 1617 if (sev_es_guest(vcpu->kvm)) 1618 return; 1619 1620 if (hcr0 == cr0) { 1621 /* Selective CR0 write remains on. */ 1622 svm_clr_intercept(svm, INTERCEPT_CR0_READ); 1623 svm_clr_intercept(svm, INTERCEPT_CR0_WRITE); 1624 } else { 1625 svm_set_intercept(svm, INTERCEPT_CR0_READ); 1626 svm_set_intercept(svm, INTERCEPT_CR0_WRITE); 1627 } 1628 } 1629 1630 static bool svm_is_valid_cr4(struct kvm_vcpu *vcpu, unsigned long cr4) 1631 { 1632 return true; 1633 } 1634 1635 void svm_set_cr4(struct kvm_vcpu *vcpu, unsigned long cr4) 1636 { 1637 unsigned long host_cr4_mce = cr4_read_shadow() & X86_CR4_MCE; 1638 unsigned long old_cr4 = vcpu->arch.cr4; 1639 1640 if (npt_enabled && ((old_cr4 ^ cr4) & X86_CR4_PGE)) 1641 svm_flush_tlb(vcpu); 1642 1643 vcpu->arch.cr4 = cr4; 1644 if (!npt_enabled) 1645 cr4 |= X86_CR4_PAE; 1646 cr4 |= host_cr4_mce; 1647 to_svm(vcpu)->vmcb->save.cr4 = cr4; 1648 vmcb_mark_dirty(to_svm(vcpu)->vmcb, VMCB_CR); 1649 1650 if ((cr4 ^ old_cr4) & (X86_CR4_OSXSAVE | X86_CR4_PKE)) 1651 kvm_update_cpuid_runtime(vcpu); 1652 } 1653 1654 static void svm_set_segment(struct kvm_vcpu *vcpu, 1655 struct kvm_segment *var, int seg) 1656 { 1657 struct vcpu_svm *svm = to_svm(vcpu); 1658 struct vmcb_seg *s = svm_seg(vcpu, seg); 1659 1660 s->base = var->base; 1661 s->limit = var->limit; 1662 s->selector = var->selector; 1663 s->attrib = (var->type & SVM_SELECTOR_TYPE_MASK); 1664 s->attrib |= (var->s & 1) << SVM_SELECTOR_S_SHIFT; 1665 s->attrib |= (var->dpl & 3) << SVM_SELECTOR_DPL_SHIFT; 1666 s->attrib |= ((var->present & 1) && !var->unusable) << SVM_SELECTOR_P_SHIFT; 1667 s->attrib |= (var->avl & 1) << SVM_SELECTOR_AVL_SHIFT; 1668 s->attrib |= (var->l & 1) << SVM_SELECTOR_L_SHIFT; 1669 s->attrib |= (var->db & 1) << SVM_SELECTOR_DB_SHIFT; 1670 s->attrib |= (var->g & 1) << SVM_SELECTOR_G_SHIFT; 1671 1672 /* 1673 * This is always accurate, except if SYSRET returned to a segment 1674 * with SS.DPL != 3. Intel does not have this quirk, and always 1675 * forces SS.DPL to 3 on sysret, so we ignore that case; fixing it 1676 * would entail passing the CPL to userspace and back. 1677 */ 1678 if (seg == VCPU_SREG_SS) 1679 /* This is symmetric with svm_get_segment() */ 1680 svm->vmcb->save.cpl = (var->dpl & 3); 1681 1682 vmcb_mark_dirty(svm->vmcb, VMCB_SEG); 1683 } 1684 1685 static void svm_update_exception_bitmap(struct kvm_vcpu *vcpu) 1686 { 1687 struct vcpu_svm *svm = to_svm(vcpu); 1688 1689 clr_exception_intercept(svm, BP_VECTOR); 1690 1691 if (vcpu->guest_debug & KVM_GUESTDBG_ENABLE) { 1692 if (vcpu->guest_debug & KVM_GUESTDBG_USE_SW_BP) 1693 set_exception_intercept(svm, BP_VECTOR); 1694 } 1695 } 1696 1697 static void new_asid(struct vcpu_svm *svm, struct svm_cpu_data *sd) 1698 { 1699 if (sd->next_asid > sd->max_asid) { 1700 ++sd->asid_generation; 1701 sd->next_asid = sd->min_asid; 1702 svm->vmcb->control.tlb_ctl = TLB_CONTROL_FLUSH_ALL_ASID; 1703 vmcb_mark_dirty(svm->vmcb, VMCB_ASID); 1704 } 1705 1706 svm->current_vmcb->asid_generation = sd->asid_generation; 1707 svm->asid = sd->next_asid++; 1708 } 1709 1710 static void svm_set_dr6(struct vcpu_svm *svm, unsigned long value) 1711 { 1712 struct vmcb *vmcb = svm->vmcb; 1713 1714 if (svm->vcpu.arch.guest_state_protected) 1715 return; 1716 1717 if (unlikely(value != vmcb->save.dr6)) { 1718 vmcb->save.dr6 = value; 1719 vmcb_mark_dirty(vmcb, VMCB_DR); 1720 } 1721 } 1722 1723 static void svm_sync_dirty_debug_regs(struct kvm_vcpu *vcpu) 1724 { 1725 struct vcpu_svm *svm = to_svm(vcpu); 1726 1727 if (vcpu->arch.guest_state_protected) 1728 return; 1729 1730 get_debugreg(vcpu->arch.db[0], 0); 1731 get_debugreg(vcpu->arch.db[1], 1); 1732 get_debugreg(vcpu->arch.db[2], 2); 1733 get_debugreg(vcpu->arch.db[3], 3); 1734 /* 1735 * We cannot reset svm->vmcb->save.dr6 to DR6_ACTIVE_LOW here, 1736 * because db_interception might need it. We can do it before vmentry. 1737 */ 1738 vcpu->arch.dr6 = svm->vmcb->save.dr6; 1739 vcpu->arch.dr7 = svm->vmcb->save.dr7; 1740 vcpu->arch.switch_db_regs &= ~KVM_DEBUGREG_WONT_EXIT; 1741 set_dr_intercepts(svm); 1742 } 1743 1744 static void svm_set_dr7(struct kvm_vcpu *vcpu, unsigned long value) 1745 { 1746 struct vcpu_svm *svm = to_svm(vcpu); 1747 1748 if (vcpu->arch.guest_state_protected) 1749 return; 1750 1751 svm->vmcb->save.dr7 = value; 1752 vmcb_mark_dirty(svm->vmcb, VMCB_DR); 1753 } 1754 1755 static int pf_interception(struct kvm_vcpu *vcpu) 1756 { 1757 struct vcpu_svm *svm = to_svm(vcpu); 1758 1759 u64 fault_address = svm->vmcb->control.exit_info_2; 1760 u64 error_code = svm->vmcb->control.exit_info_1; 1761 1762 return kvm_handle_page_fault(vcpu, error_code, fault_address, 1763 static_cpu_has(X86_FEATURE_DECODEASSISTS) ? 1764 svm->vmcb->control.insn_bytes : NULL, 1765 svm->vmcb->control.insn_len); 1766 } 1767 1768 static int npf_interception(struct kvm_vcpu *vcpu) 1769 { 1770 struct vcpu_svm *svm = to_svm(vcpu); 1771 1772 u64 fault_address = svm->vmcb->control.exit_info_2; 1773 u64 error_code = svm->vmcb->control.exit_info_1; 1774 1775 trace_kvm_page_fault(fault_address, error_code); 1776 return kvm_mmu_page_fault(vcpu, fault_address, error_code, 1777 static_cpu_has(X86_FEATURE_DECODEASSISTS) ? 1778 svm->vmcb->control.insn_bytes : NULL, 1779 svm->vmcb->control.insn_len); 1780 } 1781 1782 static int db_interception(struct kvm_vcpu *vcpu) 1783 { 1784 struct kvm_run *kvm_run = vcpu->run; 1785 struct vcpu_svm *svm = to_svm(vcpu); 1786 1787 if (!(vcpu->guest_debug & 1788 (KVM_GUESTDBG_SINGLESTEP | KVM_GUESTDBG_USE_HW_BP)) && 1789 !svm->nmi_singlestep) { 1790 u32 payload = svm->vmcb->save.dr6 ^ DR6_ACTIVE_LOW; 1791 kvm_queue_exception_p(vcpu, DB_VECTOR, payload); 1792 return 1; 1793 } 1794 1795 if (svm->nmi_singlestep) { 1796 disable_nmi_singlestep(svm); 1797 /* Make sure we check for pending NMIs upon entry */ 1798 kvm_make_request(KVM_REQ_EVENT, vcpu); 1799 } 1800 1801 if (vcpu->guest_debug & 1802 (KVM_GUESTDBG_SINGLESTEP | KVM_GUESTDBG_USE_HW_BP)) { 1803 kvm_run->exit_reason = KVM_EXIT_DEBUG; 1804 kvm_run->debug.arch.dr6 = svm->vmcb->save.dr6; 1805 kvm_run->debug.arch.dr7 = svm->vmcb->save.dr7; 1806 kvm_run->debug.arch.pc = 1807 svm->vmcb->save.cs.base + svm->vmcb->save.rip; 1808 kvm_run->debug.arch.exception = DB_VECTOR; 1809 return 0; 1810 } 1811 1812 return 1; 1813 } 1814 1815 static int bp_interception(struct kvm_vcpu *vcpu) 1816 { 1817 struct vcpu_svm *svm = to_svm(vcpu); 1818 struct kvm_run *kvm_run = vcpu->run; 1819 1820 kvm_run->exit_reason = KVM_EXIT_DEBUG; 1821 kvm_run->debug.arch.pc = svm->vmcb->save.cs.base + svm->vmcb->save.rip; 1822 kvm_run->debug.arch.exception = BP_VECTOR; 1823 return 0; 1824 } 1825 1826 static int ud_interception(struct kvm_vcpu *vcpu) 1827 { 1828 return handle_ud(vcpu); 1829 } 1830 1831 static int ac_interception(struct kvm_vcpu *vcpu) 1832 { 1833 kvm_queue_exception_e(vcpu, AC_VECTOR, 0); 1834 return 1; 1835 } 1836 1837 static bool is_erratum_383(void) 1838 { 1839 int err, i; 1840 u64 value; 1841 1842 if (!erratum_383_found) 1843 return false; 1844 1845 value = native_read_msr_safe(MSR_IA32_MC0_STATUS, &err); 1846 if (err) 1847 return false; 1848 1849 /* Bit 62 may or may not be set for this mce */ 1850 value &= ~(1ULL << 62); 1851 1852 if (value != 0xb600000000010015ULL) 1853 return false; 1854 1855 /* Clear MCi_STATUS registers */ 1856 for (i = 0; i < 6; ++i) 1857 native_write_msr_safe(MSR_IA32_MCx_STATUS(i), 0, 0); 1858 1859 value = native_read_msr_safe(MSR_IA32_MCG_STATUS, &err); 1860 if (!err) { 1861 u32 low, high; 1862 1863 value &= ~(1ULL << 2); 1864 low = lower_32_bits(value); 1865 high = upper_32_bits(value); 1866 1867 native_write_msr_safe(MSR_IA32_MCG_STATUS, low, high); 1868 } 1869 1870 /* Flush tlb to evict multi-match entries */ 1871 __flush_tlb_all(); 1872 1873 return true; 1874 } 1875 1876 static void svm_handle_mce(struct kvm_vcpu *vcpu) 1877 { 1878 if (is_erratum_383()) { 1879 /* 1880 * Erratum 383 triggered. Guest state is corrupt so kill the 1881 * guest. 1882 */ 1883 pr_err("KVM: Guest triggered AMD Erratum 383\n"); 1884 1885 kvm_make_request(KVM_REQ_TRIPLE_FAULT, vcpu); 1886 1887 return; 1888 } 1889 1890 /* 1891 * On an #MC intercept the MCE handler is not called automatically in 1892 * the host. So do it by hand here. 1893 */ 1894 kvm_machine_check(); 1895 } 1896 1897 static int mc_interception(struct kvm_vcpu *vcpu) 1898 { 1899 return 1; 1900 } 1901 1902 static int shutdown_interception(struct kvm_vcpu *vcpu) 1903 { 1904 struct kvm_run *kvm_run = vcpu->run; 1905 struct vcpu_svm *svm = to_svm(vcpu); 1906 1907 /* 1908 * The VM save area has already been encrypted so it 1909 * cannot be reinitialized - just terminate. 1910 */ 1911 if (sev_es_guest(vcpu->kvm)) 1912 return -EINVAL; 1913 1914 /* 1915 * VMCB is undefined after a SHUTDOWN intercept. INIT the vCPU to put 1916 * the VMCB in a known good state. Unfortuately, KVM doesn't have 1917 * KVM_MP_STATE_SHUTDOWN and can't add it without potentially breaking 1918 * userspace. At a platform view, INIT is acceptable behavior as 1919 * there exist bare metal platforms that automatically INIT the CPU 1920 * in response to shutdown. 1921 */ 1922 clear_page(svm->vmcb); 1923 kvm_vcpu_reset(vcpu, true); 1924 1925 kvm_run->exit_reason = KVM_EXIT_SHUTDOWN; 1926 return 0; 1927 } 1928 1929 static int io_interception(struct kvm_vcpu *vcpu) 1930 { 1931 struct vcpu_svm *svm = to_svm(vcpu); 1932 u32 io_info = svm->vmcb->control.exit_info_1; /* address size bug? */ 1933 int size, in, string; 1934 unsigned port; 1935 1936 ++vcpu->stat.io_exits; 1937 string = (io_info & SVM_IOIO_STR_MASK) != 0; 1938 in = (io_info & SVM_IOIO_TYPE_MASK) != 0; 1939 port = io_info >> 16; 1940 size = (io_info & SVM_IOIO_SIZE_MASK) >> SVM_IOIO_SIZE_SHIFT; 1941 1942 if (string) { 1943 if (sev_es_guest(vcpu->kvm)) 1944 return sev_es_string_io(svm, size, port, in); 1945 else 1946 return kvm_emulate_instruction(vcpu, 0); 1947 } 1948 1949 svm->next_rip = svm->vmcb->control.exit_info_2; 1950 1951 return kvm_fast_pio(vcpu, size, port, in); 1952 } 1953 1954 static int nmi_interception(struct kvm_vcpu *vcpu) 1955 { 1956 return 1; 1957 } 1958 1959 static int smi_interception(struct kvm_vcpu *vcpu) 1960 { 1961 return 1; 1962 } 1963 1964 static int intr_interception(struct kvm_vcpu *vcpu) 1965 { 1966 ++vcpu->stat.irq_exits; 1967 return 1; 1968 } 1969 1970 static int vmload_vmsave_interception(struct kvm_vcpu *vcpu, bool vmload) 1971 { 1972 struct vcpu_svm *svm = to_svm(vcpu); 1973 struct vmcb *vmcb12; 1974 struct kvm_host_map map; 1975 int ret; 1976 1977 if (nested_svm_check_permissions(vcpu)) 1978 return 1; 1979 1980 ret = kvm_vcpu_map(vcpu, gpa_to_gfn(svm->vmcb->save.rax), &map); 1981 if (ret) { 1982 if (ret == -EINVAL) 1983 kvm_inject_gp(vcpu, 0); 1984 return 1; 1985 } 1986 1987 vmcb12 = map.hva; 1988 1989 ret = kvm_skip_emulated_instruction(vcpu); 1990 1991 if (vmload) { 1992 svm_copy_vmloadsave_state(svm->vmcb, vmcb12); 1993 svm->sysenter_eip_hi = 0; 1994 svm->sysenter_esp_hi = 0; 1995 } else { 1996 svm_copy_vmloadsave_state(vmcb12, svm->vmcb); 1997 } 1998 1999 kvm_vcpu_unmap(vcpu, &map, true); 2000 2001 return ret; 2002 } 2003 2004 static int vmload_interception(struct kvm_vcpu *vcpu) 2005 { 2006 return vmload_vmsave_interception(vcpu, true); 2007 } 2008 2009 static int vmsave_interception(struct kvm_vcpu *vcpu) 2010 { 2011 return vmload_vmsave_interception(vcpu, false); 2012 } 2013 2014 static int vmrun_interception(struct kvm_vcpu *vcpu) 2015 { 2016 if (nested_svm_check_permissions(vcpu)) 2017 return 1; 2018 2019 return nested_svm_vmrun(vcpu); 2020 } 2021 2022 enum { 2023 NONE_SVM_INSTR, 2024 SVM_INSTR_VMRUN, 2025 SVM_INSTR_VMLOAD, 2026 SVM_INSTR_VMSAVE, 2027 }; 2028 2029 /* Return NONE_SVM_INSTR if not SVM instrs, otherwise return decode result */ 2030 static int svm_instr_opcode(struct kvm_vcpu *vcpu) 2031 { 2032 struct x86_emulate_ctxt *ctxt = vcpu->arch.emulate_ctxt; 2033 2034 if (ctxt->b != 0x1 || ctxt->opcode_len != 2) 2035 return NONE_SVM_INSTR; 2036 2037 switch (ctxt->modrm) { 2038 case 0xd8: /* VMRUN */ 2039 return SVM_INSTR_VMRUN; 2040 case 0xda: /* VMLOAD */ 2041 return SVM_INSTR_VMLOAD; 2042 case 0xdb: /* VMSAVE */ 2043 return SVM_INSTR_VMSAVE; 2044 default: 2045 break; 2046 } 2047 2048 return NONE_SVM_INSTR; 2049 } 2050 2051 static int emulate_svm_instr(struct kvm_vcpu *vcpu, int opcode) 2052 { 2053 const int guest_mode_exit_codes[] = { 2054 [SVM_INSTR_VMRUN] = SVM_EXIT_VMRUN, 2055 [SVM_INSTR_VMLOAD] = SVM_EXIT_VMLOAD, 2056 [SVM_INSTR_VMSAVE] = SVM_EXIT_VMSAVE, 2057 }; 2058 int (*const svm_instr_handlers[])(struct kvm_vcpu *vcpu) = { 2059 [SVM_INSTR_VMRUN] = vmrun_interception, 2060 [SVM_INSTR_VMLOAD] = vmload_interception, 2061 [SVM_INSTR_VMSAVE] = vmsave_interception, 2062 }; 2063 struct vcpu_svm *svm = to_svm(vcpu); 2064 int ret; 2065 2066 if (is_guest_mode(vcpu)) { 2067 /* Returns '1' or -errno on failure, '0' on success. */ 2068 ret = nested_svm_simple_vmexit(svm, guest_mode_exit_codes[opcode]); 2069 if (ret) 2070 return ret; 2071 return 1; 2072 } 2073 return svm_instr_handlers[opcode](vcpu); 2074 } 2075 2076 /* 2077 * #GP handling code. Note that #GP can be triggered under the following two 2078 * cases: 2079 * 1) SVM VM-related instructions (VMRUN/VMSAVE/VMLOAD) that trigger #GP on 2080 * some AMD CPUs when EAX of these instructions are in the reserved memory 2081 * regions (e.g. SMM memory on host). 2082 * 2) VMware backdoor 2083 */ 2084 static int gp_interception(struct kvm_vcpu *vcpu) 2085 { 2086 struct vcpu_svm *svm = to_svm(vcpu); 2087 u32 error_code = svm->vmcb->control.exit_info_1; 2088 int opcode; 2089 2090 /* Both #GP cases have zero error_code */ 2091 if (error_code) 2092 goto reinject; 2093 2094 /* All SVM instructions expect page aligned RAX */ 2095 if (svm->vmcb->save.rax & ~PAGE_MASK) 2096 goto reinject; 2097 2098 /* Decode the instruction for usage later */ 2099 if (x86_decode_emulated_instruction(vcpu, 0, NULL, 0) != EMULATION_OK) 2100 goto reinject; 2101 2102 opcode = svm_instr_opcode(vcpu); 2103 2104 if (opcode == NONE_SVM_INSTR) { 2105 if (!enable_vmware_backdoor) 2106 goto reinject; 2107 2108 /* 2109 * VMware backdoor emulation on #GP interception only handles 2110 * IN{S}, OUT{S}, and RDPMC. 2111 */ 2112 if (!is_guest_mode(vcpu)) 2113 return kvm_emulate_instruction(vcpu, 2114 EMULTYPE_VMWARE_GP | EMULTYPE_NO_DECODE); 2115 } else 2116 return emulate_svm_instr(vcpu, opcode); 2117 2118 reinject: 2119 kvm_queue_exception_e(vcpu, GP_VECTOR, error_code); 2120 return 1; 2121 } 2122 2123 void svm_set_gif(struct vcpu_svm *svm, bool value) 2124 { 2125 if (value) { 2126 /* 2127 * If VGIF is enabled, the STGI intercept is only added to 2128 * detect the opening of the SMI/NMI window; remove it now. 2129 * Likewise, clear the VINTR intercept, we will set it 2130 * again while processing KVM_REQ_EVENT if needed. 2131 */ 2132 if (vgif_enabled(svm)) 2133 svm_clr_intercept(svm, INTERCEPT_STGI); 2134 if (svm_is_intercept(svm, INTERCEPT_VINTR)) 2135 svm_clear_vintr(svm); 2136 2137 enable_gif(svm); 2138 if (svm->vcpu.arch.smi_pending || 2139 svm->vcpu.arch.nmi_pending || 2140 kvm_cpu_has_injectable_intr(&svm->vcpu)) 2141 kvm_make_request(KVM_REQ_EVENT, &svm->vcpu); 2142 } else { 2143 disable_gif(svm); 2144 2145 /* 2146 * After a CLGI no interrupts should come. But if vGIF is 2147 * in use, we still rely on the VINTR intercept (rather than 2148 * STGI) to detect an open interrupt window. 2149 */ 2150 if (!vgif_enabled(svm)) 2151 svm_clear_vintr(svm); 2152 } 2153 } 2154 2155 static int stgi_interception(struct kvm_vcpu *vcpu) 2156 { 2157 int ret; 2158 2159 if (nested_svm_check_permissions(vcpu)) 2160 return 1; 2161 2162 ret = kvm_skip_emulated_instruction(vcpu); 2163 svm_set_gif(to_svm(vcpu), true); 2164 return ret; 2165 } 2166 2167 static int clgi_interception(struct kvm_vcpu *vcpu) 2168 { 2169 int ret; 2170 2171 if (nested_svm_check_permissions(vcpu)) 2172 return 1; 2173 2174 ret = kvm_skip_emulated_instruction(vcpu); 2175 svm_set_gif(to_svm(vcpu), false); 2176 return ret; 2177 } 2178 2179 static int invlpga_interception(struct kvm_vcpu *vcpu) 2180 { 2181 gva_t gva = kvm_rax_read(vcpu); 2182 u32 asid = kvm_rcx_read(vcpu); 2183 2184 /* FIXME: Handle an address size prefix. */ 2185 if (!is_long_mode(vcpu)) 2186 gva = (u32)gva; 2187 2188 trace_kvm_invlpga(to_svm(vcpu)->vmcb->save.rip, asid, gva); 2189 2190 /* Let's treat INVLPGA the same as INVLPG (can be optimized!) */ 2191 kvm_mmu_invlpg(vcpu, gva); 2192 2193 return kvm_skip_emulated_instruction(vcpu); 2194 } 2195 2196 static int skinit_interception(struct kvm_vcpu *vcpu) 2197 { 2198 trace_kvm_skinit(to_svm(vcpu)->vmcb->save.rip, kvm_rax_read(vcpu)); 2199 2200 kvm_queue_exception(vcpu, UD_VECTOR); 2201 return 1; 2202 } 2203 2204 static int task_switch_interception(struct kvm_vcpu *vcpu) 2205 { 2206 struct vcpu_svm *svm = to_svm(vcpu); 2207 u16 tss_selector; 2208 int reason; 2209 int int_type = svm->vmcb->control.exit_int_info & 2210 SVM_EXITINTINFO_TYPE_MASK; 2211 int int_vec = svm->vmcb->control.exit_int_info & SVM_EVTINJ_VEC_MASK; 2212 uint32_t type = 2213 svm->vmcb->control.exit_int_info & SVM_EXITINTINFO_TYPE_MASK; 2214 uint32_t idt_v = 2215 svm->vmcb->control.exit_int_info & SVM_EXITINTINFO_VALID; 2216 bool has_error_code = false; 2217 u32 error_code = 0; 2218 2219 tss_selector = (u16)svm->vmcb->control.exit_info_1; 2220 2221 if (svm->vmcb->control.exit_info_2 & 2222 (1ULL << SVM_EXITINFOSHIFT_TS_REASON_IRET)) 2223 reason = TASK_SWITCH_IRET; 2224 else if (svm->vmcb->control.exit_info_2 & 2225 (1ULL << SVM_EXITINFOSHIFT_TS_REASON_JMP)) 2226 reason = TASK_SWITCH_JMP; 2227 else if (idt_v) 2228 reason = TASK_SWITCH_GATE; 2229 else 2230 reason = TASK_SWITCH_CALL; 2231 2232 if (reason == TASK_SWITCH_GATE) { 2233 switch (type) { 2234 case SVM_EXITINTINFO_TYPE_NMI: 2235 vcpu->arch.nmi_injected = false; 2236 break; 2237 case SVM_EXITINTINFO_TYPE_EXEPT: 2238 if (svm->vmcb->control.exit_info_2 & 2239 (1ULL << SVM_EXITINFOSHIFT_TS_HAS_ERROR_CODE)) { 2240 has_error_code = true; 2241 error_code = 2242 (u32)svm->vmcb->control.exit_info_2; 2243 } 2244 kvm_clear_exception_queue(vcpu); 2245 break; 2246 case SVM_EXITINTINFO_TYPE_INTR: 2247 kvm_clear_interrupt_queue(vcpu); 2248 break; 2249 default: 2250 break; 2251 } 2252 } 2253 2254 if (reason != TASK_SWITCH_GATE || 2255 int_type == SVM_EXITINTINFO_TYPE_SOFT || 2256 (int_type == SVM_EXITINTINFO_TYPE_EXEPT && 2257 (int_vec == OF_VECTOR || int_vec == BP_VECTOR))) { 2258 if (!skip_emulated_instruction(vcpu)) 2259 return 0; 2260 } 2261 2262 if (int_type != SVM_EXITINTINFO_TYPE_SOFT) 2263 int_vec = -1; 2264 2265 return kvm_task_switch(vcpu, tss_selector, int_vec, reason, 2266 has_error_code, error_code); 2267 } 2268 2269 static int iret_interception(struct kvm_vcpu *vcpu) 2270 { 2271 struct vcpu_svm *svm = to_svm(vcpu); 2272 2273 ++vcpu->stat.nmi_window_exits; 2274 vcpu->arch.hflags |= HF_IRET_MASK; 2275 if (!sev_es_guest(vcpu->kvm)) { 2276 svm_clr_intercept(svm, INTERCEPT_IRET); 2277 svm->nmi_iret_rip = kvm_rip_read(vcpu); 2278 } 2279 kvm_make_request(KVM_REQ_EVENT, vcpu); 2280 return 1; 2281 } 2282 2283 static int invlpg_interception(struct kvm_vcpu *vcpu) 2284 { 2285 if (!static_cpu_has(X86_FEATURE_DECODEASSISTS)) 2286 return kvm_emulate_instruction(vcpu, 0); 2287 2288 kvm_mmu_invlpg(vcpu, to_svm(vcpu)->vmcb->control.exit_info_1); 2289 return kvm_skip_emulated_instruction(vcpu); 2290 } 2291 2292 static int emulate_on_interception(struct kvm_vcpu *vcpu) 2293 { 2294 return kvm_emulate_instruction(vcpu, 0); 2295 } 2296 2297 static int rsm_interception(struct kvm_vcpu *vcpu) 2298 { 2299 return kvm_emulate_instruction_from_buffer(vcpu, rsm_ins_bytes, 2); 2300 } 2301 2302 static bool check_selective_cr0_intercepted(struct kvm_vcpu *vcpu, 2303 unsigned long val) 2304 { 2305 struct vcpu_svm *svm = to_svm(vcpu); 2306 unsigned long cr0 = vcpu->arch.cr0; 2307 bool ret = false; 2308 2309 if (!is_guest_mode(vcpu) || 2310 (!(vmcb12_is_intercept(&svm->nested.ctl, INTERCEPT_SELECTIVE_CR0)))) 2311 return false; 2312 2313 cr0 &= ~SVM_CR0_SELECTIVE_MASK; 2314 val &= ~SVM_CR0_SELECTIVE_MASK; 2315 2316 if (cr0 ^ val) { 2317 svm->vmcb->control.exit_code = SVM_EXIT_CR0_SEL_WRITE; 2318 ret = (nested_svm_exit_handled(svm) == NESTED_EXIT_DONE); 2319 } 2320 2321 return ret; 2322 } 2323 2324 #define CR_VALID (1ULL << 63) 2325 2326 static int cr_interception(struct kvm_vcpu *vcpu) 2327 { 2328 struct vcpu_svm *svm = to_svm(vcpu); 2329 int reg, cr; 2330 unsigned long val; 2331 int err; 2332 2333 if (!static_cpu_has(X86_FEATURE_DECODEASSISTS)) 2334 return emulate_on_interception(vcpu); 2335 2336 if (unlikely((svm->vmcb->control.exit_info_1 & CR_VALID) == 0)) 2337 return emulate_on_interception(vcpu); 2338 2339 reg = svm->vmcb->control.exit_info_1 & SVM_EXITINFO_REG_MASK; 2340 if (svm->vmcb->control.exit_code == SVM_EXIT_CR0_SEL_WRITE) 2341 cr = SVM_EXIT_WRITE_CR0 - SVM_EXIT_READ_CR0; 2342 else 2343 cr = svm->vmcb->control.exit_code - SVM_EXIT_READ_CR0; 2344 2345 err = 0; 2346 if (cr >= 16) { /* mov to cr */ 2347 cr -= 16; 2348 val = kvm_register_read(vcpu, reg); 2349 trace_kvm_cr_write(cr, val); 2350 switch (cr) { 2351 case 0: 2352 if (!check_selective_cr0_intercepted(vcpu, val)) 2353 err = kvm_set_cr0(vcpu, val); 2354 else 2355 return 1; 2356 2357 break; 2358 case 3: 2359 err = kvm_set_cr3(vcpu, val); 2360 break; 2361 case 4: 2362 err = kvm_set_cr4(vcpu, val); 2363 break; 2364 case 8: 2365 err = kvm_set_cr8(vcpu, val); 2366 break; 2367 default: 2368 WARN(1, "unhandled write to CR%d", cr); 2369 kvm_queue_exception(vcpu, UD_VECTOR); 2370 return 1; 2371 } 2372 } else { /* mov from cr */ 2373 switch (cr) { 2374 case 0: 2375 val = kvm_read_cr0(vcpu); 2376 break; 2377 case 2: 2378 val = vcpu->arch.cr2; 2379 break; 2380 case 3: 2381 val = kvm_read_cr3(vcpu); 2382 break; 2383 case 4: 2384 val = kvm_read_cr4(vcpu); 2385 break; 2386 case 8: 2387 val = kvm_get_cr8(vcpu); 2388 break; 2389 default: 2390 WARN(1, "unhandled read from CR%d", cr); 2391 kvm_queue_exception(vcpu, UD_VECTOR); 2392 return 1; 2393 } 2394 kvm_register_write(vcpu, reg, val); 2395 trace_kvm_cr_read(cr, val); 2396 } 2397 return kvm_complete_insn_gp(vcpu, err); 2398 } 2399 2400 static int cr_trap(struct kvm_vcpu *vcpu) 2401 { 2402 struct vcpu_svm *svm = to_svm(vcpu); 2403 unsigned long old_value, new_value; 2404 unsigned int cr; 2405 int ret = 0; 2406 2407 new_value = (unsigned long)svm->vmcb->control.exit_info_1; 2408 2409 cr = svm->vmcb->control.exit_code - SVM_EXIT_CR0_WRITE_TRAP; 2410 switch (cr) { 2411 case 0: 2412 old_value = kvm_read_cr0(vcpu); 2413 svm_set_cr0(vcpu, new_value); 2414 2415 kvm_post_set_cr0(vcpu, old_value, new_value); 2416 break; 2417 case 4: 2418 old_value = kvm_read_cr4(vcpu); 2419 svm_set_cr4(vcpu, new_value); 2420 2421 kvm_post_set_cr4(vcpu, old_value, new_value); 2422 break; 2423 case 8: 2424 ret = kvm_set_cr8(vcpu, new_value); 2425 break; 2426 default: 2427 WARN(1, "unhandled CR%d write trap", cr); 2428 kvm_queue_exception(vcpu, UD_VECTOR); 2429 return 1; 2430 } 2431 2432 return kvm_complete_insn_gp(vcpu, ret); 2433 } 2434 2435 static int dr_interception(struct kvm_vcpu *vcpu) 2436 { 2437 struct vcpu_svm *svm = to_svm(vcpu); 2438 int reg, dr; 2439 unsigned long val; 2440 int err = 0; 2441 2442 if (vcpu->guest_debug == 0) { 2443 /* 2444 * No more DR vmexits; force a reload of the debug registers 2445 * and reenter on this instruction. The next vmexit will 2446 * retrieve the full state of the debug registers. 2447 */ 2448 clr_dr_intercepts(svm); 2449 vcpu->arch.switch_db_regs |= KVM_DEBUGREG_WONT_EXIT; 2450 return 1; 2451 } 2452 2453 if (!boot_cpu_has(X86_FEATURE_DECODEASSISTS)) 2454 return emulate_on_interception(vcpu); 2455 2456 reg = svm->vmcb->control.exit_info_1 & SVM_EXITINFO_REG_MASK; 2457 dr = svm->vmcb->control.exit_code - SVM_EXIT_READ_DR0; 2458 if (dr >= 16) { /* mov to DRn */ 2459 dr -= 16; 2460 val = kvm_register_read(vcpu, reg); 2461 err = kvm_set_dr(vcpu, dr, val); 2462 } else { 2463 kvm_get_dr(vcpu, dr, &val); 2464 kvm_register_write(vcpu, reg, val); 2465 } 2466 2467 return kvm_complete_insn_gp(vcpu, err); 2468 } 2469 2470 static int cr8_write_interception(struct kvm_vcpu *vcpu) 2471 { 2472 int r; 2473 2474 u8 cr8_prev = kvm_get_cr8(vcpu); 2475 /* instruction emulation calls kvm_set_cr8() */ 2476 r = cr_interception(vcpu); 2477 if (lapic_in_kernel(vcpu)) 2478 return r; 2479 if (cr8_prev <= kvm_get_cr8(vcpu)) 2480 return r; 2481 vcpu->run->exit_reason = KVM_EXIT_SET_TPR; 2482 return 0; 2483 } 2484 2485 static int efer_trap(struct kvm_vcpu *vcpu) 2486 { 2487 struct msr_data msr_info; 2488 int ret; 2489 2490 /* 2491 * Clear the EFER_SVME bit from EFER. The SVM code always sets this 2492 * bit in svm_set_efer(), but __kvm_valid_efer() checks it against 2493 * whether the guest has X86_FEATURE_SVM - this avoids a failure if 2494 * the guest doesn't have X86_FEATURE_SVM. 2495 */ 2496 msr_info.host_initiated = false; 2497 msr_info.index = MSR_EFER; 2498 msr_info.data = to_svm(vcpu)->vmcb->control.exit_info_1 & ~EFER_SVME; 2499 ret = kvm_set_msr_common(vcpu, &msr_info); 2500 2501 return kvm_complete_insn_gp(vcpu, ret); 2502 } 2503 2504 static int svm_get_msr_feature(struct kvm_msr_entry *msr) 2505 { 2506 msr->data = 0; 2507 2508 switch (msr->index) { 2509 case MSR_F10H_DECFG: 2510 if (boot_cpu_has(X86_FEATURE_LFENCE_RDTSC)) 2511 msr->data |= MSR_F10H_DECFG_LFENCE_SERIALIZE; 2512 break; 2513 case MSR_IA32_PERF_CAPABILITIES: 2514 return 0; 2515 default: 2516 return KVM_MSR_RET_INVALID; 2517 } 2518 2519 return 0; 2520 } 2521 2522 static int svm_get_msr(struct kvm_vcpu *vcpu, struct msr_data *msr_info) 2523 { 2524 struct vcpu_svm *svm = to_svm(vcpu); 2525 2526 switch (msr_info->index) { 2527 case MSR_AMD64_TSC_RATIO: 2528 if (!msr_info->host_initiated && !svm->tsc_scaling_enabled) 2529 return 1; 2530 msr_info->data = svm->tsc_ratio_msr; 2531 break; 2532 case MSR_STAR: 2533 msr_info->data = svm->vmcb01.ptr->save.star; 2534 break; 2535 #ifdef CONFIG_X86_64 2536 case MSR_LSTAR: 2537 msr_info->data = svm->vmcb01.ptr->save.lstar; 2538 break; 2539 case MSR_CSTAR: 2540 msr_info->data = svm->vmcb01.ptr->save.cstar; 2541 break; 2542 case MSR_KERNEL_GS_BASE: 2543 msr_info->data = svm->vmcb01.ptr->save.kernel_gs_base; 2544 break; 2545 case MSR_SYSCALL_MASK: 2546 msr_info->data = svm->vmcb01.ptr->save.sfmask; 2547 break; 2548 #endif 2549 case MSR_IA32_SYSENTER_CS: 2550 msr_info->data = svm->vmcb01.ptr->save.sysenter_cs; 2551 break; 2552 case MSR_IA32_SYSENTER_EIP: 2553 msr_info->data = (u32)svm->vmcb01.ptr->save.sysenter_eip; 2554 if (guest_cpuid_is_intel(vcpu)) 2555 msr_info->data |= (u64)svm->sysenter_eip_hi << 32; 2556 break; 2557 case MSR_IA32_SYSENTER_ESP: 2558 msr_info->data = svm->vmcb01.ptr->save.sysenter_esp; 2559 if (guest_cpuid_is_intel(vcpu)) 2560 msr_info->data |= (u64)svm->sysenter_esp_hi << 32; 2561 break; 2562 case MSR_TSC_AUX: 2563 msr_info->data = svm->tsc_aux; 2564 break; 2565 /* 2566 * Nobody will change the following 5 values in the VMCB so we can 2567 * safely return them on rdmsr. They will always be 0 until LBRV is 2568 * implemented. 2569 */ 2570 case MSR_IA32_DEBUGCTLMSR: 2571 msr_info->data = svm->vmcb->save.dbgctl; 2572 break; 2573 case MSR_IA32_LASTBRANCHFROMIP: 2574 msr_info->data = svm->vmcb->save.br_from; 2575 break; 2576 case MSR_IA32_LASTBRANCHTOIP: 2577 msr_info->data = svm->vmcb->save.br_to; 2578 break; 2579 case MSR_IA32_LASTINTFROMIP: 2580 msr_info->data = svm->vmcb->save.last_excp_from; 2581 break; 2582 case MSR_IA32_LASTINTTOIP: 2583 msr_info->data = svm->vmcb->save.last_excp_to; 2584 break; 2585 case MSR_VM_HSAVE_PA: 2586 msr_info->data = svm->nested.hsave_msr; 2587 break; 2588 case MSR_VM_CR: 2589 msr_info->data = svm->nested.vm_cr_msr; 2590 break; 2591 case MSR_IA32_SPEC_CTRL: 2592 if (!msr_info->host_initiated && 2593 !guest_has_spec_ctrl_msr(vcpu)) 2594 return 1; 2595 2596 if (boot_cpu_has(X86_FEATURE_V_SPEC_CTRL)) 2597 msr_info->data = svm->vmcb->save.spec_ctrl; 2598 else 2599 msr_info->data = svm->spec_ctrl; 2600 break; 2601 case MSR_AMD64_VIRT_SPEC_CTRL: 2602 if (!msr_info->host_initiated && 2603 !guest_cpuid_has(vcpu, X86_FEATURE_VIRT_SSBD)) 2604 return 1; 2605 2606 msr_info->data = svm->virt_spec_ctrl; 2607 break; 2608 case MSR_F15H_IC_CFG: { 2609 2610 int family, model; 2611 2612 family = guest_cpuid_family(vcpu); 2613 model = guest_cpuid_model(vcpu); 2614 2615 if (family < 0 || model < 0) 2616 return kvm_get_msr_common(vcpu, msr_info); 2617 2618 msr_info->data = 0; 2619 2620 if (family == 0x15 && 2621 (model >= 0x2 && model < 0x20)) 2622 msr_info->data = 0x1E; 2623 } 2624 break; 2625 case MSR_F10H_DECFG: 2626 msr_info->data = svm->msr_decfg; 2627 break; 2628 default: 2629 return kvm_get_msr_common(vcpu, msr_info); 2630 } 2631 return 0; 2632 } 2633 2634 static int svm_complete_emulated_msr(struct kvm_vcpu *vcpu, int err) 2635 { 2636 struct vcpu_svm *svm = to_svm(vcpu); 2637 if (!err || !sev_es_guest(vcpu->kvm) || WARN_ON_ONCE(!svm->sev_es.ghcb)) 2638 return kvm_complete_insn_gp(vcpu, err); 2639 2640 ghcb_set_sw_exit_info_1(svm->sev_es.ghcb, 1); 2641 ghcb_set_sw_exit_info_2(svm->sev_es.ghcb, 2642 X86_TRAP_GP | 2643 SVM_EVTINJ_TYPE_EXEPT | 2644 SVM_EVTINJ_VALID); 2645 return 1; 2646 } 2647 2648 static int svm_set_vm_cr(struct kvm_vcpu *vcpu, u64 data) 2649 { 2650 struct vcpu_svm *svm = to_svm(vcpu); 2651 int svm_dis, chg_mask; 2652 2653 if (data & ~SVM_VM_CR_VALID_MASK) 2654 return 1; 2655 2656 chg_mask = SVM_VM_CR_VALID_MASK; 2657 2658 if (svm->nested.vm_cr_msr & SVM_VM_CR_SVM_DIS_MASK) 2659 chg_mask &= ~(SVM_VM_CR_SVM_LOCK_MASK | SVM_VM_CR_SVM_DIS_MASK); 2660 2661 svm->nested.vm_cr_msr &= ~chg_mask; 2662 svm->nested.vm_cr_msr |= (data & chg_mask); 2663 2664 svm_dis = svm->nested.vm_cr_msr & SVM_VM_CR_SVM_DIS_MASK; 2665 2666 /* check for svm_disable while efer.svme is set */ 2667 if (svm_dis && (vcpu->arch.efer & EFER_SVME)) 2668 return 1; 2669 2670 return 0; 2671 } 2672 2673 static int svm_set_msr(struct kvm_vcpu *vcpu, struct msr_data *msr) 2674 { 2675 struct vcpu_svm *svm = to_svm(vcpu); 2676 int r; 2677 2678 u32 ecx = msr->index; 2679 u64 data = msr->data; 2680 switch (ecx) { 2681 case MSR_AMD64_TSC_RATIO: 2682 if (!msr->host_initiated && !svm->tsc_scaling_enabled) 2683 return 1; 2684 2685 if (data & TSC_RATIO_RSVD) 2686 return 1; 2687 2688 svm->tsc_ratio_msr = data; 2689 2690 if (svm->tsc_scaling_enabled && is_guest_mode(vcpu)) 2691 nested_svm_update_tsc_ratio_msr(vcpu); 2692 2693 break; 2694 case MSR_IA32_CR_PAT: 2695 if (!kvm_mtrr_valid(vcpu, MSR_IA32_CR_PAT, data)) 2696 return 1; 2697 vcpu->arch.pat = data; 2698 svm->vmcb01.ptr->save.g_pat = data; 2699 if (is_guest_mode(vcpu)) 2700 nested_vmcb02_compute_g_pat(svm); 2701 vmcb_mark_dirty(svm->vmcb, VMCB_NPT); 2702 break; 2703 case MSR_IA32_SPEC_CTRL: 2704 if (!msr->host_initiated && 2705 !guest_has_spec_ctrl_msr(vcpu)) 2706 return 1; 2707 2708 if (kvm_spec_ctrl_test_value(data)) 2709 return 1; 2710 2711 if (boot_cpu_has(X86_FEATURE_V_SPEC_CTRL)) 2712 svm->vmcb->save.spec_ctrl = data; 2713 else 2714 svm->spec_ctrl = data; 2715 if (!data) 2716 break; 2717 2718 /* 2719 * For non-nested: 2720 * When it's written (to non-zero) for the first time, pass 2721 * it through. 2722 * 2723 * For nested: 2724 * The handling of the MSR bitmap for L2 guests is done in 2725 * nested_svm_vmrun_msrpm. 2726 * We update the L1 MSR bit as well since it will end up 2727 * touching the MSR anyway now. 2728 */ 2729 set_msr_interception(vcpu, svm->msrpm, MSR_IA32_SPEC_CTRL, 1, 1); 2730 break; 2731 case MSR_IA32_PRED_CMD: 2732 if (!msr->host_initiated && 2733 !guest_has_pred_cmd_msr(vcpu)) 2734 return 1; 2735 2736 if (data & ~PRED_CMD_IBPB) 2737 return 1; 2738 if (!boot_cpu_has(X86_FEATURE_IBPB)) 2739 return 1; 2740 if (!data) 2741 break; 2742 2743 wrmsrl(MSR_IA32_PRED_CMD, PRED_CMD_IBPB); 2744 set_msr_interception(vcpu, svm->msrpm, MSR_IA32_PRED_CMD, 0, 1); 2745 break; 2746 case MSR_AMD64_VIRT_SPEC_CTRL: 2747 if (!msr->host_initiated && 2748 !guest_cpuid_has(vcpu, X86_FEATURE_VIRT_SSBD)) 2749 return 1; 2750 2751 if (data & ~SPEC_CTRL_SSBD) 2752 return 1; 2753 2754 svm->virt_spec_ctrl = data; 2755 break; 2756 case MSR_STAR: 2757 svm->vmcb01.ptr->save.star = data; 2758 break; 2759 #ifdef CONFIG_X86_64 2760 case MSR_LSTAR: 2761 svm->vmcb01.ptr->save.lstar = data; 2762 break; 2763 case MSR_CSTAR: 2764 svm->vmcb01.ptr->save.cstar = data; 2765 break; 2766 case MSR_KERNEL_GS_BASE: 2767 svm->vmcb01.ptr->save.kernel_gs_base = data; 2768 break; 2769 case MSR_SYSCALL_MASK: 2770 svm->vmcb01.ptr->save.sfmask = data; 2771 break; 2772 #endif 2773 case MSR_IA32_SYSENTER_CS: 2774 svm->vmcb01.ptr->save.sysenter_cs = data; 2775 break; 2776 case MSR_IA32_SYSENTER_EIP: 2777 svm->vmcb01.ptr->save.sysenter_eip = (u32)data; 2778 /* 2779 * We only intercept the MSR_IA32_SYSENTER_{EIP|ESP} msrs 2780 * when we spoof an Intel vendor ID (for cross vendor migration). 2781 * In this case we use this intercept to track the high 2782 * 32 bit part of these msrs to support Intel's 2783 * implementation of SYSENTER/SYSEXIT. 2784 */ 2785 svm->sysenter_eip_hi = guest_cpuid_is_intel(vcpu) ? (data >> 32) : 0; 2786 break; 2787 case MSR_IA32_SYSENTER_ESP: 2788 svm->vmcb01.ptr->save.sysenter_esp = (u32)data; 2789 svm->sysenter_esp_hi = guest_cpuid_is_intel(vcpu) ? (data >> 32) : 0; 2790 break; 2791 case MSR_TSC_AUX: 2792 /* 2793 * TSC_AUX is usually changed only during boot and never read 2794 * directly. Intercept TSC_AUX instead of exposing it to the 2795 * guest via direct_access_msrs, and switch it via user return. 2796 */ 2797 preempt_disable(); 2798 r = kvm_set_user_return_msr(tsc_aux_uret_slot, data, -1ull); 2799 preempt_enable(); 2800 if (r) 2801 return 1; 2802 2803 svm->tsc_aux = data; 2804 break; 2805 case MSR_IA32_DEBUGCTLMSR: 2806 if (!lbrv) { 2807 vcpu_unimpl(vcpu, "%s: MSR_IA32_DEBUGCTL 0x%llx, nop\n", 2808 __func__, data); 2809 break; 2810 } 2811 if (data & DEBUGCTL_RESERVED_BITS) 2812 return 1; 2813 2814 svm->vmcb->save.dbgctl = data; 2815 vmcb_mark_dirty(svm->vmcb, VMCB_LBR); 2816 if (data & (1ULL<<0)) 2817 svm_enable_lbrv(vcpu); 2818 else 2819 svm_disable_lbrv(vcpu); 2820 break; 2821 case MSR_VM_HSAVE_PA: 2822 /* 2823 * Old kernels did not validate the value written to 2824 * MSR_VM_HSAVE_PA. Allow KVM_SET_MSR to set an invalid 2825 * value to allow live migrating buggy or malicious guests 2826 * originating from those kernels. 2827 */ 2828 if (!msr->host_initiated && !page_address_valid(vcpu, data)) 2829 return 1; 2830 2831 svm->nested.hsave_msr = data & PAGE_MASK; 2832 break; 2833 case MSR_VM_CR: 2834 return svm_set_vm_cr(vcpu, data); 2835 case MSR_VM_IGNNE: 2836 vcpu_unimpl(vcpu, "unimplemented wrmsr: 0x%x data 0x%llx\n", ecx, data); 2837 break; 2838 case MSR_F10H_DECFG: { 2839 struct kvm_msr_entry msr_entry; 2840 2841 msr_entry.index = msr->index; 2842 if (svm_get_msr_feature(&msr_entry)) 2843 return 1; 2844 2845 /* Check the supported bits */ 2846 if (data & ~msr_entry.data) 2847 return 1; 2848 2849 /* Don't allow the guest to change a bit, #GP */ 2850 if (!msr->host_initiated && (data ^ msr_entry.data)) 2851 return 1; 2852 2853 svm->msr_decfg = data; 2854 break; 2855 } 2856 default: 2857 return kvm_set_msr_common(vcpu, msr); 2858 } 2859 return 0; 2860 } 2861 2862 static int msr_interception(struct kvm_vcpu *vcpu) 2863 { 2864 if (to_svm(vcpu)->vmcb->control.exit_info_1) 2865 return kvm_emulate_wrmsr(vcpu); 2866 else 2867 return kvm_emulate_rdmsr(vcpu); 2868 } 2869 2870 static int interrupt_window_interception(struct kvm_vcpu *vcpu) 2871 { 2872 kvm_make_request(KVM_REQ_EVENT, vcpu); 2873 svm_clear_vintr(to_svm(vcpu)); 2874 2875 /* 2876 * For AVIC, the only reason to end up here is ExtINTs. 2877 * In this case AVIC was temporarily disabled for 2878 * requesting the IRQ window and we have to re-enable it. 2879 */ 2880 kvm_request_apicv_update(vcpu->kvm, true, APICV_INHIBIT_REASON_IRQWIN); 2881 2882 ++vcpu->stat.irq_window_exits; 2883 return 1; 2884 } 2885 2886 static int pause_interception(struct kvm_vcpu *vcpu) 2887 { 2888 bool in_kernel; 2889 2890 /* 2891 * CPL is not made available for an SEV-ES guest, therefore 2892 * vcpu->arch.preempted_in_kernel can never be true. Just 2893 * set in_kernel to false as well. 2894 */ 2895 in_kernel = !sev_es_guest(vcpu->kvm) && svm_get_cpl(vcpu) == 0; 2896 2897 if (!kvm_pause_in_guest(vcpu->kvm)) 2898 grow_ple_window(vcpu); 2899 2900 kvm_vcpu_on_spin(vcpu, in_kernel); 2901 return kvm_skip_emulated_instruction(vcpu); 2902 } 2903 2904 static int invpcid_interception(struct kvm_vcpu *vcpu) 2905 { 2906 struct vcpu_svm *svm = to_svm(vcpu); 2907 unsigned long type; 2908 gva_t gva; 2909 2910 if (!guest_cpuid_has(vcpu, X86_FEATURE_INVPCID)) { 2911 kvm_queue_exception(vcpu, UD_VECTOR); 2912 return 1; 2913 } 2914 2915 /* 2916 * For an INVPCID intercept: 2917 * EXITINFO1 provides the linear address of the memory operand. 2918 * EXITINFO2 provides the contents of the register operand. 2919 */ 2920 type = svm->vmcb->control.exit_info_2; 2921 gva = svm->vmcb->control.exit_info_1; 2922 2923 return kvm_handle_invpcid(vcpu, type, gva); 2924 } 2925 2926 static int (*const svm_exit_handlers[])(struct kvm_vcpu *vcpu) = { 2927 [SVM_EXIT_READ_CR0] = cr_interception, 2928 [SVM_EXIT_READ_CR3] = cr_interception, 2929 [SVM_EXIT_READ_CR4] = cr_interception, 2930 [SVM_EXIT_READ_CR8] = cr_interception, 2931 [SVM_EXIT_CR0_SEL_WRITE] = cr_interception, 2932 [SVM_EXIT_WRITE_CR0] = cr_interception, 2933 [SVM_EXIT_WRITE_CR3] = cr_interception, 2934 [SVM_EXIT_WRITE_CR4] = cr_interception, 2935 [SVM_EXIT_WRITE_CR8] = cr8_write_interception, 2936 [SVM_EXIT_READ_DR0] = dr_interception, 2937 [SVM_EXIT_READ_DR1] = dr_interception, 2938 [SVM_EXIT_READ_DR2] = dr_interception, 2939 [SVM_EXIT_READ_DR3] = dr_interception, 2940 [SVM_EXIT_READ_DR4] = dr_interception, 2941 [SVM_EXIT_READ_DR5] = dr_interception, 2942 [SVM_EXIT_READ_DR6] = dr_interception, 2943 [SVM_EXIT_READ_DR7] = dr_interception, 2944 [SVM_EXIT_WRITE_DR0] = dr_interception, 2945 [SVM_EXIT_WRITE_DR1] = dr_interception, 2946 [SVM_EXIT_WRITE_DR2] = dr_interception, 2947 [SVM_EXIT_WRITE_DR3] = dr_interception, 2948 [SVM_EXIT_WRITE_DR4] = dr_interception, 2949 [SVM_EXIT_WRITE_DR5] = dr_interception, 2950 [SVM_EXIT_WRITE_DR6] = dr_interception, 2951 [SVM_EXIT_WRITE_DR7] = dr_interception, 2952 [SVM_EXIT_EXCP_BASE + DB_VECTOR] = db_interception, 2953 [SVM_EXIT_EXCP_BASE + BP_VECTOR] = bp_interception, 2954 [SVM_EXIT_EXCP_BASE + UD_VECTOR] = ud_interception, 2955 [SVM_EXIT_EXCP_BASE + PF_VECTOR] = pf_interception, 2956 [SVM_EXIT_EXCP_BASE + MC_VECTOR] = mc_interception, 2957 [SVM_EXIT_EXCP_BASE + AC_VECTOR] = ac_interception, 2958 [SVM_EXIT_EXCP_BASE + GP_VECTOR] = gp_interception, 2959 [SVM_EXIT_INTR] = intr_interception, 2960 [SVM_EXIT_NMI] = nmi_interception, 2961 [SVM_EXIT_SMI] = smi_interception, 2962 [SVM_EXIT_VINTR] = interrupt_window_interception, 2963 [SVM_EXIT_RDPMC] = kvm_emulate_rdpmc, 2964 [SVM_EXIT_CPUID] = kvm_emulate_cpuid, 2965 [SVM_EXIT_IRET] = iret_interception, 2966 [SVM_EXIT_INVD] = kvm_emulate_invd, 2967 [SVM_EXIT_PAUSE] = pause_interception, 2968 [SVM_EXIT_HLT] = kvm_emulate_halt, 2969 [SVM_EXIT_INVLPG] = invlpg_interception, 2970 [SVM_EXIT_INVLPGA] = invlpga_interception, 2971 [SVM_EXIT_IOIO] = io_interception, 2972 [SVM_EXIT_MSR] = msr_interception, 2973 [SVM_EXIT_TASK_SWITCH] = task_switch_interception, 2974 [SVM_EXIT_SHUTDOWN] = shutdown_interception, 2975 [SVM_EXIT_VMRUN] = vmrun_interception, 2976 [SVM_EXIT_VMMCALL] = kvm_emulate_hypercall, 2977 [SVM_EXIT_VMLOAD] = vmload_interception, 2978 [SVM_EXIT_VMSAVE] = vmsave_interception, 2979 [SVM_EXIT_STGI] = stgi_interception, 2980 [SVM_EXIT_CLGI] = clgi_interception, 2981 [SVM_EXIT_SKINIT] = skinit_interception, 2982 [SVM_EXIT_RDTSCP] = kvm_handle_invalid_op, 2983 [SVM_EXIT_WBINVD] = kvm_emulate_wbinvd, 2984 [SVM_EXIT_MONITOR] = kvm_emulate_monitor, 2985 [SVM_EXIT_MWAIT] = kvm_emulate_mwait, 2986 [SVM_EXIT_XSETBV] = kvm_emulate_xsetbv, 2987 [SVM_EXIT_RDPRU] = kvm_handle_invalid_op, 2988 [SVM_EXIT_EFER_WRITE_TRAP] = efer_trap, 2989 [SVM_EXIT_CR0_WRITE_TRAP] = cr_trap, 2990 [SVM_EXIT_CR4_WRITE_TRAP] = cr_trap, 2991 [SVM_EXIT_CR8_WRITE_TRAP] = cr_trap, 2992 [SVM_EXIT_INVPCID] = invpcid_interception, 2993 [SVM_EXIT_NPF] = npf_interception, 2994 [SVM_EXIT_RSM] = rsm_interception, 2995 [SVM_EXIT_AVIC_INCOMPLETE_IPI] = avic_incomplete_ipi_interception, 2996 [SVM_EXIT_AVIC_UNACCELERATED_ACCESS] = avic_unaccelerated_access_interception, 2997 [SVM_EXIT_VMGEXIT] = sev_handle_vmgexit, 2998 }; 2999 3000 static void dump_vmcb(struct kvm_vcpu *vcpu) 3001 { 3002 struct vcpu_svm *svm = to_svm(vcpu); 3003 struct vmcb_control_area *control = &svm->vmcb->control; 3004 struct vmcb_save_area *save = &svm->vmcb->save; 3005 struct vmcb_save_area *save01 = &svm->vmcb01.ptr->save; 3006 3007 if (!dump_invalid_vmcb) { 3008 pr_warn_ratelimited("set kvm_amd.dump_invalid_vmcb=1 to dump internal KVM state.\n"); 3009 return; 3010 } 3011 3012 pr_err("VMCB %p, last attempted VMRUN on CPU %d\n", 3013 svm->current_vmcb->ptr, vcpu->arch.last_vmentry_cpu); 3014 pr_err("VMCB Control Area:\n"); 3015 pr_err("%-20s%04x\n", "cr_read:", control->intercepts[INTERCEPT_CR] & 0xffff); 3016 pr_err("%-20s%04x\n", "cr_write:", control->intercepts[INTERCEPT_CR] >> 16); 3017 pr_err("%-20s%04x\n", "dr_read:", control->intercepts[INTERCEPT_DR] & 0xffff); 3018 pr_err("%-20s%04x\n", "dr_write:", control->intercepts[INTERCEPT_DR] >> 16); 3019 pr_err("%-20s%08x\n", "exceptions:", control->intercepts[INTERCEPT_EXCEPTION]); 3020 pr_err("%-20s%08x %08x\n", "intercepts:", 3021 control->intercepts[INTERCEPT_WORD3], 3022 control->intercepts[INTERCEPT_WORD4]); 3023 pr_err("%-20s%d\n", "pause filter count:", control->pause_filter_count); 3024 pr_err("%-20s%d\n", "pause filter threshold:", 3025 control->pause_filter_thresh); 3026 pr_err("%-20s%016llx\n", "iopm_base_pa:", control->iopm_base_pa); 3027 pr_err("%-20s%016llx\n", "msrpm_base_pa:", control->msrpm_base_pa); 3028 pr_err("%-20s%016llx\n", "tsc_offset:", control->tsc_offset); 3029 pr_err("%-20s%d\n", "asid:", control->asid); 3030 pr_err("%-20s%d\n", "tlb_ctl:", control->tlb_ctl); 3031 pr_err("%-20s%08x\n", "int_ctl:", control->int_ctl); 3032 pr_err("%-20s%08x\n", "int_vector:", control->int_vector); 3033 pr_err("%-20s%08x\n", "int_state:", control->int_state); 3034 pr_err("%-20s%08x\n", "exit_code:", control->exit_code); 3035 pr_err("%-20s%016llx\n", "exit_info1:", control->exit_info_1); 3036 pr_err("%-20s%016llx\n", "exit_info2:", control->exit_info_2); 3037 pr_err("%-20s%08x\n", "exit_int_info:", control->exit_int_info); 3038 pr_err("%-20s%08x\n", "exit_int_info_err:", control->exit_int_info_err); 3039 pr_err("%-20s%lld\n", "nested_ctl:", control->nested_ctl); 3040 pr_err("%-20s%016llx\n", "nested_cr3:", control->nested_cr3); 3041 pr_err("%-20s%016llx\n", "avic_vapic_bar:", control->avic_vapic_bar); 3042 pr_err("%-20s%016llx\n", "ghcb:", control->ghcb_gpa); 3043 pr_err("%-20s%08x\n", "event_inj:", control->event_inj); 3044 pr_err("%-20s%08x\n", "event_inj_err:", control->event_inj_err); 3045 pr_err("%-20s%lld\n", "virt_ext:", control->virt_ext); 3046 pr_err("%-20s%016llx\n", "next_rip:", control->next_rip); 3047 pr_err("%-20s%016llx\n", "avic_backing_page:", control->avic_backing_page); 3048 pr_err("%-20s%016llx\n", "avic_logical_id:", control->avic_logical_id); 3049 pr_err("%-20s%016llx\n", "avic_physical_id:", control->avic_physical_id); 3050 pr_err("%-20s%016llx\n", "vmsa_pa:", control->vmsa_pa); 3051 pr_err("VMCB State Save Area:\n"); 3052 pr_err("%-5s s: %04x a: %04x l: %08x b: %016llx\n", 3053 "es:", 3054 save->es.selector, save->es.attrib, 3055 save->es.limit, save->es.base); 3056 pr_err("%-5s s: %04x a: %04x l: %08x b: %016llx\n", 3057 "cs:", 3058 save->cs.selector, save->cs.attrib, 3059 save->cs.limit, save->cs.base); 3060 pr_err("%-5s s: %04x a: %04x l: %08x b: %016llx\n", 3061 "ss:", 3062 save->ss.selector, save->ss.attrib, 3063 save->ss.limit, save->ss.base); 3064 pr_err("%-5s s: %04x a: %04x l: %08x b: %016llx\n", 3065 "ds:", 3066 save->ds.selector, save->ds.attrib, 3067 save->ds.limit, save->ds.base); 3068 pr_err("%-5s s: %04x a: %04x l: %08x b: %016llx\n", 3069 "fs:", 3070 save01->fs.selector, save01->fs.attrib, 3071 save01->fs.limit, save01->fs.base); 3072 pr_err("%-5s s: %04x a: %04x l: %08x b: %016llx\n", 3073 "gs:", 3074 save01->gs.selector, save01->gs.attrib, 3075 save01->gs.limit, save01->gs.base); 3076 pr_err("%-5s s: %04x a: %04x l: %08x b: %016llx\n", 3077 "gdtr:", 3078 save->gdtr.selector, save->gdtr.attrib, 3079 save->gdtr.limit, save->gdtr.base); 3080 pr_err("%-5s s: %04x a: %04x l: %08x b: %016llx\n", 3081 "ldtr:", 3082 save01->ldtr.selector, save01->ldtr.attrib, 3083 save01->ldtr.limit, save01->ldtr.base); 3084 pr_err("%-5s s: %04x a: %04x l: %08x b: %016llx\n", 3085 "idtr:", 3086 save->idtr.selector, save->idtr.attrib, 3087 save->idtr.limit, save->idtr.base); 3088 pr_err("%-5s s: %04x a: %04x l: %08x b: %016llx\n", 3089 "tr:", 3090 save01->tr.selector, save01->tr.attrib, 3091 save01->tr.limit, save01->tr.base); 3092 pr_err("cpl: %d efer: %016llx\n", 3093 save->cpl, save->efer); 3094 pr_err("%-15s %016llx %-13s %016llx\n", 3095 "cr0:", save->cr0, "cr2:", save->cr2); 3096 pr_err("%-15s %016llx %-13s %016llx\n", 3097 "cr3:", save->cr3, "cr4:", save->cr4); 3098 pr_err("%-15s %016llx %-13s %016llx\n", 3099 "dr6:", save->dr6, "dr7:", save->dr7); 3100 pr_err("%-15s %016llx %-13s %016llx\n", 3101 "rip:", save->rip, "rflags:", save->rflags); 3102 pr_err("%-15s %016llx %-13s %016llx\n", 3103 "rsp:", save->rsp, "rax:", save->rax); 3104 pr_err("%-15s %016llx %-13s %016llx\n", 3105 "star:", save01->star, "lstar:", save01->lstar); 3106 pr_err("%-15s %016llx %-13s %016llx\n", 3107 "cstar:", save01->cstar, "sfmask:", save01->sfmask); 3108 pr_err("%-15s %016llx %-13s %016llx\n", 3109 "kernel_gs_base:", save01->kernel_gs_base, 3110 "sysenter_cs:", save01->sysenter_cs); 3111 pr_err("%-15s %016llx %-13s %016llx\n", 3112 "sysenter_esp:", save01->sysenter_esp, 3113 "sysenter_eip:", save01->sysenter_eip); 3114 pr_err("%-15s %016llx %-13s %016llx\n", 3115 "gpat:", save->g_pat, "dbgctl:", save->dbgctl); 3116 pr_err("%-15s %016llx %-13s %016llx\n", 3117 "br_from:", save->br_from, "br_to:", save->br_to); 3118 pr_err("%-15s %016llx %-13s %016llx\n", 3119 "excp_from:", save->last_excp_from, 3120 "excp_to:", save->last_excp_to); 3121 } 3122 3123 static bool svm_check_exit_valid(struct kvm_vcpu *vcpu, u64 exit_code) 3124 { 3125 return (exit_code < ARRAY_SIZE(svm_exit_handlers) && 3126 svm_exit_handlers[exit_code]); 3127 } 3128 3129 static int svm_handle_invalid_exit(struct kvm_vcpu *vcpu, u64 exit_code) 3130 { 3131 vcpu_unimpl(vcpu, "svm: unexpected exit reason 0x%llx\n", exit_code); 3132 dump_vmcb(vcpu); 3133 vcpu->run->exit_reason = KVM_EXIT_INTERNAL_ERROR; 3134 vcpu->run->internal.suberror = KVM_INTERNAL_ERROR_UNEXPECTED_EXIT_REASON; 3135 vcpu->run->internal.ndata = 2; 3136 vcpu->run->internal.data[0] = exit_code; 3137 vcpu->run->internal.data[1] = vcpu->arch.last_vmentry_cpu; 3138 return 0; 3139 } 3140 3141 int svm_invoke_exit_handler(struct kvm_vcpu *vcpu, u64 exit_code) 3142 { 3143 if (!svm_check_exit_valid(vcpu, exit_code)) 3144 return svm_handle_invalid_exit(vcpu, exit_code); 3145 3146 #ifdef CONFIG_RETPOLINE 3147 if (exit_code == SVM_EXIT_MSR) 3148 return msr_interception(vcpu); 3149 else if (exit_code == SVM_EXIT_VINTR) 3150 return interrupt_window_interception(vcpu); 3151 else if (exit_code == SVM_EXIT_INTR) 3152 return intr_interception(vcpu); 3153 else if (exit_code == SVM_EXIT_HLT) 3154 return kvm_emulate_halt(vcpu); 3155 else if (exit_code == SVM_EXIT_NPF) 3156 return npf_interception(vcpu); 3157 #endif 3158 return svm_exit_handlers[exit_code](vcpu); 3159 } 3160 3161 static void svm_get_exit_info(struct kvm_vcpu *vcpu, u32 *reason, 3162 u64 *info1, u64 *info2, 3163 u32 *intr_info, u32 *error_code) 3164 { 3165 struct vmcb_control_area *control = &to_svm(vcpu)->vmcb->control; 3166 3167 *reason = control->exit_code; 3168 *info1 = control->exit_info_1; 3169 *info2 = control->exit_info_2; 3170 *intr_info = control->exit_int_info; 3171 if ((*intr_info & SVM_EXITINTINFO_VALID) && 3172 (*intr_info & SVM_EXITINTINFO_VALID_ERR)) 3173 *error_code = control->exit_int_info_err; 3174 else 3175 *error_code = 0; 3176 } 3177 3178 static int handle_exit(struct kvm_vcpu *vcpu, fastpath_t exit_fastpath) 3179 { 3180 struct vcpu_svm *svm = to_svm(vcpu); 3181 struct kvm_run *kvm_run = vcpu->run; 3182 u32 exit_code = svm->vmcb->control.exit_code; 3183 3184 trace_kvm_exit(vcpu, KVM_ISA_SVM); 3185 3186 /* SEV-ES guests must use the CR write traps to track CR registers. */ 3187 if (!sev_es_guest(vcpu->kvm)) { 3188 if (!svm_is_intercept(svm, INTERCEPT_CR0_WRITE)) 3189 vcpu->arch.cr0 = svm->vmcb->save.cr0; 3190 if (npt_enabled) 3191 vcpu->arch.cr3 = svm->vmcb->save.cr3; 3192 } 3193 3194 if (is_guest_mode(vcpu)) { 3195 int vmexit; 3196 3197 trace_kvm_nested_vmexit(vcpu, KVM_ISA_SVM); 3198 3199 vmexit = nested_svm_exit_special(svm); 3200 3201 if (vmexit == NESTED_EXIT_CONTINUE) 3202 vmexit = nested_svm_exit_handled(svm); 3203 3204 if (vmexit == NESTED_EXIT_DONE) 3205 return 1; 3206 } 3207 3208 if (svm->vmcb->control.exit_code == SVM_EXIT_ERR) { 3209 kvm_run->exit_reason = KVM_EXIT_FAIL_ENTRY; 3210 kvm_run->fail_entry.hardware_entry_failure_reason 3211 = svm->vmcb->control.exit_code; 3212 kvm_run->fail_entry.cpu = vcpu->arch.last_vmentry_cpu; 3213 dump_vmcb(vcpu); 3214 return 0; 3215 } 3216 3217 if (is_external_interrupt(svm->vmcb->control.exit_int_info) && 3218 exit_code != SVM_EXIT_EXCP_BASE + PF_VECTOR && 3219 exit_code != SVM_EXIT_NPF && exit_code != SVM_EXIT_TASK_SWITCH && 3220 exit_code != SVM_EXIT_INTR && exit_code != SVM_EXIT_NMI) 3221 printk(KERN_ERR "%s: unexpected exit_int_info 0x%x " 3222 "exit_code 0x%x\n", 3223 __func__, svm->vmcb->control.exit_int_info, 3224 exit_code); 3225 3226 if (exit_fastpath != EXIT_FASTPATH_NONE) 3227 return 1; 3228 3229 return svm_invoke_exit_handler(vcpu, exit_code); 3230 } 3231 3232 static void reload_tss(struct kvm_vcpu *vcpu) 3233 { 3234 struct svm_cpu_data *sd = per_cpu(svm_data, vcpu->cpu); 3235 3236 sd->tss_desc->type = 9; /* available 32/64-bit TSS */ 3237 load_TR_desc(); 3238 } 3239 3240 static void pre_svm_run(struct kvm_vcpu *vcpu) 3241 { 3242 struct svm_cpu_data *sd = per_cpu(svm_data, vcpu->cpu); 3243 struct vcpu_svm *svm = to_svm(vcpu); 3244 3245 /* 3246 * If the previous vmrun of the vmcb occurred on a different physical 3247 * cpu, then mark the vmcb dirty and assign a new asid. Hardware's 3248 * vmcb clean bits are per logical CPU, as are KVM's asid assignments. 3249 */ 3250 if (unlikely(svm->current_vmcb->cpu != vcpu->cpu)) { 3251 svm->current_vmcb->asid_generation = 0; 3252 vmcb_mark_all_dirty(svm->vmcb); 3253 svm->current_vmcb->cpu = vcpu->cpu; 3254 } 3255 3256 if (sev_guest(vcpu->kvm)) 3257 return pre_sev_run(svm, vcpu->cpu); 3258 3259 /* FIXME: handle wraparound of asid_generation */ 3260 if (svm->current_vmcb->asid_generation != sd->asid_generation) 3261 new_asid(svm, sd); 3262 } 3263 3264 static void svm_inject_nmi(struct kvm_vcpu *vcpu) 3265 { 3266 struct vcpu_svm *svm = to_svm(vcpu); 3267 3268 svm->vmcb->control.event_inj = SVM_EVTINJ_VALID | SVM_EVTINJ_TYPE_NMI; 3269 vcpu->arch.hflags |= HF_NMI_MASK; 3270 if (!sev_es_guest(vcpu->kvm)) 3271 svm_set_intercept(svm, INTERCEPT_IRET); 3272 ++vcpu->stat.nmi_injections; 3273 } 3274 3275 static void svm_set_irq(struct kvm_vcpu *vcpu) 3276 { 3277 struct vcpu_svm *svm = to_svm(vcpu); 3278 3279 BUG_ON(!(gif_set(svm))); 3280 3281 trace_kvm_inj_virq(vcpu->arch.interrupt.nr); 3282 ++vcpu->stat.irq_injections; 3283 3284 svm->vmcb->control.event_inj = vcpu->arch.interrupt.nr | 3285 SVM_EVTINJ_VALID | SVM_EVTINJ_TYPE_INTR; 3286 } 3287 3288 static void svm_update_cr8_intercept(struct kvm_vcpu *vcpu, int tpr, int irr) 3289 { 3290 struct vcpu_svm *svm = to_svm(vcpu); 3291 3292 /* 3293 * SEV-ES guests must always keep the CR intercepts cleared. CR 3294 * tracking is done using the CR write traps. 3295 */ 3296 if (sev_es_guest(vcpu->kvm)) 3297 return; 3298 3299 if (nested_svm_virtualize_tpr(vcpu)) 3300 return; 3301 3302 svm_clr_intercept(svm, INTERCEPT_CR8_WRITE); 3303 3304 if (irr == -1) 3305 return; 3306 3307 if (tpr >= irr) 3308 svm_set_intercept(svm, INTERCEPT_CR8_WRITE); 3309 } 3310 3311 bool svm_nmi_blocked(struct kvm_vcpu *vcpu) 3312 { 3313 struct vcpu_svm *svm = to_svm(vcpu); 3314 struct vmcb *vmcb = svm->vmcb; 3315 bool ret; 3316 3317 if (!gif_set(svm)) 3318 return true; 3319 3320 if (is_guest_mode(vcpu) && nested_exit_on_nmi(svm)) 3321 return false; 3322 3323 ret = (vmcb->control.int_state & SVM_INTERRUPT_SHADOW_MASK) || 3324 (vcpu->arch.hflags & HF_NMI_MASK); 3325 3326 return ret; 3327 } 3328 3329 static int svm_nmi_allowed(struct kvm_vcpu *vcpu, bool for_injection) 3330 { 3331 struct vcpu_svm *svm = to_svm(vcpu); 3332 if (svm->nested.nested_run_pending) 3333 return -EBUSY; 3334 3335 /* An NMI must not be injected into L2 if it's supposed to VM-Exit. */ 3336 if (for_injection && is_guest_mode(vcpu) && nested_exit_on_nmi(svm)) 3337 return -EBUSY; 3338 3339 return !svm_nmi_blocked(vcpu); 3340 } 3341 3342 static bool svm_get_nmi_mask(struct kvm_vcpu *vcpu) 3343 { 3344 return !!(vcpu->arch.hflags & HF_NMI_MASK); 3345 } 3346 3347 static void svm_set_nmi_mask(struct kvm_vcpu *vcpu, bool masked) 3348 { 3349 struct vcpu_svm *svm = to_svm(vcpu); 3350 3351 if (masked) { 3352 vcpu->arch.hflags |= HF_NMI_MASK; 3353 if (!sev_es_guest(vcpu->kvm)) 3354 svm_set_intercept(svm, INTERCEPT_IRET); 3355 } else { 3356 vcpu->arch.hflags &= ~HF_NMI_MASK; 3357 if (!sev_es_guest(vcpu->kvm)) 3358 svm_clr_intercept(svm, INTERCEPT_IRET); 3359 } 3360 } 3361 3362 bool svm_interrupt_blocked(struct kvm_vcpu *vcpu) 3363 { 3364 struct vcpu_svm *svm = to_svm(vcpu); 3365 struct vmcb *vmcb = svm->vmcb; 3366 3367 if (!gif_set(svm)) 3368 return true; 3369 3370 if (is_guest_mode(vcpu)) { 3371 /* As long as interrupts are being delivered... */ 3372 if ((svm->nested.ctl.int_ctl & V_INTR_MASKING_MASK) 3373 ? !(svm->vmcb01.ptr->save.rflags & X86_EFLAGS_IF) 3374 : !(kvm_get_rflags(vcpu) & X86_EFLAGS_IF)) 3375 return true; 3376 3377 /* ... vmexits aren't blocked by the interrupt shadow */ 3378 if (nested_exit_on_intr(svm)) 3379 return false; 3380 } else { 3381 if (!svm_get_if_flag(vcpu)) 3382 return true; 3383 } 3384 3385 return (vmcb->control.int_state & SVM_INTERRUPT_SHADOW_MASK); 3386 } 3387 3388 static int svm_interrupt_allowed(struct kvm_vcpu *vcpu, bool for_injection) 3389 { 3390 struct vcpu_svm *svm = to_svm(vcpu); 3391 if (svm->nested.nested_run_pending) 3392 return -EBUSY; 3393 3394 /* 3395 * An IRQ must not be injected into L2 if it's supposed to VM-Exit, 3396 * e.g. if the IRQ arrived asynchronously after checking nested events. 3397 */ 3398 if (for_injection && is_guest_mode(vcpu) && nested_exit_on_intr(svm)) 3399 return -EBUSY; 3400 3401 return !svm_interrupt_blocked(vcpu); 3402 } 3403 3404 static void svm_enable_irq_window(struct kvm_vcpu *vcpu) 3405 { 3406 struct vcpu_svm *svm = to_svm(vcpu); 3407 3408 /* 3409 * In case GIF=0 we can't rely on the CPU to tell us when GIF becomes 3410 * 1, because that's a separate STGI/VMRUN intercept. The next time we 3411 * get that intercept, this function will be called again though and 3412 * we'll get the vintr intercept. However, if the vGIF feature is 3413 * enabled, the STGI interception will not occur. Enable the irq 3414 * window under the assumption that the hardware will set the GIF. 3415 */ 3416 if (vgif_enabled(svm) || gif_set(svm)) { 3417 /* 3418 * IRQ window is not needed when AVIC is enabled, 3419 * unless we have pending ExtINT since it cannot be injected 3420 * via AVIC. In such case, we need to temporarily disable AVIC, 3421 * and fallback to injecting IRQ via V_IRQ. 3422 */ 3423 kvm_request_apicv_update(vcpu->kvm, false, APICV_INHIBIT_REASON_IRQWIN); 3424 svm_set_vintr(svm); 3425 } 3426 } 3427 3428 static void svm_enable_nmi_window(struct kvm_vcpu *vcpu) 3429 { 3430 struct vcpu_svm *svm = to_svm(vcpu); 3431 3432 if ((vcpu->arch.hflags & (HF_NMI_MASK | HF_IRET_MASK)) == HF_NMI_MASK) 3433 return; /* IRET will cause a vm exit */ 3434 3435 if (!gif_set(svm)) { 3436 if (vgif_enabled(svm)) 3437 svm_set_intercept(svm, INTERCEPT_STGI); 3438 return; /* STGI will cause a vm exit */ 3439 } 3440 3441 /* 3442 * Something prevents NMI from been injected. Single step over possible 3443 * problem (IRET or exception injection or interrupt shadow) 3444 */ 3445 svm->nmi_singlestep_guest_rflags = svm_get_rflags(vcpu); 3446 svm->nmi_singlestep = true; 3447 svm->vmcb->save.rflags |= (X86_EFLAGS_TF | X86_EFLAGS_RF); 3448 } 3449 3450 static int svm_set_tss_addr(struct kvm *kvm, unsigned int addr) 3451 { 3452 return 0; 3453 } 3454 3455 static int svm_set_identity_map_addr(struct kvm *kvm, u64 ident_addr) 3456 { 3457 return 0; 3458 } 3459 3460 void svm_flush_tlb(struct kvm_vcpu *vcpu) 3461 { 3462 struct vcpu_svm *svm = to_svm(vcpu); 3463 3464 /* 3465 * Flush only the current ASID even if the TLB flush was invoked via 3466 * kvm_flush_remote_tlbs(). Although flushing remote TLBs requires all 3467 * ASIDs to be flushed, KVM uses a single ASID for L1 and L2, and 3468 * unconditionally does a TLB flush on both nested VM-Enter and nested 3469 * VM-Exit (via kvm_mmu_reset_context()). 3470 */ 3471 if (static_cpu_has(X86_FEATURE_FLUSHBYASID)) 3472 svm->vmcb->control.tlb_ctl = TLB_CONTROL_FLUSH_ASID; 3473 else 3474 svm->current_vmcb->asid_generation--; 3475 } 3476 3477 static void svm_flush_tlb_gva(struct kvm_vcpu *vcpu, gva_t gva) 3478 { 3479 struct vcpu_svm *svm = to_svm(vcpu); 3480 3481 invlpga(gva, svm->vmcb->control.asid); 3482 } 3483 3484 static inline void sync_cr8_to_lapic(struct kvm_vcpu *vcpu) 3485 { 3486 struct vcpu_svm *svm = to_svm(vcpu); 3487 3488 if (nested_svm_virtualize_tpr(vcpu)) 3489 return; 3490 3491 if (!svm_is_intercept(svm, INTERCEPT_CR8_WRITE)) { 3492 int cr8 = svm->vmcb->control.int_ctl & V_TPR_MASK; 3493 kvm_set_cr8(vcpu, cr8); 3494 } 3495 } 3496 3497 static inline void sync_lapic_to_cr8(struct kvm_vcpu *vcpu) 3498 { 3499 struct vcpu_svm *svm = to_svm(vcpu); 3500 u64 cr8; 3501 3502 if (nested_svm_virtualize_tpr(vcpu) || 3503 kvm_vcpu_apicv_active(vcpu)) 3504 return; 3505 3506 cr8 = kvm_get_cr8(vcpu); 3507 svm->vmcb->control.int_ctl &= ~V_TPR_MASK; 3508 svm->vmcb->control.int_ctl |= cr8 & V_TPR_MASK; 3509 } 3510 3511 static void svm_complete_interrupts(struct kvm_vcpu *vcpu) 3512 { 3513 struct vcpu_svm *svm = to_svm(vcpu); 3514 u8 vector; 3515 int type; 3516 u32 exitintinfo = svm->vmcb->control.exit_int_info; 3517 unsigned int3_injected = svm->int3_injected; 3518 3519 svm->int3_injected = 0; 3520 3521 /* 3522 * If we've made progress since setting HF_IRET_MASK, we've 3523 * executed an IRET and can allow NMI injection. 3524 */ 3525 if ((vcpu->arch.hflags & HF_IRET_MASK) && 3526 (sev_es_guest(vcpu->kvm) || 3527 kvm_rip_read(vcpu) != svm->nmi_iret_rip)) { 3528 vcpu->arch.hflags &= ~(HF_NMI_MASK | HF_IRET_MASK); 3529 kvm_make_request(KVM_REQ_EVENT, vcpu); 3530 } 3531 3532 vcpu->arch.nmi_injected = false; 3533 kvm_clear_exception_queue(vcpu); 3534 kvm_clear_interrupt_queue(vcpu); 3535 3536 if (!(exitintinfo & SVM_EXITINTINFO_VALID)) 3537 return; 3538 3539 kvm_make_request(KVM_REQ_EVENT, vcpu); 3540 3541 vector = exitintinfo & SVM_EXITINTINFO_VEC_MASK; 3542 type = exitintinfo & SVM_EXITINTINFO_TYPE_MASK; 3543 3544 switch (type) { 3545 case SVM_EXITINTINFO_TYPE_NMI: 3546 vcpu->arch.nmi_injected = true; 3547 break; 3548 case SVM_EXITINTINFO_TYPE_EXEPT: 3549 /* 3550 * Never re-inject a #VC exception. 3551 */ 3552 if (vector == X86_TRAP_VC) 3553 break; 3554 3555 /* 3556 * In case of software exceptions, do not reinject the vector, 3557 * but re-execute the instruction instead. Rewind RIP first 3558 * if we emulated INT3 before. 3559 */ 3560 if (kvm_exception_is_soft(vector)) { 3561 if (vector == BP_VECTOR && int3_injected && 3562 kvm_is_linear_rip(vcpu, svm->int3_rip)) 3563 kvm_rip_write(vcpu, 3564 kvm_rip_read(vcpu) - int3_injected); 3565 break; 3566 } 3567 if (exitintinfo & SVM_EXITINTINFO_VALID_ERR) { 3568 u32 err = svm->vmcb->control.exit_int_info_err; 3569 kvm_requeue_exception_e(vcpu, vector, err); 3570 3571 } else 3572 kvm_requeue_exception(vcpu, vector); 3573 break; 3574 case SVM_EXITINTINFO_TYPE_INTR: 3575 kvm_queue_interrupt(vcpu, vector, false); 3576 break; 3577 default: 3578 break; 3579 } 3580 } 3581 3582 static void svm_cancel_injection(struct kvm_vcpu *vcpu) 3583 { 3584 struct vcpu_svm *svm = to_svm(vcpu); 3585 struct vmcb_control_area *control = &svm->vmcb->control; 3586 3587 control->exit_int_info = control->event_inj; 3588 control->exit_int_info_err = control->event_inj_err; 3589 control->event_inj = 0; 3590 svm_complete_interrupts(vcpu); 3591 } 3592 3593 static int svm_vcpu_pre_run(struct kvm_vcpu *vcpu) 3594 { 3595 return 1; 3596 } 3597 3598 static fastpath_t svm_exit_handlers_fastpath(struct kvm_vcpu *vcpu) 3599 { 3600 if (to_svm(vcpu)->vmcb->control.exit_code == SVM_EXIT_MSR && 3601 to_svm(vcpu)->vmcb->control.exit_info_1) 3602 return handle_fastpath_set_msr_irqoff(vcpu); 3603 3604 return EXIT_FASTPATH_NONE; 3605 } 3606 3607 static noinstr void svm_vcpu_enter_exit(struct kvm_vcpu *vcpu) 3608 { 3609 struct vcpu_svm *svm = to_svm(vcpu); 3610 unsigned long vmcb_pa = svm->current_vmcb->pa; 3611 3612 kvm_guest_enter_irqoff(); 3613 3614 if (sev_es_guest(vcpu->kvm)) { 3615 __svm_sev_es_vcpu_run(vmcb_pa); 3616 } else { 3617 struct svm_cpu_data *sd = per_cpu(svm_data, vcpu->cpu); 3618 3619 /* 3620 * Use a single vmcb (vmcb01 because it's always valid) for 3621 * context switching guest state via VMLOAD/VMSAVE, that way 3622 * the state doesn't need to be copied between vmcb01 and 3623 * vmcb02 when switching vmcbs for nested virtualization. 3624 */ 3625 vmload(svm->vmcb01.pa); 3626 __svm_vcpu_run(vmcb_pa, (unsigned long *)&vcpu->arch.regs); 3627 vmsave(svm->vmcb01.pa); 3628 3629 vmload(__sme_page_pa(sd->save_area)); 3630 } 3631 3632 kvm_guest_exit_irqoff(); 3633 } 3634 3635 static __no_kcsan fastpath_t svm_vcpu_run(struct kvm_vcpu *vcpu) 3636 { 3637 struct vcpu_svm *svm = to_svm(vcpu); 3638 3639 trace_kvm_entry(vcpu); 3640 3641 svm->vmcb->save.rax = vcpu->arch.regs[VCPU_REGS_RAX]; 3642 svm->vmcb->save.rsp = vcpu->arch.regs[VCPU_REGS_RSP]; 3643 svm->vmcb->save.rip = vcpu->arch.regs[VCPU_REGS_RIP]; 3644 3645 /* 3646 * Disable singlestep if we're injecting an interrupt/exception. 3647 * We don't want our modified rflags to be pushed on the stack where 3648 * we might not be able to easily reset them if we disabled NMI 3649 * singlestep later. 3650 */ 3651 if (svm->nmi_singlestep && svm->vmcb->control.event_inj) { 3652 /* 3653 * Event injection happens before external interrupts cause a 3654 * vmexit and interrupts are disabled here, so smp_send_reschedule 3655 * is enough to force an immediate vmexit. 3656 */ 3657 disable_nmi_singlestep(svm); 3658 smp_send_reschedule(vcpu->cpu); 3659 } 3660 3661 pre_svm_run(vcpu); 3662 3663 sync_lapic_to_cr8(vcpu); 3664 3665 if (unlikely(svm->asid != svm->vmcb->control.asid)) { 3666 svm->vmcb->control.asid = svm->asid; 3667 vmcb_mark_dirty(svm->vmcb, VMCB_ASID); 3668 } 3669 svm->vmcb->save.cr2 = vcpu->arch.cr2; 3670 3671 svm_hv_update_vp_id(svm->vmcb, vcpu); 3672 3673 /* 3674 * Run with all-zero DR6 unless needed, so that we can get the exact cause 3675 * of a #DB. 3676 */ 3677 if (unlikely(vcpu->arch.switch_db_regs & KVM_DEBUGREG_WONT_EXIT)) 3678 svm_set_dr6(svm, vcpu->arch.dr6); 3679 else 3680 svm_set_dr6(svm, DR6_ACTIVE_LOW); 3681 3682 clgi(); 3683 kvm_load_guest_xsave_state(vcpu); 3684 3685 kvm_wait_lapic_expire(vcpu); 3686 3687 /* 3688 * If this vCPU has touched SPEC_CTRL, restore the guest's value if 3689 * it's non-zero. Since vmentry is serialising on affected CPUs, there 3690 * is no need to worry about the conditional branch over the wrmsr 3691 * being speculatively taken. 3692 */ 3693 if (!static_cpu_has(X86_FEATURE_V_SPEC_CTRL)) 3694 x86_spec_ctrl_set_guest(svm->spec_ctrl, svm->virt_spec_ctrl); 3695 3696 svm_vcpu_enter_exit(vcpu); 3697 3698 /* 3699 * We do not use IBRS in the kernel. If this vCPU has used the 3700 * SPEC_CTRL MSR it may have left it on; save the value and 3701 * turn it off. This is much more efficient than blindly adding 3702 * it to the atomic save/restore list. Especially as the former 3703 * (Saving guest MSRs on vmexit) doesn't even exist in KVM. 3704 * 3705 * For non-nested case: 3706 * If the L01 MSR bitmap does not intercept the MSR, then we need to 3707 * save it. 3708 * 3709 * For nested case: 3710 * If the L02 MSR bitmap does not intercept the MSR, then we need to 3711 * save it. 3712 */ 3713 if (!static_cpu_has(X86_FEATURE_V_SPEC_CTRL) && 3714 unlikely(!msr_write_intercepted(vcpu, MSR_IA32_SPEC_CTRL))) 3715 svm->spec_ctrl = native_read_msr(MSR_IA32_SPEC_CTRL); 3716 3717 if (!sev_es_guest(vcpu->kvm)) 3718 reload_tss(vcpu); 3719 3720 if (!static_cpu_has(X86_FEATURE_V_SPEC_CTRL)) 3721 x86_spec_ctrl_restore_host(svm->spec_ctrl, svm->virt_spec_ctrl); 3722 3723 if (!sev_es_guest(vcpu->kvm)) { 3724 vcpu->arch.cr2 = svm->vmcb->save.cr2; 3725 vcpu->arch.regs[VCPU_REGS_RAX] = svm->vmcb->save.rax; 3726 vcpu->arch.regs[VCPU_REGS_RSP] = svm->vmcb->save.rsp; 3727 vcpu->arch.regs[VCPU_REGS_RIP] = svm->vmcb->save.rip; 3728 } 3729 vcpu->arch.regs_dirty = 0; 3730 3731 if (unlikely(svm->vmcb->control.exit_code == SVM_EXIT_NMI)) 3732 kvm_before_interrupt(vcpu, KVM_HANDLING_NMI); 3733 3734 kvm_load_host_xsave_state(vcpu); 3735 stgi(); 3736 3737 /* Any pending NMI will happen here */ 3738 3739 if (unlikely(svm->vmcb->control.exit_code == SVM_EXIT_NMI)) 3740 kvm_after_interrupt(vcpu); 3741 3742 sync_cr8_to_lapic(vcpu); 3743 3744 svm->next_rip = 0; 3745 if (is_guest_mode(vcpu)) { 3746 nested_sync_control_from_vmcb02(svm); 3747 3748 /* Track VMRUNs that have made past consistency checking */ 3749 if (svm->nested.nested_run_pending && 3750 svm->vmcb->control.exit_code != SVM_EXIT_ERR) 3751 ++vcpu->stat.nested_run; 3752 3753 svm->nested.nested_run_pending = 0; 3754 } 3755 3756 svm->vmcb->control.tlb_ctl = TLB_CONTROL_DO_NOTHING; 3757 vmcb_mark_all_clean(svm->vmcb); 3758 3759 /* if exit due to PF check for async PF */ 3760 if (svm->vmcb->control.exit_code == SVM_EXIT_EXCP_BASE + PF_VECTOR) 3761 vcpu->arch.apf.host_apf_flags = 3762 kvm_read_and_reset_apf_flags(); 3763 3764 vcpu->arch.regs_avail &= ~SVM_REGS_LAZY_LOAD_SET; 3765 3766 /* 3767 * We need to handle MC intercepts here before the vcpu has a chance to 3768 * change the physical cpu 3769 */ 3770 if (unlikely(svm->vmcb->control.exit_code == 3771 SVM_EXIT_EXCP_BASE + MC_VECTOR)) 3772 svm_handle_mce(vcpu); 3773 3774 svm_complete_interrupts(vcpu); 3775 3776 if (is_guest_mode(vcpu)) 3777 return EXIT_FASTPATH_NONE; 3778 3779 return svm_exit_handlers_fastpath(vcpu); 3780 } 3781 3782 static void svm_load_mmu_pgd(struct kvm_vcpu *vcpu, hpa_t root_hpa, 3783 int root_level) 3784 { 3785 struct vcpu_svm *svm = to_svm(vcpu); 3786 unsigned long cr3; 3787 3788 if (npt_enabled) { 3789 svm->vmcb->control.nested_cr3 = __sme_set(root_hpa); 3790 vmcb_mark_dirty(svm->vmcb, VMCB_NPT); 3791 3792 hv_track_root_tdp(vcpu, root_hpa); 3793 3794 cr3 = vcpu->arch.cr3; 3795 } else if (vcpu->arch.mmu->shadow_root_level >= PT64_ROOT_4LEVEL) { 3796 cr3 = __sme_set(root_hpa) | kvm_get_active_pcid(vcpu); 3797 } else { 3798 /* PCID in the guest should be impossible with a 32-bit MMU. */ 3799 WARN_ON_ONCE(kvm_get_active_pcid(vcpu)); 3800 cr3 = root_hpa; 3801 } 3802 3803 svm->vmcb->save.cr3 = cr3; 3804 vmcb_mark_dirty(svm->vmcb, VMCB_CR); 3805 } 3806 3807 static int is_disabled(void) 3808 { 3809 u64 vm_cr; 3810 3811 rdmsrl(MSR_VM_CR, vm_cr); 3812 if (vm_cr & (1 << SVM_VM_CR_SVM_DISABLE)) 3813 return 1; 3814 3815 return 0; 3816 } 3817 3818 static void 3819 svm_patch_hypercall(struct kvm_vcpu *vcpu, unsigned char *hypercall) 3820 { 3821 /* 3822 * Patch in the VMMCALL instruction: 3823 */ 3824 hypercall[0] = 0x0f; 3825 hypercall[1] = 0x01; 3826 hypercall[2] = 0xd9; 3827 } 3828 3829 static int __init svm_check_processor_compat(void) 3830 { 3831 return 0; 3832 } 3833 3834 static bool svm_cpu_has_accelerated_tpr(void) 3835 { 3836 return false; 3837 } 3838 3839 /* 3840 * The kvm parameter can be NULL (module initialization, or invocation before 3841 * VM creation). Be sure to check the kvm parameter before using it. 3842 */ 3843 static bool svm_has_emulated_msr(struct kvm *kvm, u32 index) 3844 { 3845 switch (index) { 3846 case MSR_IA32_MCG_EXT_CTL: 3847 case MSR_IA32_VMX_BASIC ... MSR_IA32_VMX_VMFUNC: 3848 return false; 3849 case MSR_IA32_SMBASE: 3850 /* SEV-ES guests do not support SMM, so report false */ 3851 if (kvm && sev_es_guest(kvm)) 3852 return false; 3853 break; 3854 default: 3855 break; 3856 } 3857 3858 return true; 3859 } 3860 3861 static u64 svm_get_mt_mask(struct kvm_vcpu *vcpu, gfn_t gfn, bool is_mmio) 3862 { 3863 return 0; 3864 } 3865 3866 static void svm_vcpu_after_set_cpuid(struct kvm_vcpu *vcpu) 3867 { 3868 struct vcpu_svm *svm = to_svm(vcpu); 3869 struct kvm_cpuid_entry2 *best; 3870 3871 vcpu->arch.xsaves_enabled = guest_cpuid_has(vcpu, X86_FEATURE_XSAVE) && 3872 boot_cpu_has(X86_FEATURE_XSAVE) && 3873 boot_cpu_has(X86_FEATURE_XSAVES); 3874 3875 /* Update nrips enabled cache */ 3876 svm->nrips_enabled = kvm_cpu_cap_has(X86_FEATURE_NRIPS) && 3877 guest_cpuid_has(vcpu, X86_FEATURE_NRIPS); 3878 3879 svm->tsc_scaling_enabled = tsc_scaling && guest_cpuid_has(vcpu, X86_FEATURE_TSCRATEMSR); 3880 3881 svm_recalc_instruction_intercepts(vcpu, svm); 3882 3883 /* For sev guests, the memory encryption bit is not reserved in CR3. */ 3884 if (sev_guest(vcpu->kvm)) { 3885 best = kvm_find_cpuid_entry(vcpu, 0x8000001F, 0); 3886 if (best) 3887 vcpu->arch.reserved_gpa_bits &= ~(1UL << (best->ebx & 0x3f)); 3888 } 3889 3890 if (kvm_vcpu_apicv_active(vcpu)) { 3891 /* 3892 * AVIC does not work with an x2APIC mode guest. If the X2APIC feature 3893 * is exposed to the guest, disable AVIC. 3894 */ 3895 if (guest_cpuid_has(vcpu, X86_FEATURE_X2APIC)) 3896 kvm_request_apicv_update(vcpu->kvm, false, 3897 APICV_INHIBIT_REASON_X2APIC); 3898 3899 /* 3900 * Currently, AVIC does not work with nested virtualization. 3901 * So, we disable AVIC when cpuid for SVM is set in the L1 guest. 3902 */ 3903 if (nested && guest_cpuid_has(vcpu, X86_FEATURE_SVM)) 3904 kvm_request_apicv_update(vcpu->kvm, false, 3905 APICV_INHIBIT_REASON_NESTED); 3906 } 3907 init_vmcb_after_set_cpuid(vcpu); 3908 } 3909 3910 static bool svm_has_wbinvd_exit(void) 3911 { 3912 return true; 3913 } 3914 3915 #define PRE_EX(exit) { .exit_code = (exit), \ 3916 .stage = X86_ICPT_PRE_EXCEPT, } 3917 #define POST_EX(exit) { .exit_code = (exit), \ 3918 .stage = X86_ICPT_POST_EXCEPT, } 3919 #define POST_MEM(exit) { .exit_code = (exit), \ 3920 .stage = X86_ICPT_POST_MEMACCESS, } 3921 3922 static const struct __x86_intercept { 3923 u32 exit_code; 3924 enum x86_intercept_stage stage; 3925 } x86_intercept_map[] = { 3926 [x86_intercept_cr_read] = POST_EX(SVM_EXIT_READ_CR0), 3927 [x86_intercept_cr_write] = POST_EX(SVM_EXIT_WRITE_CR0), 3928 [x86_intercept_clts] = POST_EX(SVM_EXIT_WRITE_CR0), 3929 [x86_intercept_lmsw] = POST_EX(SVM_EXIT_WRITE_CR0), 3930 [x86_intercept_smsw] = POST_EX(SVM_EXIT_READ_CR0), 3931 [x86_intercept_dr_read] = POST_EX(SVM_EXIT_READ_DR0), 3932 [x86_intercept_dr_write] = POST_EX(SVM_EXIT_WRITE_DR0), 3933 [x86_intercept_sldt] = POST_EX(SVM_EXIT_LDTR_READ), 3934 [x86_intercept_str] = POST_EX(SVM_EXIT_TR_READ), 3935 [x86_intercept_lldt] = POST_EX(SVM_EXIT_LDTR_WRITE), 3936 [x86_intercept_ltr] = POST_EX(SVM_EXIT_TR_WRITE), 3937 [x86_intercept_sgdt] = POST_EX(SVM_EXIT_GDTR_READ), 3938 [x86_intercept_sidt] = POST_EX(SVM_EXIT_IDTR_READ), 3939 [x86_intercept_lgdt] = POST_EX(SVM_EXIT_GDTR_WRITE), 3940 [x86_intercept_lidt] = POST_EX(SVM_EXIT_IDTR_WRITE), 3941 [x86_intercept_vmrun] = POST_EX(SVM_EXIT_VMRUN), 3942 [x86_intercept_vmmcall] = POST_EX(SVM_EXIT_VMMCALL), 3943 [x86_intercept_vmload] = POST_EX(SVM_EXIT_VMLOAD), 3944 [x86_intercept_vmsave] = POST_EX(SVM_EXIT_VMSAVE), 3945 [x86_intercept_stgi] = POST_EX(SVM_EXIT_STGI), 3946 [x86_intercept_clgi] = POST_EX(SVM_EXIT_CLGI), 3947 [x86_intercept_skinit] = POST_EX(SVM_EXIT_SKINIT), 3948 [x86_intercept_invlpga] = POST_EX(SVM_EXIT_INVLPGA), 3949 [x86_intercept_rdtscp] = POST_EX(SVM_EXIT_RDTSCP), 3950 [x86_intercept_monitor] = POST_MEM(SVM_EXIT_MONITOR), 3951 [x86_intercept_mwait] = POST_EX(SVM_EXIT_MWAIT), 3952 [x86_intercept_invlpg] = POST_EX(SVM_EXIT_INVLPG), 3953 [x86_intercept_invd] = POST_EX(SVM_EXIT_INVD), 3954 [x86_intercept_wbinvd] = POST_EX(SVM_EXIT_WBINVD), 3955 [x86_intercept_wrmsr] = POST_EX(SVM_EXIT_MSR), 3956 [x86_intercept_rdtsc] = POST_EX(SVM_EXIT_RDTSC), 3957 [x86_intercept_rdmsr] = POST_EX(SVM_EXIT_MSR), 3958 [x86_intercept_rdpmc] = POST_EX(SVM_EXIT_RDPMC), 3959 [x86_intercept_cpuid] = PRE_EX(SVM_EXIT_CPUID), 3960 [x86_intercept_rsm] = PRE_EX(SVM_EXIT_RSM), 3961 [x86_intercept_pause] = PRE_EX(SVM_EXIT_PAUSE), 3962 [x86_intercept_pushf] = PRE_EX(SVM_EXIT_PUSHF), 3963 [x86_intercept_popf] = PRE_EX(SVM_EXIT_POPF), 3964 [x86_intercept_intn] = PRE_EX(SVM_EXIT_SWINT), 3965 [x86_intercept_iret] = PRE_EX(SVM_EXIT_IRET), 3966 [x86_intercept_icebp] = PRE_EX(SVM_EXIT_ICEBP), 3967 [x86_intercept_hlt] = POST_EX(SVM_EXIT_HLT), 3968 [x86_intercept_in] = POST_EX(SVM_EXIT_IOIO), 3969 [x86_intercept_ins] = POST_EX(SVM_EXIT_IOIO), 3970 [x86_intercept_out] = POST_EX(SVM_EXIT_IOIO), 3971 [x86_intercept_outs] = POST_EX(SVM_EXIT_IOIO), 3972 [x86_intercept_xsetbv] = PRE_EX(SVM_EXIT_XSETBV), 3973 }; 3974 3975 #undef PRE_EX 3976 #undef POST_EX 3977 #undef POST_MEM 3978 3979 static int svm_check_intercept(struct kvm_vcpu *vcpu, 3980 struct x86_instruction_info *info, 3981 enum x86_intercept_stage stage, 3982 struct x86_exception *exception) 3983 { 3984 struct vcpu_svm *svm = to_svm(vcpu); 3985 int vmexit, ret = X86EMUL_CONTINUE; 3986 struct __x86_intercept icpt_info; 3987 struct vmcb *vmcb = svm->vmcb; 3988 3989 if (info->intercept >= ARRAY_SIZE(x86_intercept_map)) 3990 goto out; 3991 3992 icpt_info = x86_intercept_map[info->intercept]; 3993 3994 if (stage != icpt_info.stage) 3995 goto out; 3996 3997 switch (icpt_info.exit_code) { 3998 case SVM_EXIT_READ_CR0: 3999 if (info->intercept == x86_intercept_cr_read) 4000 icpt_info.exit_code += info->modrm_reg; 4001 break; 4002 case SVM_EXIT_WRITE_CR0: { 4003 unsigned long cr0, val; 4004 4005 if (info->intercept == x86_intercept_cr_write) 4006 icpt_info.exit_code += info->modrm_reg; 4007 4008 if (icpt_info.exit_code != SVM_EXIT_WRITE_CR0 || 4009 info->intercept == x86_intercept_clts) 4010 break; 4011 4012 if (!(vmcb12_is_intercept(&svm->nested.ctl, 4013 INTERCEPT_SELECTIVE_CR0))) 4014 break; 4015 4016 cr0 = vcpu->arch.cr0 & ~SVM_CR0_SELECTIVE_MASK; 4017 val = info->src_val & ~SVM_CR0_SELECTIVE_MASK; 4018 4019 if (info->intercept == x86_intercept_lmsw) { 4020 cr0 &= 0xfUL; 4021 val &= 0xfUL; 4022 /* lmsw can't clear PE - catch this here */ 4023 if (cr0 & X86_CR0_PE) 4024 val |= X86_CR0_PE; 4025 } 4026 4027 if (cr0 ^ val) 4028 icpt_info.exit_code = SVM_EXIT_CR0_SEL_WRITE; 4029 4030 break; 4031 } 4032 case SVM_EXIT_READ_DR0: 4033 case SVM_EXIT_WRITE_DR0: 4034 icpt_info.exit_code += info->modrm_reg; 4035 break; 4036 case SVM_EXIT_MSR: 4037 if (info->intercept == x86_intercept_wrmsr) 4038 vmcb->control.exit_info_1 = 1; 4039 else 4040 vmcb->control.exit_info_1 = 0; 4041 break; 4042 case SVM_EXIT_PAUSE: 4043 /* 4044 * We get this for NOP only, but pause 4045 * is rep not, check this here 4046 */ 4047 if (info->rep_prefix != REPE_PREFIX) 4048 goto out; 4049 break; 4050 case SVM_EXIT_IOIO: { 4051 u64 exit_info; 4052 u32 bytes; 4053 4054 if (info->intercept == x86_intercept_in || 4055 info->intercept == x86_intercept_ins) { 4056 exit_info = ((info->src_val & 0xffff) << 16) | 4057 SVM_IOIO_TYPE_MASK; 4058 bytes = info->dst_bytes; 4059 } else { 4060 exit_info = (info->dst_val & 0xffff) << 16; 4061 bytes = info->src_bytes; 4062 } 4063 4064 if (info->intercept == x86_intercept_outs || 4065 info->intercept == x86_intercept_ins) 4066 exit_info |= SVM_IOIO_STR_MASK; 4067 4068 if (info->rep_prefix) 4069 exit_info |= SVM_IOIO_REP_MASK; 4070 4071 bytes = min(bytes, 4u); 4072 4073 exit_info |= bytes << SVM_IOIO_SIZE_SHIFT; 4074 4075 exit_info |= (u32)info->ad_bytes << (SVM_IOIO_ASIZE_SHIFT - 1); 4076 4077 vmcb->control.exit_info_1 = exit_info; 4078 vmcb->control.exit_info_2 = info->next_rip; 4079 4080 break; 4081 } 4082 default: 4083 break; 4084 } 4085 4086 /* TODO: Advertise NRIPS to guest hypervisor unconditionally */ 4087 if (static_cpu_has(X86_FEATURE_NRIPS)) 4088 vmcb->control.next_rip = info->next_rip; 4089 vmcb->control.exit_code = icpt_info.exit_code; 4090 vmexit = nested_svm_exit_handled(svm); 4091 4092 ret = (vmexit == NESTED_EXIT_DONE) ? X86EMUL_INTERCEPTED 4093 : X86EMUL_CONTINUE; 4094 4095 out: 4096 return ret; 4097 } 4098 4099 static void svm_handle_exit_irqoff(struct kvm_vcpu *vcpu) 4100 { 4101 } 4102 4103 static void svm_sched_in(struct kvm_vcpu *vcpu, int cpu) 4104 { 4105 if (!kvm_pause_in_guest(vcpu->kvm)) 4106 shrink_ple_window(vcpu); 4107 } 4108 4109 static void svm_setup_mce(struct kvm_vcpu *vcpu) 4110 { 4111 /* [63:9] are reserved. */ 4112 vcpu->arch.mcg_cap &= 0x1ff; 4113 } 4114 4115 bool svm_smi_blocked(struct kvm_vcpu *vcpu) 4116 { 4117 struct vcpu_svm *svm = to_svm(vcpu); 4118 4119 /* Per APM Vol.2 15.22.2 "Response to SMI" */ 4120 if (!gif_set(svm)) 4121 return true; 4122 4123 return is_smm(vcpu); 4124 } 4125 4126 static int svm_smi_allowed(struct kvm_vcpu *vcpu, bool for_injection) 4127 { 4128 struct vcpu_svm *svm = to_svm(vcpu); 4129 if (svm->nested.nested_run_pending) 4130 return -EBUSY; 4131 4132 /* An SMI must not be injected into L2 if it's supposed to VM-Exit. */ 4133 if (for_injection && is_guest_mode(vcpu) && nested_exit_on_smi(svm)) 4134 return -EBUSY; 4135 4136 return !svm_smi_blocked(vcpu); 4137 } 4138 4139 static int svm_enter_smm(struct kvm_vcpu *vcpu, char *smstate) 4140 { 4141 struct vcpu_svm *svm = to_svm(vcpu); 4142 struct kvm_host_map map_save; 4143 int ret; 4144 4145 if (!is_guest_mode(vcpu)) 4146 return 0; 4147 4148 /* FED8h - SVM Guest */ 4149 put_smstate(u64, smstate, 0x7ed8, 1); 4150 /* FEE0h - SVM Guest VMCB Physical Address */ 4151 put_smstate(u64, smstate, 0x7ee0, svm->nested.vmcb12_gpa); 4152 4153 svm->vmcb->save.rax = vcpu->arch.regs[VCPU_REGS_RAX]; 4154 svm->vmcb->save.rsp = vcpu->arch.regs[VCPU_REGS_RSP]; 4155 svm->vmcb->save.rip = vcpu->arch.regs[VCPU_REGS_RIP]; 4156 4157 ret = nested_svm_vmexit(svm); 4158 if (ret) 4159 return ret; 4160 4161 /* 4162 * KVM uses VMCB01 to store L1 host state while L2 runs but 4163 * VMCB01 is going to be used during SMM and thus the state will 4164 * be lost. Temporary save non-VMLOAD/VMSAVE state to the host save 4165 * area pointed to by MSR_VM_HSAVE_PA. APM guarantees that the 4166 * format of the area is identical to guest save area offsetted 4167 * by 0x400 (matches the offset of 'struct vmcb_save_area' 4168 * within 'struct vmcb'). Note: HSAVE area may also be used by 4169 * L1 hypervisor to save additional host context (e.g. KVM does 4170 * that, see svm_prepare_guest_switch()) which must be 4171 * preserved. 4172 */ 4173 if (kvm_vcpu_map(vcpu, gpa_to_gfn(svm->nested.hsave_msr), 4174 &map_save) == -EINVAL) 4175 return 1; 4176 4177 BUILD_BUG_ON(offsetof(struct vmcb, save) != 0x400); 4178 4179 svm_copy_vmrun_state(map_save.hva + 0x400, 4180 &svm->vmcb01.ptr->save); 4181 4182 kvm_vcpu_unmap(vcpu, &map_save, true); 4183 return 0; 4184 } 4185 4186 static int svm_leave_smm(struct kvm_vcpu *vcpu, const char *smstate) 4187 { 4188 struct vcpu_svm *svm = to_svm(vcpu); 4189 struct kvm_host_map map, map_save; 4190 u64 saved_efer, vmcb12_gpa; 4191 struct vmcb *vmcb12; 4192 int ret; 4193 4194 if (!guest_cpuid_has(vcpu, X86_FEATURE_LM)) 4195 return 0; 4196 4197 /* Non-zero if SMI arrived while vCPU was in guest mode. */ 4198 if (!GET_SMSTATE(u64, smstate, 0x7ed8)) 4199 return 0; 4200 4201 if (!guest_cpuid_has(vcpu, X86_FEATURE_SVM)) 4202 return 1; 4203 4204 saved_efer = GET_SMSTATE(u64, smstate, 0x7ed0); 4205 if (!(saved_efer & EFER_SVME)) 4206 return 1; 4207 4208 vmcb12_gpa = GET_SMSTATE(u64, smstate, 0x7ee0); 4209 if (kvm_vcpu_map(vcpu, gpa_to_gfn(vmcb12_gpa), &map) == -EINVAL) 4210 return 1; 4211 4212 ret = 1; 4213 if (kvm_vcpu_map(vcpu, gpa_to_gfn(svm->nested.hsave_msr), &map_save) == -EINVAL) 4214 goto unmap_map; 4215 4216 if (svm_allocate_nested(svm)) 4217 goto unmap_save; 4218 4219 /* 4220 * Restore L1 host state from L1 HSAVE area as VMCB01 was 4221 * used during SMM (see svm_enter_smm()) 4222 */ 4223 4224 svm_copy_vmrun_state(&svm->vmcb01.ptr->save, map_save.hva + 0x400); 4225 4226 /* 4227 * Enter the nested guest now 4228 */ 4229 4230 vmcb12 = map.hva; 4231 nested_copy_vmcb_control_to_cache(svm, &vmcb12->control); 4232 nested_copy_vmcb_save_to_cache(svm, &vmcb12->save); 4233 ret = enter_svm_guest_mode(vcpu, vmcb12_gpa, vmcb12, false); 4234 4235 unmap_save: 4236 kvm_vcpu_unmap(vcpu, &map_save, true); 4237 unmap_map: 4238 kvm_vcpu_unmap(vcpu, &map, true); 4239 return ret; 4240 } 4241 4242 static void svm_enable_smi_window(struct kvm_vcpu *vcpu) 4243 { 4244 struct vcpu_svm *svm = to_svm(vcpu); 4245 4246 if (!gif_set(svm)) { 4247 if (vgif_enabled(svm)) 4248 svm_set_intercept(svm, INTERCEPT_STGI); 4249 /* STGI will cause a vm exit */ 4250 } else { 4251 /* We must be in SMM; RSM will cause a vmexit anyway. */ 4252 } 4253 } 4254 4255 static bool svm_can_emulate_instruction(struct kvm_vcpu *vcpu, void *insn, int insn_len) 4256 { 4257 bool smep, smap, is_user; 4258 unsigned long cr4; 4259 4260 /* 4261 * When the guest is an SEV-ES guest, emulation is not possible. 4262 */ 4263 if (sev_es_guest(vcpu->kvm)) 4264 return false; 4265 4266 /* 4267 * Detect and workaround Errata 1096 Fam_17h_00_0Fh. 4268 * 4269 * Errata: 4270 * When CPU raise #NPF on guest data access and vCPU CR4.SMAP=1, it is 4271 * possible that CPU microcode implementing DecodeAssist will fail 4272 * to read bytes of instruction which caused #NPF. In this case, 4273 * GuestIntrBytes field of the VMCB on a VMEXIT will incorrectly 4274 * return 0 instead of the correct guest instruction bytes. 4275 * 4276 * This happens because CPU microcode reading instruction bytes 4277 * uses a special opcode which attempts to read data using CPL=0 4278 * privileges. The microcode reads CS:RIP and if it hits a SMAP 4279 * fault, it gives up and returns no instruction bytes. 4280 * 4281 * Detection: 4282 * We reach here in case CPU supports DecodeAssist, raised #NPF and 4283 * returned 0 in GuestIntrBytes field of the VMCB. 4284 * First, errata can only be triggered in case vCPU CR4.SMAP=1. 4285 * Second, if vCPU CR4.SMEP=1, errata could only be triggered 4286 * in case vCPU CPL==3 (Because otherwise guest would have triggered 4287 * a SMEP fault instead of #NPF). 4288 * Otherwise, vCPU CR4.SMEP=0, errata could be triggered by any vCPU CPL. 4289 * As most guests enable SMAP if they have also enabled SMEP, use above 4290 * logic in order to attempt minimize false-positive of detecting errata 4291 * while still preserving all cases semantic correctness. 4292 * 4293 * Workaround: 4294 * To determine what instruction the guest was executing, the hypervisor 4295 * will have to decode the instruction at the instruction pointer. 4296 * 4297 * In non SEV guest, hypervisor will be able to read the guest 4298 * memory to decode the instruction pointer when insn_len is zero 4299 * so we return true to indicate that decoding is possible. 4300 * 4301 * But in the SEV guest, the guest memory is encrypted with the 4302 * guest specific key and hypervisor will not be able to decode the 4303 * instruction pointer so we will not able to workaround it. Lets 4304 * print the error and request to kill the guest. 4305 */ 4306 if (likely(!insn || insn_len)) 4307 return true; 4308 4309 /* 4310 * If RIP is invalid, go ahead with emulation which will cause an 4311 * internal error exit. 4312 */ 4313 if (!kvm_vcpu_gfn_to_memslot(vcpu, kvm_rip_read(vcpu) >> PAGE_SHIFT)) 4314 return true; 4315 4316 cr4 = kvm_read_cr4(vcpu); 4317 smep = cr4 & X86_CR4_SMEP; 4318 smap = cr4 & X86_CR4_SMAP; 4319 is_user = svm_get_cpl(vcpu) == 3; 4320 if (smap && (!smep || is_user)) { 4321 if (!sev_guest(vcpu->kvm)) 4322 return true; 4323 4324 pr_err_ratelimited("KVM: SEV Guest triggered AMD Erratum 1096\n"); 4325 kvm_make_request(KVM_REQ_TRIPLE_FAULT, vcpu); 4326 } 4327 4328 return false; 4329 } 4330 4331 static bool svm_apic_init_signal_blocked(struct kvm_vcpu *vcpu) 4332 { 4333 struct vcpu_svm *svm = to_svm(vcpu); 4334 4335 /* 4336 * TODO: Last condition latch INIT signals on vCPU when 4337 * vCPU is in guest-mode and vmcb12 defines intercept on INIT. 4338 * To properly emulate the INIT intercept, 4339 * svm_check_nested_events() should call nested_svm_vmexit() 4340 * if an INIT signal is pending. 4341 */ 4342 return !gif_set(svm) || 4343 (vmcb_is_intercept(&svm->vmcb->control, INTERCEPT_INIT)); 4344 } 4345 4346 static void svm_vcpu_deliver_sipi_vector(struct kvm_vcpu *vcpu, u8 vector) 4347 { 4348 if (!sev_es_guest(vcpu->kvm)) 4349 return kvm_vcpu_deliver_sipi_vector(vcpu, vector); 4350 4351 sev_vcpu_deliver_sipi_vector(vcpu, vector); 4352 } 4353 4354 static void svm_vm_destroy(struct kvm *kvm) 4355 { 4356 avic_vm_destroy(kvm); 4357 sev_vm_destroy(kvm); 4358 } 4359 4360 static int svm_vm_init(struct kvm *kvm) 4361 { 4362 if (!pause_filter_count || !pause_filter_thresh) 4363 kvm->arch.pause_in_guest = true; 4364 4365 if (enable_apicv) { 4366 int ret = avic_vm_init(kvm); 4367 if (ret) 4368 return ret; 4369 } 4370 4371 return 0; 4372 } 4373 4374 static struct kvm_x86_ops svm_x86_ops __initdata = { 4375 .name = "kvm_amd", 4376 4377 .hardware_unsetup = svm_hardware_teardown, 4378 .hardware_enable = svm_hardware_enable, 4379 .hardware_disable = svm_hardware_disable, 4380 .cpu_has_accelerated_tpr = svm_cpu_has_accelerated_tpr, 4381 .has_emulated_msr = svm_has_emulated_msr, 4382 4383 .vcpu_create = svm_create_vcpu, 4384 .vcpu_free = svm_free_vcpu, 4385 .vcpu_reset = svm_vcpu_reset, 4386 4387 .vm_size = sizeof(struct kvm_svm), 4388 .vm_init = svm_vm_init, 4389 .vm_destroy = svm_vm_destroy, 4390 4391 .prepare_guest_switch = svm_prepare_guest_switch, 4392 .vcpu_load = svm_vcpu_load, 4393 .vcpu_put = svm_vcpu_put, 4394 .vcpu_blocking = avic_vcpu_blocking, 4395 .vcpu_unblocking = avic_vcpu_unblocking, 4396 4397 .update_exception_bitmap = svm_update_exception_bitmap, 4398 .get_msr_feature = svm_get_msr_feature, 4399 .get_msr = svm_get_msr, 4400 .set_msr = svm_set_msr, 4401 .get_segment_base = svm_get_segment_base, 4402 .get_segment = svm_get_segment, 4403 .set_segment = svm_set_segment, 4404 .get_cpl = svm_get_cpl, 4405 .get_cs_db_l_bits = kvm_get_cs_db_l_bits, 4406 .set_cr0 = svm_set_cr0, 4407 .post_set_cr3 = svm_post_set_cr3, 4408 .is_valid_cr4 = svm_is_valid_cr4, 4409 .set_cr4 = svm_set_cr4, 4410 .set_efer = svm_set_efer, 4411 .get_idt = svm_get_idt, 4412 .set_idt = svm_set_idt, 4413 .get_gdt = svm_get_gdt, 4414 .set_gdt = svm_set_gdt, 4415 .set_dr7 = svm_set_dr7, 4416 .sync_dirty_debug_regs = svm_sync_dirty_debug_regs, 4417 .cache_reg = svm_cache_reg, 4418 .get_rflags = svm_get_rflags, 4419 .set_rflags = svm_set_rflags, 4420 .get_if_flag = svm_get_if_flag, 4421 4422 .tlb_flush_all = svm_flush_tlb, 4423 .tlb_flush_current = svm_flush_tlb, 4424 .tlb_flush_gva = svm_flush_tlb_gva, 4425 .tlb_flush_guest = svm_flush_tlb, 4426 4427 .vcpu_pre_run = svm_vcpu_pre_run, 4428 .run = svm_vcpu_run, 4429 .handle_exit = handle_exit, 4430 .skip_emulated_instruction = skip_emulated_instruction, 4431 .update_emulated_instruction = NULL, 4432 .set_interrupt_shadow = svm_set_interrupt_shadow, 4433 .get_interrupt_shadow = svm_get_interrupt_shadow, 4434 .patch_hypercall = svm_patch_hypercall, 4435 .set_irq = svm_set_irq, 4436 .set_nmi = svm_inject_nmi, 4437 .queue_exception = svm_queue_exception, 4438 .cancel_injection = svm_cancel_injection, 4439 .interrupt_allowed = svm_interrupt_allowed, 4440 .nmi_allowed = svm_nmi_allowed, 4441 .get_nmi_mask = svm_get_nmi_mask, 4442 .set_nmi_mask = svm_set_nmi_mask, 4443 .enable_nmi_window = svm_enable_nmi_window, 4444 .enable_irq_window = svm_enable_irq_window, 4445 .update_cr8_intercept = svm_update_cr8_intercept, 4446 .set_virtual_apic_mode = svm_set_virtual_apic_mode, 4447 .refresh_apicv_exec_ctrl = svm_refresh_apicv_exec_ctrl, 4448 .check_apicv_inhibit_reasons = svm_check_apicv_inhibit_reasons, 4449 .load_eoi_exitmap = svm_load_eoi_exitmap, 4450 .hwapic_irr_update = svm_hwapic_irr_update, 4451 .hwapic_isr_update = svm_hwapic_isr_update, 4452 .apicv_post_state_restore = avic_post_state_restore, 4453 4454 .set_tss_addr = svm_set_tss_addr, 4455 .set_identity_map_addr = svm_set_identity_map_addr, 4456 .get_mt_mask = svm_get_mt_mask, 4457 4458 .get_exit_info = svm_get_exit_info, 4459 4460 .vcpu_after_set_cpuid = svm_vcpu_after_set_cpuid, 4461 4462 .has_wbinvd_exit = svm_has_wbinvd_exit, 4463 4464 .get_l2_tsc_offset = svm_get_l2_tsc_offset, 4465 .get_l2_tsc_multiplier = svm_get_l2_tsc_multiplier, 4466 .write_tsc_offset = svm_write_tsc_offset, 4467 .write_tsc_multiplier = svm_write_tsc_multiplier, 4468 4469 .load_mmu_pgd = svm_load_mmu_pgd, 4470 4471 .check_intercept = svm_check_intercept, 4472 .handle_exit_irqoff = svm_handle_exit_irqoff, 4473 4474 .request_immediate_exit = __kvm_request_immediate_exit, 4475 4476 .sched_in = svm_sched_in, 4477 4478 .pmu_ops = &amd_pmu_ops, 4479 .nested_ops = &svm_nested_ops, 4480 4481 .deliver_posted_interrupt = svm_deliver_avic_intr, 4482 .dy_apicv_has_pending_interrupt = svm_dy_apicv_has_pending_interrupt, 4483 .update_pi_irte = svm_update_pi_irte, 4484 .setup_mce = svm_setup_mce, 4485 4486 .smi_allowed = svm_smi_allowed, 4487 .enter_smm = svm_enter_smm, 4488 .leave_smm = svm_leave_smm, 4489 .enable_smi_window = svm_enable_smi_window, 4490 4491 .mem_enc_op = svm_mem_enc_op, 4492 .mem_enc_reg_region = svm_register_enc_region, 4493 .mem_enc_unreg_region = svm_unregister_enc_region, 4494 4495 .vm_copy_enc_context_from = svm_vm_copy_asid_from, 4496 .vm_move_enc_context_from = svm_vm_migrate_from, 4497 4498 .can_emulate_instruction = svm_can_emulate_instruction, 4499 4500 .apic_init_signal_blocked = svm_apic_init_signal_blocked, 4501 4502 .msr_filter_changed = svm_msr_filter_changed, 4503 .complete_emulated_msr = svm_complete_emulated_msr, 4504 4505 .vcpu_deliver_sipi_vector = svm_vcpu_deliver_sipi_vector, 4506 }; 4507 4508 /* 4509 * The default MMIO mask is a single bit (excluding the present bit), 4510 * which could conflict with the memory encryption bit. Check for 4511 * memory encryption support and override the default MMIO mask if 4512 * memory encryption is enabled. 4513 */ 4514 static __init void svm_adjust_mmio_mask(void) 4515 { 4516 unsigned int enc_bit, mask_bit; 4517 u64 msr, mask; 4518 4519 /* If there is no memory encryption support, use existing mask */ 4520 if (cpuid_eax(0x80000000) < 0x8000001f) 4521 return; 4522 4523 /* If memory encryption is not enabled, use existing mask */ 4524 rdmsrl(MSR_AMD64_SYSCFG, msr); 4525 if (!(msr & MSR_AMD64_SYSCFG_MEM_ENCRYPT)) 4526 return; 4527 4528 enc_bit = cpuid_ebx(0x8000001f) & 0x3f; 4529 mask_bit = boot_cpu_data.x86_phys_bits; 4530 4531 /* Increment the mask bit if it is the same as the encryption bit */ 4532 if (enc_bit == mask_bit) 4533 mask_bit++; 4534 4535 /* 4536 * If the mask bit location is below 52, then some bits above the 4537 * physical addressing limit will always be reserved, so use the 4538 * rsvd_bits() function to generate the mask. This mask, along with 4539 * the present bit, will be used to generate a page fault with 4540 * PFER.RSV = 1. 4541 * 4542 * If the mask bit location is 52 (or above), then clear the mask. 4543 */ 4544 mask = (mask_bit < 52) ? rsvd_bits(mask_bit, 51) | PT_PRESENT_MASK : 0; 4545 4546 kvm_mmu_set_mmio_spte_mask(mask, mask, PT_WRITABLE_MASK | PT_USER_MASK); 4547 } 4548 4549 static __init void svm_set_cpu_caps(void) 4550 { 4551 kvm_set_cpu_caps(); 4552 4553 supported_xss = 0; 4554 4555 /* CPUID 0x80000001 and 0x8000000A (SVM features) */ 4556 if (nested) { 4557 kvm_cpu_cap_set(X86_FEATURE_SVM); 4558 4559 if (nrips) 4560 kvm_cpu_cap_set(X86_FEATURE_NRIPS); 4561 4562 if (npt_enabled) 4563 kvm_cpu_cap_set(X86_FEATURE_NPT); 4564 4565 if (tsc_scaling) 4566 kvm_cpu_cap_set(X86_FEATURE_TSCRATEMSR); 4567 4568 /* Nested VM can receive #VMEXIT instead of triggering #GP */ 4569 kvm_cpu_cap_set(X86_FEATURE_SVME_ADDR_CHK); 4570 } 4571 4572 /* CPUID 0x80000008 */ 4573 if (boot_cpu_has(X86_FEATURE_LS_CFG_SSBD) || 4574 boot_cpu_has(X86_FEATURE_AMD_SSBD)) 4575 kvm_cpu_cap_set(X86_FEATURE_VIRT_SSBD); 4576 4577 /* AMD PMU PERFCTR_CORE CPUID */ 4578 if (enable_pmu && boot_cpu_has(X86_FEATURE_PERFCTR_CORE)) 4579 kvm_cpu_cap_set(X86_FEATURE_PERFCTR_CORE); 4580 4581 /* CPUID 0x8000001F (SME/SEV features) */ 4582 sev_set_cpu_caps(); 4583 } 4584 4585 static __init int svm_hardware_setup(void) 4586 { 4587 int cpu; 4588 struct page *iopm_pages; 4589 void *iopm_va; 4590 int r; 4591 unsigned int order = get_order(IOPM_SIZE); 4592 4593 /* 4594 * NX is required for shadow paging and for NPT if the NX huge pages 4595 * mitigation is enabled. 4596 */ 4597 if (!boot_cpu_has(X86_FEATURE_NX)) { 4598 pr_err_ratelimited("NX (Execute Disable) not supported\n"); 4599 return -EOPNOTSUPP; 4600 } 4601 kvm_enable_efer_bits(EFER_NX); 4602 4603 iopm_pages = alloc_pages(GFP_KERNEL, order); 4604 4605 if (!iopm_pages) 4606 return -ENOMEM; 4607 4608 iopm_va = page_address(iopm_pages); 4609 memset(iopm_va, 0xff, PAGE_SIZE * (1 << order)); 4610 iopm_base = page_to_pfn(iopm_pages) << PAGE_SHIFT; 4611 4612 init_msrpm_offsets(); 4613 4614 supported_xcr0 &= ~(XFEATURE_MASK_BNDREGS | XFEATURE_MASK_BNDCSR); 4615 4616 if (boot_cpu_has(X86_FEATURE_FXSR_OPT)) 4617 kvm_enable_efer_bits(EFER_FFXSR); 4618 4619 if (tsc_scaling) { 4620 if (!boot_cpu_has(X86_FEATURE_TSCRATEMSR)) { 4621 tsc_scaling = false; 4622 } else { 4623 pr_info("TSC scaling supported\n"); 4624 kvm_has_tsc_control = true; 4625 kvm_max_tsc_scaling_ratio = TSC_RATIO_MAX; 4626 kvm_tsc_scaling_ratio_frac_bits = 32; 4627 } 4628 } 4629 4630 tsc_aux_uret_slot = kvm_add_user_return_msr(MSR_TSC_AUX); 4631 4632 /* Check for pause filtering support */ 4633 if (!boot_cpu_has(X86_FEATURE_PAUSEFILTER)) { 4634 pause_filter_count = 0; 4635 pause_filter_thresh = 0; 4636 } else if (!boot_cpu_has(X86_FEATURE_PFTHRESHOLD)) { 4637 pause_filter_thresh = 0; 4638 } 4639 4640 if (nested) { 4641 printk(KERN_INFO "kvm: Nested Virtualization enabled\n"); 4642 kvm_enable_efer_bits(EFER_SVME | EFER_LMSLE); 4643 } 4644 4645 /* 4646 * KVM's MMU doesn't support using 2-level paging for itself, and thus 4647 * NPT isn't supported if the host is using 2-level paging since host 4648 * CR4 is unchanged on VMRUN. 4649 */ 4650 if (!IS_ENABLED(CONFIG_X86_64) && !IS_ENABLED(CONFIG_X86_PAE)) 4651 npt_enabled = false; 4652 4653 if (!boot_cpu_has(X86_FEATURE_NPT)) 4654 npt_enabled = false; 4655 4656 /* Force VM NPT level equal to the host's paging level */ 4657 kvm_configure_mmu(npt_enabled, get_npt_level(), 4658 get_npt_level(), PG_LEVEL_1G); 4659 pr_info("kvm: Nested Paging %sabled\n", npt_enabled ? "en" : "dis"); 4660 4661 /* Note, SEV setup consumes npt_enabled. */ 4662 sev_hardware_setup(); 4663 4664 svm_hv_hardware_setup(); 4665 4666 svm_adjust_mmio_mask(); 4667 4668 for_each_possible_cpu(cpu) { 4669 r = svm_cpu_init(cpu); 4670 if (r) 4671 goto err; 4672 } 4673 4674 if (nrips) { 4675 if (!boot_cpu_has(X86_FEATURE_NRIPS)) 4676 nrips = false; 4677 } 4678 4679 enable_apicv = avic = avic && npt_enabled && boot_cpu_has(X86_FEATURE_AVIC); 4680 4681 if (enable_apicv) { 4682 pr_info("AVIC enabled\n"); 4683 4684 amd_iommu_register_ga_log_notifier(&avic_ga_log_notifier); 4685 } else { 4686 svm_x86_ops.vcpu_blocking = NULL; 4687 svm_x86_ops.vcpu_unblocking = NULL; 4688 } 4689 4690 if (vls) { 4691 if (!npt_enabled || 4692 !boot_cpu_has(X86_FEATURE_V_VMSAVE_VMLOAD) || 4693 !IS_ENABLED(CONFIG_X86_64)) { 4694 vls = false; 4695 } else { 4696 pr_info("Virtual VMLOAD VMSAVE supported\n"); 4697 } 4698 } 4699 4700 if (boot_cpu_has(X86_FEATURE_SVME_ADDR_CHK)) 4701 svm_gp_erratum_intercept = false; 4702 4703 if (vgif) { 4704 if (!boot_cpu_has(X86_FEATURE_VGIF)) 4705 vgif = false; 4706 else 4707 pr_info("Virtual GIF supported\n"); 4708 } 4709 4710 if (lbrv) { 4711 if (!boot_cpu_has(X86_FEATURE_LBRV)) 4712 lbrv = false; 4713 else 4714 pr_info("LBR virtualization supported\n"); 4715 } 4716 4717 if (!enable_pmu) 4718 pr_info("PMU virtualization is disabled\n"); 4719 4720 svm_set_cpu_caps(); 4721 4722 /* 4723 * It seems that on AMD processors PTE's accessed bit is 4724 * being set by the CPU hardware before the NPF vmexit. 4725 * This is not expected behaviour and our tests fail because 4726 * of it. 4727 * A workaround here is to disable support for 4728 * GUEST_MAXPHYADDR < HOST_MAXPHYADDR if NPT is enabled. 4729 * In this case userspace can know if there is support using 4730 * KVM_CAP_SMALLER_MAXPHYADDR extension and decide how to handle 4731 * it 4732 * If future AMD CPU models change the behaviour described above, 4733 * this variable can be changed accordingly 4734 */ 4735 allow_smaller_maxphyaddr = !npt_enabled; 4736 4737 return 0; 4738 4739 err: 4740 svm_hardware_teardown(); 4741 return r; 4742 } 4743 4744 4745 static struct kvm_x86_init_ops svm_init_ops __initdata = { 4746 .cpu_has_kvm_support = has_svm, 4747 .disabled_by_bios = is_disabled, 4748 .hardware_setup = svm_hardware_setup, 4749 .check_processor_compatibility = svm_check_processor_compat, 4750 4751 .runtime_ops = &svm_x86_ops, 4752 }; 4753 4754 static int __init svm_init(void) 4755 { 4756 __unused_size_checks(); 4757 4758 return kvm_init(&svm_init_ops, sizeof(struct vcpu_svm), 4759 __alignof__(struct vcpu_svm), THIS_MODULE); 4760 } 4761 4762 static void __exit svm_exit(void) 4763 { 4764 kvm_exit(); 4765 } 4766 4767 module_init(svm_init) 4768 module_exit(svm_exit) 4769