1 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt 2 3 #include <linux/kvm_host.h> 4 5 #include "irq.h" 6 #include "mmu.h" 7 #include "kvm_cache_regs.h" 8 #include "x86.h" 9 #include "smm.h" 10 #include "cpuid.h" 11 #include "pmu.h" 12 13 #include <linux/module.h> 14 #include <linux/mod_devicetable.h> 15 #include <linux/kernel.h> 16 #include <linux/vmalloc.h> 17 #include <linux/highmem.h> 18 #include <linux/amd-iommu.h> 19 #include <linux/sched.h> 20 #include <linux/trace_events.h> 21 #include <linux/slab.h> 22 #include <linux/hashtable.h> 23 #include <linux/objtool.h> 24 #include <linux/psp-sev.h> 25 #include <linux/file.h> 26 #include <linux/pagemap.h> 27 #include <linux/swap.h> 28 #include <linux/rwsem.h> 29 #include <linux/cc_platform.h> 30 #include <linux/smp.h> 31 32 #include <asm/apic.h> 33 #include <asm/perf_event.h> 34 #include <asm/tlbflush.h> 35 #include <asm/desc.h> 36 #include <asm/debugreg.h> 37 #include <asm/kvm_para.h> 38 #include <asm/irq_remapping.h> 39 #include <asm/spec-ctrl.h> 40 #include <asm/cpu_device_id.h> 41 #include <asm/traps.h> 42 #include <asm/reboot.h> 43 #include <asm/fpu/api.h> 44 45 #include <trace/events/ipi.h> 46 47 #include "trace.h" 48 49 #include "svm.h" 50 #include "svm_ops.h" 51 52 #include "kvm_onhyperv.h" 53 #include "svm_onhyperv.h" 54 55 MODULE_AUTHOR("Qumranet"); 56 MODULE_LICENSE("GPL"); 57 58 #ifdef MODULE 59 static const struct x86_cpu_id svm_cpu_id[] = { 60 X86_MATCH_FEATURE(X86_FEATURE_SVM, NULL), 61 {} 62 }; 63 MODULE_DEVICE_TABLE(x86cpu, svm_cpu_id); 64 #endif 65 66 #define SEG_TYPE_LDT 2 67 #define SEG_TYPE_BUSY_TSS16 3 68 69 static bool erratum_383_found __read_mostly; 70 71 u32 msrpm_offsets[MSRPM_OFFSETS] __read_mostly; 72 73 /* 74 * Set osvw_len to higher value when updated Revision Guides 75 * are published and we know what the new status bits are 76 */ 77 static uint64_t osvw_len = 4, osvw_status; 78 79 static DEFINE_PER_CPU(u64, current_tsc_ratio); 80 81 #define X2APIC_MSR(x) (APIC_BASE_MSR + (x >> 4)) 82 83 static const struct svm_direct_access_msrs { 84 u32 index; /* Index of the MSR */ 85 bool always; /* True if intercept is initially cleared */ 86 } direct_access_msrs[MAX_DIRECT_ACCESS_MSRS] = { 87 { .index = MSR_STAR, .always = true }, 88 { .index = MSR_IA32_SYSENTER_CS, .always = true }, 89 { .index = MSR_IA32_SYSENTER_EIP, .always = false }, 90 { .index = MSR_IA32_SYSENTER_ESP, .always = false }, 91 #ifdef CONFIG_X86_64 92 { .index = MSR_GS_BASE, .always = true }, 93 { .index = MSR_FS_BASE, .always = true }, 94 { .index = MSR_KERNEL_GS_BASE, .always = true }, 95 { .index = MSR_LSTAR, .always = true }, 96 { .index = MSR_CSTAR, .always = true }, 97 { .index = MSR_SYSCALL_MASK, .always = true }, 98 #endif 99 { .index = MSR_IA32_SPEC_CTRL, .always = false }, 100 { .index = MSR_IA32_PRED_CMD, .always = false }, 101 { .index = MSR_IA32_FLUSH_CMD, .always = false }, 102 { .index = MSR_IA32_DEBUGCTLMSR, .always = false }, 103 { .index = MSR_IA32_LASTBRANCHFROMIP, .always = false }, 104 { .index = MSR_IA32_LASTBRANCHTOIP, .always = false }, 105 { .index = MSR_IA32_LASTINTFROMIP, .always = false }, 106 { .index = MSR_IA32_LASTINTTOIP, .always = false }, 107 { .index = MSR_IA32_XSS, .always = false }, 108 { .index = MSR_EFER, .always = false }, 109 { .index = MSR_IA32_CR_PAT, .always = false }, 110 { .index = MSR_AMD64_SEV_ES_GHCB, .always = true }, 111 { .index = MSR_TSC_AUX, .always = false }, 112 { .index = X2APIC_MSR(APIC_ID), .always = false }, 113 { .index = X2APIC_MSR(APIC_LVR), .always = false }, 114 { .index = X2APIC_MSR(APIC_TASKPRI), .always = false }, 115 { .index = X2APIC_MSR(APIC_ARBPRI), .always = false }, 116 { .index = X2APIC_MSR(APIC_PROCPRI), .always = false }, 117 { .index = X2APIC_MSR(APIC_EOI), .always = false }, 118 { .index = X2APIC_MSR(APIC_RRR), .always = false }, 119 { .index = X2APIC_MSR(APIC_LDR), .always = false }, 120 { .index = X2APIC_MSR(APIC_DFR), .always = false }, 121 { .index = X2APIC_MSR(APIC_SPIV), .always = false }, 122 { .index = X2APIC_MSR(APIC_ISR), .always = false }, 123 { .index = X2APIC_MSR(APIC_TMR), .always = false }, 124 { .index = X2APIC_MSR(APIC_IRR), .always = false }, 125 { .index = X2APIC_MSR(APIC_ESR), .always = false }, 126 { .index = X2APIC_MSR(APIC_ICR), .always = false }, 127 { .index = X2APIC_MSR(APIC_ICR2), .always = false }, 128 129 /* 130 * Note: 131 * AMD does not virtualize APIC TSC-deadline timer mode, but it is 132 * emulated by KVM. When setting APIC LVTT (0x832) register bit 18, 133 * the AVIC hardware would generate GP fault. Therefore, always 134 * intercept the MSR 0x832, and do not setup direct_access_msr. 135 */ 136 { .index = X2APIC_MSR(APIC_LVTTHMR), .always = false }, 137 { .index = X2APIC_MSR(APIC_LVTPC), .always = false }, 138 { .index = X2APIC_MSR(APIC_LVT0), .always = false }, 139 { .index = X2APIC_MSR(APIC_LVT1), .always = false }, 140 { .index = X2APIC_MSR(APIC_LVTERR), .always = false }, 141 { .index = X2APIC_MSR(APIC_TMICT), .always = false }, 142 { .index = X2APIC_MSR(APIC_TMCCT), .always = false }, 143 { .index = X2APIC_MSR(APIC_TDCR), .always = false }, 144 { .index = MSR_INVALID, .always = false }, 145 }; 146 147 /* 148 * These 2 parameters are used to config the controls for Pause-Loop Exiting: 149 * pause_filter_count: On processors that support Pause filtering(indicated 150 * by CPUID Fn8000_000A_EDX), the VMCB provides a 16 bit pause filter 151 * count value. On VMRUN this value is loaded into an internal counter. 152 * Each time a pause instruction is executed, this counter is decremented 153 * until it reaches zero at which time a #VMEXIT is generated if pause 154 * intercept is enabled. Refer to AMD APM Vol 2 Section 15.14.4 Pause 155 * Intercept Filtering for more details. 156 * This also indicate if ple logic enabled. 157 * 158 * pause_filter_thresh: In addition, some processor families support advanced 159 * pause filtering (indicated by CPUID Fn8000_000A_EDX) upper bound on 160 * the amount of time a guest is allowed to execute in a pause loop. 161 * In this mode, a 16-bit pause filter threshold field is added in the 162 * VMCB. The threshold value is a cycle count that is used to reset the 163 * pause counter. As with simple pause filtering, VMRUN loads the pause 164 * count value from VMCB into an internal counter. Then, on each pause 165 * instruction the hardware checks the elapsed number of cycles since 166 * the most recent pause instruction against the pause filter threshold. 167 * If the elapsed cycle count is greater than the pause filter threshold, 168 * then the internal pause count is reloaded from the VMCB and execution 169 * continues. If the elapsed cycle count is less than the pause filter 170 * threshold, then the internal pause count is decremented. If the count 171 * value is less than zero and PAUSE intercept is enabled, a #VMEXIT is 172 * triggered. If advanced pause filtering is supported and pause filter 173 * threshold field is set to zero, the filter will operate in the simpler, 174 * count only mode. 175 */ 176 177 static unsigned short pause_filter_thresh = KVM_DEFAULT_PLE_GAP; 178 module_param(pause_filter_thresh, ushort, 0444); 179 180 static unsigned short pause_filter_count = KVM_SVM_DEFAULT_PLE_WINDOW; 181 module_param(pause_filter_count, ushort, 0444); 182 183 /* Default doubles per-vcpu window every exit. */ 184 static unsigned short pause_filter_count_grow = KVM_DEFAULT_PLE_WINDOW_GROW; 185 module_param(pause_filter_count_grow, ushort, 0444); 186 187 /* Default resets per-vcpu window every exit to pause_filter_count. */ 188 static unsigned short pause_filter_count_shrink = KVM_DEFAULT_PLE_WINDOW_SHRINK; 189 module_param(pause_filter_count_shrink, ushort, 0444); 190 191 /* Default is to compute the maximum so we can never overflow. */ 192 static unsigned short pause_filter_count_max = KVM_SVM_DEFAULT_PLE_WINDOW_MAX; 193 module_param(pause_filter_count_max, ushort, 0444); 194 195 /* 196 * Use nested page tables by default. Note, NPT may get forced off by 197 * svm_hardware_setup() if it's unsupported by hardware or the host kernel. 198 */ 199 bool npt_enabled = true; 200 module_param_named(npt, npt_enabled, bool, 0444); 201 202 /* allow nested virtualization in KVM/SVM */ 203 static int nested = true; 204 module_param(nested, int, S_IRUGO); 205 206 /* enable/disable Next RIP Save */ 207 int nrips = true; 208 module_param(nrips, int, 0444); 209 210 /* enable/disable Virtual VMLOAD VMSAVE */ 211 static int vls = true; 212 module_param(vls, int, 0444); 213 214 /* enable/disable Virtual GIF */ 215 int vgif = true; 216 module_param(vgif, int, 0444); 217 218 /* enable/disable LBR virtualization */ 219 int lbrv = true; 220 module_param(lbrv, int, 0444); 221 222 static int tsc_scaling = true; 223 module_param(tsc_scaling, int, 0444); 224 225 /* 226 * enable / disable AVIC. Because the defaults differ for APICv 227 * support between VMX and SVM we cannot use module_param_named. 228 */ 229 static bool avic; 230 module_param(avic, bool, 0444); 231 232 bool __read_mostly dump_invalid_vmcb; 233 module_param(dump_invalid_vmcb, bool, 0644); 234 235 236 bool intercept_smi = true; 237 module_param(intercept_smi, bool, 0444); 238 239 bool vnmi = true; 240 module_param(vnmi, bool, 0444); 241 242 static bool svm_gp_erratum_intercept = true; 243 244 static u8 rsm_ins_bytes[] = "\x0f\xaa"; 245 246 static unsigned long iopm_base; 247 248 DEFINE_PER_CPU(struct svm_cpu_data, svm_data); 249 250 /* 251 * Only MSR_TSC_AUX is switched via the user return hook. EFER is switched via 252 * the VMCB, and the SYSCALL/SYSENTER MSRs are handled by VMLOAD/VMSAVE. 253 * 254 * RDTSCP and RDPID are not used in the kernel, specifically to allow KVM to 255 * defer the restoration of TSC_AUX until the CPU returns to userspace. 256 */ 257 static int tsc_aux_uret_slot __read_mostly = -1; 258 259 static const u32 msrpm_ranges[] = {0, 0xc0000000, 0xc0010000}; 260 261 #define NUM_MSR_MAPS ARRAY_SIZE(msrpm_ranges) 262 #define MSRS_RANGE_SIZE 2048 263 #define MSRS_IN_RANGE (MSRS_RANGE_SIZE * 8 / 2) 264 265 u32 svm_msrpm_offset(u32 msr) 266 { 267 u32 offset; 268 int i; 269 270 for (i = 0; i < NUM_MSR_MAPS; i++) { 271 if (msr < msrpm_ranges[i] || 272 msr >= msrpm_ranges[i] + MSRS_IN_RANGE) 273 continue; 274 275 offset = (msr - msrpm_ranges[i]) / 4; /* 4 msrs per u8 */ 276 offset += (i * MSRS_RANGE_SIZE); /* add range offset */ 277 278 /* Now we have the u8 offset - but need the u32 offset */ 279 return offset / 4; 280 } 281 282 /* MSR not in any range */ 283 return MSR_INVALID; 284 } 285 286 static void svm_flush_tlb_current(struct kvm_vcpu *vcpu); 287 288 static int get_npt_level(void) 289 { 290 #ifdef CONFIG_X86_64 291 return pgtable_l5_enabled() ? PT64_ROOT_5LEVEL : PT64_ROOT_4LEVEL; 292 #else 293 return PT32E_ROOT_LEVEL; 294 #endif 295 } 296 297 int svm_set_efer(struct kvm_vcpu *vcpu, u64 efer) 298 { 299 struct vcpu_svm *svm = to_svm(vcpu); 300 u64 old_efer = vcpu->arch.efer; 301 vcpu->arch.efer = efer; 302 303 if (!npt_enabled) { 304 /* Shadow paging assumes NX to be available. */ 305 efer |= EFER_NX; 306 307 if (!(efer & EFER_LMA)) 308 efer &= ~EFER_LME; 309 } 310 311 if ((old_efer & EFER_SVME) != (efer & EFER_SVME)) { 312 if (!(efer & EFER_SVME)) { 313 svm_leave_nested(vcpu); 314 svm_set_gif(svm, true); 315 /* #GP intercept is still needed for vmware backdoor */ 316 if (!enable_vmware_backdoor) 317 clr_exception_intercept(svm, GP_VECTOR); 318 319 /* 320 * Free the nested guest state, unless we are in SMM. 321 * In this case we will return to the nested guest 322 * as soon as we leave SMM. 323 */ 324 if (!is_smm(vcpu)) 325 svm_free_nested(svm); 326 327 } else { 328 int ret = svm_allocate_nested(svm); 329 330 if (ret) { 331 vcpu->arch.efer = old_efer; 332 return ret; 333 } 334 335 /* 336 * Never intercept #GP for SEV guests, KVM can't 337 * decrypt guest memory to workaround the erratum. 338 */ 339 if (svm_gp_erratum_intercept && !sev_guest(vcpu->kvm)) 340 set_exception_intercept(svm, GP_VECTOR); 341 } 342 } 343 344 svm->vmcb->save.efer = efer | EFER_SVME; 345 vmcb_mark_dirty(svm->vmcb, VMCB_CR); 346 return 0; 347 } 348 349 static u32 svm_get_interrupt_shadow(struct kvm_vcpu *vcpu) 350 { 351 struct vcpu_svm *svm = to_svm(vcpu); 352 u32 ret = 0; 353 354 if (svm->vmcb->control.int_state & SVM_INTERRUPT_SHADOW_MASK) 355 ret = KVM_X86_SHADOW_INT_STI | KVM_X86_SHADOW_INT_MOV_SS; 356 return ret; 357 } 358 359 static void svm_set_interrupt_shadow(struct kvm_vcpu *vcpu, int mask) 360 { 361 struct vcpu_svm *svm = to_svm(vcpu); 362 363 if (mask == 0) 364 svm->vmcb->control.int_state &= ~SVM_INTERRUPT_SHADOW_MASK; 365 else 366 svm->vmcb->control.int_state |= SVM_INTERRUPT_SHADOW_MASK; 367 368 } 369 static bool svm_can_emulate_instruction(struct kvm_vcpu *vcpu, int emul_type, 370 void *insn, int insn_len); 371 372 static int __svm_skip_emulated_instruction(struct kvm_vcpu *vcpu, 373 bool commit_side_effects) 374 { 375 struct vcpu_svm *svm = to_svm(vcpu); 376 unsigned long old_rflags; 377 378 /* 379 * SEV-ES does not expose the next RIP. The RIP update is controlled by 380 * the type of exit and the #VC handler in the guest. 381 */ 382 if (sev_es_guest(vcpu->kvm)) 383 goto done; 384 385 if (nrips && svm->vmcb->control.next_rip != 0) { 386 WARN_ON_ONCE(!static_cpu_has(X86_FEATURE_NRIPS)); 387 svm->next_rip = svm->vmcb->control.next_rip; 388 } 389 390 if (!svm->next_rip) { 391 /* 392 * FIXME: Drop this when kvm_emulate_instruction() does the 393 * right thing and treats "can't emulate" as outright failure 394 * for EMULTYPE_SKIP. 395 */ 396 if (!svm_can_emulate_instruction(vcpu, EMULTYPE_SKIP, NULL, 0)) 397 return 0; 398 399 if (unlikely(!commit_side_effects)) 400 old_rflags = svm->vmcb->save.rflags; 401 402 if (!kvm_emulate_instruction(vcpu, EMULTYPE_SKIP)) 403 return 0; 404 405 if (unlikely(!commit_side_effects)) 406 svm->vmcb->save.rflags = old_rflags; 407 } else { 408 kvm_rip_write(vcpu, svm->next_rip); 409 } 410 411 done: 412 if (likely(commit_side_effects)) 413 svm_set_interrupt_shadow(vcpu, 0); 414 415 return 1; 416 } 417 418 static int svm_skip_emulated_instruction(struct kvm_vcpu *vcpu) 419 { 420 return __svm_skip_emulated_instruction(vcpu, true); 421 } 422 423 static int svm_update_soft_interrupt_rip(struct kvm_vcpu *vcpu) 424 { 425 unsigned long rip, old_rip = kvm_rip_read(vcpu); 426 struct vcpu_svm *svm = to_svm(vcpu); 427 428 /* 429 * Due to architectural shortcomings, the CPU doesn't always provide 430 * NextRIP, e.g. if KVM intercepted an exception that occurred while 431 * the CPU was vectoring an INTO/INT3 in the guest. Temporarily skip 432 * the instruction even if NextRIP is supported to acquire the next 433 * RIP so that it can be shoved into the NextRIP field, otherwise 434 * hardware will fail to advance guest RIP during event injection. 435 * Drop the exception/interrupt if emulation fails and effectively 436 * retry the instruction, it's the least awful option. If NRIPS is 437 * in use, the skip must not commit any side effects such as clearing 438 * the interrupt shadow or RFLAGS.RF. 439 */ 440 if (!__svm_skip_emulated_instruction(vcpu, !nrips)) 441 return -EIO; 442 443 rip = kvm_rip_read(vcpu); 444 445 /* 446 * Save the injection information, even when using next_rip, as the 447 * VMCB's next_rip will be lost (cleared on VM-Exit) if the injection 448 * doesn't complete due to a VM-Exit occurring while the CPU is 449 * vectoring the event. Decoding the instruction isn't guaranteed to 450 * work as there may be no backing instruction, e.g. if the event is 451 * being injected by L1 for L2, or if the guest is patching INT3 into 452 * a different instruction. 453 */ 454 svm->soft_int_injected = true; 455 svm->soft_int_csbase = svm->vmcb->save.cs.base; 456 svm->soft_int_old_rip = old_rip; 457 svm->soft_int_next_rip = rip; 458 459 if (nrips) 460 kvm_rip_write(vcpu, old_rip); 461 462 if (static_cpu_has(X86_FEATURE_NRIPS)) 463 svm->vmcb->control.next_rip = rip; 464 465 return 0; 466 } 467 468 static void svm_inject_exception(struct kvm_vcpu *vcpu) 469 { 470 struct kvm_queued_exception *ex = &vcpu->arch.exception; 471 struct vcpu_svm *svm = to_svm(vcpu); 472 473 kvm_deliver_exception_payload(vcpu, ex); 474 475 if (kvm_exception_is_soft(ex->vector) && 476 svm_update_soft_interrupt_rip(vcpu)) 477 return; 478 479 svm->vmcb->control.event_inj = ex->vector 480 | SVM_EVTINJ_VALID 481 | (ex->has_error_code ? SVM_EVTINJ_VALID_ERR : 0) 482 | SVM_EVTINJ_TYPE_EXEPT; 483 svm->vmcb->control.event_inj_err = ex->error_code; 484 } 485 486 static void svm_init_erratum_383(void) 487 { 488 u32 low, high; 489 int err; 490 u64 val; 491 492 if (!static_cpu_has_bug(X86_BUG_AMD_TLB_MMATCH)) 493 return; 494 495 /* Use _safe variants to not break nested virtualization */ 496 val = native_read_msr_safe(MSR_AMD64_DC_CFG, &err); 497 if (err) 498 return; 499 500 val |= (1ULL << 47); 501 502 low = lower_32_bits(val); 503 high = upper_32_bits(val); 504 505 native_write_msr_safe(MSR_AMD64_DC_CFG, low, high); 506 507 erratum_383_found = true; 508 } 509 510 static void svm_init_osvw(struct kvm_vcpu *vcpu) 511 { 512 /* 513 * Guests should see errata 400 and 415 as fixed (assuming that 514 * HLT and IO instructions are intercepted). 515 */ 516 vcpu->arch.osvw.length = (osvw_len >= 3) ? (osvw_len) : 3; 517 vcpu->arch.osvw.status = osvw_status & ~(6ULL); 518 519 /* 520 * By increasing VCPU's osvw.length to 3 we are telling the guest that 521 * all osvw.status bits inside that length, including bit 0 (which is 522 * reserved for erratum 298), are valid. However, if host processor's 523 * osvw_len is 0 then osvw_status[0] carries no information. We need to 524 * be conservative here and therefore we tell the guest that erratum 298 525 * is present (because we really don't know). 526 */ 527 if (osvw_len == 0 && boot_cpu_data.x86 == 0x10) 528 vcpu->arch.osvw.status |= 1; 529 } 530 531 static bool __kvm_is_svm_supported(void) 532 { 533 int cpu = smp_processor_id(); 534 struct cpuinfo_x86 *c = &cpu_data(cpu); 535 536 u64 vm_cr; 537 538 if (c->x86_vendor != X86_VENDOR_AMD && 539 c->x86_vendor != X86_VENDOR_HYGON) { 540 pr_err("CPU %d isn't AMD or Hygon\n", cpu); 541 return false; 542 } 543 544 if (!cpu_has(c, X86_FEATURE_SVM)) { 545 pr_err("SVM not supported by CPU %d\n", cpu); 546 return false; 547 } 548 549 if (cc_platform_has(CC_ATTR_GUEST_MEM_ENCRYPT)) { 550 pr_info("KVM is unsupported when running as an SEV guest\n"); 551 return false; 552 } 553 554 rdmsrl(MSR_VM_CR, vm_cr); 555 if (vm_cr & (1 << SVM_VM_CR_SVM_DISABLE)) { 556 pr_err("SVM disabled (by BIOS) in MSR_VM_CR on CPU %d\n", cpu); 557 return false; 558 } 559 560 return true; 561 } 562 563 static bool kvm_is_svm_supported(void) 564 { 565 bool supported; 566 567 migrate_disable(); 568 supported = __kvm_is_svm_supported(); 569 migrate_enable(); 570 571 return supported; 572 } 573 574 static int svm_check_processor_compat(void) 575 { 576 if (!__kvm_is_svm_supported()) 577 return -EIO; 578 579 return 0; 580 } 581 582 static void __svm_write_tsc_multiplier(u64 multiplier) 583 { 584 if (multiplier == __this_cpu_read(current_tsc_ratio)) 585 return; 586 587 wrmsrl(MSR_AMD64_TSC_RATIO, multiplier); 588 __this_cpu_write(current_tsc_ratio, multiplier); 589 } 590 591 static inline void kvm_cpu_svm_disable(void) 592 { 593 uint64_t efer; 594 595 wrmsrl(MSR_VM_HSAVE_PA, 0); 596 rdmsrl(MSR_EFER, efer); 597 if (efer & EFER_SVME) { 598 /* 599 * Force GIF=1 prior to disabling SVM, e.g. to ensure INIT and 600 * NMI aren't blocked. 601 */ 602 stgi(); 603 wrmsrl(MSR_EFER, efer & ~EFER_SVME); 604 } 605 } 606 607 static void svm_emergency_disable(void) 608 { 609 kvm_rebooting = true; 610 611 kvm_cpu_svm_disable(); 612 } 613 614 static void svm_hardware_disable(void) 615 { 616 /* Make sure we clean up behind us */ 617 if (tsc_scaling) 618 __svm_write_tsc_multiplier(SVM_TSC_RATIO_DEFAULT); 619 620 kvm_cpu_svm_disable(); 621 622 amd_pmu_disable_virt(); 623 } 624 625 static int svm_hardware_enable(void) 626 { 627 628 struct svm_cpu_data *sd; 629 uint64_t efer; 630 int me = raw_smp_processor_id(); 631 632 rdmsrl(MSR_EFER, efer); 633 if (efer & EFER_SVME) 634 return -EBUSY; 635 636 sd = per_cpu_ptr(&svm_data, me); 637 sd->asid_generation = 1; 638 sd->max_asid = cpuid_ebx(SVM_CPUID_FUNC) - 1; 639 sd->next_asid = sd->max_asid + 1; 640 sd->min_asid = max_sev_asid + 1; 641 642 wrmsrl(MSR_EFER, efer | EFER_SVME); 643 644 wrmsrl(MSR_VM_HSAVE_PA, sd->save_area_pa); 645 646 if (static_cpu_has(X86_FEATURE_TSCRATEMSR)) { 647 /* 648 * Set the default value, even if we don't use TSC scaling 649 * to avoid having stale value in the msr 650 */ 651 __svm_write_tsc_multiplier(SVM_TSC_RATIO_DEFAULT); 652 } 653 654 655 /* 656 * Get OSVW bits. 657 * 658 * Note that it is possible to have a system with mixed processor 659 * revisions and therefore different OSVW bits. If bits are not the same 660 * on different processors then choose the worst case (i.e. if erratum 661 * is present on one processor and not on another then assume that the 662 * erratum is present everywhere). 663 */ 664 if (cpu_has(&boot_cpu_data, X86_FEATURE_OSVW)) { 665 uint64_t len, status = 0; 666 int err; 667 668 len = native_read_msr_safe(MSR_AMD64_OSVW_ID_LENGTH, &err); 669 if (!err) 670 status = native_read_msr_safe(MSR_AMD64_OSVW_STATUS, 671 &err); 672 673 if (err) 674 osvw_status = osvw_len = 0; 675 else { 676 if (len < osvw_len) 677 osvw_len = len; 678 osvw_status |= status; 679 osvw_status &= (1ULL << osvw_len) - 1; 680 } 681 } else 682 osvw_status = osvw_len = 0; 683 684 svm_init_erratum_383(); 685 686 amd_pmu_enable_virt(); 687 688 /* 689 * If TSC_AUX virtualization is supported, TSC_AUX becomes a swap type 690 * "B" field (see sev_es_prepare_switch_to_guest()) for SEV-ES guests. 691 * Since Linux does not change the value of TSC_AUX once set, prime the 692 * TSC_AUX field now to avoid a RDMSR on every vCPU run. 693 */ 694 if (boot_cpu_has(X86_FEATURE_V_TSC_AUX)) { 695 struct sev_es_save_area *hostsa; 696 u32 __maybe_unused msr_hi; 697 698 hostsa = (struct sev_es_save_area *)(page_address(sd->save_area) + 0x400); 699 700 rdmsr(MSR_TSC_AUX, hostsa->tsc_aux, msr_hi); 701 } 702 703 return 0; 704 } 705 706 static void svm_cpu_uninit(int cpu) 707 { 708 struct svm_cpu_data *sd = per_cpu_ptr(&svm_data, cpu); 709 710 if (!sd->save_area) 711 return; 712 713 kfree(sd->sev_vmcbs); 714 __free_page(sd->save_area); 715 sd->save_area_pa = 0; 716 sd->save_area = NULL; 717 } 718 719 static int svm_cpu_init(int cpu) 720 { 721 struct svm_cpu_data *sd = per_cpu_ptr(&svm_data, cpu); 722 int ret = -ENOMEM; 723 724 memset(sd, 0, sizeof(struct svm_cpu_data)); 725 sd->save_area = alloc_page(GFP_KERNEL | __GFP_ZERO); 726 if (!sd->save_area) 727 return ret; 728 729 ret = sev_cpu_init(sd); 730 if (ret) 731 goto free_save_area; 732 733 sd->save_area_pa = __sme_page_pa(sd->save_area); 734 return 0; 735 736 free_save_area: 737 __free_page(sd->save_area); 738 sd->save_area = NULL; 739 return ret; 740 741 } 742 743 static void set_dr_intercepts(struct vcpu_svm *svm) 744 { 745 struct vmcb *vmcb = svm->vmcb01.ptr; 746 747 vmcb_set_intercept(&vmcb->control, INTERCEPT_DR0_READ); 748 vmcb_set_intercept(&vmcb->control, INTERCEPT_DR1_READ); 749 vmcb_set_intercept(&vmcb->control, INTERCEPT_DR2_READ); 750 vmcb_set_intercept(&vmcb->control, INTERCEPT_DR3_READ); 751 vmcb_set_intercept(&vmcb->control, INTERCEPT_DR4_READ); 752 vmcb_set_intercept(&vmcb->control, INTERCEPT_DR5_READ); 753 vmcb_set_intercept(&vmcb->control, INTERCEPT_DR6_READ); 754 vmcb_set_intercept(&vmcb->control, INTERCEPT_DR0_WRITE); 755 vmcb_set_intercept(&vmcb->control, INTERCEPT_DR1_WRITE); 756 vmcb_set_intercept(&vmcb->control, INTERCEPT_DR2_WRITE); 757 vmcb_set_intercept(&vmcb->control, INTERCEPT_DR3_WRITE); 758 vmcb_set_intercept(&vmcb->control, INTERCEPT_DR4_WRITE); 759 vmcb_set_intercept(&vmcb->control, INTERCEPT_DR5_WRITE); 760 vmcb_set_intercept(&vmcb->control, INTERCEPT_DR6_WRITE); 761 vmcb_set_intercept(&vmcb->control, INTERCEPT_DR7_READ); 762 vmcb_set_intercept(&vmcb->control, INTERCEPT_DR7_WRITE); 763 764 recalc_intercepts(svm); 765 } 766 767 static void clr_dr_intercepts(struct vcpu_svm *svm) 768 { 769 struct vmcb *vmcb = svm->vmcb01.ptr; 770 771 vmcb->control.intercepts[INTERCEPT_DR] = 0; 772 773 recalc_intercepts(svm); 774 } 775 776 static int direct_access_msr_slot(u32 msr) 777 { 778 u32 i; 779 780 for (i = 0; direct_access_msrs[i].index != MSR_INVALID; i++) 781 if (direct_access_msrs[i].index == msr) 782 return i; 783 784 return -ENOENT; 785 } 786 787 static void set_shadow_msr_intercept(struct kvm_vcpu *vcpu, u32 msr, int read, 788 int write) 789 { 790 struct vcpu_svm *svm = to_svm(vcpu); 791 int slot = direct_access_msr_slot(msr); 792 793 if (slot == -ENOENT) 794 return; 795 796 /* Set the shadow bitmaps to the desired intercept states */ 797 if (read) 798 set_bit(slot, svm->shadow_msr_intercept.read); 799 else 800 clear_bit(slot, svm->shadow_msr_intercept.read); 801 802 if (write) 803 set_bit(slot, svm->shadow_msr_intercept.write); 804 else 805 clear_bit(slot, svm->shadow_msr_intercept.write); 806 } 807 808 static bool valid_msr_intercept(u32 index) 809 { 810 return direct_access_msr_slot(index) != -ENOENT; 811 } 812 813 static bool msr_write_intercepted(struct kvm_vcpu *vcpu, u32 msr) 814 { 815 u8 bit_write; 816 unsigned long tmp; 817 u32 offset; 818 u32 *msrpm; 819 820 /* 821 * For non-nested case: 822 * If the L01 MSR bitmap does not intercept the MSR, then we need to 823 * save it. 824 * 825 * For nested case: 826 * If the L02 MSR bitmap does not intercept the MSR, then we need to 827 * save it. 828 */ 829 msrpm = is_guest_mode(vcpu) ? to_svm(vcpu)->nested.msrpm: 830 to_svm(vcpu)->msrpm; 831 832 offset = svm_msrpm_offset(msr); 833 bit_write = 2 * (msr & 0x0f) + 1; 834 tmp = msrpm[offset]; 835 836 BUG_ON(offset == MSR_INVALID); 837 838 return test_bit(bit_write, &tmp); 839 } 840 841 static void set_msr_interception_bitmap(struct kvm_vcpu *vcpu, u32 *msrpm, 842 u32 msr, int read, int write) 843 { 844 struct vcpu_svm *svm = to_svm(vcpu); 845 u8 bit_read, bit_write; 846 unsigned long tmp; 847 u32 offset; 848 849 /* 850 * If this warning triggers extend the direct_access_msrs list at the 851 * beginning of the file 852 */ 853 WARN_ON(!valid_msr_intercept(msr)); 854 855 /* Enforce non allowed MSRs to trap */ 856 if (read && !kvm_msr_allowed(vcpu, msr, KVM_MSR_FILTER_READ)) 857 read = 0; 858 859 if (write && !kvm_msr_allowed(vcpu, msr, KVM_MSR_FILTER_WRITE)) 860 write = 0; 861 862 offset = svm_msrpm_offset(msr); 863 bit_read = 2 * (msr & 0x0f); 864 bit_write = 2 * (msr & 0x0f) + 1; 865 tmp = msrpm[offset]; 866 867 BUG_ON(offset == MSR_INVALID); 868 869 read ? clear_bit(bit_read, &tmp) : set_bit(bit_read, &tmp); 870 write ? clear_bit(bit_write, &tmp) : set_bit(bit_write, &tmp); 871 872 msrpm[offset] = tmp; 873 874 svm_hv_vmcb_dirty_nested_enlightenments(vcpu); 875 svm->nested.force_msr_bitmap_recalc = true; 876 } 877 878 void set_msr_interception(struct kvm_vcpu *vcpu, u32 *msrpm, u32 msr, 879 int read, int write) 880 { 881 set_shadow_msr_intercept(vcpu, msr, read, write); 882 set_msr_interception_bitmap(vcpu, msrpm, msr, read, write); 883 } 884 885 u32 *svm_vcpu_alloc_msrpm(void) 886 { 887 unsigned int order = get_order(MSRPM_SIZE); 888 struct page *pages = alloc_pages(GFP_KERNEL_ACCOUNT, order); 889 u32 *msrpm; 890 891 if (!pages) 892 return NULL; 893 894 msrpm = page_address(pages); 895 memset(msrpm, 0xff, PAGE_SIZE * (1 << order)); 896 897 return msrpm; 898 } 899 900 void svm_vcpu_init_msrpm(struct kvm_vcpu *vcpu, u32 *msrpm) 901 { 902 int i; 903 904 for (i = 0; direct_access_msrs[i].index != MSR_INVALID; i++) { 905 if (!direct_access_msrs[i].always) 906 continue; 907 set_msr_interception(vcpu, msrpm, direct_access_msrs[i].index, 1, 1); 908 } 909 } 910 911 void svm_set_x2apic_msr_interception(struct vcpu_svm *svm, bool intercept) 912 { 913 int i; 914 915 if (intercept == svm->x2avic_msrs_intercepted) 916 return; 917 918 if (!x2avic_enabled) 919 return; 920 921 for (i = 0; i < MAX_DIRECT_ACCESS_MSRS; i++) { 922 int index = direct_access_msrs[i].index; 923 924 if ((index < APIC_BASE_MSR) || 925 (index > APIC_BASE_MSR + 0xff)) 926 continue; 927 set_msr_interception(&svm->vcpu, svm->msrpm, index, 928 !intercept, !intercept); 929 } 930 931 svm->x2avic_msrs_intercepted = intercept; 932 } 933 934 void svm_vcpu_free_msrpm(u32 *msrpm) 935 { 936 __free_pages(virt_to_page(msrpm), get_order(MSRPM_SIZE)); 937 } 938 939 static void svm_msr_filter_changed(struct kvm_vcpu *vcpu) 940 { 941 struct vcpu_svm *svm = to_svm(vcpu); 942 u32 i; 943 944 /* 945 * Set intercept permissions for all direct access MSRs again. They 946 * will automatically get filtered through the MSR filter, so we are 947 * back in sync after this. 948 */ 949 for (i = 0; direct_access_msrs[i].index != MSR_INVALID; i++) { 950 u32 msr = direct_access_msrs[i].index; 951 u32 read = test_bit(i, svm->shadow_msr_intercept.read); 952 u32 write = test_bit(i, svm->shadow_msr_intercept.write); 953 954 set_msr_interception_bitmap(vcpu, svm->msrpm, msr, read, write); 955 } 956 } 957 958 static void add_msr_offset(u32 offset) 959 { 960 int i; 961 962 for (i = 0; i < MSRPM_OFFSETS; ++i) { 963 964 /* Offset already in list? */ 965 if (msrpm_offsets[i] == offset) 966 return; 967 968 /* Slot used by another offset? */ 969 if (msrpm_offsets[i] != MSR_INVALID) 970 continue; 971 972 /* Add offset to list */ 973 msrpm_offsets[i] = offset; 974 975 return; 976 } 977 978 /* 979 * If this BUG triggers the msrpm_offsets table has an overflow. Just 980 * increase MSRPM_OFFSETS in this case. 981 */ 982 BUG(); 983 } 984 985 static void init_msrpm_offsets(void) 986 { 987 int i; 988 989 memset(msrpm_offsets, 0xff, sizeof(msrpm_offsets)); 990 991 for (i = 0; direct_access_msrs[i].index != MSR_INVALID; i++) { 992 u32 offset; 993 994 offset = svm_msrpm_offset(direct_access_msrs[i].index); 995 BUG_ON(offset == MSR_INVALID); 996 997 add_msr_offset(offset); 998 } 999 } 1000 1001 void svm_copy_lbrs(struct vmcb *to_vmcb, struct vmcb *from_vmcb) 1002 { 1003 to_vmcb->save.dbgctl = from_vmcb->save.dbgctl; 1004 to_vmcb->save.br_from = from_vmcb->save.br_from; 1005 to_vmcb->save.br_to = from_vmcb->save.br_to; 1006 to_vmcb->save.last_excp_from = from_vmcb->save.last_excp_from; 1007 to_vmcb->save.last_excp_to = from_vmcb->save.last_excp_to; 1008 1009 vmcb_mark_dirty(to_vmcb, VMCB_LBR); 1010 } 1011 1012 void svm_enable_lbrv(struct kvm_vcpu *vcpu) 1013 { 1014 struct vcpu_svm *svm = to_svm(vcpu); 1015 1016 svm->vmcb->control.virt_ext |= LBR_CTL_ENABLE_MASK; 1017 set_msr_interception(vcpu, svm->msrpm, MSR_IA32_LASTBRANCHFROMIP, 1, 1); 1018 set_msr_interception(vcpu, svm->msrpm, MSR_IA32_LASTBRANCHTOIP, 1, 1); 1019 set_msr_interception(vcpu, svm->msrpm, MSR_IA32_LASTINTFROMIP, 1, 1); 1020 set_msr_interception(vcpu, svm->msrpm, MSR_IA32_LASTINTTOIP, 1, 1); 1021 1022 if (sev_es_guest(vcpu->kvm)) 1023 set_msr_interception(vcpu, svm->msrpm, MSR_IA32_DEBUGCTLMSR, 1, 1); 1024 1025 /* Move the LBR msrs to the vmcb02 so that the guest can see them. */ 1026 if (is_guest_mode(vcpu)) 1027 svm_copy_lbrs(svm->vmcb, svm->vmcb01.ptr); 1028 } 1029 1030 static void svm_disable_lbrv(struct kvm_vcpu *vcpu) 1031 { 1032 struct vcpu_svm *svm = to_svm(vcpu); 1033 1034 KVM_BUG_ON(sev_es_guest(vcpu->kvm), vcpu->kvm); 1035 1036 svm->vmcb->control.virt_ext &= ~LBR_CTL_ENABLE_MASK; 1037 set_msr_interception(vcpu, svm->msrpm, MSR_IA32_LASTBRANCHFROMIP, 0, 0); 1038 set_msr_interception(vcpu, svm->msrpm, MSR_IA32_LASTBRANCHTOIP, 0, 0); 1039 set_msr_interception(vcpu, svm->msrpm, MSR_IA32_LASTINTFROMIP, 0, 0); 1040 set_msr_interception(vcpu, svm->msrpm, MSR_IA32_LASTINTTOIP, 0, 0); 1041 1042 /* 1043 * Move the LBR msrs back to the vmcb01 to avoid copying them 1044 * on nested guest entries. 1045 */ 1046 if (is_guest_mode(vcpu)) 1047 svm_copy_lbrs(svm->vmcb01.ptr, svm->vmcb); 1048 } 1049 1050 static struct vmcb *svm_get_lbr_vmcb(struct vcpu_svm *svm) 1051 { 1052 /* 1053 * If LBR virtualization is disabled, the LBR MSRs are always kept in 1054 * vmcb01. If LBR virtualization is enabled and L1 is running VMs of 1055 * its own, the MSRs are moved between vmcb01 and vmcb02 as needed. 1056 */ 1057 return svm->vmcb->control.virt_ext & LBR_CTL_ENABLE_MASK ? svm->vmcb : 1058 svm->vmcb01.ptr; 1059 } 1060 1061 void svm_update_lbrv(struct kvm_vcpu *vcpu) 1062 { 1063 struct vcpu_svm *svm = to_svm(vcpu); 1064 bool current_enable_lbrv = svm->vmcb->control.virt_ext & LBR_CTL_ENABLE_MASK; 1065 bool enable_lbrv = (svm_get_lbr_vmcb(svm)->save.dbgctl & DEBUGCTLMSR_LBR) || 1066 (is_guest_mode(vcpu) && guest_can_use(vcpu, X86_FEATURE_LBRV) && 1067 (svm->nested.ctl.virt_ext & LBR_CTL_ENABLE_MASK)); 1068 1069 if (enable_lbrv == current_enable_lbrv) 1070 return; 1071 1072 if (enable_lbrv) 1073 svm_enable_lbrv(vcpu); 1074 else 1075 svm_disable_lbrv(vcpu); 1076 } 1077 1078 void disable_nmi_singlestep(struct vcpu_svm *svm) 1079 { 1080 svm->nmi_singlestep = false; 1081 1082 if (!(svm->vcpu.guest_debug & KVM_GUESTDBG_SINGLESTEP)) { 1083 /* Clear our flags if they were not set by the guest */ 1084 if (!(svm->nmi_singlestep_guest_rflags & X86_EFLAGS_TF)) 1085 svm->vmcb->save.rflags &= ~X86_EFLAGS_TF; 1086 if (!(svm->nmi_singlestep_guest_rflags & X86_EFLAGS_RF)) 1087 svm->vmcb->save.rflags &= ~X86_EFLAGS_RF; 1088 } 1089 } 1090 1091 static void grow_ple_window(struct kvm_vcpu *vcpu) 1092 { 1093 struct vcpu_svm *svm = to_svm(vcpu); 1094 struct vmcb_control_area *control = &svm->vmcb->control; 1095 int old = control->pause_filter_count; 1096 1097 if (kvm_pause_in_guest(vcpu->kvm)) 1098 return; 1099 1100 control->pause_filter_count = __grow_ple_window(old, 1101 pause_filter_count, 1102 pause_filter_count_grow, 1103 pause_filter_count_max); 1104 1105 if (control->pause_filter_count != old) { 1106 vmcb_mark_dirty(svm->vmcb, VMCB_INTERCEPTS); 1107 trace_kvm_ple_window_update(vcpu->vcpu_id, 1108 control->pause_filter_count, old); 1109 } 1110 } 1111 1112 static void shrink_ple_window(struct kvm_vcpu *vcpu) 1113 { 1114 struct vcpu_svm *svm = to_svm(vcpu); 1115 struct vmcb_control_area *control = &svm->vmcb->control; 1116 int old = control->pause_filter_count; 1117 1118 if (kvm_pause_in_guest(vcpu->kvm)) 1119 return; 1120 1121 control->pause_filter_count = 1122 __shrink_ple_window(old, 1123 pause_filter_count, 1124 pause_filter_count_shrink, 1125 pause_filter_count); 1126 if (control->pause_filter_count != old) { 1127 vmcb_mark_dirty(svm->vmcb, VMCB_INTERCEPTS); 1128 trace_kvm_ple_window_update(vcpu->vcpu_id, 1129 control->pause_filter_count, old); 1130 } 1131 } 1132 1133 static void svm_hardware_unsetup(void) 1134 { 1135 int cpu; 1136 1137 sev_hardware_unsetup(); 1138 1139 for_each_possible_cpu(cpu) 1140 svm_cpu_uninit(cpu); 1141 1142 __free_pages(pfn_to_page(iopm_base >> PAGE_SHIFT), 1143 get_order(IOPM_SIZE)); 1144 iopm_base = 0; 1145 } 1146 1147 static void init_seg(struct vmcb_seg *seg) 1148 { 1149 seg->selector = 0; 1150 seg->attrib = SVM_SELECTOR_P_MASK | SVM_SELECTOR_S_MASK | 1151 SVM_SELECTOR_WRITE_MASK; /* Read/Write Data Segment */ 1152 seg->limit = 0xffff; 1153 seg->base = 0; 1154 } 1155 1156 static void init_sys_seg(struct vmcb_seg *seg, uint32_t type) 1157 { 1158 seg->selector = 0; 1159 seg->attrib = SVM_SELECTOR_P_MASK | type; 1160 seg->limit = 0xffff; 1161 seg->base = 0; 1162 } 1163 1164 static u64 svm_get_l2_tsc_offset(struct kvm_vcpu *vcpu) 1165 { 1166 struct vcpu_svm *svm = to_svm(vcpu); 1167 1168 return svm->nested.ctl.tsc_offset; 1169 } 1170 1171 static u64 svm_get_l2_tsc_multiplier(struct kvm_vcpu *vcpu) 1172 { 1173 struct vcpu_svm *svm = to_svm(vcpu); 1174 1175 return svm->tsc_ratio_msr; 1176 } 1177 1178 static void svm_write_tsc_offset(struct kvm_vcpu *vcpu) 1179 { 1180 struct vcpu_svm *svm = to_svm(vcpu); 1181 1182 svm->vmcb01.ptr->control.tsc_offset = vcpu->arch.l1_tsc_offset; 1183 svm->vmcb->control.tsc_offset = vcpu->arch.tsc_offset; 1184 vmcb_mark_dirty(svm->vmcb, VMCB_INTERCEPTS); 1185 } 1186 1187 void svm_write_tsc_multiplier(struct kvm_vcpu *vcpu) 1188 { 1189 preempt_disable(); 1190 if (to_svm(vcpu)->guest_state_loaded) 1191 __svm_write_tsc_multiplier(vcpu->arch.tsc_scaling_ratio); 1192 preempt_enable(); 1193 } 1194 1195 /* Evaluate instruction intercepts that depend on guest CPUID features. */ 1196 static void svm_recalc_instruction_intercepts(struct kvm_vcpu *vcpu, 1197 struct vcpu_svm *svm) 1198 { 1199 /* 1200 * Intercept INVPCID if shadow paging is enabled to sync/free shadow 1201 * roots, or if INVPCID is disabled in the guest to inject #UD. 1202 */ 1203 if (kvm_cpu_cap_has(X86_FEATURE_INVPCID)) { 1204 if (!npt_enabled || 1205 !guest_cpuid_has(&svm->vcpu, X86_FEATURE_INVPCID)) 1206 svm_set_intercept(svm, INTERCEPT_INVPCID); 1207 else 1208 svm_clr_intercept(svm, INTERCEPT_INVPCID); 1209 } 1210 1211 if (kvm_cpu_cap_has(X86_FEATURE_RDTSCP)) { 1212 if (guest_cpuid_has(vcpu, X86_FEATURE_RDTSCP)) 1213 svm_clr_intercept(svm, INTERCEPT_RDTSCP); 1214 else 1215 svm_set_intercept(svm, INTERCEPT_RDTSCP); 1216 } 1217 } 1218 1219 static inline void init_vmcb_after_set_cpuid(struct kvm_vcpu *vcpu) 1220 { 1221 struct vcpu_svm *svm = to_svm(vcpu); 1222 1223 if (guest_cpuid_is_intel(vcpu)) { 1224 /* 1225 * We must intercept SYSENTER_EIP and SYSENTER_ESP 1226 * accesses because the processor only stores 32 bits. 1227 * For the same reason we cannot use virtual VMLOAD/VMSAVE. 1228 */ 1229 svm_set_intercept(svm, INTERCEPT_VMLOAD); 1230 svm_set_intercept(svm, INTERCEPT_VMSAVE); 1231 svm->vmcb->control.virt_ext &= ~VIRTUAL_VMLOAD_VMSAVE_ENABLE_MASK; 1232 1233 set_msr_interception(vcpu, svm->msrpm, MSR_IA32_SYSENTER_EIP, 0, 0); 1234 set_msr_interception(vcpu, svm->msrpm, MSR_IA32_SYSENTER_ESP, 0, 0); 1235 } else { 1236 /* 1237 * If hardware supports Virtual VMLOAD VMSAVE then enable it 1238 * in VMCB and clear intercepts to avoid #VMEXIT. 1239 */ 1240 if (vls) { 1241 svm_clr_intercept(svm, INTERCEPT_VMLOAD); 1242 svm_clr_intercept(svm, INTERCEPT_VMSAVE); 1243 svm->vmcb->control.virt_ext |= VIRTUAL_VMLOAD_VMSAVE_ENABLE_MASK; 1244 } 1245 /* No need to intercept these MSRs */ 1246 set_msr_interception(vcpu, svm->msrpm, MSR_IA32_SYSENTER_EIP, 1, 1); 1247 set_msr_interception(vcpu, svm->msrpm, MSR_IA32_SYSENTER_ESP, 1, 1); 1248 } 1249 } 1250 1251 static void init_vmcb(struct kvm_vcpu *vcpu) 1252 { 1253 struct vcpu_svm *svm = to_svm(vcpu); 1254 struct vmcb *vmcb = svm->vmcb01.ptr; 1255 struct vmcb_control_area *control = &vmcb->control; 1256 struct vmcb_save_area *save = &vmcb->save; 1257 1258 svm_set_intercept(svm, INTERCEPT_CR0_READ); 1259 svm_set_intercept(svm, INTERCEPT_CR3_READ); 1260 svm_set_intercept(svm, INTERCEPT_CR4_READ); 1261 svm_set_intercept(svm, INTERCEPT_CR0_WRITE); 1262 svm_set_intercept(svm, INTERCEPT_CR3_WRITE); 1263 svm_set_intercept(svm, INTERCEPT_CR4_WRITE); 1264 if (!kvm_vcpu_apicv_active(vcpu)) 1265 svm_set_intercept(svm, INTERCEPT_CR8_WRITE); 1266 1267 set_dr_intercepts(svm); 1268 1269 set_exception_intercept(svm, PF_VECTOR); 1270 set_exception_intercept(svm, UD_VECTOR); 1271 set_exception_intercept(svm, MC_VECTOR); 1272 set_exception_intercept(svm, AC_VECTOR); 1273 set_exception_intercept(svm, DB_VECTOR); 1274 /* 1275 * Guest access to VMware backdoor ports could legitimately 1276 * trigger #GP because of TSS I/O permission bitmap. 1277 * We intercept those #GP and allow access to them anyway 1278 * as VMware does. 1279 */ 1280 if (enable_vmware_backdoor) 1281 set_exception_intercept(svm, GP_VECTOR); 1282 1283 svm_set_intercept(svm, INTERCEPT_INTR); 1284 svm_set_intercept(svm, INTERCEPT_NMI); 1285 1286 if (intercept_smi) 1287 svm_set_intercept(svm, INTERCEPT_SMI); 1288 1289 svm_set_intercept(svm, INTERCEPT_SELECTIVE_CR0); 1290 svm_set_intercept(svm, INTERCEPT_RDPMC); 1291 svm_set_intercept(svm, INTERCEPT_CPUID); 1292 svm_set_intercept(svm, INTERCEPT_INVD); 1293 svm_set_intercept(svm, INTERCEPT_INVLPG); 1294 svm_set_intercept(svm, INTERCEPT_INVLPGA); 1295 svm_set_intercept(svm, INTERCEPT_IOIO_PROT); 1296 svm_set_intercept(svm, INTERCEPT_MSR_PROT); 1297 svm_set_intercept(svm, INTERCEPT_TASK_SWITCH); 1298 svm_set_intercept(svm, INTERCEPT_SHUTDOWN); 1299 svm_set_intercept(svm, INTERCEPT_VMRUN); 1300 svm_set_intercept(svm, INTERCEPT_VMMCALL); 1301 svm_set_intercept(svm, INTERCEPT_VMLOAD); 1302 svm_set_intercept(svm, INTERCEPT_VMSAVE); 1303 svm_set_intercept(svm, INTERCEPT_STGI); 1304 svm_set_intercept(svm, INTERCEPT_CLGI); 1305 svm_set_intercept(svm, INTERCEPT_SKINIT); 1306 svm_set_intercept(svm, INTERCEPT_WBINVD); 1307 svm_set_intercept(svm, INTERCEPT_XSETBV); 1308 svm_set_intercept(svm, INTERCEPT_RDPRU); 1309 svm_set_intercept(svm, INTERCEPT_RSM); 1310 1311 if (!kvm_mwait_in_guest(vcpu->kvm)) { 1312 svm_set_intercept(svm, INTERCEPT_MONITOR); 1313 svm_set_intercept(svm, INTERCEPT_MWAIT); 1314 } 1315 1316 if (!kvm_hlt_in_guest(vcpu->kvm)) 1317 svm_set_intercept(svm, INTERCEPT_HLT); 1318 1319 control->iopm_base_pa = __sme_set(iopm_base); 1320 control->msrpm_base_pa = __sme_set(__pa(svm->msrpm)); 1321 control->int_ctl = V_INTR_MASKING_MASK; 1322 1323 init_seg(&save->es); 1324 init_seg(&save->ss); 1325 init_seg(&save->ds); 1326 init_seg(&save->fs); 1327 init_seg(&save->gs); 1328 1329 save->cs.selector = 0xf000; 1330 save->cs.base = 0xffff0000; 1331 /* Executable/Readable Code Segment */ 1332 save->cs.attrib = SVM_SELECTOR_READ_MASK | SVM_SELECTOR_P_MASK | 1333 SVM_SELECTOR_S_MASK | SVM_SELECTOR_CODE_MASK; 1334 save->cs.limit = 0xffff; 1335 1336 save->gdtr.base = 0; 1337 save->gdtr.limit = 0xffff; 1338 save->idtr.base = 0; 1339 save->idtr.limit = 0xffff; 1340 1341 init_sys_seg(&save->ldtr, SEG_TYPE_LDT); 1342 init_sys_seg(&save->tr, SEG_TYPE_BUSY_TSS16); 1343 1344 if (npt_enabled) { 1345 /* Setup VMCB for Nested Paging */ 1346 control->nested_ctl |= SVM_NESTED_CTL_NP_ENABLE; 1347 svm_clr_intercept(svm, INTERCEPT_INVLPG); 1348 clr_exception_intercept(svm, PF_VECTOR); 1349 svm_clr_intercept(svm, INTERCEPT_CR3_READ); 1350 svm_clr_intercept(svm, INTERCEPT_CR3_WRITE); 1351 save->g_pat = vcpu->arch.pat; 1352 save->cr3 = 0; 1353 } 1354 svm->current_vmcb->asid_generation = 0; 1355 svm->asid = 0; 1356 1357 svm->nested.vmcb12_gpa = INVALID_GPA; 1358 svm->nested.last_vmcb12_gpa = INVALID_GPA; 1359 1360 if (!kvm_pause_in_guest(vcpu->kvm)) { 1361 control->pause_filter_count = pause_filter_count; 1362 if (pause_filter_thresh) 1363 control->pause_filter_thresh = pause_filter_thresh; 1364 svm_set_intercept(svm, INTERCEPT_PAUSE); 1365 } else { 1366 svm_clr_intercept(svm, INTERCEPT_PAUSE); 1367 } 1368 1369 svm_recalc_instruction_intercepts(vcpu, svm); 1370 1371 /* 1372 * If the host supports V_SPEC_CTRL then disable the interception 1373 * of MSR_IA32_SPEC_CTRL. 1374 */ 1375 if (boot_cpu_has(X86_FEATURE_V_SPEC_CTRL)) 1376 set_msr_interception(vcpu, svm->msrpm, MSR_IA32_SPEC_CTRL, 1, 1); 1377 1378 if (kvm_vcpu_apicv_active(vcpu)) 1379 avic_init_vmcb(svm, vmcb); 1380 1381 if (vnmi) 1382 svm->vmcb->control.int_ctl |= V_NMI_ENABLE_MASK; 1383 1384 if (vgif) { 1385 svm_clr_intercept(svm, INTERCEPT_STGI); 1386 svm_clr_intercept(svm, INTERCEPT_CLGI); 1387 svm->vmcb->control.int_ctl |= V_GIF_ENABLE_MASK; 1388 } 1389 1390 if (sev_guest(vcpu->kvm)) 1391 sev_init_vmcb(svm); 1392 1393 svm_hv_init_vmcb(vmcb); 1394 init_vmcb_after_set_cpuid(vcpu); 1395 1396 vmcb_mark_all_dirty(vmcb); 1397 1398 enable_gif(svm); 1399 } 1400 1401 static void __svm_vcpu_reset(struct kvm_vcpu *vcpu) 1402 { 1403 struct vcpu_svm *svm = to_svm(vcpu); 1404 1405 svm_vcpu_init_msrpm(vcpu, svm->msrpm); 1406 1407 svm_init_osvw(vcpu); 1408 vcpu->arch.microcode_version = 0x01000065; 1409 svm->tsc_ratio_msr = kvm_caps.default_tsc_scaling_ratio; 1410 1411 svm->nmi_masked = false; 1412 svm->awaiting_iret_completion = false; 1413 1414 if (sev_es_guest(vcpu->kvm)) 1415 sev_es_vcpu_reset(svm); 1416 } 1417 1418 static void svm_vcpu_reset(struct kvm_vcpu *vcpu, bool init_event) 1419 { 1420 struct vcpu_svm *svm = to_svm(vcpu); 1421 1422 svm->spec_ctrl = 0; 1423 svm->virt_spec_ctrl = 0; 1424 1425 init_vmcb(vcpu); 1426 1427 if (!init_event) 1428 __svm_vcpu_reset(vcpu); 1429 } 1430 1431 void svm_switch_vmcb(struct vcpu_svm *svm, struct kvm_vmcb_info *target_vmcb) 1432 { 1433 svm->current_vmcb = target_vmcb; 1434 svm->vmcb = target_vmcb->ptr; 1435 } 1436 1437 static int svm_vcpu_create(struct kvm_vcpu *vcpu) 1438 { 1439 struct vcpu_svm *svm; 1440 struct page *vmcb01_page; 1441 struct page *vmsa_page = NULL; 1442 int err; 1443 1444 BUILD_BUG_ON(offsetof(struct vcpu_svm, vcpu) != 0); 1445 svm = to_svm(vcpu); 1446 1447 err = -ENOMEM; 1448 vmcb01_page = alloc_page(GFP_KERNEL_ACCOUNT | __GFP_ZERO); 1449 if (!vmcb01_page) 1450 goto out; 1451 1452 if (sev_es_guest(vcpu->kvm)) { 1453 /* 1454 * SEV-ES guests require a separate VMSA page used to contain 1455 * the encrypted register state of the guest. 1456 */ 1457 vmsa_page = alloc_page(GFP_KERNEL_ACCOUNT | __GFP_ZERO); 1458 if (!vmsa_page) 1459 goto error_free_vmcb_page; 1460 1461 /* 1462 * SEV-ES guests maintain an encrypted version of their FPU 1463 * state which is restored and saved on VMRUN and VMEXIT. 1464 * Mark vcpu->arch.guest_fpu->fpstate as scratch so it won't 1465 * do xsave/xrstor on it. 1466 */ 1467 fpstate_set_confidential(&vcpu->arch.guest_fpu); 1468 } 1469 1470 err = avic_init_vcpu(svm); 1471 if (err) 1472 goto error_free_vmsa_page; 1473 1474 svm->msrpm = svm_vcpu_alloc_msrpm(); 1475 if (!svm->msrpm) { 1476 err = -ENOMEM; 1477 goto error_free_vmsa_page; 1478 } 1479 1480 svm->x2avic_msrs_intercepted = true; 1481 1482 svm->vmcb01.ptr = page_address(vmcb01_page); 1483 svm->vmcb01.pa = __sme_set(page_to_pfn(vmcb01_page) << PAGE_SHIFT); 1484 svm_switch_vmcb(svm, &svm->vmcb01); 1485 1486 if (vmsa_page) 1487 svm->sev_es.vmsa = page_address(vmsa_page); 1488 1489 svm->guest_state_loaded = false; 1490 1491 return 0; 1492 1493 error_free_vmsa_page: 1494 if (vmsa_page) 1495 __free_page(vmsa_page); 1496 error_free_vmcb_page: 1497 __free_page(vmcb01_page); 1498 out: 1499 return err; 1500 } 1501 1502 static void svm_clear_current_vmcb(struct vmcb *vmcb) 1503 { 1504 int i; 1505 1506 for_each_online_cpu(i) 1507 cmpxchg(per_cpu_ptr(&svm_data.current_vmcb, i), vmcb, NULL); 1508 } 1509 1510 static void svm_vcpu_free(struct kvm_vcpu *vcpu) 1511 { 1512 struct vcpu_svm *svm = to_svm(vcpu); 1513 1514 /* 1515 * The vmcb page can be recycled, causing a false negative in 1516 * svm_vcpu_load(). So, ensure that no logical CPU has this 1517 * vmcb page recorded as its current vmcb. 1518 */ 1519 svm_clear_current_vmcb(svm->vmcb); 1520 1521 svm_leave_nested(vcpu); 1522 svm_free_nested(svm); 1523 1524 sev_free_vcpu(vcpu); 1525 1526 __free_page(pfn_to_page(__sme_clr(svm->vmcb01.pa) >> PAGE_SHIFT)); 1527 __free_pages(virt_to_page(svm->msrpm), get_order(MSRPM_SIZE)); 1528 } 1529 1530 static void svm_prepare_switch_to_guest(struct kvm_vcpu *vcpu) 1531 { 1532 struct vcpu_svm *svm = to_svm(vcpu); 1533 struct svm_cpu_data *sd = per_cpu_ptr(&svm_data, vcpu->cpu); 1534 1535 if (sev_es_guest(vcpu->kvm)) 1536 sev_es_unmap_ghcb(svm); 1537 1538 if (svm->guest_state_loaded) 1539 return; 1540 1541 /* 1542 * Save additional host state that will be restored on VMEXIT (sev-es) 1543 * or subsequent vmload of host save area. 1544 */ 1545 vmsave(sd->save_area_pa); 1546 if (sev_es_guest(vcpu->kvm)) { 1547 struct sev_es_save_area *hostsa; 1548 hostsa = (struct sev_es_save_area *)(page_address(sd->save_area) + 0x400); 1549 1550 sev_es_prepare_switch_to_guest(hostsa); 1551 } 1552 1553 if (tsc_scaling) 1554 __svm_write_tsc_multiplier(vcpu->arch.tsc_scaling_ratio); 1555 1556 /* 1557 * TSC_AUX is always virtualized for SEV-ES guests when the feature is 1558 * available. The user return MSR support is not required in this case 1559 * because TSC_AUX is restored on #VMEXIT from the host save area 1560 * (which has been initialized in svm_hardware_enable()). 1561 */ 1562 if (likely(tsc_aux_uret_slot >= 0) && 1563 (!boot_cpu_has(X86_FEATURE_V_TSC_AUX) || !sev_es_guest(vcpu->kvm))) 1564 kvm_set_user_return_msr(tsc_aux_uret_slot, svm->tsc_aux, -1ull); 1565 1566 svm->guest_state_loaded = true; 1567 } 1568 1569 static void svm_prepare_host_switch(struct kvm_vcpu *vcpu) 1570 { 1571 to_svm(vcpu)->guest_state_loaded = false; 1572 } 1573 1574 static void svm_vcpu_load(struct kvm_vcpu *vcpu, int cpu) 1575 { 1576 struct vcpu_svm *svm = to_svm(vcpu); 1577 struct svm_cpu_data *sd = per_cpu_ptr(&svm_data, cpu); 1578 1579 if (sd->current_vmcb != svm->vmcb) { 1580 sd->current_vmcb = svm->vmcb; 1581 1582 if (!cpu_feature_enabled(X86_FEATURE_IBPB_ON_VMEXIT)) 1583 indirect_branch_prediction_barrier(); 1584 } 1585 if (kvm_vcpu_apicv_active(vcpu)) 1586 avic_vcpu_load(vcpu, cpu); 1587 } 1588 1589 static void svm_vcpu_put(struct kvm_vcpu *vcpu) 1590 { 1591 if (kvm_vcpu_apicv_active(vcpu)) 1592 avic_vcpu_put(vcpu); 1593 1594 svm_prepare_host_switch(vcpu); 1595 1596 ++vcpu->stat.host_state_reload; 1597 } 1598 1599 static unsigned long svm_get_rflags(struct kvm_vcpu *vcpu) 1600 { 1601 struct vcpu_svm *svm = to_svm(vcpu); 1602 unsigned long rflags = svm->vmcb->save.rflags; 1603 1604 if (svm->nmi_singlestep) { 1605 /* Hide our flags if they were not set by the guest */ 1606 if (!(svm->nmi_singlestep_guest_rflags & X86_EFLAGS_TF)) 1607 rflags &= ~X86_EFLAGS_TF; 1608 if (!(svm->nmi_singlestep_guest_rflags & X86_EFLAGS_RF)) 1609 rflags &= ~X86_EFLAGS_RF; 1610 } 1611 return rflags; 1612 } 1613 1614 static void svm_set_rflags(struct kvm_vcpu *vcpu, unsigned long rflags) 1615 { 1616 if (to_svm(vcpu)->nmi_singlestep) 1617 rflags |= (X86_EFLAGS_TF | X86_EFLAGS_RF); 1618 1619 /* 1620 * Any change of EFLAGS.VM is accompanied by a reload of SS 1621 * (caused by either a task switch or an inter-privilege IRET), 1622 * so we do not need to update the CPL here. 1623 */ 1624 to_svm(vcpu)->vmcb->save.rflags = rflags; 1625 } 1626 1627 static bool svm_get_if_flag(struct kvm_vcpu *vcpu) 1628 { 1629 struct vmcb *vmcb = to_svm(vcpu)->vmcb; 1630 1631 return sev_es_guest(vcpu->kvm) 1632 ? vmcb->control.int_state & SVM_GUEST_INTERRUPT_MASK 1633 : kvm_get_rflags(vcpu) & X86_EFLAGS_IF; 1634 } 1635 1636 static void svm_cache_reg(struct kvm_vcpu *vcpu, enum kvm_reg reg) 1637 { 1638 kvm_register_mark_available(vcpu, reg); 1639 1640 switch (reg) { 1641 case VCPU_EXREG_PDPTR: 1642 /* 1643 * When !npt_enabled, mmu->pdptrs[] is already available since 1644 * it is always updated per SDM when moving to CRs. 1645 */ 1646 if (npt_enabled) 1647 load_pdptrs(vcpu, kvm_read_cr3(vcpu)); 1648 break; 1649 default: 1650 KVM_BUG_ON(1, vcpu->kvm); 1651 } 1652 } 1653 1654 static void svm_set_vintr(struct vcpu_svm *svm) 1655 { 1656 struct vmcb_control_area *control; 1657 1658 /* 1659 * The following fields are ignored when AVIC is enabled 1660 */ 1661 WARN_ON(kvm_vcpu_apicv_activated(&svm->vcpu)); 1662 1663 svm_set_intercept(svm, INTERCEPT_VINTR); 1664 1665 /* 1666 * Recalculating intercepts may have cleared the VINTR intercept. If 1667 * V_INTR_MASKING is enabled in vmcb12, then the effective RFLAGS.IF 1668 * for L1 physical interrupts is L1's RFLAGS.IF at the time of VMRUN. 1669 * Requesting an interrupt window if save.RFLAGS.IF=0 is pointless as 1670 * interrupts will never be unblocked while L2 is running. 1671 */ 1672 if (!svm_is_intercept(svm, INTERCEPT_VINTR)) 1673 return; 1674 1675 /* 1676 * This is just a dummy VINTR to actually cause a vmexit to happen. 1677 * Actual injection of virtual interrupts happens through EVENTINJ. 1678 */ 1679 control = &svm->vmcb->control; 1680 control->int_vector = 0x0; 1681 control->int_ctl &= ~V_INTR_PRIO_MASK; 1682 control->int_ctl |= V_IRQ_MASK | 1683 ((/*control->int_vector >> 4*/ 0xf) << V_INTR_PRIO_SHIFT); 1684 vmcb_mark_dirty(svm->vmcb, VMCB_INTR); 1685 } 1686 1687 static void svm_clear_vintr(struct vcpu_svm *svm) 1688 { 1689 svm_clr_intercept(svm, INTERCEPT_VINTR); 1690 1691 /* Drop int_ctl fields related to VINTR injection. */ 1692 svm->vmcb->control.int_ctl &= ~V_IRQ_INJECTION_BITS_MASK; 1693 if (is_guest_mode(&svm->vcpu)) { 1694 svm->vmcb01.ptr->control.int_ctl &= ~V_IRQ_INJECTION_BITS_MASK; 1695 1696 WARN_ON((svm->vmcb->control.int_ctl & V_TPR_MASK) != 1697 (svm->nested.ctl.int_ctl & V_TPR_MASK)); 1698 1699 svm->vmcb->control.int_ctl |= svm->nested.ctl.int_ctl & 1700 V_IRQ_INJECTION_BITS_MASK; 1701 1702 svm->vmcb->control.int_vector = svm->nested.ctl.int_vector; 1703 } 1704 1705 vmcb_mark_dirty(svm->vmcb, VMCB_INTR); 1706 } 1707 1708 static struct vmcb_seg *svm_seg(struct kvm_vcpu *vcpu, int seg) 1709 { 1710 struct vmcb_save_area *save = &to_svm(vcpu)->vmcb->save; 1711 struct vmcb_save_area *save01 = &to_svm(vcpu)->vmcb01.ptr->save; 1712 1713 switch (seg) { 1714 case VCPU_SREG_CS: return &save->cs; 1715 case VCPU_SREG_DS: return &save->ds; 1716 case VCPU_SREG_ES: return &save->es; 1717 case VCPU_SREG_FS: return &save01->fs; 1718 case VCPU_SREG_GS: return &save01->gs; 1719 case VCPU_SREG_SS: return &save->ss; 1720 case VCPU_SREG_TR: return &save01->tr; 1721 case VCPU_SREG_LDTR: return &save01->ldtr; 1722 } 1723 BUG(); 1724 return NULL; 1725 } 1726 1727 static u64 svm_get_segment_base(struct kvm_vcpu *vcpu, int seg) 1728 { 1729 struct vmcb_seg *s = svm_seg(vcpu, seg); 1730 1731 return s->base; 1732 } 1733 1734 static void svm_get_segment(struct kvm_vcpu *vcpu, 1735 struct kvm_segment *var, int seg) 1736 { 1737 struct vmcb_seg *s = svm_seg(vcpu, seg); 1738 1739 var->base = s->base; 1740 var->limit = s->limit; 1741 var->selector = s->selector; 1742 var->type = s->attrib & SVM_SELECTOR_TYPE_MASK; 1743 var->s = (s->attrib >> SVM_SELECTOR_S_SHIFT) & 1; 1744 var->dpl = (s->attrib >> SVM_SELECTOR_DPL_SHIFT) & 3; 1745 var->present = (s->attrib >> SVM_SELECTOR_P_SHIFT) & 1; 1746 var->avl = (s->attrib >> SVM_SELECTOR_AVL_SHIFT) & 1; 1747 var->l = (s->attrib >> SVM_SELECTOR_L_SHIFT) & 1; 1748 var->db = (s->attrib >> SVM_SELECTOR_DB_SHIFT) & 1; 1749 1750 /* 1751 * AMD CPUs circa 2014 track the G bit for all segments except CS. 1752 * However, the SVM spec states that the G bit is not observed by the 1753 * CPU, and some VMware virtual CPUs drop the G bit for all segments. 1754 * So let's synthesize a legal G bit for all segments, this helps 1755 * running KVM nested. It also helps cross-vendor migration, because 1756 * Intel's vmentry has a check on the 'G' bit. 1757 */ 1758 var->g = s->limit > 0xfffff; 1759 1760 /* 1761 * AMD's VMCB does not have an explicit unusable field, so emulate it 1762 * for cross vendor migration purposes by "not present" 1763 */ 1764 var->unusable = !var->present; 1765 1766 switch (seg) { 1767 case VCPU_SREG_TR: 1768 /* 1769 * Work around a bug where the busy flag in the tr selector 1770 * isn't exposed 1771 */ 1772 var->type |= 0x2; 1773 break; 1774 case VCPU_SREG_DS: 1775 case VCPU_SREG_ES: 1776 case VCPU_SREG_FS: 1777 case VCPU_SREG_GS: 1778 /* 1779 * The accessed bit must always be set in the segment 1780 * descriptor cache, although it can be cleared in the 1781 * descriptor, the cached bit always remains at 1. Since 1782 * Intel has a check on this, set it here to support 1783 * cross-vendor migration. 1784 */ 1785 if (!var->unusable) 1786 var->type |= 0x1; 1787 break; 1788 case VCPU_SREG_SS: 1789 /* 1790 * On AMD CPUs sometimes the DB bit in the segment 1791 * descriptor is left as 1, although the whole segment has 1792 * been made unusable. Clear it here to pass an Intel VMX 1793 * entry check when cross vendor migrating. 1794 */ 1795 if (var->unusable) 1796 var->db = 0; 1797 /* This is symmetric with svm_set_segment() */ 1798 var->dpl = to_svm(vcpu)->vmcb->save.cpl; 1799 break; 1800 } 1801 } 1802 1803 static int svm_get_cpl(struct kvm_vcpu *vcpu) 1804 { 1805 struct vmcb_save_area *save = &to_svm(vcpu)->vmcb->save; 1806 1807 return save->cpl; 1808 } 1809 1810 static void svm_get_cs_db_l_bits(struct kvm_vcpu *vcpu, int *db, int *l) 1811 { 1812 struct kvm_segment cs; 1813 1814 svm_get_segment(vcpu, &cs, VCPU_SREG_CS); 1815 *db = cs.db; 1816 *l = cs.l; 1817 } 1818 1819 static void svm_get_idt(struct kvm_vcpu *vcpu, struct desc_ptr *dt) 1820 { 1821 struct vcpu_svm *svm = to_svm(vcpu); 1822 1823 dt->size = svm->vmcb->save.idtr.limit; 1824 dt->address = svm->vmcb->save.idtr.base; 1825 } 1826 1827 static void svm_set_idt(struct kvm_vcpu *vcpu, struct desc_ptr *dt) 1828 { 1829 struct vcpu_svm *svm = to_svm(vcpu); 1830 1831 svm->vmcb->save.idtr.limit = dt->size; 1832 svm->vmcb->save.idtr.base = dt->address ; 1833 vmcb_mark_dirty(svm->vmcb, VMCB_DT); 1834 } 1835 1836 static void svm_get_gdt(struct kvm_vcpu *vcpu, struct desc_ptr *dt) 1837 { 1838 struct vcpu_svm *svm = to_svm(vcpu); 1839 1840 dt->size = svm->vmcb->save.gdtr.limit; 1841 dt->address = svm->vmcb->save.gdtr.base; 1842 } 1843 1844 static void svm_set_gdt(struct kvm_vcpu *vcpu, struct desc_ptr *dt) 1845 { 1846 struct vcpu_svm *svm = to_svm(vcpu); 1847 1848 svm->vmcb->save.gdtr.limit = dt->size; 1849 svm->vmcb->save.gdtr.base = dt->address ; 1850 vmcb_mark_dirty(svm->vmcb, VMCB_DT); 1851 } 1852 1853 static void sev_post_set_cr3(struct kvm_vcpu *vcpu, unsigned long cr3) 1854 { 1855 struct vcpu_svm *svm = to_svm(vcpu); 1856 1857 /* 1858 * For guests that don't set guest_state_protected, the cr3 update is 1859 * handled via kvm_mmu_load() while entering the guest. For guests 1860 * that do (SEV-ES/SEV-SNP), the cr3 update needs to be written to 1861 * VMCB save area now, since the save area will become the initial 1862 * contents of the VMSA, and future VMCB save area updates won't be 1863 * seen. 1864 */ 1865 if (sev_es_guest(vcpu->kvm)) { 1866 svm->vmcb->save.cr3 = cr3; 1867 vmcb_mark_dirty(svm->vmcb, VMCB_CR); 1868 } 1869 } 1870 1871 static bool svm_is_valid_cr0(struct kvm_vcpu *vcpu, unsigned long cr0) 1872 { 1873 return true; 1874 } 1875 1876 void svm_set_cr0(struct kvm_vcpu *vcpu, unsigned long cr0) 1877 { 1878 struct vcpu_svm *svm = to_svm(vcpu); 1879 u64 hcr0 = cr0; 1880 bool old_paging = is_paging(vcpu); 1881 1882 #ifdef CONFIG_X86_64 1883 if (vcpu->arch.efer & EFER_LME) { 1884 if (!is_paging(vcpu) && (cr0 & X86_CR0_PG)) { 1885 vcpu->arch.efer |= EFER_LMA; 1886 if (!vcpu->arch.guest_state_protected) 1887 svm->vmcb->save.efer |= EFER_LMA | EFER_LME; 1888 } 1889 1890 if (is_paging(vcpu) && !(cr0 & X86_CR0_PG)) { 1891 vcpu->arch.efer &= ~EFER_LMA; 1892 if (!vcpu->arch.guest_state_protected) 1893 svm->vmcb->save.efer &= ~(EFER_LMA | EFER_LME); 1894 } 1895 } 1896 #endif 1897 vcpu->arch.cr0 = cr0; 1898 1899 if (!npt_enabled) { 1900 hcr0 |= X86_CR0_PG | X86_CR0_WP; 1901 if (old_paging != is_paging(vcpu)) 1902 svm_set_cr4(vcpu, kvm_read_cr4(vcpu)); 1903 } 1904 1905 /* 1906 * re-enable caching here because the QEMU bios 1907 * does not do it - this results in some delay at 1908 * reboot 1909 */ 1910 if (kvm_check_has_quirk(vcpu->kvm, KVM_X86_QUIRK_CD_NW_CLEARED)) 1911 hcr0 &= ~(X86_CR0_CD | X86_CR0_NW); 1912 1913 svm->vmcb->save.cr0 = hcr0; 1914 vmcb_mark_dirty(svm->vmcb, VMCB_CR); 1915 1916 /* 1917 * SEV-ES guests must always keep the CR intercepts cleared. CR 1918 * tracking is done using the CR write traps. 1919 */ 1920 if (sev_es_guest(vcpu->kvm)) 1921 return; 1922 1923 if (hcr0 == cr0) { 1924 /* Selective CR0 write remains on. */ 1925 svm_clr_intercept(svm, INTERCEPT_CR0_READ); 1926 svm_clr_intercept(svm, INTERCEPT_CR0_WRITE); 1927 } else { 1928 svm_set_intercept(svm, INTERCEPT_CR0_READ); 1929 svm_set_intercept(svm, INTERCEPT_CR0_WRITE); 1930 } 1931 } 1932 1933 static bool svm_is_valid_cr4(struct kvm_vcpu *vcpu, unsigned long cr4) 1934 { 1935 return true; 1936 } 1937 1938 void svm_set_cr4(struct kvm_vcpu *vcpu, unsigned long cr4) 1939 { 1940 unsigned long host_cr4_mce = cr4_read_shadow() & X86_CR4_MCE; 1941 unsigned long old_cr4 = vcpu->arch.cr4; 1942 1943 if (npt_enabled && ((old_cr4 ^ cr4) & X86_CR4_PGE)) 1944 svm_flush_tlb_current(vcpu); 1945 1946 vcpu->arch.cr4 = cr4; 1947 if (!npt_enabled) { 1948 cr4 |= X86_CR4_PAE; 1949 1950 if (!is_paging(vcpu)) 1951 cr4 &= ~(X86_CR4_SMEP | X86_CR4_SMAP | X86_CR4_PKE); 1952 } 1953 cr4 |= host_cr4_mce; 1954 to_svm(vcpu)->vmcb->save.cr4 = cr4; 1955 vmcb_mark_dirty(to_svm(vcpu)->vmcb, VMCB_CR); 1956 1957 if ((cr4 ^ old_cr4) & (X86_CR4_OSXSAVE | X86_CR4_PKE)) 1958 kvm_update_cpuid_runtime(vcpu); 1959 } 1960 1961 static void svm_set_segment(struct kvm_vcpu *vcpu, 1962 struct kvm_segment *var, int seg) 1963 { 1964 struct vcpu_svm *svm = to_svm(vcpu); 1965 struct vmcb_seg *s = svm_seg(vcpu, seg); 1966 1967 s->base = var->base; 1968 s->limit = var->limit; 1969 s->selector = var->selector; 1970 s->attrib = (var->type & SVM_SELECTOR_TYPE_MASK); 1971 s->attrib |= (var->s & 1) << SVM_SELECTOR_S_SHIFT; 1972 s->attrib |= (var->dpl & 3) << SVM_SELECTOR_DPL_SHIFT; 1973 s->attrib |= ((var->present & 1) && !var->unusable) << SVM_SELECTOR_P_SHIFT; 1974 s->attrib |= (var->avl & 1) << SVM_SELECTOR_AVL_SHIFT; 1975 s->attrib |= (var->l & 1) << SVM_SELECTOR_L_SHIFT; 1976 s->attrib |= (var->db & 1) << SVM_SELECTOR_DB_SHIFT; 1977 s->attrib |= (var->g & 1) << SVM_SELECTOR_G_SHIFT; 1978 1979 /* 1980 * This is always accurate, except if SYSRET returned to a segment 1981 * with SS.DPL != 3. Intel does not have this quirk, and always 1982 * forces SS.DPL to 3 on sysret, so we ignore that case; fixing it 1983 * would entail passing the CPL to userspace and back. 1984 */ 1985 if (seg == VCPU_SREG_SS) 1986 /* This is symmetric with svm_get_segment() */ 1987 svm->vmcb->save.cpl = (var->dpl & 3); 1988 1989 vmcb_mark_dirty(svm->vmcb, VMCB_SEG); 1990 } 1991 1992 static void svm_update_exception_bitmap(struct kvm_vcpu *vcpu) 1993 { 1994 struct vcpu_svm *svm = to_svm(vcpu); 1995 1996 clr_exception_intercept(svm, BP_VECTOR); 1997 1998 if (vcpu->guest_debug & KVM_GUESTDBG_ENABLE) { 1999 if (vcpu->guest_debug & KVM_GUESTDBG_USE_SW_BP) 2000 set_exception_intercept(svm, BP_VECTOR); 2001 } 2002 } 2003 2004 static void new_asid(struct vcpu_svm *svm, struct svm_cpu_data *sd) 2005 { 2006 if (sd->next_asid > sd->max_asid) { 2007 ++sd->asid_generation; 2008 sd->next_asid = sd->min_asid; 2009 svm->vmcb->control.tlb_ctl = TLB_CONTROL_FLUSH_ALL_ASID; 2010 vmcb_mark_dirty(svm->vmcb, VMCB_ASID); 2011 } 2012 2013 svm->current_vmcb->asid_generation = sd->asid_generation; 2014 svm->asid = sd->next_asid++; 2015 } 2016 2017 static void svm_set_dr6(struct vcpu_svm *svm, unsigned long value) 2018 { 2019 struct vmcb *vmcb = svm->vmcb; 2020 2021 if (svm->vcpu.arch.guest_state_protected) 2022 return; 2023 2024 if (unlikely(value != vmcb->save.dr6)) { 2025 vmcb->save.dr6 = value; 2026 vmcb_mark_dirty(vmcb, VMCB_DR); 2027 } 2028 } 2029 2030 static void svm_sync_dirty_debug_regs(struct kvm_vcpu *vcpu) 2031 { 2032 struct vcpu_svm *svm = to_svm(vcpu); 2033 2034 if (WARN_ON_ONCE(sev_es_guest(vcpu->kvm))) 2035 return; 2036 2037 get_debugreg(vcpu->arch.db[0], 0); 2038 get_debugreg(vcpu->arch.db[1], 1); 2039 get_debugreg(vcpu->arch.db[2], 2); 2040 get_debugreg(vcpu->arch.db[3], 3); 2041 /* 2042 * We cannot reset svm->vmcb->save.dr6 to DR6_ACTIVE_LOW here, 2043 * because db_interception might need it. We can do it before vmentry. 2044 */ 2045 vcpu->arch.dr6 = svm->vmcb->save.dr6; 2046 vcpu->arch.dr7 = svm->vmcb->save.dr7; 2047 vcpu->arch.switch_db_regs &= ~KVM_DEBUGREG_WONT_EXIT; 2048 set_dr_intercepts(svm); 2049 } 2050 2051 static void svm_set_dr7(struct kvm_vcpu *vcpu, unsigned long value) 2052 { 2053 struct vcpu_svm *svm = to_svm(vcpu); 2054 2055 if (vcpu->arch.guest_state_protected) 2056 return; 2057 2058 svm->vmcb->save.dr7 = value; 2059 vmcb_mark_dirty(svm->vmcb, VMCB_DR); 2060 } 2061 2062 static int pf_interception(struct kvm_vcpu *vcpu) 2063 { 2064 struct vcpu_svm *svm = to_svm(vcpu); 2065 2066 u64 fault_address = svm->vmcb->control.exit_info_2; 2067 u64 error_code = svm->vmcb->control.exit_info_1; 2068 2069 return kvm_handle_page_fault(vcpu, error_code, fault_address, 2070 static_cpu_has(X86_FEATURE_DECODEASSISTS) ? 2071 svm->vmcb->control.insn_bytes : NULL, 2072 svm->vmcb->control.insn_len); 2073 } 2074 2075 static int npf_interception(struct kvm_vcpu *vcpu) 2076 { 2077 struct vcpu_svm *svm = to_svm(vcpu); 2078 2079 u64 fault_address = svm->vmcb->control.exit_info_2; 2080 u64 error_code = svm->vmcb->control.exit_info_1; 2081 2082 trace_kvm_page_fault(vcpu, fault_address, error_code); 2083 return kvm_mmu_page_fault(vcpu, fault_address, error_code, 2084 static_cpu_has(X86_FEATURE_DECODEASSISTS) ? 2085 svm->vmcb->control.insn_bytes : NULL, 2086 svm->vmcb->control.insn_len); 2087 } 2088 2089 static int db_interception(struct kvm_vcpu *vcpu) 2090 { 2091 struct kvm_run *kvm_run = vcpu->run; 2092 struct vcpu_svm *svm = to_svm(vcpu); 2093 2094 if (!(vcpu->guest_debug & 2095 (KVM_GUESTDBG_SINGLESTEP | KVM_GUESTDBG_USE_HW_BP)) && 2096 !svm->nmi_singlestep) { 2097 u32 payload = svm->vmcb->save.dr6 ^ DR6_ACTIVE_LOW; 2098 kvm_queue_exception_p(vcpu, DB_VECTOR, payload); 2099 return 1; 2100 } 2101 2102 if (svm->nmi_singlestep) { 2103 disable_nmi_singlestep(svm); 2104 /* Make sure we check for pending NMIs upon entry */ 2105 kvm_make_request(KVM_REQ_EVENT, vcpu); 2106 } 2107 2108 if (vcpu->guest_debug & 2109 (KVM_GUESTDBG_SINGLESTEP | KVM_GUESTDBG_USE_HW_BP)) { 2110 kvm_run->exit_reason = KVM_EXIT_DEBUG; 2111 kvm_run->debug.arch.dr6 = svm->vmcb->save.dr6; 2112 kvm_run->debug.arch.dr7 = svm->vmcb->save.dr7; 2113 kvm_run->debug.arch.pc = 2114 svm->vmcb->save.cs.base + svm->vmcb->save.rip; 2115 kvm_run->debug.arch.exception = DB_VECTOR; 2116 return 0; 2117 } 2118 2119 return 1; 2120 } 2121 2122 static int bp_interception(struct kvm_vcpu *vcpu) 2123 { 2124 struct vcpu_svm *svm = to_svm(vcpu); 2125 struct kvm_run *kvm_run = vcpu->run; 2126 2127 kvm_run->exit_reason = KVM_EXIT_DEBUG; 2128 kvm_run->debug.arch.pc = svm->vmcb->save.cs.base + svm->vmcb->save.rip; 2129 kvm_run->debug.arch.exception = BP_VECTOR; 2130 return 0; 2131 } 2132 2133 static int ud_interception(struct kvm_vcpu *vcpu) 2134 { 2135 return handle_ud(vcpu); 2136 } 2137 2138 static int ac_interception(struct kvm_vcpu *vcpu) 2139 { 2140 kvm_queue_exception_e(vcpu, AC_VECTOR, 0); 2141 return 1; 2142 } 2143 2144 static bool is_erratum_383(void) 2145 { 2146 int err, i; 2147 u64 value; 2148 2149 if (!erratum_383_found) 2150 return false; 2151 2152 value = native_read_msr_safe(MSR_IA32_MC0_STATUS, &err); 2153 if (err) 2154 return false; 2155 2156 /* Bit 62 may or may not be set for this mce */ 2157 value &= ~(1ULL << 62); 2158 2159 if (value != 0xb600000000010015ULL) 2160 return false; 2161 2162 /* Clear MCi_STATUS registers */ 2163 for (i = 0; i < 6; ++i) 2164 native_write_msr_safe(MSR_IA32_MCx_STATUS(i), 0, 0); 2165 2166 value = native_read_msr_safe(MSR_IA32_MCG_STATUS, &err); 2167 if (!err) { 2168 u32 low, high; 2169 2170 value &= ~(1ULL << 2); 2171 low = lower_32_bits(value); 2172 high = upper_32_bits(value); 2173 2174 native_write_msr_safe(MSR_IA32_MCG_STATUS, low, high); 2175 } 2176 2177 /* Flush tlb to evict multi-match entries */ 2178 __flush_tlb_all(); 2179 2180 return true; 2181 } 2182 2183 static void svm_handle_mce(struct kvm_vcpu *vcpu) 2184 { 2185 if (is_erratum_383()) { 2186 /* 2187 * Erratum 383 triggered. Guest state is corrupt so kill the 2188 * guest. 2189 */ 2190 pr_err("Guest triggered AMD Erratum 383\n"); 2191 2192 kvm_make_request(KVM_REQ_TRIPLE_FAULT, vcpu); 2193 2194 return; 2195 } 2196 2197 /* 2198 * On an #MC intercept the MCE handler is not called automatically in 2199 * the host. So do it by hand here. 2200 */ 2201 kvm_machine_check(); 2202 } 2203 2204 static int mc_interception(struct kvm_vcpu *vcpu) 2205 { 2206 return 1; 2207 } 2208 2209 static int shutdown_interception(struct kvm_vcpu *vcpu) 2210 { 2211 struct kvm_run *kvm_run = vcpu->run; 2212 struct vcpu_svm *svm = to_svm(vcpu); 2213 2214 /* 2215 * The VM save area has already been encrypted so it 2216 * cannot be reinitialized - just terminate. 2217 */ 2218 if (sev_es_guest(vcpu->kvm)) 2219 return -EINVAL; 2220 2221 /* 2222 * VMCB is undefined after a SHUTDOWN intercept. INIT the vCPU to put 2223 * the VMCB in a known good state. Unfortuately, KVM doesn't have 2224 * KVM_MP_STATE_SHUTDOWN and can't add it without potentially breaking 2225 * userspace. At a platform view, INIT is acceptable behavior as 2226 * there exist bare metal platforms that automatically INIT the CPU 2227 * in response to shutdown. 2228 */ 2229 clear_page(svm->vmcb); 2230 kvm_vcpu_reset(vcpu, true); 2231 2232 kvm_run->exit_reason = KVM_EXIT_SHUTDOWN; 2233 return 0; 2234 } 2235 2236 static int io_interception(struct kvm_vcpu *vcpu) 2237 { 2238 struct vcpu_svm *svm = to_svm(vcpu); 2239 u32 io_info = svm->vmcb->control.exit_info_1; /* address size bug? */ 2240 int size, in, string; 2241 unsigned port; 2242 2243 ++vcpu->stat.io_exits; 2244 string = (io_info & SVM_IOIO_STR_MASK) != 0; 2245 in = (io_info & SVM_IOIO_TYPE_MASK) != 0; 2246 port = io_info >> 16; 2247 size = (io_info & SVM_IOIO_SIZE_MASK) >> SVM_IOIO_SIZE_SHIFT; 2248 2249 if (string) { 2250 if (sev_es_guest(vcpu->kvm)) 2251 return sev_es_string_io(svm, size, port, in); 2252 else 2253 return kvm_emulate_instruction(vcpu, 0); 2254 } 2255 2256 svm->next_rip = svm->vmcb->control.exit_info_2; 2257 2258 return kvm_fast_pio(vcpu, size, port, in); 2259 } 2260 2261 static int nmi_interception(struct kvm_vcpu *vcpu) 2262 { 2263 return 1; 2264 } 2265 2266 static int smi_interception(struct kvm_vcpu *vcpu) 2267 { 2268 return 1; 2269 } 2270 2271 static int intr_interception(struct kvm_vcpu *vcpu) 2272 { 2273 ++vcpu->stat.irq_exits; 2274 return 1; 2275 } 2276 2277 static int vmload_vmsave_interception(struct kvm_vcpu *vcpu, bool vmload) 2278 { 2279 struct vcpu_svm *svm = to_svm(vcpu); 2280 struct vmcb *vmcb12; 2281 struct kvm_host_map map; 2282 int ret; 2283 2284 if (nested_svm_check_permissions(vcpu)) 2285 return 1; 2286 2287 ret = kvm_vcpu_map(vcpu, gpa_to_gfn(svm->vmcb->save.rax), &map); 2288 if (ret) { 2289 if (ret == -EINVAL) 2290 kvm_inject_gp(vcpu, 0); 2291 return 1; 2292 } 2293 2294 vmcb12 = map.hva; 2295 2296 ret = kvm_skip_emulated_instruction(vcpu); 2297 2298 if (vmload) { 2299 svm_copy_vmloadsave_state(svm->vmcb, vmcb12); 2300 svm->sysenter_eip_hi = 0; 2301 svm->sysenter_esp_hi = 0; 2302 } else { 2303 svm_copy_vmloadsave_state(vmcb12, svm->vmcb); 2304 } 2305 2306 kvm_vcpu_unmap(vcpu, &map, true); 2307 2308 return ret; 2309 } 2310 2311 static int vmload_interception(struct kvm_vcpu *vcpu) 2312 { 2313 return vmload_vmsave_interception(vcpu, true); 2314 } 2315 2316 static int vmsave_interception(struct kvm_vcpu *vcpu) 2317 { 2318 return vmload_vmsave_interception(vcpu, false); 2319 } 2320 2321 static int vmrun_interception(struct kvm_vcpu *vcpu) 2322 { 2323 if (nested_svm_check_permissions(vcpu)) 2324 return 1; 2325 2326 return nested_svm_vmrun(vcpu); 2327 } 2328 2329 enum { 2330 NONE_SVM_INSTR, 2331 SVM_INSTR_VMRUN, 2332 SVM_INSTR_VMLOAD, 2333 SVM_INSTR_VMSAVE, 2334 }; 2335 2336 /* Return NONE_SVM_INSTR if not SVM instrs, otherwise return decode result */ 2337 static int svm_instr_opcode(struct kvm_vcpu *vcpu) 2338 { 2339 struct x86_emulate_ctxt *ctxt = vcpu->arch.emulate_ctxt; 2340 2341 if (ctxt->b != 0x1 || ctxt->opcode_len != 2) 2342 return NONE_SVM_INSTR; 2343 2344 switch (ctxt->modrm) { 2345 case 0xd8: /* VMRUN */ 2346 return SVM_INSTR_VMRUN; 2347 case 0xda: /* VMLOAD */ 2348 return SVM_INSTR_VMLOAD; 2349 case 0xdb: /* VMSAVE */ 2350 return SVM_INSTR_VMSAVE; 2351 default: 2352 break; 2353 } 2354 2355 return NONE_SVM_INSTR; 2356 } 2357 2358 static int emulate_svm_instr(struct kvm_vcpu *vcpu, int opcode) 2359 { 2360 const int guest_mode_exit_codes[] = { 2361 [SVM_INSTR_VMRUN] = SVM_EXIT_VMRUN, 2362 [SVM_INSTR_VMLOAD] = SVM_EXIT_VMLOAD, 2363 [SVM_INSTR_VMSAVE] = SVM_EXIT_VMSAVE, 2364 }; 2365 int (*const svm_instr_handlers[])(struct kvm_vcpu *vcpu) = { 2366 [SVM_INSTR_VMRUN] = vmrun_interception, 2367 [SVM_INSTR_VMLOAD] = vmload_interception, 2368 [SVM_INSTR_VMSAVE] = vmsave_interception, 2369 }; 2370 struct vcpu_svm *svm = to_svm(vcpu); 2371 int ret; 2372 2373 if (is_guest_mode(vcpu)) { 2374 /* Returns '1' or -errno on failure, '0' on success. */ 2375 ret = nested_svm_simple_vmexit(svm, guest_mode_exit_codes[opcode]); 2376 if (ret) 2377 return ret; 2378 return 1; 2379 } 2380 return svm_instr_handlers[opcode](vcpu); 2381 } 2382 2383 /* 2384 * #GP handling code. Note that #GP can be triggered under the following two 2385 * cases: 2386 * 1) SVM VM-related instructions (VMRUN/VMSAVE/VMLOAD) that trigger #GP on 2387 * some AMD CPUs when EAX of these instructions are in the reserved memory 2388 * regions (e.g. SMM memory on host). 2389 * 2) VMware backdoor 2390 */ 2391 static int gp_interception(struct kvm_vcpu *vcpu) 2392 { 2393 struct vcpu_svm *svm = to_svm(vcpu); 2394 u32 error_code = svm->vmcb->control.exit_info_1; 2395 int opcode; 2396 2397 /* Both #GP cases have zero error_code */ 2398 if (error_code) 2399 goto reinject; 2400 2401 /* Decode the instruction for usage later */ 2402 if (x86_decode_emulated_instruction(vcpu, 0, NULL, 0) != EMULATION_OK) 2403 goto reinject; 2404 2405 opcode = svm_instr_opcode(vcpu); 2406 2407 if (opcode == NONE_SVM_INSTR) { 2408 if (!enable_vmware_backdoor) 2409 goto reinject; 2410 2411 /* 2412 * VMware backdoor emulation on #GP interception only handles 2413 * IN{S}, OUT{S}, and RDPMC. 2414 */ 2415 if (!is_guest_mode(vcpu)) 2416 return kvm_emulate_instruction(vcpu, 2417 EMULTYPE_VMWARE_GP | EMULTYPE_NO_DECODE); 2418 } else { 2419 /* All SVM instructions expect page aligned RAX */ 2420 if (svm->vmcb->save.rax & ~PAGE_MASK) 2421 goto reinject; 2422 2423 return emulate_svm_instr(vcpu, opcode); 2424 } 2425 2426 reinject: 2427 kvm_queue_exception_e(vcpu, GP_VECTOR, error_code); 2428 return 1; 2429 } 2430 2431 void svm_set_gif(struct vcpu_svm *svm, bool value) 2432 { 2433 if (value) { 2434 /* 2435 * If VGIF is enabled, the STGI intercept is only added to 2436 * detect the opening of the SMI/NMI window; remove it now. 2437 * Likewise, clear the VINTR intercept, we will set it 2438 * again while processing KVM_REQ_EVENT if needed. 2439 */ 2440 if (vgif) 2441 svm_clr_intercept(svm, INTERCEPT_STGI); 2442 if (svm_is_intercept(svm, INTERCEPT_VINTR)) 2443 svm_clear_vintr(svm); 2444 2445 enable_gif(svm); 2446 if (svm->vcpu.arch.smi_pending || 2447 svm->vcpu.arch.nmi_pending || 2448 kvm_cpu_has_injectable_intr(&svm->vcpu) || 2449 kvm_apic_has_pending_init_or_sipi(&svm->vcpu)) 2450 kvm_make_request(KVM_REQ_EVENT, &svm->vcpu); 2451 } else { 2452 disable_gif(svm); 2453 2454 /* 2455 * After a CLGI no interrupts should come. But if vGIF is 2456 * in use, we still rely on the VINTR intercept (rather than 2457 * STGI) to detect an open interrupt window. 2458 */ 2459 if (!vgif) 2460 svm_clear_vintr(svm); 2461 } 2462 } 2463 2464 static int stgi_interception(struct kvm_vcpu *vcpu) 2465 { 2466 int ret; 2467 2468 if (nested_svm_check_permissions(vcpu)) 2469 return 1; 2470 2471 ret = kvm_skip_emulated_instruction(vcpu); 2472 svm_set_gif(to_svm(vcpu), true); 2473 return ret; 2474 } 2475 2476 static int clgi_interception(struct kvm_vcpu *vcpu) 2477 { 2478 int ret; 2479 2480 if (nested_svm_check_permissions(vcpu)) 2481 return 1; 2482 2483 ret = kvm_skip_emulated_instruction(vcpu); 2484 svm_set_gif(to_svm(vcpu), false); 2485 return ret; 2486 } 2487 2488 static int invlpga_interception(struct kvm_vcpu *vcpu) 2489 { 2490 gva_t gva = kvm_rax_read(vcpu); 2491 u32 asid = kvm_rcx_read(vcpu); 2492 2493 /* FIXME: Handle an address size prefix. */ 2494 if (!is_long_mode(vcpu)) 2495 gva = (u32)gva; 2496 2497 trace_kvm_invlpga(to_svm(vcpu)->vmcb->save.rip, asid, gva); 2498 2499 /* Let's treat INVLPGA the same as INVLPG (can be optimized!) */ 2500 kvm_mmu_invlpg(vcpu, gva); 2501 2502 return kvm_skip_emulated_instruction(vcpu); 2503 } 2504 2505 static int skinit_interception(struct kvm_vcpu *vcpu) 2506 { 2507 trace_kvm_skinit(to_svm(vcpu)->vmcb->save.rip, kvm_rax_read(vcpu)); 2508 2509 kvm_queue_exception(vcpu, UD_VECTOR); 2510 return 1; 2511 } 2512 2513 static int task_switch_interception(struct kvm_vcpu *vcpu) 2514 { 2515 struct vcpu_svm *svm = to_svm(vcpu); 2516 u16 tss_selector; 2517 int reason; 2518 int int_type = svm->vmcb->control.exit_int_info & 2519 SVM_EXITINTINFO_TYPE_MASK; 2520 int int_vec = svm->vmcb->control.exit_int_info & SVM_EVTINJ_VEC_MASK; 2521 uint32_t type = 2522 svm->vmcb->control.exit_int_info & SVM_EXITINTINFO_TYPE_MASK; 2523 uint32_t idt_v = 2524 svm->vmcb->control.exit_int_info & SVM_EXITINTINFO_VALID; 2525 bool has_error_code = false; 2526 u32 error_code = 0; 2527 2528 tss_selector = (u16)svm->vmcb->control.exit_info_1; 2529 2530 if (svm->vmcb->control.exit_info_2 & 2531 (1ULL << SVM_EXITINFOSHIFT_TS_REASON_IRET)) 2532 reason = TASK_SWITCH_IRET; 2533 else if (svm->vmcb->control.exit_info_2 & 2534 (1ULL << SVM_EXITINFOSHIFT_TS_REASON_JMP)) 2535 reason = TASK_SWITCH_JMP; 2536 else if (idt_v) 2537 reason = TASK_SWITCH_GATE; 2538 else 2539 reason = TASK_SWITCH_CALL; 2540 2541 if (reason == TASK_SWITCH_GATE) { 2542 switch (type) { 2543 case SVM_EXITINTINFO_TYPE_NMI: 2544 vcpu->arch.nmi_injected = false; 2545 break; 2546 case SVM_EXITINTINFO_TYPE_EXEPT: 2547 if (svm->vmcb->control.exit_info_2 & 2548 (1ULL << SVM_EXITINFOSHIFT_TS_HAS_ERROR_CODE)) { 2549 has_error_code = true; 2550 error_code = 2551 (u32)svm->vmcb->control.exit_info_2; 2552 } 2553 kvm_clear_exception_queue(vcpu); 2554 break; 2555 case SVM_EXITINTINFO_TYPE_INTR: 2556 case SVM_EXITINTINFO_TYPE_SOFT: 2557 kvm_clear_interrupt_queue(vcpu); 2558 break; 2559 default: 2560 break; 2561 } 2562 } 2563 2564 if (reason != TASK_SWITCH_GATE || 2565 int_type == SVM_EXITINTINFO_TYPE_SOFT || 2566 (int_type == SVM_EXITINTINFO_TYPE_EXEPT && 2567 (int_vec == OF_VECTOR || int_vec == BP_VECTOR))) { 2568 if (!svm_skip_emulated_instruction(vcpu)) 2569 return 0; 2570 } 2571 2572 if (int_type != SVM_EXITINTINFO_TYPE_SOFT) 2573 int_vec = -1; 2574 2575 return kvm_task_switch(vcpu, tss_selector, int_vec, reason, 2576 has_error_code, error_code); 2577 } 2578 2579 static void svm_clr_iret_intercept(struct vcpu_svm *svm) 2580 { 2581 if (!sev_es_guest(svm->vcpu.kvm)) 2582 svm_clr_intercept(svm, INTERCEPT_IRET); 2583 } 2584 2585 static void svm_set_iret_intercept(struct vcpu_svm *svm) 2586 { 2587 if (!sev_es_guest(svm->vcpu.kvm)) 2588 svm_set_intercept(svm, INTERCEPT_IRET); 2589 } 2590 2591 static int iret_interception(struct kvm_vcpu *vcpu) 2592 { 2593 struct vcpu_svm *svm = to_svm(vcpu); 2594 2595 WARN_ON_ONCE(sev_es_guest(vcpu->kvm)); 2596 2597 ++vcpu->stat.nmi_window_exits; 2598 svm->awaiting_iret_completion = true; 2599 2600 svm_clr_iret_intercept(svm); 2601 svm->nmi_iret_rip = kvm_rip_read(vcpu); 2602 2603 kvm_make_request(KVM_REQ_EVENT, vcpu); 2604 return 1; 2605 } 2606 2607 static int invlpg_interception(struct kvm_vcpu *vcpu) 2608 { 2609 if (!static_cpu_has(X86_FEATURE_DECODEASSISTS)) 2610 return kvm_emulate_instruction(vcpu, 0); 2611 2612 kvm_mmu_invlpg(vcpu, to_svm(vcpu)->vmcb->control.exit_info_1); 2613 return kvm_skip_emulated_instruction(vcpu); 2614 } 2615 2616 static int emulate_on_interception(struct kvm_vcpu *vcpu) 2617 { 2618 return kvm_emulate_instruction(vcpu, 0); 2619 } 2620 2621 static int rsm_interception(struct kvm_vcpu *vcpu) 2622 { 2623 return kvm_emulate_instruction_from_buffer(vcpu, rsm_ins_bytes, 2); 2624 } 2625 2626 static bool check_selective_cr0_intercepted(struct kvm_vcpu *vcpu, 2627 unsigned long val) 2628 { 2629 struct vcpu_svm *svm = to_svm(vcpu); 2630 unsigned long cr0 = vcpu->arch.cr0; 2631 bool ret = false; 2632 2633 if (!is_guest_mode(vcpu) || 2634 (!(vmcb12_is_intercept(&svm->nested.ctl, INTERCEPT_SELECTIVE_CR0)))) 2635 return false; 2636 2637 cr0 &= ~SVM_CR0_SELECTIVE_MASK; 2638 val &= ~SVM_CR0_SELECTIVE_MASK; 2639 2640 if (cr0 ^ val) { 2641 svm->vmcb->control.exit_code = SVM_EXIT_CR0_SEL_WRITE; 2642 ret = (nested_svm_exit_handled(svm) == NESTED_EXIT_DONE); 2643 } 2644 2645 return ret; 2646 } 2647 2648 #define CR_VALID (1ULL << 63) 2649 2650 static int cr_interception(struct kvm_vcpu *vcpu) 2651 { 2652 struct vcpu_svm *svm = to_svm(vcpu); 2653 int reg, cr; 2654 unsigned long val; 2655 int err; 2656 2657 if (!static_cpu_has(X86_FEATURE_DECODEASSISTS)) 2658 return emulate_on_interception(vcpu); 2659 2660 if (unlikely((svm->vmcb->control.exit_info_1 & CR_VALID) == 0)) 2661 return emulate_on_interception(vcpu); 2662 2663 reg = svm->vmcb->control.exit_info_1 & SVM_EXITINFO_REG_MASK; 2664 if (svm->vmcb->control.exit_code == SVM_EXIT_CR0_SEL_WRITE) 2665 cr = SVM_EXIT_WRITE_CR0 - SVM_EXIT_READ_CR0; 2666 else 2667 cr = svm->vmcb->control.exit_code - SVM_EXIT_READ_CR0; 2668 2669 err = 0; 2670 if (cr >= 16) { /* mov to cr */ 2671 cr -= 16; 2672 val = kvm_register_read(vcpu, reg); 2673 trace_kvm_cr_write(cr, val); 2674 switch (cr) { 2675 case 0: 2676 if (!check_selective_cr0_intercepted(vcpu, val)) 2677 err = kvm_set_cr0(vcpu, val); 2678 else 2679 return 1; 2680 2681 break; 2682 case 3: 2683 err = kvm_set_cr3(vcpu, val); 2684 break; 2685 case 4: 2686 err = kvm_set_cr4(vcpu, val); 2687 break; 2688 case 8: 2689 err = kvm_set_cr8(vcpu, val); 2690 break; 2691 default: 2692 WARN(1, "unhandled write to CR%d", cr); 2693 kvm_queue_exception(vcpu, UD_VECTOR); 2694 return 1; 2695 } 2696 } else { /* mov from cr */ 2697 switch (cr) { 2698 case 0: 2699 val = kvm_read_cr0(vcpu); 2700 break; 2701 case 2: 2702 val = vcpu->arch.cr2; 2703 break; 2704 case 3: 2705 val = kvm_read_cr3(vcpu); 2706 break; 2707 case 4: 2708 val = kvm_read_cr4(vcpu); 2709 break; 2710 case 8: 2711 val = kvm_get_cr8(vcpu); 2712 break; 2713 default: 2714 WARN(1, "unhandled read from CR%d", cr); 2715 kvm_queue_exception(vcpu, UD_VECTOR); 2716 return 1; 2717 } 2718 kvm_register_write(vcpu, reg, val); 2719 trace_kvm_cr_read(cr, val); 2720 } 2721 return kvm_complete_insn_gp(vcpu, err); 2722 } 2723 2724 static int cr_trap(struct kvm_vcpu *vcpu) 2725 { 2726 struct vcpu_svm *svm = to_svm(vcpu); 2727 unsigned long old_value, new_value; 2728 unsigned int cr; 2729 int ret = 0; 2730 2731 new_value = (unsigned long)svm->vmcb->control.exit_info_1; 2732 2733 cr = svm->vmcb->control.exit_code - SVM_EXIT_CR0_WRITE_TRAP; 2734 switch (cr) { 2735 case 0: 2736 old_value = kvm_read_cr0(vcpu); 2737 svm_set_cr0(vcpu, new_value); 2738 2739 kvm_post_set_cr0(vcpu, old_value, new_value); 2740 break; 2741 case 4: 2742 old_value = kvm_read_cr4(vcpu); 2743 svm_set_cr4(vcpu, new_value); 2744 2745 kvm_post_set_cr4(vcpu, old_value, new_value); 2746 break; 2747 case 8: 2748 ret = kvm_set_cr8(vcpu, new_value); 2749 break; 2750 default: 2751 WARN(1, "unhandled CR%d write trap", cr); 2752 kvm_queue_exception(vcpu, UD_VECTOR); 2753 return 1; 2754 } 2755 2756 return kvm_complete_insn_gp(vcpu, ret); 2757 } 2758 2759 static int dr_interception(struct kvm_vcpu *vcpu) 2760 { 2761 struct vcpu_svm *svm = to_svm(vcpu); 2762 int reg, dr; 2763 unsigned long val; 2764 int err = 0; 2765 2766 /* 2767 * SEV-ES intercepts DR7 only to disable guest debugging and the guest issues a VMGEXIT 2768 * for DR7 write only. KVM cannot change DR7 (always swapped as type 'A') so return early. 2769 */ 2770 if (sev_es_guest(vcpu->kvm)) 2771 return 1; 2772 2773 if (vcpu->guest_debug == 0) { 2774 /* 2775 * No more DR vmexits; force a reload of the debug registers 2776 * and reenter on this instruction. The next vmexit will 2777 * retrieve the full state of the debug registers. 2778 */ 2779 clr_dr_intercepts(svm); 2780 vcpu->arch.switch_db_regs |= KVM_DEBUGREG_WONT_EXIT; 2781 return 1; 2782 } 2783 2784 if (!boot_cpu_has(X86_FEATURE_DECODEASSISTS)) 2785 return emulate_on_interception(vcpu); 2786 2787 reg = svm->vmcb->control.exit_info_1 & SVM_EXITINFO_REG_MASK; 2788 dr = svm->vmcb->control.exit_code - SVM_EXIT_READ_DR0; 2789 if (dr >= 16) { /* mov to DRn */ 2790 dr -= 16; 2791 val = kvm_register_read(vcpu, reg); 2792 err = kvm_set_dr(vcpu, dr, val); 2793 } else { 2794 kvm_get_dr(vcpu, dr, &val); 2795 kvm_register_write(vcpu, reg, val); 2796 } 2797 2798 return kvm_complete_insn_gp(vcpu, err); 2799 } 2800 2801 static int cr8_write_interception(struct kvm_vcpu *vcpu) 2802 { 2803 int r; 2804 2805 u8 cr8_prev = kvm_get_cr8(vcpu); 2806 /* instruction emulation calls kvm_set_cr8() */ 2807 r = cr_interception(vcpu); 2808 if (lapic_in_kernel(vcpu)) 2809 return r; 2810 if (cr8_prev <= kvm_get_cr8(vcpu)) 2811 return r; 2812 vcpu->run->exit_reason = KVM_EXIT_SET_TPR; 2813 return 0; 2814 } 2815 2816 static int efer_trap(struct kvm_vcpu *vcpu) 2817 { 2818 struct msr_data msr_info; 2819 int ret; 2820 2821 /* 2822 * Clear the EFER_SVME bit from EFER. The SVM code always sets this 2823 * bit in svm_set_efer(), but __kvm_valid_efer() checks it against 2824 * whether the guest has X86_FEATURE_SVM - this avoids a failure if 2825 * the guest doesn't have X86_FEATURE_SVM. 2826 */ 2827 msr_info.host_initiated = false; 2828 msr_info.index = MSR_EFER; 2829 msr_info.data = to_svm(vcpu)->vmcb->control.exit_info_1 & ~EFER_SVME; 2830 ret = kvm_set_msr_common(vcpu, &msr_info); 2831 2832 return kvm_complete_insn_gp(vcpu, ret); 2833 } 2834 2835 static int svm_get_msr_feature(struct kvm_msr_entry *msr) 2836 { 2837 msr->data = 0; 2838 2839 switch (msr->index) { 2840 case MSR_AMD64_DE_CFG: 2841 if (cpu_feature_enabled(X86_FEATURE_LFENCE_RDTSC)) 2842 msr->data |= MSR_AMD64_DE_CFG_LFENCE_SERIALIZE; 2843 break; 2844 default: 2845 return KVM_MSR_RET_INVALID; 2846 } 2847 2848 return 0; 2849 } 2850 2851 static int svm_get_msr(struct kvm_vcpu *vcpu, struct msr_data *msr_info) 2852 { 2853 struct vcpu_svm *svm = to_svm(vcpu); 2854 2855 switch (msr_info->index) { 2856 case MSR_AMD64_TSC_RATIO: 2857 if (!msr_info->host_initiated && 2858 !guest_can_use(vcpu, X86_FEATURE_TSCRATEMSR)) 2859 return 1; 2860 msr_info->data = svm->tsc_ratio_msr; 2861 break; 2862 case MSR_STAR: 2863 msr_info->data = svm->vmcb01.ptr->save.star; 2864 break; 2865 #ifdef CONFIG_X86_64 2866 case MSR_LSTAR: 2867 msr_info->data = svm->vmcb01.ptr->save.lstar; 2868 break; 2869 case MSR_CSTAR: 2870 msr_info->data = svm->vmcb01.ptr->save.cstar; 2871 break; 2872 case MSR_GS_BASE: 2873 msr_info->data = svm->vmcb01.ptr->save.gs.base; 2874 break; 2875 case MSR_FS_BASE: 2876 msr_info->data = svm->vmcb01.ptr->save.fs.base; 2877 break; 2878 case MSR_KERNEL_GS_BASE: 2879 msr_info->data = svm->vmcb01.ptr->save.kernel_gs_base; 2880 break; 2881 case MSR_SYSCALL_MASK: 2882 msr_info->data = svm->vmcb01.ptr->save.sfmask; 2883 break; 2884 #endif 2885 case MSR_IA32_SYSENTER_CS: 2886 msr_info->data = svm->vmcb01.ptr->save.sysenter_cs; 2887 break; 2888 case MSR_IA32_SYSENTER_EIP: 2889 msr_info->data = (u32)svm->vmcb01.ptr->save.sysenter_eip; 2890 if (guest_cpuid_is_intel(vcpu)) 2891 msr_info->data |= (u64)svm->sysenter_eip_hi << 32; 2892 break; 2893 case MSR_IA32_SYSENTER_ESP: 2894 msr_info->data = svm->vmcb01.ptr->save.sysenter_esp; 2895 if (guest_cpuid_is_intel(vcpu)) 2896 msr_info->data |= (u64)svm->sysenter_esp_hi << 32; 2897 break; 2898 case MSR_TSC_AUX: 2899 msr_info->data = svm->tsc_aux; 2900 break; 2901 case MSR_IA32_DEBUGCTLMSR: 2902 msr_info->data = svm_get_lbr_vmcb(svm)->save.dbgctl; 2903 break; 2904 case MSR_IA32_LASTBRANCHFROMIP: 2905 msr_info->data = svm_get_lbr_vmcb(svm)->save.br_from; 2906 break; 2907 case MSR_IA32_LASTBRANCHTOIP: 2908 msr_info->data = svm_get_lbr_vmcb(svm)->save.br_to; 2909 break; 2910 case MSR_IA32_LASTINTFROMIP: 2911 msr_info->data = svm_get_lbr_vmcb(svm)->save.last_excp_from; 2912 break; 2913 case MSR_IA32_LASTINTTOIP: 2914 msr_info->data = svm_get_lbr_vmcb(svm)->save.last_excp_to; 2915 break; 2916 case MSR_VM_HSAVE_PA: 2917 msr_info->data = svm->nested.hsave_msr; 2918 break; 2919 case MSR_VM_CR: 2920 msr_info->data = svm->nested.vm_cr_msr; 2921 break; 2922 case MSR_IA32_SPEC_CTRL: 2923 if (!msr_info->host_initiated && 2924 !guest_has_spec_ctrl_msr(vcpu)) 2925 return 1; 2926 2927 if (boot_cpu_has(X86_FEATURE_V_SPEC_CTRL)) 2928 msr_info->data = svm->vmcb->save.spec_ctrl; 2929 else 2930 msr_info->data = svm->spec_ctrl; 2931 break; 2932 case MSR_AMD64_VIRT_SPEC_CTRL: 2933 if (!msr_info->host_initiated && 2934 !guest_cpuid_has(vcpu, X86_FEATURE_VIRT_SSBD)) 2935 return 1; 2936 2937 msr_info->data = svm->virt_spec_ctrl; 2938 break; 2939 case MSR_F15H_IC_CFG: { 2940 2941 int family, model; 2942 2943 family = guest_cpuid_family(vcpu); 2944 model = guest_cpuid_model(vcpu); 2945 2946 if (family < 0 || model < 0) 2947 return kvm_get_msr_common(vcpu, msr_info); 2948 2949 msr_info->data = 0; 2950 2951 if (family == 0x15 && 2952 (model >= 0x2 && model < 0x20)) 2953 msr_info->data = 0x1E; 2954 } 2955 break; 2956 case MSR_AMD64_DE_CFG: 2957 msr_info->data = svm->msr_decfg; 2958 break; 2959 default: 2960 return kvm_get_msr_common(vcpu, msr_info); 2961 } 2962 return 0; 2963 } 2964 2965 static int svm_complete_emulated_msr(struct kvm_vcpu *vcpu, int err) 2966 { 2967 struct vcpu_svm *svm = to_svm(vcpu); 2968 if (!err || !sev_es_guest(vcpu->kvm) || WARN_ON_ONCE(!svm->sev_es.ghcb)) 2969 return kvm_complete_insn_gp(vcpu, err); 2970 2971 ghcb_set_sw_exit_info_1(svm->sev_es.ghcb, 1); 2972 ghcb_set_sw_exit_info_2(svm->sev_es.ghcb, 2973 X86_TRAP_GP | 2974 SVM_EVTINJ_TYPE_EXEPT | 2975 SVM_EVTINJ_VALID); 2976 return 1; 2977 } 2978 2979 static int svm_set_vm_cr(struct kvm_vcpu *vcpu, u64 data) 2980 { 2981 struct vcpu_svm *svm = to_svm(vcpu); 2982 int svm_dis, chg_mask; 2983 2984 if (data & ~SVM_VM_CR_VALID_MASK) 2985 return 1; 2986 2987 chg_mask = SVM_VM_CR_VALID_MASK; 2988 2989 if (svm->nested.vm_cr_msr & SVM_VM_CR_SVM_DIS_MASK) 2990 chg_mask &= ~(SVM_VM_CR_SVM_LOCK_MASK | SVM_VM_CR_SVM_DIS_MASK); 2991 2992 svm->nested.vm_cr_msr &= ~chg_mask; 2993 svm->nested.vm_cr_msr |= (data & chg_mask); 2994 2995 svm_dis = svm->nested.vm_cr_msr & SVM_VM_CR_SVM_DIS_MASK; 2996 2997 /* check for svm_disable while efer.svme is set */ 2998 if (svm_dis && (vcpu->arch.efer & EFER_SVME)) 2999 return 1; 3000 3001 return 0; 3002 } 3003 3004 static int svm_set_msr(struct kvm_vcpu *vcpu, struct msr_data *msr) 3005 { 3006 struct vcpu_svm *svm = to_svm(vcpu); 3007 int ret = 0; 3008 3009 u32 ecx = msr->index; 3010 u64 data = msr->data; 3011 switch (ecx) { 3012 case MSR_AMD64_TSC_RATIO: 3013 3014 if (!guest_can_use(vcpu, X86_FEATURE_TSCRATEMSR)) { 3015 3016 if (!msr->host_initiated) 3017 return 1; 3018 /* 3019 * In case TSC scaling is not enabled, always 3020 * leave this MSR at the default value. 3021 * 3022 * Due to bug in qemu 6.2.0, it would try to set 3023 * this msr to 0 if tsc scaling is not enabled. 3024 * Ignore this value as well. 3025 */ 3026 if (data != 0 && data != svm->tsc_ratio_msr) 3027 return 1; 3028 break; 3029 } 3030 3031 if (data & SVM_TSC_RATIO_RSVD) 3032 return 1; 3033 3034 svm->tsc_ratio_msr = data; 3035 3036 if (guest_can_use(vcpu, X86_FEATURE_TSCRATEMSR) && 3037 is_guest_mode(vcpu)) 3038 nested_svm_update_tsc_ratio_msr(vcpu); 3039 3040 break; 3041 case MSR_IA32_CR_PAT: 3042 ret = kvm_set_msr_common(vcpu, msr); 3043 if (ret) 3044 break; 3045 3046 svm->vmcb01.ptr->save.g_pat = data; 3047 if (is_guest_mode(vcpu)) 3048 nested_vmcb02_compute_g_pat(svm); 3049 vmcb_mark_dirty(svm->vmcb, VMCB_NPT); 3050 break; 3051 case MSR_IA32_SPEC_CTRL: 3052 if (!msr->host_initiated && 3053 !guest_has_spec_ctrl_msr(vcpu)) 3054 return 1; 3055 3056 if (kvm_spec_ctrl_test_value(data)) 3057 return 1; 3058 3059 if (boot_cpu_has(X86_FEATURE_V_SPEC_CTRL)) 3060 svm->vmcb->save.spec_ctrl = data; 3061 else 3062 svm->spec_ctrl = data; 3063 if (!data) 3064 break; 3065 3066 /* 3067 * For non-nested: 3068 * When it's written (to non-zero) for the first time, pass 3069 * it through. 3070 * 3071 * For nested: 3072 * The handling of the MSR bitmap for L2 guests is done in 3073 * nested_svm_vmrun_msrpm. 3074 * We update the L1 MSR bit as well since it will end up 3075 * touching the MSR anyway now. 3076 */ 3077 set_msr_interception(vcpu, svm->msrpm, MSR_IA32_SPEC_CTRL, 1, 1); 3078 break; 3079 case MSR_AMD64_VIRT_SPEC_CTRL: 3080 if (!msr->host_initiated && 3081 !guest_cpuid_has(vcpu, X86_FEATURE_VIRT_SSBD)) 3082 return 1; 3083 3084 if (data & ~SPEC_CTRL_SSBD) 3085 return 1; 3086 3087 svm->virt_spec_ctrl = data; 3088 break; 3089 case MSR_STAR: 3090 svm->vmcb01.ptr->save.star = data; 3091 break; 3092 #ifdef CONFIG_X86_64 3093 case MSR_LSTAR: 3094 svm->vmcb01.ptr->save.lstar = data; 3095 break; 3096 case MSR_CSTAR: 3097 svm->vmcb01.ptr->save.cstar = data; 3098 break; 3099 case MSR_GS_BASE: 3100 svm->vmcb01.ptr->save.gs.base = data; 3101 break; 3102 case MSR_FS_BASE: 3103 svm->vmcb01.ptr->save.fs.base = data; 3104 break; 3105 case MSR_KERNEL_GS_BASE: 3106 svm->vmcb01.ptr->save.kernel_gs_base = data; 3107 break; 3108 case MSR_SYSCALL_MASK: 3109 svm->vmcb01.ptr->save.sfmask = data; 3110 break; 3111 #endif 3112 case MSR_IA32_SYSENTER_CS: 3113 svm->vmcb01.ptr->save.sysenter_cs = data; 3114 break; 3115 case MSR_IA32_SYSENTER_EIP: 3116 svm->vmcb01.ptr->save.sysenter_eip = (u32)data; 3117 /* 3118 * We only intercept the MSR_IA32_SYSENTER_{EIP|ESP} msrs 3119 * when we spoof an Intel vendor ID (for cross vendor migration). 3120 * In this case we use this intercept to track the high 3121 * 32 bit part of these msrs to support Intel's 3122 * implementation of SYSENTER/SYSEXIT. 3123 */ 3124 svm->sysenter_eip_hi = guest_cpuid_is_intel(vcpu) ? (data >> 32) : 0; 3125 break; 3126 case MSR_IA32_SYSENTER_ESP: 3127 svm->vmcb01.ptr->save.sysenter_esp = (u32)data; 3128 svm->sysenter_esp_hi = guest_cpuid_is_intel(vcpu) ? (data >> 32) : 0; 3129 break; 3130 case MSR_TSC_AUX: 3131 /* 3132 * TSC_AUX is always virtualized for SEV-ES guests when the 3133 * feature is available. The user return MSR support is not 3134 * required in this case because TSC_AUX is restored on #VMEXIT 3135 * from the host save area (which has been initialized in 3136 * svm_hardware_enable()). 3137 */ 3138 if (boot_cpu_has(X86_FEATURE_V_TSC_AUX) && sev_es_guest(vcpu->kvm)) 3139 break; 3140 3141 /* 3142 * TSC_AUX is usually changed only during boot and never read 3143 * directly. Intercept TSC_AUX instead of exposing it to the 3144 * guest via direct_access_msrs, and switch it via user return. 3145 */ 3146 preempt_disable(); 3147 ret = kvm_set_user_return_msr(tsc_aux_uret_slot, data, -1ull); 3148 preempt_enable(); 3149 if (ret) 3150 break; 3151 3152 svm->tsc_aux = data; 3153 break; 3154 case MSR_IA32_DEBUGCTLMSR: 3155 if (!lbrv) { 3156 kvm_pr_unimpl_wrmsr(vcpu, ecx, data); 3157 break; 3158 } 3159 if (data & DEBUGCTL_RESERVED_BITS) 3160 return 1; 3161 3162 svm_get_lbr_vmcb(svm)->save.dbgctl = data; 3163 svm_update_lbrv(vcpu); 3164 break; 3165 case MSR_VM_HSAVE_PA: 3166 /* 3167 * Old kernels did not validate the value written to 3168 * MSR_VM_HSAVE_PA. Allow KVM_SET_MSR to set an invalid 3169 * value to allow live migrating buggy or malicious guests 3170 * originating from those kernels. 3171 */ 3172 if (!msr->host_initiated && !page_address_valid(vcpu, data)) 3173 return 1; 3174 3175 svm->nested.hsave_msr = data & PAGE_MASK; 3176 break; 3177 case MSR_VM_CR: 3178 return svm_set_vm_cr(vcpu, data); 3179 case MSR_VM_IGNNE: 3180 kvm_pr_unimpl_wrmsr(vcpu, ecx, data); 3181 break; 3182 case MSR_AMD64_DE_CFG: { 3183 struct kvm_msr_entry msr_entry; 3184 3185 msr_entry.index = msr->index; 3186 if (svm_get_msr_feature(&msr_entry)) 3187 return 1; 3188 3189 /* Check the supported bits */ 3190 if (data & ~msr_entry.data) 3191 return 1; 3192 3193 /* Don't allow the guest to change a bit, #GP */ 3194 if (!msr->host_initiated && (data ^ msr_entry.data)) 3195 return 1; 3196 3197 svm->msr_decfg = data; 3198 break; 3199 } 3200 default: 3201 return kvm_set_msr_common(vcpu, msr); 3202 } 3203 return ret; 3204 } 3205 3206 static int msr_interception(struct kvm_vcpu *vcpu) 3207 { 3208 if (to_svm(vcpu)->vmcb->control.exit_info_1) 3209 return kvm_emulate_wrmsr(vcpu); 3210 else 3211 return kvm_emulate_rdmsr(vcpu); 3212 } 3213 3214 static int interrupt_window_interception(struct kvm_vcpu *vcpu) 3215 { 3216 kvm_make_request(KVM_REQ_EVENT, vcpu); 3217 svm_clear_vintr(to_svm(vcpu)); 3218 3219 /* 3220 * If not running nested, for AVIC, the only reason to end up here is ExtINTs. 3221 * In this case AVIC was temporarily disabled for 3222 * requesting the IRQ window and we have to re-enable it. 3223 * 3224 * If running nested, still remove the VM wide AVIC inhibit to 3225 * support case in which the interrupt window was requested when the 3226 * vCPU was not running nested. 3227 3228 * All vCPUs which run still run nested, will remain to have their 3229 * AVIC still inhibited due to per-cpu AVIC inhibition. 3230 */ 3231 kvm_clear_apicv_inhibit(vcpu->kvm, APICV_INHIBIT_REASON_IRQWIN); 3232 3233 ++vcpu->stat.irq_window_exits; 3234 return 1; 3235 } 3236 3237 static int pause_interception(struct kvm_vcpu *vcpu) 3238 { 3239 bool in_kernel; 3240 /* 3241 * CPL is not made available for an SEV-ES guest, therefore 3242 * vcpu->arch.preempted_in_kernel can never be true. Just 3243 * set in_kernel to false as well. 3244 */ 3245 in_kernel = !sev_es_guest(vcpu->kvm) && svm_get_cpl(vcpu) == 0; 3246 3247 grow_ple_window(vcpu); 3248 3249 kvm_vcpu_on_spin(vcpu, in_kernel); 3250 return kvm_skip_emulated_instruction(vcpu); 3251 } 3252 3253 static int invpcid_interception(struct kvm_vcpu *vcpu) 3254 { 3255 struct vcpu_svm *svm = to_svm(vcpu); 3256 unsigned long type; 3257 gva_t gva; 3258 3259 if (!guest_cpuid_has(vcpu, X86_FEATURE_INVPCID)) { 3260 kvm_queue_exception(vcpu, UD_VECTOR); 3261 return 1; 3262 } 3263 3264 /* 3265 * For an INVPCID intercept: 3266 * EXITINFO1 provides the linear address of the memory operand. 3267 * EXITINFO2 provides the contents of the register operand. 3268 */ 3269 type = svm->vmcb->control.exit_info_2; 3270 gva = svm->vmcb->control.exit_info_1; 3271 3272 return kvm_handle_invpcid(vcpu, type, gva); 3273 } 3274 3275 static int (*const svm_exit_handlers[])(struct kvm_vcpu *vcpu) = { 3276 [SVM_EXIT_READ_CR0] = cr_interception, 3277 [SVM_EXIT_READ_CR3] = cr_interception, 3278 [SVM_EXIT_READ_CR4] = cr_interception, 3279 [SVM_EXIT_READ_CR8] = cr_interception, 3280 [SVM_EXIT_CR0_SEL_WRITE] = cr_interception, 3281 [SVM_EXIT_WRITE_CR0] = cr_interception, 3282 [SVM_EXIT_WRITE_CR3] = cr_interception, 3283 [SVM_EXIT_WRITE_CR4] = cr_interception, 3284 [SVM_EXIT_WRITE_CR8] = cr8_write_interception, 3285 [SVM_EXIT_READ_DR0] = dr_interception, 3286 [SVM_EXIT_READ_DR1] = dr_interception, 3287 [SVM_EXIT_READ_DR2] = dr_interception, 3288 [SVM_EXIT_READ_DR3] = dr_interception, 3289 [SVM_EXIT_READ_DR4] = dr_interception, 3290 [SVM_EXIT_READ_DR5] = dr_interception, 3291 [SVM_EXIT_READ_DR6] = dr_interception, 3292 [SVM_EXIT_READ_DR7] = dr_interception, 3293 [SVM_EXIT_WRITE_DR0] = dr_interception, 3294 [SVM_EXIT_WRITE_DR1] = dr_interception, 3295 [SVM_EXIT_WRITE_DR2] = dr_interception, 3296 [SVM_EXIT_WRITE_DR3] = dr_interception, 3297 [SVM_EXIT_WRITE_DR4] = dr_interception, 3298 [SVM_EXIT_WRITE_DR5] = dr_interception, 3299 [SVM_EXIT_WRITE_DR6] = dr_interception, 3300 [SVM_EXIT_WRITE_DR7] = dr_interception, 3301 [SVM_EXIT_EXCP_BASE + DB_VECTOR] = db_interception, 3302 [SVM_EXIT_EXCP_BASE + BP_VECTOR] = bp_interception, 3303 [SVM_EXIT_EXCP_BASE + UD_VECTOR] = ud_interception, 3304 [SVM_EXIT_EXCP_BASE + PF_VECTOR] = pf_interception, 3305 [SVM_EXIT_EXCP_BASE + MC_VECTOR] = mc_interception, 3306 [SVM_EXIT_EXCP_BASE + AC_VECTOR] = ac_interception, 3307 [SVM_EXIT_EXCP_BASE + GP_VECTOR] = gp_interception, 3308 [SVM_EXIT_INTR] = intr_interception, 3309 [SVM_EXIT_NMI] = nmi_interception, 3310 [SVM_EXIT_SMI] = smi_interception, 3311 [SVM_EXIT_VINTR] = interrupt_window_interception, 3312 [SVM_EXIT_RDPMC] = kvm_emulate_rdpmc, 3313 [SVM_EXIT_CPUID] = kvm_emulate_cpuid, 3314 [SVM_EXIT_IRET] = iret_interception, 3315 [SVM_EXIT_INVD] = kvm_emulate_invd, 3316 [SVM_EXIT_PAUSE] = pause_interception, 3317 [SVM_EXIT_HLT] = kvm_emulate_halt, 3318 [SVM_EXIT_INVLPG] = invlpg_interception, 3319 [SVM_EXIT_INVLPGA] = invlpga_interception, 3320 [SVM_EXIT_IOIO] = io_interception, 3321 [SVM_EXIT_MSR] = msr_interception, 3322 [SVM_EXIT_TASK_SWITCH] = task_switch_interception, 3323 [SVM_EXIT_SHUTDOWN] = shutdown_interception, 3324 [SVM_EXIT_VMRUN] = vmrun_interception, 3325 [SVM_EXIT_VMMCALL] = kvm_emulate_hypercall, 3326 [SVM_EXIT_VMLOAD] = vmload_interception, 3327 [SVM_EXIT_VMSAVE] = vmsave_interception, 3328 [SVM_EXIT_STGI] = stgi_interception, 3329 [SVM_EXIT_CLGI] = clgi_interception, 3330 [SVM_EXIT_SKINIT] = skinit_interception, 3331 [SVM_EXIT_RDTSCP] = kvm_handle_invalid_op, 3332 [SVM_EXIT_WBINVD] = kvm_emulate_wbinvd, 3333 [SVM_EXIT_MONITOR] = kvm_emulate_monitor, 3334 [SVM_EXIT_MWAIT] = kvm_emulate_mwait, 3335 [SVM_EXIT_XSETBV] = kvm_emulate_xsetbv, 3336 [SVM_EXIT_RDPRU] = kvm_handle_invalid_op, 3337 [SVM_EXIT_EFER_WRITE_TRAP] = efer_trap, 3338 [SVM_EXIT_CR0_WRITE_TRAP] = cr_trap, 3339 [SVM_EXIT_CR4_WRITE_TRAP] = cr_trap, 3340 [SVM_EXIT_CR8_WRITE_TRAP] = cr_trap, 3341 [SVM_EXIT_INVPCID] = invpcid_interception, 3342 [SVM_EXIT_NPF] = npf_interception, 3343 [SVM_EXIT_RSM] = rsm_interception, 3344 [SVM_EXIT_AVIC_INCOMPLETE_IPI] = avic_incomplete_ipi_interception, 3345 [SVM_EXIT_AVIC_UNACCELERATED_ACCESS] = avic_unaccelerated_access_interception, 3346 [SVM_EXIT_VMGEXIT] = sev_handle_vmgexit, 3347 }; 3348 3349 static void dump_vmcb(struct kvm_vcpu *vcpu) 3350 { 3351 struct vcpu_svm *svm = to_svm(vcpu); 3352 struct vmcb_control_area *control = &svm->vmcb->control; 3353 struct vmcb_save_area *save = &svm->vmcb->save; 3354 struct vmcb_save_area *save01 = &svm->vmcb01.ptr->save; 3355 3356 if (!dump_invalid_vmcb) { 3357 pr_warn_ratelimited("set kvm_amd.dump_invalid_vmcb=1 to dump internal KVM state.\n"); 3358 return; 3359 } 3360 3361 pr_err("VMCB %p, last attempted VMRUN on CPU %d\n", 3362 svm->current_vmcb->ptr, vcpu->arch.last_vmentry_cpu); 3363 pr_err("VMCB Control Area:\n"); 3364 pr_err("%-20s%04x\n", "cr_read:", control->intercepts[INTERCEPT_CR] & 0xffff); 3365 pr_err("%-20s%04x\n", "cr_write:", control->intercepts[INTERCEPT_CR] >> 16); 3366 pr_err("%-20s%04x\n", "dr_read:", control->intercepts[INTERCEPT_DR] & 0xffff); 3367 pr_err("%-20s%04x\n", "dr_write:", control->intercepts[INTERCEPT_DR] >> 16); 3368 pr_err("%-20s%08x\n", "exceptions:", control->intercepts[INTERCEPT_EXCEPTION]); 3369 pr_err("%-20s%08x %08x\n", "intercepts:", 3370 control->intercepts[INTERCEPT_WORD3], 3371 control->intercepts[INTERCEPT_WORD4]); 3372 pr_err("%-20s%d\n", "pause filter count:", control->pause_filter_count); 3373 pr_err("%-20s%d\n", "pause filter threshold:", 3374 control->pause_filter_thresh); 3375 pr_err("%-20s%016llx\n", "iopm_base_pa:", control->iopm_base_pa); 3376 pr_err("%-20s%016llx\n", "msrpm_base_pa:", control->msrpm_base_pa); 3377 pr_err("%-20s%016llx\n", "tsc_offset:", control->tsc_offset); 3378 pr_err("%-20s%d\n", "asid:", control->asid); 3379 pr_err("%-20s%d\n", "tlb_ctl:", control->tlb_ctl); 3380 pr_err("%-20s%08x\n", "int_ctl:", control->int_ctl); 3381 pr_err("%-20s%08x\n", "int_vector:", control->int_vector); 3382 pr_err("%-20s%08x\n", "int_state:", control->int_state); 3383 pr_err("%-20s%08x\n", "exit_code:", control->exit_code); 3384 pr_err("%-20s%016llx\n", "exit_info1:", control->exit_info_1); 3385 pr_err("%-20s%016llx\n", "exit_info2:", control->exit_info_2); 3386 pr_err("%-20s%08x\n", "exit_int_info:", control->exit_int_info); 3387 pr_err("%-20s%08x\n", "exit_int_info_err:", control->exit_int_info_err); 3388 pr_err("%-20s%lld\n", "nested_ctl:", control->nested_ctl); 3389 pr_err("%-20s%016llx\n", "nested_cr3:", control->nested_cr3); 3390 pr_err("%-20s%016llx\n", "avic_vapic_bar:", control->avic_vapic_bar); 3391 pr_err("%-20s%016llx\n", "ghcb:", control->ghcb_gpa); 3392 pr_err("%-20s%08x\n", "event_inj:", control->event_inj); 3393 pr_err("%-20s%08x\n", "event_inj_err:", control->event_inj_err); 3394 pr_err("%-20s%lld\n", "virt_ext:", control->virt_ext); 3395 pr_err("%-20s%016llx\n", "next_rip:", control->next_rip); 3396 pr_err("%-20s%016llx\n", "avic_backing_page:", control->avic_backing_page); 3397 pr_err("%-20s%016llx\n", "avic_logical_id:", control->avic_logical_id); 3398 pr_err("%-20s%016llx\n", "avic_physical_id:", control->avic_physical_id); 3399 pr_err("%-20s%016llx\n", "vmsa_pa:", control->vmsa_pa); 3400 pr_err("VMCB State Save Area:\n"); 3401 pr_err("%-5s s: %04x a: %04x l: %08x b: %016llx\n", 3402 "es:", 3403 save->es.selector, save->es.attrib, 3404 save->es.limit, save->es.base); 3405 pr_err("%-5s s: %04x a: %04x l: %08x b: %016llx\n", 3406 "cs:", 3407 save->cs.selector, save->cs.attrib, 3408 save->cs.limit, save->cs.base); 3409 pr_err("%-5s s: %04x a: %04x l: %08x b: %016llx\n", 3410 "ss:", 3411 save->ss.selector, save->ss.attrib, 3412 save->ss.limit, save->ss.base); 3413 pr_err("%-5s s: %04x a: %04x l: %08x b: %016llx\n", 3414 "ds:", 3415 save->ds.selector, save->ds.attrib, 3416 save->ds.limit, save->ds.base); 3417 pr_err("%-5s s: %04x a: %04x l: %08x b: %016llx\n", 3418 "fs:", 3419 save01->fs.selector, save01->fs.attrib, 3420 save01->fs.limit, save01->fs.base); 3421 pr_err("%-5s s: %04x a: %04x l: %08x b: %016llx\n", 3422 "gs:", 3423 save01->gs.selector, save01->gs.attrib, 3424 save01->gs.limit, save01->gs.base); 3425 pr_err("%-5s s: %04x a: %04x l: %08x b: %016llx\n", 3426 "gdtr:", 3427 save->gdtr.selector, save->gdtr.attrib, 3428 save->gdtr.limit, save->gdtr.base); 3429 pr_err("%-5s s: %04x a: %04x l: %08x b: %016llx\n", 3430 "ldtr:", 3431 save01->ldtr.selector, save01->ldtr.attrib, 3432 save01->ldtr.limit, save01->ldtr.base); 3433 pr_err("%-5s s: %04x a: %04x l: %08x b: %016llx\n", 3434 "idtr:", 3435 save->idtr.selector, save->idtr.attrib, 3436 save->idtr.limit, save->idtr.base); 3437 pr_err("%-5s s: %04x a: %04x l: %08x b: %016llx\n", 3438 "tr:", 3439 save01->tr.selector, save01->tr.attrib, 3440 save01->tr.limit, save01->tr.base); 3441 pr_err("vmpl: %d cpl: %d efer: %016llx\n", 3442 save->vmpl, save->cpl, save->efer); 3443 pr_err("%-15s %016llx %-13s %016llx\n", 3444 "cr0:", save->cr0, "cr2:", save->cr2); 3445 pr_err("%-15s %016llx %-13s %016llx\n", 3446 "cr3:", save->cr3, "cr4:", save->cr4); 3447 pr_err("%-15s %016llx %-13s %016llx\n", 3448 "dr6:", save->dr6, "dr7:", save->dr7); 3449 pr_err("%-15s %016llx %-13s %016llx\n", 3450 "rip:", save->rip, "rflags:", save->rflags); 3451 pr_err("%-15s %016llx %-13s %016llx\n", 3452 "rsp:", save->rsp, "rax:", save->rax); 3453 pr_err("%-15s %016llx %-13s %016llx\n", 3454 "star:", save01->star, "lstar:", save01->lstar); 3455 pr_err("%-15s %016llx %-13s %016llx\n", 3456 "cstar:", save01->cstar, "sfmask:", save01->sfmask); 3457 pr_err("%-15s %016llx %-13s %016llx\n", 3458 "kernel_gs_base:", save01->kernel_gs_base, 3459 "sysenter_cs:", save01->sysenter_cs); 3460 pr_err("%-15s %016llx %-13s %016llx\n", 3461 "sysenter_esp:", save01->sysenter_esp, 3462 "sysenter_eip:", save01->sysenter_eip); 3463 pr_err("%-15s %016llx %-13s %016llx\n", 3464 "gpat:", save->g_pat, "dbgctl:", save->dbgctl); 3465 pr_err("%-15s %016llx %-13s %016llx\n", 3466 "br_from:", save->br_from, "br_to:", save->br_to); 3467 pr_err("%-15s %016llx %-13s %016llx\n", 3468 "excp_from:", save->last_excp_from, 3469 "excp_to:", save->last_excp_to); 3470 } 3471 3472 static bool svm_check_exit_valid(u64 exit_code) 3473 { 3474 return (exit_code < ARRAY_SIZE(svm_exit_handlers) && 3475 svm_exit_handlers[exit_code]); 3476 } 3477 3478 static int svm_handle_invalid_exit(struct kvm_vcpu *vcpu, u64 exit_code) 3479 { 3480 vcpu_unimpl(vcpu, "svm: unexpected exit reason 0x%llx\n", exit_code); 3481 dump_vmcb(vcpu); 3482 vcpu->run->exit_reason = KVM_EXIT_INTERNAL_ERROR; 3483 vcpu->run->internal.suberror = KVM_INTERNAL_ERROR_UNEXPECTED_EXIT_REASON; 3484 vcpu->run->internal.ndata = 2; 3485 vcpu->run->internal.data[0] = exit_code; 3486 vcpu->run->internal.data[1] = vcpu->arch.last_vmentry_cpu; 3487 return 0; 3488 } 3489 3490 int svm_invoke_exit_handler(struct kvm_vcpu *vcpu, u64 exit_code) 3491 { 3492 if (!svm_check_exit_valid(exit_code)) 3493 return svm_handle_invalid_exit(vcpu, exit_code); 3494 3495 #ifdef CONFIG_RETPOLINE 3496 if (exit_code == SVM_EXIT_MSR) 3497 return msr_interception(vcpu); 3498 else if (exit_code == SVM_EXIT_VINTR) 3499 return interrupt_window_interception(vcpu); 3500 else if (exit_code == SVM_EXIT_INTR) 3501 return intr_interception(vcpu); 3502 else if (exit_code == SVM_EXIT_HLT) 3503 return kvm_emulate_halt(vcpu); 3504 else if (exit_code == SVM_EXIT_NPF) 3505 return npf_interception(vcpu); 3506 #endif 3507 return svm_exit_handlers[exit_code](vcpu); 3508 } 3509 3510 static void svm_get_exit_info(struct kvm_vcpu *vcpu, u32 *reason, 3511 u64 *info1, u64 *info2, 3512 u32 *intr_info, u32 *error_code) 3513 { 3514 struct vmcb_control_area *control = &to_svm(vcpu)->vmcb->control; 3515 3516 *reason = control->exit_code; 3517 *info1 = control->exit_info_1; 3518 *info2 = control->exit_info_2; 3519 *intr_info = control->exit_int_info; 3520 if ((*intr_info & SVM_EXITINTINFO_VALID) && 3521 (*intr_info & SVM_EXITINTINFO_VALID_ERR)) 3522 *error_code = control->exit_int_info_err; 3523 else 3524 *error_code = 0; 3525 } 3526 3527 static int svm_handle_exit(struct kvm_vcpu *vcpu, fastpath_t exit_fastpath) 3528 { 3529 struct vcpu_svm *svm = to_svm(vcpu); 3530 struct kvm_run *kvm_run = vcpu->run; 3531 u32 exit_code = svm->vmcb->control.exit_code; 3532 3533 /* SEV-ES guests must use the CR write traps to track CR registers. */ 3534 if (!sev_es_guest(vcpu->kvm)) { 3535 if (!svm_is_intercept(svm, INTERCEPT_CR0_WRITE)) 3536 vcpu->arch.cr0 = svm->vmcb->save.cr0; 3537 if (npt_enabled) 3538 vcpu->arch.cr3 = svm->vmcb->save.cr3; 3539 } 3540 3541 if (is_guest_mode(vcpu)) { 3542 int vmexit; 3543 3544 trace_kvm_nested_vmexit(vcpu, KVM_ISA_SVM); 3545 3546 vmexit = nested_svm_exit_special(svm); 3547 3548 if (vmexit == NESTED_EXIT_CONTINUE) 3549 vmexit = nested_svm_exit_handled(svm); 3550 3551 if (vmexit == NESTED_EXIT_DONE) 3552 return 1; 3553 } 3554 3555 if (svm->vmcb->control.exit_code == SVM_EXIT_ERR) { 3556 kvm_run->exit_reason = KVM_EXIT_FAIL_ENTRY; 3557 kvm_run->fail_entry.hardware_entry_failure_reason 3558 = svm->vmcb->control.exit_code; 3559 kvm_run->fail_entry.cpu = vcpu->arch.last_vmentry_cpu; 3560 dump_vmcb(vcpu); 3561 return 0; 3562 } 3563 3564 if (exit_fastpath != EXIT_FASTPATH_NONE) 3565 return 1; 3566 3567 return svm_invoke_exit_handler(vcpu, exit_code); 3568 } 3569 3570 static void pre_svm_run(struct kvm_vcpu *vcpu) 3571 { 3572 struct svm_cpu_data *sd = per_cpu_ptr(&svm_data, vcpu->cpu); 3573 struct vcpu_svm *svm = to_svm(vcpu); 3574 3575 /* 3576 * If the previous vmrun of the vmcb occurred on a different physical 3577 * cpu, then mark the vmcb dirty and assign a new asid. Hardware's 3578 * vmcb clean bits are per logical CPU, as are KVM's asid assignments. 3579 */ 3580 if (unlikely(svm->current_vmcb->cpu != vcpu->cpu)) { 3581 svm->current_vmcb->asid_generation = 0; 3582 vmcb_mark_all_dirty(svm->vmcb); 3583 svm->current_vmcb->cpu = vcpu->cpu; 3584 } 3585 3586 if (sev_guest(vcpu->kvm)) 3587 return pre_sev_run(svm, vcpu->cpu); 3588 3589 /* FIXME: handle wraparound of asid_generation */ 3590 if (svm->current_vmcb->asid_generation != sd->asid_generation) 3591 new_asid(svm, sd); 3592 } 3593 3594 static void svm_inject_nmi(struct kvm_vcpu *vcpu) 3595 { 3596 struct vcpu_svm *svm = to_svm(vcpu); 3597 3598 svm->vmcb->control.event_inj = SVM_EVTINJ_VALID | SVM_EVTINJ_TYPE_NMI; 3599 3600 if (svm->nmi_l1_to_l2) 3601 return; 3602 3603 svm->nmi_masked = true; 3604 svm_set_iret_intercept(svm); 3605 ++vcpu->stat.nmi_injections; 3606 } 3607 3608 static bool svm_is_vnmi_pending(struct kvm_vcpu *vcpu) 3609 { 3610 struct vcpu_svm *svm = to_svm(vcpu); 3611 3612 if (!is_vnmi_enabled(svm)) 3613 return false; 3614 3615 return !!(svm->vmcb->control.int_ctl & V_NMI_PENDING_MASK); 3616 } 3617 3618 static bool svm_set_vnmi_pending(struct kvm_vcpu *vcpu) 3619 { 3620 struct vcpu_svm *svm = to_svm(vcpu); 3621 3622 if (!is_vnmi_enabled(svm)) 3623 return false; 3624 3625 if (svm->vmcb->control.int_ctl & V_NMI_PENDING_MASK) 3626 return false; 3627 3628 svm->vmcb->control.int_ctl |= V_NMI_PENDING_MASK; 3629 vmcb_mark_dirty(svm->vmcb, VMCB_INTR); 3630 3631 /* 3632 * Because the pending NMI is serviced by hardware, KVM can't know when 3633 * the NMI is "injected", but for all intents and purposes, passing the 3634 * NMI off to hardware counts as injection. 3635 */ 3636 ++vcpu->stat.nmi_injections; 3637 3638 return true; 3639 } 3640 3641 static void svm_inject_irq(struct kvm_vcpu *vcpu, bool reinjected) 3642 { 3643 struct vcpu_svm *svm = to_svm(vcpu); 3644 u32 type; 3645 3646 if (vcpu->arch.interrupt.soft) { 3647 if (svm_update_soft_interrupt_rip(vcpu)) 3648 return; 3649 3650 type = SVM_EVTINJ_TYPE_SOFT; 3651 } else { 3652 type = SVM_EVTINJ_TYPE_INTR; 3653 } 3654 3655 trace_kvm_inj_virq(vcpu->arch.interrupt.nr, 3656 vcpu->arch.interrupt.soft, reinjected); 3657 ++vcpu->stat.irq_injections; 3658 3659 svm->vmcb->control.event_inj = vcpu->arch.interrupt.nr | 3660 SVM_EVTINJ_VALID | type; 3661 } 3662 3663 void svm_complete_interrupt_delivery(struct kvm_vcpu *vcpu, int delivery_mode, 3664 int trig_mode, int vector) 3665 { 3666 /* 3667 * apic->apicv_active must be read after vcpu->mode. 3668 * Pairs with smp_store_release in vcpu_enter_guest. 3669 */ 3670 bool in_guest_mode = (smp_load_acquire(&vcpu->mode) == IN_GUEST_MODE); 3671 3672 /* Note, this is called iff the local APIC is in-kernel. */ 3673 if (!READ_ONCE(vcpu->arch.apic->apicv_active)) { 3674 /* Process the interrupt via kvm_check_and_inject_events(). */ 3675 kvm_make_request(KVM_REQ_EVENT, vcpu); 3676 kvm_vcpu_kick(vcpu); 3677 return; 3678 } 3679 3680 trace_kvm_apicv_accept_irq(vcpu->vcpu_id, delivery_mode, trig_mode, vector); 3681 if (in_guest_mode) { 3682 /* 3683 * Signal the doorbell to tell hardware to inject the IRQ. If 3684 * the vCPU exits the guest before the doorbell chimes, hardware 3685 * will automatically process AVIC interrupts at the next VMRUN. 3686 */ 3687 avic_ring_doorbell(vcpu); 3688 } else { 3689 /* 3690 * Wake the vCPU if it was blocking. KVM will then detect the 3691 * pending IRQ when checking if the vCPU has a wake event. 3692 */ 3693 kvm_vcpu_wake_up(vcpu); 3694 } 3695 } 3696 3697 static void svm_deliver_interrupt(struct kvm_lapic *apic, int delivery_mode, 3698 int trig_mode, int vector) 3699 { 3700 kvm_lapic_set_irr(vector, apic); 3701 3702 /* 3703 * Pairs with the smp_mb_*() after setting vcpu->guest_mode in 3704 * vcpu_enter_guest() to ensure the write to the vIRR is ordered before 3705 * the read of guest_mode. This guarantees that either VMRUN will see 3706 * and process the new vIRR entry, or that svm_complete_interrupt_delivery 3707 * will signal the doorbell if the CPU has already entered the guest. 3708 */ 3709 smp_mb__after_atomic(); 3710 svm_complete_interrupt_delivery(apic->vcpu, delivery_mode, trig_mode, vector); 3711 } 3712 3713 static void svm_update_cr8_intercept(struct kvm_vcpu *vcpu, int tpr, int irr) 3714 { 3715 struct vcpu_svm *svm = to_svm(vcpu); 3716 3717 /* 3718 * SEV-ES guests must always keep the CR intercepts cleared. CR 3719 * tracking is done using the CR write traps. 3720 */ 3721 if (sev_es_guest(vcpu->kvm)) 3722 return; 3723 3724 if (nested_svm_virtualize_tpr(vcpu)) 3725 return; 3726 3727 svm_clr_intercept(svm, INTERCEPT_CR8_WRITE); 3728 3729 if (irr == -1) 3730 return; 3731 3732 if (tpr >= irr) 3733 svm_set_intercept(svm, INTERCEPT_CR8_WRITE); 3734 } 3735 3736 static bool svm_get_nmi_mask(struct kvm_vcpu *vcpu) 3737 { 3738 struct vcpu_svm *svm = to_svm(vcpu); 3739 3740 if (is_vnmi_enabled(svm)) 3741 return svm->vmcb->control.int_ctl & V_NMI_BLOCKING_MASK; 3742 else 3743 return svm->nmi_masked; 3744 } 3745 3746 static void svm_set_nmi_mask(struct kvm_vcpu *vcpu, bool masked) 3747 { 3748 struct vcpu_svm *svm = to_svm(vcpu); 3749 3750 if (is_vnmi_enabled(svm)) { 3751 if (masked) 3752 svm->vmcb->control.int_ctl |= V_NMI_BLOCKING_MASK; 3753 else 3754 svm->vmcb->control.int_ctl &= ~V_NMI_BLOCKING_MASK; 3755 3756 } else { 3757 svm->nmi_masked = masked; 3758 if (masked) 3759 svm_set_iret_intercept(svm); 3760 else 3761 svm_clr_iret_intercept(svm); 3762 } 3763 } 3764 3765 bool svm_nmi_blocked(struct kvm_vcpu *vcpu) 3766 { 3767 struct vcpu_svm *svm = to_svm(vcpu); 3768 struct vmcb *vmcb = svm->vmcb; 3769 3770 if (!gif_set(svm)) 3771 return true; 3772 3773 if (is_guest_mode(vcpu) && nested_exit_on_nmi(svm)) 3774 return false; 3775 3776 if (svm_get_nmi_mask(vcpu)) 3777 return true; 3778 3779 return vmcb->control.int_state & SVM_INTERRUPT_SHADOW_MASK; 3780 } 3781 3782 static int svm_nmi_allowed(struct kvm_vcpu *vcpu, bool for_injection) 3783 { 3784 struct vcpu_svm *svm = to_svm(vcpu); 3785 if (svm->nested.nested_run_pending) 3786 return -EBUSY; 3787 3788 if (svm_nmi_blocked(vcpu)) 3789 return 0; 3790 3791 /* An NMI must not be injected into L2 if it's supposed to VM-Exit. */ 3792 if (for_injection && is_guest_mode(vcpu) && nested_exit_on_nmi(svm)) 3793 return -EBUSY; 3794 return 1; 3795 } 3796 3797 bool svm_interrupt_blocked(struct kvm_vcpu *vcpu) 3798 { 3799 struct vcpu_svm *svm = to_svm(vcpu); 3800 struct vmcb *vmcb = svm->vmcb; 3801 3802 if (!gif_set(svm)) 3803 return true; 3804 3805 if (is_guest_mode(vcpu)) { 3806 /* As long as interrupts are being delivered... */ 3807 if ((svm->nested.ctl.int_ctl & V_INTR_MASKING_MASK) 3808 ? !(svm->vmcb01.ptr->save.rflags & X86_EFLAGS_IF) 3809 : !(kvm_get_rflags(vcpu) & X86_EFLAGS_IF)) 3810 return true; 3811 3812 /* ... vmexits aren't blocked by the interrupt shadow */ 3813 if (nested_exit_on_intr(svm)) 3814 return false; 3815 } else { 3816 if (!svm_get_if_flag(vcpu)) 3817 return true; 3818 } 3819 3820 return (vmcb->control.int_state & SVM_INTERRUPT_SHADOW_MASK); 3821 } 3822 3823 static int svm_interrupt_allowed(struct kvm_vcpu *vcpu, bool for_injection) 3824 { 3825 struct vcpu_svm *svm = to_svm(vcpu); 3826 3827 if (svm->nested.nested_run_pending) 3828 return -EBUSY; 3829 3830 if (svm_interrupt_blocked(vcpu)) 3831 return 0; 3832 3833 /* 3834 * An IRQ must not be injected into L2 if it's supposed to VM-Exit, 3835 * e.g. if the IRQ arrived asynchronously after checking nested events. 3836 */ 3837 if (for_injection && is_guest_mode(vcpu) && nested_exit_on_intr(svm)) 3838 return -EBUSY; 3839 3840 return 1; 3841 } 3842 3843 static void svm_enable_irq_window(struct kvm_vcpu *vcpu) 3844 { 3845 struct vcpu_svm *svm = to_svm(vcpu); 3846 3847 /* 3848 * In case GIF=0 we can't rely on the CPU to tell us when GIF becomes 3849 * 1, because that's a separate STGI/VMRUN intercept. The next time we 3850 * get that intercept, this function will be called again though and 3851 * we'll get the vintr intercept. However, if the vGIF feature is 3852 * enabled, the STGI interception will not occur. Enable the irq 3853 * window under the assumption that the hardware will set the GIF. 3854 */ 3855 if (vgif || gif_set(svm)) { 3856 /* 3857 * IRQ window is not needed when AVIC is enabled, 3858 * unless we have pending ExtINT since it cannot be injected 3859 * via AVIC. In such case, KVM needs to temporarily disable AVIC, 3860 * and fallback to injecting IRQ via V_IRQ. 3861 * 3862 * If running nested, AVIC is already locally inhibited 3863 * on this vCPU, therefore there is no need to request 3864 * the VM wide AVIC inhibition. 3865 */ 3866 if (!is_guest_mode(vcpu)) 3867 kvm_set_apicv_inhibit(vcpu->kvm, APICV_INHIBIT_REASON_IRQWIN); 3868 3869 svm_set_vintr(svm); 3870 } 3871 } 3872 3873 static void svm_enable_nmi_window(struct kvm_vcpu *vcpu) 3874 { 3875 struct vcpu_svm *svm = to_svm(vcpu); 3876 3877 /* 3878 * If NMIs are outright masked, i.e. the vCPU is already handling an 3879 * NMI, and KVM has not yet intercepted an IRET, then there is nothing 3880 * more to do at this time as KVM has already enabled IRET intercepts. 3881 * If KVM has already intercepted IRET, then single-step over the IRET, 3882 * as NMIs aren't architecturally unmasked until the IRET completes. 3883 * 3884 * If vNMI is enabled, KVM should never request an NMI window if NMIs 3885 * are masked, as KVM allows at most one to-be-injected NMI and one 3886 * pending NMI. If two NMIs arrive simultaneously, KVM will inject one 3887 * NMI and set V_NMI_PENDING for the other, but if and only if NMIs are 3888 * unmasked. KVM _will_ request an NMI window in some situations, e.g. 3889 * if the vCPU is in an STI shadow or if GIF=0, KVM can't immediately 3890 * inject the NMI. In those situations, KVM needs to single-step over 3891 * the STI shadow or intercept STGI. 3892 */ 3893 if (svm_get_nmi_mask(vcpu)) { 3894 WARN_ON_ONCE(is_vnmi_enabled(svm)); 3895 3896 if (!svm->awaiting_iret_completion) 3897 return; /* IRET will cause a vm exit */ 3898 } 3899 3900 /* 3901 * SEV-ES guests are responsible for signaling when a vCPU is ready to 3902 * receive a new NMI, as SEV-ES guests can't be single-stepped, i.e. 3903 * KVM can't intercept and single-step IRET to detect when NMIs are 3904 * unblocked (architecturally speaking). See SVM_VMGEXIT_NMI_COMPLETE. 3905 * 3906 * Note, GIF is guaranteed to be '1' for SEV-ES guests as hardware 3907 * ignores SEV-ES guest writes to EFER.SVME *and* CLGI/STGI are not 3908 * supported NAEs in the GHCB protocol. 3909 */ 3910 if (sev_es_guest(vcpu->kvm)) 3911 return; 3912 3913 if (!gif_set(svm)) { 3914 if (vgif) 3915 svm_set_intercept(svm, INTERCEPT_STGI); 3916 return; /* STGI will cause a vm exit */ 3917 } 3918 3919 /* 3920 * Something prevents NMI from been injected. Single step over possible 3921 * problem (IRET or exception injection or interrupt shadow) 3922 */ 3923 svm->nmi_singlestep_guest_rflags = svm_get_rflags(vcpu); 3924 svm->nmi_singlestep = true; 3925 svm->vmcb->save.rflags |= (X86_EFLAGS_TF | X86_EFLAGS_RF); 3926 } 3927 3928 static void svm_flush_tlb_asid(struct kvm_vcpu *vcpu) 3929 { 3930 struct vcpu_svm *svm = to_svm(vcpu); 3931 3932 /* 3933 * Unlike VMX, SVM doesn't provide a way to flush only NPT TLB entries. 3934 * A TLB flush for the current ASID flushes both "host" and "guest" TLB 3935 * entries, and thus is a superset of Hyper-V's fine grained flushing. 3936 */ 3937 kvm_hv_vcpu_purge_flush_tlb(vcpu); 3938 3939 /* 3940 * Flush only the current ASID even if the TLB flush was invoked via 3941 * kvm_flush_remote_tlbs(). Although flushing remote TLBs requires all 3942 * ASIDs to be flushed, KVM uses a single ASID for L1 and L2, and 3943 * unconditionally does a TLB flush on both nested VM-Enter and nested 3944 * VM-Exit (via kvm_mmu_reset_context()). 3945 */ 3946 if (static_cpu_has(X86_FEATURE_FLUSHBYASID)) 3947 svm->vmcb->control.tlb_ctl = TLB_CONTROL_FLUSH_ASID; 3948 else 3949 svm->current_vmcb->asid_generation--; 3950 } 3951 3952 static void svm_flush_tlb_current(struct kvm_vcpu *vcpu) 3953 { 3954 hpa_t root_tdp = vcpu->arch.mmu->root.hpa; 3955 3956 /* 3957 * When running on Hyper-V with EnlightenedNptTlb enabled, explicitly 3958 * flush the NPT mappings via hypercall as flushing the ASID only 3959 * affects virtual to physical mappings, it does not invalidate guest 3960 * physical to host physical mappings. 3961 */ 3962 if (svm_hv_is_enlightened_tlb_enabled(vcpu) && VALID_PAGE(root_tdp)) 3963 hyperv_flush_guest_mapping(root_tdp); 3964 3965 svm_flush_tlb_asid(vcpu); 3966 } 3967 3968 static void svm_flush_tlb_all(struct kvm_vcpu *vcpu) 3969 { 3970 /* 3971 * When running on Hyper-V with EnlightenedNptTlb enabled, remote TLB 3972 * flushes should be routed to hv_flush_remote_tlbs() without requesting 3973 * a "regular" remote flush. Reaching this point means either there's 3974 * a KVM bug or a prior hv_flush_remote_tlbs() call failed, both of 3975 * which might be fatal to the guest. Yell, but try to recover. 3976 */ 3977 if (WARN_ON_ONCE(svm_hv_is_enlightened_tlb_enabled(vcpu))) 3978 hv_flush_remote_tlbs(vcpu->kvm); 3979 3980 svm_flush_tlb_asid(vcpu); 3981 } 3982 3983 static void svm_flush_tlb_gva(struct kvm_vcpu *vcpu, gva_t gva) 3984 { 3985 struct vcpu_svm *svm = to_svm(vcpu); 3986 3987 invlpga(gva, svm->vmcb->control.asid); 3988 } 3989 3990 static inline void sync_cr8_to_lapic(struct kvm_vcpu *vcpu) 3991 { 3992 struct vcpu_svm *svm = to_svm(vcpu); 3993 3994 if (nested_svm_virtualize_tpr(vcpu)) 3995 return; 3996 3997 if (!svm_is_intercept(svm, INTERCEPT_CR8_WRITE)) { 3998 int cr8 = svm->vmcb->control.int_ctl & V_TPR_MASK; 3999 kvm_set_cr8(vcpu, cr8); 4000 } 4001 } 4002 4003 static inline void sync_lapic_to_cr8(struct kvm_vcpu *vcpu) 4004 { 4005 struct vcpu_svm *svm = to_svm(vcpu); 4006 u64 cr8; 4007 4008 if (nested_svm_virtualize_tpr(vcpu) || 4009 kvm_vcpu_apicv_active(vcpu)) 4010 return; 4011 4012 cr8 = kvm_get_cr8(vcpu); 4013 svm->vmcb->control.int_ctl &= ~V_TPR_MASK; 4014 svm->vmcb->control.int_ctl |= cr8 & V_TPR_MASK; 4015 } 4016 4017 static void svm_complete_soft_interrupt(struct kvm_vcpu *vcpu, u8 vector, 4018 int type) 4019 { 4020 bool is_exception = (type == SVM_EXITINTINFO_TYPE_EXEPT); 4021 bool is_soft = (type == SVM_EXITINTINFO_TYPE_SOFT); 4022 struct vcpu_svm *svm = to_svm(vcpu); 4023 4024 /* 4025 * If NRIPS is enabled, KVM must snapshot the pre-VMRUN next_rip that's 4026 * associated with the original soft exception/interrupt. next_rip is 4027 * cleared on all exits that can occur while vectoring an event, so KVM 4028 * needs to manually set next_rip for re-injection. Unlike the !nrips 4029 * case below, this needs to be done if and only if KVM is re-injecting 4030 * the same event, i.e. if the event is a soft exception/interrupt, 4031 * otherwise next_rip is unused on VMRUN. 4032 */ 4033 if (nrips && (is_soft || (is_exception && kvm_exception_is_soft(vector))) && 4034 kvm_is_linear_rip(vcpu, svm->soft_int_old_rip + svm->soft_int_csbase)) 4035 svm->vmcb->control.next_rip = svm->soft_int_next_rip; 4036 /* 4037 * If NRIPS isn't enabled, KVM must manually advance RIP prior to 4038 * injecting the soft exception/interrupt. That advancement needs to 4039 * be unwound if vectoring didn't complete. Note, the new event may 4040 * not be the injected event, e.g. if KVM injected an INTn, the INTn 4041 * hit a #NP in the guest, and the #NP encountered a #PF, the #NP will 4042 * be the reported vectored event, but RIP still needs to be unwound. 4043 */ 4044 else if (!nrips && (is_soft || is_exception) && 4045 kvm_is_linear_rip(vcpu, svm->soft_int_next_rip + svm->soft_int_csbase)) 4046 kvm_rip_write(vcpu, svm->soft_int_old_rip); 4047 } 4048 4049 static void svm_complete_interrupts(struct kvm_vcpu *vcpu) 4050 { 4051 struct vcpu_svm *svm = to_svm(vcpu); 4052 u8 vector; 4053 int type; 4054 u32 exitintinfo = svm->vmcb->control.exit_int_info; 4055 bool nmi_l1_to_l2 = svm->nmi_l1_to_l2; 4056 bool soft_int_injected = svm->soft_int_injected; 4057 4058 svm->nmi_l1_to_l2 = false; 4059 svm->soft_int_injected = false; 4060 4061 /* 4062 * If we've made progress since setting awaiting_iret_completion, we've 4063 * executed an IRET and can allow NMI injection. 4064 */ 4065 if (svm->awaiting_iret_completion && 4066 kvm_rip_read(vcpu) != svm->nmi_iret_rip) { 4067 svm->awaiting_iret_completion = false; 4068 svm->nmi_masked = false; 4069 kvm_make_request(KVM_REQ_EVENT, vcpu); 4070 } 4071 4072 vcpu->arch.nmi_injected = false; 4073 kvm_clear_exception_queue(vcpu); 4074 kvm_clear_interrupt_queue(vcpu); 4075 4076 if (!(exitintinfo & SVM_EXITINTINFO_VALID)) 4077 return; 4078 4079 kvm_make_request(KVM_REQ_EVENT, vcpu); 4080 4081 vector = exitintinfo & SVM_EXITINTINFO_VEC_MASK; 4082 type = exitintinfo & SVM_EXITINTINFO_TYPE_MASK; 4083 4084 if (soft_int_injected) 4085 svm_complete_soft_interrupt(vcpu, vector, type); 4086 4087 switch (type) { 4088 case SVM_EXITINTINFO_TYPE_NMI: 4089 vcpu->arch.nmi_injected = true; 4090 svm->nmi_l1_to_l2 = nmi_l1_to_l2; 4091 break; 4092 case SVM_EXITINTINFO_TYPE_EXEPT: 4093 /* 4094 * Never re-inject a #VC exception. 4095 */ 4096 if (vector == X86_TRAP_VC) 4097 break; 4098 4099 if (exitintinfo & SVM_EXITINTINFO_VALID_ERR) { 4100 u32 err = svm->vmcb->control.exit_int_info_err; 4101 kvm_requeue_exception_e(vcpu, vector, err); 4102 4103 } else 4104 kvm_requeue_exception(vcpu, vector); 4105 break; 4106 case SVM_EXITINTINFO_TYPE_INTR: 4107 kvm_queue_interrupt(vcpu, vector, false); 4108 break; 4109 case SVM_EXITINTINFO_TYPE_SOFT: 4110 kvm_queue_interrupt(vcpu, vector, true); 4111 break; 4112 default: 4113 break; 4114 } 4115 4116 } 4117 4118 static void svm_cancel_injection(struct kvm_vcpu *vcpu) 4119 { 4120 struct vcpu_svm *svm = to_svm(vcpu); 4121 struct vmcb_control_area *control = &svm->vmcb->control; 4122 4123 control->exit_int_info = control->event_inj; 4124 control->exit_int_info_err = control->event_inj_err; 4125 control->event_inj = 0; 4126 svm_complete_interrupts(vcpu); 4127 } 4128 4129 static int svm_vcpu_pre_run(struct kvm_vcpu *vcpu) 4130 { 4131 return 1; 4132 } 4133 4134 static fastpath_t svm_exit_handlers_fastpath(struct kvm_vcpu *vcpu) 4135 { 4136 if (to_svm(vcpu)->vmcb->control.exit_code == SVM_EXIT_MSR && 4137 to_svm(vcpu)->vmcb->control.exit_info_1) 4138 return handle_fastpath_set_msr_irqoff(vcpu); 4139 4140 return EXIT_FASTPATH_NONE; 4141 } 4142 4143 static noinstr void svm_vcpu_enter_exit(struct kvm_vcpu *vcpu, bool spec_ctrl_intercepted) 4144 { 4145 struct vcpu_svm *svm = to_svm(vcpu); 4146 4147 guest_state_enter_irqoff(); 4148 4149 amd_clear_divider(); 4150 4151 if (sev_es_guest(vcpu->kvm)) 4152 __svm_sev_es_vcpu_run(svm, spec_ctrl_intercepted); 4153 else 4154 __svm_vcpu_run(svm, spec_ctrl_intercepted); 4155 4156 guest_state_exit_irqoff(); 4157 } 4158 4159 static __no_kcsan fastpath_t svm_vcpu_run(struct kvm_vcpu *vcpu) 4160 { 4161 struct vcpu_svm *svm = to_svm(vcpu); 4162 bool spec_ctrl_intercepted = msr_write_intercepted(vcpu, MSR_IA32_SPEC_CTRL); 4163 4164 trace_kvm_entry(vcpu); 4165 4166 svm->vmcb->save.rax = vcpu->arch.regs[VCPU_REGS_RAX]; 4167 svm->vmcb->save.rsp = vcpu->arch.regs[VCPU_REGS_RSP]; 4168 svm->vmcb->save.rip = vcpu->arch.regs[VCPU_REGS_RIP]; 4169 4170 /* 4171 * Disable singlestep if we're injecting an interrupt/exception. 4172 * We don't want our modified rflags to be pushed on the stack where 4173 * we might not be able to easily reset them if we disabled NMI 4174 * singlestep later. 4175 */ 4176 if (svm->nmi_singlestep && svm->vmcb->control.event_inj) { 4177 /* 4178 * Event injection happens before external interrupts cause a 4179 * vmexit and interrupts are disabled here, so smp_send_reschedule 4180 * is enough to force an immediate vmexit. 4181 */ 4182 disable_nmi_singlestep(svm); 4183 smp_send_reschedule(vcpu->cpu); 4184 } 4185 4186 pre_svm_run(vcpu); 4187 4188 sync_lapic_to_cr8(vcpu); 4189 4190 if (unlikely(svm->asid != svm->vmcb->control.asid)) { 4191 svm->vmcb->control.asid = svm->asid; 4192 vmcb_mark_dirty(svm->vmcb, VMCB_ASID); 4193 } 4194 svm->vmcb->save.cr2 = vcpu->arch.cr2; 4195 4196 svm_hv_update_vp_id(svm->vmcb, vcpu); 4197 4198 /* 4199 * Run with all-zero DR6 unless needed, so that we can get the exact cause 4200 * of a #DB. 4201 */ 4202 if (unlikely(vcpu->arch.switch_db_regs & KVM_DEBUGREG_WONT_EXIT)) 4203 svm_set_dr6(svm, vcpu->arch.dr6); 4204 else 4205 svm_set_dr6(svm, DR6_ACTIVE_LOW); 4206 4207 clgi(); 4208 kvm_load_guest_xsave_state(vcpu); 4209 4210 kvm_wait_lapic_expire(vcpu); 4211 4212 /* 4213 * If this vCPU has touched SPEC_CTRL, restore the guest's value if 4214 * it's non-zero. Since vmentry is serialising on affected CPUs, there 4215 * is no need to worry about the conditional branch over the wrmsr 4216 * being speculatively taken. 4217 */ 4218 if (!static_cpu_has(X86_FEATURE_V_SPEC_CTRL)) 4219 x86_spec_ctrl_set_guest(svm->virt_spec_ctrl); 4220 4221 svm_vcpu_enter_exit(vcpu, spec_ctrl_intercepted); 4222 4223 if (!static_cpu_has(X86_FEATURE_V_SPEC_CTRL)) 4224 x86_spec_ctrl_restore_host(svm->virt_spec_ctrl); 4225 4226 if (!sev_es_guest(vcpu->kvm)) { 4227 vcpu->arch.cr2 = svm->vmcb->save.cr2; 4228 vcpu->arch.regs[VCPU_REGS_RAX] = svm->vmcb->save.rax; 4229 vcpu->arch.regs[VCPU_REGS_RSP] = svm->vmcb->save.rsp; 4230 vcpu->arch.regs[VCPU_REGS_RIP] = svm->vmcb->save.rip; 4231 } 4232 vcpu->arch.regs_dirty = 0; 4233 4234 if (unlikely(svm->vmcb->control.exit_code == SVM_EXIT_NMI)) 4235 kvm_before_interrupt(vcpu, KVM_HANDLING_NMI); 4236 4237 kvm_load_host_xsave_state(vcpu); 4238 stgi(); 4239 4240 /* Any pending NMI will happen here */ 4241 4242 if (unlikely(svm->vmcb->control.exit_code == SVM_EXIT_NMI)) 4243 kvm_after_interrupt(vcpu); 4244 4245 sync_cr8_to_lapic(vcpu); 4246 4247 svm->next_rip = 0; 4248 if (is_guest_mode(vcpu)) { 4249 nested_sync_control_from_vmcb02(svm); 4250 4251 /* Track VMRUNs that have made past consistency checking */ 4252 if (svm->nested.nested_run_pending && 4253 svm->vmcb->control.exit_code != SVM_EXIT_ERR) 4254 ++vcpu->stat.nested_run; 4255 4256 svm->nested.nested_run_pending = 0; 4257 } 4258 4259 svm->vmcb->control.tlb_ctl = TLB_CONTROL_DO_NOTHING; 4260 vmcb_mark_all_clean(svm->vmcb); 4261 4262 /* if exit due to PF check for async PF */ 4263 if (svm->vmcb->control.exit_code == SVM_EXIT_EXCP_BASE + PF_VECTOR) 4264 vcpu->arch.apf.host_apf_flags = 4265 kvm_read_and_reset_apf_flags(); 4266 4267 vcpu->arch.regs_avail &= ~SVM_REGS_LAZY_LOAD_SET; 4268 4269 /* 4270 * We need to handle MC intercepts here before the vcpu has a chance to 4271 * change the physical cpu 4272 */ 4273 if (unlikely(svm->vmcb->control.exit_code == 4274 SVM_EXIT_EXCP_BASE + MC_VECTOR)) 4275 svm_handle_mce(vcpu); 4276 4277 trace_kvm_exit(vcpu, KVM_ISA_SVM); 4278 4279 svm_complete_interrupts(vcpu); 4280 4281 if (is_guest_mode(vcpu)) 4282 return EXIT_FASTPATH_NONE; 4283 4284 return svm_exit_handlers_fastpath(vcpu); 4285 } 4286 4287 static void svm_load_mmu_pgd(struct kvm_vcpu *vcpu, hpa_t root_hpa, 4288 int root_level) 4289 { 4290 struct vcpu_svm *svm = to_svm(vcpu); 4291 unsigned long cr3; 4292 4293 if (npt_enabled) { 4294 svm->vmcb->control.nested_cr3 = __sme_set(root_hpa); 4295 vmcb_mark_dirty(svm->vmcb, VMCB_NPT); 4296 4297 hv_track_root_tdp(vcpu, root_hpa); 4298 4299 cr3 = vcpu->arch.cr3; 4300 } else if (root_level >= PT64_ROOT_4LEVEL) { 4301 cr3 = __sme_set(root_hpa) | kvm_get_active_pcid(vcpu); 4302 } else { 4303 /* PCID in the guest should be impossible with a 32-bit MMU. */ 4304 WARN_ON_ONCE(kvm_get_active_pcid(vcpu)); 4305 cr3 = root_hpa; 4306 } 4307 4308 svm->vmcb->save.cr3 = cr3; 4309 vmcb_mark_dirty(svm->vmcb, VMCB_CR); 4310 } 4311 4312 static void 4313 svm_patch_hypercall(struct kvm_vcpu *vcpu, unsigned char *hypercall) 4314 { 4315 /* 4316 * Patch in the VMMCALL instruction: 4317 */ 4318 hypercall[0] = 0x0f; 4319 hypercall[1] = 0x01; 4320 hypercall[2] = 0xd9; 4321 } 4322 4323 /* 4324 * The kvm parameter can be NULL (module initialization, or invocation before 4325 * VM creation). Be sure to check the kvm parameter before using it. 4326 */ 4327 static bool svm_has_emulated_msr(struct kvm *kvm, u32 index) 4328 { 4329 switch (index) { 4330 case MSR_IA32_MCG_EXT_CTL: 4331 case KVM_FIRST_EMULATED_VMX_MSR ... KVM_LAST_EMULATED_VMX_MSR: 4332 return false; 4333 case MSR_IA32_SMBASE: 4334 if (!IS_ENABLED(CONFIG_KVM_SMM)) 4335 return false; 4336 /* SEV-ES guests do not support SMM, so report false */ 4337 if (kvm && sev_es_guest(kvm)) 4338 return false; 4339 break; 4340 default: 4341 break; 4342 } 4343 4344 return true; 4345 } 4346 4347 static void svm_vcpu_after_set_cpuid(struct kvm_vcpu *vcpu) 4348 { 4349 struct vcpu_svm *svm = to_svm(vcpu); 4350 4351 /* 4352 * SVM doesn't provide a way to disable just XSAVES in the guest, KVM 4353 * can only disable all variants of by disallowing CR4.OSXSAVE from 4354 * being set. As a result, if the host has XSAVE and XSAVES, and the 4355 * guest has XSAVE enabled, the guest can execute XSAVES without 4356 * faulting. Treat XSAVES as enabled in this case regardless of 4357 * whether it's advertised to the guest so that KVM context switches 4358 * XSS on VM-Enter/VM-Exit. Failure to do so would effectively give 4359 * the guest read/write access to the host's XSS. 4360 */ 4361 if (boot_cpu_has(X86_FEATURE_XSAVE) && 4362 boot_cpu_has(X86_FEATURE_XSAVES) && 4363 guest_cpuid_has(vcpu, X86_FEATURE_XSAVE)) 4364 kvm_governed_feature_set(vcpu, X86_FEATURE_XSAVES); 4365 4366 kvm_governed_feature_check_and_set(vcpu, X86_FEATURE_NRIPS); 4367 kvm_governed_feature_check_and_set(vcpu, X86_FEATURE_TSCRATEMSR); 4368 kvm_governed_feature_check_and_set(vcpu, X86_FEATURE_LBRV); 4369 4370 /* 4371 * Intercept VMLOAD if the vCPU mode is Intel in order to emulate that 4372 * VMLOAD drops bits 63:32 of SYSENTER (ignoring the fact that exposing 4373 * SVM on Intel is bonkers and extremely unlikely to work). 4374 */ 4375 if (!guest_cpuid_is_intel(vcpu)) 4376 kvm_governed_feature_check_and_set(vcpu, X86_FEATURE_V_VMSAVE_VMLOAD); 4377 4378 kvm_governed_feature_check_and_set(vcpu, X86_FEATURE_PAUSEFILTER); 4379 kvm_governed_feature_check_and_set(vcpu, X86_FEATURE_PFTHRESHOLD); 4380 kvm_governed_feature_check_and_set(vcpu, X86_FEATURE_VGIF); 4381 kvm_governed_feature_check_and_set(vcpu, X86_FEATURE_VNMI); 4382 4383 svm_recalc_instruction_intercepts(vcpu, svm); 4384 4385 if (boot_cpu_has(X86_FEATURE_IBPB)) 4386 set_msr_interception(vcpu, svm->msrpm, MSR_IA32_PRED_CMD, 0, 4387 !!guest_has_pred_cmd_msr(vcpu)); 4388 4389 if (boot_cpu_has(X86_FEATURE_FLUSH_L1D)) 4390 set_msr_interception(vcpu, svm->msrpm, MSR_IA32_FLUSH_CMD, 0, 4391 !!guest_cpuid_has(vcpu, X86_FEATURE_FLUSH_L1D)); 4392 4393 if (sev_guest(vcpu->kvm)) 4394 sev_vcpu_after_set_cpuid(svm); 4395 4396 init_vmcb_after_set_cpuid(vcpu); 4397 } 4398 4399 static bool svm_has_wbinvd_exit(void) 4400 { 4401 return true; 4402 } 4403 4404 #define PRE_EX(exit) { .exit_code = (exit), \ 4405 .stage = X86_ICPT_PRE_EXCEPT, } 4406 #define POST_EX(exit) { .exit_code = (exit), \ 4407 .stage = X86_ICPT_POST_EXCEPT, } 4408 #define POST_MEM(exit) { .exit_code = (exit), \ 4409 .stage = X86_ICPT_POST_MEMACCESS, } 4410 4411 static const struct __x86_intercept { 4412 u32 exit_code; 4413 enum x86_intercept_stage stage; 4414 } x86_intercept_map[] = { 4415 [x86_intercept_cr_read] = POST_EX(SVM_EXIT_READ_CR0), 4416 [x86_intercept_cr_write] = POST_EX(SVM_EXIT_WRITE_CR0), 4417 [x86_intercept_clts] = POST_EX(SVM_EXIT_WRITE_CR0), 4418 [x86_intercept_lmsw] = POST_EX(SVM_EXIT_WRITE_CR0), 4419 [x86_intercept_smsw] = POST_EX(SVM_EXIT_READ_CR0), 4420 [x86_intercept_dr_read] = POST_EX(SVM_EXIT_READ_DR0), 4421 [x86_intercept_dr_write] = POST_EX(SVM_EXIT_WRITE_DR0), 4422 [x86_intercept_sldt] = POST_EX(SVM_EXIT_LDTR_READ), 4423 [x86_intercept_str] = POST_EX(SVM_EXIT_TR_READ), 4424 [x86_intercept_lldt] = POST_EX(SVM_EXIT_LDTR_WRITE), 4425 [x86_intercept_ltr] = POST_EX(SVM_EXIT_TR_WRITE), 4426 [x86_intercept_sgdt] = POST_EX(SVM_EXIT_GDTR_READ), 4427 [x86_intercept_sidt] = POST_EX(SVM_EXIT_IDTR_READ), 4428 [x86_intercept_lgdt] = POST_EX(SVM_EXIT_GDTR_WRITE), 4429 [x86_intercept_lidt] = POST_EX(SVM_EXIT_IDTR_WRITE), 4430 [x86_intercept_vmrun] = POST_EX(SVM_EXIT_VMRUN), 4431 [x86_intercept_vmmcall] = POST_EX(SVM_EXIT_VMMCALL), 4432 [x86_intercept_vmload] = POST_EX(SVM_EXIT_VMLOAD), 4433 [x86_intercept_vmsave] = POST_EX(SVM_EXIT_VMSAVE), 4434 [x86_intercept_stgi] = POST_EX(SVM_EXIT_STGI), 4435 [x86_intercept_clgi] = POST_EX(SVM_EXIT_CLGI), 4436 [x86_intercept_skinit] = POST_EX(SVM_EXIT_SKINIT), 4437 [x86_intercept_invlpga] = POST_EX(SVM_EXIT_INVLPGA), 4438 [x86_intercept_rdtscp] = POST_EX(SVM_EXIT_RDTSCP), 4439 [x86_intercept_monitor] = POST_MEM(SVM_EXIT_MONITOR), 4440 [x86_intercept_mwait] = POST_EX(SVM_EXIT_MWAIT), 4441 [x86_intercept_invlpg] = POST_EX(SVM_EXIT_INVLPG), 4442 [x86_intercept_invd] = POST_EX(SVM_EXIT_INVD), 4443 [x86_intercept_wbinvd] = POST_EX(SVM_EXIT_WBINVD), 4444 [x86_intercept_wrmsr] = POST_EX(SVM_EXIT_MSR), 4445 [x86_intercept_rdtsc] = POST_EX(SVM_EXIT_RDTSC), 4446 [x86_intercept_rdmsr] = POST_EX(SVM_EXIT_MSR), 4447 [x86_intercept_rdpmc] = POST_EX(SVM_EXIT_RDPMC), 4448 [x86_intercept_cpuid] = PRE_EX(SVM_EXIT_CPUID), 4449 [x86_intercept_rsm] = PRE_EX(SVM_EXIT_RSM), 4450 [x86_intercept_pause] = PRE_EX(SVM_EXIT_PAUSE), 4451 [x86_intercept_pushf] = PRE_EX(SVM_EXIT_PUSHF), 4452 [x86_intercept_popf] = PRE_EX(SVM_EXIT_POPF), 4453 [x86_intercept_intn] = PRE_EX(SVM_EXIT_SWINT), 4454 [x86_intercept_iret] = PRE_EX(SVM_EXIT_IRET), 4455 [x86_intercept_icebp] = PRE_EX(SVM_EXIT_ICEBP), 4456 [x86_intercept_hlt] = POST_EX(SVM_EXIT_HLT), 4457 [x86_intercept_in] = POST_EX(SVM_EXIT_IOIO), 4458 [x86_intercept_ins] = POST_EX(SVM_EXIT_IOIO), 4459 [x86_intercept_out] = POST_EX(SVM_EXIT_IOIO), 4460 [x86_intercept_outs] = POST_EX(SVM_EXIT_IOIO), 4461 [x86_intercept_xsetbv] = PRE_EX(SVM_EXIT_XSETBV), 4462 }; 4463 4464 #undef PRE_EX 4465 #undef POST_EX 4466 #undef POST_MEM 4467 4468 static int svm_check_intercept(struct kvm_vcpu *vcpu, 4469 struct x86_instruction_info *info, 4470 enum x86_intercept_stage stage, 4471 struct x86_exception *exception) 4472 { 4473 struct vcpu_svm *svm = to_svm(vcpu); 4474 int vmexit, ret = X86EMUL_CONTINUE; 4475 struct __x86_intercept icpt_info; 4476 struct vmcb *vmcb = svm->vmcb; 4477 4478 if (info->intercept >= ARRAY_SIZE(x86_intercept_map)) 4479 goto out; 4480 4481 icpt_info = x86_intercept_map[info->intercept]; 4482 4483 if (stage != icpt_info.stage) 4484 goto out; 4485 4486 switch (icpt_info.exit_code) { 4487 case SVM_EXIT_READ_CR0: 4488 if (info->intercept == x86_intercept_cr_read) 4489 icpt_info.exit_code += info->modrm_reg; 4490 break; 4491 case SVM_EXIT_WRITE_CR0: { 4492 unsigned long cr0, val; 4493 4494 if (info->intercept == x86_intercept_cr_write) 4495 icpt_info.exit_code += info->modrm_reg; 4496 4497 if (icpt_info.exit_code != SVM_EXIT_WRITE_CR0 || 4498 info->intercept == x86_intercept_clts) 4499 break; 4500 4501 if (!(vmcb12_is_intercept(&svm->nested.ctl, 4502 INTERCEPT_SELECTIVE_CR0))) 4503 break; 4504 4505 cr0 = vcpu->arch.cr0 & ~SVM_CR0_SELECTIVE_MASK; 4506 val = info->src_val & ~SVM_CR0_SELECTIVE_MASK; 4507 4508 if (info->intercept == x86_intercept_lmsw) { 4509 cr0 &= 0xfUL; 4510 val &= 0xfUL; 4511 /* lmsw can't clear PE - catch this here */ 4512 if (cr0 & X86_CR0_PE) 4513 val |= X86_CR0_PE; 4514 } 4515 4516 if (cr0 ^ val) 4517 icpt_info.exit_code = SVM_EXIT_CR0_SEL_WRITE; 4518 4519 break; 4520 } 4521 case SVM_EXIT_READ_DR0: 4522 case SVM_EXIT_WRITE_DR0: 4523 icpt_info.exit_code += info->modrm_reg; 4524 break; 4525 case SVM_EXIT_MSR: 4526 if (info->intercept == x86_intercept_wrmsr) 4527 vmcb->control.exit_info_1 = 1; 4528 else 4529 vmcb->control.exit_info_1 = 0; 4530 break; 4531 case SVM_EXIT_PAUSE: 4532 /* 4533 * We get this for NOP only, but pause 4534 * is rep not, check this here 4535 */ 4536 if (info->rep_prefix != REPE_PREFIX) 4537 goto out; 4538 break; 4539 case SVM_EXIT_IOIO: { 4540 u64 exit_info; 4541 u32 bytes; 4542 4543 if (info->intercept == x86_intercept_in || 4544 info->intercept == x86_intercept_ins) { 4545 exit_info = ((info->src_val & 0xffff) << 16) | 4546 SVM_IOIO_TYPE_MASK; 4547 bytes = info->dst_bytes; 4548 } else { 4549 exit_info = (info->dst_val & 0xffff) << 16; 4550 bytes = info->src_bytes; 4551 } 4552 4553 if (info->intercept == x86_intercept_outs || 4554 info->intercept == x86_intercept_ins) 4555 exit_info |= SVM_IOIO_STR_MASK; 4556 4557 if (info->rep_prefix) 4558 exit_info |= SVM_IOIO_REP_MASK; 4559 4560 bytes = min(bytes, 4u); 4561 4562 exit_info |= bytes << SVM_IOIO_SIZE_SHIFT; 4563 4564 exit_info |= (u32)info->ad_bytes << (SVM_IOIO_ASIZE_SHIFT - 1); 4565 4566 vmcb->control.exit_info_1 = exit_info; 4567 vmcb->control.exit_info_2 = info->next_rip; 4568 4569 break; 4570 } 4571 default: 4572 break; 4573 } 4574 4575 /* TODO: Advertise NRIPS to guest hypervisor unconditionally */ 4576 if (static_cpu_has(X86_FEATURE_NRIPS)) 4577 vmcb->control.next_rip = info->next_rip; 4578 vmcb->control.exit_code = icpt_info.exit_code; 4579 vmexit = nested_svm_exit_handled(svm); 4580 4581 ret = (vmexit == NESTED_EXIT_DONE) ? X86EMUL_INTERCEPTED 4582 : X86EMUL_CONTINUE; 4583 4584 out: 4585 return ret; 4586 } 4587 4588 static void svm_handle_exit_irqoff(struct kvm_vcpu *vcpu) 4589 { 4590 if (to_svm(vcpu)->vmcb->control.exit_code == SVM_EXIT_INTR) 4591 vcpu->arch.at_instruction_boundary = true; 4592 } 4593 4594 static void svm_sched_in(struct kvm_vcpu *vcpu, int cpu) 4595 { 4596 if (!kvm_pause_in_guest(vcpu->kvm)) 4597 shrink_ple_window(vcpu); 4598 } 4599 4600 static void svm_setup_mce(struct kvm_vcpu *vcpu) 4601 { 4602 /* [63:9] are reserved. */ 4603 vcpu->arch.mcg_cap &= 0x1ff; 4604 } 4605 4606 #ifdef CONFIG_KVM_SMM 4607 bool svm_smi_blocked(struct kvm_vcpu *vcpu) 4608 { 4609 struct vcpu_svm *svm = to_svm(vcpu); 4610 4611 /* Per APM Vol.2 15.22.2 "Response to SMI" */ 4612 if (!gif_set(svm)) 4613 return true; 4614 4615 return is_smm(vcpu); 4616 } 4617 4618 static int svm_smi_allowed(struct kvm_vcpu *vcpu, bool for_injection) 4619 { 4620 struct vcpu_svm *svm = to_svm(vcpu); 4621 if (svm->nested.nested_run_pending) 4622 return -EBUSY; 4623 4624 if (svm_smi_blocked(vcpu)) 4625 return 0; 4626 4627 /* An SMI must not be injected into L2 if it's supposed to VM-Exit. */ 4628 if (for_injection && is_guest_mode(vcpu) && nested_exit_on_smi(svm)) 4629 return -EBUSY; 4630 4631 return 1; 4632 } 4633 4634 static int svm_enter_smm(struct kvm_vcpu *vcpu, union kvm_smram *smram) 4635 { 4636 struct vcpu_svm *svm = to_svm(vcpu); 4637 struct kvm_host_map map_save; 4638 int ret; 4639 4640 if (!is_guest_mode(vcpu)) 4641 return 0; 4642 4643 /* 4644 * 32-bit SMRAM format doesn't preserve EFER and SVM state. Userspace is 4645 * responsible for ensuring nested SVM and SMIs are mutually exclusive. 4646 */ 4647 4648 if (!guest_cpuid_has(vcpu, X86_FEATURE_LM)) 4649 return 1; 4650 4651 smram->smram64.svm_guest_flag = 1; 4652 smram->smram64.svm_guest_vmcb_gpa = svm->nested.vmcb12_gpa; 4653 4654 svm->vmcb->save.rax = vcpu->arch.regs[VCPU_REGS_RAX]; 4655 svm->vmcb->save.rsp = vcpu->arch.regs[VCPU_REGS_RSP]; 4656 svm->vmcb->save.rip = vcpu->arch.regs[VCPU_REGS_RIP]; 4657 4658 ret = nested_svm_simple_vmexit(svm, SVM_EXIT_SW); 4659 if (ret) 4660 return ret; 4661 4662 /* 4663 * KVM uses VMCB01 to store L1 host state while L2 runs but 4664 * VMCB01 is going to be used during SMM and thus the state will 4665 * be lost. Temporary save non-VMLOAD/VMSAVE state to the host save 4666 * area pointed to by MSR_VM_HSAVE_PA. APM guarantees that the 4667 * format of the area is identical to guest save area offsetted 4668 * by 0x400 (matches the offset of 'struct vmcb_save_area' 4669 * within 'struct vmcb'). Note: HSAVE area may also be used by 4670 * L1 hypervisor to save additional host context (e.g. KVM does 4671 * that, see svm_prepare_switch_to_guest()) which must be 4672 * preserved. 4673 */ 4674 if (kvm_vcpu_map(vcpu, gpa_to_gfn(svm->nested.hsave_msr), &map_save)) 4675 return 1; 4676 4677 BUILD_BUG_ON(offsetof(struct vmcb, save) != 0x400); 4678 4679 svm_copy_vmrun_state(map_save.hva + 0x400, 4680 &svm->vmcb01.ptr->save); 4681 4682 kvm_vcpu_unmap(vcpu, &map_save, true); 4683 return 0; 4684 } 4685 4686 static int svm_leave_smm(struct kvm_vcpu *vcpu, const union kvm_smram *smram) 4687 { 4688 struct vcpu_svm *svm = to_svm(vcpu); 4689 struct kvm_host_map map, map_save; 4690 struct vmcb *vmcb12; 4691 int ret; 4692 4693 const struct kvm_smram_state_64 *smram64 = &smram->smram64; 4694 4695 if (!guest_cpuid_has(vcpu, X86_FEATURE_LM)) 4696 return 0; 4697 4698 /* Non-zero if SMI arrived while vCPU was in guest mode. */ 4699 if (!smram64->svm_guest_flag) 4700 return 0; 4701 4702 if (!guest_cpuid_has(vcpu, X86_FEATURE_SVM)) 4703 return 1; 4704 4705 if (!(smram64->efer & EFER_SVME)) 4706 return 1; 4707 4708 if (kvm_vcpu_map(vcpu, gpa_to_gfn(smram64->svm_guest_vmcb_gpa), &map)) 4709 return 1; 4710 4711 ret = 1; 4712 if (kvm_vcpu_map(vcpu, gpa_to_gfn(svm->nested.hsave_msr), &map_save)) 4713 goto unmap_map; 4714 4715 if (svm_allocate_nested(svm)) 4716 goto unmap_save; 4717 4718 /* 4719 * Restore L1 host state from L1 HSAVE area as VMCB01 was 4720 * used during SMM (see svm_enter_smm()) 4721 */ 4722 4723 svm_copy_vmrun_state(&svm->vmcb01.ptr->save, map_save.hva + 0x400); 4724 4725 /* 4726 * Enter the nested guest now 4727 */ 4728 4729 vmcb_mark_all_dirty(svm->vmcb01.ptr); 4730 4731 vmcb12 = map.hva; 4732 nested_copy_vmcb_control_to_cache(svm, &vmcb12->control); 4733 nested_copy_vmcb_save_to_cache(svm, &vmcb12->save); 4734 ret = enter_svm_guest_mode(vcpu, smram64->svm_guest_vmcb_gpa, vmcb12, false); 4735 4736 if (ret) 4737 goto unmap_save; 4738 4739 svm->nested.nested_run_pending = 1; 4740 4741 unmap_save: 4742 kvm_vcpu_unmap(vcpu, &map_save, true); 4743 unmap_map: 4744 kvm_vcpu_unmap(vcpu, &map, true); 4745 return ret; 4746 } 4747 4748 static void svm_enable_smi_window(struct kvm_vcpu *vcpu) 4749 { 4750 struct vcpu_svm *svm = to_svm(vcpu); 4751 4752 if (!gif_set(svm)) { 4753 if (vgif) 4754 svm_set_intercept(svm, INTERCEPT_STGI); 4755 /* STGI will cause a vm exit */ 4756 } else { 4757 /* We must be in SMM; RSM will cause a vmexit anyway. */ 4758 } 4759 } 4760 #endif 4761 4762 static bool svm_can_emulate_instruction(struct kvm_vcpu *vcpu, int emul_type, 4763 void *insn, int insn_len) 4764 { 4765 bool smep, smap, is_user; 4766 u64 error_code; 4767 4768 /* Emulation is always possible when KVM has access to all guest state. */ 4769 if (!sev_guest(vcpu->kvm)) 4770 return true; 4771 4772 /* #UD and #GP should never be intercepted for SEV guests. */ 4773 WARN_ON_ONCE(emul_type & (EMULTYPE_TRAP_UD | 4774 EMULTYPE_TRAP_UD_FORCED | 4775 EMULTYPE_VMWARE_GP)); 4776 4777 /* 4778 * Emulation is impossible for SEV-ES guests as KVM doesn't have access 4779 * to guest register state. 4780 */ 4781 if (sev_es_guest(vcpu->kvm)) 4782 return false; 4783 4784 /* 4785 * Emulation is possible if the instruction is already decoded, e.g. 4786 * when completing I/O after returning from userspace. 4787 */ 4788 if (emul_type & EMULTYPE_NO_DECODE) 4789 return true; 4790 4791 /* 4792 * Emulation is possible for SEV guests if and only if a prefilled 4793 * buffer containing the bytes of the intercepted instruction is 4794 * available. SEV guest memory is encrypted with a guest specific key 4795 * and cannot be decrypted by KVM, i.e. KVM would read cyphertext and 4796 * decode garbage. 4797 * 4798 * If KVM is NOT trying to simply skip an instruction, inject #UD if 4799 * KVM reached this point without an instruction buffer. In practice, 4800 * this path should never be hit by a well-behaved guest, e.g. KVM 4801 * doesn't intercept #UD or #GP for SEV guests, but this path is still 4802 * theoretically reachable, e.g. via unaccelerated fault-like AVIC 4803 * access, and needs to be handled by KVM to avoid putting the guest 4804 * into an infinite loop. Injecting #UD is somewhat arbitrary, but 4805 * its the least awful option given lack of insight into the guest. 4806 * 4807 * If KVM is trying to skip an instruction, simply resume the guest. 4808 * If a #NPF occurs while the guest is vectoring an INT3/INTO, then KVM 4809 * will attempt to re-inject the INT3/INTO and skip the instruction. 4810 * In that scenario, retrying the INT3/INTO and hoping the guest will 4811 * make forward progress is the only option that has a chance of 4812 * success (and in practice it will work the vast majority of the time). 4813 */ 4814 if (unlikely(!insn)) { 4815 if (!(emul_type & EMULTYPE_SKIP)) 4816 kvm_queue_exception(vcpu, UD_VECTOR); 4817 return false; 4818 } 4819 4820 /* 4821 * Emulate for SEV guests if the insn buffer is not empty. The buffer 4822 * will be empty if the DecodeAssist microcode cannot fetch bytes for 4823 * the faulting instruction because the code fetch itself faulted, e.g. 4824 * the guest attempted to fetch from emulated MMIO or a guest page 4825 * table used to translate CS:RIP resides in emulated MMIO. 4826 */ 4827 if (likely(insn_len)) 4828 return true; 4829 4830 /* 4831 * Detect and workaround Errata 1096 Fam_17h_00_0Fh. 4832 * 4833 * Errata: 4834 * When CPU raises #NPF on guest data access and vCPU CR4.SMAP=1, it is 4835 * possible that CPU microcode implementing DecodeAssist will fail to 4836 * read guest memory at CS:RIP and vmcb.GuestIntrBytes will incorrectly 4837 * be '0'. This happens because microcode reads CS:RIP using a _data_ 4838 * loap uop with CPL=0 privileges. If the load hits a SMAP #PF, ucode 4839 * gives up and does not fill the instruction bytes buffer. 4840 * 4841 * As above, KVM reaches this point iff the VM is an SEV guest, the CPU 4842 * supports DecodeAssist, a #NPF was raised, KVM's page fault handler 4843 * triggered emulation (e.g. for MMIO), and the CPU returned 0 in the 4844 * GuestIntrBytes field of the VMCB. 4845 * 4846 * This does _not_ mean that the erratum has been encountered, as the 4847 * DecodeAssist will also fail if the load for CS:RIP hits a legitimate 4848 * #PF, e.g. if the guest attempt to execute from emulated MMIO and 4849 * encountered a reserved/not-present #PF. 4850 * 4851 * To hit the erratum, the following conditions must be true: 4852 * 1. CR4.SMAP=1 (obviously). 4853 * 2. CR4.SMEP=0 || CPL=3. If SMEP=1 and CPL<3, the erratum cannot 4854 * have been hit as the guest would have encountered a SMEP 4855 * violation #PF, not a #NPF. 4856 * 3. The #NPF is not due to a code fetch, in which case failure to 4857 * retrieve the instruction bytes is legitimate (see abvoe). 4858 * 4859 * In addition, don't apply the erratum workaround if the #NPF occurred 4860 * while translating guest page tables (see below). 4861 */ 4862 error_code = to_svm(vcpu)->vmcb->control.exit_info_1; 4863 if (error_code & (PFERR_GUEST_PAGE_MASK | PFERR_FETCH_MASK)) 4864 goto resume_guest; 4865 4866 smep = kvm_is_cr4_bit_set(vcpu, X86_CR4_SMEP); 4867 smap = kvm_is_cr4_bit_set(vcpu, X86_CR4_SMAP); 4868 is_user = svm_get_cpl(vcpu) == 3; 4869 if (smap && (!smep || is_user)) { 4870 pr_err_ratelimited("SEV Guest triggered AMD Erratum 1096\n"); 4871 4872 /* 4873 * If the fault occurred in userspace, arbitrarily inject #GP 4874 * to avoid killing the guest and to hopefully avoid confusing 4875 * the guest kernel too much, e.g. injecting #PF would not be 4876 * coherent with respect to the guest's page tables. Request 4877 * triple fault if the fault occurred in the kernel as there's 4878 * no fault that KVM can inject without confusing the guest. 4879 * In practice, the triple fault is moot as no sane SEV kernel 4880 * will execute from user memory while also running with SMAP=1. 4881 */ 4882 if (is_user) 4883 kvm_inject_gp(vcpu, 0); 4884 else 4885 kvm_make_request(KVM_REQ_TRIPLE_FAULT, vcpu); 4886 } 4887 4888 resume_guest: 4889 /* 4890 * If the erratum was not hit, simply resume the guest and let it fault 4891 * again. While awful, e.g. the vCPU may get stuck in an infinite loop 4892 * if the fault is at CPL=0, it's the lesser of all evils. Exiting to 4893 * userspace will kill the guest, and letting the emulator read garbage 4894 * will yield random behavior and potentially corrupt the guest. 4895 * 4896 * Simply resuming the guest is technically not a violation of the SEV 4897 * architecture. AMD's APM states that all code fetches and page table 4898 * accesses for SEV guest are encrypted, regardless of the C-Bit. The 4899 * APM also states that encrypted accesses to MMIO are "ignored", but 4900 * doesn't explicitly define "ignored", i.e. doing nothing and letting 4901 * the guest spin is technically "ignoring" the access. 4902 */ 4903 return false; 4904 } 4905 4906 static bool svm_apic_init_signal_blocked(struct kvm_vcpu *vcpu) 4907 { 4908 struct vcpu_svm *svm = to_svm(vcpu); 4909 4910 return !gif_set(svm); 4911 } 4912 4913 static void svm_vcpu_deliver_sipi_vector(struct kvm_vcpu *vcpu, u8 vector) 4914 { 4915 if (!sev_es_guest(vcpu->kvm)) 4916 return kvm_vcpu_deliver_sipi_vector(vcpu, vector); 4917 4918 sev_vcpu_deliver_sipi_vector(vcpu, vector); 4919 } 4920 4921 static void svm_vm_destroy(struct kvm *kvm) 4922 { 4923 avic_vm_destroy(kvm); 4924 sev_vm_destroy(kvm); 4925 } 4926 4927 static int svm_vm_init(struct kvm *kvm) 4928 { 4929 if (!pause_filter_count || !pause_filter_thresh) 4930 kvm->arch.pause_in_guest = true; 4931 4932 if (enable_apicv) { 4933 int ret = avic_vm_init(kvm); 4934 if (ret) 4935 return ret; 4936 } 4937 4938 return 0; 4939 } 4940 4941 static struct kvm_x86_ops svm_x86_ops __initdata = { 4942 .name = KBUILD_MODNAME, 4943 4944 .check_processor_compatibility = svm_check_processor_compat, 4945 4946 .hardware_unsetup = svm_hardware_unsetup, 4947 .hardware_enable = svm_hardware_enable, 4948 .hardware_disable = svm_hardware_disable, 4949 .has_emulated_msr = svm_has_emulated_msr, 4950 4951 .vcpu_create = svm_vcpu_create, 4952 .vcpu_free = svm_vcpu_free, 4953 .vcpu_reset = svm_vcpu_reset, 4954 4955 .vm_size = sizeof(struct kvm_svm), 4956 .vm_init = svm_vm_init, 4957 .vm_destroy = svm_vm_destroy, 4958 4959 .prepare_switch_to_guest = svm_prepare_switch_to_guest, 4960 .vcpu_load = svm_vcpu_load, 4961 .vcpu_put = svm_vcpu_put, 4962 .vcpu_blocking = avic_vcpu_blocking, 4963 .vcpu_unblocking = avic_vcpu_unblocking, 4964 4965 .update_exception_bitmap = svm_update_exception_bitmap, 4966 .get_msr_feature = svm_get_msr_feature, 4967 .get_msr = svm_get_msr, 4968 .set_msr = svm_set_msr, 4969 .get_segment_base = svm_get_segment_base, 4970 .get_segment = svm_get_segment, 4971 .set_segment = svm_set_segment, 4972 .get_cpl = svm_get_cpl, 4973 .get_cs_db_l_bits = svm_get_cs_db_l_bits, 4974 .is_valid_cr0 = svm_is_valid_cr0, 4975 .set_cr0 = svm_set_cr0, 4976 .post_set_cr3 = sev_post_set_cr3, 4977 .is_valid_cr4 = svm_is_valid_cr4, 4978 .set_cr4 = svm_set_cr4, 4979 .set_efer = svm_set_efer, 4980 .get_idt = svm_get_idt, 4981 .set_idt = svm_set_idt, 4982 .get_gdt = svm_get_gdt, 4983 .set_gdt = svm_set_gdt, 4984 .set_dr7 = svm_set_dr7, 4985 .sync_dirty_debug_regs = svm_sync_dirty_debug_regs, 4986 .cache_reg = svm_cache_reg, 4987 .get_rflags = svm_get_rflags, 4988 .set_rflags = svm_set_rflags, 4989 .get_if_flag = svm_get_if_flag, 4990 4991 .flush_tlb_all = svm_flush_tlb_all, 4992 .flush_tlb_current = svm_flush_tlb_current, 4993 .flush_tlb_gva = svm_flush_tlb_gva, 4994 .flush_tlb_guest = svm_flush_tlb_asid, 4995 4996 .vcpu_pre_run = svm_vcpu_pre_run, 4997 .vcpu_run = svm_vcpu_run, 4998 .handle_exit = svm_handle_exit, 4999 .skip_emulated_instruction = svm_skip_emulated_instruction, 5000 .update_emulated_instruction = NULL, 5001 .set_interrupt_shadow = svm_set_interrupt_shadow, 5002 .get_interrupt_shadow = svm_get_interrupt_shadow, 5003 .patch_hypercall = svm_patch_hypercall, 5004 .inject_irq = svm_inject_irq, 5005 .inject_nmi = svm_inject_nmi, 5006 .is_vnmi_pending = svm_is_vnmi_pending, 5007 .set_vnmi_pending = svm_set_vnmi_pending, 5008 .inject_exception = svm_inject_exception, 5009 .cancel_injection = svm_cancel_injection, 5010 .interrupt_allowed = svm_interrupt_allowed, 5011 .nmi_allowed = svm_nmi_allowed, 5012 .get_nmi_mask = svm_get_nmi_mask, 5013 .set_nmi_mask = svm_set_nmi_mask, 5014 .enable_nmi_window = svm_enable_nmi_window, 5015 .enable_irq_window = svm_enable_irq_window, 5016 .update_cr8_intercept = svm_update_cr8_intercept, 5017 .set_virtual_apic_mode = avic_refresh_virtual_apic_mode, 5018 .refresh_apicv_exec_ctrl = avic_refresh_apicv_exec_ctrl, 5019 .apicv_post_state_restore = avic_apicv_post_state_restore, 5020 .required_apicv_inhibits = AVIC_REQUIRED_APICV_INHIBITS, 5021 5022 .get_exit_info = svm_get_exit_info, 5023 5024 .vcpu_after_set_cpuid = svm_vcpu_after_set_cpuid, 5025 5026 .has_wbinvd_exit = svm_has_wbinvd_exit, 5027 5028 .get_l2_tsc_offset = svm_get_l2_tsc_offset, 5029 .get_l2_tsc_multiplier = svm_get_l2_tsc_multiplier, 5030 .write_tsc_offset = svm_write_tsc_offset, 5031 .write_tsc_multiplier = svm_write_tsc_multiplier, 5032 5033 .load_mmu_pgd = svm_load_mmu_pgd, 5034 5035 .check_intercept = svm_check_intercept, 5036 .handle_exit_irqoff = svm_handle_exit_irqoff, 5037 5038 .request_immediate_exit = __kvm_request_immediate_exit, 5039 5040 .sched_in = svm_sched_in, 5041 5042 .nested_ops = &svm_nested_ops, 5043 5044 .deliver_interrupt = svm_deliver_interrupt, 5045 .pi_update_irte = avic_pi_update_irte, 5046 .setup_mce = svm_setup_mce, 5047 5048 #ifdef CONFIG_KVM_SMM 5049 .smi_allowed = svm_smi_allowed, 5050 .enter_smm = svm_enter_smm, 5051 .leave_smm = svm_leave_smm, 5052 .enable_smi_window = svm_enable_smi_window, 5053 #endif 5054 5055 .mem_enc_ioctl = sev_mem_enc_ioctl, 5056 .mem_enc_register_region = sev_mem_enc_register_region, 5057 .mem_enc_unregister_region = sev_mem_enc_unregister_region, 5058 .guest_memory_reclaimed = sev_guest_memory_reclaimed, 5059 5060 .vm_copy_enc_context_from = sev_vm_copy_enc_context_from, 5061 .vm_move_enc_context_from = sev_vm_move_enc_context_from, 5062 5063 .can_emulate_instruction = svm_can_emulate_instruction, 5064 5065 .apic_init_signal_blocked = svm_apic_init_signal_blocked, 5066 5067 .msr_filter_changed = svm_msr_filter_changed, 5068 .complete_emulated_msr = svm_complete_emulated_msr, 5069 5070 .vcpu_deliver_sipi_vector = svm_vcpu_deliver_sipi_vector, 5071 .vcpu_get_apicv_inhibit_reasons = avic_vcpu_get_apicv_inhibit_reasons, 5072 }; 5073 5074 /* 5075 * The default MMIO mask is a single bit (excluding the present bit), 5076 * which could conflict with the memory encryption bit. Check for 5077 * memory encryption support and override the default MMIO mask if 5078 * memory encryption is enabled. 5079 */ 5080 static __init void svm_adjust_mmio_mask(void) 5081 { 5082 unsigned int enc_bit, mask_bit; 5083 u64 msr, mask; 5084 5085 /* If there is no memory encryption support, use existing mask */ 5086 if (cpuid_eax(0x80000000) < 0x8000001f) 5087 return; 5088 5089 /* If memory encryption is not enabled, use existing mask */ 5090 rdmsrl(MSR_AMD64_SYSCFG, msr); 5091 if (!(msr & MSR_AMD64_SYSCFG_MEM_ENCRYPT)) 5092 return; 5093 5094 enc_bit = cpuid_ebx(0x8000001f) & 0x3f; 5095 mask_bit = boot_cpu_data.x86_phys_bits; 5096 5097 /* Increment the mask bit if it is the same as the encryption bit */ 5098 if (enc_bit == mask_bit) 5099 mask_bit++; 5100 5101 /* 5102 * If the mask bit location is below 52, then some bits above the 5103 * physical addressing limit will always be reserved, so use the 5104 * rsvd_bits() function to generate the mask. This mask, along with 5105 * the present bit, will be used to generate a page fault with 5106 * PFER.RSV = 1. 5107 * 5108 * If the mask bit location is 52 (or above), then clear the mask. 5109 */ 5110 mask = (mask_bit < 52) ? rsvd_bits(mask_bit, 51) | PT_PRESENT_MASK : 0; 5111 5112 kvm_mmu_set_mmio_spte_mask(mask, mask, PT_WRITABLE_MASK | PT_USER_MASK); 5113 } 5114 5115 static __init void svm_set_cpu_caps(void) 5116 { 5117 kvm_set_cpu_caps(); 5118 5119 kvm_caps.supported_perf_cap = 0; 5120 kvm_caps.supported_xss = 0; 5121 5122 /* CPUID 0x80000001 and 0x8000000A (SVM features) */ 5123 if (nested) { 5124 kvm_cpu_cap_set(X86_FEATURE_SVM); 5125 kvm_cpu_cap_set(X86_FEATURE_VMCBCLEAN); 5126 5127 if (nrips) 5128 kvm_cpu_cap_set(X86_FEATURE_NRIPS); 5129 5130 if (npt_enabled) 5131 kvm_cpu_cap_set(X86_FEATURE_NPT); 5132 5133 if (tsc_scaling) 5134 kvm_cpu_cap_set(X86_FEATURE_TSCRATEMSR); 5135 5136 if (vls) 5137 kvm_cpu_cap_set(X86_FEATURE_V_VMSAVE_VMLOAD); 5138 if (lbrv) 5139 kvm_cpu_cap_set(X86_FEATURE_LBRV); 5140 5141 if (boot_cpu_has(X86_FEATURE_PAUSEFILTER)) 5142 kvm_cpu_cap_set(X86_FEATURE_PAUSEFILTER); 5143 5144 if (boot_cpu_has(X86_FEATURE_PFTHRESHOLD)) 5145 kvm_cpu_cap_set(X86_FEATURE_PFTHRESHOLD); 5146 5147 if (vgif) 5148 kvm_cpu_cap_set(X86_FEATURE_VGIF); 5149 5150 if (vnmi) 5151 kvm_cpu_cap_set(X86_FEATURE_VNMI); 5152 5153 /* Nested VM can receive #VMEXIT instead of triggering #GP */ 5154 kvm_cpu_cap_set(X86_FEATURE_SVME_ADDR_CHK); 5155 } 5156 5157 /* CPUID 0x80000008 */ 5158 if (boot_cpu_has(X86_FEATURE_LS_CFG_SSBD) || 5159 boot_cpu_has(X86_FEATURE_AMD_SSBD)) 5160 kvm_cpu_cap_set(X86_FEATURE_VIRT_SSBD); 5161 5162 if (enable_pmu) { 5163 /* 5164 * Enumerate support for PERFCTR_CORE if and only if KVM has 5165 * access to enough counters to virtualize "core" support, 5166 * otherwise limit vPMU support to the legacy number of counters. 5167 */ 5168 if (kvm_pmu_cap.num_counters_gp < AMD64_NUM_COUNTERS_CORE) 5169 kvm_pmu_cap.num_counters_gp = min(AMD64_NUM_COUNTERS, 5170 kvm_pmu_cap.num_counters_gp); 5171 else 5172 kvm_cpu_cap_check_and_set(X86_FEATURE_PERFCTR_CORE); 5173 5174 if (kvm_pmu_cap.version != 2 || 5175 !kvm_cpu_cap_has(X86_FEATURE_PERFCTR_CORE)) 5176 kvm_cpu_cap_clear(X86_FEATURE_PERFMON_V2); 5177 } 5178 5179 /* CPUID 0x8000001F (SME/SEV features) */ 5180 sev_set_cpu_caps(); 5181 5182 /* Don't advertise Bus Lock Detect to guest if SVM support is absent */ 5183 kvm_cpu_cap_clear(X86_FEATURE_BUS_LOCK_DETECT); 5184 } 5185 5186 static __init int svm_hardware_setup(void) 5187 { 5188 int cpu; 5189 struct page *iopm_pages; 5190 void *iopm_va; 5191 int r; 5192 unsigned int order = get_order(IOPM_SIZE); 5193 5194 /* 5195 * NX is required for shadow paging and for NPT if the NX huge pages 5196 * mitigation is enabled. 5197 */ 5198 if (!boot_cpu_has(X86_FEATURE_NX)) { 5199 pr_err_ratelimited("NX (Execute Disable) not supported\n"); 5200 return -EOPNOTSUPP; 5201 } 5202 kvm_enable_efer_bits(EFER_NX); 5203 5204 iopm_pages = alloc_pages(GFP_KERNEL, order); 5205 5206 if (!iopm_pages) 5207 return -ENOMEM; 5208 5209 iopm_va = page_address(iopm_pages); 5210 memset(iopm_va, 0xff, PAGE_SIZE * (1 << order)); 5211 iopm_base = page_to_pfn(iopm_pages) << PAGE_SHIFT; 5212 5213 init_msrpm_offsets(); 5214 5215 kvm_caps.supported_xcr0 &= ~(XFEATURE_MASK_BNDREGS | 5216 XFEATURE_MASK_BNDCSR); 5217 5218 if (boot_cpu_has(X86_FEATURE_FXSR_OPT)) 5219 kvm_enable_efer_bits(EFER_FFXSR); 5220 5221 if (tsc_scaling) { 5222 if (!boot_cpu_has(X86_FEATURE_TSCRATEMSR)) { 5223 tsc_scaling = false; 5224 } else { 5225 pr_info("TSC scaling supported\n"); 5226 kvm_caps.has_tsc_control = true; 5227 } 5228 } 5229 kvm_caps.max_tsc_scaling_ratio = SVM_TSC_RATIO_MAX; 5230 kvm_caps.tsc_scaling_ratio_frac_bits = 32; 5231 5232 tsc_aux_uret_slot = kvm_add_user_return_msr(MSR_TSC_AUX); 5233 5234 if (boot_cpu_has(X86_FEATURE_AUTOIBRS)) 5235 kvm_enable_efer_bits(EFER_AUTOIBRS); 5236 5237 /* Check for pause filtering support */ 5238 if (!boot_cpu_has(X86_FEATURE_PAUSEFILTER)) { 5239 pause_filter_count = 0; 5240 pause_filter_thresh = 0; 5241 } else if (!boot_cpu_has(X86_FEATURE_PFTHRESHOLD)) { 5242 pause_filter_thresh = 0; 5243 } 5244 5245 if (nested) { 5246 pr_info("Nested Virtualization enabled\n"); 5247 kvm_enable_efer_bits(EFER_SVME | EFER_LMSLE); 5248 } 5249 5250 /* 5251 * KVM's MMU doesn't support using 2-level paging for itself, and thus 5252 * NPT isn't supported if the host is using 2-level paging since host 5253 * CR4 is unchanged on VMRUN. 5254 */ 5255 if (!IS_ENABLED(CONFIG_X86_64) && !IS_ENABLED(CONFIG_X86_PAE)) 5256 npt_enabled = false; 5257 5258 if (!boot_cpu_has(X86_FEATURE_NPT)) 5259 npt_enabled = false; 5260 5261 /* Force VM NPT level equal to the host's paging level */ 5262 kvm_configure_mmu(npt_enabled, get_npt_level(), 5263 get_npt_level(), PG_LEVEL_1G); 5264 pr_info("Nested Paging %sabled\n", npt_enabled ? "en" : "dis"); 5265 5266 /* Setup shadow_me_value and shadow_me_mask */ 5267 kvm_mmu_set_me_spte_mask(sme_me_mask, sme_me_mask); 5268 5269 svm_adjust_mmio_mask(); 5270 5271 nrips = nrips && boot_cpu_has(X86_FEATURE_NRIPS); 5272 5273 if (lbrv) { 5274 if (!boot_cpu_has(X86_FEATURE_LBRV)) 5275 lbrv = false; 5276 else 5277 pr_info("LBR virtualization supported\n"); 5278 } 5279 /* 5280 * Note, SEV setup consumes npt_enabled and enable_mmio_caching (which 5281 * may be modified by svm_adjust_mmio_mask()), as well as nrips. 5282 */ 5283 sev_hardware_setup(); 5284 5285 svm_hv_hardware_setup(); 5286 5287 for_each_possible_cpu(cpu) { 5288 r = svm_cpu_init(cpu); 5289 if (r) 5290 goto err; 5291 } 5292 5293 enable_apicv = avic = avic && avic_hardware_setup(); 5294 5295 if (!enable_apicv) { 5296 svm_x86_ops.vcpu_blocking = NULL; 5297 svm_x86_ops.vcpu_unblocking = NULL; 5298 svm_x86_ops.vcpu_get_apicv_inhibit_reasons = NULL; 5299 } else if (!x2avic_enabled) { 5300 svm_x86_ops.allow_apicv_in_x2apic_without_x2apic_virtualization = true; 5301 } 5302 5303 if (vls) { 5304 if (!npt_enabled || 5305 !boot_cpu_has(X86_FEATURE_V_VMSAVE_VMLOAD) || 5306 !IS_ENABLED(CONFIG_X86_64)) { 5307 vls = false; 5308 } else { 5309 pr_info("Virtual VMLOAD VMSAVE supported\n"); 5310 } 5311 } 5312 5313 if (boot_cpu_has(X86_FEATURE_SVME_ADDR_CHK)) 5314 svm_gp_erratum_intercept = false; 5315 5316 if (vgif) { 5317 if (!boot_cpu_has(X86_FEATURE_VGIF)) 5318 vgif = false; 5319 else 5320 pr_info("Virtual GIF supported\n"); 5321 } 5322 5323 vnmi = vgif && vnmi && boot_cpu_has(X86_FEATURE_VNMI); 5324 if (vnmi) 5325 pr_info("Virtual NMI enabled\n"); 5326 5327 if (!vnmi) { 5328 svm_x86_ops.is_vnmi_pending = NULL; 5329 svm_x86_ops.set_vnmi_pending = NULL; 5330 } 5331 5332 if (!enable_pmu) 5333 pr_info("PMU virtualization is disabled\n"); 5334 5335 svm_set_cpu_caps(); 5336 5337 /* 5338 * It seems that on AMD processors PTE's accessed bit is 5339 * being set by the CPU hardware before the NPF vmexit. 5340 * This is not expected behaviour and our tests fail because 5341 * of it. 5342 * A workaround here is to disable support for 5343 * GUEST_MAXPHYADDR < HOST_MAXPHYADDR if NPT is enabled. 5344 * In this case userspace can know if there is support using 5345 * KVM_CAP_SMALLER_MAXPHYADDR extension and decide how to handle 5346 * it 5347 * If future AMD CPU models change the behaviour described above, 5348 * this variable can be changed accordingly 5349 */ 5350 allow_smaller_maxphyaddr = !npt_enabled; 5351 5352 return 0; 5353 5354 err: 5355 svm_hardware_unsetup(); 5356 return r; 5357 } 5358 5359 5360 static struct kvm_x86_init_ops svm_init_ops __initdata = { 5361 .hardware_setup = svm_hardware_setup, 5362 5363 .runtime_ops = &svm_x86_ops, 5364 .pmu_ops = &amd_pmu_ops, 5365 }; 5366 5367 static void __svm_exit(void) 5368 { 5369 kvm_x86_vendor_exit(); 5370 5371 cpu_emergency_unregister_virt_callback(svm_emergency_disable); 5372 } 5373 5374 static int __init svm_init(void) 5375 { 5376 int r; 5377 5378 __unused_size_checks(); 5379 5380 if (!kvm_is_svm_supported()) 5381 return -EOPNOTSUPP; 5382 5383 r = kvm_x86_vendor_init(&svm_init_ops); 5384 if (r) 5385 return r; 5386 5387 cpu_emergency_register_virt_callback(svm_emergency_disable); 5388 5389 /* 5390 * Common KVM initialization _must_ come last, after this, /dev/kvm is 5391 * exposed to userspace! 5392 */ 5393 r = kvm_init(sizeof(struct vcpu_svm), __alignof__(struct vcpu_svm), 5394 THIS_MODULE); 5395 if (r) 5396 goto err_kvm_init; 5397 5398 return 0; 5399 5400 err_kvm_init: 5401 __svm_exit(); 5402 return r; 5403 } 5404 5405 static void __exit svm_exit(void) 5406 { 5407 kvm_exit(); 5408 __svm_exit(); 5409 } 5410 5411 module_init(svm_init) 5412 module_exit(svm_exit) 5413