1 #define pr_fmt(fmt) "SVM: " fmt 2 3 #include <linux/kvm_host.h> 4 5 #include "irq.h" 6 #include "mmu.h" 7 #include "kvm_cache_regs.h" 8 #include "x86.h" 9 #include "cpuid.h" 10 #include "pmu.h" 11 12 #include <linux/module.h> 13 #include <linux/mod_devicetable.h> 14 #include <linux/kernel.h> 15 #include <linux/vmalloc.h> 16 #include <linux/highmem.h> 17 #include <linux/amd-iommu.h> 18 #include <linux/sched.h> 19 #include <linux/trace_events.h> 20 #include <linux/slab.h> 21 #include <linux/hashtable.h> 22 #include <linux/objtool.h> 23 #include <linux/psp-sev.h> 24 #include <linux/file.h> 25 #include <linux/pagemap.h> 26 #include <linux/swap.h> 27 #include <linux/rwsem.h> 28 29 #include <asm/apic.h> 30 #include <asm/perf_event.h> 31 #include <asm/tlbflush.h> 32 #include <asm/desc.h> 33 #include <asm/debugreg.h> 34 #include <asm/kvm_para.h> 35 #include <asm/irq_remapping.h> 36 #include <asm/spec-ctrl.h> 37 #include <asm/cpu_device_id.h> 38 #include <asm/traps.h> 39 40 #include <asm/virtext.h> 41 #include "trace.h" 42 43 #include "svm.h" 44 #include "svm_ops.h" 45 46 #include "kvm_onhyperv.h" 47 #include "svm_onhyperv.h" 48 49 MODULE_AUTHOR("Qumranet"); 50 MODULE_LICENSE("GPL"); 51 52 #ifdef MODULE 53 static const struct x86_cpu_id svm_cpu_id[] = { 54 X86_MATCH_FEATURE(X86_FEATURE_SVM, NULL), 55 {} 56 }; 57 MODULE_DEVICE_TABLE(x86cpu, svm_cpu_id); 58 #endif 59 60 #define SEG_TYPE_LDT 2 61 #define SEG_TYPE_BUSY_TSS16 3 62 63 #define SVM_FEATURE_LBRV (1 << 1) 64 #define SVM_FEATURE_SVML (1 << 2) 65 #define SVM_FEATURE_TSC_RATE (1 << 4) 66 #define SVM_FEATURE_VMCB_CLEAN (1 << 5) 67 #define SVM_FEATURE_FLUSH_ASID (1 << 6) 68 #define SVM_FEATURE_DECODE_ASSIST (1 << 7) 69 #define SVM_FEATURE_PAUSE_FILTER (1 << 10) 70 71 #define DEBUGCTL_RESERVED_BITS (~(0x3fULL)) 72 73 #define TSC_RATIO_RSVD 0xffffff0000000000ULL 74 #define TSC_RATIO_MIN 0x0000000000000001ULL 75 #define TSC_RATIO_MAX 0x000000ffffffffffULL 76 77 static bool erratum_383_found __read_mostly; 78 79 u32 msrpm_offsets[MSRPM_OFFSETS] __read_mostly; 80 81 /* 82 * Set osvw_len to higher value when updated Revision Guides 83 * are published and we know what the new status bits are 84 */ 85 static uint64_t osvw_len = 4, osvw_status; 86 87 static DEFINE_PER_CPU(u64, current_tsc_ratio); 88 #define TSC_RATIO_DEFAULT 0x0100000000ULL 89 90 static const struct svm_direct_access_msrs { 91 u32 index; /* Index of the MSR */ 92 bool always; /* True if intercept is initially cleared */ 93 } direct_access_msrs[MAX_DIRECT_ACCESS_MSRS] = { 94 { .index = MSR_STAR, .always = true }, 95 { .index = MSR_IA32_SYSENTER_CS, .always = true }, 96 { .index = MSR_IA32_SYSENTER_EIP, .always = false }, 97 { .index = MSR_IA32_SYSENTER_ESP, .always = false }, 98 #ifdef CONFIG_X86_64 99 { .index = MSR_GS_BASE, .always = true }, 100 { .index = MSR_FS_BASE, .always = true }, 101 { .index = MSR_KERNEL_GS_BASE, .always = true }, 102 { .index = MSR_LSTAR, .always = true }, 103 { .index = MSR_CSTAR, .always = true }, 104 { .index = MSR_SYSCALL_MASK, .always = true }, 105 #endif 106 { .index = MSR_IA32_SPEC_CTRL, .always = false }, 107 { .index = MSR_IA32_PRED_CMD, .always = false }, 108 { .index = MSR_IA32_LASTBRANCHFROMIP, .always = false }, 109 { .index = MSR_IA32_LASTBRANCHTOIP, .always = false }, 110 { .index = MSR_IA32_LASTINTFROMIP, .always = false }, 111 { .index = MSR_IA32_LASTINTTOIP, .always = false }, 112 { .index = MSR_EFER, .always = false }, 113 { .index = MSR_IA32_CR_PAT, .always = false }, 114 { .index = MSR_AMD64_SEV_ES_GHCB, .always = true }, 115 { .index = MSR_INVALID, .always = false }, 116 }; 117 118 /* 119 * These 2 parameters are used to config the controls for Pause-Loop Exiting: 120 * pause_filter_count: On processors that support Pause filtering(indicated 121 * by CPUID Fn8000_000A_EDX), the VMCB provides a 16 bit pause filter 122 * count value. On VMRUN this value is loaded into an internal counter. 123 * Each time a pause instruction is executed, this counter is decremented 124 * until it reaches zero at which time a #VMEXIT is generated if pause 125 * intercept is enabled. Refer to AMD APM Vol 2 Section 15.14.4 Pause 126 * Intercept Filtering for more details. 127 * This also indicate if ple logic enabled. 128 * 129 * pause_filter_thresh: In addition, some processor families support advanced 130 * pause filtering (indicated by CPUID Fn8000_000A_EDX) upper bound on 131 * the amount of time a guest is allowed to execute in a pause loop. 132 * In this mode, a 16-bit pause filter threshold field is added in the 133 * VMCB. The threshold value is a cycle count that is used to reset the 134 * pause counter. As with simple pause filtering, VMRUN loads the pause 135 * count value from VMCB into an internal counter. Then, on each pause 136 * instruction the hardware checks the elapsed number of cycles since 137 * the most recent pause instruction against the pause filter threshold. 138 * If the elapsed cycle count is greater than the pause filter threshold, 139 * then the internal pause count is reloaded from the VMCB and execution 140 * continues. If the elapsed cycle count is less than the pause filter 141 * threshold, then the internal pause count is decremented. If the count 142 * value is less than zero and PAUSE intercept is enabled, a #VMEXIT is 143 * triggered. If advanced pause filtering is supported and pause filter 144 * threshold field is set to zero, the filter will operate in the simpler, 145 * count only mode. 146 */ 147 148 static unsigned short pause_filter_thresh = KVM_DEFAULT_PLE_GAP; 149 module_param(pause_filter_thresh, ushort, 0444); 150 151 static unsigned short pause_filter_count = KVM_SVM_DEFAULT_PLE_WINDOW; 152 module_param(pause_filter_count, ushort, 0444); 153 154 /* Default doubles per-vcpu window every exit. */ 155 static unsigned short pause_filter_count_grow = KVM_DEFAULT_PLE_WINDOW_GROW; 156 module_param(pause_filter_count_grow, ushort, 0444); 157 158 /* Default resets per-vcpu window every exit to pause_filter_count. */ 159 static unsigned short pause_filter_count_shrink = KVM_DEFAULT_PLE_WINDOW_SHRINK; 160 module_param(pause_filter_count_shrink, ushort, 0444); 161 162 /* Default is to compute the maximum so we can never overflow. */ 163 static unsigned short pause_filter_count_max = KVM_SVM_DEFAULT_PLE_WINDOW_MAX; 164 module_param(pause_filter_count_max, ushort, 0444); 165 166 /* 167 * Use nested page tables by default. Note, NPT may get forced off by 168 * svm_hardware_setup() if it's unsupported by hardware or the host kernel. 169 */ 170 bool npt_enabled = true; 171 module_param_named(npt, npt_enabled, bool, 0444); 172 173 /* allow nested virtualization in KVM/SVM */ 174 static int nested = true; 175 module_param(nested, int, S_IRUGO); 176 177 /* enable/disable Next RIP Save */ 178 static int nrips = true; 179 module_param(nrips, int, 0444); 180 181 /* enable/disable Virtual VMLOAD VMSAVE */ 182 static int vls = true; 183 module_param(vls, int, 0444); 184 185 /* enable/disable Virtual GIF */ 186 static int vgif = true; 187 module_param(vgif, int, 0444); 188 189 /* 190 * enable / disable AVIC. Because the defaults differ for APICv 191 * support between VMX and SVM we cannot use module_param_named. 192 */ 193 static bool avic; 194 module_param(avic, bool, 0444); 195 196 bool __read_mostly dump_invalid_vmcb; 197 module_param(dump_invalid_vmcb, bool, 0644); 198 199 200 bool intercept_smi = true; 201 module_param(intercept_smi, bool, 0444); 202 203 204 static bool svm_gp_erratum_intercept = true; 205 206 static u8 rsm_ins_bytes[] = "\x0f\xaa"; 207 208 static unsigned long iopm_base; 209 210 struct kvm_ldttss_desc { 211 u16 limit0; 212 u16 base0; 213 unsigned base1:8, type:5, dpl:2, p:1; 214 unsigned limit1:4, zero0:3, g:1, base2:8; 215 u32 base3; 216 u32 zero1; 217 } __attribute__((packed)); 218 219 DEFINE_PER_CPU(struct svm_cpu_data *, svm_data); 220 221 /* 222 * Only MSR_TSC_AUX is switched via the user return hook. EFER is switched via 223 * the VMCB, and the SYSCALL/SYSENTER MSRs are handled by VMLOAD/VMSAVE. 224 * 225 * RDTSCP and RDPID are not used in the kernel, specifically to allow KVM to 226 * defer the restoration of TSC_AUX until the CPU returns to userspace. 227 */ 228 static int tsc_aux_uret_slot __read_mostly = -1; 229 230 static const u32 msrpm_ranges[] = {0, 0xc0000000, 0xc0010000}; 231 232 #define NUM_MSR_MAPS ARRAY_SIZE(msrpm_ranges) 233 #define MSRS_RANGE_SIZE 2048 234 #define MSRS_IN_RANGE (MSRS_RANGE_SIZE * 8 / 2) 235 236 u32 svm_msrpm_offset(u32 msr) 237 { 238 u32 offset; 239 int i; 240 241 for (i = 0; i < NUM_MSR_MAPS; i++) { 242 if (msr < msrpm_ranges[i] || 243 msr >= msrpm_ranges[i] + MSRS_IN_RANGE) 244 continue; 245 246 offset = (msr - msrpm_ranges[i]) / 4; /* 4 msrs per u8 */ 247 offset += (i * MSRS_RANGE_SIZE); /* add range offset */ 248 249 /* Now we have the u8 offset - but need the u32 offset */ 250 return offset / 4; 251 } 252 253 /* MSR not in any range */ 254 return MSR_INVALID; 255 } 256 257 #define MAX_INST_SIZE 15 258 259 static int get_max_npt_level(void) 260 { 261 #ifdef CONFIG_X86_64 262 return pgtable_l5_enabled() ? PT64_ROOT_5LEVEL : PT64_ROOT_4LEVEL; 263 #else 264 return PT32E_ROOT_LEVEL; 265 #endif 266 } 267 268 int svm_set_efer(struct kvm_vcpu *vcpu, u64 efer) 269 { 270 struct vcpu_svm *svm = to_svm(vcpu); 271 u64 old_efer = vcpu->arch.efer; 272 vcpu->arch.efer = efer; 273 274 if (!npt_enabled) { 275 /* Shadow paging assumes NX to be available. */ 276 efer |= EFER_NX; 277 278 if (!(efer & EFER_LMA)) 279 efer &= ~EFER_LME; 280 } 281 282 if ((old_efer & EFER_SVME) != (efer & EFER_SVME)) { 283 if (!(efer & EFER_SVME)) { 284 svm_leave_nested(svm); 285 svm_set_gif(svm, true); 286 /* #GP intercept is still needed for vmware backdoor */ 287 if (!enable_vmware_backdoor) 288 clr_exception_intercept(svm, GP_VECTOR); 289 290 /* 291 * Free the nested guest state, unless we are in SMM. 292 * In this case we will return to the nested guest 293 * as soon as we leave SMM. 294 */ 295 if (!is_smm(vcpu)) 296 svm_free_nested(svm); 297 298 } else { 299 int ret = svm_allocate_nested(svm); 300 301 if (ret) { 302 vcpu->arch.efer = old_efer; 303 return ret; 304 } 305 306 if (svm_gp_erratum_intercept) 307 set_exception_intercept(svm, GP_VECTOR); 308 } 309 } 310 311 svm->vmcb->save.efer = efer | EFER_SVME; 312 vmcb_mark_dirty(svm->vmcb, VMCB_CR); 313 return 0; 314 } 315 316 static int is_external_interrupt(u32 info) 317 { 318 info &= SVM_EVTINJ_TYPE_MASK | SVM_EVTINJ_VALID; 319 return info == (SVM_EVTINJ_VALID | SVM_EVTINJ_TYPE_INTR); 320 } 321 322 static u32 svm_get_interrupt_shadow(struct kvm_vcpu *vcpu) 323 { 324 struct vcpu_svm *svm = to_svm(vcpu); 325 u32 ret = 0; 326 327 if (svm->vmcb->control.int_state & SVM_INTERRUPT_SHADOW_MASK) 328 ret = KVM_X86_SHADOW_INT_STI | KVM_X86_SHADOW_INT_MOV_SS; 329 return ret; 330 } 331 332 static void svm_set_interrupt_shadow(struct kvm_vcpu *vcpu, int mask) 333 { 334 struct vcpu_svm *svm = to_svm(vcpu); 335 336 if (mask == 0) 337 svm->vmcb->control.int_state &= ~SVM_INTERRUPT_SHADOW_MASK; 338 else 339 svm->vmcb->control.int_state |= SVM_INTERRUPT_SHADOW_MASK; 340 341 } 342 343 static int skip_emulated_instruction(struct kvm_vcpu *vcpu) 344 { 345 struct vcpu_svm *svm = to_svm(vcpu); 346 347 /* 348 * SEV-ES does not expose the next RIP. The RIP update is controlled by 349 * the type of exit and the #VC handler in the guest. 350 */ 351 if (sev_es_guest(vcpu->kvm)) 352 goto done; 353 354 if (nrips && svm->vmcb->control.next_rip != 0) { 355 WARN_ON_ONCE(!static_cpu_has(X86_FEATURE_NRIPS)); 356 svm->next_rip = svm->vmcb->control.next_rip; 357 } 358 359 if (!svm->next_rip) { 360 if (!kvm_emulate_instruction(vcpu, EMULTYPE_SKIP)) 361 return 0; 362 } else { 363 kvm_rip_write(vcpu, svm->next_rip); 364 } 365 366 done: 367 svm_set_interrupt_shadow(vcpu, 0); 368 369 return 1; 370 } 371 372 static void svm_queue_exception(struct kvm_vcpu *vcpu) 373 { 374 struct vcpu_svm *svm = to_svm(vcpu); 375 unsigned nr = vcpu->arch.exception.nr; 376 bool has_error_code = vcpu->arch.exception.has_error_code; 377 u32 error_code = vcpu->arch.exception.error_code; 378 379 kvm_deliver_exception_payload(vcpu); 380 381 if (nr == BP_VECTOR && !nrips) { 382 unsigned long rip, old_rip = kvm_rip_read(vcpu); 383 384 /* 385 * For guest debugging where we have to reinject #BP if some 386 * INT3 is guest-owned: 387 * Emulate nRIP by moving RIP forward. Will fail if injection 388 * raises a fault that is not intercepted. Still better than 389 * failing in all cases. 390 */ 391 (void)skip_emulated_instruction(vcpu); 392 rip = kvm_rip_read(vcpu); 393 svm->int3_rip = rip + svm->vmcb->save.cs.base; 394 svm->int3_injected = rip - old_rip; 395 } 396 397 svm->vmcb->control.event_inj = nr 398 | SVM_EVTINJ_VALID 399 | (has_error_code ? SVM_EVTINJ_VALID_ERR : 0) 400 | SVM_EVTINJ_TYPE_EXEPT; 401 svm->vmcb->control.event_inj_err = error_code; 402 } 403 404 static void svm_init_erratum_383(void) 405 { 406 u32 low, high; 407 int err; 408 u64 val; 409 410 if (!static_cpu_has_bug(X86_BUG_AMD_TLB_MMATCH)) 411 return; 412 413 /* Use _safe variants to not break nested virtualization */ 414 val = native_read_msr_safe(MSR_AMD64_DC_CFG, &err); 415 if (err) 416 return; 417 418 val |= (1ULL << 47); 419 420 low = lower_32_bits(val); 421 high = upper_32_bits(val); 422 423 native_write_msr_safe(MSR_AMD64_DC_CFG, low, high); 424 425 erratum_383_found = true; 426 } 427 428 static void svm_init_osvw(struct kvm_vcpu *vcpu) 429 { 430 /* 431 * Guests should see errata 400 and 415 as fixed (assuming that 432 * HLT and IO instructions are intercepted). 433 */ 434 vcpu->arch.osvw.length = (osvw_len >= 3) ? (osvw_len) : 3; 435 vcpu->arch.osvw.status = osvw_status & ~(6ULL); 436 437 /* 438 * By increasing VCPU's osvw.length to 3 we are telling the guest that 439 * all osvw.status bits inside that length, including bit 0 (which is 440 * reserved for erratum 298), are valid. However, if host processor's 441 * osvw_len is 0 then osvw_status[0] carries no information. We need to 442 * be conservative here and therefore we tell the guest that erratum 298 443 * is present (because we really don't know). 444 */ 445 if (osvw_len == 0 && boot_cpu_data.x86 == 0x10) 446 vcpu->arch.osvw.status |= 1; 447 } 448 449 static int has_svm(void) 450 { 451 const char *msg; 452 453 if (!cpu_has_svm(&msg)) { 454 printk(KERN_INFO "has_svm: %s\n", msg); 455 return 0; 456 } 457 458 if (sev_active()) { 459 pr_info("KVM is unsupported when running as an SEV guest\n"); 460 return 0; 461 } 462 463 return 1; 464 } 465 466 static void svm_hardware_disable(void) 467 { 468 /* Make sure we clean up behind us */ 469 if (static_cpu_has(X86_FEATURE_TSCRATEMSR)) 470 wrmsrl(MSR_AMD64_TSC_RATIO, TSC_RATIO_DEFAULT); 471 472 cpu_svm_disable(); 473 474 amd_pmu_disable_virt(); 475 } 476 477 static int svm_hardware_enable(void) 478 { 479 480 struct svm_cpu_data *sd; 481 uint64_t efer; 482 struct desc_struct *gdt; 483 int me = raw_smp_processor_id(); 484 485 rdmsrl(MSR_EFER, efer); 486 if (efer & EFER_SVME) 487 return -EBUSY; 488 489 if (!has_svm()) { 490 pr_err("%s: err EOPNOTSUPP on %d\n", __func__, me); 491 return -EINVAL; 492 } 493 sd = per_cpu(svm_data, me); 494 if (!sd) { 495 pr_err("%s: svm_data is NULL on %d\n", __func__, me); 496 return -EINVAL; 497 } 498 499 sd->asid_generation = 1; 500 sd->max_asid = cpuid_ebx(SVM_CPUID_FUNC) - 1; 501 sd->next_asid = sd->max_asid + 1; 502 sd->min_asid = max_sev_asid + 1; 503 504 gdt = get_current_gdt_rw(); 505 sd->tss_desc = (struct kvm_ldttss_desc *)(gdt + GDT_ENTRY_TSS); 506 507 wrmsrl(MSR_EFER, efer | EFER_SVME); 508 509 wrmsrl(MSR_VM_HSAVE_PA, __sme_page_pa(sd->save_area)); 510 511 if (static_cpu_has(X86_FEATURE_TSCRATEMSR)) { 512 wrmsrl(MSR_AMD64_TSC_RATIO, TSC_RATIO_DEFAULT); 513 __this_cpu_write(current_tsc_ratio, TSC_RATIO_DEFAULT); 514 } 515 516 517 /* 518 * Get OSVW bits. 519 * 520 * Note that it is possible to have a system with mixed processor 521 * revisions and therefore different OSVW bits. If bits are not the same 522 * on different processors then choose the worst case (i.e. if erratum 523 * is present on one processor and not on another then assume that the 524 * erratum is present everywhere). 525 */ 526 if (cpu_has(&boot_cpu_data, X86_FEATURE_OSVW)) { 527 uint64_t len, status = 0; 528 int err; 529 530 len = native_read_msr_safe(MSR_AMD64_OSVW_ID_LENGTH, &err); 531 if (!err) 532 status = native_read_msr_safe(MSR_AMD64_OSVW_STATUS, 533 &err); 534 535 if (err) 536 osvw_status = osvw_len = 0; 537 else { 538 if (len < osvw_len) 539 osvw_len = len; 540 osvw_status |= status; 541 osvw_status &= (1ULL << osvw_len) - 1; 542 } 543 } else 544 osvw_status = osvw_len = 0; 545 546 svm_init_erratum_383(); 547 548 amd_pmu_enable_virt(); 549 550 return 0; 551 } 552 553 static void svm_cpu_uninit(int cpu) 554 { 555 struct svm_cpu_data *sd = per_cpu(svm_data, cpu); 556 557 if (!sd) 558 return; 559 560 per_cpu(svm_data, cpu) = NULL; 561 kfree(sd->sev_vmcbs); 562 __free_page(sd->save_area); 563 kfree(sd); 564 } 565 566 static int svm_cpu_init(int cpu) 567 { 568 struct svm_cpu_data *sd; 569 int ret = -ENOMEM; 570 571 sd = kzalloc(sizeof(struct svm_cpu_data), GFP_KERNEL); 572 if (!sd) 573 return ret; 574 sd->cpu = cpu; 575 sd->save_area = alloc_page(GFP_KERNEL); 576 if (!sd->save_area) 577 goto free_cpu_data; 578 579 clear_page(page_address(sd->save_area)); 580 581 ret = sev_cpu_init(sd); 582 if (ret) 583 goto free_save_area; 584 585 per_cpu(svm_data, cpu) = sd; 586 587 return 0; 588 589 free_save_area: 590 __free_page(sd->save_area); 591 free_cpu_data: 592 kfree(sd); 593 return ret; 594 595 } 596 597 static int direct_access_msr_slot(u32 msr) 598 { 599 u32 i; 600 601 for (i = 0; direct_access_msrs[i].index != MSR_INVALID; i++) 602 if (direct_access_msrs[i].index == msr) 603 return i; 604 605 return -ENOENT; 606 } 607 608 static void set_shadow_msr_intercept(struct kvm_vcpu *vcpu, u32 msr, int read, 609 int write) 610 { 611 struct vcpu_svm *svm = to_svm(vcpu); 612 int slot = direct_access_msr_slot(msr); 613 614 if (slot == -ENOENT) 615 return; 616 617 /* Set the shadow bitmaps to the desired intercept states */ 618 if (read) 619 set_bit(slot, svm->shadow_msr_intercept.read); 620 else 621 clear_bit(slot, svm->shadow_msr_intercept.read); 622 623 if (write) 624 set_bit(slot, svm->shadow_msr_intercept.write); 625 else 626 clear_bit(slot, svm->shadow_msr_intercept.write); 627 } 628 629 static bool valid_msr_intercept(u32 index) 630 { 631 return direct_access_msr_slot(index) != -ENOENT; 632 } 633 634 static bool msr_write_intercepted(struct kvm_vcpu *vcpu, u32 msr) 635 { 636 u8 bit_write; 637 unsigned long tmp; 638 u32 offset; 639 u32 *msrpm; 640 641 msrpm = is_guest_mode(vcpu) ? to_svm(vcpu)->nested.msrpm: 642 to_svm(vcpu)->msrpm; 643 644 offset = svm_msrpm_offset(msr); 645 bit_write = 2 * (msr & 0x0f) + 1; 646 tmp = msrpm[offset]; 647 648 BUG_ON(offset == MSR_INVALID); 649 650 return !!test_bit(bit_write, &tmp); 651 } 652 653 static void set_msr_interception_bitmap(struct kvm_vcpu *vcpu, u32 *msrpm, 654 u32 msr, int read, int write) 655 { 656 u8 bit_read, bit_write; 657 unsigned long tmp; 658 u32 offset; 659 660 /* 661 * If this warning triggers extend the direct_access_msrs list at the 662 * beginning of the file 663 */ 664 WARN_ON(!valid_msr_intercept(msr)); 665 666 /* Enforce non allowed MSRs to trap */ 667 if (read && !kvm_msr_allowed(vcpu, msr, KVM_MSR_FILTER_READ)) 668 read = 0; 669 670 if (write && !kvm_msr_allowed(vcpu, msr, KVM_MSR_FILTER_WRITE)) 671 write = 0; 672 673 offset = svm_msrpm_offset(msr); 674 bit_read = 2 * (msr & 0x0f); 675 bit_write = 2 * (msr & 0x0f) + 1; 676 tmp = msrpm[offset]; 677 678 BUG_ON(offset == MSR_INVALID); 679 680 read ? clear_bit(bit_read, &tmp) : set_bit(bit_read, &tmp); 681 write ? clear_bit(bit_write, &tmp) : set_bit(bit_write, &tmp); 682 683 msrpm[offset] = tmp; 684 685 svm_hv_vmcb_dirty_nested_enlightenments(vcpu); 686 687 } 688 689 void set_msr_interception(struct kvm_vcpu *vcpu, u32 *msrpm, u32 msr, 690 int read, int write) 691 { 692 set_shadow_msr_intercept(vcpu, msr, read, write); 693 set_msr_interception_bitmap(vcpu, msrpm, msr, read, write); 694 } 695 696 u32 *svm_vcpu_alloc_msrpm(void) 697 { 698 unsigned int order = get_order(MSRPM_SIZE); 699 struct page *pages = alloc_pages(GFP_KERNEL_ACCOUNT, order); 700 u32 *msrpm; 701 702 if (!pages) 703 return NULL; 704 705 msrpm = page_address(pages); 706 memset(msrpm, 0xff, PAGE_SIZE * (1 << order)); 707 708 return msrpm; 709 } 710 711 void svm_vcpu_init_msrpm(struct kvm_vcpu *vcpu, u32 *msrpm) 712 { 713 int i; 714 715 for (i = 0; direct_access_msrs[i].index != MSR_INVALID; i++) { 716 if (!direct_access_msrs[i].always) 717 continue; 718 set_msr_interception(vcpu, msrpm, direct_access_msrs[i].index, 1, 1); 719 } 720 } 721 722 723 void svm_vcpu_free_msrpm(u32 *msrpm) 724 { 725 __free_pages(virt_to_page(msrpm), get_order(MSRPM_SIZE)); 726 } 727 728 static void svm_msr_filter_changed(struct kvm_vcpu *vcpu) 729 { 730 struct vcpu_svm *svm = to_svm(vcpu); 731 u32 i; 732 733 /* 734 * Set intercept permissions for all direct access MSRs again. They 735 * will automatically get filtered through the MSR filter, so we are 736 * back in sync after this. 737 */ 738 for (i = 0; direct_access_msrs[i].index != MSR_INVALID; i++) { 739 u32 msr = direct_access_msrs[i].index; 740 u32 read = test_bit(i, svm->shadow_msr_intercept.read); 741 u32 write = test_bit(i, svm->shadow_msr_intercept.write); 742 743 set_msr_interception_bitmap(vcpu, svm->msrpm, msr, read, write); 744 } 745 } 746 747 static void add_msr_offset(u32 offset) 748 { 749 int i; 750 751 for (i = 0; i < MSRPM_OFFSETS; ++i) { 752 753 /* Offset already in list? */ 754 if (msrpm_offsets[i] == offset) 755 return; 756 757 /* Slot used by another offset? */ 758 if (msrpm_offsets[i] != MSR_INVALID) 759 continue; 760 761 /* Add offset to list */ 762 msrpm_offsets[i] = offset; 763 764 return; 765 } 766 767 /* 768 * If this BUG triggers the msrpm_offsets table has an overflow. Just 769 * increase MSRPM_OFFSETS in this case. 770 */ 771 BUG(); 772 } 773 774 static void init_msrpm_offsets(void) 775 { 776 int i; 777 778 memset(msrpm_offsets, 0xff, sizeof(msrpm_offsets)); 779 780 for (i = 0; direct_access_msrs[i].index != MSR_INVALID; i++) { 781 u32 offset; 782 783 offset = svm_msrpm_offset(direct_access_msrs[i].index); 784 BUG_ON(offset == MSR_INVALID); 785 786 add_msr_offset(offset); 787 } 788 } 789 790 static void svm_enable_lbrv(struct kvm_vcpu *vcpu) 791 { 792 struct vcpu_svm *svm = to_svm(vcpu); 793 794 svm->vmcb->control.virt_ext |= LBR_CTL_ENABLE_MASK; 795 set_msr_interception(vcpu, svm->msrpm, MSR_IA32_LASTBRANCHFROMIP, 1, 1); 796 set_msr_interception(vcpu, svm->msrpm, MSR_IA32_LASTBRANCHTOIP, 1, 1); 797 set_msr_interception(vcpu, svm->msrpm, MSR_IA32_LASTINTFROMIP, 1, 1); 798 set_msr_interception(vcpu, svm->msrpm, MSR_IA32_LASTINTTOIP, 1, 1); 799 } 800 801 static void svm_disable_lbrv(struct kvm_vcpu *vcpu) 802 { 803 struct vcpu_svm *svm = to_svm(vcpu); 804 805 svm->vmcb->control.virt_ext &= ~LBR_CTL_ENABLE_MASK; 806 set_msr_interception(vcpu, svm->msrpm, MSR_IA32_LASTBRANCHFROMIP, 0, 0); 807 set_msr_interception(vcpu, svm->msrpm, MSR_IA32_LASTBRANCHTOIP, 0, 0); 808 set_msr_interception(vcpu, svm->msrpm, MSR_IA32_LASTINTFROMIP, 0, 0); 809 set_msr_interception(vcpu, svm->msrpm, MSR_IA32_LASTINTTOIP, 0, 0); 810 } 811 812 void disable_nmi_singlestep(struct vcpu_svm *svm) 813 { 814 svm->nmi_singlestep = false; 815 816 if (!(svm->vcpu.guest_debug & KVM_GUESTDBG_SINGLESTEP)) { 817 /* Clear our flags if they were not set by the guest */ 818 if (!(svm->nmi_singlestep_guest_rflags & X86_EFLAGS_TF)) 819 svm->vmcb->save.rflags &= ~X86_EFLAGS_TF; 820 if (!(svm->nmi_singlestep_guest_rflags & X86_EFLAGS_RF)) 821 svm->vmcb->save.rflags &= ~X86_EFLAGS_RF; 822 } 823 } 824 825 static void grow_ple_window(struct kvm_vcpu *vcpu) 826 { 827 struct vcpu_svm *svm = to_svm(vcpu); 828 struct vmcb_control_area *control = &svm->vmcb->control; 829 int old = control->pause_filter_count; 830 831 control->pause_filter_count = __grow_ple_window(old, 832 pause_filter_count, 833 pause_filter_count_grow, 834 pause_filter_count_max); 835 836 if (control->pause_filter_count != old) { 837 vmcb_mark_dirty(svm->vmcb, VMCB_INTERCEPTS); 838 trace_kvm_ple_window_update(vcpu->vcpu_id, 839 control->pause_filter_count, old); 840 } 841 } 842 843 static void shrink_ple_window(struct kvm_vcpu *vcpu) 844 { 845 struct vcpu_svm *svm = to_svm(vcpu); 846 struct vmcb_control_area *control = &svm->vmcb->control; 847 int old = control->pause_filter_count; 848 849 control->pause_filter_count = 850 __shrink_ple_window(old, 851 pause_filter_count, 852 pause_filter_count_shrink, 853 pause_filter_count); 854 if (control->pause_filter_count != old) { 855 vmcb_mark_dirty(svm->vmcb, VMCB_INTERCEPTS); 856 trace_kvm_ple_window_update(vcpu->vcpu_id, 857 control->pause_filter_count, old); 858 } 859 } 860 861 /* 862 * The default MMIO mask is a single bit (excluding the present bit), 863 * which could conflict with the memory encryption bit. Check for 864 * memory encryption support and override the default MMIO mask if 865 * memory encryption is enabled. 866 */ 867 static __init void svm_adjust_mmio_mask(void) 868 { 869 unsigned int enc_bit, mask_bit; 870 u64 msr, mask; 871 872 /* If there is no memory encryption support, use existing mask */ 873 if (cpuid_eax(0x80000000) < 0x8000001f) 874 return; 875 876 /* If memory encryption is not enabled, use existing mask */ 877 rdmsrl(MSR_AMD64_SYSCFG, msr); 878 if (!(msr & MSR_AMD64_SYSCFG_MEM_ENCRYPT)) 879 return; 880 881 enc_bit = cpuid_ebx(0x8000001f) & 0x3f; 882 mask_bit = boot_cpu_data.x86_phys_bits; 883 884 /* Increment the mask bit if it is the same as the encryption bit */ 885 if (enc_bit == mask_bit) 886 mask_bit++; 887 888 /* 889 * If the mask bit location is below 52, then some bits above the 890 * physical addressing limit will always be reserved, so use the 891 * rsvd_bits() function to generate the mask. This mask, along with 892 * the present bit, will be used to generate a page fault with 893 * PFER.RSV = 1. 894 * 895 * If the mask bit location is 52 (or above), then clear the mask. 896 */ 897 mask = (mask_bit < 52) ? rsvd_bits(mask_bit, 51) | PT_PRESENT_MASK : 0; 898 899 kvm_mmu_set_mmio_spte_mask(mask, mask, PT_WRITABLE_MASK | PT_USER_MASK); 900 } 901 902 static void svm_hardware_teardown(void) 903 { 904 int cpu; 905 906 sev_hardware_teardown(); 907 908 for_each_possible_cpu(cpu) 909 svm_cpu_uninit(cpu); 910 911 __free_pages(pfn_to_page(iopm_base >> PAGE_SHIFT), 912 get_order(IOPM_SIZE)); 913 iopm_base = 0; 914 } 915 916 static __init void svm_set_cpu_caps(void) 917 { 918 kvm_set_cpu_caps(); 919 920 supported_xss = 0; 921 922 /* CPUID 0x80000001 and 0x8000000A (SVM features) */ 923 if (nested) { 924 kvm_cpu_cap_set(X86_FEATURE_SVM); 925 926 if (nrips) 927 kvm_cpu_cap_set(X86_FEATURE_NRIPS); 928 929 if (npt_enabled) 930 kvm_cpu_cap_set(X86_FEATURE_NPT); 931 932 /* Nested VM can receive #VMEXIT instead of triggering #GP */ 933 kvm_cpu_cap_set(X86_FEATURE_SVME_ADDR_CHK); 934 } 935 936 /* CPUID 0x80000008 */ 937 if (boot_cpu_has(X86_FEATURE_LS_CFG_SSBD) || 938 boot_cpu_has(X86_FEATURE_AMD_SSBD)) 939 kvm_cpu_cap_set(X86_FEATURE_VIRT_SSBD); 940 941 /* CPUID 0x8000001F (SME/SEV features) */ 942 sev_set_cpu_caps(); 943 } 944 945 static __init int svm_hardware_setup(void) 946 { 947 int cpu; 948 struct page *iopm_pages; 949 void *iopm_va; 950 int r; 951 unsigned int order = get_order(IOPM_SIZE); 952 953 /* 954 * NX is required for shadow paging and for NPT if the NX huge pages 955 * mitigation is enabled. 956 */ 957 if (!boot_cpu_has(X86_FEATURE_NX)) { 958 pr_err_ratelimited("NX (Execute Disable) not supported\n"); 959 return -EOPNOTSUPP; 960 } 961 kvm_enable_efer_bits(EFER_NX); 962 963 iopm_pages = alloc_pages(GFP_KERNEL, order); 964 965 if (!iopm_pages) 966 return -ENOMEM; 967 968 iopm_va = page_address(iopm_pages); 969 memset(iopm_va, 0xff, PAGE_SIZE * (1 << order)); 970 iopm_base = page_to_pfn(iopm_pages) << PAGE_SHIFT; 971 972 init_msrpm_offsets(); 973 974 supported_xcr0 &= ~(XFEATURE_MASK_BNDREGS | XFEATURE_MASK_BNDCSR); 975 976 if (boot_cpu_has(X86_FEATURE_FXSR_OPT)) 977 kvm_enable_efer_bits(EFER_FFXSR); 978 979 if (boot_cpu_has(X86_FEATURE_TSCRATEMSR)) { 980 kvm_has_tsc_control = true; 981 kvm_max_tsc_scaling_ratio = TSC_RATIO_MAX; 982 kvm_tsc_scaling_ratio_frac_bits = 32; 983 } 984 985 tsc_aux_uret_slot = kvm_add_user_return_msr(MSR_TSC_AUX); 986 987 /* Check for pause filtering support */ 988 if (!boot_cpu_has(X86_FEATURE_PAUSEFILTER)) { 989 pause_filter_count = 0; 990 pause_filter_thresh = 0; 991 } else if (!boot_cpu_has(X86_FEATURE_PFTHRESHOLD)) { 992 pause_filter_thresh = 0; 993 } 994 995 if (nested) { 996 printk(KERN_INFO "kvm: Nested Virtualization enabled\n"); 997 kvm_enable_efer_bits(EFER_SVME | EFER_LMSLE); 998 } 999 1000 /* 1001 * KVM's MMU doesn't support using 2-level paging for itself, and thus 1002 * NPT isn't supported if the host is using 2-level paging since host 1003 * CR4 is unchanged on VMRUN. 1004 */ 1005 if (!IS_ENABLED(CONFIG_X86_64) && !IS_ENABLED(CONFIG_X86_PAE)) 1006 npt_enabled = false; 1007 1008 if (!boot_cpu_has(X86_FEATURE_NPT)) 1009 npt_enabled = false; 1010 1011 /* Force VM NPT level equal to the host's max NPT level */ 1012 kvm_configure_mmu(npt_enabled, get_max_npt_level(), 1013 get_max_npt_level(), PG_LEVEL_1G); 1014 pr_info("kvm: Nested Paging %sabled\n", npt_enabled ? "en" : "dis"); 1015 1016 /* Note, SEV setup consumes npt_enabled. */ 1017 sev_hardware_setup(); 1018 1019 svm_hv_hardware_setup(); 1020 1021 svm_adjust_mmio_mask(); 1022 1023 for_each_possible_cpu(cpu) { 1024 r = svm_cpu_init(cpu); 1025 if (r) 1026 goto err; 1027 } 1028 1029 if (nrips) { 1030 if (!boot_cpu_has(X86_FEATURE_NRIPS)) 1031 nrips = false; 1032 } 1033 1034 enable_apicv = avic = avic && npt_enabled && boot_cpu_has(X86_FEATURE_AVIC); 1035 1036 if (enable_apicv) { 1037 pr_info("AVIC enabled\n"); 1038 1039 amd_iommu_register_ga_log_notifier(&avic_ga_log_notifier); 1040 } 1041 1042 if (vls) { 1043 if (!npt_enabled || 1044 !boot_cpu_has(X86_FEATURE_V_VMSAVE_VMLOAD) || 1045 !IS_ENABLED(CONFIG_X86_64)) { 1046 vls = false; 1047 } else { 1048 pr_info("Virtual VMLOAD VMSAVE supported\n"); 1049 } 1050 } 1051 1052 if (boot_cpu_has(X86_FEATURE_SVME_ADDR_CHK)) 1053 svm_gp_erratum_intercept = false; 1054 1055 if (vgif) { 1056 if (!boot_cpu_has(X86_FEATURE_VGIF)) 1057 vgif = false; 1058 else 1059 pr_info("Virtual GIF supported\n"); 1060 } 1061 1062 svm_set_cpu_caps(); 1063 1064 /* 1065 * It seems that on AMD processors PTE's accessed bit is 1066 * being set by the CPU hardware before the NPF vmexit. 1067 * This is not expected behaviour and our tests fail because 1068 * of it. 1069 * A workaround here is to disable support for 1070 * GUEST_MAXPHYADDR < HOST_MAXPHYADDR if NPT is enabled. 1071 * In this case userspace can know if there is support using 1072 * KVM_CAP_SMALLER_MAXPHYADDR extension and decide how to handle 1073 * it 1074 * If future AMD CPU models change the behaviour described above, 1075 * this variable can be changed accordingly 1076 */ 1077 allow_smaller_maxphyaddr = !npt_enabled; 1078 1079 return 0; 1080 1081 err: 1082 svm_hardware_teardown(); 1083 return r; 1084 } 1085 1086 static void init_seg(struct vmcb_seg *seg) 1087 { 1088 seg->selector = 0; 1089 seg->attrib = SVM_SELECTOR_P_MASK | SVM_SELECTOR_S_MASK | 1090 SVM_SELECTOR_WRITE_MASK; /* Read/Write Data Segment */ 1091 seg->limit = 0xffff; 1092 seg->base = 0; 1093 } 1094 1095 static void init_sys_seg(struct vmcb_seg *seg, uint32_t type) 1096 { 1097 seg->selector = 0; 1098 seg->attrib = SVM_SELECTOR_P_MASK | type; 1099 seg->limit = 0xffff; 1100 seg->base = 0; 1101 } 1102 1103 static u64 svm_get_l2_tsc_offset(struct kvm_vcpu *vcpu) 1104 { 1105 struct vcpu_svm *svm = to_svm(vcpu); 1106 1107 return svm->nested.ctl.tsc_offset; 1108 } 1109 1110 static u64 svm_get_l2_tsc_multiplier(struct kvm_vcpu *vcpu) 1111 { 1112 return kvm_default_tsc_scaling_ratio; 1113 } 1114 1115 static void svm_write_tsc_offset(struct kvm_vcpu *vcpu, u64 offset) 1116 { 1117 struct vcpu_svm *svm = to_svm(vcpu); 1118 1119 svm->vmcb01.ptr->control.tsc_offset = vcpu->arch.l1_tsc_offset; 1120 svm->vmcb->control.tsc_offset = offset; 1121 vmcb_mark_dirty(svm->vmcb, VMCB_INTERCEPTS); 1122 } 1123 1124 static void svm_write_tsc_multiplier(struct kvm_vcpu *vcpu, u64 multiplier) 1125 { 1126 wrmsrl(MSR_AMD64_TSC_RATIO, multiplier); 1127 } 1128 1129 /* Evaluate instruction intercepts that depend on guest CPUID features. */ 1130 static void svm_recalc_instruction_intercepts(struct kvm_vcpu *vcpu, 1131 struct vcpu_svm *svm) 1132 { 1133 /* 1134 * Intercept INVPCID if shadow paging is enabled to sync/free shadow 1135 * roots, or if INVPCID is disabled in the guest to inject #UD. 1136 */ 1137 if (kvm_cpu_cap_has(X86_FEATURE_INVPCID)) { 1138 if (!npt_enabled || 1139 !guest_cpuid_has(&svm->vcpu, X86_FEATURE_INVPCID)) 1140 svm_set_intercept(svm, INTERCEPT_INVPCID); 1141 else 1142 svm_clr_intercept(svm, INTERCEPT_INVPCID); 1143 } 1144 1145 if (kvm_cpu_cap_has(X86_FEATURE_RDTSCP)) { 1146 if (guest_cpuid_has(vcpu, X86_FEATURE_RDTSCP)) 1147 svm_clr_intercept(svm, INTERCEPT_RDTSCP); 1148 else 1149 svm_set_intercept(svm, INTERCEPT_RDTSCP); 1150 } 1151 } 1152 1153 static void init_vmcb(struct kvm_vcpu *vcpu) 1154 { 1155 struct vcpu_svm *svm = to_svm(vcpu); 1156 struct vmcb_control_area *control = &svm->vmcb->control; 1157 struct vmcb_save_area *save = &svm->vmcb->save; 1158 1159 svm_set_intercept(svm, INTERCEPT_CR0_READ); 1160 svm_set_intercept(svm, INTERCEPT_CR3_READ); 1161 svm_set_intercept(svm, INTERCEPT_CR4_READ); 1162 svm_set_intercept(svm, INTERCEPT_CR0_WRITE); 1163 svm_set_intercept(svm, INTERCEPT_CR3_WRITE); 1164 svm_set_intercept(svm, INTERCEPT_CR4_WRITE); 1165 if (!kvm_vcpu_apicv_active(vcpu)) 1166 svm_set_intercept(svm, INTERCEPT_CR8_WRITE); 1167 1168 set_dr_intercepts(svm); 1169 1170 set_exception_intercept(svm, PF_VECTOR); 1171 set_exception_intercept(svm, UD_VECTOR); 1172 set_exception_intercept(svm, MC_VECTOR); 1173 set_exception_intercept(svm, AC_VECTOR); 1174 set_exception_intercept(svm, DB_VECTOR); 1175 /* 1176 * Guest access to VMware backdoor ports could legitimately 1177 * trigger #GP because of TSS I/O permission bitmap. 1178 * We intercept those #GP and allow access to them anyway 1179 * as VMware does. 1180 */ 1181 if (enable_vmware_backdoor) 1182 set_exception_intercept(svm, GP_VECTOR); 1183 1184 svm_set_intercept(svm, INTERCEPT_INTR); 1185 svm_set_intercept(svm, INTERCEPT_NMI); 1186 1187 if (intercept_smi) 1188 svm_set_intercept(svm, INTERCEPT_SMI); 1189 1190 svm_set_intercept(svm, INTERCEPT_SELECTIVE_CR0); 1191 svm_set_intercept(svm, INTERCEPT_RDPMC); 1192 svm_set_intercept(svm, INTERCEPT_CPUID); 1193 svm_set_intercept(svm, INTERCEPT_INVD); 1194 svm_set_intercept(svm, INTERCEPT_INVLPG); 1195 svm_set_intercept(svm, INTERCEPT_INVLPGA); 1196 svm_set_intercept(svm, INTERCEPT_IOIO_PROT); 1197 svm_set_intercept(svm, INTERCEPT_MSR_PROT); 1198 svm_set_intercept(svm, INTERCEPT_TASK_SWITCH); 1199 svm_set_intercept(svm, INTERCEPT_SHUTDOWN); 1200 svm_set_intercept(svm, INTERCEPT_VMRUN); 1201 svm_set_intercept(svm, INTERCEPT_VMMCALL); 1202 svm_set_intercept(svm, INTERCEPT_VMLOAD); 1203 svm_set_intercept(svm, INTERCEPT_VMSAVE); 1204 svm_set_intercept(svm, INTERCEPT_STGI); 1205 svm_set_intercept(svm, INTERCEPT_CLGI); 1206 svm_set_intercept(svm, INTERCEPT_SKINIT); 1207 svm_set_intercept(svm, INTERCEPT_WBINVD); 1208 svm_set_intercept(svm, INTERCEPT_XSETBV); 1209 svm_set_intercept(svm, INTERCEPT_RDPRU); 1210 svm_set_intercept(svm, INTERCEPT_RSM); 1211 1212 if (!kvm_mwait_in_guest(vcpu->kvm)) { 1213 svm_set_intercept(svm, INTERCEPT_MONITOR); 1214 svm_set_intercept(svm, INTERCEPT_MWAIT); 1215 } 1216 1217 if (!kvm_hlt_in_guest(vcpu->kvm)) 1218 svm_set_intercept(svm, INTERCEPT_HLT); 1219 1220 control->iopm_base_pa = __sme_set(iopm_base); 1221 control->msrpm_base_pa = __sme_set(__pa(svm->msrpm)); 1222 control->int_ctl = V_INTR_MASKING_MASK; 1223 1224 init_seg(&save->es); 1225 init_seg(&save->ss); 1226 init_seg(&save->ds); 1227 init_seg(&save->fs); 1228 init_seg(&save->gs); 1229 1230 save->cs.selector = 0xf000; 1231 save->cs.base = 0xffff0000; 1232 /* Executable/Readable Code Segment */ 1233 save->cs.attrib = SVM_SELECTOR_READ_MASK | SVM_SELECTOR_P_MASK | 1234 SVM_SELECTOR_S_MASK | SVM_SELECTOR_CODE_MASK; 1235 save->cs.limit = 0xffff; 1236 1237 save->gdtr.base = 0; 1238 save->gdtr.limit = 0xffff; 1239 save->idtr.base = 0; 1240 save->idtr.limit = 0xffff; 1241 1242 init_sys_seg(&save->ldtr, SEG_TYPE_LDT); 1243 init_sys_seg(&save->tr, SEG_TYPE_BUSY_TSS16); 1244 1245 if (npt_enabled) { 1246 /* Setup VMCB for Nested Paging */ 1247 control->nested_ctl |= SVM_NESTED_CTL_NP_ENABLE; 1248 svm_clr_intercept(svm, INTERCEPT_INVLPG); 1249 clr_exception_intercept(svm, PF_VECTOR); 1250 svm_clr_intercept(svm, INTERCEPT_CR3_READ); 1251 svm_clr_intercept(svm, INTERCEPT_CR3_WRITE); 1252 save->g_pat = vcpu->arch.pat; 1253 save->cr3 = 0; 1254 } 1255 svm->current_vmcb->asid_generation = 0; 1256 svm->asid = 0; 1257 1258 svm->nested.vmcb12_gpa = INVALID_GPA; 1259 svm->nested.last_vmcb12_gpa = INVALID_GPA; 1260 1261 if (!kvm_pause_in_guest(vcpu->kvm)) { 1262 control->pause_filter_count = pause_filter_count; 1263 if (pause_filter_thresh) 1264 control->pause_filter_thresh = pause_filter_thresh; 1265 svm_set_intercept(svm, INTERCEPT_PAUSE); 1266 } else { 1267 svm_clr_intercept(svm, INTERCEPT_PAUSE); 1268 } 1269 1270 svm_recalc_instruction_intercepts(vcpu, svm); 1271 1272 /* 1273 * If the host supports V_SPEC_CTRL then disable the interception 1274 * of MSR_IA32_SPEC_CTRL. 1275 */ 1276 if (boot_cpu_has(X86_FEATURE_V_SPEC_CTRL)) 1277 set_msr_interception(vcpu, svm->msrpm, MSR_IA32_SPEC_CTRL, 1, 1); 1278 1279 if (kvm_vcpu_apicv_active(vcpu)) 1280 avic_init_vmcb(svm); 1281 1282 if (vgif) { 1283 svm_clr_intercept(svm, INTERCEPT_STGI); 1284 svm_clr_intercept(svm, INTERCEPT_CLGI); 1285 svm->vmcb->control.int_ctl |= V_GIF_ENABLE_MASK; 1286 } 1287 1288 if (sev_guest(vcpu->kvm)) { 1289 svm->vmcb->control.nested_ctl |= SVM_NESTED_CTL_SEV_ENABLE; 1290 clr_exception_intercept(svm, UD_VECTOR); 1291 1292 if (sev_es_guest(vcpu->kvm)) { 1293 /* Perform SEV-ES specific VMCB updates */ 1294 sev_es_init_vmcb(svm); 1295 } 1296 } 1297 1298 svm_hv_init_vmcb(svm->vmcb); 1299 1300 vmcb_mark_all_dirty(svm->vmcb); 1301 1302 enable_gif(svm); 1303 1304 } 1305 1306 static void svm_vcpu_reset(struct kvm_vcpu *vcpu, bool init_event) 1307 { 1308 struct vcpu_svm *svm = to_svm(vcpu); 1309 1310 svm->spec_ctrl = 0; 1311 svm->virt_spec_ctrl = 0; 1312 1313 init_vmcb(vcpu); 1314 } 1315 1316 void svm_switch_vmcb(struct vcpu_svm *svm, struct kvm_vmcb_info *target_vmcb) 1317 { 1318 svm->current_vmcb = target_vmcb; 1319 svm->vmcb = target_vmcb->ptr; 1320 } 1321 1322 static int svm_create_vcpu(struct kvm_vcpu *vcpu) 1323 { 1324 struct vcpu_svm *svm; 1325 struct page *vmcb01_page; 1326 struct page *vmsa_page = NULL; 1327 int err; 1328 1329 BUILD_BUG_ON(offsetof(struct vcpu_svm, vcpu) != 0); 1330 svm = to_svm(vcpu); 1331 1332 err = -ENOMEM; 1333 vmcb01_page = alloc_page(GFP_KERNEL_ACCOUNT | __GFP_ZERO); 1334 if (!vmcb01_page) 1335 goto out; 1336 1337 if (sev_es_guest(vcpu->kvm)) { 1338 /* 1339 * SEV-ES guests require a separate VMSA page used to contain 1340 * the encrypted register state of the guest. 1341 */ 1342 vmsa_page = alloc_page(GFP_KERNEL_ACCOUNT | __GFP_ZERO); 1343 if (!vmsa_page) 1344 goto error_free_vmcb_page; 1345 1346 /* 1347 * SEV-ES guests maintain an encrypted version of their FPU 1348 * state which is restored and saved on VMRUN and VMEXIT. 1349 * Free the fpu structure to prevent KVM from attempting to 1350 * access the FPU state. 1351 */ 1352 kvm_free_guest_fpu(vcpu); 1353 } 1354 1355 err = avic_init_vcpu(svm); 1356 if (err) 1357 goto error_free_vmsa_page; 1358 1359 /* We initialize this flag to true to make sure that the is_running 1360 * bit would be set the first time the vcpu is loaded. 1361 */ 1362 if (irqchip_in_kernel(vcpu->kvm) && kvm_apicv_activated(vcpu->kvm)) 1363 svm->avic_is_running = true; 1364 1365 svm->msrpm = svm_vcpu_alloc_msrpm(); 1366 if (!svm->msrpm) { 1367 err = -ENOMEM; 1368 goto error_free_vmsa_page; 1369 } 1370 1371 svm->vmcb01.ptr = page_address(vmcb01_page); 1372 svm->vmcb01.pa = __sme_set(page_to_pfn(vmcb01_page) << PAGE_SHIFT); 1373 1374 if (vmsa_page) 1375 svm->vmsa = page_address(vmsa_page); 1376 1377 svm->guest_state_loaded = false; 1378 1379 svm_switch_vmcb(svm, &svm->vmcb01); 1380 init_vmcb(vcpu); 1381 1382 svm_vcpu_init_msrpm(vcpu, svm->msrpm); 1383 1384 svm_init_osvw(vcpu); 1385 vcpu->arch.microcode_version = 0x01000065; 1386 1387 if (sev_es_guest(vcpu->kvm)) 1388 /* Perform SEV-ES specific VMCB creation updates */ 1389 sev_es_create_vcpu(svm); 1390 1391 return 0; 1392 1393 error_free_vmsa_page: 1394 if (vmsa_page) 1395 __free_page(vmsa_page); 1396 error_free_vmcb_page: 1397 __free_page(vmcb01_page); 1398 out: 1399 return err; 1400 } 1401 1402 static void svm_clear_current_vmcb(struct vmcb *vmcb) 1403 { 1404 int i; 1405 1406 for_each_online_cpu(i) 1407 cmpxchg(&per_cpu(svm_data, i)->current_vmcb, vmcb, NULL); 1408 } 1409 1410 static void svm_free_vcpu(struct kvm_vcpu *vcpu) 1411 { 1412 struct vcpu_svm *svm = to_svm(vcpu); 1413 1414 /* 1415 * The vmcb page can be recycled, causing a false negative in 1416 * svm_vcpu_load(). So, ensure that no logical CPU has this 1417 * vmcb page recorded as its current vmcb. 1418 */ 1419 svm_clear_current_vmcb(svm->vmcb); 1420 1421 svm_free_nested(svm); 1422 1423 sev_free_vcpu(vcpu); 1424 1425 __free_page(pfn_to_page(__sme_clr(svm->vmcb01.pa) >> PAGE_SHIFT)); 1426 __free_pages(virt_to_page(svm->msrpm), get_order(MSRPM_SIZE)); 1427 } 1428 1429 static void svm_prepare_guest_switch(struct kvm_vcpu *vcpu) 1430 { 1431 struct vcpu_svm *svm = to_svm(vcpu); 1432 struct svm_cpu_data *sd = per_cpu(svm_data, vcpu->cpu); 1433 1434 if (sev_es_guest(vcpu->kvm)) 1435 sev_es_unmap_ghcb(svm); 1436 1437 if (svm->guest_state_loaded) 1438 return; 1439 1440 /* 1441 * Save additional host state that will be restored on VMEXIT (sev-es) 1442 * or subsequent vmload of host save area. 1443 */ 1444 if (sev_es_guest(vcpu->kvm)) { 1445 sev_es_prepare_guest_switch(svm, vcpu->cpu); 1446 } else { 1447 vmsave(__sme_page_pa(sd->save_area)); 1448 } 1449 1450 if (static_cpu_has(X86_FEATURE_TSCRATEMSR)) { 1451 u64 tsc_ratio = vcpu->arch.tsc_scaling_ratio; 1452 if (tsc_ratio != __this_cpu_read(current_tsc_ratio)) { 1453 __this_cpu_write(current_tsc_ratio, tsc_ratio); 1454 wrmsrl(MSR_AMD64_TSC_RATIO, tsc_ratio); 1455 } 1456 } 1457 1458 if (likely(tsc_aux_uret_slot >= 0)) 1459 kvm_set_user_return_msr(tsc_aux_uret_slot, svm->tsc_aux, -1ull); 1460 1461 svm->guest_state_loaded = true; 1462 } 1463 1464 static void svm_prepare_host_switch(struct kvm_vcpu *vcpu) 1465 { 1466 to_svm(vcpu)->guest_state_loaded = false; 1467 } 1468 1469 static void svm_vcpu_load(struct kvm_vcpu *vcpu, int cpu) 1470 { 1471 struct vcpu_svm *svm = to_svm(vcpu); 1472 struct svm_cpu_data *sd = per_cpu(svm_data, cpu); 1473 1474 if (sd->current_vmcb != svm->vmcb) { 1475 sd->current_vmcb = svm->vmcb; 1476 indirect_branch_prediction_barrier(); 1477 } 1478 if (kvm_vcpu_apicv_active(vcpu)) 1479 avic_vcpu_load(vcpu, cpu); 1480 } 1481 1482 static void svm_vcpu_put(struct kvm_vcpu *vcpu) 1483 { 1484 if (kvm_vcpu_apicv_active(vcpu)) 1485 avic_vcpu_put(vcpu); 1486 1487 svm_prepare_host_switch(vcpu); 1488 1489 ++vcpu->stat.host_state_reload; 1490 } 1491 1492 static unsigned long svm_get_rflags(struct kvm_vcpu *vcpu) 1493 { 1494 struct vcpu_svm *svm = to_svm(vcpu); 1495 unsigned long rflags = svm->vmcb->save.rflags; 1496 1497 if (svm->nmi_singlestep) { 1498 /* Hide our flags if they were not set by the guest */ 1499 if (!(svm->nmi_singlestep_guest_rflags & X86_EFLAGS_TF)) 1500 rflags &= ~X86_EFLAGS_TF; 1501 if (!(svm->nmi_singlestep_guest_rflags & X86_EFLAGS_RF)) 1502 rflags &= ~X86_EFLAGS_RF; 1503 } 1504 return rflags; 1505 } 1506 1507 static void svm_set_rflags(struct kvm_vcpu *vcpu, unsigned long rflags) 1508 { 1509 if (to_svm(vcpu)->nmi_singlestep) 1510 rflags |= (X86_EFLAGS_TF | X86_EFLAGS_RF); 1511 1512 /* 1513 * Any change of EFLAGS.VM is accompanied by a reload of SS 1514 * (caused by either a task switch or an inter-privilege IRET), 1515 * so we do not need to update the CPL here. 1516 */ 1517 to_svm(vcpu)->vmcb->save.rflags = rflags; 1518 } 1519 1520 static void svm_cache_reg(struct kvm_vcpu *vcpu, enum kvm_reg reg) 1521 { 1522 switch (reg) { 1523 case VCPU_EXREG_PDPTR: 1524 BUG_ON(!npt_enabled); 1525 load_pdptrs(vcpu, vcpu->arch.walk_mmu, kvm_read_cr3(vcpu)); 1526 break; 1527 default: 1528 KVM_BUG_ON(1, vcpu->kvm); 1529 } 1530 } 1531 1532 static void svm_set_vintr(struct vcpu_svm *svm) 1533 { 1534 struct vmcb_control_area *control; 1535 1536 /* 1537 * The following fields are ignored when AVIC is enabled 1538 */ 1539 WARN_ON(kvm_apicv_activated(svm->vcpu.kvm)); 1540 1541 svm_set_intercept(svm, INTERCEPT_VINTR); 1542 1543 /* 1544 * This is just a dummy VINTR to actually cause a vmexit to happen. 1545 * Actual injection of virtual interrupts happens through EVENTINJ. 1546 */ 1547 control = &svm->vmcb->control; 1548 control->int_vector = 0x0; 1549 control->int_ctl &= ~V_INTR_PRIO_MASK; 1550 control->int_ctl |= V_IRQ_MASK | 1551 ((/*control->int_vector >> 4*/ 0xf) << V_INTR_PRIO_SHIFT); 1552 vmcb_mark_dirty(svm->vmcb, VMCB_INTR); 1553 } 1554 1555 static void svm_clear_vintr(struct vcpu_svm *svm) 1556 { 1557 svm_clr_intercept(svm, INTERCEPT_VINTR); 1558 1559 /* Drop int_ctl fields related to VINTR injection. */ 1560 svm->vmcb->control.int_ctl &= ~V_IRQ_INJECTION_BITS_MASK; 1561 if (is_guest_mode(&svm->vcpu)) { 1562 svm->vmcb01.ptr->control.int_ctl &= ~V_IRQ_INJECTION_BITS_MASK; 1563 1564 WARN_ON((svm->vmcb->control.int_ctl & V_TPR_MASK) != 1565 (svm->nested.ctl.int_ctl & V_TPR_MASK)); 1566 1567 svm->vmcb->control.int_ctl |= svm->nested.ctl.int_ctl & 1568 V_IRQ_INJECTION_BITS_MASK; 1569 } 1570 1571 vmcb_mark_dirty(svm->vmcb, VMCB_INTR); 1572 } 1573 1574 static struct vmcb_seg *svm_seg(struct kvm_vcpu *vcpu, int seg) 1575 { 1576 struct vmcb_save_area *save = &to_svm(vcpu)->vmcb->save; 1577 struct vmcb_save_area *save01 = &to_svm(vcpu)->vmcb01.ptr->save; 1578 1579 switch (seg) { 1580 case VCPU_SREG_CS: return &save->cs; 1581 case VCPU_SREG_DS: return &save->ds; 1582 case VCPU_SREG_ES: return &save->es; 1583 case VCPU_SREG_FS: return &save01->fs; 1584 case VCPU_SREG_GS: return &save01->gs; 1585 case VCPU_SREG_SS: return &save->ss; 1586 case VCPU_SREG_TR: return &save01->tr; 1587 case VCPU_SREG_LDTR: return &save01->ldtr; 1588 } 1589 BUG(); 1590 return NULL; 1591 } 1592 1593 static u64 svm_get_segment_base(struct kvm_vcpu *vcpu, int seg) 1594 { 1595 struct vmcb_seg *s = svm_seg(vcpu, seg); 1596 1597 return s->base; 1598 } 1599 1600 static void svm_get_segment(struct kvm_vcpu *vcpu, 1601 struct kvm_segment *var, int seg) 1602 { 1603 struct vmcb_seg *s = svm_seg(vcpu, seg); 1604 1605 var->base = s->base; 1606 var->limit = s->limit; 1607 var->selector = s->selector; 1608 var->type = s->attrib & SVM_SELECTOR_TYPE_MASK; 1609 var->s = (s->attrib >> SVM_SELECTOR_S_SHIFT) & 1; 1610 var->dpl = (s->attrib >> SVM_SELECTOR_DPL_SHIFT) & 3; 1611 var->present = (s->attrib >> SVM_SELECTOR_P_SHIFT) & 1; 1612 var->avl = (s->attrib >> SVM_SELECTOR_AVL_SHIFT) & 1; 1613 var->l = (s->attrib >> SVM_SELECTOR_L_SHIFT) & 1; 1614 var->db = (s->attrib >> SVM_SELECTOR_DB_SHIFT) & 1; 1615 1616 /* 1617 * AMD CPUs circa 2014 track the G bit for all segments except CS. 1618 * However, the SVM spec states that the G bit is not observed by the 1619 * CPU, and some VMware virtual CPUs drop the G bit for all segments. 1620 * So let's synthesize a legal G bit for all segments, this helps 1621 * running KVM nested. It also helps cross-vendor migration, because 1622 * Intel's vmentry has a check on the 'G' bit. 1623 */ 1624 var->g = s->limit > 0xfffff; 1625 1626 /* 1627 * AMD's VMCB does not have an explicit unusable field, so emulate it 1628 * for cross vendor migration purposes by "not present" 1629 */ 1630 var->unusable = !var->present; 1631 1632 switch (seg) { 1633 case VCPU_SREG_TR: 1634 /* 1635 * Work around a bug where the busy flag in the tr selector 1636 * isn't exposed 1637 */ 1638 var->type |= 0x2; 1639 break; 1640 case VCPU_SREG_DS: 1641 case VCPU_SREG_ES: 1642 case VCPU_SREG_FS: 1643 case VCPU_SREG_GS: 1644 /* 1645 * The accessed bit must always be set in the segment 1646 * descriptor cache, although it can be cleared in the 1647 * descriptor, the cached bit always remains at 1. Since 1648 * Intel has a check on this, set it here to support 1649 * cross-vendor migration. 1650 */ 1651 if (!var->unusable) 1652 var->type |= 0x1; 1653 break; 1654 case VCPU_SREG_SS: 1655 /* 1656 * On AMD CPUs sometimes the DB bit in the segment 1657 * descriptor is left as 1, although the whole segment has 1658 * been made unusable. Clear it here to pass an Intel VMX 1659 * entry check when cross vendor migrating. 1660 */ 1661 if (var->unusable) 1662 var->db = 0; 1663 /* This is symmetric with svm_set_segment() */ 1664 var->dpl = to_svm(vcpu)->vmcb->save.cpl; 1665 break; 1666 } 1667 } 1668 1669 static int svm_get_cpl(struct kvm_vcpu *vcpu) 1670 { 1671 struct vmcb_save_area *save = &to_svm(vcpu)->vmcb->save; 1672 1673 return save->cpl; 1674 } 1675 1676 static void svm_get_idt(struct kvm_vcpu *vcpu, struct desc_ptr *dt) 1677 { 1678 struct vcpu_svm *svm = to_svm(vcpu); 1679 1680 dt->size = svm->vmcb->save.idtr.limit; 1681 dt->address = svm->vmcb->save.idtr.base; 1682 } 1683 1684 static void svm_set_idt(struct kvm_vcpu *vcpu, struct desc_ptr *dt) 1685 { 1686 struct vcpu_svm *svm = to_svm(vcpu); 1687 1688 svm->vmcb->save.idtr.limit = dt->size; 1689 svm->vmcb->save.idtr.base = dt->address ; 1690 vmcb_mark_dirty(svm->vmcb, VMCB_DT); 1691 } 1692 1693 static void svm_get_gdt(struct kvm_vcpu *vcpu, struct desc_ptr *dt) 1694 { 1695 struct vcpu_svm *svm = to_svm(vcpu); 1696 1697 dt->size = svm->vmcb->save.gdtr.limit; 1698 dt->address = svm->vmcb->save.gdtr.base; 1699 } 1700 1701 static void svm_set_gdt(struct kvm_vcpu *vcpu, struct desc_ptr *dt) 1702 { 1703 struct vcpu_svm *svm = to_svm(vcpu); 1704 1705 svm->vmcb->save.gdtr.limit = dt->size; 1706 svm->vmcb->save.gdtr.base = dt->address ; 1707 vmcb_mark_dirty(svm->vmcb, VMCB_DT); 1708 } 1709 1710 void svm_set_cr0(struct kvm_vcpu *vcpu, unsigned long cr0) 1711 { 1712 struct vcpu_svm *svm = to_svm(vcpu); 1713 u64 hcr0 = cr0; 1714 1715 #ifdef CONFIG_X86_64 1716 if (vcpu->arch.efer & EFER_LME && !vcpu->arch.guest_state_protected) { 1717 if (!is_paging(vcpu) && (cr0 & X86_CR0_PG)) { 1718 vcpu->arch.efer |= EFER_LMA; 1719 svm->vmcb->save.efer |= EFER_LMA | EFER_LME; 1720 } 1721 1722 if (is_paging(vcpu) && !(cr0 & X86_CR0_PG)) { 1723 vcpu->arch.efer &= ~EFER_LMA; 1724 svm->vmcb->save.efer &= ~(EFER_LMA | EFER_LME); 1725 } 1726 } 1727 #endif 1728 vcpu->arch.cr0 = cr0; 1729 1730 if (!npt_enabled) 1731 hcr0 |= X86_CR0_PG | X86_CR0_WP; 1732 1733 /* 1734 * re-enable caching here because the QEMU bios 1735 * does not do it - this results in some delay at 1736 * reboot 1737 */ 1738 if (kvm_check_has_quirk(vcpu->kvm, KVM_X86_QUIRK_CD_NW_CLEARED)) 1739 hcr0 &= ~(X86_CR0_CD | X86_CR0_NW); 1740 1741 svm->vmcb->save.cr0 = hcr0; 1742 vmcb_mark_dirty(svm->vmcb, VMCB_CR); 1743 1744 /* 1745 * SEV-ES guests must always keep the CR intercepts cleared. CR 1746 * tracking is done using the CR write traps. 1747 */ 1748 if (sev_es_guest(vcpu->kvm)) 1749 return; 1750 1751 if (hcr0 == cr0) { 1752 /* Selective CR0 write remains on. */ 1753 svm_clr_intercept(svm, INTERCEPT_CR0_READ); 1754 svm_clr_intercept(svm, INTERCEPT_CR0_WRITE); 1755 } else { 1756 svm_set_intercept(svm, INTERCEPT_CR0_READ); 1757 svm_set_intercept(svm, INTERCEPT_CR0_WRITE); 1758 } 1759 } 1760 1761 static bool svm_is_valid_cr4(struct kvm_vcpu *vcpu, unsigned long cr4) 1762 { 1763 return true; 1764 } 1765 1766 void svm_set_cr4(struct kvm_vcpu *vcpu, unsigned long cr4) 1767 { 1768 unsigned long host_cr4_mce = cr4_read_shadow() & X86_CR4_MCE; 1769 unsigned long old_cr4 = vcpu->arch.cr4; 1770 1771 if (npt_enabled && ((old_cr4 ^ cr4) & X86_CR4_PGE)) 1772 svm_flush_tlb(vcpu); 1773 1774 vcpu->arch.cr4 = cr4; 1775 if (!npt_enabled) 1776 cr4 |= X86_CR4_PAE; 1777 cr4 |= host_cr4_mce; 1778 to_svm(vcpu)->vmcb->save.cr4 = cr4; 1779 vmcb_mark_dirty(to_svm(vcpu)->vmcb, VMCB_CR); 1780 1781 if ((cr4 ^ old_cr4) & (X86_CR4_OSXSAVE | X86_CR4_PKE)) 1782 kvm_update_cpuid_runtime(vcpu); 1783 } 1784 1785 static void svm_set_segment(struct kvm_vcpu *vcpu, 1786 struct kvm_segment *var, int seg) 1787 { 1788 struct vcpu_svm *svm = to_svm(vcpu); 1789 struct vmcb_seg *s = svm_seg(vcpu, seg); 1790 1791 s->base = var->base; 1792 s->limit = var->limit; 1793 s->selector = var->selector; 1794 s->attrib = (var->type & SVM_SELECTOR_TYPE_MASK); 1795 s->attrib |= (var->s & 1) << SVM_SELECTOR_S_SHIFT; 1796 s->attrib |= (var->dpl & 3) << SVM_SELECTOR_DPL_SHIFT; 1797 s->attrib |= ((var->present & 1) && !var->unusable) << SVM_SELECTOR_P_SHIFT; 1798 s->attrib |= (var->avl & 1) << SVM_SELECTOR_AVL_SHIFT; 1799 s->attrib |= (var->l & 1) << SVM_SELECTOR_L_SHIFT; 1800 s->attrib |= (var->db & 1) << SVM_SELECTOR_DB_SHIFT; 1801 s->attrib |= (var->g & 1) << SVM_SELECTOR_G_SHIFT; 1802 1803 /* 1804 * This is always accurate, except if SYSRET returned to a segment 1805 * with SS.DPL != 3. Intel does not have this quirk, and always 1806 * forces SS.DPL to 3 on sysret, so we ignore that case; fixing it 1807 * would entail passing the CPL to userspace and back. 1808 */ 1809 if (seg == VCPU_SREG_SS) 1810 /* This is symmetric with svm_get_segment() */ 1811 svm->vmcb->save.cpl = (var->dpl & 3); 1812 1813 vmcb_mark_dirty(svm->vmcb, VMCB_SEG); 1814 } 1815 1816 static void svm_update_exception_bitmap(struct kvm_vcpu *vcpu) 1817 { 1818 struct vcpu_svm *svm = to_svm(vcpu); 1819 1820 clr_exception_intercept(svm, BP_VECTOR); 1821 1822 if (vcpu->guest_debug & KVM_GUESTDBG_ENABLE) { 1823 if (vcpu->guest_debug & KVM_GUESTDBG_USE_SW_BP) 1824 set_exception_intercept(svm, BP_VECTOR); 1825 } 1826 } 1827 1828 static void new_asid(struct vcpu_svm *svm, struct svm_cpu_data *sd) 1829 { 1830 if (sd->next_asid > sd->max_asid) { 1831 ++sd->asid_generation; 1832 sd->next_asid = sd->min_asid; 1833 svm->vmcb->control.tlb_ctl = TLB_CONTROL_FLUSH_ALL_ASID; 1834 vmcb_mark_dirty(svm->vmcb, VMCB_ASID); 1835 } 1836 1837 svm->current_vmcb->asid_generation = sd->asid_generation; 1838 svm->asid = sd->next_asid++; 1839 } 1840 1841 static void svm_set_dr6(struct vcpu_svm *svm, unsigned long value) 1842 { 1843 struct vmcb *vmcb = svm->vmcb; 1844 1845 if (svm->vcpu.arch.guest_state_protected) 1846 return; 1847 1848 if (unlikely(value != vmcb->save.dr6)) { 1849 vmcb->save.dr6 = value; 1850 vmcb_mark_dirty(vmcb, VMCB_DR); 1851 } 1852 } 1853 1854 static void svm_sync_dirty_debug_regs(struct kvm_vcpu *vcpu) 1855 { 1856 struct vcpu_svm *svm = to_svm(vcpu); 1857 1858 if (vcpu->arch.guest_state_protected) 1859 return; 1860 1861 get_debugreg(vcpu->arch.db[0], 0); 1862 get_debugreg(vcpu->arch.db[1], 1); 1863 get_debugreg(vcpu->arch.db[2], 2); 1864 get_debugreg(vcpu->arch.db[3], 3); 1865 /* 1866 * We cannot reset svm->vmcb->save.dr6 to DR6_ACTIVE_LOW here, 1867 * because db_interception might need it. We can do it before vmentry. 1868 */ 1869 vcpu->arch.dr6 = svm->vmcb->save.dr6; 1870 vcpu->arch.dr7 = svm->vmcb->save.dr7; 1871 vcpu->arch.switch_db_regs &= ~KVM_DEBUGREG_WONT_EXIT; 1872 set_dr_intercepts(svm); 1873 } 1874 1875 static void svm_set_dr7(struct kvm_vcpu *vcpu, unsigned long value) 1876 { 1877 struct vcpu_svm *svm = to_svm(vcpu); 1878 1879 if (vcpu->arch.guest_state_protected) 1880 return; 1881 1882 svm->vmcb->save.dr7 = value; 1883 vmcb_mark_dirty(svm->vmcb, VMCB_DR); 1884 } 1885 1886 static int pf_interception(struct kvm_vcpu *vcpu) 1887 { 1888 struct vcpu_svm *svm = to_svm(vcpu); 1889 1890 u64 fault_address = svm->vmcb->control.exit_info_2; 1891 u64 error_code = svm->vmcb->control.exit_info_1; 1892 1893 return kvm_handle_page_fault(vcpu, error_code, fault_address, 1894 static_cpu_has(X86_FEATURE_DECODEASSISTS) ? 1895 svm->vmcb->control.insn_bytes : NULL, 1896 svm->vmcb->control.insn_len); 1897 } 1898 1899 static int npf_interception(struct kvm_vcpu *vcpu) 1900 { 1901 struct vcpu_svm *svm = to_svm(vcpu); 1902 1903 u64 fault_address = svm->vmcb->control.exit_info_2; 1904 u64 error_code = svm->vmcb->control.exit_info_1; 1905 1906 trace_kvm_page_fault(fault_address, error_code); 1907 return kvm_mmu_page_fault(vcpu, fault_address, error_code, 1908 static_cpu_has(X86_FEATURE_DECODEASSISTS) ? 1909 svm->vmcb->control.insn_bytes : NULL, 1910 svm->vmcb->control.insn_len); 1911 } 1912 1913 static int db_interception(struct kvm_vcpu *vcpu) 1914 { 1915 struct kvm_run *kvm_run = vcpu->run; 1916 struct vcpu_svm *svm = to_svm(vcpu); 1917 1918 if (!(vcpu->guest_debug & 1919 (KVM_GUESTDBG_SINGLESTEP | KVM_GUESTDBG_USE_HW_BP)) && 1920 !svm->nmi_singlestep) { 1921 u32 payload = svm->vmcb->save.dr6 ^ DR6_ACTIVE_LOW; 1922 kvm_queue_exception_p(vcpu, DB_VECTOR, payload); 1923 return 1; 1924 } 1925 1926 if (svm->nmi_singlestep) { 1927 disable_nmi_singlestep(svm); 1928 /* Make sure we check for pending NMIs upon entry */ 1929 kvm_make_request(KVM_REQ_EVENT, vcpu); 1930 } 1931 1932 if (vcpu->guest_debug & 1933 (KVM_GUESTDBG_SINGLESTEP | KVM_GUESTDBG_USE_HW_BP)) { 1934 kvm_run->exit_reason = KVM_EXIT_DEBUG; 1935 kvm_run->debug.arch.dr6 = svm->vmcb->save.dr6; 1936 kvm_run->debug.arch.dr7 = svm->vmcb->save.dr7; 1937 kvm_run->debug.arch.pc = 1938 svm->vmcb->save.cs.base + svm->vmcb->save.rip; 1939 kvm_run->debug.arch.exception = DB_VECTOR; 1940 return 0; 1941 } 1942 1943 return 1; 1944 } 1945 1946 static int bp_interception(struct kvm_vcpu *vcpu) 1947 { 1948 struct vcpu_svm *svm = to_svm(vcpu); 1949 struct kvm_run *kvm_run = vcpu->run; 1950 1951 kvm_run->exit_reason = KVM_EXIT_DEBUG; 1952 kvm_run->debug.arch.pc = svm->vmcb->save.cs.base + svm->vmcb->save.rip; 1953 kvm_run->debug.arch.exception = BP_VECTOR; 1954 return 0; 1955 } 1956 1957 static int ud_interception(struct kvm_vcpu *vcpu) 1958 { 1959 return handle_ud(vcpu); 1960 } 1961 1962 static int ac_interception(struct kvm_vcpu *vcpu) 1963 { 1964 kvm_queue_exception_e(vcpu, AC_VECTOR, 0); 1965 return 1; 1966 } 1967 1968 static bool is_erratum_383(void) 1969 { 1970 int err, i; 1971 u64 value; 1972 1973 if (!erratum_383_found) 1974 return false; 1975 1976 value = native_read_msr_safe(MSR_IA32_MC0_STATUS, &err); 1977 if (err) 1978 return false; 1979 1980 /* Bit 62 may or may not be set for this mce */ 1981 value &= ~(1ULL << 62); 1982 1983 if (value != 0xb600000000010015ULL) 1984 return false; 1985 1986 /* Clear MCi_STATUS registers */ 1987 for (i = 0; i < 6; ++i) 1988 native_write_msr_safe(MSR_IA32_MCx_STATUS(i), 0, 0); 1989 1990 value = native_read_msr_safe(MSR_IA32_MCG_STATUS, &err); 1991 if (!err) { 1992 u32 low, high; 1993 1994 value &= ~(1ULL << 2); 1995 low = lower_32_bits(value); 1996 high = upper_32_bits(value); 1997 1998 native_write_msr_safe(MSR_IA32_MCG_STATUS, low, high); 1999 } 2000 2001 /* Flush tlb to evict multi-match entries */ 2002 __flush_tlb_all(); 2003 2004 return true; 2005 } 2006 2007 static void svm_handle_mce(struct kvm_vcpu *vcpu) 2008 { 2009 if (is_erratum_383()) { 2010 /* 2011 * Erratum 383 triggered. Guest state is corrupt so kill the 2012 * guest. 2013 */ 2014 pr_err("KVM: Guest triggered AMD Erratum 383\n"); 2015 2016 kvm_make_request(KVM_REQ_TRIPLE_FAULT, vcpu); 2017 2018 return; 2019 } 2020 2021 /* 2022 * On an #MC intercept the MCE handler is not called automatically in 2023 * the host. So do it by hand here. 2024 */ 2025 kvm_machine_check(); 2026 } 2027 2028 static int mc_interception(struct kvm_vcpu *vcpu) 2029 { 2030 return 1; 2031 } 2032 2033 static int shutdown_interception(struct kvm_vcpu *vcpu) 2034 { 2035 struct kvm_run *kvm_run = vcpu->run; 2036 struct vcpu_svm *svm = to_svm(vcpu); 2037 2038 /* 2039 * The VM save area has already been encrypted so it 2040 * cannot be reinitialized - just terminate. 2041 */ 2042 if (sev_es_guest(vcpu->kvm)) 2043 return -EINVAL; 2044 2045 /* 2046 * VMCB is undefined after a SHUTDOWN intercept. INIT the vCPU to put 2047 * the VMCB in a known good state. Unfortuately, KVM doesn't have 2048 * KVM_MP_STATE_SHUTDOWN and can't add it without potentially breaking 2049 * userspace. At a platform view, INIT is acceptable behavior as 2050 * there exist bare metal platforms that automatically INIT the CPU 2051 * in response to shutdown. 2052 */ 2053 clear_page(svm->vmcb); 2054 kvm_vcpu_reset(vcpu, true); 2055 2056 kvm_run->exit_reason = KVM_EXIT_SHUTDOWN; 2057 return 0; 2058 } 2059 2060 static int io_interception(struct kvm_vcpu *vcpu) 2061 { 2062 struct vcpu_svm *svm = to_svm(vcpu); 2063 u32 io_info = svm->vmcb->control.exit_info_1; /* address size bug? */ 2064 int size, in, string; 2065 unsigned port; 2066 2067 ++vcpu->stat.io_exits; 2068 string = (io_info & SVM_IOIO_STR_MASK) != 0; 2069 in = (io_info & SVM_IOIO_TYPE_MASK) != 0; 2070 port = io_info >> 16; 2071 size = (io_info & SVM_IOIO_SIZE_MASK) >> SVM_IOIO_SIZE_SHIFT; 2072 2073 if (string) { 2074 if (sev_es_guest(vcpu->kvm)) 2075 return sev_es_string_io(svm, size, port, in); 2076 else 2077 return kvm_emulate_instruction(vcpu, 0); 2078 } 2079 2080 svm->next_rip = svm->vmcb->control.exit_info_2; 2081 2082 return kvm_fast_pio(vcpu, size, port, in); 2083 } 2084 2085 static int nmi_interception(struct kvm_vcpu *vcpu) 2086 { 2087 return 1; 2088 } 2089 2090 static int smi_interception(struct kvm_vcpu *vcpu) 2091 { 2092 return 1; 2093 } 2094 2095 static int intr_interception(struct kvm_vcpu *vcpu) 2096 { 2097 ++vcpu->stat.irq_exits; 2098 return 1; 2099 } 2100 2101 static int vmload_vmsave_interception(struct kvm_vcpu *vcpu, bool vmload) 2102 { 2103 struct vcpu_svm *svm = to_svm(vcpu); 2104 struct vmcb *vmcb12; 2105 struct kvm_host_map map; 2106 int ret; 2107 2108 if (nested_svm_check_permissions(vcpu)) 2109 return 1; 2110 2111 ret = kvm_vcpu_map(vcpu, gpa_to_gfn(svm->vmcb->save.rax), &map); 2112 if (ret) { 2113 if (ret == -EINVAL) 2114 kvm_inject_gp(vcpu, 0); 2115 return 1; 2116 } 2117 2118 vmcb12 = map.hva; 2119 2120 ret = kvm_skip_emulated_instruction(vcpu); 2121 2122 if (vmload) { 2123 svm_copy_vmloadsave_state(svm->vmcb, vmcb12); 2124 svm->sysenter_eip_hi = 0; 2125 svm->sysenter_esp_hi = 0; 2126 } else { 2127 svm_copy_vmloadsave_state(vmcb12, svm->vmcb); 2128 } 2129 2130 kvm_vcpu_unmap(vcpu, &map, true); 2131 2132 return ret; 2133 } 2134 2135 static int vmload_interception(struct kvm_vcpu *vcpu) 2136 { 2137 return vmload_vmsave_interception(vcpu, true); 2138 } 2139 2140 static int vmsave_interception(struct kvm_vcpu *vcpu) 2141 { 2142 return vmload_vmsave_interception(vcpu, false); 2143 } 2144 2145 static int vmrun_interception(struct kvm_vcpu *vcpu) 2146 { 2147 if (nested_svm_check_permissions(vcpu)) 2148 return 1; 2149 2150 return nested_svm_vmrun(vcpu); 2151 } 2152 2153 enum { 2154 NONE_SVM_INSTR, 2155 SVM_INSTR_VMRUN, 2156 SVM_INSTR_VMLOAD, 2157 SVM_INSTR_VMSAVE, 2158 }; 2159 2160 /* Return NONE_SVM_INSTR if not SVM instrs, otherwise return decode result */ 2161 static int svm_instr_opcode(struct kvm_vcpu *vcpu) 2162 { 2163 struct x86_emulate_ctxt *ctxt = vcpu->arch.emulate_ctxt; 2164 2165 if (ctxt->b != 0x1 || ctxt->opcode_len != 2) 2166 return NONE_SVM_INSTR; 2167 2168 switch (ctxt->modrm) { 2169 case 0xd8: /* VMRUN */ 2170 return SVM_INSTR_VMRUN; 2171 case 0xda: /* VMLOAD */ 2172 return SVM_INSTR_VMLOAD; 2173 case 0xdb: /* VMSAVE */ 2174 return SVM_INSTR_VMSAVE; 2175 default: 2176 break; 2177 } 2178 2179 return NONE_SVM_INSTR; 2180 } 2181 2182 static int emulate_svm_instr(struct kvm_vcpu *vcpu, int opcode) 2183 { 2184 const int guest_mode_exit_codes[] = { 2185 [SVM_INSTR_VMRUN] = SVM_EXIT_VMRUN, 2186 [SVM_INSTR_VMLOAD] = SVM_EXIT_VMLOAD, 2187 [SVM_INSTR_VMSAVE] = SVM_EXIT_VMSAVE, 2188 }; 2189 int (*const svm_instr_handlers[])(struct kvm_vcpu *vcpu) = { 2190 [SVM_INSTR_VMRUN] = vmrun_interception, 2191 [SVM_INSTR_VMLOAD] = vmload_interception, 2192 [SVM_INSTR_VMSAVE] = vmsave_interception, 2193 }; 2194 struct vcpu_svm *svm = to_svm(vcpu); 2195 int ret; 2196 2197 if (is_guest_mode(vcpu)) { 2198 /* Returns '1' or -errno on failure, '0' on success. */ 2199 ret = nested_svm_simple_vmexit(svm, guest_mode_exit_codes[opcode]); 2200 if (ret) 2201 return ret; 2202 return 1; 2203 } 2204 return svm_instr_handlers[opcode](vcpu); 2205 } 2206 2207 /* 2208 * #GP handling code. Note that #GP can be triggered under the following two 2209 * cases: 2210 * 1) SVM VM-related instructions (VMRUN/VMSAVE/VMLOAD) that trigger #GP on 2211 * some AMD CPUs when EAX of these instructions are in the reserved memory 2212 * regions (e.g. SMM memory on host). 2213 * 2) VMware backdoor 2214 */ 2215 static int gp_interception(struct kvm_vcpu *vcpu) 2216 { 2217 struct vcpu_svm *svm = to_svm(vcpu); 2218 u32 error_code = svm->vmcb->control.exit_info_1; 2219 int opcode; 2220 2221 /* Both #GP cases have zero error_code */ 2222 if (error_code) 2223 goto reinject; 2224 2225 /* Decode the instruction for usage later */ 2226 if (x86_decode_emulated_instruction(vcpu, 0, NULL, 0) != EMULATION_OK) 2227 goto reinject; 2228 2229 opcode = svm_instr_opcode(vcpu); 2230 2231 if (opcode == NONE_SVM_INSTR) { 2232 if (!enable_vmware_backdoor) 2233 goto reinject; 2234 2235 /* 2236 * VMware backdoor emulation on #GP interception only handles 2237 * IN{S}, OUT{S}, and RDPMC. 2238 */ 2239 if (!is_guest_mode(vcpu)) 2240 return kvm_emulate_instruction(vcpu, 2241 EMULTYPE_VMWARE_GP | EMULTYPE_NO_DECODE); 2242 } else 2243 return emulate_svm_instr(vcpu, opcode); 2244 2245 reinject: 2246 kvm_queue_exception_e(vcpu, GP_VECTOR, error_code); 2247 return 1; 2248 } 2249 2250 void svm_set_gif(struct vcpu_svm *svm, bool value) 2251 { 2252 if (value) { 2253 /* 2254 * If VGIF is enabled, the STGI intercept is only added to 2255 * detect the opening of the SMI/NMI window; remove it now. 2256 * Likewise, clear the VINTR intercept, we will set it 2257 * again while processing KVM_REQ_EVENT if needed. 2258 */ 2259 if (vgif_enabled(svm)) 2260 svm_clr_intercept(svm, INTERCEPT_STGI); 2261 if (svm_is_intercept(svm, INTERCEPT_VINTR)) 2262 svm_clear_vintr(svm); 2263 2264 enable_gif(svm); 2265 if (svm->vcpu.arch.smi_pending || 2266 svm->vcpu.arch.nmi_pending || 2267 kvm_cpu_has_injectable_intr(&svm->vcpu)) 2268 kvm_make_request(KVM_REQ_EVENT, &svm->vcpu); 2269 } else { 2270 disable_gif(svm); 2271 2272 /* 2273 * After a CLGI no interrupts should come. But if vGIF is 2274 * in use, we still rely on the VINTR intercept (rather than 2275 * STGI) to detect an open interrupt window. 2276 */ 2277 if (!vgif_enabled(svm)) 2278 svm_clear_vintr(svm); 2279 } 2280 } 2281 2282 static int stgi_interception(struct kvm_vcpu *vcpu) 2283 { 2284 int ret; 2285 2286 if (nested_svm_check_permissions(vcpu)) 2287 return 1; 2288 2289 ret = kvm_skip_emulated_instruction(vcpu); 2290 svm_set_gif(to_svm(vcpu), true); 2291 return ret; 2292 } 2293 2294 static int clgi_interception(struct kvm_vcpu *vcpu) 2295 { 2296 int ret; 2297 2298 if (nested_svm_check_permissions(vcpu)) 2299 return 1; 2300 2301 ret = kvm_skip_emulated_instruction(vcpu); 2302 svm_set_gif(to_svm(vcpu), false); 2303 return ret; 2304 } 2305 2306 static int invlpga_interception(struct kvm_vcpu *vcpu) 2307 { 2308 gva_t gva = kvm_rax_read(vcpu); 2309 u32 asid = kvm_rcx_read(vcpu); 2310 2311 /* FIXME: Handle an address size prefix. */ 2312 if (!is_long_mode(vcpu)) 2313 gva = (u32)gva; 2314 2315 trace_kvm_invlpga(to_svm(vcpu)->vmcb->save.rip, asid, gva); 2316 2317 /* Let's treat INVLPGA the same as INVLPG (can be optimized!) */ 2318 kvm_mmu_invlpg(vcpu, gva); 2319 2320 return kvm_skip_emulated_instruction(vcpu); 2321 } 2322 2323 static int skinit_interception(struct kvm_vcpu *vcpu) 2324 { 2325 trace_kvm_skinit(to_svm(vcpu)->vmcb->save.rip, kvm_rax_read(vcpu)); 2326 2327 kvm_queue_exception(vcpu, UD_VECTOR); 2328 return 1; 2329 } 2330 2331 static int task_switch_interception(struct kvm_vcpu *vcpu) 2332 { 2333 struct vcpu_svm *svm = to_svm(vcpu); 2334 u16 tss_selector; 2335 int reason; 2336 int int_type = svm->vmcb->control.exit_int_info & 2337 SVM_EXITINTINFO_TYPE_MASK; 2338 int int_vec = svm->vmcb->control.exit_int_info & SVM_EVTINJ_VEC_MASK; 2339 uint32_t type = 2340 svm->vmcb->control.exit_int_info & SVM_EXITINTINFO_TYPE_MASK; 2341 uint32_t idt_v = 2342 svm->vmcb->control.exit_int_info & SVM_EXITINTINFO_VALID; 2343 bool has_error_code = false; 2344 u32 error_code = 0; 2345 2346 tss_selector = (u16)svm->vmcb->control.exit_info_1; 2347 2348 if (svm->vmcb->control.exit_info_2 & 2349 (1ULL << SVM_EXITINFOSHIFT_TS_REASON_IRET)) 2350 reason = TASK_SWITCH_IRET; 2351 else if (svm->vmcb->control.exit_info_2 & 2352 (1ULL << SVM_EXITINFOSHIFT_TS_REASON_JMP)) 2353 reason = TASK_SWITCH_JMP; 2354 else if (idt_v) 2355 reason = TASK_SWITCH_GATE; 2356 else 2357 reason = TASK_SWITCH_CALL; 2358 2359 if (reason == TASK_SWITCH_GATE) { 2360 switch (type) { 2361 case SVM_EXITINTINFO_TYPE_NMI: 2362 vcpu->arch.nmi_injected = false; 2363 break; 2364 case SVM_EXITINTINFO_TYPE_EXEPT: 2365 if (svm->vmcb->control.exit_info_2 & 2366 (1ULL << SVM_EXITINFOSHIFT_TS_HAS_ERROR_CODE)) { 2367 has_error_code = true; 2368 error_code = 2369 (u32)svm->vmcb->control.exit_info_2; 2370 } 2371 kvm_clear_exception_queue(vcpu); 2372 break; 2373 case SVM_EXITINTINFO_TYPE_INTR: 2374 kvm_clear_interrupt_queue(vcpu); 2375 break; 2376 default: 2377 break; 2378 } 2379 } 2380 2381 if (reason != TASK_SWITCH_GATE || 2382 int_type == SVM_EXITINTINFO_TYPE_SOFT || 2383 (int_type == SVM_EXITINTINFO_TYPE_EXEPT && 2384 (int_vec == OF_VECTOR || int_vec == BP_VECTOR))) { 2385 if (!skip_emulated_instruction(vcpu)) 2386 return 0; 2387 } 2388 2389 if (int_type != SVM_EXITINTINFO_TYPE_SOFT) 2390 int_vec = -1; 2391 2392 return kvm_task_switch(vcpu, tss_selector, int_vec, reason, 2393 has_error_code, error_code); 2394 } 2395 2396 static int iret_interception(struct kvm_vcpu *vcpu) 2397 { 2398 struct vcpu_svm *svm = to_svm(vcpu); 2399 2400 ++vcpu->stat.nmi_window_exits; 2401 vcpu->arch.hflags |= HF_IRET_MASK; 2402 if (!sev_es_guest(vcpu->kvm)) { 2403 svm_clr_intercept(svm, INTERCEPT_IRET); 2404 svm->nmi_iret_rip = kvm_rip_read(vcpu); 2405 } 2406 kvm_make_request(KVM_REQ_EVENT, vcpu); 2407 return 1; 2408 } 2409 2410 static int invlpg_interception(struct kvm_vcpu *vcpu) 2411 { 2412 if (!static_cpu_has(X86_FEATURE_DECODEASSISTS)) 2413 return kvm_emulate_instruction(vcpu, 0); 2414 2415 kvm_mmu_invlpg(vcpu, to_svm(vcpu)->vmcb->control.exit_info_1); 2416 return kvm_skip_emulated_instruction(vcpu); 2417 } 2418 2419 static int emulate_on_interception(struct kvm_vcpu *vcpu) 2420 { 2421 return kvm_emulate_instruction(vcpu, 0); 2422 } 2423 2424 static int rsm_interception(struct kvm_vcpu *vcpu) 2425 { 2426 return kvm_emulate_instruction_from_buffer(vcpu, rsm_ins_bytes, 2); 2427 } 2428 2429 static bool check_selective_cr0_intercepted(struct kvm_vcpu *vcpu, 2430 unsigned long val) 2431 { 2432 struct vcpu_svm *svm = to_svm(vcpu); 2433 unsigned long cr0 = vcpu->arch.cr0; 2434 bool ret = false; 2435 2436 if (!is_guest_mode(vcpu) || 2437 (!(vmcb_is_intercept(&svm->nested.ctl, INTERCEPT_SELECTIVE_CR0)))) 2438 return false; 2439 2440 cr0 &= ~SVM_CR0_SELECTIVE_MASK; 2441 val &= ~SVM_CR0_SELECTIVE_MASK; 2442 2443 if (cr0 ^ val) { 2444 svm->vmcb->control.exit_code = SVM_EXIT_CR0_SEL_WRITE; 2445 ret = (nested_svm_exit_handled(svm) == NESTED_EXIT_DONE); 2446 } 2447 2448 return ret; 2449 } 2450 2451 #define CR_VALID (1ULL << 63) 2452 2453 static int cr_interception(struct kvm_vcpu *vcpu) 2454 { 2455 struct vcpu_svm *svm = to_svm(vcpu); 2456 int reg, cr; 2457 unsigned long val; 2458 int err; 2459 2460 if (!static_cpu_has(X86_FEATURE_DECODEASSISTS)) 2461 return emulate_on_interception(vcpu); 2462 2463 if (unlikely((svm->vmcb->control.exit_info_1 & CR_VALID) == 0)) 2464 return emulate_on_interception(vcpu); 2465 2466 reg = svm->vmcb->control.exit_info_1 & SVM_EXITINFO_REG_MASK; 2467 if (svm->vmcb->control.exit_code == SVM_EXIT_CR0_SEL_WRITE) 2468 cr = SVM_EXIT_WRITE_CR0 - SVM_EXIT_READ_CR0; 2469 else 2470 cr = svm->vmcb->control.exit_code - SVM_EXIT_READ_CR0; 2471 2472 err = 0; 2473 if (cr >= 16) { /* mov to cr */ 2474 cr -= 16; 2475 val = kvm_register_read(vcpu, reg); 2476 trace_kvm_cr_write(cr, val); 2477 switch (cr) { 2478 case 0: 2479 if (!check_selective_cr0_intercepted(vcpu, val)) 2480 err = kvm_set_cr0(vcpu, val); 2481 else 2482 return 1; 2483 2484 break; 2485 case 3: 2486 err = kvm_set_cr3(vcpu, val); 2487 break; 2488 case 4: 2489 err = kvm_set_cr4(vcpu, val); 2490 break; 2491 case 8: 2492 err = kvm_set_cr8(vcpu, val); 2493 break; 2494 default: 2495 WARN(1, "unhandled write to CR%d", cr); 2496 kvm_queue_exception(vcpu, UD_VECTOR); 2497 return 1; 2498 } 2499 } else { /* mov from cr */ 2500 switch (cr) { 2501 case 0: 2502 val = kvm_read_cr0(vcpu); 2503 break; 2504 case 2: 2505 val = vcpu->arch.cr2; 2506 break; 2507 case 3: 2508 val = kvm_read_cr3(vcpu); 2509 break; 2510 case 4: 2511 val = kvm_read_cr4(vcpu); 2512 break; 2513 case 8: 2514 val = kvm_get_cr8(vcpu); 2515 break; 2516 default: 2517 WARN(1, "unhandled read from CR%d", cr); 2518 kvm_queue_exception(vcpu, UD_VECTOR); 2519 return 1; 2520 } 2521 kvm_register_write(vcpu, reg, val); 2522 trace_kvm_cr_read(cr, val); 2523 } 2524 return kvm_complete_insn_gp(vcpu, err); 2525 } 2526 2527 static int cr_trap(struct kvm_vcpu *vcpu) 2528 { 2529 struct vcpu_svm *svm = to_svm(vcpu); 2530 unsigned long old_value, new_value; 2531 unsigned int cr; 2532 int ret = 0; 2533 2534 new_value = (unsigned long)svm->vmcb->control.exit_info_1; 2535 2536 cr = svm->vmcb->control.exit_code - SVM_EXIT_CR0_WRITE_TRAP; 2537 switch (cr) { 2538 case 0: 2539 old_value = kvm_read_cr0(vcpu); 2540 svm_set_cr0(vcpu, new_value); 2541 2542 kvm_post_set_cr0(vcpu, old_value, new_value); 2543 break; 2544 case 4: 2545 old_value = kvm_read_cr4(vcpu); 2546 svm_set_cr4(vcpu, new_value); 2547 2548 kvm_post_set_cr4(vcpu, old_value, new_value); 2549 break; 2550 case 8: 2551 ret = kvm_set_cr8(vcpu, new_value); 2552 break; 2553 default: 2554 WARN(1, "unhandled CR%d write trap", cr); 2555 kvm_queue_exception(vcpu, UD_VECTOR); 2556 return 1; 2557 } 2558 2559 return kvm_complete_insn_gp(vcpu, ret); 2560 } 2561 2562 static int dr_interception(struct kvm_vcpu *vcpu) 2563 { 2564 struct vcpu_svm *svm = to_svm(vcpu); 2565 int reg, dr; 2566 unsigned long val; 2567 int err = 0; 2568 2569 if (vcpu->guest_debug == 0) { 2570 /* 2571 * No more DR vmexits; force a reload of the debug registers 2572 * and reenter on this instruction. The next vmexit will 2573 * retrieve the full state of the debug registers. 2574 */ 2575 clr_dr_intercepts(svm); 2576 vcpu->arch.switch_db_regs |= KVM_DEBUGREG_WONT_EXIT; 2577 return 1; 2578 } 2579 2580 if (!boot_cpu_has(X86_FEATURE_DECODEASSISTS)) 2581 return emulate_on_interception(vcpu); 2582 2583 reg = svm->vmcb->control.exit_info_1 & SVM_EXITINFO_REG_MASK; 2584 dr = svm->vmcb->control.exit_code - SVM_EXIT_READ_DR0; 2585 if (dr >= 16) { /* mov to DRn */ 2586 dr -= 16; 2587 val = kvm_register_read(vcpu, reg); 2588 err = kvm_set_dr(vcpu, dr, val); 2589 } else { 2590 kvm_get_dr(vcpu, dr, &val); 2591 kvm_register_write(vcpu, reg, val); 2592 } 2593 2594 return kvm_complete_insn_gp(vcpu, err); 2595 } 2596 2597 static int cr8_write_interception(struct kvm_vcpu *vcpu) 2598 { 2599 int r; 2600 2601 u8 cr8_prev = kvm_get_cr8(vcpu); 2602 /* instruction emulation calls kvm_set_cr8() */ 2603 r = cr_interception(vcpu); 2604 if (lapic_in_kernel(vcpu)) 2605 return r; 2606 if (cr8_prev <= kvm_get_cr8(vcpu)) 2607 return r; 2608 vcpu->run->exit_reason = KVM_EXIT_SET_TPR; 2609 return 0; 2610 } 2611 2612 static int efer_trap(struct kvm_vcpu *vcpu) 2613 { 2614 struct msr_data msr_info; 2615 int ret; 2616 2617 /* 2618 * Clear the EFER_SVME bit from EFER. The SVM code always sets this 2619 * bit in svm_set_efer(), but __kvm_valid_efer() checks it against 2620 * whether the guest has X86_FEATURE_SVM - this avoids a failure if 2621 * the guest doesn't have X86_FEATURE_SVM. 2622 */ 2623 msr_info.host_initiated = false; 2624 msr_info.index = MSR_EFER; 2625 msr_info.data = to_svm(vcpu)->vmcb->control.exit_info_1 & ~EFER_SVME; 2626 ret = kvm_set_msr_common(vcpu, &msr_info); 2627 2628 return kvm_complete_insn_gp(vcpu, ret); 2629 } 2630 2631 static int svm_get_msr_feature(struct kvm_msr_entry *msr) 2632 { 2633 msr->data = 0; 2634 2635 switch (msr->index) { 2636 case MSR_F10H_DECFG: 2637 if (boot_cpu_has(X86_FEATURE_LFENCE_RDTSC)) 2638 msr->data |= MSR_F10H_DECFG_LFENCE_SERIALIZE; 2639 break; 2640 case MSR_IA32_PERF_CAPABILITIES: 2641 return 0; 2642 default: 2643 return KVM_MSR_RET_INVALID; 2644 } 2645 2646 return 0; 2647 } 2648 2649 static int svm_get_msr(struct kvm_vcpu *vcpu, struct msr_data *msr_info) 2650 { 2651 struct vcpu_svm *svm = to_svm(vcpu); 2652 2653 switch (msr_info->index) { 2654 case MSR_STAR: 2655 msr_info->data = svm->vmcb01.ptr->save.star; 2656 break; 2657 #ifdef CONFIG_X86_64 2658 case MSR_LSTAR: 2659 msr_info->data = svm->vmcb01.ptr->save.lstar; 2660 break; 2661 case MSR_CSTAR: 2662 msr_info->data = svm->vmcb01.ptr->save.cstar; 2663 break; 2664 case MSR_KERNEL_GS_BASE: 2665 msr_info->data = svm->vmcb01.ptr->save.kernel_gs_base; 2666 break; 2667 case MSR_SYSCALL_MASK: 2668 msr_info->data = svm->vmcb01.ptr->save.sfmask; 2669 break; 2670 #endif 2671 case MSR_IA32_SYSENTER_CS: 2672 msr_info->data = svm->vmcb01.ptr->save.sysenter_cs; 2673 break; 2674 case MSR_IA32_SYSENTER_EIP: 2675 msr_info->data = (u32)svm->vmcb01.ptr->save.sysenter_eip; 2676 if (guest_cpuid_is_intel(vcpu)) 2677 msr_info->data |= (u64)svm->sysenter_eip_hi << 32; 2678 break; 2679 case MSR_IA32_SYSENTER_ESP: 2680 msr_info->data = svm->vmcb01.ptr->save.sysenter_esp; 2681 if (guest_cpuid_is_intel(vcpu)) 2682 msr_info->data |= (u64)svm->sysenter_esp_hi << 32; 2683 break; 2684 case MSR_TSC_AUX: 2685 msr_info->data = svm->tsc_aux; 2686 break; 2687 /* 2688 * Nobody will change the following 5 values in the VMCB so we can 2689 * safely return them on rdmsr. They will always be 0 until LBRV is 2690 * implemented. 2691 */ 2692 case MSR_IA32_DEBUGCTLMSR: 2693 msr_info->data = svm->vmcb->save.dbgctl; 2694 break; 2695 case MSR_IA32_LASTBRANCHFROMIP: 2696 msr_info->data = svm->vmcb->save.br_from; 2697 break; 2698 case MSR_IA32_LASTBRANCHTOIP: 2699 msr_info->data = svm->vmcb->save.br_to; 2700 break; 2701 case MSR_IA32_LASTINTFROMIP: 2702 msr_info->data = svm->vmcb->save.last_excp_from; 2703 break; 2704 case MSR_IA32_LASTINTTOIP: 2705 msr_info->data = svm->vmcb->save.last_excp_to; 2706 break; 2707 case MSR_VM_HSAVE_PA: 2708 msr_info->data = svm->nested.hsave_msr; 2709 break; 2710 case MSR_VM_CR: 2711 msr_info->data = svm->nested.vm_cr_msr; 2712 break; 2713 case MSR_IA32_SPEC_CTRL: 2714 if (!msr_info->host_initiated && 2715 !guest_has_spec_ctrl_msr(vcpu)) 2716 return 1; 2717 2718 if (boot_cpu_has(X86_FEATURE_V_SPEC_CTRL)) 2719 msr_info->data = svm->vmcb->save.spec_ctrl; 2720 else 2721 msr_info->data = svm->spec_ctrl; 2722 break; 2723 case MSR_AMD64_VIRT_SPEC_CTRL: 2724 if (!msr_info->host_initiated && 2725 !guest_cpuid_has(vcpu, X86_FEATURE_VIRT_SSBD)) 2726 return 1; 2727 2728 msr_info->data = svm->virt_spec_ctrl; 2729 break; 2730 case MSR_F15H_IC_CFG: { 2731 2732 int family, model; 2733 2734 family = guest_cpuid_family(vcpu); 2735 model = guest_cpuid_model(vcpu); 2736 2737 if (family < 0 || model < 0) 2738 return kvm_get_msr_common(vcpu, msr_info); 2739 2740 msr_info->data = 0; 2741 2742 if (family == 0x15 && 2743 (model >= 0x2 && model < 0x20)) 2744 msr_info->data = 0x1E; 2745 } 2746 break; 2747 case MSR_F10H_DECFG: 2748 msr_info->data = svm->msr_decfg; 2749 break; 2750 default: 2751 return kvm_get_msr_common(vcpu, msr_info); 2752 } 2753 return 0; 2754 } 2755 2756 static int svm_complete_emulated_msr(struct kvm_vcpu *vcpu, int err) 2757 { 2758 struct vcpu_svm *svm = to_svm(vcpu); 2759 if (!err || !sev_es_guest(vcpu->kvm) || WARN_ON_ONCE(!svm->ghcb)) 2760 return kvm_complete_insn_gp(vcpu, err); 2761 2762 ghcb_set_sw_exit_info_1(svm->ghcb, 1); 2763 ghcb_set_sw_exit_info_2(svm->ghcb, 2764 X86_TRAP_GP | 2765 SVM_EVTINJ_TYPE_EXEPT | 2766 SVM_EVTINJ_VALID); 2767 return 1; 2768 } 2769 2770 static int svm_set_vm_cr(struct kvm_vcpu *vcpu, u64 data) 2771 { 2772 struct vcpu_svm *svm = to_svm(vcpu); 2773 int svm_dis, chg_mask; 2774 2775 if (data & ~SVM_VM_CR_VALID_MASK) 2776 return 1; 2777 2778 chg_mask = SVM_VM_CR_VALID_MASK; 2779 2780 if (svm->nested.vm_cr_msr & SVM_VM_CR_SVM_DIS_MASK) 2781 chg_mask &= ~(SVM_VM_CR_SVM_LOCK_MASK | SVM_VM_CR_SVM_DIS_MASK); 2782 2783 svm->nested.vm_cr_msr &= ~chg_mask; 2784 svm->nested.vm_cr_msr |= (data & chg_mask); 2785 2786 svm_dis = svm->nested.vm_cr_msr & SVM_VM_CR_SVM_DIS_MASK; 2787 2788 /* check for svm_disable while efer.svme is set */ 2789 if (svm_dis && (vcpu->arch.efer & EFER_SVME)) 2790 return 1; 2791 2792 return 0; 2793 } 2794 2795 static int svm_set_msr(struct kvm_vcpu *vcpu, struct msr_data *msr) 2796 { 2797 struct vcpu_svm *svm = to_svm(vcpu); 2798 int r; 2799 2800 u32 ecx = msr->index; 2801 u64 data = msr->data; 2802 switch (ecx) { 2803 case MSR_IA32_CR_PAT: 2804 if (!kvm_mtrr_valid(vcpu, MSR_IA32_CR_PAT, data)) 2805 return 1; 2806 vcpu->arch.pat = data; 2807 svm->vmcb01.ptr->save.g_pat = data; 2808 if (is_guest_mode(vcpu)) 2809 nested_vmcb02_compute_g_pat(svm); 2810 vmcb_mark_dirty(svm->vmcb, VMCB_NPT); 2811 break; 2812 case MSR_IA32_SPEC_CTRL: 2813 if (!msr->host_initiated && 2814 !guest_has_spec_ctrl_msr(vcpu)) 2815 return 1; 2816 2817 if (kvm_spec_ctrl_test_value(data)) 2818 return 1; 2819 2820 if (boot_cpu_has(X86_FEATURE_V_SPEC_CTRL)) 2821 svm->vmcb->save.spec_ctrl = data; 2822 else 2823 svm->spec_ctrl = data; 2824 if (!data) 2825 break; 2826 2827 /* 2828 * For non-nested: 2829 * When it's written (to non-zero) for the first time, pass 2830 * it through. 2831 * 2832 * For nested: 2833 * The handling of the MSR bitmap for L2 guests is done in 2834 * nested_svm_vmrun_msrpm. 2835 * We update the L1 MSR bit as well since it will end up 2836 * touching the MSR anyway now. 2837 */ 2838 set_msr_interception(vcpu, svm->msrpm, MSR_IA32_SPEC_CTRL, 1, 1); 2839 break; 2840 case MSR_IA32_PRED_CMD: 2841 if (!msr->host_initiated && 2842 !guest_has_pred_cmd_msr(vcpu)) 2843 return 1; 2844 2845 if (data & ~PRED_CMD_IBPB) 2846 return 1; 2847 if (!boot_cpu_has(X86_FEATURE_IBPB)) 2848 return 1; 2849 if (!data) 2850 break; 2851 2852 wrmsrl(MSR_IA32_PRED_CMD, PRED_CMD_IBPB); 2853 set_msr_interception(vcpu, svm->msrpm, MSR_IA32_PRED_CMD, 0, 1); 2854 break; 2855 case MSR_AMD64_VIRT_SPEC_CTRL: 2856 if (!msr->host_initiated && 2857 !guest_cpuid_has(vcpu, X86_FEATURE_VIRT_SSBD)) 2858 return 1; 2859 2860 if (data & ~SPEC_CTRL_SSBD) 2861 return 1; 2862 2863 svm->virt_spec_ctrl = data; 2864 break; 2865 case MSR_STAR: 2866 svm->vmcb01.ptr->save.star = data; 2867 break; 2868 #ifdef CONFIG_X86_64 2869 case MSR_LSTAR: 2870 svm->vmcb01.ptr->save.lstar = data; 2871 break; 2872 case MSR_CSTAR: 2873 svm->vmcb01.ptr->save.cstar = data; 2874 break; 2875 case MSR_KERNEL_GS_BASE: 2876 svm->vmcb01.ptr->save.kernel_gs_base = data; 2877 break; 2878 case MSR_SYSCALL_MASK: 2879 svm->vmcb01.ptr->save.sfmask = data; 2880 break; 2881 #endif 2882 case MSR_IA32_SYSENTER_CS: 2883 svm->vmcb01.ptr->save.sysenter_cs = data; 2884 break; 2885 case MSR_IA32_SYSENTER_EIP: 2886 svm->vmcb01.ptr->save.sysenter_eip = (u32)data; 2887 /* 2888 * We only intercept the MSR_IA32_SYSENTER_{EIP|ESP} msrs 2889 * when we spoof an Intel vendor ID (for cross vendor migration). 2890 * In this case we use this intercept to track the high 2891 * 32 bit part of these msrs to support Intel's 2892 * implementation of SYSENTER/SYSEXIT. 2893 */ 2894 svm->sysenter_eip_hi = guest_cpuid_is_intel(vcpu) ? (data >> 32) : 0; 2895 break; 2896 case MSR_IA32_SYSENTER_ESP: 2897 svm->vmcb01.ptr->save.sysenter_esp = (u32)data; 2898 svm->sysenter_esp_hi = guest_cpuid_is_intel(vcpu) ? (data >> 32) : 0; 2899 break; 2900 case MSR_TSC_AUX: 2901 /* 2902 * TSC_AUX is usually changed only during boot and never read 2903 * directly. Intercept TSC_AUX instead of exposing it to the 2904 * guest via direct_access_msrs, and switch it via user return. 2905 */ 2906 preempt_disable(); 2907 r = kvm_set_user_return_msr(tsc_aux_uret_slot, data, -1ull); 2908 preempt_enable(); 2909 if (r) 2910 return 1; 2911 2912 svm->tsc_aux = data; 2913 break; 2914 case MSR_IA32_DEBUGCTLMSR: 2915 if (!boot_cpu_has(X86_FEATURE_LBRV)) { 2916 vcpu_unimpl(vcpu, "%s: MSR_IA32_DEBUGCTL 0x%llx, nop\n", 2917 __func__, data); 2918 break; 2919 } 2920 if (data & DEBUGCTL_RESERVED_BITS) 2921 return 1; 2922 2923 svm->vmcb->save.dbgctl = data; 2924 vmcb_mark_dirty(svm->vmcb, VMCB_LBR); 2925 if (data & (1ULL<<0)) 2926 svm_enable_lbrv(vcpu); 2927 else 2928 svm_disable_lbrv(vcpu); 2929 break; 2930 case MSR_VM_HSAVE_PA: 2931 /* 2932 * Old kernels did not validate the value written to 2933 * MSR_VM_HSAVE_PA. Allow KVM_SET_MSR to set an invalid 2934 * value to allow live migrating buggy or malicious guests 2935 * originating from those kernels. 2936 */ 2937 if (!msr->host_initiated && !page_address_valid(vcpu, data)) 2938 return 1; 2939 2940 svm->nested.hsave_msr = data & PAGE_MASK; 2941 break; 2942 case MSR_VM_CR: 2943 return svm_set_vm_cr(vcpu, data); 2944 case MSR_VM_IGNNE: 2945 vcpu_unimpl(vcpu, "unimplemented wrmsr: 0x%x data 0x%llx\n", ecx, data); 2946 break; 2947 case MSR_F10H_DECFG: { 2948 struct kvm_msr_entry msr_entry; 2949 2950 msr_entry.index = msr->index; 2951 if (svm_get_msr_feature(&msr_entry)) 2952 return 1; 2953 2954 /* Check the supported bits */ 2955 if (data & ~msr_entry.data) 2956 return 1; 2957 2958 /* Don't allow the guest to change a bit, #GP */ 2959 if (!msr->host_initiated && (data ^ msr_entry.data)) 2960 return 1; 2961 2962 svm->msr_decfg = data; 2963 break; 2964 } 2965 default: 2966 return kvm_set_msr_common(vcpu, msr); 2967 } 2968 return 0; 2969 } 2970 2971 static int msr_interception(struct kvm_vcpu *vcpu) 2972 { 2973 if (to_svm(vcpu)->vmcb->control.exit_info_1) 2974 return kvm_emulate_wrmsr(vcpu); 2975 else 2976 return kvm_emulate_rdmsr(vcpu); 2977 } 2978 2979 static int interrupt_window_interception(struct kvm_vcpu *vcpu) 2980 { 2981 kvm_make_request(KVM_REQ_EVENT, vcpu); 2982 svm_clear_vintr(to_svm(vcpu)); 2983 2984 /* 2985 * For AVIC, the only reason to end up here is ExtINTs. 2986 * In this case AVIC was temporarily disabled for 2987 * requesting the IRQ window and we have to re-enable it. 2988 */ 2989 kvm_request_apicv_update(vcpu->kvm, true, APICV_INHIBIT_REASON_IRQWIN); 2990 2991 ++vcpu->stat.irq_window_exits; 2992 return 1; 2993 } 2994 2995 static int pause_interception(struct kvm_vcpu *vcpu) 2996 { 2997 bool in_kernel; 2998 2999 /* 3000 * CPL is not made available for an SEV-ES guest, therefore 3001 * vcpu->arch.preempted_in_kernel can never be true. Just 3002 * set in_kernel to false as well. 3003 */ 3004 in_kernel = !sev_es_guest(vcpu->kvm) && svm_get_cpl(vcpu) == 0; 3005 3006 if (!kvm_pause_in_guest(vcpu->kvm)) 3007 grow_ple_window(vcpu); 3008 3009 kvm_vcpu_on_spin(vcpu, in_kernel); 3010 return kvm_skip_emulated_instruction(vcpu); 3011 } 3012 3013 static int invpcid_interception(struct kvm_vcpu *vcpu) 3014 { 3015 struct vcpu_svm *svm = to_svm(vcpu); 3016 unsigned long type; 3017 gva_t gva; 3018 3019 if (!guest_cpuid_has(vcpu, X86_FEATURE_INVPCID)) { 3020 kvm_queue_exception(vcpu, UD_VECTOR); 3021 return 1; 3022 } 3023 3024 /* 3025 * For an INVPCID intercept: 3026 * EXITINFO1 provides the linear address of the memory operand. 3027 * EXITINFO2 provides the contents of the register operand. 3028 */ 3029 type = svm->vmcb->control.exit_info_2; 3030 gva = svm->vmcb->control.exit_info_1; 3031 3032 if (type > 3) { 3033 kvm_inject_gp(vcpu, 0); 3034 return 1; 3035 } 3036 3037 return kvm_handle_invpcid(vcpu, type, gva); 3038 } 3039 3040 static int (*const svm_exit_handlers[])(struct kvm_vcpu *vcpu) = { 3041 [SVM_EXIT_READ_CR0] = cr_interception, 3042 [SVM_EXIT_READ_CR3] = cr_interception, 3043 [SVM_EXIT_READ_CR4] = cr_interception, 3044 [SVM_EXIT_READ_CR8] = cr_interception, 3045 [SVM_EXIT_CR0_SEL_WRITE] = cr_interception, 3046 [SVM_EXIT_WRITE_CR0] = cr_interception, 3047 [SVM_EXIT_WRITE_CR3] = cr_interception, 3048 [SVM_EXIT_WRITE_CR4] = cr_interception, 3049 [SVM_EXIT_WRITE_CR8] = cr8_write_interception, 3050 [SVM_EXIT_READ_DR0] = dr_interception, 3051 [SVM_EXIT_READ_DR1] = dr_interception, 3052 [SVM_EXIT_READ_DR2] = dr_interception, 3053 [SVM_EXIT_READ_DR3] = dr_interception, 3054 [SVM_EXIT_READ_DR4] = dr_interception, 3055 [SVM_EXIT_READ_DR5] = dr_interception, 3056 [SVM_EXIT_READ_DR6] = dr_interception, 3057 [SVM_EXIT_READ_DR7] = dr_interception, 3058 [SVM_EXIT_WRITE_DR0] = dr_interception, 3059 [SVM_EXIT_WRITE_DR1] = dr_interception, 3060 [SVM_EXIT_WRITE_DR2] = dr_interception, 3061 [SVM_EXIT_WRITE_DR3] = dr_interception, 3062 [SVM_EXIT_WRITE_DR4] = dr_interception, 3063 [SVM_EXIT_WRITE_DR5] = dr_interception, 3064 [SVM_EXIT_WRITE_DR6] = dr_interception, 3065 [SVM_EXIT_WRITE_DR7] = dr_interception, 3066 [SVM_EXIT_EXCP_BASE + DB_VECTOR] = db_interception, 3067 [SVM_EXIT_EXCP_BASE + BP_VECTOR] = bp_interception, 3068 [SVM_EXIT_EXCP_BASE + UD_VECTOR] = ud_interception, 3069 [SVM_EXIT_EXCP_BASE + PF_VECTOR] = pf_interception, 3070 [SVM_EXIT_EXCP_BASE + MC_VECTOR] = mc_interception, 3071 [SVM_EXIT_EXCP_BASE + AC_VECTOR] = ac_interception, 3072 [SVM_EXIT_EXCP_BASE + GP_VECTOR] = gp_interception, 3073 [SVM_EXIT_INTR] = intr_interception, 3074 [SVM_EXIT_NMI] = nmi_interception, 3075 [SVM_EXIT_SMI] = smi_interception, 3076 [SVM_EXIT_VINTR] = interrupt_window_interception, 3077 [SVM_EXIT_RDPMC] = kvm_emulate_rdpmc, 3078 [SVM_EXIT_CPUID] = kvm_emulate_cpuid, 3079 [SVM_EXIT_IRET] = iret_interception, 3080 [SVM_EXIT_INVD] = kvm_emulate_invd, 3081 [SVM_EXIT_PAUSE] = pause_interception, 3082 [SVM_EXIT_HLT] = kvm_emulate_halt, 3083 [SVM_EXIT_INVLPG] = invlpg_interception, 3084 [SVM_EXIT_INVLPGA] = invlpga_interception, 3085 [SVM_EXIT_IOIO] = io_interception, 3086 [SVM_EXIT_MSR] = msr_interception, 3087 [SVM_EXIT_TASK_SWITCH] = task_switch_interception, 3088 [SVM_EXIT_SHUTDOWN] = shutdown_interception, 3089 [SVM_EXIT_VMRUN] = vmrun_interception, 3090 [SVM_EXIT_VMMCALL] = kvm_emulate_hypercall, 3091 [SVM_EXIT_VMLOAD] = vmload_interception, 3092 [SVM_EXIT_VMSAVE] = vmsave_interception, 3093 [SVM_EXIT_STGI] = stgi_interception, 3094 [SVM_EXIT_CLGI] = clgi_interception, 3095 [SVM_EXIT_SKINIT] = skinit_interception, 3096 [SVM_EXIT_RDTSCP] = kvm_handle_invalid_op, 3097 [SVM_EXIT_WBINVD] = kvm_emulate_wbinvd, 3098 [SVM_EXIT_MONITOR] = kvm_emulate_monitor, 3099 [SVM_EXIT_MWAIT] = kvm_emulate_mwait, 3100 [SVM_EXIT_XSETBV] = kvm_emulate_xsetbv, 3101 [SVM_EXIT_RDPRU] = kvm_handle_invalid_op, 3102 [SVM_EXIT_EFER_WRITE_TRAP] = efer_trap, 3103 [SVM_EXIT_CR0_WRITE_TRAP] = cr_trap, 3104 [SVM_EXIT_CR4_WRITE_TRAP] = cr_trap, 3105 [SVM_EXIT_CR8_WRITE_TRAP] = cr_trap, 3106 [SVM_EXIT_INVPCID] = invpcid_interception, 3107 [SVM_EXIT_NPF] = npf_interception, 3108 [SVM_EXIT_RSM] = rsm_interception, 3109 [SVM_EXIT_AVIC_INCOMPLETE_IPI] = avic_incomplete_ipi_interception, 3110 [SVM_EXIT_AVIC_UNACCELERATED_ACCESS] = avic_unaccelerated_access_interception, 3111 [SVM_EXIT_VMGEXIT] = sev_handle_vmgexit, 3112 }; 3113 3114 static void dump_vmcb(struct kvm_vcpu *vcpu) 3115 { 3116 struct vcpu_svm *svm = to_svm(vcpu); 3117 struct vmcb_control_area *control = &svm->vmcb->control; 3118 struct vmcb_save_area *save = &svm->vmcb->save; 3119 struct vmcb_save_area *save01 = &svm->vmcb01.ptr->save; 3120 3121 if (!dump_invalid_vmcb) { 3122 pr_warn_ratelimited("set kvm_amd.dump_invalid_vmcb=1 to dump internal KVM state.\n"); 3123 return; 3124 } 3125 3126 pr_err("VMCB %p, last attempted VMRUN on CPU %d\n", 3127 svm->current_vmcb->ptr, vcpu->arch.last_vmentry_cpu); 3128 pr_err("VMCB Control Area:\n"); 3129 pr_err("%-20s%04x\n", "cr_read:", control->intercepts[INTERCEPT_CR] & 0xffff); 3130 pr_err("%-20s%04x\n", "cr_write:", control->intercepts[INTERCEPT_CR] >> 16); 3131 pr_err("%-20s%04x\n", "dr_read:", control->intercepts[INTERCEPT_DR] & 0xffff); 3132 pr_err("%-20s%04x\n", "dr_write:", control->intercepts[INTERCEPT_DR] >> 16); 3133 pr_err("%-20s%08x\n", "exceptions:", control->intercepts[INTERCEPT_EXCEPTION]); 3134 pr_err("%-20s%08x %08x\n", "intercepts:", 3135 control->intercepts[INTERCEPT_WORD3], 3136 control->intercepts[INTERCEPT_WORD4]); 3137 pr_err("%-20s%d\n", "pause filter count:", control->pause_filter_count); 3138 pr_err("%-20s%d\n", "pause filter threshold:", 3139 control->pause_filter_thresh); 3140 pr_err("%-20s%016llx\n", "iopm_base_pa:", control->iopm_base_pa); 3141 pr_err("%-20s%016llx\n", "msrpm_base_pa:", control->msrpm_base_pa); 3142 pr_err("%-20s%016llx\n", "tsc_offset:", control->tsc_offset); 3143 pr_err("%-20s%d\n", "asid:", control->asid); 3144 pr_err("%-20s%d\n", "tlb_ctl:", control->tlb_ctl); 3145 pr_err("%-20s%08x\n", "int_ctl:", control->int_ctl); 3146 pr_err("%-20s%08x\n", "int_vector:", control->int_vector); 3147 pr_err("%-20s%08x\n", "int_state:", control->int_state); 3148 pr_err("%-20s%08x\n", "exit_code:", control->exit_code); 3149 pr_err("%-20s%016llx\n", "exit_info1:", control->exit_info_1); 3150 pr_err("%-20s%016llx\n", "exit_info2:", control->exit_info_2); 3151 pr_err("%-20s%08x\n", "exit_int_info:", control->exit_int_info); 3152 pr_err("%-20s%08x\n", "exit_int_info_err:", control->exit_int_info_err); 3153 pr_err("%-20s%lld\n", "nested_ctl:", control->nested_ctl); 3154 pr_err("%-20s%016llx\n", "nested_cr3:", control->nested_cr3); 3155 pr_err("%-20s%016llx\n", "avic_vapic_bar:", control->avic_vapic_bar); 3156 pr_err("%-20s%016llx\n", "ghcb:", control->ghcb_gpa); 3157 pr_err("%-20s%08x\n", "event_inj:", control->event_inj); 3158 pr_err("%-20s%08x\n", "event_inj_err:", control->event_inj_err); 3159 pr_err("%-20s%lld\n", "virt_ext:", control->virt_ext); 3160 pr_err("%-20s%016llx\n", "next_rip:", control->next_rip); 3161 pr_err("%-20s%016llx\n", "avic_backing_page:", control->avic_backing_page); 3162 pr_err("%-20s%016llx\n", "avic_logical_id:", control->avic_logical_id); 3163 pr_err("%-20s%016llx\n", "avic_physical_id:", control->avic_physical_id); 3164 pr_err("%-20s%016llx\n", "vmsa_pa:", control->vmsa_pa); 3165 pr_err("VMCB State Save Area:\n"); 3166 pr_err("%-5s s: %04x a: %04x l: %08x b: %016llx\n", 3167 "es:", 3168 save->es.selector, save->es.attrib, 3169 save->es.limit, save->es.base); 3170 pr_err("%-5s s: %04x a: %04x l: %08x b: %016llx\n", 3171 "cs:", 3172 save->cs.selector, save->cs.attrib, 3173 save->cs.limit, save->cs.base); 3174 pr_err("%-5s s: %04x a: %04x l: %08x b: %016llx\n", 3175 "ss:", 3176 save->ss.selector, save->ss.attrib, 3177 save->ss.limit, save->ss.base); 3178 pr_err("%-5s s: %04x a: %04x l: %08x b: %016llx\n", 3179 "ds:", 3180 save->ds.selector, save->ds.attrib, 3181 save->ds.limit, save->ds.base); 3182 pr_err("%-5s s: %04x a: %04x l: %08x b: %016llx\n", 3183 "fs:", 3184 save01->fs.selector, save01->fs.attrib, 3185 save01->fs.limit, save01->fs.base); 3186 pr_err("%-5s s: %04x a: %04x l: %08x b: %016llx\n", 3187 "gs:", 3188 save01->gs.selector, save01->gs.attrib, 3189 save01->gs.limit, save01->gs.base); 3190 pr_err("%-5s s: %04x a: %04x l: %08x b: %016llx\n", 3191 "gdtr:", 3192 save->gdtr.selector, save->gdtr.attrib, 3193 save->gdtr.limit, save->gdtr.base); 3194 pr_err("%-5s s: %04x a: %04x l: %08x b: %016llx\n", 3195 "ldtr:", 3196 save01->ldtr.selector, save01->ldtr.attrib, 3197 save01->ldtr.limit, save01->ldtr.base); 3198 pr_err("%-5s s: %04x a: %04x l: %08x b: %016llx\n", 3199 "idtr:", 3200 save->idtr.selector, save->idtr.attrib, 3201 save->idtr.limit, save->idtr.base); 3202 pr_err("%-5s s: %04x a: %04x l: %08x b: %016llx\n", 3203 "tr:", 3204 save01->tr.selector, save01->tr.attrib, 3205 save01->tr.limit, save01->tr.base); 3206 pr_err("cpl: %d efer: %016llx\n", 3207 save->cpl, save->efer); 3208 pr_err("%-15s %016llx %-13s %016llx\n", 3209 "cr0:", save->cr0, "cr2:", save->cr2); 3210 pr_err("%-15s %016llx %-13s %016llx\n", 3211 "cr3:", save->cr3, "cr4:", save->cr4); 3212 pr_err("%-15s %016llx %-13s %016llx\n", 3213 "dr6:", save->dr6, "dr7:", save->dr7); 3214 pr_err("%-15s %016llx %-13s %016llx\n", 3215 "rip:", save->rip, "rflags:", save->rflags); 3216 pr_err("%-15s %016llx %-13s %016llx\n", 3217 "rsp:", save->rsp, "rax:", save->rax); 3218 pr_err("%-15s %016llx %-13s %016llx\n", 3219 "star:", save01->star, "lstar:", save01->lstar); 3220 pr_err("%-15s %016llx %-13s %016llx\n", 3221 "cstar:", save01->cstar, "sfmask:", save01->sfmask); 3222 pr_err("%-15s %016llx %-13s %016llx\n", 3223 "kernel_gs_base:", save01->kernel_gs_base, 3224 "sysenter_cs:", save01->sysenter_cs); 3225 pr_err("%-15s %016llx %-13s %016llx\n", 3226 "sysenter_esp:", save01->sysenter_esp, 3227 "sysenter_eip:", save01->sysenter_eip); 3228 pr_err("%-15s %016llx %-13s %016llx\n", 3229 "gpat:", save->g_pat, "dbgctl:", save->dbgctl); 3230 pr_err("%-15s %016llx %-13s %016llx\n", 3231 "br_from:", save->br_from, "br_to:", save->br_to); 3232 pr_err("%-15s %016llx %-13s %016llx\n", 3233 "excp_from:", save->last_excp_from, 3234 "excp_to:", save->last_excp_to); 3235 } 3236 3237 static bool svm_check_exit_valid(struct kvm_vcpu *vcpu, u64 exit_code) 3238 { 3239 return (exit_code < ARRAY_SIZE(svm_exit_handlers) && 3240 svm_exit_handlers[exit_code]); 3241 } 3242 3243 static int svm_handle_invalid_exit(struct kvm_vcpu *vcpu, u64 exit_code) 3244 { 3245 vcpu_unimpl(vcpu, "svm: unexpected exit reason 0x%llx\n", exit_code); 3246 dump_vmcb(vcpu); 3247 vcpu->run->exit_reason = KVM_EXIT_INTERNAL_ERROR; 3248 vcpu->run->internal.suberror = KVM_INTERNAL_ERROR_UNEXPECTED_EXIT_REASON; 3249 vcpu->run->internal.ndata = 2; 3250 vcpu->run->internal.data[0] = exit_code; 3251 vcpu->run->internal.data[1] = vcpu->arch.last_vmentry_cpu; 3252 return 0; 3253 } 3254 3255 int svm_invoke_exit_handler(struct kvm_vcpu *vcpu, u64 exit_code) 3256 { 3257 if (!svm_check_exit_valid(vcpu, exit_code)) 3258 return svm_handle_invalid_exit(vcpu, exit_code); 3259 3260 #ifdef CONFIG_RETPOLINE 3261 if (exit_code == SVM_EXIT_MSR) 3262 return msr_interception(vcpu); 3263 else if (exit_code == SVM_EXIT_VINTR) 3264 return interrupt_window_interception(vcpu); 3265 else if (exit_code == SVM_EXIT_INTR) 3266 return intr_interception(vcpu); 3267 else if (exit_code == SVM_EXIT_HLT) 3268 return kvm_emulate_halt(vcpu); 3269 else if (exit_code == SVM_EXIT_NPF) 3270 return npf_interception(vcpu); 3271 #endif 3272 return svm_exit_handlers[exit_code](vcpu); 3273 } 3274 3275 static void svm_get_exit_info(struct kvm_vcpu *vcpu, u64 *info1, u64 *info2, 3276 u32 *intr_info, u32 *error_code) 3277 { 3278 struct vmcb_control_area *control = &to_svm(vcpu)->vmcb->control; 3279 3280 *info1 = control->exit_info_1; 3281 *info2 = control->exit_info_2; 3282 *intr_info = control->exit_int_info; 3283 if ((*intr_info & SVM_EXITINTINFO_VALID) && 3284 (*intr_info & SVM_EXITINTINFO_VALID_ERR)) 3285 *error_code = control->exit_int_info_err; 3286 else 3287 *error_code = 0; 3288 } 3289 3290 static int handle_exit(struct kvm_vcpu *vcpu, fastpath_t exit_fastpath) 3291 { 3292 struct vcpu_svm *svm = to_svm(vcpu); 3293 struct kvm_run *kvm_run = vcpu->run; 3294 u32 exit_code = svm->vmcb->control.exit_code; 3295 3296 trace_kvm_exit(exit_code, vcpu, KVM_ISA_SVM); 3297 3298 /* SEV-ES guests must use the CR write traps to track CR registers. */ 3299 if (!sev_es_guest(vcpu->kvm)) { 3300 if (!svm_is_intercept(svm, INTERCEPT_CR0_WRITE)) 3301 vcpu->arch.cr0 = svm->vmcb->save.cr0; 3302 if (npt_enabled) 3303 vcpu->arch.cr3 = svm->vmcb->save.cr3; 3304 } 3305 3306 if (is_guest_mode(vcpu)) { 3307 int vmexit; 3308 3309 trace_kvm_nested_vmexit(exit_code, vcpu, KVM_ISA_SVM); 3310 3311 vmexit = nested_svm_exit_special(svm); 3312 3313 if (vmexit == NESTED_EXIT_CONTINUE) 3314 vmexit = nested_svm_exit_handled(svm); 3315 3316 if (vmexit == NESTED_EXIT_DONE) 3317 return 1; 3318 } 3319 3320 if (svm->vmcb->control.exit_code == SVM_EXIT_ERR) { 3321 kvm_run->exit_reason = KVM_EXIT_FAIL_ENTRY; 3322 kvm_run->fail_entry.hardware_entry_failure_reason 3323 = svm->vmcb->control.exit_code; 3324 kvm_run->fail_entry.cpu = vcpu->arch.last_vmentry_cpu; 3325 dump_vmcb(vcpu); 3326 return 0; 3327 } 3328 3329 if (is_external_interrupt(svm->vmcb->control.exit_int_info) && 3330 exit_code != SVM_EXIT_EXCP_BASE + PF_VECTOR && 3331 exit_code != SVM_EXIT_NPF && exit_code != SVM_EXIT_TASK_SWITCH && 3332 exit_code != SVM_EXIT_INTR && exit_code != SVM_EXIT_NMI) 3333 printk(KERN_ERR "%s: unexpected exit_int_info 0x%x " 3334 "exit_code 0x%x\n", 3335 __func__, svm->vmcb->control.exit_int_info, 3336 exit_code); 3337 3338 if (exit_fastpath != EXIT_FASTPATH_NONE) 3339 return 1; 3340 3341 return svm_invoke_exit_handler(vcpu, exit_code); 3342 } 3343 3344 static void reload_tss(struct kvm_vcpu *vcpu) 3345 { 3346 struct svm_cpu_data *sd = per_cpu(svm_data, vcpu->cpu); 3347 3348 sd->tss_desc->type = 9; /* available 32/64-bit TSS */ 3349 load_TR_desc(); 3350 } 3351 3352 static void pre_svm_run(struct kvm_vcpu *vcpu) 3353 { 3354 struct svm_cpu_data *sd = per_cpu(svm_data, vcpu->cpu); 3355 struct vcpu_svm *svm = to_svm(vcpu); 3356 3357 /* 3358 * If the previous vmrun of the vmcb occurred on a different physical 3359 * cpu, then mark the vmcb dirty and assign a new asid. Hardware's 3360 * vmcb clean bits are per logical CPU, as are KVM's asid assignments. 3361 */ 3362 if (unlikely(svm->current_vmcb->cpu != vcpu->cpu)) { 3363 svm->current_vmcb->asid_generation = 0; 3364 vmcb_mark_all_dirty(svm->vmcb); 3365 svm->current_vmcb->cpu = vcpu->cpu; 3366 } 3367 3368 if (sev_guest(vcpu->kvm)) 3369 return pre_sev_run(svm, vcpu->cpu); 3370 3371 /* FIXME: handle wraparound of asid_generation */ 3372 if (svm->current_vmcb->asid_generation != sd->asid_generation) 3373 new_asid(svm, sd); 3374 } 3375 3376 static void svm_inject_nmi(struct kvm_vcpu *vcpu) 3377 { 3378 struct vcpu_svm *svm = to_svm(vcpu); 3379 3380 svm->vmcb->control.event_inj = SVM_EVTINJ_VALID | SVM_EVTINJ_TYPE_NMI; 3381 vcpu->arch.hflags |= HF_NMI_MASK; 3382 if (!sev_es_guest(vcpu->kvm)) 3383 svm_set_intercept(svm, INTERCEPT_IRET); 3384 ++vcpu->stat.nmi_injections; 3385 } 3386 3387 static void svm_set_irq(struct kvm_vcpu *vcpu) 3388 { 3389 struct vcpu_svm *svm = to_svm(vcpu); 3390 3391 BUG_ON(!(gif_set(svm))); 3392 3393 trace_kvm_inj_virq(vcpu->arch.interrupt.nr); 3394 ++vcpu->stat.irq_injections; 3395 3396 svm->vmcb->control.event_inj = vcpu->arch.interrupt.nr | 3397 SVM_EVTINJ_VALID | SVM_EVTINJ_TYPE_INTR; 3398 } 3399 3400 static void svm_update_cr8_intercept(struct kvm_vcpu *vcpu, int tpr, int irr) 3401 { 3402 struct vcpu_svm *svm = to_svm(vcpu); 3403 3404 /* 3405 * SEV-ES guests must always keep the CR intercepts cleared. CR 3406 * tracking is done using the CR write traps. 3407 */ 3408 if (sev_es_guest(vcpu->kvm)) 3409 return; 3410 3411 if (nested_svm_virtualize_tpr(vcpu)) 3412 return; 3413 3414 svm_clr_intercept(svm, INTERCEPT_CR8_WRITE); 3415 3416 if (irr == -1) 3417 return; 3418 3419 if (tpr >= irr) 3420 svm_set_intercept(svm, INTERCEPT_CR8_WRITE); 3421 } 3422 3423 bool svm_nmi_blocked(struct kvm_vcpu *vcpu) 3424 { 3425 struct vcpu_svm *svm = to_svm(vcpu); 3426 struct vmcb *vmcb = svm->vmcb; 3427 bool ret; 3428 3429 if (!gif_set(svm)) 3430 return true; 3431 3432 if (is_guest_mode(vcpu) && nested_exit_on_nmi(svm)) 3433 return false; 3434 3435 ret = (vmcb->control.int_state & SVM_INTERRUPT_SHADOW_MASK) || 3436 (vcpu->arch.hflags & HF_NMI_MASK); 3437 3438 return ret; 3439 } 3440 3441 static int svm_nmi_allowed(struct kvm_vcpu *vcpu, bool for_injection) 3442 { 3443 struct vcpu_svm *svm = to_svm(vcpu); 3444 if (svm->nested.nested_run_pending) 3445 return -EBUSY; 3446 3447 /* An NMI must not be injected into L2 if it's supposed to VM-Exit. */ 3448 if (for_injection && is_guest_mode(vcpu) && nested_exit_on_nmi(svm)) 3449 return -EBUSY; 3450 3451 return !svm_nmi_blocked(vcpu); 3452 } 3453 3454 static bool svm_get_nmi_mask(struct kvm_vcpu *vcpu) 3455 { 3456 return !!(vcpu->arch.hflags & HF_NMI_MASK); 3457 } 3458 3459 static void svm_set_nmi_mask(struct kvm_vcpu *vcpu, bool masked) 3460 { 3461 struct vcpu_svm *svm = to_svm(vcpu); 3462 3463 if (masked) { 3464 vcpu->arch.hflags |= HF_NMI_MASK; 3465 if (!sev_es_guest(vcpu->kvm)) 3466 svm_set_intercept(svm, INTERCEPT_IRET); 3467 } else { 3468 vcpu->arch.hflags &= ~HF_NMI_MASK; 3469 if (!sev_es_guest(vcpu->kvm)) 3470 svm_clr_intercept(svm, INTERCEPT_IRET); 3471 } 3472 } 3473 3474 bool svm_interrupt_blocked(struct kvm_vcpu *vcpu) 3475 { 3476 struct vcpu_svm *svm = to_svm(vcpu); 3477 struct vmcb *vmcb = svm->vmcb; 3478 3479 if (!gif_set(svm)) 3480 return true; 3481 3482 if (sev_es_guest(vcpu->kvm)) { 3483 /* 3484 * SEV-ES guests to not expose RFLAGS. Use the VMCB interrupt mask 3485 * bit to determine the state of the IF flag. 3486 */ 3487 if (!(vmcb->control.int_state & SVM_GUEST_INTERRUPT_MASK)) 3488 return true; 3489 } else if (is_guest_mode(vcpu)) { 3490 /* As long as interrupts are being delivered... */ 3491 if ((svm->nested.ctl.int_ctl & V_INTR_MASKING_MASK) 3492 ? !(svm->vmcb01.ptr->save.rflags & X86_EFLAGS_IF) 3493 : !(kvm_get_rflags(vcpu) & X86_EFLAGS_IF)) 3494 return true; 3495 3496 /* ... vmexits aren't blocked by the interrupt shadow */ 3497 if (nested_exit_on_intr(svm)) 3498 return false; 3499 } else { 3500 if (!(kvm_get_rflags(vcpu) & X86_EFLAGS_IF)) 3501 return true; 3502 } 3503 3504 return (vmcb->control.int_state & SVM_INTERRUPT_SHADOW_MASK); 3505 } 3506 3507 static int svm_interrupt_allowed(struct kvm_vcpu *vcpu, bool for_injection) 3508 { 3509 struct vcpu_svm *svm = to_svm(vcpu); 3510 if (svm->nested.nested_run_pending) 3511 return -EBUSY; 3512 3513 /* 3514 * An IRQ must not be injected into L2 if it's supposed to VM-Exit, 3515 * e.g. if the IRQ arrived asynchronously after checking nested events. 3516 */ 3517 if (for_injection && is_guest_mode(vcpu) && nested_exit_on_intr(svm)) 3518 return -EBUSY; 3519 3520 return !svm_interrupt_blocked(vcpu); 3521 } 3522 3523 static void svm_enable_irq_window(struct kvm_vcpu *vcpu) 3524 { 3525 struct vcpu_svm *svm = to_svm(vcpu); 3526 3527 /* 3528 * In case GIF=0 we can't rely on the CPU to tell us when GIF becomes 3529 * 1, because that's a separate STGI/VMRUN intercept. The next time we 3530 * get that intercept, this function will be called again though and 3531 * we'll get the vintr intercept. However, if the vGIF feature is 3532 * enabled, the STGI interception will not occur. Enable the irq 3533 * window under the assumption that the hardware will set the GIF. 3534 */ 3535 if (vgif_enabled(svm) || gif_set(svm)) { 3536 /* 3537 * IRQ window is not needed when AVIC is enabled, 3538 * unless we have pending ExtINT since it cannot be injected 3539 * via AVIC. In such case, we need to temporarily disable AVIC, 3540 * and fallback to injecting IRQ via V_IRQ. 3541 */ 3542 kvm_request_apicv_update(vcpu->kvm, false, APICV_INHIBIT_REASON_IRQWIN); 3543 svm_set_vintr(svm); 3544 } 3545 } 3546 3547 static void svm_enable_nmi_window(struct kvm_vcpu *vcpu) 3548 { 3549 struct vcpu_svm *svm = to_svm(vcpu); 3550 3551 if ((vcpu->arch.hflags & (HF_NMI_MASK | HF_IRET_MASK)) == HF_NMI_MASK) 3552 return; /* IRET will cause a vm exit */ 3553 3554 if (!gif_set(svm)) { 3555 if (vgif_enabled(svm)) 3556 svm_set_intercept(svm, INTERCEPT_STGI); 3557 return; /* STGI will cause a vm exit */ 3558 } 3559 3560 /* 3561 * Something prevents NMI from been injected. Single step over possible 3562 * problem (IRET or exception injection or interrupt shadow) 3563 */ 3564 svm->nmi_singlestep_guest_rflags = svm_get_rflags(vcpu); 3565 svm->nmi_singlestep = true; 3566 svm->vmcb->save.rflags |= (X86_EFLAGS_TF | X86_EFLAGS_RF); 3567 } 3568 3569 static int svm_set_tss_addr(struct kvm *kvm, unsigned int addr) 3570 { 3571 return 0; 3572 } 3573 3574 static int svm_set_identity_map_addr(struct kvm *kvm, u64 ident_addr) 3575 { 3576 return 0; 3577 } 3578 3579 void svm_flush_tlb(struct kvm_vcpu *vcpu) 3580 { 3581 struct vcpu_svm *svm = to_svm(vcpu); 3582 3583 /* 3584 * Flush only the current ASID even if the TLB flush was invoked via 3585 * kvm_flush_remote_tlbs(). Although flushing remote TLBs requires all 3586 * ASIDs to be flushed, KVM uses a single ASID for L1 and L2, and 3587 * unconditionally does a TLB flush on both nested VM-Enter and nested 3588 * VM-Exit (via kvm_mmu_reset_context()). 3589 */ 3590 if (static_cpu_has(X86_FEATURE_FLUSHBYASID)) 3591 svm->vmcb->control.tlb_ctl = TLB_CONTROL_FLUSH_ASID; 3592 else 3593 svm->current_vmcb->asid_generation--; 3594 } 3595 3596 static void svm_flush_tlb_gva(struct kvm_vcpu *vcpu, gva_t gva) 3597 { 3598 struct vcpu_svm *svm = to_svm(vcpu); 3599 3600 invlpga(gva, svm->vmcb->control.asid); 3601 } 3602 3603 static inline void sync_cr8_to_lapic(struct kvm_vcpu *vcpu) 3604 { 3605 struct vcpu_svm *svm = to_svm(vcpu); 3606 3607 if (nested_svm_virtualize_tpr(vcpu)) 3608 return; 3609 3610 if (!svm_is_intercept(svm, INTERCEPT_CR8_WRITE)) { 3611 int cr8 = svm->vmcb->control.int_ctl & V_TPR_MASK; 3612 kvm_set_cr8(vcpu, cr8); 3613 } 3614 } 3615 3616 static inline void sync_lapic_to_cr8(struct kvm_vcpu *vcpu) 3617 { 3618 struct vcpu_svm *svm = to_svm(vcpu); 3619 u64 cr8; 3620 3621 if (nested_svm_virtualize_tpr(vcpu) || 3622 kvm_vcpu_apicv_active(vcpu)) 3623 return; 3624 3625 cr8 = kvm_get_cr8(vcpu); 3626 svm->vmcb->control.int_ctl &= ~V_TPR_MASK; 3627 svm->vmcb->control.int_ctl |= cr8 & V_TPR_MASK; 3628 } 3629 3630 static void svm_complete_interrupts(struct kvm_vcpu *vcpu) 3631 { 3632 struct vcpu_svm *svm = to_svm(vcpu); 3633 u8 vector; 3634 int type; 3635 u32 exitintinfo = svm->vmcb->control.exit_int_info; 3636 unsigned int3_injected = svm->int3_injected; 3637 3638 svm->int3_injected = 0; 3639 3640 /* 3641 * If we've made progress since setting HF_IRET_MASK, we've 3642 * executed an IRET and can allow NMI injection. 3643 */ 3644 if ((vcpu->arch.hflags & HF_IRET_MASK) && 3645 (sev_es_guest(vcpu->kvm) || 3646 kvm_rip_read(vcpu) != svm->nmi_iret_rip)) { 3647 vcpu->arch.hflags &= ~(HF_NMI_MASK | HF_IRET_MASK); 3648 kvm_make_request(KVM_REQ_EVENT, vcpu); 3649 } 3650 3651 vcpu->arch.nmi_injected = false; 3652 kvm_clear_exception_queue(vcpu); 3653 kvm_clear_interrupt_queue(vcpu); 3654 3655 if (!(exitintinfo & SVM_EXITINTINFO_VALID)) 3656 return; 3657 3658 kvm_make_request(KVM_REQ_EVENT, vcpu); 3659 3660 vector = exitintinfo & SVM_EXITINTINFO_VEC_MASK; 3661 type = exitintinfo & SVM_EXITINTINFO_TYPE_MASK; 3662 3663 switch (type) { 3664 case SVM_EXITINTINFO_TYPE_NMI: 3665 vcpu->arch.nmi_injected = true; 3666 break; 3667 case SVM_EXITINTINFO_TYPE_EXEPT: 3668 /* 3669 * Never re-inject a #VC exception. 3670 */ 3671 if (vector == X86_TRAP_VC) 3672 break; 3673 3674 /* 3675 * In case of software exceptions, do not reinject the vector, 3676 * but re-execute the instruction instead. Rewind RIP first 3677 * if we emulated INT3 before. 3678 */ 3679 if (kvm_exception_is_soft(vector)) { 3680 if (vector == BP_VECTOR && int3_injected && 3681 kvm_is_linear_rip(vcpu, svm->int3_rip)) 3682 kvm_rip_write(vcpu, 3683 kvm_rip_read(vcpu) - int3_injected); 3684 break; 3685 } 3686 if (exitintinfo & SVM_EXITINTINFO_VALID_ERR) { 3687 u32 err = svm->vmcb->control.exit_int_info_err; 3688 kvm_requeue_exception_e(vcpu, vector, err); 3689 3690 } else 3691 kvm_requeue_exception(vcpu, vector); 3692 break; 3693 case SVM_EXITINTINFO_TYPE_INTR: 3694 kvm_queue_interrupt(vcpu, vector, false); 3695 break; 3696 default: 3697 break; 3698 } 3699 } 3700 3701 static void svm_cancel_injection(struct kvm_vcpu *vcpu) 3702 { 3703 struct vcpu_svm *svm = to_svm(vcpu); 3704 struct vmcb_control_area *control = &svm->vmcb->control; 3705 3706 control->exit_int_info = control->event_inj; 3707 control->exit_int_info_err = control->event_inj_err; 3708 control->event_inj = 0; 3709 svm_complete_interrupts(vcpu); 3710 } 3711 3712 static fastpath_t svm_exit_handlers_fastpath(struct kvm_vcpu *vcpu) 3713 { 3714 if (to_svm(vcpu)->vmcb->control.exit_code == SVM_EXIT_MSR && 3715 to_svm(vcpu)->vmcb->control.exit_info_1) 3716 return handle_fastpath_set_msr_irqoff(vcpu); 3717 3718 return EXIT_FASTPATH_NONE; 3719 } 3720 3721 static noinstr void svm_vcpu_enter_exit(struct kvm_vcpu *vcpu) 3722 { 3723 struct vcpu_svm *svm = to_svm(vcpu); 3724 unsigned long vmcb_pa = svm->current_vmcb->pa; 3725 3726 kvm_guest_enter_irqoff(); 3727 3728 if (sev_es_guest(vcpu->kvm)) { 3729 __svm_sev_es_vcpu_run(vmcb_pa); 3730 } else { 3731 struct svm_cpu_data *sd = per_cpu(svm_data, vcpu->cpu); 3732 3733 /* 3734 * Use a single vmcb (vmcb01 because it's always valid) for 3735 * context switching guest state via VMLOAD/VMSAVE, that way 3736 * the state doesn't need to be copied between vmcb01 and 3737 * vmcb02 when switching vmcbs for nested virtualization. 3738 */ 3739 vmload(svm->vmcb01.pa); 3740 __svm_vcpu_run(vmcb_pa, (unsigned long *)&vcpu->arch.regs); 3741 vmsave(svm->vmcb01.pa); 3742 3743 vmload(__sme_page_pa(sd->save_area)); 3744 } 3745 3746 kvm_guest_exit_irqoff(); 3747 } 3748 3749 static __no_kcsan fastpath_t svm_vcpu_run(struct kvm_vcpu *vcpu) 3750 { 3751 struct vcpu_svm *svm = to_svm(vcpu); 3752 3753 trace_kvm_entry(vcpu); 3754 3755 svm->vmcb->save.rax = vcpu->arch.regs[VCPU_REGS_RAX]; 3756 svm->vmcb->save.rsp = vcpu->arch.regs[VCPU_REGS_RSP]; 3757 svm->vmcb->save.rip = vcpu->arch.regs[VCPU_REGS_RIP]; 3758 3759 /* 3760 * Disable singlestep if we're injecting an interrupt/exception. 3761 * We don't want our modified rflags to be pushed on the stack where 3762 * we might not be able to easily reset them if we disabled NMI 3763 * singlestep later. 3764 */ 3765 if (svm->nmi_singlestep && svm->vmcb->control.event_inj) { 3766 /* 3767 * Event injection happens before external interrupts cause a 3768 * vmexit and interrupts are disabled here, so smp_send_reschedule 3769 * is enough to force an immediate vmexit. 3770 */ 3771 disable_nmi_singlestep(svm); 3772 smp_send_reschedule(vcpu->cpu); 3773 } 3774 3775 pre_svm_run(vcpu); 3776 3777 WARN_ON_ONCE(kvm_apicv_activated(vcpu->kvm) != kvm_vcpu_apicv_active(vcpu)); 3778 3779 sync_lapic_to_cr8(vcpu); 3780 3781 if (unlikely(svm->asid != svm->vmcb->control.asid)) { 3782 svm->vmcb->control.asid = svm->asid; 3783 vmcb_mark_dirty(svm->vmcb, VMCB_ASID); 3784 } 3785 svm->vmcb->save.cr2 = vcpu->arch.cr2; 3786 3787 svm_hv_update_vp_id(svm->vmcb, vcpu); 3788 3789 /* 3790 * Run with all-zero DR6 unless needed, so that we can get the exact cause 3791 * of a #DB. 3792 */ 3793 if (unlikely(vcpu->arch.switch_db_regs & KVM_DEBUGREG_WONT_EXIT)) 3794 svm_set_dr6(svm, vcpu->arch.dr6); 3795 else 3796 svm_set_dr6(svm, DR6_ACTIVE_LOW); 3797 3798 clgi(); 3799 kvm_load_guest_xsave_state(vcpu); 3800 3801 kvm_wait_lapic_expire(vcpu); 3802 3803 /* 3804 * If this vCPU has touched SPEC_CTRL, restore the guest's value if 3805 * it's non-zero. Since vmentry is serialising on affected CPUs, there 3806 * is no need to worry about the conditional branch over the wrmsr 3807 * being speculatively taken. 3808 */ 3809 if (!static_cpu_has(X86_FEATURE_V_SPEC_CTRL)) 3810 x86_spec_ctrl_set_guest(svm->spec_ctrl, svm->virt_spec_ctrl); 3811 3812 svm_vcpu_enter_exit(vcpu); 3813 3814 /* 3815 * We do not use IBRS in the kernel. If this vCPU has used the 3816 * SPEC_CTRL MSR it may have left it on; save the value and 3817 * turn it off. This is much more efficient than blindly adding 3818 * it to the atomic save/restore list. Especially as the former 3819 * (Saving guest MSRs on vmexit) doesn't even exist in KVM. 3820 * 3821 * For non-nested case: 3822 * If the L01 MSR bitmap does not intercept the MSR, then we need to 3823 * save it. 3824 * 3825 * For nested case: 3826 * If the L02 MSR bitmap does not intercept the MSR, then we need to 3827 * save it. 3828 */ 3829 if (!static_cpu_has(X86_FEATURE_V_SPEC_CTRL) && 3830 unlikely(!msr_write_intercepted(vcpu, MSR_IA32_SPEC_CTRL))) 3831 svm->spec_ctrl = native_read_msr(MSR_IA32_SPEC_CTRL); 3832 3833 if (!sev_es_guest(vcpu->kvm)) 3834 reload_tss(vcpu); 3835 3836 if (!static_cpu_has(X86_FEATURE_V_SPEC_CTRL)) 3837 x86_spec_ctrl_restore_host(svm->spec_ctrl, svm->virt_spec_ctrl); 3838 3839 if (!sev_es_guest(vcpu->kvm)) { 3840 vcpu->arch.cr2 = svm->vmcb->save.cr2; 3841 vcpu->arch.regs[VCPU_REGS_RAX] = svm->vmcb->save.rax; 3842 vcpu->arch.regs[VCPU_REGS_RSP] = svm->vmcb->save.rsp; 3843 vcpu->arch.regs[VCPU_REGS_RIP] = svm->vmcb->save.rip; 3844 } 3845 3846 if (unlikely(svm->vmcb->control.exit_code == SVM_EXIT_NMI)) 3847 kvm_before_interrupt(vcpu); 3848 3849 kvm_load_host_xsave_state(vcpu); 3850 stgi(); 3851 3852 /* Any pending NMI will happen here */ 3853 3854 if (unlikely(svm->vmcb->control.exit_code == SVM_EXIT_NMI)) 3855 kvm_after_interrupt(vcpu); 3856 3857 sync_cr8_to_lapic(vcpu); 3858 3859 svm->next_rip = 0; 3860 if (is_guest_mode(vcpu)) { 3861 nested_sync_control_from_vmcb02(svm); 3862 3863 /* Track VMRUNs that have made past consistency checking */ 3864 if (svm->nested.nested_run_pending && 3865 svm->vmcb->control.exit_code != SVM_EXIT_ERR) 3866 ++vcpu->stat.nested_run; 3867 3868 svm->nested.nested_run_pending = 0; 3869 } 3870 3871 svm->vmcb->control.tlb_ctl = TLB_CONTROL_DO_NOTHING; 3872 vmcb_mark_all_clean(svm->vmcb); 3873 3874 /* if exit due to PF check for async PF */ 3875 if (svm->vmcb->control.exit_code == SVM_EXIT_EXCP_BASE + PF_VECTOR) 3876 vcpu->arch.apf.host_apf_flags = 3877 kvm_read_and_reset_apf_flags(); 3878 3879 if (npt_enabled) 3880 kvm_register_clear_available(vcpu, VCPU_EXREG_PDPTR); 3881 3882 /* 3883 * We need to handle MC intercepts here before the vcpu has a chance to 3884 * change the physical cpu 3885 */ 3886 if (unlikely(svm->vmcb->control.exit_code == 3887 SVM_EXIT_EXCP_BASE + MC_VECTOR)) 3888 svm_handle_mce(vcpu); 3889 3890 svm_complete_interrupts(vcpu); 3891 3892 if (is_guest_mode(vcpu)) 3893 return EXIT_FASTPATH_NONE; 3894 3895 return svm_exit_handlers_fastpath(vcpu); 3896 } 3897 3898 static void svm_load_mmu_pgd(struct kvm_vcpu *vcpu, hpa_t root_hpa, 3899 int root_level) 3900 { 3901 struct vcpu_svm *svm = to_svm(vcpu); 3902 unsigned long cr3; 3903 3904 if (npt_enabled) { 3905 svm->vmcb->control.nested_cr3 = __sme_set(root_hpa); 3906 vmcb_mark_dirty(svm->vmcb, VMCB_NPT); 3907 3908 hv_track_root_tdp(vcpu, root_hpa); 3909 3910 /* Loading L2's CR3 is handled by enter_svm_guest_mode. */ 3911 if (!test_bit(VCPU_EXREG_CR3, (ulong *)&vcpu->arch.regs_avail)) 3912 return; 3913 cr3 = vcpu->arch.cr3; 3914 } else if (vcpu->arch.mmu->shadow_root_level >= PT64_ROOT_4LEVEL) { 3915 cr3 = __sme_set(root_hpa) | kvm_get_active_pcid(vcpu); 3916 } else { 3917 /* PCID in the guest should be impossible with a 32-bit MMU. */ 3918 WARN_ON_ONCE(kvm_get_active_pcid(vcpu)); 3919 cr3 = root_hpa; 3920 } 3921 3922 svm->vmcb->save.cr3 = cr3; 3923 vmcb_mark_dirty(svm->vmcb, VMCB_CR); 3924 } 3925 3926 static int is_disabled(void) 3927 { 3928 u64 vm_cr; 3929 3930 rdmsrl(MSR_VM_CR, vm_cr); 3931 if (vm_cr & (1 << SVM_VM_CR_SVM_DISABLE)) 3932 return 1; 3933 3934 return 0; 3935 } 3936 3937 static void 3938 svm_patch_hypercall(struct kvm_vcpu *vcpu, unsigned char *hypercall) 3939 { 3940 /* 3941 * Patch in the VMMCALL instruction: 3942 */ 3943 hypercall[0] = 0x0f; 3944 hypercall[1] = 0x01; 3945 hypercall[2] = 0xd9; 3946 } 3947 3948 static int __init svm_check_processor_compat(void) 3949 { 3950 return 0; 3951 } 3952 3953 static bool svm_cpu_has_accelerated_tpr(void) 3954 { 3955 return false; 3956 } 3957 3958 /* 3959 * The kvm parameter can be NULL (module initialization, or invocation before 3960 * VM creation). Be sure to check the kvm parameter before using it. 3961 */ 3962 static bool svm_has_emulated_msr(struct kvm *kvm, u32 index) 3963 { 3964 switch (index) { 3965 case MSR_IA32_MCG_EXT_CTL: 3966 case MSR_IA32_VMX_BASIC ... MSR_IA32_VMX_VMFUNC: 3967 return false; 3968 case MSR_IA32_SMBASE: 3969 /* SEV-ES guests do not support SMM, so report false */ 3970 if (kvm && sev_es_guest(kvm)) 3971 return false; 3972 break; 3973 default: 3974 break; 3975 } 3976 3977 return true; 3978 } 3979 3980 static u64 svm_get_mt_mask(struct kvm_vcpu *vcpu, gfn_t gfn, bool is_mmio) 3981 { 3982 return 0; 3983 } 3984 3985 static void svm_vcpu_after_set_cpuid(struct kvm_vcpu *vcpu) 3986 { 3987 struct vcpu_svm *svm = to_svm(vcpu); 3988 struct kvm_cpuid_entry2 *best; 3989 3990 vcpu->arch.xsaves_enabled = guest_cpuid_has(vcpu, X86_FEATURE_XSAVE) && 3991 boot_cpu_has(X86_FEATURE_XSAVE) && 3992 boot_cpu_has(X86_FEATURE_XSAVES); 3993 3994 /* Update nrips enabled cache */ 3995 svm->nrips_enabled = kvm_cpu_cap_has(X86_FEATURE_NRIPS) && 3996 guest_cpuid_has(vcpu, X86_FEATURE_NRIPS); 3997 3998 svm_recalc_instruction_intercepts(vcpu, svm); 3999 4000 /* For sev guests, the memory encryption bit is not reserved in CR3. */ 4001 if (sev_guest(vcpu->kvm)) { 4002 best = kvm_find_cpuid_entry(vcpu, 0x8000001F, 0); 4003 if (best) 4004 vcpu->arch.reserved_gpa_bits &= ~(1UL << (best->ebx & 0x3f)); 4005 } 4006 4007 if (kvm_vcpu_apicv_active(vcpu)) { 4008 /* 4009 * AVIC does not work with an x2APIC mode guest. If the X2APIC feature 4010 * is exposed to the guest, disable AVIC. 4011 */ 4012 if (guest_cpuid_has(vcpu, X86_FEATURE_X2APIC)) 4013 kvm_request_apicv_update(vcpu->kvm, false, 4014 APICV_INHIBIT_REASON_X2APIC); 4015 4016 /* 4017 * Currently, AVIC does not work with nested virtualization. 4018 * So, we disable AVIC when cpuid for SVM is set in the L1 guest. 4019 */ 4020 if (nested && guest_cpuid_has(vcpu, X86_FEATURE_SVM)) 4021 kvm_request_apicv_update(vcpu->kvm, false, 4022 APICV_INHIBIT_REASON_NESTED); 4023 } 4024 4025 if (guest_cpuid_is_intel(vcpu)) { 4026 /* 4027 * We must intercept SYSENTER_EIP and SYSENTER_ESP 4028 * accesses because the processor only stores 32 bits. 4029 * For the same reason we cannot use virtual VMLOAD/VMSAVE. 4030 */ 4031 svm_set_intercept(svm, INTERCEPT_VMLOAD); 4032 svm_set_intercept(svm, INTERCEPT_VMSAVE); 4033 svm->vmcb->control.virt_ext &= ~VIRTUAL_VMLOAD_VMSAVE_ENABLE_MASK; 4034 4035 set_msr_interception(vcpu, svm->msrpm, MSR_IA32_SYSENTER_EIP, 0, 0); 4036 set_msr_interception(vcpu, svm->msrpm, MSR_IA32_SYSENTER_ESP, 0, 0); 4037 } else { 4038 /* 4039 * If hardware supports Virtual VMLOAD VMSAVE then enable it 4040 * in VMCB and clear intercepts to avoid #VMEXIT. 4041 */ 4042 if (vls) { 4043 svm_clr_intercept(svm, INTERCEPT_VMLOAD); 4044 svm_clr_intercept(svm, INTERCEPT_VMSAVE); 4045 svm->vmcb->control.virt_ext |= VIRTUAL_VMLOAD_VMSAVE_ENABLE_MASK; 4046 } 4047 /* No need to intercept these MSRs */ 4048 set_msr_interception(vcpu, svm->msrpm, MSR_IA32_SYSENTER_EIP, 1, 1); 4049 set_msr_interception(vcpu, svm->msrpm, MSR_IA32_SYSENTER_ESP, 1, 1); 4050 } 4051 } 4052 4053 static bool svm_has_wbinvd_exit(void) 4054 { 4055 return true; 4056 } 4057 4058 #define PRE_EX(exit) { .exit_code = (exit), \ 4059 .stage = X86_ICPT_PRE_EXCEPT, } 4060 #define POST_EX(exit) { .exit_code = (exit), \ 4061 .stage = X86_ICPT_POST_EXCEPT, } 4062 #define POST_MEM(exit) { .exit_code = (exit), \ 4063 .stage = X86_ICPT_POST_MEMACCESS, } 4064 4065 static const struct __x86_intercept { 4066 u32 exit_code; 4067 enum x86_intercept_stage stage; 4068 } x86_intercept_map[] = { 4069 [x86_intercept_cr_read] = POST_EX(SVM_EXIT_READ_CR0), 4070 [x86_intercept_cr_write] = POST_EX(SVM_EXIT_WRITE_CR0), 4071 [x86_intercept_clts] = POST_EX(SVM_EXIT_WRITE_CR0), 4072 [x86_intercept_lmsw] = POST_EX(SVM_EXIT_WRITE_CR0), 4073 [x86_intercept_smsw] = POST_EX(SVM_EXIT_READ_CR0), 4074 [x86_intercept_dr_read] = POST_EX(SVM_EXIT_READ_DR0), 4075 [x86_intercept_dr_write] = POST_EX(SVM_EXIT_WRITE_DR0), 4076 [x86_intercept_sldt] = POST_EX(SVM_EXIT_LDTR_READ), 4077 [x86_intercept_str] = POST_EX(SVM_EXIT_TR_READ), 4078 [x86_intercept_lldt] = POST_EX(SVM_EXIT_LDTR_WRITE), 4079 [x86_intercept_ltr] = POST_EX(SVM_EXIT_TR_WRITE), 4080 [x86_intercept_sgdt] = POST_EX(SVM_EXIT_GDTR_READ), 4081 [x86_intercept_sidt] = POST_EX(SVM_EXIT_IDTR_READ), 4082 [x86_intercept_lgdt] = POST_EX(SVM_EXIT_GDTR_WRITE), 4083 [x86_intercept_lidt] = POST_EX(SVM_EXIT_IDTR_WRITE), 4084 [x86_intercept_vmrun] = POST_EX(SVM_EXIT_VMRUN), 4085 [x86_intercept_vmmcall] = POST_EX(SVM_EXIT_VMMCALL), 4086 [x86_intercept_vmload] = POST_EX(SVM_EXIT_VMLOAD), 4087 [x86_intercept_vmsave] = POST_EX(SVM_EXIT_VMSAVE), 4088 [x86_intercept_stgi] = POST_EX(SVM_EXIT_STGI), 4089 [x86_intercept_clgi] = POST_EX(SVM_EXIT_CLGI), 4090 [x86_intercept_skinit] = POST_EX(SVM_EXIT_SKINIT), 4091 [x86_intercept_invlpga] = POST_EX(SVM_EXIT_INVLPGA), 4092 [x86_intercept_rdtscp] = POST_EX(SVM_EXIT_RDTSCP), 4093 [x86_intercept_monitor] = POST_MEM(SVM_EXIT_MONITOR), 4094 [x86_intercept_mwait] = POST_EX(SVM_EXIT_MWAIT), 4095 [x86_intercept_invlpg] = POST_EX(SVM_EXIT_INVLPG), 4096 [x86_intercept_invd] = POST_EX(SVM_EXIT_INVD), 4097 [x86_intercept_wbinvd] = POST_EX(SVM_EXIT_WBINVD), 4098 [x86_intercept_wrmsr] = POST_EX(SVM_EXIT_MSR), 4099 [x86_intercept_rdtsc] = POST_EX(SVM_EXIT_RDTSC), 4100 [x86_intercept_rdmsr] = POST_EX(SVM_EXIT_MSR), 4101 [x86_intercept_rdpmc] = POST_EX(SVM_EXIT_RDPMC), 4102 [x86_intercept_cpuid] = PRE_EX(SVM_EXIT_CPUID), 4103 [x86_intercept_rsm] = PRE_EX(SVM_EXIT_RSM), 4104 [x86_intercept_pause] = PRE_EX(SVM_EXIT_PAUSE), 4105 [x86_intercept_pushf] = PRE_EX(SVM_EXIT_PUSHF), 4106 [x86_intercept_popf] = PRE_EX(SVM_EXIT_POPF), 4107 [x86_intercept_intn] = PRE_EX(SVM_EXIT_SWINT), 4108 [x86_intercept_iret] = PRE_EX(SVM_EXIT_IRET), 4109 [x86_intercept_icebp] = PRE_EX(SVM_EXIT_ICEBP), 4110 [x86_intercept_hlt] = POST_EX(SVM_EXIT_HLT), 4111 [x86_intercept_in] = POST_EX(SVM_EXIT_IOIO), 4112 [x86_intercept_ins] = POST_EX(SVM_EXIT_IOIO), 4113 [x86_intercept_out] = POST_EX(SVM_EXIT_IOIO), 4114 [x86_intercept_outs] = POST_EX(SVM_EXIT_IOIO), 4115 [x86_intercept_xsetbv] = PRE_EX(SVM_EXIT_XSETBV), 4116 }; 4117 4118 #undef PRE_EX 4119 #undef POST_EX 4120 #undef POST_MEM 4121 4122 static int svm_check_intercept(struct kvm_vcpu *vcpu, 4123 struct x86_instruction_info *info, 4124 enum x86_intercept_stage stage, 4125 struct x86_exception *exception) 4126 { 4127 struct vcpu_svm *svm = to_svm(vcpu); 4128 int vmexit, ret = X86EMUL_CONTINUE; 4129 struct __x86_intercept icpt_info; 4130 struct vmcb *vmcb = svm->vmcb; 4131 4132 if (info->intercept >= ARRAY_SIZE(x86_intercept_map)) 4133 goto out; 4134 4135 icpt_info = x86_intercept_map[info->intercept]; 4136 4137 if (stage != icpt_info.stage) 4138 goto out; 4139 4140 switch (icpt_info.exit_code) { 4141 case SVM_EXIT_READ_CR0: 4142 if (info->intercept == x86_intercept_cr_read) 4143 icpt_info.exit_code += info->modrm_reg; 4144 break; 4145 case SVM_EXIT_WRITE_CR0: { 4146 unsigned long cr0, val; 4147 4148 if (info->intercept == x86_intercept_cr_write) 4149 icpt_info.exit_code += info->modrm_reg; 4150 4151 if (icpt_info.exit_code != SVM_EXIT_WRITE_CR0 || 4152 info->intercept == x86_intercept_clts) 4153 break; 4154 4155 if (!(vmcb_is_intercept(&svm->nested.ctl, 4156 INTERCEPT_SELECTIVE_CR0))) 4157 break; 4158 4159 cr0 = vcpu->arch.cr0 & ~SVM_CR0_SELECTIVE_MASK; 4160 val = info->src_val & ~SVM_CR0_SELECTIVE_MASK; 4161 4162 if (info->intercept == x86_intercept_lmsw) { 4163 cr0 &= 0xfUL; 4164 val &= 0xfUL; 4165 /* lmsw can't clear PE - catch this here */ 4166 if (cr0 & X86_CR0_PE) 4167 val |= X86_CR0_PE; 4168 } 4169 4170 if (cr0 ^ val) 4171 icpt_info.exit_code = SVM_EXIT_CR0_SEL_WRITE; 4172 4173 break; 4174 } 4175 case SVM_EXIT_READ_DR0: 4176 case SVM_EXIT_WRITE_DR0: 4177 icpt_info.exit_code += info->modrm_reg; 4178 break; 4179 case SVM_EXIT_MSR: 4180 if (info->intercept == x86_intercept_wrmsr) 4181 vmcb->control.exit_info_1 = 1; 4182 else 4183 vmcb->control.exit_info_1 = 0; 4184 break; 4185 case SVM_EXIT_PAUSE: 4186 /* 4187 * We get this for NOP only, but pause 4188 * is rep not, check this here 4189 */ 4190 if (info->rep_prefix != REPE_PREFIX) 4191 goto out; 4192 break; 4193 case SVM_EXIT_IOIO: { 4194 u64 exit_info; 4195 u32 bytes; 4196 4197 if (info->intercept == x86_intercept_in || 4198 info->intercept == x86_intercept_ins) { 4199 exit_info = ((info->src_val & 0xffff) << 16) | 4200 SVM_IOIO_TYPE_MASK; 4201 bytes = info->dst_bytes; 4202 } else { 4203 exit_info = (info->dst_val & 0xffff) << 16; 4204 bytes = info->src_bytes; 4205 } 4206 4207 if (info->intercept == x86_intercept_outs || 4208 info->intercept == x86_intercept_ins) 4209 exit_info |= SVM_IOIO_STR_MASK; 4210 4211 if (info->rep_prefix) 4212 exit_info |= SVM_IOIO_REP_MASK; 4213 4214 bytes = min(bytes, 4u); 4215 4216 exit_info |= bytes << SVM_IOIO_SIZE_SHIFT; 4217 4218 exit_info |= (u32)info->ad_bytes << (SVM_IOIO_ASIZE_SHIFT - 1); 4219 4220 vmcb->control.exit_info_1 = exit_info; 4221 vmcb->control.exit_info_2 = info->next_rip; 4222 4223 break; 4224 } 4225 default: 4226 break; 4227 } 4228 4229 /* TODO: Advertise NRIPS to guest hypervisor unconditionally */ 4230 if (static_cpu_has(X86_FEATURE_NRIPS)) 4231 vmcb->control.next_rip = info->next_rip; 4232 vmcb->control.exit_code = icpt_info.exit_code; 4233 vmexit = nested_svm_exit_handled(svm); 4234 4235 ret = (vmexit == NESTED_EXIT_DONE) ? X86EMUL_INTERCEPTED 4236 : X86EMUL_CONTINUE; 4237 4238 out: 4239 return ret; 4240 } 4241 4242 static void svm_handle_exit_irqoff(struct kvm_vcpu *vcpu) 4243 { 4244 } 4245 4246 static void svm_sched_in(struct kvm_vcpu *vcpu, int cpu) 4247 { 4248 if (!kvm_pause_in_guest(vcpu->kvm)) 4249 shrink_ple_window(vcpu); 4250 } 4251 4252 static void svm_setup_mce(struct kvm_vcpu *vcpu) 4253 { 4254 /* [63:9] are reserved. */ 4255 vcpu->arch.mcg_cap &= 0x1ff; 4256 } 4257 4258 bool svm_smi_blocked(struct kvm_vcpu *vcpu) 4259 { 4260 struct vcpu_svm *svm = to_svm(vcpu); 4261 4262 /* Per APM Vol.2 15.22.2 "Response to SMI" */ 4263 if (!gif_set(svm)) 4264 return true; 4265 4266 return is_smm(vcpu); 4267 } 4268 4269 static int svm_smi_allowed(struct kvm_vcpu *vcpu, bool for_injection) 4270 { 4271 struct vcpu_svm *svm = to_svm(vcpu); 4272 if (svm->nested.nested_run_pending) 4273 return -EBUSY; 4274 4275 /* An SMI must not be injected into L2 if it's supposed to VM-Exit. */ 4276 if (for_injection && is_guest_mode(vcpu) && nested_exit_on_smi(svm)) 4277 return -EBUSY; 4278 4279 return !svm_smi_blocked(vcpu); 4280 } 4281 4282 static int svm_enter_smm(struct kvm_vcpu *vcpu, char *smstate) 4283 { 4284 struct vcpu_svm *svm = to_svm(vcpu); 4285 struct kvm_host_map map_save; 4286 int ret; 4287 4288 if (is_guest_mode(vcpu)) { 4289 /* FED8h - SVM Guest */ 4290 put_smstate(u64, smstate, 0x7ed8, 1); 4291 /* FEE0h - SVM Guest VMCB Physical Address */ 4292 put_smstate(u64, smstate, 0x7ee0, svm->nested.vmcb12_gpa); 4293 4294 svm->vmcb->save.rax = vcpu->arch.regs[VCPU_REGS_RAX]; 4295 svm->vmcb->save.rsp = vcpu->arch.regs[VCPU_REGS_RSP]; 4296 svm->vmcb->save.rip = vcpu->arch.regs[VCPU_REGS_RIP]; 4297 4298 ret = nested_svm_vmexit(svm); 4299 if (ret) 4300 return ret; 4301 4302 /* 4303 * KVM uses VMCB01 to store L1 host state while L2 runs but 4304 * VMCB01 is going to be used during SMM and thus the state will 4305 * be lost. Temporary save non-VMLOAD/VMSAVE state to the host save 4306 * area pointed to by MSR_VM_HSAVE_PA. APM guarantees that the 4307 * format of the area is identical to guest save area offsetted 4308 * by 0x400 (matches the offset of 'struct vmcb_save_area' 4309 * within 'struct vmcb'). Note: HSAVE area may also be used by 4310 * L1 hypervisor to save additional host context (e.g. KVM does 4311 * that, see svm_prepare_guest_switch()) which must be 4312 * preserved. 4313 */ 4314 if (kvm_vcpu_map(vcpu, gpa_to_gfn(svm->nested.hsave_msr), 4315 &map_save) == -EINVAL) 4316 return 1; 4317 4318 BUILD_BUG_ON(offsetof(struct vmcb, save) != 0x400); 4319 4320 svm_copy_vmrun_state(map_save.hva + 0x400, 4321 &svm->vmcb01.ptr->save); 4322 4323 kvm_vcpu_unmap(vcpu, &map_save, true); 4324 } 4325 return 0; 4326 } 4327 4328 static int svm_leave_smm(struct kvm_vcpu *vcpu, const char *smstate) 4329 { 4330 struct vcpu_svm *svm = to_svm(vcpu); 4331 struct kvm_host_map map, map_save; 4332 int ret = 0; 4333 4334 if (guest_cpuid_has(vcpu, X86_FEATURE_LM)) { 4335 u64 saved_efer = GET_SMSTATE(u64, smstate, 0x7ed0); 4336 u64 guest = GET_SMSTATE(u64, smstate, 0x7ed8); 4337 u64 vmcb12_gpa = GET_SMSTATE(u64, smstate, 0x7ee0); 4338 struct vmcb *vmcb12; 4339 4340 if (guest) { 4341 if (!guest_cpuid_has(vcpu, X86_FEATURE_SVM)) 4342 return 1; 4343 4344 if (!(saved_efer & EFER_SVME)) 4345 return 1; 4346 4347 if (kvm_vcpu_map(vcpu, 4348 gpa_to_gfn(vmcb12_gpa), &map) == -EINVAL) 4349 return 1; 4350 4351 if (svm_allocate_nested(svm)) 4352 return 1; 4353 4354 vmcb12 = map.hva; 4355 4356 nested_load_control_from_vmcb12(svm, &vmcb12->control); 4357 4358 ret = enter_svm_guest_mode(vcpu, vmcb12_gpa, vmcb12); 4359 kvm_vcpu_unmap(vcpu, &map, true); 4360 4361 /* 4362 * Restore L1 host state from L1 HSAVE area as VMCB01 was 4363 * used during SMM (see svm_enter_smm()) 4364 */ 4365 if (kvm_vcpu_map(vcpu, gpa_to_gfn(svm->nested.hsave_msr), 4366 &map_save) == -EINVAL) 4367 return 1; 4368 4369 svm_copy_vmrun_state(&svm->vmcb01.ptr->save, 4370 map_save.hva + 0x400); 4371 4372 kvm_vcpu_unmap(vcpu, &map_save, true); 4373 } 4374 } 4375 4376 return ret; 4377 } 4378 4379 static void svm_enable_smi_window(struct kvm_vcpu *vcpu) 4380 { 4381 struct vcpu_svm *svm = to_svm(vcpu); 4382 4383 if (!gif_set(svm)) { 4384 if (vgif_enabled(svm)) 4385 svm_set_intercept(svm, INTERCEPT_STGI); 4386 /* STGI will cause a vm exit */ 4387 } else { 4388 /* We must be in SMM; RSM will cause a vmexit anyway. */ 4389 } 4390 } 4391 4392 static bool svm_can_emulate_instruction(struct kvm_vcpu *vcpu, void *insn, int insn_len) 4393 { 4394 bool smep, smap, is_user; 4395 unsigned long cr4; 4396 4397 /* 4398 * When the guest is an SEV-ES guest, emulation is not possible. 4399 */ 4400 if (sev_es_guest(vcpu->kvm)) 4401 return false; 4402 4403 /* 4404 * Detect and workaround Errata 1096 Fam_17h_00_0Fh. 4405 * 4406 * Errata: 4407 * When CPU raise #NPF on guest data access and vCPU CR4.SMAP=1, it is 4408 * possible that CPU microcode implementing DecodeAssist will fail 4409 * to read bytes of instruction which caused #NPF. In this case, 4410 * GuestIntrBytes field of the VMCB on a VMEXIT will incorrectly 4411 * return 0 instead of the correct guest instruction bytes. 4412 * 4413 * This happens because CPU microcode reading instruction bytes 4414 * uses a special opcode which attempts to read data using CPL=0 4415 * privileges. The microcode reads CS:RIP and if it hits a SMAP 4416 * fault, it gives up and returns no instruction bytes. 4417 * 4418 * Detection: 4419 * We reach here in case CPU supports DecodeAssist, raised #NPF and 4420 * returned 0 in GuestIntrBytes field of the VMCB. 4421 * First, errata can only be triggered in case vCPU CR4.SMAP=1. 4422 * Second, if vCPU CR4.SMEP=1, errata could only be triggered 4423 * in case vCPU CPL==3 (Because otherwise guest would have triggered 4424 * a SMEP fault instead of #NPF). 4425 * Otherwise, vCPU CR4.SMEP=0, errata could be triggered by any vCPU CPL. 4426 * As most guests enable SMAP if they have also enabled SMEP, use above 4427 * logic in order to attempt minimize false-positive of detecting errata 4428 * while still preserving all cases semantic correctness. 4429 * 4430 * Workaround: 4431 * To determine what instruction the guest was executing, the hypervisor 4432 * will have to decode the instruction at the instruction pointer. 4433 * 4434 * In non SEV guest, hypervisor will be able to read the guest 4435 * memory to decode the instruction pointer when insn_len is zero 4436 * so we return true to indicate that decoding is possible. 4437 * 4438 * But in the SEV guest, the guest memory is encrypted with the 4439 * guest specific key and hypervisor will not be able to decode the 4440 * instruction pointer so we will not able to workaround it. Lets 4441 * print the error and request to kill the guest. 4442 */ 4443 if (likely(!insn || insn_len)) 4444 return true; 4445 4446 /* 4447 * If RIP is invalid, go ahead with emulation which will cause an 4448 * internal error exit. 4449 */ 4450 if (!kvm_vcpu_gfn_to_memslot(vcpu, kvm_rip_read(vcpu) >> PAGE_SHIFT)) 4451 return true; 4452 4453 cr4 = kvm_read_cr4(vcpu); 4454 smep = cr4 & X86_CR4_SMEP; 4455 smap = cr4 & X86_CR4_SMAP; 4456 is_user = svm_get_cpl(vcpu) == 3; 4457 if (smap && (!smep || is_user)) { 4458 if (!sev_guest(vcpu->kvm)) 4459 return true; 4460 4461 pr_err_ratelimited("KVM: SEV Guest triggered AMD Erratum 1096\n"); 4462 kvm_make_request(KVM_REQ_TRIPLE_FAULT, vcpu); 4463 } 4464 4465 return false; 4466 } 4467 4468 static bool svm_apic_init_signal_blocked(struct kvm_vcpu *vcpu) 4469 { 4470 struct vcpu_svm *svm = to_svm(vcpu); 4471 4472 /* 4473 * TODO: Last condition latch INIT signals on vCPU when 4474 * vCPU is in guest-mode and vmcb12 defines intercept on INIT. 4475 * To properly emulate the INIT intercept, 4476 * svm_check_nested_events() should call nested_svm_vmexit() 4477 * if an INIT signal is pending. 4478 */ 4479 return !gif_set(svm) || 4480 (vmcb_is_intercept(&svm->vmcb->control, INTERCEPT_INIT)); 4481 } 4482 4483 static void svm_vcpu_deliver_sipi_vector(struct kvm_vcpu *vcpu, u8 vector) 4484 { 4485 if (!sev_es_guest(vcpu->kvm)) 4486 return kvm_vcpu_deliver_sipi_vector(vcpu, vector); 4487 4488 sev_vcpu_deliver_sipi_vector(vcpu, vector); 4489 } 4490 4491 static void svm_vm_destroy(struct kvm *kvm) 4492 { 4493 avic_vm_destroy(kvm); 4494 sev_vm_destroy(kvm); 4495 } 4496 4497 static int svm_vm_init(struct kvm *kvm) 4498 { 4499 if (!pause_filter_count || !pause_filter_thresh) 4500 kvm->arch.pause_in_guest = true; 4501 4502 if (enable_apicv) { 4503 int ret = avic_vm_init(kvm); 4504 if (ret) 4505 return ret; 4506 } 4507 4508 return 0; 4509 } 4510 4511 static struct kvm_x86_ops svm_x86_ops __initdata = { 4512 .hardware_unsetup = svm_hardware_teardown, 4513 .hardware_enable = svm_hardware_enable, 4514 .hardware_disable = svm_hardware_disable, 4515 .cpu_has_accelerated_tpr = svm_cpu_has_accelerated_tpr, 4516 .has_emulated_msr = svm_has_emulated_msr, 4517 4518 .vcpu_create = svm_create_vcpu, 4519 .vcpu_free = svm_free_vcpu, 4520 .vcpu_reset = svm_vcpu_reset, 4521 4522 .vm_size = sizeof(struct kvm_svm), 4523 .vm_init = svm_vm_init, 4524 .vm_destroy = svm_vm_destroy, 4525 4526 .prepare_guest_switch = svm_prepare_guest_switch, 4527 .vcpu_load = svm_vcpu_load, 4528 .vcpu_put = svm_vcpu_put, 4529 .vcpu_blocking = svm_vcpu_blocking, 4530 .vcpu_unblocking = svm_vcpu_unblocking, 4531 4532 .update_exception_bitmap = svm_update_exception_bitmap, 4533 .get_msr_feature = svm_get_msr_feature, 4534 .get_msr = svm_get_msr, 4535 .set_msr = svm_set_msr, 4536 .get_segment_base = svm_get_segment_base, 4537 .get_segment = svm_get_segment, 4538 .set_segment = svm_set_segment, 4539 .get_cpl = svm_get_cpl, 4540 .get_cs_db_l_bits = kvm_get_cs_db_l_bits, 4541 .set_cr0 = svm_set_cr0, 4542 .is_valid_cr4 = svm_is_valid_cr4, 4543 .set_cr4 = svm_set_cr4, 4544 .set_efer = svm_set_efer, 4545 .get_idt = svm_get_idt, 4546 .set_idt = svm_set_idt, 4547 .get_gdt = svm_get_gdt, 4548 .set_gdt = svm_set_gdt, 4549 .set_dr7 = svm_set_dr7, 4550 .sync_dirty_debug_regs = svm_sync_dirty_debug_regs, 4551 .cache_reg = svm_cache_reg, 4552 .get_rflags = svm_get_rflags, 4553 .set_rflags = svm_set_rflags, 4554 4555 .tlb_flush_all = svm_flush_tlb, 4556 .tlb_flush_current = svm_flush_tlb, 4557 .tlb_flush_gva = svm_flush_tlb_gva, 4558 .tlb_flush_guest = svm_flush_tlb, 4559 4560 .run = svm_vcpu_run, 4561 .handle_exit = handle_exit, 4562 .skip_emulated_instruction = skip_emulated_instruction, 4563 .update_emulated_instruction = NULL, 4564 .set_interrupt_shadow = svm_set_interrupt_shadow, 4565 .get_interrupt_shadow = svm_get_interrupt_shadow, 4566 .patch_hypercall = svm_patch_hypercall, 4567 .set_irq = svm_set_irq, 4568 .set_nmi = svm_inject_nmi, 4569 .queue_exception = svm_queue_exception, 4570 .cancel_injection = svm_cancel_injection, 4571 .interrupt_allowed = svm_interrupt_allowed, 4572 .nmi_allowed = svm_nmi_allowed, 4573 .get_nmi_mask = svm_get_nmi_mask, 4574 .set_nmi_mask = svm_set_nmi_mask, 4575 .enable_nmi_window = svm_enable_nmi_window, 4576 .enable_irq_window = svm_enable_irq_window, 4577 .update_cr8_intercept = svm_update_cr8_intercept, 4578 .set_virtual_apic_mode = svm_set_virtual_apic_mode, 4579 .refresh_apicv_exec_ctrl = svm_refresh_apicv_exec_ctrl, 4580 .check_apicv_inhibit_reasons = svm_check_apicv_inhibit_reasons, 4581 .load_eoi_exitmap = svm_load_eoi_exitmap, 4582 .hwapic_irr_update = svm_hwapic_irr_update, 4583 .hwapic_isr_update = svm_hwapic_isr_update, 4584 .sync_pir_to_irr = kvm_lapic_find_highest_irr, 4585 .apicv_post_state_restore = avic_post_state_restore, 4586 4587 .set_tss_addr = svm_set_tss_addr, 4588 .set_identity_map_addr = svm_set_identity_map_addr, 4589 .get_mt_mask = svm_get_mt_mask, 4590 4591 .get_exit_info = svm_get_exit_info, 4592 4593 .vcpu_after_set_cpuid = svm_vcpu_after_set_cpuid, 4594 4595 .has_wbinvd_exit = svm_has_wbinvd_exit, 4596 4597 .get_l2_tsc_offset = svm_get_l2_tsc_offset, 4598 .get_l2_tsc_multiplier = svm_get_l2_tsc_multiplier, 4599 .write_tsc_offset = svm_write_tsc_offset, 4600 .write_tsc_multiplier = svm_write_tsc_multiplier, 4601 4602 .load_mmu_pgd = svm_load_mmu_pgd, 4603 4604 .check_intercept = svm_check_intercept, 4605 .handle_exit_irqoff = svm_handle_exit_irqoff, 4606 4607 .request_immediate_exit = __kvm_request_immediate_exit, 4608 4609 .sched_in = svm_sched_in, 4610 4611 .pmu_ops = &amd_pmu_ops, 4612 .nested_ops = &svm_nested_ops, 4613 4614 .deliver_posted_interrupt = svm_deliver_avic_intr, 4615 .dy_apicv_has_pending_interrupt = svm_dy_apicv_has_pending_interrupt, 4616 .update_pi_irte = svm_update_pi_irte, 4617 .setup_mce = svm_setup_mce, 4618 4619 .smi_allowed = svm_smi_allowed, 4620 .enter_smm = svm_enter_smm, 4621 .leave_smm = svm_leave_smm, 4622 .enable_smi_window = svm_enable_smi_window, 4623 4624 .mem_enc_op = svm_mem_enc_op, 4625 .mem_enc_reg_region = svm_register_enc_region, 4626 .mem_enc_unreg_region = svm_unregister_enc_region, 4627 4628 .vm_copy_enc_context_from = svm_vm_copy_asid_from, 4629 4630 .can_emulate_instruction = svm_can_emulate_instruction, 4631 4632 .apic_init_signal_blocked = svm_apic_init_signal_blocked, 4633 4634 .msr_filter_changed = svm_msr_filter_changed, 4635 .complete_emulated_msr = svm_complete_emulated_msr, 4636 4637 .vcpu_deliver_sipi_vector = svm_vcpu_deliver_sipi_vector, 4638 }; 4639 4640 static struct kvm_x86_init_ops svm_init_ops __initdata = { 4641 .cpu_has_kvm_support = has_svm, 4642 .disabled_by_bios = is_disabled, 4643 .hardware_setup = svm_hardware_setup, 4644 .check_processor_compatibility = svm_check_processor_compat, 4645 4646 .runtime_ops = &svm_x86_ops, 4647 }; 4648 4649 static int __init svm_init(void) 4650 { 4651 __unused_size_checks(); 4652 4653 return kvm_init(&svm_init_ops, sizeof(struct vcpu_svm), 4654 __alignof__(struct vcpu_svm), THIS_MODULE); 4655 } 4656 4657 static void __exit svm_exit(void) 4658 { 4659 kvm_exit(); 4660 } 4661 4662 module_init(svm_init) 4663 module_exit(svm_exit) 4664