1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * Kernel-based Virtual Machine driver for Linux 4 * 5 * This module enables machines with Intel VT-x extensions to run virtual 6 * machines without emulation or binary translation. 7 * 8 * Copyright (C) 2006 Qumranet, Inc. 9 * Copyright 2010 Red Hat, Inc. and/or its affiliates. 10 * 11 * Authors: 12 * Avi Kivity <avi@qumranet.com> 13 * Yaniv Kamay <yaniv@qumranet.com> 14 */ 15 16 #include <linux/highmem.h> 17 #include <linux/hrtimer.h> 18 #include <linux/kernel.h> 19 #include <linux/kvm_host.h> 20 #include <linux/module.h> 21 #include <linux/moduleparam.h> 22 #include <linux/mod_devicetable.h> 23 #include <linux/mm.h> 24 #include <linux/objtool.h> 25 #include <linux/sched.h> 26 #include <linux/sched/smt.h> 27 #include <linux/slab.h> 28 #include <linux/tboot.h> 29 #include <linux/trace_events.h> 30 #include <linux/entry-kvm.h> 31 32 #include <asm/apic.h> 33 #include <asm/asm.h> 34 #include <asm/cpu.h> 35 #include <asm/cpu_device_id.h> 36 #include <asm/debugreg.h> 37 #include <asm/desc.h> 38 #include <asm/fpu/internal.h> 39 #include <asm/io.h> 40 #include <asm/irq_remapping.h> 41 #include <asm/kexec.h> 42 #include <asm/perf_event.h> 43 #include <asm/mce.h> 44 #include <asm/mmu_context.h> 45 #include <asm/mshyperv.h> 46 #include <asm/mwait.h> 47 #include <asm/spec-ctrl.h> 48 #include <asm/virtext.h> 49 #include <asm/vmx.h> 50 51 #include "capabilities.h" 52 #include "cpuid.h" 53 #include "evmcs.h" 54 #include "irq.h" 55 #include "kvm_cache_regs.h" 56 #include "lapic.h" 57 #include "mmu.h" 58 #include "nested.h" 59 #include "ops.h" 60 #include "pmu.h" 61 #include "trace.h" 62 #include "vmcs.h" 63 #include "vmcs12.h" 64 #include "vmx.h" 65 #include "x86.h" 66 67 MODULE_AUTHOR("Qumranet"); 68 MODULE_LICENSE("GPL"); 69 70 #ifdef MODULE 71 static const struct x86_cpu_id vmx_cpu_id[] = { 72 X86_MATCH_FEATURE(X86_FEATURE_VMX, NULL), 73 {} 74 }; 75 MODULE_DEVICE_TABLE(x86cpu, vmx_cpu_id); 76 #endif 77 78 bool __read_mostly enable_vpid = 1; 79 module_param_named(vpid, enable_vpid, bool, 0444); 80 81 static bool __read_mostly enable_vnmi = 1; 82 module_param_named(vnmi, enable_vnmi, bool, S_IRUGO); 83 84 bool __read_mostly flexpriority_enabled = 1; 85 module_param_named(flexpriority, flexpriority_enabled, bool, S_IRUGO); 86 87 bool __read_mostly enable_ept = 1; 88 module_param_named(ept, enable_ept, bool, S_IRUGO); 89 90 bool __read_mostly enable_unrestricted_guest = 1; 91 module_param_named(unrestricted_guest, 92 enable_unrestricted_guest, bool, S_IRUGO); 93 94 bool __read_mostly enable_ept_ad_bits = 1; 95 module_param_named(eptad, enable_ept_ad_bits, bool, S_IRUGO); 96 97 static bool __read_mostly emulate_invalid_guest_state = true; 98 module_param(emulate_invalid_guest_state, bool, S_IRUGO); 99 100 static bool __read_mostly fasteoi = 1; 101 module_param(fasteoi, bool, S_IRUGO); 102 103 bool __read_mostly enable_apicv = 1; 104 module_param(enable_apicv, bool, S_IRUGO); 105 106 /* 107 * If nested=1, nested virtualization is supported, i.e., guests may use 108 * VMX and be a hypervisor for its own guests. If nested=0, guests may not 109 * use VMX instructions. 110 */ 111 static bool __read_mostly nested = 1; 112 module_param(nested, bool, S_IRUGO); 113 114 bool __read_mostly enable_pml = 1; 115 module_param_named(pml, enable_pml, bool, S_IRUGO); 116 117 static bool __read_mostly dump_invalid_vmcs = 0; 118 module_param(dump_invalid_vmcs, bool, 0644); 119 120 #define MSR_BITMAP_MODE_X2APIC 1 121 #define MSR_BITMAP_MODE_X2APIC_APICV 2 122 123 #define KVM_VMX_TSC_MULTIPLIER_MAX 0xffffffffffffffffULL 124 125 /* Guest_tsc -> host_tsc conversion requires 64-bit division. */ 126 static int __read_mostly cpu_preemption_timer_multi; 127 static bool __read_mostly enable_preemption_timer = 1; 128 #ifdef CONFIG_X86_64 129 module_param_named(preemption_timer, enable_preemption_timer, bool, S_IRUGO); 130 #endif 131 132 extern bool __read_mostly allow_smaller_maxphyaddr; 133 module_param(allow_smaller_maxphyaddr, bool, S_IRUGO); 134 135 #define KVM_VM_CR0_ALWAYS_OFF (X86_CR0_NW | X86_CR0_CD) 136 #define KVM_VM_CR0_ALWAYS_ON_UNRESTRICTED_GUEST X86_CR0_NE 137 #define KVM_VM_CR0_ALWAYS_ON \ 138 (KVM_VM_CR0_ALWAYS_ON_UNRESTRICTED_GUEST | \ 139 X86_CR0_WP | X86_CR0_PG | X86_CR0_PE) 140 141 #define KVM_VM_CR4_ALWAYS_ON_UNRESTRICTED_GUEST X86_CR4_VMXE 142 #define KVM_PMODE_VM_CR4_ALWAYS_ON (X86_CR4_PAE | X86_CR4_VMXE) 143 #define KVM_RMODE_VM_CR4_ALWAYS_ON (X86_CR4_VME | X86_CR4_PAE | X86_CR4_VMXE) 144 145 #define RMODE_GUEST_OWNED_EFLAGS_BITS (~(X86_EFLAGS_IOPL | X86_EFLAGS_VM)) 146 147 #define MSR_IA32_RTIT_STATUS_MASK (~(RTIT_STATUS_FILTEREN | \ 148 RTIT_STATUS_CONTEXTEN | RTIT_STATUS_TRIGGEREN | \ 149 RTIT_STATUS_ERROR | RTIT_STATUS_STOPPED | \ 150 RTIT_STATUS_BYTECNT)) 151 152 #define MSR_IA32_RTIT_OUTPUT_BASE_MASK \ 153 (~((1UL << cpuid_query_maxphyaddr(vcpu)) - 1) | 0x7f) 154 155 /* 156 * These 2 parameters are used to config the controls for Pause-Loop Exiting: 157 * ple_gap: upper bound on the amount of time between two successive 158 * executions of PAUSE in a loop. Also indicate if ple enabled. 159 * According to test, this time is usually smaller than 128 cycles. 160 * ple_window: upper bound on the amount of time a guest is allowed to execute 161 * in a PAUSE loop. Tests indicate that most spinlocks are held for 162 * less than 2^12 cycles 163 * Time is measured based on a counter that runs at the same rate as the TSC, 164 * refer SDM volume 3b section 21.6.13 & 22.1.3. 165 */ 166 static unsigned int ple_gap = KVM_DEFAULT_PLE_GAP; 167 module_param(ple_gap, uint, 0444); 168 169 static unsigned int ple_window = KVM_VMX_DEFAULT_PLE_WINDOW; 170 module_param(ple_window, uint, 0444); 171 172 /* Default doubles per-vcpu window every exit. */ 173 static unsigned int ple_window_grow = KVM_DEFAULT_PLE_WINDOW_GROW; 174 module_param(ple_window_grow, uint, 0444); 175 176 /* Default resets per-vcpu window every exit to ple_window. */ 177 static unsigned int ple_window_shrink = KVM_DEFAULT_PLE_WINDOW_SHRINK; 178 module_param(ple_window_shrink, uint, 0444); 179 180 /* Default is to compute the maximum so we can never overflow. */ 181 static unsigned int ple_window_max = KVM_VMX_DEFAULT_PLE_WINDOW_MAX; 182 module_param(ple_window_max, uint, 0444); 183 184 /* Default is SYSTEM mode, 1 for host-guest mode */ 185 int __read_mostly pt_mode = PT_MODE_SYSTEM; 186 module_param(pt_mode, int, S_IRUGO); 187 188 static DEFINE_STATIC_KEY_FALSE(vmx_l1d_should_flush); 189 static DEFINE_STATIC_KEY_FALSE(vmx_l1d_flush_cond); 190 static DEFINE_MUTEX(vmx_l1d_flush_mutex); 191 192 /* Storage for pre module init parameter parsing */ 193 static enum vmx_l1d_flush_state __read_mostly vmentry_l1d_flush_param = VMENTER_L1D_FLUSH_AUTO; 194 195 static const struct { 196 const char *option; 197 bool for_parse; 198 } vmentry_l1d_param[] = { 199 [VMENTER_L1D_FLUSH_AUTO] = {"auto", true}, 200 [VMENTER_L1D_FLUSH_NEVER] = {"never", true}, 201 [VMENTER_L1D_FLUSH_COND] = {"cond", true}, 202 [VMENTER_L1D_FLUSH_ALWAYS] = {"always", true}, 203 [VMENTER_L1D_FLUSH_EPT_DISABLED] = {"EPT disabled", false}, 204 [VMENTER_L1D_FLUSH_NOT_REQUIRED] = {"not required", false}, 205 }; 206 207 #define L1D_CACHE_ORDER 4 208 static void *vmx_l1d_flush_pages; 209 210 static int vmx_setup_l1d_flush(enum vmx_l1d_flush_state l1tf) 211 { 212 struct page *page; 213 unsigned int i; 214 215 if (!boot_cpu_has_bug(X86_BUG_L1TF)) { 216 l1tf_vmx_mitigation = VMENTER_L1D_FLUSH_NOT_REQUIRED; 217 return 0; 218 } 219 220 if (!enable_ept) { 221 l1tf_vmx_mitigation = VMENTER_L1D_FLUSH_EPT_DISABLED; 222 return 0; 223 } 224 225 if (boot_cpu_has(X86_FEATURE_ARCH_CAPABILITIES)) { 226 u64 msr; 227 228 rdmsrl(MSR_IA32_ARCH_CAPABILITIES, msr); 229 if (msr & ARCH_CAP_SKIP_VMENTRY_L1DFLUSH) { 230 l1tf_vmx_mitigation = VMENTER_L1D_FLUSH_NOT_REQUIRED; 231 return 0; 232 } 233 } 234 235 /* If set to auto use the default l1tf mitigation method */ 236 if (l1tf == VMENTER_L1D_FLUSH_AUTO) { 237 switch (l1tf_mitigation) { 238 case L1TF_MITIGATION_OFF: 239 l1tf = VMENTER_L1D_FLUSH_NEVER; 240 break; 241 case L1TF_MITIGATION_FLUSH_NOWARN: 242 case L1TF_MITIGATION_FLUSH: 243 case L1TF_MITIGATION_FLUSH_NOSMT: 244 l1tf = VMENTER_L1D_FLUSH_COND; 245 break; 246 case L1TF_MITIGATION_FULL: 247 case L1TF_MITIGATION_FULL_FORCE: 248 l1tf = VMENTER_L1D_FLUSH_ALWAYS; 249 break; 250 } 251 } else if (l1tf_mitigation == L1TF_MITIGATION_FULL_FORCE) { 252 l1tf = VMENTER_L1D_FLUSH_ALWAYS; 253 } 254 255 if (l1tf != VMENTER_L1D_FLUSH_NEVER && !vmx_l1d_flush_pages && 256 !boot_cpu_has(X86_FEATURE_FLUSH_L1D)) { 257 /* 258 * This allocation for vmx_l1d_flush_pages is not tied to a VM 259 * lifetime and so should not be charged to a memcg. 260 */ 261 page = alloc_pages(GFP_KERNEL, L1D_CACHE_ORDER); 262 if (!page) 263 return -ENOMEM; 264 vmx_l1d_flush_pages = page_address(page); 265 266 /* 267 * Initialize each page with a different pattern in 268 * order to protect against KSM in the nested 269 * virtualization case. 270 */ 271 for (i = 0; i < 1u << L1D_CACHE_ORDER; ++i) { 272 memset(vmx_l1d_flush_pages + i * PAGE_SIZE, i + 1, 273 PAGE_SIZE); 274 } 275 } 276 277 l1tf_vmx_mitigation = l1tf; 278 279 if (l1tf != VMENTER_L1D_FLUSH_NEVER) 280 static_branch_enable(&vmx_l1d_should_flush); 281 else 282 static_branch_disable(&vmx_l1d_should_flush); 283 284 if (l1tf == VMENTER_L1D_FLUSH_COND) 285 static_branch_enable(&vmx_l1d_flush_cond); 286 else 287 static_branch_disable(&vmx_l1d_flush_cond); 288 return 0; 289 } 290 291 static int vmentry_l1d_flush_parse(const char *s) 292 { 293 unsigned int i; 294 295 if (s) { 296 for (i = 0; i < ARRAY_SIZE(vmentry_l1d_param); i++) { 297 if (vmentry_l1d_param[i].for_parse && 298 sysfs_streq(s, vmentry_l1d_param[i].option)) 299 return i; 300 } 301 } 302 return -EINVAL; 303 } 304 305 static int vmentry_l1d_flush_set(const char *s, const struct kernel_param *kp) 306 { 307 int l1tf, ret; 308 309 l1tf = vmentry_l1d_flush_parse(s); 310 if (l1tf < 0) 311 return l1tf; 312 313 if (!boot_cpu_has(X86_BUG_L1TF)) 314 return 0; 315 316 /* 317 * Has vmx_init() run already? If not then this is the pre init 318 * parameter parsing. In that case just store the value and let 319 * vmx_init() do the proper setup after enable_ept has been 320 * established. 321 */ 322 if (l1tf_vmx_mitigation == VMENTER_L1D_FLUSH_AUTO) { 323 vmentry_l1d_flush_param = l1tf; 324 return 0; 325 } 326 327 mutex_lock(&vmx_l1d_flush_mutex); 328 ret = vmx_setup_l1d_flush(l1tf); 329 mutex_unlock(&vmx_l1d_flush_mutex); 330 return ret; 331 } 332 333 static int vmentry_l1d_flush_get(char *s, const struct kernel_param *kp) 334 { 335 if (WARN_ON_ONCE(l1tf_vmx_mitigation >= ARRAY_SIZE(vmentry_l1d_param))) 336 return sprintf(s, "???\n"); 337 338 return sprintf(s, "%s\n", vmentry_l1d_param[l1tf_vmx_mitigation].option); 339 } 340 341 static const struct kernel_param_ops vmentry_l1d_flush_ops = { 342 .set = vmentry_l1d_flush_set, 343 .get = vmentry_l1d_flush_get, 344 }; 345 module_param_cb(vmentry_l1d_flush, &vmentry_l1d_flush_ops, NULL, 0644); 346 347 static bool guest_state_valid(struct kvm_vcpu *vcpu); 348 static u32 vmx_segment_access_rights(struct kvm_segment *var); 349 static __always_inline void vmx_disable_intercept_for_msr(unsigned long *msr_bitmap, 350 u32 msr, int type); 351 352 void vmx_vmexit(void); 353 354 #define vmx_insn_failed(fmt...) \ 355 do { \ 356 WARN_ONCE(1, fmt); \ 357 pr_warn_ratelimited(fmt); \ 358 } while (0) 359 360 asmlinkage void vmread_error(unsigned long field, bool fault) 361 { 362 if (fault) 363 kvm_spurious_fault(); 364 else 365 vmx_insn_failed("kvm: vmread failed: field=%lx\n", field); 366 } 367 368 noinline void vmwrite_error(unsigned long field, unsigned long value) 369 { 370 vmx_insn_failed("kvm: vmwrite failed: field=%lx val=%lx err=%d\n", 371 field, value, vmcs_read32(VM_INSTRUCTION_ERROR)); 372 } 373 374 noinline void vmclear_error(struct vmcs *vmcs, u64 phys_addr) 375 { 376 vmx_insn_failed("kvm: vmclear failed: %p/%llx\n", vmcs, phys_addr); 377 } 378 379 noinline void vmptrld_error(struct vmcs *vmcs, u64 phys_addr) 380 { 381 vmx_insn_failed("kvm: vmptrld failed: %p/%llx\n", vmcs, phys_addr); 382 } 383 384 noinline void invvpid_error(unsigned long ext, u16 vpid, gva_t gva) 385 { 386 vmx_insn_failed("kvm: invvpid failed: ext=0x%lx vpid=%u gva=0x%lx\n", 387 ext, vpid, gva); 388 } 389 390 noinline void invept_error(unsigned long ext, u64 eptp, gpa_t gpa) 391 { 392 vmx_insn_failed("kvm: invept failed: ext=0x%lx eptp=%llx gpa=0x%llx\n", 393 ext, eptp, gpa); 394 } 395 396 static DEFINE_PER_CPU(struct vmcs *, vmxarea); 397 DEFINE_PER_CPU(struct vmcs *, current_vmcs); 398 /* 399 * We maintain a per-CPU linked-list of VMCS loaded on that CPU. This is needed 400 * when a CPU is brought down, and we need to VMCLEAR all VMCSs loaded on it. 401 */ 402 static DEFINE_PER_CPU(struct list_head, loaded_vmcss_on_cpu); 403 404 /* 405 * We maintian a per-CPU linked-list of vCPU, so in wakeup_handler() we 406 * can find which vCPU should be waken up. 407 */ 408 static DEFINE_PER_CPU(struct list_head, blocked_vcpu_on_cpu); 409 static DEFINE_PER_CPU(spinlock_t, blocked_vcpu_on_cpu_lock); 410 411 static DECLARE_BITMAP(vmx_vpid_bitmap, VMX_NR_VPIDS); 412 static DEFINE_SPINLOCK(vmx_vpid_lock); 413 414 struct vmcs_config vmcs_config; 415 struct vmx_capability vmx_capability; 416 417 #define VMX_SEGMENT_FIELD(seg) \ 418 [VCPU_SREG_##seg] = { \ 419 .selector = GUEST_##seg##_SELECTOR, \ 420 .base = GUEST_##seg##_BASE, \ 421 .limit = GUEST_##seg##_LIMIT, \ 422 .ar_bytes = GUEST_##seg##_AR_BYTES, \ 423 } 424 425 static const struct kvm_vmx_segment_field { 426 unsigned selector; 427 unsigned base; 428 unsigned limit; 429 unsigned ar_bytes; 430 } kvm_vmx_segment_fields[] = { 431 VMX_SEGMENT_FIELD(CS), 432 VMX_SEGMENT_FIELD(DS), 433 VMX_SEGMENT_FIELD(ES), 434 VMX_SEGMENT_FIELD(FS), 435 VMX_SEGMENT_FIELD(GS), 436 VMX_SEGMENT_FIELD(SS), 437 VMX_SEGMENT_FIELD(TR), 438 VMX_SEGMENT_FIELD(LDTR), 439 }; 440 441 static inline void vmx_segment_cache_clear(struct vcpu_vmx *vmx) 442 { 443 vmx->segment_cache.bitmask = 0; 444 } 445 446 static unsigned long host_idt_base; 447 448 /* 449 * Though SYSCALL is only supported in 64-bit mode on Intel CPUs, kvm 450 * will emulate SYSCALL in legacy mode if the vendor string in guest 451 * CPUID.0:{EBX,ECX,EDX} is "AuthenticAMD" or "AMDisbetter!" To 452 * support this emulation, IA32_STAR must always be included in 453 * vmx_msr_index[], even in i386 builds. 454 */ 455 const u32 vmx_msr_index[] = { 456 #ifdef CONFIG_X86_64 457 MSR_SYSCALL_MASK, MSR_LSTAR, MSR_CSTAR, 458 #endif 459 MSR_EFER, MSR_TSC_AUX, MSR_STAR, 460 MSR_IA32_TSX_CTRL, 461 }; 462 463 #if IS_ENABLED(CONFIG_HYPERV) 464 static bool __read_mostly enlightened_vmcs = true; 465 module_param(enlightened_vmcs, bool, 0444); 466 467 /* check_ept_pointer() should be under protection of ept_pointer_lock. */ 468 static void check_ept_pointer_match(struct kvm *kvm) 469 { 470 struct kvm_vcpu *vcpu; 471 u64 tmp_eptp = INVALID_PAGE; 472 int i; 473 474 kvm_for_each_vcpu(i, vcpu, kvm) { 475 if (!VALID_PAGE(tmp_eptp)) { 476 tmp_eptp = to_vmx(vcpu)->ept_pointer; 477 } else if (tmp_eptp != to_vmx(vcpu)->ept_pointer) { 478 to_kvm_vmx(kvm)->ept_pointers_match 479 = EPT_POINTERS_MISMATCH; 480 return; 481 } 482 } 483 484 to_kvm_vmx(kvm)->ept_pointers_match = EPT_POINTERS_MATCH; 485 } 486 487 static int kvm_fill_hv_flush_list_func(struct hv_guest_mapping_flush_list *flush, 488 void *data) 489 { 490 struct kvm_tlb_range *range = data; 491 492 return hyperv_fill_flush_guest_mapping_list(flush, range->start_gfn, 493 range->pages); 494 } 495 496 static inline int __hv_remote_flush_tlb_with_range(struct kvm *kvm, 497 struct kvm_vcpu *vcpu, struct kvm_tlb_range *range) 498 { 499 u64 ept_pointer = to_vmx(vcpu)->ept_pointer; 500 501 /* 502 * FLUSH_GUEST_PHYSICAL_ADDRESS_SPACE hypercall needs address 503 * of the base of EPT PML4 table, strip off EPT configuration 504 * information. 505 */ 506 if (range) 507 return hyperv_flush_guest_mapping_range(ept_pointer & PAGE_MASK, 508 kvm_fill_hv_flush_list_func, (void *)range); 509 else 510 return hyperv_flush_guest_mapping(ept_pointer & PAGE_MASK); 511 } 512 513 static int hv_remote_flush_tlb_with_range(struct kvm *kvm, 514 struct kvm_tlb_range *range) 515 { 516 struct kvm_vcpu *vcpu; 517 int ret = 0, i; 518 519 spin_lock(&to_kvm_vmx(kvm)->ept_pointer_lock); 520 521 if (to_kvm_vmx(kvm)->ept_pointers_match == EPT_POINTERS_CHECK) 522 check_ept_pointer_match(kvm); 523 524 if (to_kvm_vmx(kvm)->ept_pointers_match != EPT_POINTERS_MATCH) { 525 kvm_for_each_vcpu(i, vcpu, kvm) { 526 /* If ept_pointer is invalid pointer, bypass flush request. */ 527 if (VALID_PAGE(to_vmx(vcpu)->ept_pointer)) 528 ret |= __hv_remote_flush_tlb_with_range( 529 kvm, vcpu, range); 530 } 531 } else { 532 ret = __hv_remote_flush_tlb_with_range(kvm, 533 kvm_get_vcpu(kvm, 0), range); 534 } 535 536 spin_unlock(&to_kvm_vmx(kvm)->ept_pointer_lock); 537 return ret; 538 } 539 static int hv_remote_flush_tlb(struct kvm *kvm) 540 { 541 return hv_remote_flush_tlb_with_range(kvm, NULL); 542 } 543 544 static int hv_enable_direct_tlbflush(struct kvm_vcpu *vcpu) 545 { 546 struct hv_enlightened_vmcs *evmcs; 547 struct hv_partition_assist_pg **p_hv_pa_pg = 548 &vcpu->kvm->arch.hyperv.hv_pa_pg; 549 /* 550 * Synthetic VM-Exit is not enabled in current code and so All 551 * evmcs in singe VM shares same assist page. 552 */ 553 if (!*p_hv_pa_pg) 554 *p_hv_pa_pg = kzalloc(PAGE_SIZE, GFP_KERNEL); 555 556 if (!*p_hv_pa_pg) 557 return -ENOMEM; 558 559 evmcs = (struct hv_enlightened_vmcs *)to_vmx(vcpu)->loaded_vmcs->vmcs; 560 561 evmcs->partition_assist_page = 562 __pa(*p_hv_pa_pg); 563 evmcs->hv_vm_id = (unsigned long)vcpu->kvm; 564 evmcs->hv_enlightenments_control.nested_flush_hypercall = 1; 565 566 return 0; 567 } 568 569 #endif /* IS_ENABLED(CONFIG_HYPERV) */ 570 571 /* 572 * Comment's format: document - errata name - stepping - processor name. 573 * Refer from 574 * https://www.virtualbox.org/svn/vbox/trunk/src/VBox/VMM/VMMR0/HMR0.cpp 575 */ 576 static u32 vmx_preemption_cpu_tfms[] = { 577 /* 323344.pdf - BA86 - D0 - Xeon 7500 Series */ 578 0x000206E6, 579 /* 323056.pdf - AAX65 - C2 - Xeon L3406 */ 580 /* 322814.pdf - AAT59 - C2 - i7-600, i5-500, i5-400 and i3-300 Mobile */ 581 /* 322911.pdf - AAU65 - C2 - i5-600, i3-500 Desktop and Pentium G6950 */ 582 0x00020652, 583 /* 322911.pdf - AAU65 - K0 - i5-600, i3-500 Desktop and Pentium G6950 */ 584 0x00020655, 585 /* 322373.pdf - AAO95 - B1 - Xeon 3400 Series */ 586 /* 322166.pdf - AAN92 - B1 - i7-800 and i5-700 Desktop */ 587 /* 588 * 320767.pdf - AAP86 - B1 - 589 * i7-900 Mobile Extreme, i7-800 and i7-700 Mobile 590 */ 591 0x000106E5, 592 /* 321333.pdf - AAM126 - C0 - Xeon 3500 */ 593 0x000106A0, 594 /* 321333.pdf - AAM126 - C1 - Xeon 3500 */ 595 0x000106A1, 596 /* 320836.pdf - AAJ124 - C0 - i7-900 Desktop Extreme and i7-900 Desktop */ 597 0x000106A4, 598 /* 321333.pdf - AAM126 - D0 - Xeon 3500 */ 599 /* 321324.pdf - AAK139 - D0 - Xeon 5500 */ 600 /* 320836.pdf - AAJ124 - D0 - i7-900 Extreme and i7-900 Desktop */ 601 0x000106A5, 602 /* Xeon E3-1220 V2 */ 603 0x000306A8, 604 }; 605 606 static inline bool cpu_has_broken_vmx_preemption_timer(void) 607 { 608 u32 eax = cpuid_eax(0x00000001), i; 609 610 /* Clear the reserved bits */ 611 eax &= ~(0x3U << 14 | 0xfU << 28); 612 for (i = 0; i < ARRAY_SIZE(vmx_preemption_cpu_tfms); i++) 613 if (eax == vmx_preemption_cpu_tfms[i]) 614 return true; 615 616 return false; 617 } 618 619 static inline bool cpu_need_virtualize_apic_accesses(struct kvm_vcpu *vcpu) 620 { 621 return flexpriority_enabled && lapic_in_kernel(vcpu); 622 } 623 624 static inline bool report_flexpriority(void) 625 { 626 return flexpriority_enabled; 627 } 628 629 static inline int __find_msr_index(struct vcpu_vmx *vmx, u32 msr) 630 { 631 int i; 632 633 for (i = 0; i < vmx->nmsrs; ++i) 634 if (vmx_msr_index[vmx->guest_msrs[i].index] == msr) 635 return i; 636 return -1; 637 } 638 639 struct shared_msr_entry *find_msr_entry(struct vcpu_vmx *vmx, u32 msr) 640 { 641 int i; 642 643 i = __find_msr_index(vmx, msr); 644 if (i >= 0) 645 return &vmx->guest_msrs[i]; 646 return NULL; 647 } 648 649 static int vmx_set_guest_msr(struct vcpu_vmx *vmx, struct shared_msr_entry *msr, u64 data) 650 { 651 int ret = 0; 652 653 u64 old_msr_data = msr->data; 654 msr->data = data; 655 if (msr - vmx->guest_msrs < vmx->save_nmsrs) { 656 preempt_disable(); 657 ret = kvm_set_shared_msr(msr->index, msr->data, 658 msr->mask); 659 preempt_enable(); 660 if (ret) 661 msr->data = old_msr_data; 662 } 663 return ret; 664 } 665 666 #ifdef CONFIG_KEXEC_CORE 667 static void crash_vmclear_local_loaded_vmcss(void) 668 { 669 int cpu = raw_smp_processor_id(); 670 struct loaded_vmcs *v; 671 672 list_for_each_entry(v, &per_cpu(loaded_vmcss_on_cpu, cpu), 673 loaded_vmcss_on_cpu_link) 674 vmcs_clear(v->vmcs); 675 } 676 #endif /* CONFIG_KEXEC_CORE */ 677 678 static void __loaded_vmcs_clear(void *arg) 679 { 680 struct loaded_vmcs *loaded_vmcs = arg; 681 int cpu = raw_smp_processor_id(); 682 683 if (loaded_vmcs->cpu != cpu) 684 return; /* vcpu migration can race with cpu offline */ 685 if (per_cpu(current_vmcs, cpu) == loaded_vmcs->vmcs) 686 per_cpu(current_vmcs, cpu) = NULL; 687 688 vmcs_clear(loaded_vmcs->vmcs); 689 if (loaded_vmcs->shadow_vmcs && loaded_vmcs->launched) 690 vmcs_clear(loaded_vmcs->shadow_vmcs); 691 692 list_del(&loaded_vmcs->loaded_vmcss_on_cpu_link); 693 694 /* 695 * Ensure all writes to loaded_vmcs, including deleting it from its 696 * current percpu list, complete before setting loaded_vmcs->vcpu to 697 * -1, otherwise a different cpu can see vcpu == -1 first and add 698 * loaded_vmcs to its percpu list before it's deleted from this cpu's 699 * list. Pairs with the smp_rmb() in vmx_vcpu_load_vmcs(). 700 */ 701 smp_wmb(); 702 703 loaded_vmcs->cpu = -1; 704 loaded_vmcs->launched = 0; 705 } 706 707 void loaded_vmcs_clear(struct loaded_vmcs *loaded_vmcs) 708 { 709 int cpu = loaded_vmcs->cpu; 710 711 if (cpu != -1) 712 smp_call_function_single(cpu, 713 __loaded_vmcs_clear, loaded_vmcs, 1); 714 } 715 716 static bool vmx_segment_cache_test_set(struct vcpu_vmx *vmx, unsigned seg, 717 unsigned field) 718 { 719 bool ret; 720 u32 mask = 1 << (seg * SEG_FIELD_NR + field); 721 722 if (!kvm_register_is_available(&vmx->vcpu, VCPU_EXREG_SEGMENTS)) { 723 kvm_register_mark_available(&vmx->vcpu, VCPU_EXREG_SEGMENTS); 724 vmx->segment_cache.bitmask = 0; 725 } 726 ret = vmx->segment_cache.bitmask & mask; 727 vmx->segment_cache.bitmask |= mask; 728 return ret; 729 } 730 731 static u16 vmx_read_guest_seg_selector(struct vcpu_vmx *vmx, unsigned seg) 732 { 733 u16 *p = &vmx->segment_cache.seg[seg].selector; 734 735 if (!vmx_segment_cache_test_set(vmx, seg, SEG_FIELD_SEL)) 736 *p = vmcs_read16(kvm_vmx_segment_fields[seg].selector); 737 return *p; 738 } 739 740 static ulong vmx_read_guest_seg_base(struct vcpu_vmx *vmx, unsigned seg) 741 { 742 ulong *p = &vmx->segment_cache.seg[seg].base; 743 744 if (!vmx_segment_cache_test_set(vmx, seg, SEG_FIELD_BASE)) 745 *p = vmcs_readl(kvm_vmx_segment_fields[seg].base); 746 return *p; 747 } 748 749 static u32 vmx_read_guest_seg_limit(struct vcpu_vmx *vmx, unsigned seg) 750 { 751 u32 *p = &vmx->segment_cache.seg[seg].limit; 752 753 if (!vmx_segment_cache_test_set(vmx, seg, SEG_FIELD_LIMIT)) 754 *p = vmcs_read32(kvm_vmx_segment_fields[seg].limit); 755 return *p; 756 } 757 758 static u32 vmx_read_guest_seg_ar(struct vcpu_vmx *vmx, unsigned seg) 759 { 760 u32 *p = &vmx->segment_cache.seg[seg].ar; 761 762 if (!vmx_segment_cache_test_set(vmx, seg, SEG_FIELD_AR)) 763 *p = vmcs_read32(kvm_vmx_segment_fields[seg].ar_bytes); 764 return *p; 765 } 766 767 void update_exception_bitmap(struct kvm_vcpu *vcpu) 768 { 769 u32 eb; 770 771 eb = (1u << PF_VECTOR) | (1u << UD_VECTOR) | (1u << MC_VECTOR) | 772 (1u << DB_VECTOR) | (1u << AC_VECTOR); 773 /* 774 * Guest access to VMware backdoor ports could legitimately 775 * trigger #GP because of TSS I/O permission bitmap. 776 * We intercept those #GP and allow access to them anyway 777 * as VMware does. 778 */ 779 if (enable_vmware_backdoor) 780 eb |= (1u << GP_VECTOR); 781 if ((vcpu->guest_debug & 782 (KVM_GUESTDBG_ENABLE | KVM_GUESTDBG_USE_SW_BP)) == 783 (KVM_GUESTDBG_ENABLE | KVM_GUESTDBG_USE_SW_BP)) 784 eb |= 1u << BP_VECTOR; 785 if (to_vmx(vcpu)->rmode.vm86_active) 786 eb = ~0; 787 if (!vmx_need_pf_intercept(vcpu)) 788 eb &= ~(1u << PF_VECTOR); 789 790 /* When we are running a nested L2 guest and L1 specified for it a 791 * certain exception bitmap, we must trap the same exceptions and pass 792 * them to L1. When running L2, we will only handle the exceptions 793 * specified above if L1 did not want them. 794 */ 795 if (is_guest_mode(vcpu)) 796 eb |= get_vmcs12(vcpu)->exception_bitmap; 797 else { 798 /* 799 * If EPT is enabled, #PF is only trapped if MAXPHYADDR is mismatched 800 * between guest and host. In that case we only care about present 801 * faults. For vmcs02, however, PFEC_MASK and PFEC_MATCH are set in 802 * prepare_vmcs02_rare. 803 */ 804 bool selective_pf_trap = enable_ept && (eb & (1u << PF_VECTOR)); 805 int mask = selective_pf_trap ? PFERR_PRESENT_MASK : 0; 806 vmcs_write32(PAGE_FAULT_ERROR_CODE_MASK, mask); 807 vmcs_write32(PAGE_FAULT_ERROR_CODE_MATCH, mask); 808 } 809 810 vmcs_write32(EXCEPTION_BITMAP, eb); 811 } 812 813 /* 814 * Check if MSR is intercepted for currently loaded MSR bitmap. 815 */ 816 static bool msr_write_intercepted(struct kvm_vcpu *vcpu, u32 msr) 817 { 818 unsigned long *msr_bitmap; 819 int f = sizeof(unsigned long); 820 821 if (!cpu_has_vmx_msr_bitmap()) 822 return true; 823 824 msr_bitmap = to_vmx(vcpu)->loaded_vmcs->msr_bitmap; 825 826 if (msr <= 0x1fff) { 827 return !!test_bit(msr, msr_bitmap + 0x800 / f); 828 } else if ((msr >= 0xc0000000) && (msr <= 0xc0001fff)) { 829 msr &= 0x1fff; 830 return !!test_bit(msr, msr_bitmap + 0xc00 / f); 831 } 832 833 return true; 834 } 835 836 static void clear_atomic_switch_msr_special(struct vcpu_vmx *vmx, 837 unsigned long entry, unsigned long exit) 838 { 839 vm_entry_controls_clearbit(vmx, entry); 840 vm_exit_controls_clearbit(vmx, exit); 841 } 842 843 int vmx_find_msr_index(struct vmx_msrs *m, u32 msr) 844 { 845 unsigned int i; 846 847 for (i = 0; i < m->nr; ++i) { 848 if (m->val[i].index == msr) 849 return i; 850 } 851 return -ENOENT; 852 } 853 854 static void clear_atomic_switch_msr(struct vcpu_vmx *vmx, unsigned msr) 855 { 856 int i; 857 struct msr_autoload *m = &vmx->msr_autoload; 858 859 switch (msr) { 860 case MSR_EFER: 861 if (cpu_has_load_ia32_efer()) { 862 clear_atomic_switch_msr_special(vmx, 863 VM_ENTRY_LOAD_IA32_EFER, 864 VM_EXIT_LOAD_IA32_EFER); 865 return; 866 } 867 break; 868 case MSR_CORE_PERF_GLOBAL_CTRL: 869 if (cpu_has_load_perf_global_ctrl()) { 870 clear_atomic_switch_msr_special(vmx, 871 VM_ENTRY_LOAD_IA32_PERF_GLOBAL_CTRL, 872 VM_EXIT_LOAD_IA32_PERF_GLOBAL_CTRL); 873 return; 874 } 875 break; 876 } 877 i = vmx_find_msr_index(&m->guest, msr); 878 if (i < 0) 879 goto skip_guest; 880 --m->guest.nr; 881 m->guest.val[i] = m->guest.val[m->guest.nr]; 882 vmcs_write32(VM_ENTRY_MSR_LOAD_COUNT, m->guest.nr); 883 884 skip_guest: 885 i = vmx_find_msr_index(&m->host, msr); 886 if (i < 0) 887 return; 888 889 --m->host.nr; 890 m->host.val[i] = m->host.val[m->host.nr]; 891 vmcs_write32(VM_EXIT_MSR_LOAD_COUNT, m->host.nr); 892 } 893 894 static void add_atomic_switch_msr_special(struct vcpu_vmx *vmx, 895 unsigned long entry, unsigned long exit, 896 unsigned long guest_val_vmcs, unsigned long host_val_vmcs, 897 u64 guest_val, u64 host_val) 898 { 899 vmcs_write64(guest_val_vmcs, guest_val); 900 if (host_val_vmcs != HOST_IA32_EFER) 901 vmcs_write64(host_val_vmcs, host_val); 902 vm_entry_controls_setbit(vmx, entry); 903 vm_exit_controls_setbit(vmx, exit); 904 } 905 906 static void add_atomic_switch_msr(struct vcpu_vmx *vmx, unsigned msr, 907 u64 guest_val, u64 host_val, bool entry_only) 908 { 909 int i, j = 0; 910 struct msr_autoload *m = &vmx->msr_autoload; 911 912 switch (msr) { 913 case MSR_EFER: 914 if (cpu_has_load_ia32_efer()) { 915 add_atomic_switch_msr_special(vmx, 916 VM_ENTRY_LOAD_IA32_EFER, 917 VM_EXIT_LOAD_IA32_EFER, 918 GUEST_IA32_EFER, 919 HOST_IA32_EFER, 920 guest_val, host_val); 921 return; 922 } 923 break; 924 case MSR_CORE_PERF_GLOBAL_CTRL: 925 if (cpu_has_load_perf_global_ctrl()) { 926 add_atomic_switch_msr_special(vmx, 927 VM_ENTRY_LOAD_IA32_PERF_GLOBAL_CTRL, 928 VM_EXIT_LOAD_IA32_PERF_GLOBAL_CTRL, 929 GUEST_IA32_PERF_GLOBAL_CTRL, 930 HOST_IA32_PERF_GLOBAL_CTRL, 931 guest_val, host_val); 932 return; 933 } 934 break; 935 case MSR_IA32_PEBS_ENABLE: 936 /* PEBS needs a quiescent period after being disabled (to write 937 * a record). Disabling PEBS through VMX MSR swapping doesn't 938 * provide that period, so a CPU could write host's record into 939 * guest's memory. 940 */ 941 wrmsrl(MSR_IA32_PEBS_ENABLE, 0); 942 } 943 944 i = vmx_find_msr_index(&m->guest, msr); 945 if (!entry_only) 946 j = vmx_find_msr_index(&m->host, msr); 947 948 if ((i < 0 && m->guest.nr == NR_LOADSTORE_MSRS) || 949 (j < 0 && m->host.nr == NR_LOADSTORE_MSRS)) { 950 printk_once(KERN_WARNING "Not enough msr switch entries. " 951 "Can't add msr %x\n", msr); 952 return; 953 } 954 if (i < 0) { 955 i = m->guest.nr++; 956 vmcs_write32(VM_ENTRY_MSR_LOAD_COUNT, m->guest.nr); 957 } 958 m->guest.val[i].index = msr; 959 m->guest.val[i].value = guest_val; 960 961 if (entry_only) 962 return; 963 964 if (j < 0) { 965 j = m->host.nr++; 966 vmcs_write32(VM_EXIT_MSR_LOAD_COUNT, m->host.nr); 967 } 968 m->host.val[j].index = msr; 969 m->host.val[j].value = host_val; 970 } 971 972 static bool update_transition_efer(struct vcpu_vmx *vmx, int efer_offset) 973 { 974 u64 guest_efer = vmx->vcpu.arch.efer; 975 u64 ignore_bits = 0; 976 977 /* Shadow paging assumes NX to be available. */ 978 if (!enable_ept) 979 guest_efer |= EFER_NX; 980 981 /* 982 * LMA and LME handled by hardware; SCE meaningless outside long mode. 983 */ 984 ignore_bits |= EFER_SCE; 985 #ifdef CONFIG_X86_64 986 ignore_bits |= EFER_LMA | EFER_LME; 987 /* SCE is meaningful only in long mode on Intel */ 988 if (guest_efer & EFER_LMA) 989 ignore_bits &= ~(u64)EFER_SCE; 990 #endif 991 992 /* 993 * On EPT, we can't emulate NX, so we must switch EFER atomically. 994 * On CPUs that support "load IA32_EFER", always switch EFER 995 * atomically, since it's faster than switching it manually. 996 */ 997 if (cpu_has_load_ia32_efer() || 998 (enable_ept && ((vmx->vcpu.arch.efer ^ host_efer) & EFER_NX))) { 999 if (!(guest_efer & EFER_LMA)) 1000 guest_efer &= ~EFER_LME; 1001 if (guest_efer != host_efer) 1002 add_atomic_switch_msr(vmx, MSR_EFER, 1003 guest_efer, host_efer, false); 1004 else 1005 clear_atomic_switch_msr(vmx, MSR_EFER); 1006 return false; 1007 } else { 1008 clear_atomic_switch_msr(vmx, MSR_EFER); 1009 1010 guest_efer &= ~ignore_bits; 1011 guest_efer |= host_efer & ignore_bits; 1012 1013 vmx->guest_msrs[efer_offset].data = guest_efer; 1014 vmx->guest_msrs[efer_offset].mask = ~ignore_bits; 1015 1016 return true; 1017 } 1018 } 1019 1020 #ifdef CONFIG_X86_32 1021 /* 1022 * On 32-bit kernels, VM exits still load the FS and GS bases from the 1023 * VMCS rather than the segment table. KVM uses this helper to figure 1024 * out the current bases to poke them into the VMCS before entry. 1025 */ 1026 static unsigned long segment_base(u16 selector) 1027 { 1028 struct desc_struct *table; 1029 unsigned long v; 1030 1031 if (!(selector & ~SEGMENT_RPL_MASK)) 1032 return 0; 1033 1034 table = get_current_gdt_ro(); 1035 1036 if ((selector & SEGMENT_TI_MASK) == SEGMENT_LDT) { 1037 u16 ldt_selector = kvm_read_ldt(); 1038 1039 if (!(ldt_selector & ~SEGMENT_RPL_MASK)) 1040 return 0; 1041 1042 table = (struct desc_struct *)segment_base(ldt_selector); 1043 } 1044 v = get_desc_base(&table[selector >> 3]); 1045 return v; 1046 } 1047 #endif 1048 1049 static inline bool pt_can_write_msr(struct vcpu_vmx *vmx) 1050 { 1051 return vmx_pt_mode_is_host_guest() && 1052 !(vmx->pt_desc.guest.ctl & RTIT_CTL_TRACEEN); 1053 } 1054 1055 static inline void pt_load_msr(struct pt_ctx *ctx, u32 addr_range) 1056 { 1057 u32 i; 1058 1059 wrmsrl(MSR_IA32_RTIT_STATUS, ctx->status); 1060 wrmsrl(MSR_IA32_RTIT_OUTPUT_BASE, ctx->output_base); 1061 wrmsrl(MSR_IA32_RTIT_OUTPUT_MASK, ctx->output_mask); 1062 wrmsrl(MSR_IA32_RTIT_CR3_MATCH, ctx->cr3_match); 1063 for (i = 0; i < addr_range; i++) { 1064 wrmsrl(MSR_IA32_RTIT_ADDR0_A + i * 2, ctx->addr_a[i]); 1065 wrmsrl(MSR_IA32_RTIT_ADDR0_B + i * 2, ctx->addr_b[i]); 1066 } 1067 } 1068 1069 static inline void pt_save_msr(struct pt_ctx *ctx, u32 addr_range) 1070 { 1071 u32 i; 1072 1073 rdmsrl(MSR_IA32_RTIT_STATUS, ctx->status); 1074 rdmsrl(MSR_IA32_RTIT_OUTPUT_BASE, ctx->output_base); 1075 rdmsrl(MSR_IA32_RTIT_OUTPUT_MASK, ctx->output_mask); 1076 rdmsrl(MSR_IA32_RTIT_CR3_MATCH, ctx->cr3_match); 1077 for (i = 0; i < addr_range; i++) { 1078 rdmsrl(MSR_IA32_RTIT_ADDR0_A + i * 2, ctx->addr_a[i]); 1079 rdmsrl(MSR_IA32_RTIT_ADDR0_B + i * 2, ctx->addr_b[i]); 1080 } 1081 } 1082 1083 static void pt_guest_enter(struct vcpu_vmx *vmx) 1084 { 1085 if (vmx_pt_mode_is_system()) 1086 return; 1087 1088 /* 1089 * GUEST_IA32_RTIT_CTL is already set in the VMCS. 1090 * Save host state before VM entry. 1091 */ 1092 rdmsrl(MSR_IA32_RTIT_CTL, vmx->pt_desc.host.ctl); 1093 if (vmx->pt_desc.guest.ctl & RTIT_CTL_TRACEEN) { 1094 wrmsrl(MSR_IA32_RTIT_CTL, 0); 1095 pt_save_msr(&vmx->pt_desc.host, vmx->pt_desc.addr_range); 1096 pt_load_msr(&vmx->pt_desc.guest, vmx->pt_desc.addr_range); 1097 } 1098 } 1099 1100 static void pt_guest_exit(struct vcpu_vmx *vmx) 1101 { 1102 if (vmx_pt_mode_is_system()) 1103 return; 1104 1105 if (vmx->pt_desc.guest.ctl & RTIT_CTL_TRACEEN) { 1106 pt_save_msr(&vmx->pt_desc.guest, vmx->pt_desc.addr_range); 1107 pt_load_msr(&vmx->pt_desc.host, vmx->pt_desc.addr_range); 1108 } 1109 1110 /* Reload host state (IA32_RTIT_CTL will be cleared on VM exit). */ 1111 wrmsrl(MSR_IA32_RTIT_CTL, vmx->pt_desc.host.ctl); 1112 } 1113 1114 void vmx_set_host_fs_gs(struct vmcs_host_state *host, u16 fs_sel, u16 gs_sel, 1115 unsigned long fs_base, unsigned long gs_base) 1116 { 1117 if (unlikely(fs_sel != host->fs_sel)) { 1118 if (!(fs_sel & 7)) 1119 vmcs_write16(HOST_FS_SELECTOR, fs_sel); 1120 else 1121 vmcs_write16(HOST_FS_SELECTOR, 0); 1122 host->fs_sel = fs_sel; 1123 } 1124 if (unlikely(gs_sel != host->gs_sel)) { 1125 if (!(gs_sel & 7)) 1126 vmcs_write16(HOST_GS_SELECTOR, gs_sel); 1127 else 1128 vmcs_write16(HOST_GS_SELECTOR, 0); 1129 host->gs_sel = gs_sel; 1130 } 1131 if (unlikely(fs_base != host->fs_base)) { 1132 vmcs_writel(HOST_FS_BASE, fs_base); 1133 host->fs_base = fs_base; 1134 } 1135 if (unlikely(gs_base != host->gs_base)) { 1136 vmcs_writel(HOST_GS_BASE, gs_base); 1137 host->gs_base = gs_base; 1138 } 1139 } 1140 1141 void vmx_prepare_switch_to_guest(struct kvm_vcpu *vcpu) 1142 { 1143 struct vcpu_vmx *vmx = to_vmx(vcpu); 1144 struct vmcs_host_state *host_state; 1145 #ifdef CONFIG_X86_64 1146 int cpu = raw_smp_processor_id(); 1147 #endif 1148 unsigned long fs_base, gs_base; 1149 u16 fs_sel, gs_sel; 1150 int i; 1151 1152 vmx->req_immediate_exit = false; 1153 1154 /* 1155 * Note that guest MSRs to be saved/restored can also be changed 1156 * when guest state is loaded. This happens when guest transitions 1157 * to/from long-mode by setting MSR_EFER.LMA. 1158 */ 1159 if (!vmx->guest_msrs_ready) { 1160 vmx->guest_msrs_ready = true; 1161 for (i = 0; i < vmx->save_nmsrs; ++i) 1162 kvm_set_shared_msr(vmx->guest_msrs[i].index, 1163 vmx->guest_msrs[i].data, 1164 vmx->guest_msrs[i].mask); 1165 1166 } 1167 1168 if (vmx->nested.need_vmcs12_to_shadow_sync) 1169 nested_sync_vmcs12_to_shadow(vcpu); 1170 1171 if (vmx->guest_state_loaded) 1172 return; 1173 1174 host_state = &vmx->loaded_vmcs->host_state; 1175 1176 /* 1177 * Set host fs and gs selectors. Unfortunately, 22.2.3 does not 1178 * allow segment selectors with cpl > 0 or ti == 1. 1179 */ 1180 host_state->ldt_sel = kvm_read_ldt(); 1181 1182 #ifdef CONFIG_X86_64 1183 savesegment(ds, host_state->ds_sel); 1184 savesegment(es, host_state->es_sel); 1185 1186 gs_base = cpu_kernelmode_gs_base(cpu); 1187 if (likely(is_64bit_mm(current->mm))) { 1188 current_save_fsgs(); 1189 fs_sel = current->thread.fsindex; 1190 gs_sel = current->thread.gsindex; 1191 fs_base = current->thread.fsbase; 1192 vmx->msr_host_kernel_gs_base = current->thread.gsbase; 1193 } else { 1194 savesegment(fs, fs_sel); 1195 savesegment(gs, gs_sel); 1196 fs_base = read_msr(MSR_FS_BASE); 1197 vmx->msr_host_kernel_gs_base = read_msr(MSR_KERNEL_GS_BASE); 1198 } 1199 1200 wrmsrl(MSR_KERNEL_GS_BASE, vmx->msr_guest_kernel_gs_base); 1201 #else 1202 savesegment(fs, fs_sel); 1203 savesegment(gs, gs_sel); 1204 fs_base = segment_base(fs_sel); 1205 gs_base = segment_base(gs_sel); 1206 #endif 1207 1208 vmx_set_host_fs_gs(host_state, fs_sel, gs_sel, fs_base, gs_base); 1209 vmx->guest_state_loaded = true; 1210 } 1211 1212 static void vmx_prepare_switch_to_host(struct vcpu_vmx *vmx) 1213 { 1214 struct vmcs_host_state *host_state; 1215 1216 if (!vmx->guest_state_loaded) 1217 return; 1218 1219 host_state = &vmx->loaded_vmcs->host_state; 1220 1221 ++vmx->vcpu.stat.host_state_reload; 1222 1223 #ifdef CONFIG_X86_64 1224 rdmsrl(MSR_KERNEL_GS_BASE, vmx->msr_guest_kernel_gs_base); 1225 #endif 1226 if (host_state->ldt_sel || (host_state->gs_sel & 7)) { 1227 kvm_load_ldt(host_state->ldt_sel); 1228 #ifdef CONFIG_X86_64 1229 load_gs_index(host_state->gs_sel); 1230 #else 1231 loadsegment(gs, host_state->gs_sel); 1232 #endif 1233 } 1234 if (host_state->fs_sel & 7) 1235 loadsegment(fs, host_state->fs_sel); 1236 #ifdef CONFIG_X86_64 1237 if (unlikely(host_state->ds_sel | host_state->es_sel)) { 1238 loadsegment(ds, host_state->ds_sel); 1239 loadsegment(es, host_state->es_sel); 1240 } 1241 #endif 1242 invalidate_tss_limit(); 1243 #ifdef CONFIG_X86_64 1244 wrmsrl(MSR_KERNEL_GS_BASE, vmx->msr_host_kernel_gs_base); 1245 #endif 1246 load_fixmap_gdt(raw_smp_processor_id()); 1247 vmx->guest_state_loaded = false; 1248 vmx->guest_msrs_ready = false; 1249 } 1250 1251 #ifdef CONFIG_X86_64 1252 static u64 vmx_read_guest_kernel_gs_base(struct vcpu_vmx *vmx) 1253 { 1254 preempt_disable(); 1255 if (vmx->guest_state_loaded) 1256 rdmsrl(MSR_KERNEL_GS_BASE, vmx->msr_guest_kernel_gs_base); 1257 preempt_enable(); 1258 return vmx->msr_guest_kernel_gs_base; 1259 } 1260 1261 static void vmx_write_guest_kernel_gs_base(struct vcpu_vmx *vmx, u64 data) 1262 { 1263 preempt_disable(); 1264 if (vmx->guest_state_loaded) 1265 wrmsrl(MSR_KERNEL_GS_BASE, data); 1266 preempt_enable(); 1267 vmx->msr_guest_kernel_gs_base = data; 1268 } 1269 #endif 1270 1271 static void vmx_vcpu_pi_load(struct kvm_vcpu *vcpu, int cpu) 1272 { 1273 struct pi_desc *pi_desc = vcpu_to_pi_desc(vcpu); 1274 struct pi_desc old, new; 1275 unsigned int dest; 1276 1277 /* 1278 * In case of hot-plug or hot-unplug, we may have to undo 1279 * vmx_vcpu_pi_put even if there is no assigned device. And we 1280 * always keep PI.NDST up to date for simplicity: it makes the 1281 * code easier, and CPU migration is not a fast path. 1282 */ 1283 if (!pi_test_sn(pi_desc) && vcpu->cpu == cpu) 1284 return; 1285 1286 /* 1287 * If the 'nv' field is POSTED_INTR_WAKEUP_VECTOR, do not change 1288 * PI.NDST: pi_post_block is the one expected to change PID.NDST and the 1289 * wakeup handler expects the vCPU to be on the blocked_vcpu_list that 1290 * matches PI.NDST. Otherwise, a vcpu may not be able to be woken up 1291 * correctly. 1292 */ 1293 if (pi_desc->nv == POSTED_INTR_WAKEUP_VECTOR || vcpu->cpu == cpu) { 1294 pi_clear_sn(pi_desc); 1295 goto after_clear_sn; 1296 } 1297 1298 /* The full case. */ 1299 do { 1300 old.control = new.control = pi_desc->control; 1301 1302 dest = cpu_physical_id(cpu); 1303 1304 if (x2apic_enabled()) 1305 new.ndst = dest; 1306 else 1307 new.ndst = (dest << 8) & 0xFF00; 1308 1309 new.sn = 0; 1310 } while (cmpxchg64(&pi_desc->control, old.control, 1311 new.control) != old.control); 1312 1313 after_clear_sn: 1314 1315 /* 1316 * Clear SN before reading the bitmap. The VT-d firmware 1317 * writes the bitmap and reads SN atomically (5.2.3 in the 1318 * spec), so it doesn't really have a memory barrier that 1319 * pairs with this, but we cannot do that and we need one. 1320 */ 1321 smp_mb__after_atomic(); 1322 1323 if (!pi_is_pir_empty(pi_desc)) 1324 pi_set_on(pi_desc); 1325 } 1326 1327 void vmx_vcpu_load_vmcs(struct kvm_vcpu *vcpu, int cpu, 1328 struct loaded_vmcs *buddy) 1329 { 1330 struct vcpu_vmx *vmx = to_vmx(vcpu); 1331 bool already_loaded = vmx->loaded_vmcs->cpu == cpu; 1332 struct vmcs *prev; 1333 1334 if (!already_loaded) { 1335 loaded_vmcs_clear(vmx->loaded_vmcs); 1336 local_irq_disable(); 1337 1338 /* 1339 * Ensure loaded_vmcs->cpu is read before adding loaded_vmcs to 1340 * this cpu's percpu list, otherwise it may not yet be deleted 1341 * from its previous cpu's percpu list. Pairs with the 1342 * smb_wmb() in __loaded_vmcs_clear(). 1343 */ 1344 smp_rmb(); 1345 1346 list_add(&vmx->loaded_vmcs->loaded_vmcss_on_cpu_link, 1347 &per_cpu(loaded_vmcss_on_cpu, cpu)); 1348 local_irq_enable(); 1349 } 1350 1351 prev = per_cpu(current_vmcs, cpu); 1352 if (prev != vmx->loaded_vmcs->vmcs) { 1353 per_cpu(current_vmcs, cpu) = vmx->loaded_vmcs->vmcs; 1354 vmcs_load(vmx->loaded_vmcs->vmcs); 1355 1356 /* 1357 * No indirect branch prediction barrier needed when switching 1358 * the active VMCS within a guest, e.g. on nested VM-Enter. 1359 * The L1 VMM can protect itself with retpolines, IBPB or IBRS. 1360 */ 1361 if (!buddy || WARN_ON_ONCE(buddy->vmcs != prev)) 1362 indirect_branch_prediction_barrier(); 1363 } 1364 1365 if (!already_loaded) { 1366 void *gdt = get_current_gdt_ro(); 1367 unsigned long sysenter_esp; 1368 1369 /* 1370 * Flush all EPTP/VPID contexts, the new pCPU may have stale 1371 * TLB entries from its previous association with the vCPU. 1372 */ 1373 kvm_make_request(KVM_REQ_TLB_FLUSH, vcpu); 1374 1375 /* 1376 * Linux uses per-cpu TSS and GDT, so set these when switching 1377 * processors. See 22.2.4. 1378 */ 1379 vmcs_writel(HOST_TR_BASE, 1380 (unsigned long)&get_cpu_entry_area(cpu)->tss.x86_tss); 1381 vmcs_writel(HOST_GDTR_BASE, (unsigned long)gdt); /* 22.2.4 */ 1382 1383 rdmsrl(MSR_IA32_SYSENTER_ESP, sysenter_esp); 1384 vmcs_writel(HOST_IA32_SYSENTER_ESP, sysenter_esp); /* 22.2.3 */ 1385 1386 vmx->loaded_vmcs->cpu = cpu; 1387 } 1388 1389 /* Setup TSC multiplier */ 1390 if (kvm_has_tsc_control && 1391 vmx->current_tsc_ratio != vcpu->arch.tsc_scaling_ratio) 1392 decache_tsc_multiplier(vmx); 1393 } 1394 1395 /* 1396 * Switches to specified vcpu, until a matching vcpu_put(), but assumes 1397 * vcpu mutex is already taken. 1398 */ 1399 static void vmx_vcpu_load(struct kvm_vcpu *vcpu, int cpu) 1400 { 1401 struct vcpu_vmx *vmx = to_vmx(vcpu); 1402 1403 vmx_vcpu_load_vmcs(vcpu, cpu, NULL); 1404 1405 vmx_vcpu_pi_load(vcpu, cpu); 1406 1407 vmx->host_debugctlmsr = get_debugctlmsr(); 1408 } 1409 1410 static void vmx_vcpu_pi_put(struct kvm_vcpu *vcpu) 1411 { 1412 struct pi_desc *pi_desc = vcpu_to_pi_desc(vcpu); 1413 1414 if (!kvm_arch_has_assigned_device(vcpu->kvm) || 1415 !irq_remapping_cap(IRQ_POSTING_CAP) || 1416 !kvm_vcpu_apicv_active(vcpu)) 1417 return; 1418 1419 /* Set SN when the vCPU is preempted */ 1420 if (vcpu->preempted) 1421 pi_set_sn(pi_desc); 1422 } 1423 1424 static void vmx_vcpu_put(struct kvm_vcpu *vcpu) 1425 { 1426 vmx_vcpu_pi_put(vcpu); 1427 1428 vmx_prepare_switch_to_host(to_vmx(vcpu)); 1429 } 1430 1431 static bool emulation_required(struct kvm_vcpu *vcpu) 1432 { 1433 return emulate_invalid_guest_state && !guest_state_valid(vcpu); 1434 } 1435 1436 unsigned long vmx_get_rflags(struct kvm_vcpu *vcpu) 1437 { 1438 struct vcpu_vmx *vmx = to_vmx(vcpu); 1439 unsigned long rflags, save_rflags; 1440 1441 if (!kvm_register_is_available(vcpu, VCPU_EXREG_RFLAGS)) { 1442 kvm_register_mark_available(vcpu, VCPU_EXREG_RFLAGS); 1443 rflags = vmcs_readl(GUEST_RFLAGS); 1444 if (vmx->rmode.vm86_active) { 1445 rflags &= RMODE_GUEST_OWNED_EFLAGS_BITS; 1446 save_rflags = vmx->rmode.save_rflags; 1447 rflags |= save_rflags & ~RMODE_GUEST_OWNED_EFLAGS_BITS; 1448 } 1449 vmx->rflags = rflags; 1450 } 1451 return vmx->rflags; 1452 } 1453 1454 void vmx_set_rflags(struct kvm_vcpu *vcpu, unsigned long rflags) 1455 { 1456 struct vcpu_vmx *vmx = to_vmx(vcpu); 1457 unsigned long old_rflags; 1458 1459 if (enable_unrestricted_guest) { 1460 kvm_register_mark_available(vcpu, VCPU_EXREG_RFLAGS); 1461 vmx->rflags = rflags; 1462 vmcs_writel(GUEST_RFLAGS, rflags); 1463 return; 1464 } 1465 1466 old_rflags = vmx_get_rflags(vcpu); 1467 vmx->rflags = rflags; 1468 if (vmx->rmode.vm86_active) { 1469 vmx->rmode.save_rflags = rflags; 1470 rflags |= X86_EFLAGS_IOPL | X86_EFLAGS_VM; 1471 } 1472 vmcs_writel(GUEST_RFLAGS, rflags); 1473 1474 if ((old_rflags ^ vmx->rflags) & X86_EFLAGS_VM) 1475 vmx->emulation_required = emulation_required(vcpu); 1476 } 1477 1478 u32 vmx_get_interrupt_shadow(struct kvm_vcpu *vcpu) 1479 { 1480 u32 interruptibility = vmcs_read32(GUEST_INTERRUPTIBILITY_INFO); 1481 int ret = 0; 1482 1483 if (interruptibility & GUEST_INTR_STATE_STI) 1484 ret |= KVM_X86_SHADOW_INT_STI; 1485 if (interruptibility & GUEST_INTR_STATE_MOV_SS) 1486 ret |= KVM_X86_SHADOW_INT_MOV_SS; 1487 1488 return ret; 1489 } 1490 1491 void vmx_set_interrupt_shadow(struct kvm_vcpu *vcpu, int mask) 1492 { 1493 u32 interruptibility_old = vmcs_read32(GUEST_INTERRUPTIBILITY_INFO); 1494 u32 interruptibility = interruptibility_old; 1495 1496 interruptibility &= ~(GUEST_INTR_STATE_STI | GUEST_INTR_STATE_MOV_SS); 1497 1498 if (mask & KVM_X86_SHADOW_INT_MOV_SS) 1499 interruptibility |= GUEST_INTR_STATE_MOV_SS; 1500 else if (mask & KVM_X86_SHADOW_INT_STI) 1501 interruptibility |= GUEST_INTR_STATE_STI; 1502 1503 if ((interruptibility != interruptibility_old)) 1504 vmcs_write32(GUEST_INTERRUPTIBILITY_INFO, interruptibility); 1505 } 1506 1507 static int vmx_rtit_ctl_check(struct kvm_vcpu *vcpu, u64 data) 1508 { 1509 struct vcpu_vmx *vmx = to_vmx(vcpu); 1510 unsigned long value; 1511 1512 /* 1513 * Any MSR write that attempts to change bits marked reserved will 1514 * case a #GP fault. 1515 */ 1516 if (data & vmx->pt_desc.ctl_bitmask) 1517 return 1; 1518 1519 /* 1520 * Any attempt to modify IA32_RTIT_CTL while TraceEn is set will 1521 * result in a #GP unless the same write also clears TraceEn. 1522 */ 1523 if ((vmx->pt_desc.guest.ctl & RTIT_CTL_TRACEEN) && 1524 ((vmx->pt_desc.guest.ctl ^ data) & ~RTIT_CTL_TRACEEN)) 1525 return 1; 1526 1527 /* 1528 * WRMSR to IA32_RTIT_CTL that sets TraceEn but clears this bit 1529 * and FabricEn would cause #GP, if 1530 * CPUID.(EAX=14H, ECX=0):ECX.SNGLRGNOUT[bit 2] = 0 1531 */ 1532 if ((data & RTIT_CTL_TRACEEN) && !(data & RTIT_CTL_TOPA) && 1533 !(data & RTIT_CTL_FABRIC_EN) && 1534 !intel_pt_validate_cap(vmx->pt_desc.caps, 1535 PT_CAP_single_range_output)) 1536 return 1; 1537 1538 /* 1539 * MTCFreq, CycThresh and PSBFreq encodings check, any MSR write that 1540 * utilize encodings marked reserved will casue a #GP fault. 1541 */ 1542 value = intel_pt_validate_cap(vmx->pt_desc.caps, PT_CAP_mtc_periods); 1543 if (intel_pt_validate_cap(vmx->pt_desc.caps, PT_CAP_mtc) && 1544 !test_bit((data & RTIT_CTL_MTC_RANGE) >> 1545 RTIT_CTL_MTC_RANGE_OFFSET, &value)) 1546 return 1; 1547 value = intel_pt_validate_cap(vmx->pt_desc.caps, 1548 PT_CAP_cycle_thresholds); 1549 if (intel_pt_validate_cap(vmx->pt_desc.caps, PT_CAP_psb_cyc) && 1550 !test_bit((data & RTIT_CTL_CYC_THRESH) >> 1551 RTIT_CTL_CYC_THRESH_OFFSET, &value)) 1552 return 1; 1553 value = intel_pt_validate_cap(vmx->pt_desc.caps, PT_CAP_psb_periods); 1554 if (intel_pt_validate_cap(vmx->pt_desc.caps, PT_CAP_psb_cyc) && 1555 !test_bit((data & RTIT_CTL_PSB_FREQ) >> 1556 RTIT_CTL_PSB_FREQ_OFFSET, &value)) 1557 return 1; 1558 1559 /* 1560 * If ADDRx_CFG is reserved or the encodings is >2 will 1561 * cause a #GP fault. 1562 */ 1563 value = (data & RTIT_CTL_ADDR0) >> RTIT_CTL_ADDR0_OFFSET; 1564 if ((value && (vmx->pt_desc.addr_range < 1)) || (value > 2)) 1565 return 1; 1566 value = (data & RTIT_CTL_ADDR1) >> RTIT_CTL_ADDR1_OFFSET; 1567 if ((value && (vmx->pt_desc.addr_range < 2)) || (value > 2)) 1568 return 1; 1569 value = (data & RTIT_CTL_ADDR2) >> RTIT_CTL_ADDR2_OFFSET; 1570 if ((value && (vmx->pt_desc.addr_range < 3)) || (value > 2)) 1571 return 1; 1572 value = (data & RTIT_CTL_ADDR3) >> RTIT_CTL_ADDR3_OFFSET; 1573 if ((value && (vmx->pt_desc.addr_range < 4)) || (value > 2)) 1574 return 1; 1575 1576 return 0; 1577 } 1578 1579 static int skip_emulated_instruction(struct kvm_vcpu *vcpu) 1580 { 1581 unsigned long rip, orig_rip; 1582 1583 /* 1584 * Using VMCS.VM_EXIT_INSTRUCTION_LEN on EPT misconfig depends on 1585 * undefined behavior: Intel's SDM doesn't mandate the VMCS field be 1586 * set when EPT misconfig occurs. In practice, real hardware updates 1587 * VM_EXIT_INSTRUCTION_LEN on EPT misconfig, but other hypervisors 1588 * (namely Hyper-V) don't set it due to it being undefined behavior, 1589 * i.e. we end up advancing IP with some random value. 1590 */ 1591 if (!static_cpu_has(X86_FEATURE_HYPERVISOR) || 1592 to_vmx(vcpu)->exit_reason != EXIT_REASON_EPT_MISCONFIG) { 1593 orig_rip = kvm_rip_read(vcpu); 1594 rip = orig_rip + vmcs_read32(VM_EXIT_INSTRUCTION_LEN); 1595 #ifdef CONFIG_X86_64 1596 /* 1597 * We need to mask out the high 32 bits of RIP if not in 64-bit 1598 * mode, but just finding out that we are in 64-bit mode is 1599 * quite expensive. Only do it if there was a carry. 1600 */ 1601 if (unlikely(((rip ^ orig_rip) >> 31) == 3) && !is_64_bit_mode(vcpu)) 1602 rip = (u32)rip; 1603 #endif 1604 kvm_rip_write(vcpu, rip); 1605 } else { 1606 if (!kvm_emulate_instruction(vcpu, EMULTYPE_SKIP)) 1607 return 0; 1608 } 1609 1610 /* skipping an emulated instruction also counts */ 1611 vmx_set_interrupt_shadow(vcpu, 0); 1612 1613 return 1; 1614 } 1615 1616 /* 1617 * Handles kvm_read/write_guest_virt*() result and either injects #PF or returns 1618 * KVM_EXIT_INTERNAL_ERROR for cases not currently handled by KVM. Return value 1619 * indicates whether exit to userspace is needed. 1620 */ 1621 int vmx_handle_memory_failure(struct kvm_vcpu *vcpu, int r, 1622 struct x86_exception *e) 1623 { 1624 if (r == X86EMUL_PROPAGATE_FAULT) { 1625 kvm_inject_emulated_page_fault(vcpu, e); 1626 return 1; 1627 } 1628 1629 /* 1630 * In case kvm_read/write_guest_virt*() failed with X86EMUL_IO_NEEDED 1631 * while handling a VMX instruction KVM could've handled the request 1632 * correctly by exiting to userspace and performing I/O but there 1633 * doesn't seem to be a real use-case behind such requests, just return 1634 * KVM_EXIT_INTERNAL_ERROR for now. 1635 */ 1636 vcpu->run->exit_reason = KVM_EXIT_INTERNAL_ERROR; 1637 vcpu->run->internal.suberror = KVM_INTERNAL_ERROR_EMULATION; 1638 vcpu->run->internal.ndata = 0; 1639 1640 return 0; 1641 } 1642 1643 /* 1644 * Recognizes a pending MTF VM-exit and records the nested state for later 1645 * delivery. 1646 */ 1647 static void vmx_update_emulated_instruction(struct kvm_vcpu *vcpu) 1648 { 1649 struct vmcs12 *vmcs12 = get_vmcs12(vcpu); 1650 struct vcpu_vmx *vmx = to_vmx(vcpu); 1651 1652 if (!is_guest_mode(vcpu)) 1653 return; 1654 1655 /* 1656 * Per the SDM, MTF takes priority over debug-trap exceptions besides 1657 * T-bit traps. As instruction emulation is completed (i.e. at the 1658 * instruction boundary), any #DB exception pending delivery must be a 1659 * debug-trap. Record the pending MTF state to be delivered in 1660 * vmx_check_nested_events(). 1661 */ 1662 if (nested_cpu_has_mtf(vmcs12) && 1663 (!vcpu->arch.exception.pending || 1664 vcpu->arch.exception.nr == DB_VECTOR)) 1665 vmx->nested.mtf_pending = true; 1666 else 1667 vmx->nested.mtf_pending = false; 1668 } 1669 1670 static int vmx_skip_emulated_instruction(struct kvm_vcpu *vcpu) 1671 { 1672 vmx_update_emulated_instruction(vcpu); 1673 return skip_emulated_instruction(vcpu); 1674 } 1675 1676 static void vmx_clear_hlt(struct kvm_vcpu *vcpu) 1677 { 1678 /* 1679 * Ensure that we clear the HLT state in the VMCS. We don't need to 1680 * explicitly skip the instruction because if the HLT state is set, 1681 * then the instruction is already executing and RIP has already been 1682 * advanced. 1683 */ 1684 if (kvm_hlt_in_guest(vcpu->kvm) && 1685 vmcs_read32(GUEST_ACTIVITY_STATE) == GUEST_ACTIVITY_HLT) 1686 vmcs_write32(GUEST_ACTIVITY_STATE, GUEST_ACTIVITY_ACTIVE); 1687 } 1688 1689 static void vmx_queue_exception(struct kvm_vcpu *vcpu) 1690 { 1691 struct vcpu_vmx *vmx = to_vmx(vcpu); 1692 unsigned nr = vcpu->arch.exception.nr; 1693 bool has_error_code = vcpu->arch.exception.has_error_code; 1694 u32 error_code = vcpu->arch.exception.error_code; 1695 u32 intr_info = nr | INTR_INFO_VALID_MASK; 1696 1697 kvm_deliver_exception_payload(vcpu); 1698 1699 if (has_error_code) { 1700 vmcs_write32(VM_ENTRY_EXCEPTION_ERROR_CODE, error_code); 1701 intr_info |= INTR_INFO_DELIVER_CODE_MASK; 1702 } 1703 1704 if (vmx->rmode.vm86_active) { 1705 int inc_eip = 0; 1706 if (kvm_exception_is_soft(nr)) 1707 inc_eip = vcpu->arch.event_exit_inst_len; 1708 kvm_inject_realmode_interrupt(vcpu, nr, inc_eip); 1709 return; 1710 } 1711 1712 WARN_ON_ONCE(vmx->emulation_required); 1713 1714 if (kvm_exception_is_soft(nr)) { 1715 vmcs_write32(VM_ENTRY_INSTRUCTION_LEN, 1716 vmx->vcpu.arch.event_exit_inst_len); 1717 intr_info |= INTR_TYPE_SOFT_EXCEPTION; 1718 } else 1719 intr_info |= INTR_TYPE_HARD_EXCEPTION; 1720 1721 vmcs_write32(VM_ENTRY_INTR_INFO_FIELD, intr_info); 1722 1723 vmx_clear_hlt(vcpu); 1724 } 1725 1726 /* 1727 * Swap MSR entry in host/guest MSR entry array. 1728 */ 1729 static void move_msr_up(struct vcpu_vmx *vmx, int from, int to) 1730 { 1731 struct shared_msr_entry tmp; 1732 1733 tmp = vmx->guest_msrs[to]; 1734 vmx->guest_msrs[to] = vmx->guest_msrs[from]; 1735 vmx->guest_msrs[from] = tmp; 1736 } 1737 1738 /* 1739 * Set up the vmcs to automatically save and restore system 1740 * msrs. Don't touch the 64-bit msrs if the guest is in legacy 1741 * mode, as fiddling with msrs is very expensive. 1742 */ 1743 static void setup_msrs(struct vcpu_vmx *vmx) 1744 { 1745 int save_nmsrs, index; 1746 1747 save_nmsrs = 0; 1748 #ifdef CONFIG_X86_64 1749 /* 1750 * The SYSCALL MSRs are only needed on long mode guests, and only 1751 * when EFER.SCE is set. 1752 */ 1753 if (is_long_mode(&vmx->vcpu) && (vmx->vcpu.arch.efer & EFER_SCE)) { 1754 index = __find_msr_index(vmx, MSR_STAR); 1755 if (index >= 0) 1756 move_msr_up(vmx, index, save_nmsrs++); 1757 index = __find_msr_index(vmx, MSR_LSTAR); 1758 if (index >= 0) 1759 move_msr_up(vmx, index, save_nmsrs++); 1760 index = __find_msr_index(vmx, MSR_SYSCALL_MASK); 1761 if (index >= 0) 1762 move_msr_up(vmx, index, save_nmsrs++); 1763 } 1764 #endif 1765 index = __find_msr_index(vmx, MSR_EFER); 1766 if (index >= 0 && update_transition_efer(vmx, index)) 1767 move_msr_up(vmx, index, save_nmsrs++); 1768 index = __find_msr_index(vmx, MSR_TSC_AUX); 1769 if (index >= 0 && guest_cpuid_has(&vmx->vcpu, X86_FEATURE_RDTSCP)) 1770 move_msr_up(vmx, index, save_nmsrs++); 1771 index = __find_msr_index(vmx, MSR_IA32_TSX_CTRL); 1772 if (index >= 0) 1773 move_msr_up(vmx, index, save_nmsrs++); 1774 1775 vmx->save_nmsrs = save_nmsrs; 1776 vmx->guest_msrs_ready = false; 1777 1778 if (cpu_has_vmx_msr_bitmap()) 1779 vmx_update_msr_bitmap(&vmx->vcpu); 1780 } 1781 1782 static u64 vmx_write_l1_tsc_offset(struct kvm_vcpu *vcpu, u64 offset) 1783 { 1784 struct vmcs12 *vmcs12 = get_vmcs12(vcpu); 1785 u64 g_tsc_offset = 0; 1786 1787 /* 1788 * We're here if L1 chose not to trap WRMSR to TSC. According 1789 * to the spec, this should set L1's TSC; The offset that L1 1790 * set for L2 remains unchanged, and still needs to be added 1791 * to the newly set TSC to get L2's TSC. 1792 */ 1793 if (is_guest_mode(vcpu) && 1794 (vmcs12->cpu_based_vm_exec_control & CPU_BASED_USE_TSC_OFFSETTING)) 1795 g_tsc_offset = vmcs12->tsc_offset; 1796 1797 trace_kvm_write_tsc_offset(vcpu->vcpu_id, 1798 vcpu->arch.tsc_offset - g_tsc_offset, 1799 offset); 1800 vmcs_write64(TSC_OFFSET, offset + g_tsc_offset); 1801 return offset + g_tsc_offset; 1802 } 1803 1804 /* 1805 * nested_vmx_allowed() checks whether a guest should be allowed to use VMX 1806 * instructions and MSRs (i.e., nested VMX). Nested VMX is disabled for 1807 * all guests if the "nested" module option is off, and can also be disabled 1808 * for a single guest by disabling its VMX cpuid bit. 1809 */ 1810 bool nested_vmx_allowed(struct kvm_vcpu *vcpu) 1811 { 1812 return nested && guest_cpuid_has(vcpu, X86_FEATURE_VMX); 1813 } 1814 1815 static inline bool vmx_feature_control_msr_valid(struct kvm_vcpu *vcpu, 1816 uint64_t val) 1817 { 1818 uint64_t valid_bits = to_vmx(vcpu)->msr_ia32_feature_control_valid_bits; 1819 1820 return !(val & ~valid_bits); 1821 } 1822 1823 static int vmx_get_msr_feature(struct kvm_msr_entry *msr) 1824 { 1825 switch (msr->index) { 1826 case MSR_IA32_VMX_BASIC ... MSR_IA32_VMX_VMFUNC: 1827 if (!nested) 1828 return 1; 1829 return vmx_get_vmx_msr(&vmcs_config.nested, msr->index, &msr->data); 1830 case MSR_IA32_PERF_CAPABILITIES: 1831 msr->data = vmx_get_perf_capabilities(); 1832 return 0; 1833 default: 1834 return KVM_MSR_RET_INVALID; 1835 } 1836 } 1837 1838 /* 1839 * Reads an msr value (of 'msr_index') into 'pdata'. 1840 * Returns 0 on success, non-0 otherwise. 1841 * Assumes vcpu_load() was already called. 1842 */ 1843 static int vmx_get_msr(struct kvm_vcpu *vcpu, struct msr_data *msr_info) 1844 { 1845 struct vcpu_vmx *vmx = to_vmx(vcpu); 1846 struct shared_msr_entry *msr; 1847 u32 index; 1848 1849 switch (msr_info->index) { 1850 #ifdef CONFIG_X86_64 1851 case MSR_FS_BASE: 1852 msr_info->data = vmcs_readl(GUEST_FS_BASE); 1853 break; 1854 case MSR_GS_BASE: 1855 msr_info->data = vmcs_readl(GUEST_GS_BASE); 1856 break; 1857 case MSR_KERNEL_GS_BASE: 1858 msr_info->data = vmx_read_guest_kernel_gs_base(vmx); 1859 break; 1860 #endif 1861 case MSR_EFER: 1862 return kvm_get_msr_common(vcpu, msr_info); 1863 case MSR_IA32_TSX_CTRL: 1864 if (!msr_info->host_initiated && 1865 !(vcpu->arch.arch_capabilities & ARCH_CAP_TSX_CTRL_MSR)) 1866 return 1; 1867 goto find_shared_msr; 1868 case MSR_IA32_UMWAIT_CONTROL: 1869 if (!msr_info->host_initiated && !vmx_has_waitpkg(vmx)) 1870 return 1; 1871 1872 msr_info->data = vmx->msr_ia32_umwait_control; 1873 break; 1874 case MSR_IA32_SPEC_CTRL: 1875 if (!msr_info->host_initiated && 1876 !guest_cpuid_has(vcpu, X86_FEATURE_SPEC_CTRL)) 1877 return 1; 1878 1879 msr_info->data = to_vmx(vcpu)->spec_ctrl; 1880 break; 1881 case MSR_IA32_SYSENTER_CS: 1882 msr_info->data = vmcs_read32(GUEST_SYSENTER_CS); 1883 break; 1884 case MSR_IA32_SYSENTER_EIP: 1885 msr_info->data = vmcs_readl(GUEST_SYSENTER_EIP); 1886 break; 1887 case MSR_IA32_SYSENTER_ESP: 1888 msr_info->data = vmcs_readl(GUEST_SYSENTER_ESP); 1889 break; 1890 case MSR_IA32_BNDCFGS: 1891 if (!kvm_mpx_supported() || 1892 (!msr_info->host_initiated && 1893 !guest_cpuid_has(vcpu, X86_FEATURE_MPX))) 1894 return 1; 1895 msr_info->data = vmcs_read64(GUEST_BNDCFGS); 1896 break; 1897 case MSR_IA32_MCG_EXT_CTL: 1898 if (!msr_info->host_initiated && 1899 !(vmx->msr_ia32_feature_control & 1900 FEAT_CTL_LMCE_ENABLED)) 1901 return 1; 1902 msr_info->data = vcpu->arch.mcg_ext_ctl; 1903 break; 1904 case MSR_IA32_FEAT_CTL: 1905 msr_info->data = vmx->msr_ia32_feature_control; 1906 break; 1907 case MSR_IA32_VMX_BASIC ... MSR_IA32_VMX_VMFUNC: 1908 if (!nested_vmx_allowed(vcpu)) 1909 return 1; 1910 if (vmx_get_vmx_msr(&vmx->nested.msrs, msr_info->index, 1911 &msr_info->data)) 1912 return 1; 1913 /* 1914 * Enlightened VMCS v1 doesn't have certain fields, but buggy 1915 * Hyper-V versions are still trying to use corresponding 1916 * features when they are exposed. Filter out the essential 1917 * minimum. 1918 */ 1919 if (!msr_info->host_initiated && 1920 vmx->nested.enlightened_vmcs_enabled) 1921 nested_evmcs_filter_control_msr(msr_info->index, 1922 &msr_info->data); 1923 break; 1924 case MSR_IA32_RTIT_CTL: 1925 if (!vmx_pt_mode_is_host_guest()) 1926 return 1; 1927 msr_info->data = vmx->pt_desc.guest.ctl; 1928 break; 1929 case MSR_IA32_RTIT_STATUS: 1930 if (!vmx_pt_mode_is_host_guest()) 1931 return 1; 1932 msr_info->data = vmx->pt_desc.guest.status; 1933 break; 1934 case MSR_IA32_RTIT_CR3_MATCH: 1935 if (!vmx_pt_mode_is_host_guest() || 1936 !intel_pt_validate_cap(vmx->pt_desc.caps, 1937 PT_CAP_cr3_filtering)) 1938 return 1; 1939 msr_info->data = vmx->pt_desc.guest.cr3_match; 1940 break; 1941 case MSR_IA32_RTIT_OUTPUT_BASE: 1942 if (!vmx_pt_mode_is_host_guest() || 1943 (!intel_pt_validate_cap(vmx->pt_desc.caps, 1944 PT_CAP_topa_output) && 1945 !intel_pt_validate_cap(vmx->pt_desc.caps, 1946 PT_CAP_single_range_output))) 1947 return 1; 1948 msr_info->data = vmx->pt_desc.guest.output_base; 1949 break; 1950 case MSR_IA32_RTIT_OUTPUT_MASK: 1951 if (!vmx_pt_mode_is_host_guest() || 1952 (!intel_pt_validate_cap(vmx->pt_desc.caps, 1953 PT_CAP_topa_output) && 1954 !intel_pt_validate_cap(vmx->pt_desc.caps, 1955 PT_CAP_single_range_output))) 1956 return 1; 1957 msr_info->data = vmx->pt_desc.guest.output_mask; 1958 break; 1959 case MSR_IA32_RTIT_ADDR0_A ... MSR_IA32_RTIT_ADDR3_B: 1960 index = msr_info->index - MSR_IA32_RTIT_ADDR0_A; 1961 if (!vmx_pt_mode_is_host_guest() || 1962 (index >= 2 * intel_pt_validate_cap(vmx->pt_desc.caps, 1963 PT_CAP_num_address_ranges))) 1964 return 1; 1965 if (index % 2) 1966 msr_info->data = vmx->pt_desc.guest.addr_b[index / 2]; 1967 else 1968 msr_info->data = vmx->pt_desc.guest.addr_a[index / 2]; 1969 break; 1970 case MSR_TSC_AUX: 1971 if (!msr_info->host_initiated && 1972 !guest_cpuid_has(vcpu, X86_FEATURE_RDTSCP)) 1973 return 1; 1974 goto find_shared_msr; 1975 default: 1976 find_shared_msr: 1977 msr = find_msr_entry(vmx, msr_info->index); 1978 if (msr) { 1979 msr_info->data = msr->data; 1980 break; 1981 } 1982 return kvm_get_msr_common(vcpu, msr_info); 1983 } 1984 1985 return 0; 1986 } 1987 1988 static u64 nested_vmx_truncate_sysenter_addr(struct kvm_vcpu *vcpu, 1989 u64 data) 1990 { 1991 #ifdef CONFIG_X86_64 1992 if (!guest_cpuid_has(vcpu, X86_FEATURE_LM)) 1993 return (u32)data; 1994 #endif 1995 return (unsigned long)data; 1996 } 1997 1998 /* 1999 * Writes msr value into the appropriate "register". 2000 * Returns 0 on success, non-0 otherwise. 2001 * Assumes vcpu_load() was already called. 2002 */ 2003 static int vmx_set_msr(struct kvm_vcpu *vcpu, struct msr_data *msr_info) 2004 { 2005 struct vcpu_vmx *vmx = to_vmx(vcpu); 2006 struct shared_msr_entry *msr; 2007 int ret = 0; 2008 u32 msr_index = msr_info->index; 2009 u64 data = msr_info->data; 2010 u32 index; 2011 2012 switch (msr_index) { 2013 case MSR_EFER: 2014 ret = kvm_set_msr_common(vcpu, msr_info); 2015 break; 2016 #ifdef CONFIG_X86_64 2017 case MSR_FS_BASE: 2018 vmx_segment_cache_clear(vmx); 2019 vmcs_writel(GUEST_FS_BASE, data); 2020 break; 2021 case MSR_GS_BASE: 2022 vmx_segment_cache_clear(vmx); 2023 vmcs_writel(GUEST_GS_BASE, data); 2024 break; 2025 case MSR_KERNEL_GS_BASE: 2026 vmx_write_guest_kernel_gs_base(vmx, data); 2027 break; 2028 #endif 2029 case MSR_IA32_SYSENTER_CS: 2030 if (is_guest_mode(vcpu)) 2031 get_vmcs12(vcpu)->guest_sysenter_cs = data; 2032 vmcs_write32(GUEST_SYSENTER_CS, data); 2033 break; 2034 case MSR_IA32_SYSENTER_EIP: 2035 if (is_guest_mode(vcpu)) { 2036 data = nested_vmx_truncate_sysenter_addr(vcpu, data); 2037 get_vmcs12(vcpu)->guest_sysenter_eip = data; 2038 } 2039 vmcs_writel(GUEST_SYSENTER_EIP, data); 2040 break; 2041 case MSR_IA32_SYSENTER_ESP: 2042 if (is_guest_mode(vcpu)) { 2043 data = nested_vmx_truncate_sysenter_addr(vcpu, data); 2044 get_vmcs12(vcpu)->guest_sysenter_esp = data; 2045 } 2046 vmcs_writel(GUEST_SYSENTER_ESP, data); 2047 break; 2048 case MSR_IA32_DEBUGCTLMSR: 2049 if (is_guest_mode(vcpu) && get_vmcs12(vcpu)->vm_exit_controls & 2050 VM_EXIT_SAVE_DEBUG_CONTROLS) 2051 get_vmcs12(vcpu)->guest_ia32_debugctl = data; 2052 2053 ret = kvm_set_msr_common(vcpu, msr_info); 2054 break; 2055 2056 case MSR_IA32_BNDCFGS: 2057 if (!kvm_mpx_supported() || 2058 (!msr_info->host_initiated && 2059 !guest_cpuid_has(vcpu, X86_FEATURE_MPX))) 2060 return 1; 2061 if (is_noncanonical_address(data & PAGE_MASK, vcpu) || 2062 (data & MSR_IA32_BNDCFGS_RSVD)) 2063 return 1; 2064 vmcs_write64(GUEST_BNDCFGS, data); 2065 break; 2066 case MSR_IA32_UMWAIT_CONTROL: 2067 if (!msr_info->host_initiated && !vmx_has_waitpkg(vmx)) 2068 return 1; 2069 2070 /* The reserved bit 1 and non-32 bit [63:32] should be zero */ 2071 if (data & (BIT_ULL(1) | GENMASK_ULL(63, 32))) 2072 return 1; 2073 2074 vmx->msr_ia32_umwait_control = data; 2075 break; 2076 case MSR_IA32_SPEC_CTRL: 2077 if (!msr_info->host_initiated && 2078 !guest_cpuid_has(vcpu, X86_FEATURE_SPEC_CTRL)) 2079 return 1; 2080 2081 if (kvm_spec_ctrl_test_value(data)) 2082 return 1; 2083 2084 vmx->spec_ctrl = data; 2085 if (!data) 2086 break; 2087 2088 /* 2089 * For non-nested: 2090 * When it's written (to non-zero) for the first time, pass 2091 * it through. 2092 * 2093 * For nested: 2094 * The handling of the MSR bitmap for L2 guests is done in 2095 * nested_vmx_prepare_msr_bitmap. We should not touch the 2096 * vmcs02.msr_bitmap here since it gets completely overwritten 2097 * in the merging. We update the vmcs01 here for L1 as well 2098 * since it will end up touching the MSR anyway now. 2099 */ 2100 vmx_disable_intercept_for_msr(vmx->vmcs01.msr_bitmap, 2101 MSR_IA32_SPEC_CTRL, 2102 MSR_TYPE_RW); 2103 break; 2104 case MSR_IA32_TSX_CTRL: 2105 if (!msr_info->host_initiated && 2106 !(vcpu->arch.arch_capabilities & ARCH_CAP_TSX_CTRL_MSR)) 2107 return 1; 2108 if (data & ~(TSX_CTRL_RTM_DISABLE | TSX_CTRL_CPUID_CLEAR)) 2109 return 1; 2110 goto find_shared_msr; 2111 case MSR_IA32_PRED_CMD: 2112 if (!msr_info->host_initiated && 2113 !guest_cpuid_has(vcpu, X86_FEATURE_SPEC_CTRL)) 2114 return 1; 2115 2116 if (data & ~PRED_CMD_IBPB) 2117 return 1; 2118 if (!boot_cpu_has(X86_FEATURE_SPEC_CTRL)) 2119 return 1; 2120 if (!data) 2121 break; 2122 2123 wrmsrl(MSR_IA32_PRED_CMD, PRED_CMD_IBPB); 2124 2125 /* 2126 * For non-nested: 2127 * When it's written (to non-zero) for the first time, pass 2128 * it through. 2129 * 2130 * For nested: 2131 * The handling of the MSR bitmap for L2 guests is done in 2132 * nested_vmx_prepare_msr_bitmap. We should not touch the 2133 * vmcs02.msr_bitmap here since it gets completely overwritten 2134 * in the merging. 2135 */ 2136 vmx_disable_intercept_for_msr(vmx->vmcs01.msr_bitmap, MSR_IA32_PRED_CMD, 2137 MSR_TYPE_W); 2138 break; 2139 case MSR_IA32_CR_PAT: 2140 if (!kvm_pat_valid(data)) 2141 return 1; 2142 2143 if (is_guest_mode(vcpu) && 2144 get_vmcs12(vcpu)->vm_exit_controls & VM_EXIT_SAVE_IA32_PAT) 2145 get_vmcs12(vcpu)->guest_ia32_pat = data; 2146 2147 if (vmcs_config.vmentry_ctrl & VM_ENTRY_LOAD_IA32_PAT) { 2148 vmcs_write64(GUEST_IA32_PAT, data); 2149 vcpu->arch.pat = data; 2150 break; 2151 } 2152 ret = kvm_set_msr_common(vcpu, msr_info); 2153 break; 2154 case MSR_IA32_TSC_ADJUST: 2155 ret = kvm_set_msr_common(vcpu, msr_info); 2156 break; 2157 case MSR_IA32_MCG_EXT_CTL: 2158 if ((!msr_info->host_initiated && 2159 !(to_vmx(vcpu)->msr_ia32_feature_control & 2160 FEAT_CTL_LMCE_ENABLED)) || 2161 (data & ~MCG_EXT_CTL_LMCE_EN)) 2162 return 1; 2163 vcpu->arch.mcg_ext_ctl = data; 2164 break; 2165 case MSR_IA32_FEAT_CTL: 2166 if (!vmx_feature_control_msr_valid(vcpu, data) || 2167 (to_vmx(vcpu)->msr_ia32_feature_control & 2168 FEAT_CTL_LOCKED && !msr_info->host_initiated)) 2169 return 1; 2170 vmx->msr_ia32_feature_control = data; 2171 if (msr_info->host_initiated && data == 0) 2172 vmx_leave_nested(vcpu); 2173 break; 2174 case MSR_IA32_VMX_BASIC ... MSR_IA32_VMX_VMFUNC: 2175 if (!msr_info->host_initiated) 2176 return 1; /* they are read-only */ 2177 if (!nested_vmx_allowed(vcpu)) 2178 return 1; 2179 return vmx_set_vmx_msr(vcpu, msr_index, data); 2180 case MSR_IA32_RTIT_CTL: 2181 if (!vmx_pt_mode_is_host_guest() || 2182 vmx_rtit_ctl_check(vcpu, data) || 2183 vmx->nested.vmxon) 2184 return 1; 2185 vmcs_write64(GUEST_IA32_RTIT_CTL, data); 2186 vmx->pt_desc.guest.ctl = data; 2187 pt_update_intercept_for_msr(vmx); 2188 break; 2189 case MSR_IA32_RTIT_STATUS: 2190 if (!pt_can_write_msr(vmx)) 2191 return 1; 2192 if (data & MSR_IA32_RTIT_STATUS_MASK) 2193 return 1; 2194 vmx->pt_desc.guest.status = data; 2195 break; 2196 case MSR_IA32_RTIT_CR3_MATCH: 2197 if (!pt_can_write_msr(vmx)) 2198 return 1; 2199 if (!intel_pt_validate_cap(vmx->pt_desc.caps, 2200 PT_CAP_cr3_filtering)) 2201 return 1; 2202 vmx->pt_desc.guest.cr3_match = data; 2203 break; 2204 case MSR_IA32_RTIT_OUTPUT_BASE: 2205 if (!pt_can_write_msr(vmx)) 2206 return 1; 2207 if (!intel_pt_validate_cap(vmx->pt_desc.caps, 2208 PT_CAP_topa_output) && 2209 !intel_pt_validate_cap(vmx->pt_desc.caps, 2210 PT_CAP_single_range_output)) 2211 return 1; 2212 if (data & MSR_IA32_RTIT_OUTPUT_BASE_MASK) 2213 return 1; 2214 vmx->pt_desc.guest.output_base = data; 2215 break; 2216 case MSR_IA32_RTIT_OUTPUT_MASK: 2217 if (!pt_can_write_msr(vmx)) 2218 return 1; 2219 if (!intel_pt_validate_cap(vmx->pt_desc.caps, 2220 PT_CAP_topa_output) && 2221 !intel_pt_validate_cap(vmx->pt_desc.caps, 2222 PT_CAP_single_range_output)) 2223 return 1; 2224 vmx->pt_desc.guest.output_mask = data; 2225 break; 2226 case MSR_IA32_RTIT_ADDR0_A ... MSR_IA32_RTIT_ADDR3_B: 2227 if (!pt_can_write_msr(vmx)) 2228 return 1; 2229 index = msr_info->index - MSR_IA32_RTIT_ADDR0_A; 2230 if (index >= 2 * intel_pt_validate_cap(vmx->pt_desc.caps, 2231 PT_CAP_num_address_ranges)) 2232 return 1; 2233 if (is_noncanonical_address(data, vcpu)) 2234 return 1; 2235 if (index % 2) 2236 vmx->pt_desc.guest.addr_b[index / 2] = data; 2237 else 2238 vmx->pt_desc.guest.addr_a[index / 2] = data; 2239 break; 2240 case MSR_TSC_AUX: 2241 if (!msr_info->host_initiated && 2242 !guest_cpuid_has(vcpu, X86_FEATURE_RDTSCP)) 2243 return 1; 2244 /* Check reserved bit, higher 32 bits should be zero */ 2245 if ((data >> 32) != 0) 2246 return 1; 2247 goto find_shared_msr; 2248 2249 default: 2250 find_shared_msr: 2251 msr = find_msr_entry(vmx, msr_index); 2252 if (msr) 2253 ret = vmx_set_guest_msr(vmx, msr, data); 2254 else 2255 ret = kvm_set_msr_common(vcpu, msr_info); 2256 } 2257 2258 return ret; 2259 } 2260 2261 static void vmx_cache_reg(struct kvm_vcpu *vcpu, enum kvm_reg reg) 2262 { 2263 unsigned long guest_owned_bits; 2264 2265 kvm_register_mark_available(vcpu, reg); 2266 2267 switch (reg) { 2268 case VCPU_REGS_RSP: 2269 vcpu->arch.regs[VCPU_REGS_RSP] = vmcs_readl(GUEST_RSP); 2270 break; 2271 case VCPU_REGS_RIP: 2272 vcpu->arch.regs[VCPU_REGS_RIP] = vmcs_readl(GUEST_RIP); 2273 break; 2274 case VCPU_EXREG_PDPTR: 2275 if (enable_ept) 2276 ept_save_pdptrs(vcpu); 2277 break; 2278 case VCPU_EXREG_CR0: 2279 guest_owned_bits = vcpu->arch.cr0_guest_owned_bits; 2280 2281 vcpu->arch.cr0 &= ~guest_owned_bits; 2282 vcpu->arch.cr0 |= vmcs_readl(GUEST_CR0) & guest_owned_bits; 2283 break; 2284 case VCPU_EXREG_CR3: 2285 if (enable_unrestricted_guest || (enable_ept && is_paging(vcpu))) 2286 vcpu->arch.cr3 = vmcs_readl(GUEST_CR3); 2287 break; 2288 case VCPU_EXREG_CR4: 2289 guest_owned_bits = vcpu->arch.cr4_guest_owned_bits; 2290 2291 vcpu->arch.cr4 &= ~guest_owned_bits; 2292 vcpu->arch.cr4 |= vmcs_readl(GUEST_CR4) & guest_owned_bits; 2293 break; 2294 default: 2295 WARN_ON_ONCE(1); 2296 break; 2297 } 2298 } 2299 2300 static __init int cpu_has_kvm_support(void) 2301 { 2302 return cpu_has_vmx(); 2303 } 2304 2305 static __init int vmx_disabled_by_bios(void) 2306 { 2307 return !boot_cpu_has(X86_FEATURE_MSR_IA32_FEAT_CTL) || 2308 !boot_cpu_has(X86_FEATURE_VMX); 2309 } 2310 2311 static int kvm_cpu_vmxon(u64 vmxon_pointer) 2312 { 2313 u64 msr; 2314 2315 cr4_set_bits(X86_CR4_VMXE); 2316 intel_pt_handle_vmx(1); 2317 2318 asm_volatile_goto("1: vmxon %[vmxon_pointer]\n\t" 2319 _ASM_EXTABLE(1b, %l[fault]) 2320 : : [vmxon_pointer] "m"(vmxon_pointer) 2321 : : fault); 2322 return 0; 2323 2324 fault: 2325 WARN_ONCE(1, "VMXON faulted, MSR_IA32_FEAT_CTL (0x3a) = 0x%llx\n", 2326 rdmsrl_safe(MSR_IA32_FEAT_CTL, &msr) ? 0xdeadbeef : msr); 2327 intel_pt_handle_vmx(0); 2328 cr4_clear_bits(X86_CR4_VMXE); 2329 2330 return -EFAULT; 2331 } 2332 2333 static int hardware_enable(void) 2334 { 2335 int cpu = raw_smp_processor_id(); 2336 u64 phys_addr = __pa(per_cpu(vmxarea, cpu)); 2337 int r; 2338 2339 if (cr4_read_shadow() & X86_CR4_VMXE) 2340 return -EBUSY; 2341 2342 /* 2343 * This can happen if we hot-added a CPU but failed to allocate 2344 * VP assist page for it. 2345 */ 2346 if (static_branch_unlikely(&enable_evmcs) && 2347 !hv_get_vp_assist_page(cpu)) 2348 return -EFAULT; 2349 2350 r = kvm_cpu_vmxon(phys_addr); 2351 if (r) 2352 return r; 2353 2354 if (enable_ept) 2355 ept_sync_global(); 2356 2357 return 0; 2358 } 2359 2360 static void vmclear_local_loaded_vmcss(void) 2361 { 2362 int cpu = raw_smp_processor_id(); 2363 struct loaded_vmcs *v, *n; 2364 2365 list_for_each_entry_safe(v, n, &per_cpu(loaded_vmcss_on_cpu, cpu), 2366 loaded_vmcss_on_cpu_link) 2367 __loaded_vmcs_clear(v); 2368 } 2369 2370 2371 /* Just like cpu_vmxoff(), but with the __kvm_handle_fault_on_reboot() 2372 * tricks. 2373 */ 2374 static void kvm_cpu_vmxoff(void) 2375 { 2376 asm volatile (__ex("vmxoff")); 2377 2378 intel_pt_handle_vmx(0); 2379 cr4_clear_bits(X86_CR4_VMXE); 2380 } 2381 2382 static void hardware_disable(void) 2383 { 2384 vmclear_local_loaded_vmcss(); 2385 kvm_cpu_vmxoff(); 2386 } 2387 2388 /* 2389 * There is no X86_FEATURE for SGX yet, but anyway we need to query CPUID 2390 * directly instead of going through cpu_has(), to ensure KVM is trapping 2391 * ENCLS whenever it's supported in hardware. It does not matter whether 2392 * the host OS supports or has enabled SGX. 2393 */ 2394 static bool cpu_has_sgx(void) 2395 { 2396 return cpuid_eax(0) >= 0x12 && (cpuid_eax(0x12) & BIT(0)); 2397 } 2398 2399 static __init int adjust_vmx_controls(u32 ctl_min, u32 ctl_opt, 2400 u32 msr, u32 *result) 2401 { 2402 u32 vmx_msr_low, vmx_msr_high; 2403 u32 ctl = ctl_min | ctl_opt; 2404 2405 rdmsr(msr, vmx_msr_low, vmx_msr_high); 2406 2407 ctl &= vmx_msr_high; /* bit == 0 in high word ==> must be zero */ 2408 ctl |= vmx_msr_low; /* bit == 1 in low word ==> must be one */ 2409 2410 /* Ensure minimum (required) set of control bits are supported. */ 2411 if (ctl_min & ~ctl) 2412 return -EIO; 2413 2414 *result = ctl; 2415 return 0; 2416 } 2417 2418 static __init int setup_vmcs_config(struct vmcs_config *vmcs_conf, 2419 struct vmx_capability *vmx_cap) 2420 { 2421 u32 vmx_msr_low, vmx_msr_high; 2422 u32 min, opt, min2, opt2; 2423 u32 _pin_based_exec_control = 0; 2424 u32 _cpu_based_exec_control = 0; 2425 u32 _cpu_based_2nd_exec_control = 0; 2426 u32 _vmexit_control = 0; 2427 u32 _vmentry_control = 0; 2428 2429 memset(vmcs_conf, 0, sizeof(*vmcs_conf)); 2430 min = CPU_BASED_HLT_EXITING | 2431 #ifdef CONFIG_X86_64 2432 CPU_BASED_CR8_LOAD_EXITING | 2433 CPU_BASED_CR8_STORE_EXITING | 2434 #endif 2435 CPU_BASED_CR3_LOAD_EXITING | 2436 CPU_BASED_CR3_STORE_EXITING | 2437 CPU_BASED_UNCOND_IO_EXITING | 2438 CPU_BASED_MOV_DR_EXITING | 2439 CPU_BASED_USE_TSC_OFFSETTING | 2440 CPU_BASED_MWAIT_EXITING | 2441 CPU_BASED_MONITOR_EXITING | 2442 CPU_BASED_INVLPG_EXITING | 2443 CPU_BASED_RDPMC_EXITING; 2444 2445 opt = CPU_BASED_TPR_SHADOW | 2446 CPU_BASED_USE_MSR_BITMAPS | 2447 CPU_BASED_ACTIVATE_SECONDARY_CONTROLS; 2448 if (adjust_vmx_controls(min, opt, MSR_IA32_VMX_PROCBASED_CTLS, 2449 &_cpu_based_exec_control) < 0) 2450 return -EIO; 2451 #ifdef CONFIG_X86_64 2452 if ((_cpu_based_exec_control & CPU_BASED_TPR_SHADOW)) 2453 _cpu_based_exec_control &= ~CPU_BASED_CR8_LOAD_EXITING & 2454 ~CPU_BASED_CR8_STORE_EXITING; 2455 #endif 2456 if (_cpu_based_exec_control & CPU_BASED_ACTIVATE_SECONDARY_CONTROLS) { 2457 min2 = 0; 2458 opt2 = SECONDARY_EXEC_VIRTUALIZE_APIC_ACCESSES | 2459 SECONDARY_EXEC_VIRTUALIZE_X2APIC_MODE | 2460 SECONDARY_EXEC_WBINVD_EXITING | 2461 SECONDARY_EXEC_ENABLE_VPID | 2462 SECONDARY_EXEC_ENABLE_EPT | 2463 SECONDARY_EXEC_UNRESTRICTED_GUEST | 2464 SECONDARY_EXEC_PAUSE_LOOP_EXITING | 2465 SECONDARY_EXEC_DESC | 2466 SECONDARY_EXEC_RDTSCP | 2467 SECONDARY_EXEC_ENABLE_INVPCID | 2468 SECONDARY_EXEC_APIC_REGISTER_VIRT | 2469 SECONDARY_EXEC_VIRTUAL_INTR_DELIVERY | 2470 SECONDARY_EXEC_SHADOW_VMCS | 2471 SECONDARY_EXEC_XSAVES | 2472 SECONDARY_EXEC_RDSEED_EXITING | 2473 SECONDARY_EXEC_RDRAND_EXITING | 2474 SECONDARY_EXEC_ENABLE_PML | 2475 SECONDARY_EXEC_TSC_SCALING | 2476 SECONDARY_EXEC_ENABLE_USR_WAIT_PAUSE | 2477 SECONDARY_EXEC_PT_USE_GPA | 2478 SECONDARY_EXEC_PT_CONCEAL_VMX | 2479 SECONDARY_EXEC_ENABLE_VMFUNC; 2480 if (cpu_has_sgx()) 2481 opt2 |= SECONDARY_EXEC_ENCLS_EXITING; 2482 if (adjust_vmx_controls(min2, opt2, 2483 MSR_IA32_VMX_PROCBASED_CTLS2, 2484 &_cpu_based_2nd_exec_control) < 0) 2485 return -EIO; 2486 } 2487 #ifndef CONFIG_X86_64 2488 if (!(_cpu_based_2nd_exec_control & 2489 SECONDARY_EXEC_VIRTUALIZE_APIC_ACCESSES)) 2490 _cpu_based_exec_control &= ~CPU_BASED_TPR_SHADOW; 2491 #endif 2492 2493 if (!(_cpu_based_exec_control & CPU_BASED_TPR_SHADOW)) 2494 _cpu_based_2nd_exec_control &= ~( 2495 SECONDARY_EXEC_APIC_REGISTER_VIRT | 2496 SECONDARY_EXEC_VIRTUALIZE_X2APIC_MODE | 2497 SECONDARY_EXEC_VIRTUAL_INTR_DELIVERY); 2498 2499 rdmsr_safe(MSR_IA32_VMX_EPT_VPID_CAP, 2500 &vmx_cap->ept, &vmx_cap->vpid); 2501 2502 if (_cpu_based_2nd_exec_control & SECONDARY_EXEC_ENABLE_EPT) { 2503 /* CR3 accesses and invlpg don't need to cause VM Exits when EPT 2504 enabled */ 2505 _cpu_based_exec_control &= ~(CPU_BASED_CR3_LOAD_EXITING | 2506 CPU_BASED_CR3_STORE_EXITING | 2507 CPU_BASED_INVLPG_EXITING); 2508 } else if (vmx_cap->ept) { 2509 vmx_cap->ept = 0; 2510 pr_warn_once("EPT CAP should not exist if not support " 2511 "1-setting enable EPT VM-execution control\n"); 2512 } 2513 if (!(_cpu_based_2nd_exec_control & SECONDARY_EXEC_ENABLE_VPID) && 2514 vmx_cap->vpid) { 2515 vmx_cap->vpid = 0; 2516 pr_warn_once("VPID CAP should not exist if not support " 2517 "1-setting enable VPID VM-execution control\n"); 2518 } 2519 2520 min = VM_EXIT_SAVE_DEBUG_CONTROLS | VM_EXIT_ACK_INTR_ON_EXIT; 2521 #ifdef CONFIG_X86_64 2522 min |= VM_EXIT_HOST_ADDR_SPACE_SIZE; 2523 #endif 2524 opt = VM_EXIT_LOAD_IA32_PERF_GLOBAL_CTRL | 2525 VM_EXIT_LOAD_IA32_PAT | 2526 VM_EXIT_LOAD_IA32_EFER | 2527 VM_EXIT_CLEAR_BNDCFGS | 2528 VM_EXIT_PT_CONCEAL_PIP | 2529 VM_EXIT_CLEAR_IA32_RTIT_CTL; 2530 if (adjust_vmx_controls(min, opt, MSR_IA32_VMX_EXIT_CTLS, 2531 &_vmexit_control) < 0) 2532 return -EIO; 2533 2534 min = PIN_BASED_EXT_INTR_MASK | PIN_BASED_NMI_EXITING; 2535 opt = PIN_BASED_VIRTUAL_NMIS | PIN_BASED_POSTED_INTR | 2536 PIN_BASED_VMX_PREEMPTION_TIMER; 2537 if (adjust_vmx_controls(min, opt, MSR_IA32_VMX_PINBASED_CTLS, 2538 &_pin_based_exec_control) < 0) 2539 return -EIO; 2540 2541 if (cpu_has_broken_vmx_preemption_timer()) 2542 _pin_based_exec_control &= ~PIN_BASED_VMX_PREEMPTION_TIMER; 2543 if (!(_cpu_based_2nd_exec_control & 2544 SECONDARY_EXEC_VIRTUAL_INTR_DELIVERY)) 2545 _pin_based_exec_control &= ~PIN_BASED_POSTED_INTR; 2546 2547 min = VM_ENTRY_LOAD_DEBUG_CONTROLS; 2548 opt = VM_ENTRY_LOAD_IA32_PERF_GLOBAL_CTRL | 2549 VM_ENTRY_LOAD_IA32_PAT | 2550 VM_ENTRY_LOAD_IA32_EFER | 2551 VM_ENTRY_LOAD_BNDCFGS | 2552 VM_ENTRY_PT_CONCEAL_PIP | 2553 VM_ENTRY_LOAD_IA32_RTIT_CTL; 2554 if (adjust_vmx_controls(min, opt, MSR_IA32_VMX_ENTRY_CTLS, 2555 &_vmentry_control) < 0) 2556 return -EIO; 2557 2558 /* 2559 * Some cpus support VM_{ENTRY,EXIT}_IA32_PERF_GLOBAL_CTRL but they 2560 * can't be used due to an errata where VM Exit may incorrectly clear 2561 * IA32_PERF_GLOBAL_CTRL[34:32]. Workaround the errata by using the 2562 * MSR load mechanism to switch IA32_PERF_GLOBAL_CTRL. 2563 */ 2564 if (boot_cpu_data.x86 == 0x6) { 2565 switch (boot_cpu_data.x86_model) { 2566 case 26: /* AAK155 */ 2567 case 30: /* AAP115 */ 2568 case 37: /* AAT100 */ 2569 case 44: /* BC86,AAY89,BD102 */ 2570 case 46: /* BA97 */ 2571 _vmentry_control &= ~VM_ENTRY_LOAD_IA32_PERF_GLOBAL_CTRL; 2572 _vmexit_control &= ~VM_EXIT_LOAD_IA32_PERF_GLOBAL_CTRL; 2573 pr_warn_once("kvm: VM_EXIT_LOAD_IA32_PERF_GLOBAL_CTRL " 2574 "does not work properly. Using workaround\n"); 2575 break; 2576 default: 2577 break; 2578 } 2579 } 2580 2581 2582 rdmsr(MSR_IA32_VMX_BASIC, vmx_msr_low, vmx_msr_high); 2583 2584 /* IA-32 SDM Vol 3B: VMCS size is never greater than 4kB. */ 2585 if ((vmx_msr_high & 0x1fff) > PAGE_SIZE) 2586 return -EIO; 2587 2588 #ifdef CONFIG_X86_64 2589 /* IA-32 SDM Vol 3B: 64-bit CPUs always have VMX_BASIC_MSR[48]==0. */ 2590 if (vmx_msr_high & (1u<<16)) 2591 return -EIO; 2592 #endif 2593 2594 /* Require Write-Back (WB) memory type for VMCS accesses. */ 2595 if (((vmx_msr_high >> 18) & 15) != 6) 2596 return -EIO; 2597 2598 vmcs_conf->size = vmx_msr_high & 0x1fff; 2599 vmcs_conf->order = get_order(vmcs_conf->size); 2600 vmcs_conf->basic_cap = vmx_msr_high & ~0x1fff; 2601 2602 vmcs_conf->revision_id = vmx_msr_low; 2603 2604 vmcs_conf->pin_based_exec_ctrl = _pin_based_exec_control; 2605 vmcs_conf->cpu_based_exec_ctrl = _cpu_based_exec_control; 2606 vmcs_conf->cpu_based_2nd_exec_ctrl = _cpu_based_2nd_exec_control; 2607 vmcs_conf->vmexit_ctrl = _vmexit_control; 2608 vmcs_conf->vmentry_ctrl = _vmentry_control; 2609 2610 if (static_branch_unlikely(&enable_evmcs)) 2611 evmcs_sanitize_exec_ctrls(vmcs_conf); 2612 2613 return 0; 2614 } 2615 2616 struct vmcs *alloc_vmcs_cpu(bool shadow, int cpu, gfp_t flags) 2617 { 2618 int node = cpu_to_node(cpu); 2619 struct page *pages; 2620 struct vmcs *vmcs; 2621 2622 pages = __alloc_pages_node(node, flags, vmcs_config.order); 2623 if (!pages) 2624 return NULL; 2625 vmcs = page_address(pages); 2626 memset(vmcs, 0, vmcs_config.size); 2627 2628 /* KVM supports Enlightened VMCS v1 only */ 2629 if (static_branch_unlikely(&enable_evmcs)) 2630 vmcs->hdr.revision_id = KVM_EVMCS_VERSION; 2631 else 2632 vmcs->hdr.revision_id = vmcs_config.revision_id; 2633 2634 if (shadow) 2635 vmcs->hdr.shadow_vmcs = 1; 2636 return vmcs; 2637 } 2638 2639 void free_vmcs(struct vmcs *vmcs) 2640 { 2641 free_pages((unsigned long)vmcs, vmcs_config.order); 2642 } 2643 2644 /* 2645 * Free a VMCS, but before that VMCLEAR it on the CPU where it was last loaded 2646 */ 2647 void free_loaded_vmcs(struct loaded_vmcs *loaded_vmcs) 2648 { 2649 if (!loaded_vmcs->vmcs) 2650 return; 2651 loaded_vmcs_clear(loaded_vmcs); 2652 free_vmcs(loaded_vmcs->vmcs); 2653 loaded_vmcs->vmcs = NULL; 2654 if (loaded_vmcs->msr_bitmap) 2655 free_page((unsigned long)loaded_vmcs->msr_bitmap); 2656 WARN_ON(loaded_vmcs->shadow_vmcs != NULL); 2657 } 2658 2659 int alloc_loaded_vmcs(struct loaded_vmcs *loaded_vmcs) 2660 { 2661 loaded_vmcs->vmcs = alloc_vmcs(false); 2662 if (!loaded_vmcs->vmcs) 2663 return -ENOMEM; 2664 2665 vmcs_clear(loaded_vmcs->vmcs); 2666 2667 loaded_vmcs->shadow_vmcs = NULL; 2668 loaded_vmcs->hv_timer_soft_disabled = false; 2669 loaded_vmcs->cpu = -1; 2670 loaded_vmcs->launched = 0; 2671 2672 if (cpu_has_vmx_msr_bitmap()) { 2673 loaded_vmcs->msr_bitmap = (unsigned long *) 2674 __get_free_page(GFP_KERNEL_ACCOUNT); 2675 if (!loaded_vmcs->msr_bitmap) 2676 goto out_vmcs; 2677 memset(loaded_vmcs->msr_bitmap, 0xff, PAGE_SIZE); 2678 2679 if (IS_ENABLED(CONFIG_HYPERV) && 2680 static_branch_unlikely(&enable_evmcs) && 2681 (ms_hyperv.nested_features & HV_X64_NESTED_MSR_BITMAP)) { 2682 struct hv_enlightened_vmcs *evmcs = 2683 (struct hv_enlightened_vmcs *)loaded_vmcs->vmcs; 2684 2685 evmcs->hv_enlightenments_control.msr_bitmap = 1; 2686 } 2687 } 2688 2689 memset(&loaded_vmcs->host_state, 0, sizeof(struct vmcs_host_state)); 2690 memset(&loaded_vmcs->controls_shadow, 0, 2691 sizeof(struct vmcs_controls_shadow)); 2692 2693 return 0; 2694 2695 out_vmcs: 2696 free_loaded_vmcs(loaded_vmcs); 2697 return -ENOMEM; 2698 } 2699 2700 static void free_kvm_area(void) 2701 { 2702 int cpu; 2703 2704 for_each_possible_cpu(cpu) { 2705 free_vmcs(per_cpu(vmxarea, cpu)); 2706 per_cpu(vmxarea, cpu) = NULL; 2707 } 2708 } 2709 2710 static __init int alloc_kvm_area(void) 2711 { 2712 int cpu; 2713 2714 for_each_possible_cpu(cpu) { 2715 struct vmcs *vmcs; 2716 2717 vmcs = alloc_vmcs_cpu(false, cpu, GFP_KERNEL); 2718 if (!vmcs) { 2719 free_kvm_area(); 2720 return -ENOMEM; 2721 } 2722 2723 /* 2724 * When eVMCS is enabled, alloc_vmcs_cpu() sets 2725 * vmcs->revision_id to KVM_EVMCS_VERSION instead of 2726 * revision_id reported by MSR_IA32_VMX_BASIC. 2727 * 2728 * However, even though not explicitly documented by 2729 * TLFS, VMXArea passed as VMXON argument should 2730 * still be marked with revision_id reported by 2731 * physical CPU. 2732 */ 2733 if (static_branch_unlikely(&enable_evmcs)) 2734 vmcs->hdr.revision_id = vmcs_config.revision_id; 2735 2736 per_cpu(vmxarea, cpu) = vmcs; 2737 } 2738 return 0; 2739 } 2740 2741 static void fix_pmode_seg(struct kvm_vcpu *vcpu, int seg, 2742 struct kvm_segment *save) 2743 { 2744 if (!emulate_invalid_guest_state) { 2745 /* 2746 * CS and SS RPL should be equal during guest entry according 2747 * to VMX spec, but in reality it is not always so. Since vcpu 2748 * is in the middle of the transition from real mode to 2749 * protected mode it is safe to assume that RPL 0 is a good 2750 * default value. 2751 */ 2752 if (seg == VCPU_SREG_CS || seg == VCPU_SREG_SS) 2753 save->selector &= ~SEGMENT_RPL_MASK; 2754 save->dpl = save->selector & SEGMENT_RPL_MASK; 2755 save->s = 1; 2756 } 2757 vmx_set_segment(vcpu, save, seg); 2758 } 2759 2760 static void enter_pmode(struct kvm_vcpu *vcpu) 2761 { 2762 unsigned long flags; 2763 struct vcpu_vmx *vmx = to_vmx(vcpu); 2764 2765 /* 2766 * Update real mode segment cache. It may be not up-to-date if sement 2767 * register was written while vcpu was in a guest mode. 2768 */ 2769 vmx_get_segment(vcpu, &vmx->rmode.segs[VCPU_SREG_ES], VCPU_SREG_ES); 2770 vmx_get_segment(vcpu, &vmx->rmode.segs[VCPU_SREG_DS], VCPU_SREG_DS); 2771 vmx_get_segment(vcpu, &vmx->rmode.segs[VCPU_SREG_FS], VCPU_SREG_FS); 2772 vmx_get_segment(vcpu, &vmx->rmode.segs[VCPU_SREG_GS], VCPU_SREG_GS); 2773 vmx_get_segment(vcpu, &vmx->rmode.segs[VCPU_SREG_SS], VCPU_SREG_SS); 2774 vmx_get_segment(vcpu, &vmx->rmode.segs[VCPU_SREG_CS], VCPU_SREG_CS); 2775 2776 vmx->rmode.vm86_active = 0; 2777 2778 vmx_set_segment(vcpu, &vmx->rmode.segs[VCPU_SREG_TR], VCPU_SREG_TR); 2779 2780 flags = vmcs_readl(GUEST_RFLAGS); 2781 flags &= RMODE_GUEST_OWNED_EFLAGS_BITS; 2782 flags |= vmx->rmode.save_rflags & ~RMODE_GUEST_OWNED_EFLAGS_BITS; 2783 vmcs_writel(GUEST_RFLAGS, flags); 2784 2785 vmcs_writel(GUEST_CR4, (vmcs_readl(GUEST_CR4) & ~X86_CR4_VME) | 2786 (vmcs_readl(CR4_READ_SHADOW) & X86_CR4_VME)); 2787 2788 update_exception_bitmap(vcpu); 2789 2790 fix_pmode_seg(vcpu, VCPU_SREG_CS, &vmx->rmode.segs[VCPU_SREG_CS]); 2791 fix_pmode_seg(vcpu, VCPU_SREG_SS, &vmx->rmode.segs[VCPU_SREG_SS]); 2792 fix_pmode_seg(vcpu, VCPU_SREG_ES, &vmx->rmode.segs[VCPU_SREG_ES]); 2793 fix_pmode_seg(vcpu, VCPU_SREG_DS, &vmx->rmode.segs[VCPU_SREG_DS]); 2794 fix_pmode_seg(vcpu, VCPU_SREG_FS, &vmx->rmode.segs[VCPU_SREG_FS]); 2795 fix_pmode_seg(vcpu, VCPU_SREG_GS, &vmx->rmode.segs[VCPU_SREG_GS]); 2796 } 2797 2798 static void fix_rmode_seg(int seg, struct kvm_segment *save) 2799 { 2800 const struct kvm_vmx_segment_field *sf = &kvm_vmx_segment_fields[seg]; 2801 struct kvm_segment var = *save; 2802 2803 var.dpl = 0x3; 2804 if (seg == VCPU_SREG_CS) 2805 var.type = 0x3; 2806 2807 if (!emulate_invalid_guest_state) { 2808 var.selector = var.base >> 4; 2809 var.base = var.base & 0xffff0; 2810 var.limit = 0xffff; 2811 var.g = 0; 2812 var.db = 0; 2813 var.present = 1; 2814 var.s = 1; 2815 var.l = 0; 2816 var.unusable = 0; 2817 var.type = 0x3; 2818 var.avl = 0; 2819 if (save->base & 0xf) 2820 printk_once(KERN_WARNING "kvm: segment base is not " 2821 "paragraph aligned when entering " 2822 "protected mode (seg=%d)", seg); 2823 } 2824 2825 vmcs_write16(sf->selector, var.selector); 2826 vmcs_writel(sf->base, var.base); 2827 vmcs_write32(sf->limit, var.limit); 2828 vmcs_write32(sf->ar_bytes, vmx_segment_access_rights(&var)); 2829 } 2830 2831 static void enter_rmode(struct kvm_vcpu *vcpu) 2832 { 2833 unsigned long flags; 2834 struct vcpu_vmx *vmx = to_vmx(vcpu); 2835 struct kvm_vmx *kvm_vmx = to_kvm_vmx(vcpu->kvm); 2836 2837 vmx_get_segment(vcpu, &vmx->rmode.segs[VCPU_SREG_TR], VCPU_SREG_TR); 2838 vmx_get_segment(vcpu, &vmx->rmode.segs[VCPU_SREG_ES], VCPU_SREG_ES); 2839 vmx_get_segment(vcpu, &vmx->rmode.segs[VCPU_SREG_DS], VCPU_SREG_DS); 2840 vmx_get_segment(vcpu, &vmx->rmode.segs[VCPU_SREG_FS], VCPU_SREG_FS); 2841 vmx_get_segment(vcpu, &vmx->rmode.segs[VCPU_SREG_GS], VCPU_SREG_GS); 2842 vmx_get_segment(vcpu, &vmx->rmode.segs[VCPU_SREG_SS], VCPU_SREG_SS); 2843 vmx_get_segment(vcpu, &vmx->rmode.segs[VCPU_SREG_CS], VCPU_SREG_CS); 2844 2845 vmx->rmode.vm86_active = 1; 2846 2847 /* 2848 * Very old userspace does not call KVM_SET_TSS_ADDR before entering 2849 * vcpu. Warn the user that an update is overdue. 2850 */ 2851 if (!kvm_vmx->tss_addr) 2852 printk_once(KERN_WARNING "kvm: KVM_SET_TSS_ADDR need to be " 2853 "called before entering vcpu\n"); 2854 2855 vmx_segment_cache_clear(vmx); 2856 2857 vmcs_writel(GUEST_TR_BASE, kvm_vmx->tss_addr); 2858 vmcs_write32(GUEST_TR_LIMIT, RMODE_TSS_SIZE - 1); 2859 vmcs_write32(GUEST_TR_AR_BYTES, 0x008b); 2860 2861 flags = vmcs_readl(GUEST_RFLAGS); 2862 vmx->rmode.save_rflags = flags; 2863 2864 flags |= X86_EFLAGS_IOPL | X86_EFLAGS_VM; 2865 2866 vmcs_writel(GUEST_RFLAGS, flags); 2867 vmcs_writel(GUEST_CR4, vmcs_readl(GUEST_CR4) | X86_CR4_VME); 2868 update_exception_bitmap(vcpu); 2869 2870 fix_rmode_seg(VCPU_SREG_SS, &vmx->rmode.segs[VCPU_SREG_SS]); 2871 fix_rmode_seg(VCPU_SREG_CS, &vmx->rmode.segs[VCPU_SREG_CS]); 2872 fix_rmode_seg(VCPU_SREG_ES, &vmx->rmode.segs[VCPU_SREG_ES]); 2873 fix_rmode_seg(VCPU_SREG_DS, &vmx->rmode.segs[VCPU_SREG_DS]); 2874 fix_rmode_seg(VCPU_SREG_GS, &vmx->rmode.segs[VCPU_SREG_GS]); 2875 fix_rmode_seg(VCPU_SREG_FS, &vmx->rmode.segs[VCPU_SREG_FS]); 2876 2877 kvm_mmu_reset_context(vcpu); 2878 } 2879 2880 void vmx_set_efer(struct kvm_vcpu *vcpu, u64 efer) 2881 { 2882 struct vcpu_vmx *vmx = to_vmx(vcpu); 2883 struct shared_msr_entry *msr = find_msr_entry(vmx, MSR_EFER); 2884 2885 if (!msr) 2886 return; 2887 2888 vcpu->arch.efer = efer; 2889 if (efer & EFER_LMA) { 2890 vm_entry_controls_setbit(to_vmx(vcpu), VM_ENTRY_IA32E_MODE); 2891 msr->data = efer; 2892 } else { 2893 vm_entry_controls_clearbit(to_vmx(vcpu), VM_ENTRY_IA32E_MODE); 2894 2895 msr->data = efer & ~EFER_LME; 2896 } 2897 setup_msrs(vmx); 2898 } 2899 2900 #ifdef CONFIG_X86_64 2901 2902 static void enter_lmode(struct kvm_vcpu *vcpu) 2903 { 2904 u32 guest_tr_ar; 2905 2906 vmx_segment_cache_clear(to_vmx(vcpu)); 2907 2908 guest_tr_ar = vmcs_read32(GUEST_TR_AR_BYTES); 2909 if ((guest_tr_ar & VMX_AR_TYPE_MASK) != VMX_AR_TYPE_BUSY_64_TSS) { 2910 pr_debug_ratelimited("%s: tss fixup for long mode. \n", 2911 __func__); 2912 vmcs_write32(GUEST_TR_AR_BYTES, 2913 (guest_tr_ar & ~VMX_AR_TYPE_MASK) 2914 | VMX_AR_TYPE_BUSY_64_TSS); 2915 } 2916 vmx_set_efer(vcpu, vcpu->arch.efer | EFER_LMA); 2917 } 2918 2919 static void exit_lmode(struct kvm_vcpu *vcpu) 2920 { 2921 vm_entry_controls_clearbit(to_vmx(vcpu), VM_ENTRY_IA32E_MODE); 2922 vmx_set_efer(vcpu, vcpu->arch.efer & ~EFER_LMA); 2923 } 2924 2925 #endif 2926 2927 static void vmx_flush_tlb_all(struct kvm_vcpu *vcpu) 2928 { 2929 struct vcpu_vmx *vmx = to_vmx(vcpu); 2930 2931 /* 2932 * INVEPT must be issued when EPT is enabled, irrespective of VPID, as 2933 * the CPU is not required to invalidate guest-physical mappings on 2934 * VM-Entry, even if VPID is disabled. Guest-physical mappings are 2935 * associated with the root EPT structure and not any particular VPID 2936 * (INVVPID also isn't required to invalidate guest-physical mappings). 2937 */ 2938 if (enable_ept) { 2939 ept_sync_global(); 2940 } else if (enable_vpid) { 2941 if (cpu_has_vmx_invvpid_global()) { 2942 vpid_sync_vcpu_global(); 2943 } else { 2944 vpid_sync_vcpu_single(vmx->vpid); 2945 vpid_sync_vcpu_single(vmx->nested.vpid02); 2946 } 2947 } 2948 } 2949 2950 static void vmx_flush_tlb_current(struct kvm_vcpu *vcpu) 2951 { 2952 struct kvm_mmu *mmu = vcpu->arch.mmu; 2953 u64 root_hpa = mmu->root_hpa; 2954 2955 /* No flush required if the current context is invalid. */ 2956 if (!VALID_PAGE(root_hpa)) 2957 return; 2958 2959 if (enable_ept) 2960 ept_sync_context(construct_eptp(vcpu, root_hpa, 2961 mmu->shadow_root_level)); 2962 else if (!is_guest_mode(vcpu)) 2963 vpid_sync_context(to_vmx(vcpu)->vpid); 2964 else 2965 vpid_sync_context(nested_get_vpid02(vcpu)); 2966 } 2967 2968 static void vmx_flush_tlb_gva(struct kvm_vcpu *vcpu, gva_t addr) 2969 { 2970 /* 2971 * vpid_sync_vcpu_addr() is a nop if vmx->vpid==0, see the comment in 2972 * vmx_flush_tlb_guest() for an explanation of why this is ok. 2973 */ 2974 vpid_sync_vcpu_addr(to_vmx(vcpu)->vpid, addr); 2975 } 2976 2977 static void vmx_flush_tlb_guest(struct kvm_vcpu *vcpu) 2978 { 2979 /* 2980 * vpid_sync_context() is a nop if vmx->vpid==0, e.g. if enable_vpid==0 2981 * or a vpid couldn't be allocated for this vCPU. VM-Enter and VM-Exit 2982 * are required to flush GVA->{G,H}PA mappings from the TLB if vpid is 2983 * disabled (VM-Enter with vpid enabled and vpid==0 is disallowed), 2984 * i.e. no explicit INVVPID is necessary. 2985 */ 2986 vpid_sync_context(to_vmx(vcpu)->vpid); 2987 } 2988 2989 void vmx_ept_load_pdptrs(struct kvm_vcpu *vcpu) 2990 { 2991 struct kvm_mmu *mmu = vcpu->arch.walk_mmu; 2992 2993 if (!kvm_register_is_dirty(vcpu, VCPU_EXREG_PDPTR)) 2994 return; 2995 2996 if (is_pae_paging(vcpu)) { 2997 vmcs_write64(GUEST_PDPTR0, mmu->pdptrs[0]); 2998 vmcs_write64(GUEST_PDPTR1, mmu->pdptrs[1]); 2999 vmcs_write64(GUEST_PDPTR2, mmu->pdptrs[2]); 3000 vmcs_write64(GUEST_PDPTR3, mmu->pdptrs[3]); 3001 } 3002 } 3003 3004 void ept_save_pdptrs(struct kvm_vcpu *vcpu) 3005 { 3006 struct kvm_mmu *mmu = vcpu->arch.walk_mmu; 3007 3008 if (WARN_ON_ONCE(!is_pae_paging(vcpu))) 3009 return; 3010 3011 mmu->pdptrs[0] = vmcs_read64(GUEST_PDPTR0); 3012 mmu->pdptrs[1] = vmcs_read64(GUEST_PDPTR1); 3013 mmu->pdptrs[2] = vmcs_read64(GUEST_PDPTR2); 3014 mmu->pdptrs[3] = vmcs_read64(GUEST_PDPTR3); 3015 3016 kvm_register_mark_dirty(vcpu, VCPU_EXREG_PDPTR); 3017 } 3018 3019 static void ept_update_paging_mode_cr0(unsigned long *hw_cr0, 3020 unsigned long cr0, 3021 struct kvm_vcpu *vcpu) 3022 { 3023 struct vcpu_vmx *vmx = to_vmx(vcpu); 3024 3025 if (!kvm_register_is_available(vcpu, VCPU_EXREG_CR3)) 3026 vmx_cache_reg(vcpu, VCPU_EXREG_CR3); 3027 if (!(cr0 & X86_CR0_PG)) { 3028 /* From paging/starting to nonpaging */ 3029 exec_controls_setbit(vmx, CPU_BASED_CR3_LOAD_EXITING | 3030 CPU_BASED_CR3_STORE_EXITING); 3031 vcpu->arch.cr0 = cr0; 3032 vmx_set_cr4(vcpu, kvm_read_cr4(vcpu)); 3033 } else if (!is_paging(vcpu)) { 3034 /* From nonpaging to paging */ 3035 exec_controls_clearbit(vmx, CPU_BASED_CR3_LOAD_EXITING | 3036 CPU_BASED_CR3_STORE_EXITING); 3037 vcpu->arch.cr0 = cr0; 3038 vmx_set_cr4(vcpu, kvm_read_cr4(vcpu)); 3039 } 3040 3041 if (!(cr0 & X86_CR0_WP)) 3042 *hw_cr0 &= ~X86_CR0_WP; 3043 } 3044 3045 void vmx_set_cr0(struct kvm_vcpu *vcpu, unsigned long cr0) 3046 { 3047 struct vcpu_vmx *vmx = to_vmx(vcpu); 3048 unsigned long hw_cr0; 3049 3050 hw_cr0 = (cr0 & ~KVM_VM_CR0_ALWAYS_OFF); 3051 if (enable_unrestricted_guest) 3052 hw_cr0 |= KVM_VM_CR0_ALWAYS_ON_UNRESTRICTED_GUEST; 3053 else { 3054 hw_cr0 |= KVM_VM_CR0_ALWAYS_ON; 3055 3056 if (vmx->rmode.vm86_active && (cr0 & X86_CR0_PE)) 3057 enter_pmode(vcpu); 3058 3059 if (!vmx->rmode.vm86_active && !(cr0 & X86_CR0_PE)) 3060 enter_rmode(vcpu); 3061 } 3062 3063 #ifdef CONFIG_X86_64 3064 if (vcpu->arch.efer & EFER_LME) { 3065 if (!is_paging(vcpu) && (cr0 & X86_CR0_PG)) 3066 enter_lmode(vcpu); 3067 if (is_paging(vcpu) && !(cr0 & X86_CR0_PG)) 3068 exit_lmode(vcpu); 3069 } 3070 #endif 3071 3072 if (enable_ept && !enable_unrestricted_guest) 3073 ept_update_paging_mode_cr0(&hw_cr0, cr0, vcpu); 3074 3075 vmcs_writel(CR0_READ_SHADOW, cr0); 3076 vmcs_writel(GUEST_CR0, hw_cr0); 3077 vcpu->arch.cr0 = cr0; 3078 kvm_register_mark_available(vcpu, VCPU_EXREG_CR0); 3079 3080 /* depends on vcpu->arch.cr0 to be set to a new value */ 3081 vmx->emulation_required = emulation_required(vcpu); 3082 } 3083 3084 static int vmx_get_max_tdp_level(void) 3085 { 3086 if (cpu_has_vmx_ept_5levels()) 3087 return 5; 3088 return 4; 3089 } 3090 3091 u64 construct_eptp(struct kvm_vcpu *vcpu, unsigned long root_hpa, 3092 int root_level) 3093 { 3094 u64 eptp = VMX_EPTP_MT_WB; 3095 3096 eptp |= (root_level == 5) ? VMX_EPTP_PWL_5 : VMX_EPTP_PWL_4; 3097 3098 if (enable_ept_ad_bits && 3099 (!is_guest_mode(vcpu) || nested_ept_ad_enabled(vcpu))) 3100 eptp |= VMX_EPTP_AD_ENABLE_BIT; 3101 eptp |= (root_hpa & PAGE_MASK); 3102 3103 return eptp; 3104 } 3105 3106 static void vmx_load_mmu_pgd(struct kvm_vcpu *vcpu, unsigned long pgd, 3107 int pgd_level) 3108 { 3109 struct kvm *kvm = vcpu->kvm; 3110 bool update_guest_cr3 = true; 3111 unsigned long guest_cr3; 3112 u64 eptp; 3113 3114 if (enable_ept) { 3115 eptp = construct_eptp(vcpu, pgd, pgd_level); 3116 vmcs_write64(EPT_POINTER, eptp); 3117 3118 if (kvm_x86_ops.tlb_remote_flush) { 3119 spin_lock(&to_kvm_vmx(kvm)->ept_pointer_lock); 3120 to_vmx(vcpu)->ept_pointer = eptp; 3121 to_kvm_vmx(kvm)->ept_pointers_match 3122 = EPT_POINTERS_CHECK; 3123 spin_unlock(&to_kvm_vmx(kvm)->ept_pointer_lock); 3124 } 3125 3126 if (!enable_unrestricted_guest && !is_paging(vcpu)) 3127 guest_cr3 = to_kvm_vmx(kvm)->ept_identity_map_addr; 3128 else if (test_bit(VCPU_EXREG_CR3, (ulong *)&vcpu->arch.regs_avail)) 3129 guest_cr3 = vcpu->arch.cr3; 3130 else /* vmcs01.GUEST_CR3 is already up-to-date. */ 3131 update_guest_cr3 = false; 3132 vmx_ept_load_pdptrs(vcpu); 3133 } else { 3134 guest_cr3 = pgd; 3135 } 3136 3137 if (update_guest_cr3) 3138 vmcs_writel(GUEST_CR3, guest_cr3); 3139 } 3140 3141 int vmx_set_cr4(struct kvm_vcpu *vcpu, unsigned long cr4) 3142 { 3143 struct vcpu_vmx *vmx = to_vmx(vcpu); 3144 /* 3145 * Pass through host's Machine Check Enable value to hw_cr4, which 3146 * is in force while we are in guest mode. Do not let guests control 3147 * this bit, even if host CR4.MCE == 0. 3148 */ 3149 unsigned long hw_cr4; 3150 3151 hw_cr4 = (cr4_read_shadow() & X86_CR4_MCE) | (cr4 & ~X86_CR4_MCE); 3152 if (enable_unrestricted_guest) 3153 hw_cr4 |= KVM_VM_CR4_ALWAYS_ON_UNRESTRICTED_GUEST; 3154 else if (vmx->rmode.vm86_active) 3155 hw_cr4 |= KVM_RMODE_VM_CR4_ALWAYS_ON; 3156 else 3157 hw_cr4 |= KVM_PMODE_VM_CR4_ALWAYS_ON; 3158 3159 if (!boot_cpu_has(X86_FEATURE_UMIP) && vmx_umip_emulated()) { 3160 if (cr4 & X86_CR4_UMIP) { 3161 secondary_exec_controls_setbit(vmx, SECONDARY_EXEC_DESC); 3162 hw_cr4 &= ~X86_CR4_UMIP; 3163 } else if (!is_guest_mode(vcpu) || 3164 !nested_cpu_has2(get_vmcs12(vcpu), SECONDARY_EXEC_DESC)) { 3165 secondary_exec_controls_clearbit(vmx, SECONDARY_EXEC_DESC); 3166 } 3167 } 3168 3169 if (cr4 & X86_CR4_VMXE) { 3170 /* 3171 * To use VMXON (and later other VMX instructions), a guest 3172 * must first be able to turn on cr4.VMXE (see handle_vmon()). 3173 * So basically the check on whether to allow nested VMX 3174 * is here. We operate under the default treatment of SMM, 3175 * so VMX cannot be enabled under SMM. 3176 */ 3177 if (!nested_vmx_allowed(vcpu) || is_smm(vcpu)) 3178 return 1; 3179 } 3180 3181 if (vmx->nested.vmxon && !nested_cr4_valid(vcpu, cr4)) 3182 return 1; 3183 3184 vcpu->arch.cr4 = cr4; 3185 kvm_register_mark_available(vcpu, VCPU_EXREG_CR4); 3186 3187 if (!enable_unrestricted_guest) { 3188 if (enable_ept) { 3189 if (!is_paging(vcpu)) { 3190 hw_cr4 &= ~X86_CR4_PAE; 3191 hw_cr4 |= X86_CR4_PSE; 3192 } else if (!(cr4 & X86_CR4_PAE)) { 3193 hw_cr4 &= ~X86_CR4_PAE; 3194 } 3195 } 3196 3197 /* 3198 * SMEP/SMAP/PKU is disabled if CPU is in non-paging mode in 3199 * hardware. To emulate this behavior, SMEP/SMAP/PKU needs 3200 * to be manually disabled when guest switches to non-paging 3201 * mode. 3202 * 3203 * If !enable_unrestricted_guest, the CPU is always running 3204 * with CR0.PG=1 and CR4 needs to be modified. 3205 * If enable_unrestricted_guest, the CPU automatically 3206 * disables SMEP/SMAP/PKU when the guest sets CR0.PG=0. 3207 */ 3208 if (!is_paging(vcpu)) 3209 hw_cr4 &= ~(X86_CR4_SMEP | X86_CR4_SMAP | X86_CR4_PKE); 3210 } 3211 3212 vmcs_writel(CR4_READ_SHADOW, cr4); 3213 vmcs_writel(GUEST_CR4, hw_cr4); 3214 return 0; 3215 } 3216 3217 void vmx_get_segment(struct kvm_vcpu *vcpu, struct kvm_segment *var, int seg) 3218 { 3219 struct vcpu_vmx *vmx = to_vmx(vcpu); 3220 u32 ar; 3221 3222 if (vmx->rmode.vm86_active && seg != VCPU_SREG_LDTR) { 3223 *var = vmx->rmode.segs[seg]; 3224 if (seg == VCPU_SREG_TR 3225 || var->selector == vmx_read_guest_seg_selector(vmx, seg)) 3226 return; 3227 var->base = vmx_read_guest_seg_base(vmx, seg); 3228 var->selector = vmx_read_guest_seg_selector(vmx, seg); 3229 return; 3230 } 3231 var->base = vmx_read_guest_seg_base(vmx, seg); 3232 var->limit = vmx_read_guest_seg_limit(vmx, seg); 3233 var->selector = vmx_read_guest_seg_selector(vmx, seg); 3234 ar = vmx_read_guest_seg_ar(vmx, seg); 3235 var->unusable = (ar >> 16) & 1; 3236 var->type = ar & 15; 3237 var->s = (ar >> 4) & 1; 3238 var->dpl = (ar >> 5) & 3; 3239 /* 3240 * Some userspaces do not preserve unusable property. Since usable 3241 * segment has to be present according to VMX spec we can use present 3242 * property to amend userspace bug by making unusable segment always 3243 * nonpresent. vmx_segment_access_rights() already marks nonpresent 3244 * segment as unusable. 3245 */ 3246 var->present = !var->unusable; 3247 var->avl = (ar >> 12) & 1; 3248 var->l = (ar >> 13) & 1; 3249 var->db = (ar >> 14) & 1; 3250 var->g = (ar >> 15) & 1; 3251 } 3252 3253 static u64 vmx_get_segment_base(struct kvm_vcpu *vcpu, int seg) 3254 { 3255 struct kvm_segment s; 3256 3257 if (to_vmx(vcpu)->rmode.vm86_active) { 3258 vmx_get_segment(vcpu, &s, seg); 3259 return s.base; 3260 } 3261 return vmx_read_guest_seg_base(to_vmx(vcpu), seg); 3262 } 3263 3264 int vmx_get_cpl(struct kvm_vcpu *vcpu) 3265 { 3266 struct vcpu_vmx *vmx = to_vmx(vcpu); 3267 3268 if (unlikely(vmx->rmode.vm86_active)) 3269 return 0; 3270 else { 3271 int ar = vmx_read_guest_seg_ar(vmx, VCPU_SREG_SS); 3272 return VMX_AR_DPL(ar); 3273 } 3274 } 3275 3276 static u32 vmx_segment_access_rights(struct kvm_segment *var) 3277 { 3278 u32 ar; 3279 3280 if (var->unusable || !var->present) 3281 ar = 1 << 16; 3282 else { 3283 ar = var->type & 15; 3284 ar |= (var->s & 1) << 4; 3285 ar |= (var->dpl & 3) << 5; 3286 ar |= (var->present & 1) << 7; 3287 ar |= (var->avl & 1) << 12; 3288 ar |= (var->l & 1) << 13; 3289 ar |= (var->db & 1) << 14; 3290 ar |= (var->g & 1) << 15; 3291 } 3292 3293 return ar; 3294 } 3295 3296 void vmx_set_segment(struct kvm_vcpu *vcpu, struct kvm_segment *var, int seg) 3297 { 3298 struct vcpu_vmx *vmx = to_vmx(vcpu); 3299 const struct kvm_vmx_segment_field *sf = &kvm_vmx_segment_fields[seg]; 3300 3301 vmx_segment_cache_clear(vmx); 3302 3303 if (vmx->rmode.vm86_active && seg != VCPU_SREG_LDTR) { 3304 vmx->rmode.segs[seg] = *var; 3305 if (seg == VCPU_SREG_TR) 3306 vmcs_write16(sf->selector, var->selector); 3307 else if (var->s) 3308 fix_rmode_seg(seg, &vmx->rmode.segs[seg]); 3309 goto out; 3310 } 3311 3312 vmcs_writel(sf->base, var->base); 3313 vmcs_write32(sf->limit, var->limit); 3314 vmcs_write16(sf->selector, var->selector); 3315 3316 /* 3317 * Fix the "Accessed" bit in AR field of segment registers for older 3318 * qemu binaries. 3319 * IA32 arch specifies that at the time of processor reset the 3320 * "Accessed" bit in the AR field of segment registers is 1. And qemu 3321 * is setting it to 0 in the userland code. This causes invalid guest 3322 * state vmexit when "unrestricted guest" mode is turned on. 3323 * Fix for this setup issue in cpu_reset is being pushed in the qemu 3324 * tree. Newer qemu binaries with that qemu fix would not need this 3325 * kvm hack. 3326 */ 3327 if (enable_unrestricted_guest && (seg != VCPU_SREG_LDTR)) 3328 var->type |= 0x1; /* Accessed */ 3329 3330 vmcs_write32(sf->ar_bytes, vmx_segment_access_rights(var)); 3331 3332 out: 3333 vmx->emulation_required = emulation_required(vcpu); 3334 } 3335 3336 static void vmx_get_cs_db_l_bits(struct kvm_vcpu *vcpu, int *db, int *l) 3337 { 3338 u32 ar = vmx_read_guest_seg_ar(to_vmx(vcpu), VCPU_SREG_CS); 3339 3340 *db = (ar >> 14) & 1; 3341 *l = (ar >> 13) & 1; 3342 } 3343 3344 static void vmx_get_idt(struct kvm_vcpu *vcpu, struct desc_ptr *dt) 3345 { 3346 dt->size = vmcs_read32(GUEST_IDTR_LIMIT); 3347 dt->address = vmcs_readl(GUEST_IDTR_BASE); 3348 } 3349 3350 static void vmx_set_idt(struct kvm_vcpu *vcpu, struct desc_ptr *dt) 3351 { 3352 vmcs_write32(GUEST_IDTR_LIMIT, dt->size); 3353 vmcs_writel(GUEST_IDTR_BASE, dt->address); 3354 } 3355 3356 static void vmx_get_gdt(struct kvm_vcpu *vcpu, struct desc_ptr *dt) 3357 { 3358 dt->size = vmcs_read32(GUEST_GDTR_LIMIT); 3359 dt->address = vmcs_readl(GUEST_GDTR_BASE); 3360 } 3361 3362 static void vmx_set_gdt(struct kvm_vcpu *vcpu, struct desc_ptr *dt) 3363 { 3364 vmcs_write32(GUEST_GDTR_LIMIT, dt->size); 3365 vmcs_writel(GUEST_GDTR_BASE, dt->address); 3366 } 3367 3368 static bool rmode_segment_valid(struct kvm_vcpu *vcpu, int seg) 3369 { 3370 struct kvm_segment var; 3371 u32 ar; 3372 3373 vmx_get_segment(vcpu, &var, seg); 3374 var.dpl = 0x3; 3375 if (seg == VCPU_SREG_CS) 3376 var.type = 0x3; 3377 ar = vmx_segment_access_rights(&var); 3378 3379 if (var.base != (var.selector << 4)) 3380 return false; 3381 if (var.limit != 0xffff) 3382 return false; 3383 if (ar != 0xf3) 3384 return false; 3385 3386 return true; 3387 } 3388 3389 static bool code_segment_valid(struct kvm_vcpu *vcpu) 3390 { 3391 struct kvm_segment cs; 3392 unsigned int cs_rpl; 3393 3394 vmx_get_segment(vcpu, &cs, VCPU_SREG_CS); 3395 cs_rpl = cs.selector & SEGMENT_RPL_MASK; 3396 3397 if (cs.unusable) 3398 return false; 3399 if (~cs.type & (VMX_AR_TYPE_CODE_MASK|VMX_AR_TYPE_ACCESSES_MASK)) 3400 return false; 3401 if (!cs.s) 3402 return false; 3403 if (cs.type & VMX_AR_TYPE_WRITEABLE_MASK) { 3404 if (cs.dpl > cs_rpl) 3405 return false; 3406 } else { 3407 if (cs.dpl != cs_rpl) 3408 return false; 3409 } 3410 if (!cs.present) 3411 return false; 3412 3413 /* TODO: Add Reserved field check, this'll require a new member in the kvm_segment_field structure */ 3414 return true; 3415 } 3416 3417 static bool stack_segment_valid(struct kvm_vcpu *vcpu) 3418 { 3419 struct kvm_segment ss; 3420 unsigned int ss_rpl; 3421 3422 vmx_get_segment(vcpu, &ss, VCPU_SREG_SS); 3423 ss_rpl = ss.selector & SEGMENT_RPL_MASK; 3424 3425 if (ss.unusable) 3426 return true; 3427 if (ss.type != 3 && ss.type != 7) 3428 return false; 3429 if (!ss.s) 3430 return false; 3431 if (ss.dpl != ss_rpl) /* DPL != RPL */ 3432 return false; 3433 if (!ss.present) 3434 return false; 3435 3436 return true; 3437 } 3438 3439 static bool data_segment_valid(struct kvm_vcpu *vcpu, int seg) 3440 { 3441 struct kvm_segment var; 3442 unsigned int rpl; 3443 3444 vmx_get_segment(vcpu, &var, seg); 3445 rpl = var.selector & SEGMENT_RPL_MASK; 3446 3447 if (var.unusable) 3448 return true; 3449 if (!var.s) 3450 return false; 3451 if (!var.present) 3452 return false; 3453 if (~var.type & (VMX_AR_TYPE_CODE_MASK|VMX_AR_TYPE_WRITEABLE_MASK)) { 3454 if (var.dpl < rpl) /* DPL < RPL */ 3455 return false; 3456 } 3457 3458 /* TODO: Add other members to kvm_segment_field to allow checking for other access 3459 * rights flags 3460 */ 3461 return true; 3462 } 3463 3464 static bool tr_valid(struct kvm_vcpu *vcpu) 3465 { 3466 struct kvm_segment tr; 3467 3468 vmx_get_segment(vcpu, &tr, VCPU_SREG_TR); 3469 3470 if (tr.unusable) 3471 return false; 3472 if (tr.selector & SEGMENT_TI_MASK) /* TI = 1 */ 3473 return false; 3474 if (tr.type != 3 && tr.type != 11) /* TODO: Check if guest is in IA32e mode */ 3475 return false; 3476 if (!tr.present) 3477 return false; 3478 3479 return true; 3480 } 3481 3482 static bool ldtr_valid(struct kvm_vcpu *vcpu) 3483 { 3484 struct kvm_segment ldtr; 3485 3486 vmx_get_segment(vcpu, &ldtr, VCPU_SREG_LDTR); 3487 3488 if (ldtr.unusable) 3489 return true; 3490 if (ldtr.selector & SEGMENT_TI_MASK) /* TI = 1 */ 3491 return false; 3492 if (ldtr.type != 2) 3493 return false; 3494 if (!ldtr.present) 3495 return false; 3496 3497 return true; 3498 } 3499 3500 static bool cs_ss_rpl_check(struct kvm_vcpu *vcpu) 3501 { 3502 struct kvm_segment cs, ss; 3503 3504 vmx_get_segment(vcpu, &cs, VCPU_SREG_CS); 3505 vmx_get_segment(vcpu, &ss, VCPU_SREG_SS); 3506 3507 return ((cs.selector & SEGMENT_RPL_MASK) == 3508 (ss.selector & SEGMENT_RPL_MASK)); 3509 } 3510 3511 /* 3512 * Check if guest state is valid. Returns true if valid, false if 3513 * not. 3514 * We assume that registers are always usable 3515 */ 3516 static bool guest_state_valid(struct kvm_vcpu *vcpu) 3517 { 3518 if (enable_unrestricted_guest) 3519 return true; 3520 3521 /* real mode guest state checks */ 3522 if (!is_protmode(vcpu) || (vmx_get_rflags(vcpu) & X86_EFLAGS_VM)) { 3523 if (!rmode_segment_valid(vcpu, VCPU_SREG_CS)) 3524 return false; 3525 if (!rmode_segment_valid(vcpu, VCPU_SREG_SS)) 3526 return false; 3527 if (!rmode_segment_valid(vcpu, VCPU_SREG_DS)) 3528 return false; 3529 if (!rmode_segment_valid(vcpu, VCPU_SREG_ES)) 3530 return false; 3531 if (!rmode_segment_valid(vcpu, VCPU_SREG_FS)) 3532 return false; 3533 if (!rmode_segment_valid(vcpu, VCPU_SREG_GS)) 3534 return false; 3535 } else { 3536 /* protected mode guest state checks */ 3537 if (!cs_ss_rpl_check(vcpu)) 3538 return false; 3539 if (!code_segment_valid(vcpu)) 3540 return false; 3541 if (!stack_segment_valid(vcpu)) 3542 return false; 3543 if (!data_segment_valid(vcpu, VCPU_SREG_DS)) 3544 return false; 3545 if (!data_segment_valid(vcpu, VCPU_SREG_ES)) 3546 return false; 3547 if (!data_segment_valid(vcpu, VCPU_SREG_FS)) 3548 return false; 3549 if (!data_segment_valid(vcpu, VCPU_SREG_GS)) 3550 return false; 3551 if (!tr_valid(vcpu)) 3552 return false; 3553 if (!ldtr_valid(vcpu)) 3554 return false; 3555 } 3556 /* TODO: 3557 * - Add checks on RIP 3558 * - Add checks on RFLAGS 3559 */ 3560 3561 return true; 3562 } 3563 3564 static int init_rmode_tss(struct kvm *kvm) 3565 { 3566 gfn_t fn; 3567 u16 data = 0; 3568 int idx, r; 3569 3570 idx = srcu_read_lock(&kvm->srcu); 3571 fn = to_kvm_vmx(kvm)->tss_addr >> PAGE_SHIFT; 3572 r = kvm_clear_guest_page(kvm, fn, 0, PAGE_SIZE); 3573 if (r < 0) 3574 goto out; 3575 data = TSS_BASE_SIZE + TSS_REDIRECTION_SIZE; 3576 r = kvm_write_guest_page(kvm, fn++, &data, 3577 TSS_IOPB_BASE_OFFSET, sizeof(u16)); 3578 if (r < 0) 3579 goto out; 3580 r = kvm_clear_guest_page(kvm, fn++, 0, PAGE_SIZE); 3581 if (r < 0) 3582 goto out; 3583 r = kvm_clear_guest_page(kvm, fn, 0, PAGE_SIZE); 3584 if (r < 0) 3585 goto out; 3586 data = ~0; 3587 r = kvm_write_guest_page(kvm, fn, &data, 3588 RMODE_TSS_SIZE - 2 * PAGE_SIZE - 1, 3589 sizeof(u8)); 3590 out: 3591 srcu_read_unlock(&kvm->srcu, idx); 3592 return r; 3593 } 3594 3595 static int init_rmode_identity_map(struct kvm *kvm) 3596 { 3597 struct kvm_vmx *kvm_vmx = to_kvm_vmx(kvm); 3598 int i, r = 0; 3599 kvm_pfn_t identity_map_pfn; 3600 u32 tmp; 3601 3602 /* Protect kvm_vmx->ept_identity_pagetable_done. */ 3603 mutex_lock(&kvm->slots_lock); 3604 3605 if (likely(kvm_vmx->ept_identity_pagetable_done)) 3606 goto out; 3607 3608 if (!kvm_vmx->ept_identity_map_addr) 3609 kvm_vmx->ept_identity_map_addr = VMX_EPT_IDENTITY_PAGETABLE_ADDR; 3610 identity_map_pfn = kvm_vmx->ept_identity_map_addr >> PAGE_SHIFT; 3611 3612 r = __x86_set_memory_region(kvm, IDENTITY_PAGETABLE_PRIVATE_MEMSLOT, 3613 kvm_vmx->ept_identity_map_addr, PAGE_SIZE); 3614 if (r < 0) 3615 goto out; 3616 3617 r = kvm_clear_guest_page(kvm, identity_map_pfn, 0, PAGE_SIZE); 3618 if (r < 0) 3619 goto out; 3620 /* Set up identity-mapping pagetable for EPT in real mode */ 3621 for (i = 0; i < PT32_ENT_PER_PAGE; i++) { 3622 tmp = (i << 22) + (_PAGE_PRESENT | _PAGE_RW | _PAGE_USER | 3623 _PAGE_ACCESSED | _PAGE_DIRTY | _PAGE_PSE); 3624 r = kvm_write_guest_page(kvm, identity_map_pfn, 3625 &tmp, i * sizeof(tmp), sizeof(tmp)); 3626 if (r < 0) 3627 goto out; 3628 } 3629 kvm_vmx->ept_identity_pagetable_done = true; 3630 3631 out: 3632 mutex_unlock(&kvm->slots_lock); 3633 return r; 3634 } 3635 3636 static void seg_setup(int seg) 3637 { 3638 const struct kvm_vmx_segment_field *sf = &kvm_vmx_segment_fields[seg]; 3639 unsigned int ar; 3640 3641 vmcs_write16(sf->selector, 0); 3642 vmcs_writel(sf->base, 0); 3643 vmcs_write32(sf->limit, 0xffff); 3644 ar = 0x93; 3645 if (seg == VCPU_SREG_CS) 3646 ar |= 0x08; /* code segment */ 3647 3648 vmcs_write32(sf->ar_bytes, ar); 3649 } 3650 3651 static int alloc_apic_access_page(struct kvm *kvm) 3652 { 3653 struct page *page; 3654 int r = 0; 3655 3656 mutex_lock(&kvm->slots_lock); 3657 if (kvm->arch.apic_access_page_done) 3658 goto out; 3659 r = __x86_set_memory_region(kvm, APIC_ACCESS_PAGE_PRIVATE_MEMSLOT, 3660 APIC_DEFAULT_PHYS_BASE, PAGE_SIZE); 3661 if (r) 3662 goto out; 3663 3664 page = gfn_to_page(kvm, APIC_DEFAULT_PHYS_BASE >> PAGE_SHIFT); 3665 if (is_error_page(page)) { 3666 r = -EFAULT; 3667 goto out; 3668 } 3669 3670 /* 3671 * Do not pin the page in memory, so that memory hot-unplug 3672 * is able to migrate it. 3673 */ 3674 put_page(page); 3675 kvm->arch.apic_access_page_done = true; 3676 out: 3677 mutex_unlock(&kvm->slots_lock); 3678 return r; 3679 } 3680 3681 int allocate_vpid(void) 3682 { 3683 int vpid; 3684 3685 if (!enable_vpid) 3686 return 0; 3687 spin_lock(&vmx_vpid_lock); 3688 vpid = find_first_zero_bit(vmx_vpid_bitmap, VMX_NR_VPIDS); 3689 if (vpid < VMX_NR_VPIDS) 3690 __set_bit(vpid, vmx_vpid_bitmap); 3691 else 3692 vpid = 0; 3693 spin_unlock(&vmx_vpid_lock); 3694 return vpid; 3695 } 3696 3697 void free_vpid(int vpid) 3698 { 3699 if (!enable_vpid || vpid == 0) 3700 return; 3701 spin_lock(&vmx_vpid_lock); 3702 __clear_bit(vpid, vmx_vpid_bitmap); 3703 spin_unlock(&vmx_vpid_lock); 3704 } 3705 3706 static __always_inline void vmx_disable_intercept_for_msr(unsigned long *msr_bitmap, 3707 u32 msr, int type) 3708 { 3709 int f = sizeof(unsigned long); 3710 3711 if (!cpu_has_vmx_msr_bitmap()) 3712 return; 3713 3714 if (static_branch_unlikely(&enable_evmcs)) 3715 evmcs_touch_msr_bitmap(); 3716 3717 /* 3718 * See Intel PRM Vol. 3, 20.6.9 (MSR-Bitmap Address). Early manuals 3719 * have the write-low and read-high bitmap offsets the wrong way round. 3720 * We can control MSRs 0x00000000-0x00001fff and 0xc0000000-0xc0001fff. 3721 */ 3722 if (msr <= 0x1fff) { 3723 if (type & MSR_TYPE_R) 3724 /* read-low */ 3725 __clear_bit(msr, msr_bitmap + 0x000 / f); 3726 3727 if (type & MSR_TYPE_W) 3728 /* write-low */ 3729 __clear_bit(msr, msr_bitmap + 0x800 / f); 3730 3731 } else if ((msr >= 0xc0000000) && (msr <= 0xc0001fff)) { 3732 msr &= 0x1fff; 3733 if (type & MSR_TYPE_R) 3734 /* read-high */ 3735 __clear_bit(msr, msr_bitmap + 0x400 / f); 3736 3737 if (type & MSR_TYPE_W) 3738 /* write-high */ 3739 __clear_bit(msr, msr_bitmap + 0xc00 / f); 3740 3741 } 3742 } 3743 3744 static __always_inline void vmx_enable_intercept_for_msr(unsigned long *msr_bitmap, 3745 u32 msr, int type) 3746 { 3747 int f = sizeof(unsigned long); 3748 3749 if (!cpu_has_vmx_msr_bitmap()) 3750 return; 3751 3752 if (static_branch_unlikely(&enable_evmcs)) 3753 evmcs_touch_msr_bitmap(); 3754 3755 /* 3756 * See Intel PRM Vol. 3, 20.6.9 (MSR-Bitmap Address). Early manuals 3757 * have the write-low and read-high bitmap offsets the wrong way round. 3758 * We can control MSRs 0x00000000-0x00001fff and 0xc0000000-0xc0001fff. 3759 */ 3760 if (msr <= 0x1fff) { 3761 if (type & MSR_TYPE_R) 3762 /* read-low */ 3763 __set_bit(msr, msr_bitmap + 0x000 / f); 3764 3765 if (type & MSR_TYPE_W) 3766 /* write-low */ 3767 __set_bit(msr, msr_bitmap + 0x800 / f); 3768 3769 } else if ((msr >= 0xc0000000) && (msr <= 0xc0001fff)) { 3770 msr &= 0x1fff; 3771 if (type & MSR_TYPE_R) 3772 /* read-high */ 3773 __set_bit(msr, msr_bitmap + 0x400 / f); 3774 3775 if (type & MSR_TYPE_W) 3776 /* write-high */ 3777 __set_bit(msr, msr_bitmap + 0xc00 / f); 3778 3779 } 3780 } 3781 3782 static __always_inline void vmx_set_intercept_for_msr(unsigned long *msr_bitmap, 3783 u32 msr, int type, bool value) 3784 { 3785 if (value) 3786 vmx_enable_intercept_for_msr(msr_bitmap, msr, type); 3787 else 3788 vmx_disable_intercept_for_msr(msr_bitmap, msr, type); 3789 } 3790 3791 static u8 vmx_msr_bitmap_mode(struct kvm_vcpu *vcpu) 3792 { 3793 u8 mode = 0; 3794 3795 if (cpu_has_secondary_exec_ctrls() && 3796 (secondary_exec_controls_get(to_vmx(vcpu)) & 3797 SECONDARY_EXEC_VIRTUALIZE_X2APIC_MODE)) { 3798 mode |= MSR_BITMAP_MODE_X2APIC; 3799 if (enable_apicv && kvm_vcpu_apicv_active(vcpu)) 3800 mode |= MSR_BITMAP_MODE_X2APIC_APICV; 3801 } 3802 3803 return mode; 3804 } 3805 3806 static void vmx_update_msr_bitmap_x2apic(unsigned long *msr_bitmap, 3807 u8 mode) 3808 { 3809 int msr; 3810 3811 for (msr = 0x800; msr <= 0x8ff; msr += BITS_PER_LONG) { 3812 unsigned word = msr / BITS_PER_LONG; 3813 msr_bitmap[word] = (mode & MSR_BITMAP_MODE_X2APIC_APICV) ? 0 : ~0; 3814 msr_bitmap[word + (0x800 / sizeof(long))] = ~0; 3815 } 3816 3817 if (mode & MSR_BITMAP_MODE_X2APIC) { 3818 /* 3819 * TPR reads and writes can be virtualized even if virtual interrupt 3820 * delivery is not in use. 3821 */ 3822 vmx_disable_intercept_for_msr(msr_bitmap, X2APIC_MSR(APIC_TASKPRI), MSR_TYPE_RW); 3823 if (mode & MSR_BITMAP_MODE_X2APIC_APICV) { 3824 vmx_enable_intercept_for_msr(msr_bitmap, X2APIC_MSR(APIC_TMCCT), MSR_TYPE_R); 3825 vmx_disable_intercept_for_msr(msr_bitmap, X2APIC_MSR(APIC_EOI), MSR_TYPE_W); 3826 vmx_disable_intercept_for_msr(msr_bitmap, X2APIC_MSR(APIC_SELF_IPI), MSR_TYPE_W); 3827 } 3828 } 3829 } 3830 3831 void vmx_update_msr_bitmap(struct kvm_vcpu *vcpu) 3832 { 3833 struct vcpu_vmx *vmx = to_vmx(vcpu); 3834 unsigned long *msr_bitmap = vmx->vmcs01.msr_bitmap; 3835 u8 mode = vmx_msr_bitmap_mode(vcpu); 3836 u8 changed = mode ^ vmx->msr_bitmap_mode; 3837 3838 if (!changed) 3839 return; 3840 3841 if (changed & (MSR_BITMAP_MODE_X2APIC | MSR_BITMAP_MODE_X2APIC_APICV)) 3842 vmx_update_msr_bitmap_x2apic(msr_bitmap, mode); 3843 3844 vmx->msr_bitmap_mode = mode; 3845 } 3846 3847 void pt_update_intercept_for_msr(struct vcpu_vmx *vmx) 3848 { 3849 unsigned long *msr_bitmap = vmx->vmcs01.msr_bitmap; 3850 bool flag = !(vmx->pt_desc.guest.ctl & RTIT_CTL_TRACEEN); 3851 u32 i; 3852 3853 vmx_set_intercept_for_msr(msr_bitmap, MSR_IA32_RTIT_STATUS, 3854 MSR_TYPE_RW, flag); 3855 vmx_set_intercept_for_msr(msr_bitmap, MSR_IA32_RTIT_OUTPUT_BASE, 3856 MSR_TYPE_RW, flag); 3857 vmx_set_intercept_for_msr(msr_bitmap, MSR_IA32_RTIT_OUTPUT_MASK, 3858 MSR_TYPE_RW, flag); 3859 vmx_set_intercept_for_msr(msr_bitmap, MSR_IA32_RTIT_CR3_MATCH, 3860 MSR_TYPE_RW, flag); 3861 for (i = 0; i < vmx->pt_desc.addr_range; i++) { 3862 vmx_set_intercept_for_msr(msr_bitmap, 3863 MSR_IA32_RTIT_ADDR0_A + i * 2, MSR_TYPE_RW, flag); 3864 vmx_set_intercept_for_msr(msr_bitmap, 3865 MSR_IA32_RTIT_ADDR0_B + i * 2, MSR_TYPE_RW, flag); 3866 } 3867 } 3868 3869 static bool vmx_guest_apic_has_interrupt(struct kvm_vcpu *vcpu) 3870 { 3871 struct vcpu_vmx *vmx = to_vmx(vcpu); 3872 void *vapic_page; 3873 u32 vppr; 3874 int rvi; 3875 3876 if (WARN_ON_ONCE(!is_guest_mode(vcpu)) || 3877 !nested_cpu_has_vid(get_vmcs12(vcpu)) || 3878 WARN_ON_ONCE(!vmx->nested.virtual_apic_map.gfn)) 3879 return false; 3880 3881 rvi = vmx_get_rvi(); 3882 3883 vapic_page = vmx->nested.virtual_apic_map.hva; 3884 vppr = *((u32 *)(vapic_page + APIC_PROCPRI)); 3885 3886 return ((rvi & 0xf0) > (vppr & 0xf0)); 3887 } 3888 3889 static inline bool kvm_vcpu_trigger_posted_interrupt(struct kvm_vcpu *vcpu, 3890 bool nested) 3891 { 3892 #ifdef CONFIG_SMP 3893 int pi_vec = nested ? POSTED_INTR_NESTED_VECTOR : POSTED_INTR_VECTOR; 3894 3895 if (vcpu->mode == IN_GUEST_MODE) { 3896 /* 3897 * The vector of interrupt to be delivered to vcpu had 3898 * been set in PIR before this function. 3899 * 3900 * Following cases will be reached in this block, and 3901 * we always send a notification event in all cases as 3902 * explained below. 3903 * 3904 * Case 1: vcpu keeps in non-root mode. Sending a 3905 * notification event posts the interrupt to vcpu. 3906 * 3907 * Case 2: vcpu exits to root mode and is still 3908 * runnable. PIR will be synced to vIRR before the 3909 * next vcpu entry. Sending a notification event in 3910 * this case has no effect, as vcpu is not in root 3911 * mode. 3912 * 3913 * Case 3: vcpu exits to root mode and is blocked. 3914 * vcpu_block() has already synced PIR to vIRR and 3915 * never blocks vcpu if vIRR is not cleared. Therefore, 3916 * a blocked vcpu here does not wait for any requested 3917 * interrupts in PIR, and sending a notification event 3918 * which has no effect is safe here. 3919 */ 3920 3921 apic->send_IPI_mask(get_cpu_mask(vcpu->cpu), pi_vec); 3922 return true; 3923 } 3924 #endif 3925 return false; 3926 } 3927 3928 static int vmx_deliver_nested_posted_interrupt(struct kvm_vcpu *vcpu, 3929 int vector) 3930 { 3931 struct vcpu_vmx *vmx = to_vmx(vcpu); 3932 3933 if (is_guest_mode(vcpu) && 3934 vector == vmx->nested.posted_intr_nv) { 3935 /* 3936 * If a posted intr is not recognized by hardware, 3937 * we will accomplish it in the next vmentry. 3938 */ 3939 vmx->nested.pi_pending = true; 3940 kvm_make_request(KVM_REQ_EVENT, vcpu); 3941 /* the PIR and ON have been set by L1. */ 3942 if (!kvm_vcpu_trigger_posted_interrupt(vcpu, true)) 3943 kvm_vcpu_kick(vcpu); 3944 return 0; 3945 } 3946 return -1; 3947 } 3948 /* 3949 * Send interrupt to vcpu via posted interrupt way. 3950 * 1. If target vcpu is running(non-root mode), send posted interrupt 3951 * notification to vcpu and hardware will sync PIR to vIRR atomically. 3952 * 2. If target vcpu isn't running(root mode), kick it to pick up the 3953 * interrupt from PIR in next vmentry. 3954 */ 3955 static int vmx_deliver_posted_interrupt(struct kvm_vcpu *vcpu, int vector) 3956 { 3957 struct vcpu_vmx *vmx = to_vmx(vcpu); 3958 int r; 3959 3960 r = vmx_deliver_nested_posted_interrupt(vcpu, vector); 3961 if (!r) 3962 return 0; 3963 3964 if (!vcpu->arch.apicv_active) 3965 return -1; 3966 3967 if (pi_test_and_set_pir(vector, &vmx->pi_desc)) 3968 return 0; 3969 3970 /* If a previous notification has sent the IPI, nothing to do. */ 3971 if (pi_test_and_set_on(&vmx->pi_desc)) 3972 return 0; 3973 3974 if (vcpu != kvm_get_running_vcpu() && 3975 !kvm_vcpu_trigger_posted_interrupt(vcpu, false)) 3976 kvm_vcpu_kick(vcpu); 3977 3978 return 0; 3979 } 3980 3981 /* 3982 * Set up the vmcs's constant host-state fields, i.e., host-state fields that 3983 * will not change in the lifetime of the guest. 3984 * Note that host-state that does change is set elsewhere. E.g., host-state 3985 * that is set differently for each CPU is set in vmx_vcpu_load(), not here. 3986 */ 3987 void vmx_set_constant_host_state(struct vcpu_vmx *vmx) 3988 { 3989 u32 low32, high32; 3990 unsigned long tmpl; 3991 unsigned long cr0, cr3, cr4; 3992 3993 cr0 = read_cr0(); 3994 WARN_ON(cr0 & X86_CR0_TS); 3995 vmcs_writel(HOST_CR0, cr0); /* 22.2.3 */ 3996 3997 /* 3998 * Save the most likely value for this task's CR3 in the VMCS. 3999 * We can't use __get_current_cr3_fast() because we're not atomic. 4000 */ 4001 cr3 = __read_cr3(); 4002 vmcs_writel(HOST_CR3, cr3); /* 22.2.3 FIXME: shadow tables */ 4003 vmx->loaded_vmcs->host_state.cr3 = cr3; 4004 4005 /* Save the most likely value for this task's CR4 in the VMCS. */ 4006 cr4 = cr4_read_shadow(); 4007 vmcs_writel(HOST_CR4, cr4); /* 22.2.3, 22.2.5 */ 4008 vmx->loaded_vmcs->host_state.cr4 = cr4; 4009 4010 vmcs_write16(HOST_CS_SELECTOR, __KERNEL_CS); /* 22.2.4 */ 4011 #ifdef CONFIG_X86_64 4012 /* 4013 * Load null selectors, so we can avoid reloading them in 4014 * vmx_prepare_switch_to_host(), in case userspace uses 4015 * the null selectors too (the expected case). 4016 */ 4017 vmcs_write16(HOST_DS_SELECTOR, 0); 4018 vmcs_write16(HOST_ES_SELECTOR, 0); 4019 #else 4020 vmcs_write16(HOST_DS_SELECTOR, __KERNEL_DS); /* 22.2.4 */ 4021 vmcs_write16(HOST_ES_SELECTOR, __KERNEL_DS); /* 22.2.4 */ 4022 #endif 4023 vmcs_write16(HOST_SS_SELECTOR, __KERNEL_DS); /* 22.2.4 */ 4024 vmcs_write16(HOST_TR_SELECTOR, GDT_ENTRY_TSS*8); /* 22.2.4 */ 4025 4026 vmcs_writel(HOST_IDTR_BASE, host_idt_base); /* 22.2.4 */ 4027 4028 vmcs_writel(HOST_RIP, (unsigned long)vmx_vmexit); /* 22.2.5 */ 4029 4030 rdmsr(MSR_IA32_SYSENTER_CS, low32, high32); 4031 vmcs_write32(HOST_IA32_SYSENTER_CS, low32); 4032 rdmsrl(MSR_IA32_SYSENTER_EIP, tmpl); 4033 vmcs_writel(HOST_IA32_SYSENTER_EIP, tmpl); /* 22.2.3 */ 4034 4035 if (vmcs_config.vmexit_ctrl & VM_EXIT_LOAD_IA32_PAT) { 4036 rdmsr(MSR_IA32_CR_PAT, low32, high32); 4037 vmcs_write64(HOST_IA32_PAT, low32 | ((u64) high32 << 32)); 4038 } 4039 4040 if (cpu_has_load_ia32_efer()) 4041 vmcs_write64(HOST_IA32_EFER, host_efer); 4042 } 4043 4044 void set_cr4_guest_host_mask(struct vcpu_vmx *vmx) 4045 { 4046 vmx->vcpu.arch.cr4_guest_owned_bits = KVM_POSSIBLE_CR4_GUEST_BITS; 4047 if (!enable_ept) 4048 vmx->vcpu.arch.cr4_guest_owned_bits &= ~X86_CR4_PGE; 4049 if (is_guest_mode(&vmx->vcpu)) 4050 vmx->vcpu.arch.cr4_guest_owned_bits &= 4051 ~get_vmcs12(&vmx->vcpu)->cr4_guest_host_mask; 4052 vmcs_writel(CR4_GUEST_HOST_MASK, ~vmx->vcpu.arch.cr4_guest_owned_bits); 4053 } 4054 4055 u32 vmx_pin_based_exec_ctrl(struct vcpu_vmx *vmx) 4056 { 4057 u32 pin_based_exec_ctrl = vmcs_config.pin_based_exec_ctrl; 4058 4059 if (!kvm_vcpu_apicv_active(&vmx->vcpu)) 4060 pin_based_exec_ctrl &= ~PIN_BASED_POSTED_INTR; 4061 4062 if (!enable_vnmi) 4063 pin_based_exec_ctrl &= ~PIN_BASED_VIRTUAL_NMIS; 4064 4065 if (!enable_preemption_timer) 4066 pin_based_exec_ctrl &= ~PIN_BASED_VMX_PREEMPTION_TIMER; 4067 4068 return pin_based_exec_ctrl; 4069 } 4070 4071 static void vmx_refresh_apicv_exec_ctrl(struct kvm_vcpu *vcpu) 4072 { 4073 struct vcpu_vmx *vmx = to_vmx(vcpu); 4074 4075 pin_controls_set(vmx, vmx_pin_based_exec_ctrl(vmx)); 4076 if (cpu_has_secondary_exec_ctrls()) { 4077 if (kvm_vcpu_apicv_active(vcpu)) 4078 secondary_exec_controls_setbit(vmx, 4079 SECONDARY_EXEC_APIC_REGISTER_VIRT | 4080 SECONDARY_EXEC_VIRTUAL_INTR_DELIVERY); 4081 else 4082 secondary_exec_controls_clearbit(vmx, 4083 SECONDARY_EXEC_APIC_REGISTER_VIRT | 4084 SECONDARY_EXEC_VIRTUAL_INTR_DELIVERY); 4085 } 4086 4087 if (cpu_has_vmx_msr_bitmap()) 4088 vmx_update_msr_bitmap(vcpu); 4089 } 4090 4091 u32 vmx_exec_control(struct vcpu_vmx *vmx) 4092 { 4093 u32 exec_control = vmcs_config.cpu_based_exec_ctrl; 4094 4095 if (vmx->vcpu.arch.switch_db_regs & KVM_DEBUGREG_WONT_EXIT) 4096 exec_control &= ~CPU_BASED_MOV_DR_EXITING; 4097 4098 if (!cpu_need_tpr_shadow(&vmx->vcpu)) { 4099 exec_control &= ~CPU_BASED_TPR_SHADOW; 4100 #ifdef CONFIG_X86_64 4101 exec_control |= CPU_BASED_CR8_STORE_EXITING | 4102 CPU_BASED_CR8_LOAD_EXITING; 4103 #endif 4104 } 4105 if (!enable_ept) 4106 exec_control |= CPU_BASED_CR3_STORE_EXITING | 4107 CPU_BASED_CR3_LOAD_EXITING | 4108 CPU_BASED_INVLPG_EXITING; 4109 if (kvm_mwait_in_guest(vmx->vcpu.kvm)) 4110 exec_control &= ~(CPU_BASED_MWAIT_EXITING | 4111 CPU_BASED_MONITOR_EXITING); 4112 if (kvm_hlt_in_guest(vmx->vcpu.kvm)) 4113 exec_control &= ~CPU_BASED_HLT_EXITING; 4114 return exec_control; 4115 } 4116 4117 4118 static void vmx_compute_secondary_exec_control(struct vcpu_vmx *vmx) 4119 { 4120 struct kvm_vcpu *vcpu = &vmx->vcpu; 4121 4122 u32 exec_control = vmcs_config.cpu_based_2nd_exec_ctrl; 4123 4124 if (vmx_pt_mode_is_system()) 4125 exec_control &= ~(SECONDARY_EXEC_PT_USE_GPA | SECONDARY_EXEC_PT_CONCEAL_VMX); 4126 if (!cpu_need_virtualize_apic_accesses(vcpu)) 4127 exec_control &= ~SECONDARY_EXEC_VIRTUALIZE_APIC_ACCESSES; 4128 if (vmx->vpid == 0) 4129 exec_control &= ~SECONDARY_EXEC_ENABLE_VPID; 4130 if (!enable_ept) { 4131 exec_control &= ~SECONDARY_EXEC_ENABLE_EPT; 4132 enable_unrestricted_guest = 0; 4133 } 4134 if (!enable_unrestricted_guest) 4135 exec_control &= ~SECONDARY_EXEC_UNRESTRICTED_GUEST; 4136 if (kvm_pause_in_guest(vmx->vcpu.kvm)) 4137 exec_control &= ~SECONDARY_EXEC_PAUSE_LOOP_EXITING; 4138 if (!kvm_vcpu_apicv_active(vcpu)) 4139 exec_control &= ~(SECONDARY_EXEC_APIC_REGISTER_VIRT | 4140 SECONDARY_EXEC_VIRTUAL_INTR_DELIVERY); 4141 exec_control &= ~SECONDARY_EXEC_VIRTUALIZE_X2APIC_MODE; 4142 4143 /* SECONDARY_EXEC_DESC is enabled/disabled on writes to CR4.UMIP, 4144 * in vmx_set_cr4. */ 4145 exec_control &= ~SECONDARY_EXEC_DESC; 4146 4147 /* SECONDARY_EXEC_SHADOW_VMCS is enabled when L1 executes VMPTRLD 4148 (handle_vmptrld). 4149 We can NOT enable shadow_vmcs here because we don't have yet 4150 a current VMCS12 4151 */ 4152 exec_control &= ~SECONDARY_EXEC_SHADOW_VMCS; 4153 4154 if (!enable_pml) 4155 exec_control &= ~SECONDARY_EXEC_ENABLE_PML; 4156 4157 if (vmx_xsaves_supported()) { 4158 /* Exposing XSAVES only when XSAVE is exposed */ 4159 bool xsaves_enabled = 4160 boot_cpu_has(X86_FEATURE_XSAVE) && 4161 guest_cpuid_has(vcpu, X86_FEATURE_XSAVE) && 4162 guest_cpuid_has(vcpu, X86_FEATURE_XSAVES); 4163 4164 vcpu->arch.xsaves_enabled = xsaves_enabled; 4165 4166 if (!xsaves_enabled) 4167 exec_control &= ~SECONDARY_EXEC_XSAVES; 4168 4169 if (nested) { 4170 if (xsaves_enabled) 4171 vmx->nested.msrs.secondary_ctls_high |= 4172 SECONDARY_EXEC_XSAVES; 4173 else 4174 vmx->nested.msrs.secondary_ctls_high &= 4175 ~SECONDARY_EXEC_XSAVES; 4176 } 4177 } 4178 4179 if (cpu_has_vmx_rdtscp()) { 4180 bool rdtscp_enabled = guest_cpuid_has(vcpu, X86_FEATURE_RDTSCP); 4181 if (!rdtscp_enabled) 4182 exec_control &= ~SECONDARY_EXEC_RDTSCP; 4183 4184 if (nested) { 4185 if (rdtscp_enabled) 4186 vmx->nested.msrs.secondary_ctls_high |= 4187 SECONDARY_EXEC_RDTSCP; 4188 else 4189 vmx->nested.msrs.secondary_ctls_high &= 4190 ~SECONDARY_EXEC_RDTSCP; 4191 } 4192 } 4193 4194 if (cpu_has_vmx_invpcid()) { 4195 /* Exposing INVPCID only when PCID is exposed */ 4196 bool invpcid_enabled = 4197 guest_cpuid_has(vcpu, X86_FEATURE_INVPCID) && 4198 guest_cpuid_has(vcpu, X86_FEATURE_PCID); 4199 4200 if (!invpcid_enabled) { 4201 exec_control &= ~SECONDARY_EXEC_ENABLE_INVPCID; 4202 guest_cpuid_clear(vcpu, X86_FEATURE_INVPCID); 4203 } 4204 4205 if (nested) { 4206 if (invpcid_enabled) 4207 vmx->nested.msrs.secondary_ctls_high |= 4208 SECONDARY_EXEC_ENABLE_INVPCID; 4209 else 4210 vmx->nested.msrs.secondary_ctls_high &= 4211 ~SECONDARY_EXEC_ENABLE_INVPCID; 4212 } 4213 } 4214 4215 if (vmx_rdrand_supported()) { 4216 bool rdrand_enabled = guest_cpuid_has(vcpu, X86_FEATURE_RDRAND); 4217 if (rdrand_enabled) 4218 exec_control &= ~SECONDARY_EXEC_RDRAND_EXITING; 4219 4220 if (nested) { 4221 if (rdrand_enabled) 4222 vmx->nested.msrs.secondary_ctls_high |= 4223 SECONDARY_EXEC_RDRAND_EXITING; 4224 else 4225 vmx->nested.msrs.secondary_ctls_high &= 4226 ~SECONDARY_EXEC_RDRAND_EXITING; 4227 } 4228 } 4229 4230 if (vmx_rdseed_supported()) { 4231 bool rdseed_enabled = guest_cpuid_has(vcpu, X86_FEATURE_RDSEED); 4232 if (rdseed_enabled) 4233 exec_control &= ~SECONDARY_EXEC_RDSEED_EXITING; 4234 4235 if (nested) { 4236 if (rdseed_enabled) 4237 vmx->nested.msrs.secondary_ctls_high |= 4238 SECONDARY_EXEC_RDSEED_EXITING; 4239 else 4240 vmx->nested.msrs.secondary_ctls_high &= 4241 ~SECONDARY_EXEC_RDSEED_EXITING; 4242 } 4243 } 4244 4245 if (vmx_waitpkg_supported()) { 4246 bool waitpkg_enabled = 4247 guest_cpuid_has(vcpu, X86_FEATURE_WAITPKG); 4248 4249 if (!waitpkg_enabled) 4250 exec_control &= ~SECONDARY_EXEC_ENABLE_USR_WAIT_PAUSE; 4251 4252 if (nested) { 4253 if (waitpkg_enabled) 4254 vmx->nested.msrs.secondary_ctls_high |= 4255 SECONDARY_EXEC_ENABLE_USR_WAIT_PAUSE; 4256 else 4257 vmx->nested.msrs.secondary_ctls_high &= 4258 ~SECONDARY_EXEC_ENABLE_USR_WAIT_PAUSE; 4259 } 4260 } 4261 4262 vmx->secondary_exec_control = exec_control; 4263 } 4264 4265 static void ept_set_mmio_spte_mask(void) 4266 { 4267 /* 4268 * EPT Misconfigurations can be generated if the value of bits 2:0 4269 * of an EPT paging-structure entry is 110b (write/execute). 4270 */ 4271 kvm_mmu_set_mmio_spte_mask(VMX_EPT_MISCONFIG_WX_VALUE, 0); 4272 } 4273 4274 #define VMX_XSS_EXIT_BITMAP 0 4275 4276 /* 4277 * Noting that the initialization of Guest-state Area of VMCS is in 4278 * vmx_vcpu_reset(). 4279 */ 4280 static void init_vmcs(struct vcpu_vmx *vmx) 4281 { 4282 if (nested) 4283 nested_vmx_set_vmcs_shadowing_bitmap(); 4284 4285 if (cpu_has_vmx_msr_bitmap()) 4286 vmcs_write64(MSR_BITMAP, __pa(vmx->vmcs01.msr_bitmap)); 4287 4288 vmcs_write64(VMCS_LINK_POINTER, -1ull); /* 22.3.1.5 */ 4289 4290 /* Control */ 4291 pin_controls_set(vmx, vmx_pin_based_exec_ctrl(vmx)); 4292 4293 exec_controls_set(vmx, vmx_exec_control(vmx)); 4294 4295 if (cpu_has_secondary_exec_ctrls()) { 4296 vmx_compute_secondary_exec_control(vmx); 4297 secondary_exec_controls_set(vmx, vmx->secondary_exec_control); 4298 } 4299 4300 if (kvm_vcpu_apicv_active(&vmx->vcpu)) { 4301 vmcs_write64(EOI_EXIT_BITMAP0, 0); 4302 vmcs_write64(EOI_EXIT_BITMAP1, 0); 4303 vmcs_write64(EOI_EXIT_BITMAP2, 0); 4304 vmcs_write64(EOI_EXIT_BITMAP3, 0); 4305 4306 vmcs_write16(GUEST_INTR_STATUS, 0); 4307 4308 vmcs_write16(POSTED_INTR_NV, POSTED_INTR_VECTOR); 4309 vmcs_write64(POSTED_INTR_DESC_ADDR, __pa((&vmx->pi_desc))); 4310 } 4311 4312 if (!kvm_pause_in_guest(vmx->vcpu.kvm)) { 4313 vmcs_write32(PLE_GAP, ple_gap); 4314 vmx->ple_window = ple_window; 4315 vmx->ple_window_dirty = true; 4316 } 4317 4318 vmcs_write32(PAGE_FAULT_ERROR_CODE_MASK, 0); 4319 vmcs_write32(PAGE_FAULT_ERROR_CODE_MATCH, 0); 4320 vmcs_write32(CR3_TARGET_COUNT, 0); /* 22.2.1 */ 4321 4322 vmcs_write16(HOST_FS_SELECTOR, 0); /* 22.2.4 */ 4323 vmcs_write16(HOST_GS_SELECTOR, 0); /* 22.2.4 */ 4324 vmx_set_constant_host_state(vmx); 4325 vmcs_writel(HOST_FS_BASE, 0); /* 22.2.4 */ 4326 vmcs_writel(HOST_GS_BASE, 0); /* 22.2.4 */ 4327 4328 if (cpu_has_vmx_vmfunc()) 4329 vmcs_write64(VM_FUNCTION_CONTROL, 0); 4330 4331 vmcs_write32(VM_EXIT_MSR_STORE_COUNT, 0); 4332 vmcs_write32(VM_EXIT_MSR_LOAD_COUNT, 0); 4333 vmcs_write64(VM_EXIT_MSR_LOAD_ADDR, __pa(vmx->msr_autoload.host.val)); 4334 vmcs_write32(VM_ENTRY_MSR_LOAD_COUNT, 0); 4335 vmcs_write64(VM_ENTRY_MSR_LOAD_ADDR, __pa(vmx->msr_autoload.guest.val)); 4336 4337 if (vmcs_config.vmentry_ctrl & VM_ENTRY_LOAD_IA32_PAT) 4338 vmcs_write64(GUEST_IA32_PAT, vmx->vcpu.arch.pat); 4339 4340 vm_exit_controls_set(vmx, vmx_vmexit_ctrl()); 4341 4342 /* 22.2.1, 20.8.1 */ 4343 vm_entry_controls_set(vmx, vmx_vmentry_ctrl()); 4344 4345 vmx->vcpu.arch.cr0_guest_owned_bits = KVM_POSSIBLE_CR0_GUEST_BITS; 4346 vmcs_writel(CR0_GUEST_HOST_MASK, ~vmx->vcpu.arch.cr0_guest_owned_bits); 4347 4348 set_cr4_guest_host_mask(vmx); 4349 4350 if (vmx->vpid != 0) 4351 vmcs_write16(VIRTUAL_PROCESSOR_ID, vmx->vpid); 4352 4353 if (vmx_xsaves_supported()) 4354 vmcs_write64(XSS_EXIT_BITMAP, VMX_XSS_EXIT_BITMAP); 4355 4356 if (enable_pml) { 4357 vmcs_write64(PML_ADDRESS, page_to_phys(vmx->pml_pg)); 4358 vmcs_write16(GUEST_PML_INDEX, PML_ENTITY_NUM - 1); 4359 } 4360 4361 if (cpu_has_vmx_encls_vmexit()) 4362 vmcs_write64(ENCLS_EXITING_BITMAP, -1ull); 4363 4364 if (vmx_pt_mode_is_host_guest()) { 4365 memset(&vmx->pt_desc, 0, sizeof(vmx->pt_desc)); 4366 /* Bit[6~0] are forced to 1, writes are ignored. */ 4367 vmx->pt_desc.guest.output_mask = 0x7F; 4368 vmcs_write64(GUEST_IA32_RTIT_CTL, 0); 4369 } 4370 } 4371 4372 static void vmx_vcpu_reset(struct kvm_vcpu *vcpu, bool init_event) 4373 { 4374 struct vcpu_vmx *vmx = to_vmx(vcpu); 4375 struct msr_data apic_base_msr; 4376 u64 cr0; 4377 4378 vmx->rmode.vm86_active = 0; 4379 vmx->spec_ctrl = 0; 4380 4381 vmx->msr_ia32_umwait_control = 0; 4382 4383 vmx->vcpu.arch.regs[VCPU_REGS_RDX] = get_rdx_init_val(); 4384 vmx->hv_deadline_tsc = -1; 4385 kvm_set_cr8(vcpu, 0); 4386 4387 if (!init_event) { 4388 apic_base_msr.data = APIC_DEFAULT_PHYS_BASE | 4389 MSR_IA32_APICBASE_ENABLE; 4390 if (kvm_vcpu_is_reset_bsp(vcpu)) 4391 apic_base_msr.data |= MSR_IA32_APICBASE_BSP; 4392 apic_base_msr.host_initiated = true; 4393 kvm_set_apic_base(vcpu, &apic_base_msr); 4394 } 4395 4396 vmx_segment_cache_clear(vmx); 4397 4398 seg_setup(VCPU_SREG_CS); 4399 vmcs_write16(GUEST_CS_SELECTOR, 0xf000); 4400 vmcs_writel(GUEST_CS_BASE, 0xffff0000ul); 4401 4402 seg_setup(VCPU_SREG_DS); 4403 seg_setup(VCPU_SREG_ES); 4404 seg_setup(VCPU_SREG_FS); 4405 seg_setup(VCPU_SREG_GS); 4406 seg_setup(VCPU_SREG_SS); 4407 4408 vmcs_write16(GUEST_TR_SELECTOR, 0); 4409 vmcs_writel(GUEST_TR_BASE, 0); 4410 vmcs_write32(GUEST_TR_LIMIT, 0xffff); 4411 vmcs_write32(GUEST_TR_AR_BYTES, 0x008b); 4412 4413 vmcs_write16(GUEST_LDTR_SELECTOR, 0); 4414 vmcs_writel(GUEST_LDTR_BASE, 0); 4415 vmcs_write32(GUEST_LDTR_LIMIT, 0xffff); 4416 vmcs_write32(GUEST_LDTR_AR_BYTES, 0x00082); 4417 4418 if (!init_event) { 4419 vmcs_write32(GUEST_SYSENTER_CS, 0); 4420 vmcs_writel(GUEST_SYSENTER_ESP, 0); 4421 vmcs_writel(GUEST_SYSENTER_EIP, 0); 4422 vmcs_write64(GUEST_IA32_DEBUGCTL, 0); 4423 } 4424 4425 kvm_set_rflags(vcpu, X86_EFLAGS_FIXED); 4426 kvm_rip_write(vcpu, 0xfff0); 4427 4428 vmcs_writel(GUEST_GDTR_BASE, 0); 4429 vmcs_write32(GUEST_GDTR_LIMIT, 0xffff); 4430 4431 vmcs_writel(GUEST_IDTR_BASE, 0); 4432 vmcs_write32(GUEST_IDTR_LIMIT, 0xffff); 4433 4434 vmcs_write32(GUEST_ACTIVITY_STATE, GUEST_ACTIVITY_ACTIVE); 4435 vmcs_write32(GUEST_INTERRUPTIBILITY_INFO, 0); 4436 vmcs_writel(GUEST_PENDING_DBG_EXCEPTIONS, 0); 4437 if (kvm_mpx_supported()) 4438 vmcs_write64(GUEST_BNDCFGS, 0); 4439 4440 setup_msrs(vmx); 4441 4442 vmcs_write32(VM_ENTRY_INTR_INFO_FIELD, 0); /* 22.2.1 */ 4443 4444 if (cpu_has_vmx_tpr_shadow() && !init_event) { 4445 vmcs_write64(VIRTUAL_APIC_PAGE_ADDR, 0); 4446 if (cpu_need_tpr_shadow(vcpu)) 4447 vmcs_write64(VIRTUAL_APIC_PAGE_ADDR, 4448 __pa(vcpu->arch.apic->regs)); 4449 vmcs_write32(TPR_THRESHOLD, 0); 4450 } 4451 4452 kvm_make_request(KVM_REQ_APIC_PAGE_RELOAD, vcpu); 4453 4454 cr0 = X86_CR0_NW | X86_CR0_CD | X86_CR0_ET; 4455 vmx->vcpu.arch.cr0 = cr0; 4456 vmx_set_cr0(vcpu, cr0); /* enter rmode */ 4457 vmx_set_cr4(vcpu, 0); 4458 vmx_set_efer(vcpu, 0); 4459 4460 update_exception_bitmap(vcpu); 4461 4462 vpid_sync_context(vmx->vpid); 4463 if (init_event) 4464 vmx_clear_hlt(vcpu); 4465 } 4466 4467 static void enable_irq_window(struct kvm_vcpu *vcpu) 4468 { 4469 exec_controls_setbit(to_vmx(vcpu), CPU_BASED_INTR_WINDOW_EXITING); 4470 } 4471 4472 static void enable_nmi_window(struct kvm_vcpu *vcpu) 4473 { 4474 if (!enable_vnmi || 4475 vmcs_read32(GUEST_INTERRUPTIBILITY_INFO) & GUEST_INTR_STATE_STI) { 4476 enable_irq_window(vcpu); 4477 return; 4478 } 4479 4480 exec_controls_setbit(to_vmx(vcpu), CPU_BASED_NMI_WINDOW_EXITING); 4481 } 4482 4483 static void vmx_inject_irq(struct kvm_vcpu *vcpu) 4484 { 4485 struct vcpu_vmx *vmx = to_vmx(vcpu); 4486 uint32_t intr; 4487 int irq = vcpu->arch.interrupt.nr; 4488 4489 trace_kvm_inj_virq(irq); 4490 4491 ++vcpu->stat.irq_injections; 4492 if (vmx->rmode.vm86_active) { 4493 int inc_eip = 0; 4494 if (vcpu->arch.interrupt.soft) 4495 inc_eip = vcpu->arch.event_exit_inst_len; 4496 kvm_inject_realmode_interrupt(vcpu, irq, inc_eip); 4497 return; 4498 } 4499 intr = irq | INTR_INFO_VALID_MASK; 4500 if (vcpu->arch.interrupt.soft) { 4501 intr |= INTR_TYPE_SOFT_INTR; 4502 vmcs_write32(VM_ENTRY_INSTRUCTION_LEN, 4503 vmx->vcpu.arch.event_exit_inst_len); 4504 } else 4505 intr |= INTR_TYPE_EXT_INTR; 4506 vmcs_write32(VM_ENTRY_INTR_INFO_FIELD, intr); 4507 4508 vmx_clear_hlt(vcpu); 4509 } 4510 4511 static void vmx_inject_nmi(struct kvm_vcpu *vcpu) 4512 { 4513 struct vcpu_vmx *vmx = to_vmx(vcpu); 4514 4515 if (!enable_vnmi) { 4516 /* 4517 * Tracking the NMI-blocked state in software is built upon 4518 * finding the next open IRQ window. This, in turn, depends on 4519 * well-behaving guests: They have to keep IRQs disabled at 4520 * least as long as the NMI handler runs. Otherwise we may 4521 * cause NMI nesting, maybe breaking the guest. But as this is 4522 * highly unlikely, we can live with the residual risk. 4523 */ 4524 vmx->loaded_vmcs->soft_vnmi_blocked = 1; 4525 vmx->loaded_vmcs->vnmi_blocked_time = 0; 4526 } 4527 4528 ++vcpu->stat.nmi_injections; 4529 vmx->loaded_vmcs->nmi_known_unmasked = false; 4530 4531 if (vmx->rmode.vm86_active) { 4532 kvm_inject_realmode_interrupt(vcpu, NMI_VECTOR, 0); 4533 return; 4534 } 4535 4536 vmcs_write32(VM_ENTRY_INTR_INFO_FIELD, 4537 INTR_TYPE_NMI_INTR | INTR_INFO_VALID_MASK | NMI_VECTOR); 4538 4539 vmx_clear_hlt(vcpu); 4540 } 4541 4542 bool vmx_get_nmi_mask(struct kvm_vcpu *vcpu) 4543 { 4544 struct vcpu_vmx *vmx = to_vmx(vcpu); 4545 bool masked; 4546 4547 if (!enable_vnmi) 4548 return vmx->loaded_vmcs->soft_vnmi_blocked; 4549 if (vmx->loaded_vmcs->nmi_known_unmasked) 4550 return false; 4551 masked = vmcs_read32(GUEST_INTERRUPTIBILITY_INFO) & GUEST_INTR_STATE_NMI; 4552 vmx->loaded_vmcs->nmi_known_unmasked = !masked; 4553 return masked; 4554 } 4555 4556 void vmx_set_nmi_mask(struct kvm_vcpu *vcpu, bool masked) 4557 { 4558 struct vcpu_vmx *vmx = to_vmx(vcpu); 4559 4560 if (!enable_vnmi) { 4561 if (vmx->loaded_vmcs->soft_vnmi_blocked != masked) { 4562 vmx->loaded_vmcs->soft_vnmi_blocked = masked; 4563 vmx->loaded_vmcs->vnmi_blocked_time = 0; 4564 } 4565 } else { 4566 vmx->loaded_vmcs->nmi_known_unmasked = !masked; 4567 if (masked) 4568 vmcs_set_bits(GUEST_INTERRUPTIBILITY_INFO, 4569 GUEST_INTR_STATE_NMI); 4570 else 4571 vmcs_clear_bits(GUEST_INTERRUPTIBILITY_INFO, 4572 GUEST_INTR_STATE_NMI); 4573 } 4574 } 4575 4576 bool vmx_nmi_blocked(struct kvm_vcpu *vcpu) 4577 { 4578 if (is_guest_mode(vcpu) && nested_exit_on_nmi(vcpu)) 4579 return false; 4580 4581 if (!enable_vnmi && to_vmx(vcpu)->loaded_vmcs->soft_vnmi_blocked) 4582 return true; 4583 4584 return (vmcs_read32(GUEST_INTERRUPTIBILITY_INFO) & 4585 (GUEST_INTR_STATE_MOV_SS | GUEST_INTR_STATE_STI | 4586 GUEST_INTR_STATE_NMI)); 4587 } 4588 4589 static int vmx_nmi_allowed(struct kvm_vcpu *vcpu, bool for_injection) 4590 { 4591 if (to_vmx(vcpu)->nested.nested_run_pending) 4592 return -EBUSY; 4593 4594 /* An NMI must not be injected into L2 if it's supposed to VM-Exit. */ 4595 if (for_injection && is_guest_mode(vcpu) && nested_exit_on_nmi(vcpu)) 4596 return -EBUSY; 4597 4598 return !vmx_nmi_blocked(vcpu); 4599 } 4600 4601 bool vmx_interrupt_blocked(struct kvm_vcpu *vcpu) 4602 { 4603 if (is_guest_mode(vcpu) && nested_exit_on_intr(vcpu)) 4604 return false; 4605 4606 return !(vmx_get_rflags(vcpu) & X86_EFLAGS_IF) || 4607 (vmcs_read32(GUEST_INTERRUPTIBILITY_INFO) & 4608 (GUEST_INTR_STATE_STI | GUEST_INTR_STATE_MOV_SS)); 4609 } 4610 4611 static int vmx_interrupt_allowed(struct kvm_vcpu *vcpu, bool for_injection) 4612 { 4613 if (to_vmx(vcpu)->nested.nested_run_pending) 4614 return -EBUSY; 4615 4616 /* 4617 * An IRQ must not be injected into L2 if it's supposed to VM-Exit, 4618 * e.g. if the IRQ arrived asynchronously after checking nested events. 4619 */ 4620 if (for_injection && is_guest_mode(vcpu) && nested_exit_on_intr(vcpu)) 4621 return -EBUSY; 4622 4623 return !vmx_interrupt_blocked(vcpu); 4624 } 4625 4626 static int vmx_set_tss_addr(struct kvm *kvm, unsigned int addr) 4627 { 4628 int ret; 4629 4630 if (enable_unrestricted_guest) 4631 return 0; 4632 4633 mutex_lock(&kvm->slots_lock); 4634 ret = __x86_set_memory_region(kvm, TSS_PRIVATE_MEMSLOT, addr, 4635 PAGE_SIZE * 3); 4636 mutex_unlock(&kvm->slots_lock); 4637 4638 if (ret) 4639 return ret; 4640 to_kvm_vmx(kvm)->tss_addr = addr; 4641 return init_rmode_tss(kvm); 4642 } 4643 4644 static int vmx_set_identity_map_addr(struct kvm *kvm, u64 ident_addr) 4645 { 4646 to_kvm_vmx(kvm)->ept_identity_map_addr = ident_addr; 4647 return 0; 4648 } 4649 4650 static bool rmode_exception(struct kvm_vcpu *vcpu, int vec) 4651 { 4652 switch (vec) { 4653 case BP_VECTOR: 4654 /* 4655 * Update instruction length as we may reinject the exception 4656 * from user space while in guest debugging mode. 4657 */ 4658 to_vmx(vcpu)->vcpu.arch.event_exit_inst_len = 4659 vmcs_read32(VM_EXIT_INSTRUCTION_LEN); 4660 if (vcpu->guest_debug & KVM_GUESTDBG_USE_SW_BP) 4661 return false; 4662 fallthrough; 4663 case DB_VECTOR: 4664 return !(vcpu->guest_debug & 4665 (KVM_GUESTDBG_SINGLESTEP | KVM_GUESTDBG_USE_HW_BP)); 4666 case DE_VECTOR: 4667 case OF_VECTOR: 4668 case BR_VECTOR: 4669 case UD_VECTOR: 4670 case DF_VECTOR: 4671 case SS_VECTOR: 4672 case GP_VECTOR: 4673 case MF_VECTOR: 4674 return true; 4675 } 4676 return false; 4677 } 4678 4679 static int handle_rmode_exception(struct kvm_vcpu *vcpu, 4680 int vec, u32 err_code) 4681 { 4682 /* 4683 * Instruction with address size override prefix opcode 0x67 4684 * Cause the #SS fault with 0 error code in VM86 mode. 4685 */ 4686 if (((vec == GP_VECTOR) || (vec == SS_VECTOR)) && err_code == 0) { 4687 if (kvm_emulate_instruction(vcpu, 0)) { 4688 if (vcpu->arch.halt_request) { 4689 vcpu->arch.halt_request = 0; 4690 return kvm_vcpu_halt(vcpu); 4691 } 4692 return 1; 4693 } 4694 return 0; 4695 } 4696 4697 /* 4698 * Forward all other exceptions that are valid in real mode. 4699 * FIXME: Breaks guest debugging in real mode, needs to be fixed with 4700 * the required debugging infrastructure rework. 4701 */ 4702 kvm_queue_exception(vcpu, vec); 4703 return 1; 4704 } 4705 4706 /* 4707 * Trigger machine check on the host. We assume all the MSRs are already set up 4708 * by the CPU and that we still run on the same CPU as the MCE occurred on. 4709 * We pass a fake environment to the machine check handler because we want 4710 * the guest to be always treated like user space, no matter what context 4711 * it used internally. 4712 */ 4713 static void kvm_machine_check(void) 4714 { 4715 #if defined(CONFIG_X86_MCE) 4716 struct pt_regs regs = { 4717 .cs = 3, /* Fake ring 3 no matter what the guest ran on */ 4718 .flags = X86_EFLAGS_IF, 4719 }; 4720 4721 do_machine_check(®s); 4722 #endif 4723 } 4724 4725 static int handle_machine_check(struct kvm_vcpu *vcpu) 4726 { 4727 /* handled by vmx_vcpu_run() */ 4728 return 1; 4729 } 4730 4731 /* 4732 * If the host has split lock detection disabled, then #AC is 4733 * unconditionally injected into the guest, which is the pre split lock 4734 * detection behaviour. 4735 * 4736 * If the host has split lock detection enabled then #AC is 4737 * only injected into the guest when: 4738 * - Guest CPL == 3 (user mode) 4739 * - Guest has #AC detection enabled in CR0 4740 * - Guest EFLAGS has AC bit set 4741 */ 4742 static inline bool guest_inject_ac(struct kvm_vcpu *vcpu) 4743 { 4744 if (!boot_cpu_has(X86_FEATURE_SPLIT_LOCK_DETECT)) 4745 return true; 4746 4747 return vmx_get_cpl(vcpu) == 3 && kvm_read_cr0_bits(vcpu, X86_CR0_AM) && 4748 (kvm_get_rflags(vcpu) & X86_EFLAGS_AC); 4749 } 4750 4751 static int handle_exception_nmi(struct kvm_vcpu *vcpu) 4752 { 4753 struct vcpu_vmx *vmx = to_vmx(vcpu); 4754 struct kvm_run *kvm_run = vcpu->run; 4755 u32 intr_info, ex_no, error_code; 4756 unsigned long cr2, rip, dr6; 4757 u32 vect_info; 4758 4759 vect_info = vmx->idt_vectoring_info; 4760 intr_info = vmx_get_intr_info(vcpu); 4761 4762 if (is_machine_check(intr_info) || is_nmi(intr_info)) 4763 return 1; /* handled by handle_exception_nmi_irqoff() */ 4764 4765 if (is_invalid_opcode(intr_info)) 4766 return handle_ud(vcpu); 4767 4768 error_code = 0; 4769 if (intr_info & INTR_INFO_DELIVER_CODE_MASK) 4770 error_code = vmcs_read32(VM_EXIT_INTR_ERROR_CODE); 4771 4772 if (!vmx->rmode.vm86_active && is_gp_fault(intr_info)) { 4773 WARN_ON_ONCE(!enable_vmware_backdoor); 4774 4775 /* 4776 * VMware backdoor emulation on #GP interception only handles 4777 * IN{S}, OUT{S}, and RDPMC, none of which generate a non-zero 4778 * error code on #GP. 4779 */ 4780 if (error_code) { 4781 kvm_queue_exception_e(vcpu, GP_VECTOR, error_code); 4782 return 1; 4783 } 4784 return kvm_emulate_instruction(vcpu, EMULTYPE_VMWARE_GP); 4785 } 4786 4787 /* 4788 * The #PF with PFEC.RSVD = 1 indicates the guest is accessing 4789 * MMIO, it is better to report an internal error. 4790 * See the comments in vmx_handle_exit. 4791 */ 4792 if ((vect_info & VECTORING_INFO_VALID_MASK) && 4793 !(is_page_fault(intr_info) && !(error_code & PFERR_RSVD_MASK))) { 4794 vcpu->run->exit_reason = KVM_EXIT_INTERNAL_ERROR; 4795 vcpu->run->internal.suberror = KVM_INTERNAL_ERROR_SIMUL_EX; 4796 vcpu->run->internal.ndata = 4; 4797 vcpu->run->internal.data[0] = vect_info; 4798 vcpu->run->internal.data[1] = intr_info; 4799 vcpu->run->internal.data[2] = error_code; 4800 vcpu->run->internal.data[3] = vcpu->arch.last_vmentry_cpu; 4801 return 0; 4802 } 4803 4804 if (is_page_fault(intr_info)) { 4805 cr2 = vmx_get_exit_qual(vcpu); 4806 if (enable_ept && !vcpu->arch.apf.host_apf_flags) { 4807 /* 4808 * EPT will cause page fault only if we need to 4809 * detect illegal GPAs. 4810 */ 4811 WARN_ON_ONCE(!allow_smaller_maxphyaddr); 4812 kvm_fixup_and_inject_pf_error(vcpu, cr2, error_code); 4813 return 1; 4814 } else 4815 return kvm_handle_page_fault(vcpu, error_code, cr2, NULL, 0); 4816 } 4817 4818 ex_no = intr_info & INTR_INFO_VECTOR_MASK; 4819 4820 if (vmx->rmode.vm86_active && rmode_exception(vcpu, ex_no)) 4821 return handle_rmode_exception(vcpu, ex_no, error_code); 4822 4823 switch (ex_no) { 4824 case DB_VECTOR: 4825 dr6 = vmx_get_exit_qual(vcpu); 4826 if (!(vcpu->guest_debug & 4827 (KVM_GUESTDBG_SINGLESTEP | KVM_GUESTDBG_USE_HW_BP))) { 4828 if (is_icebp(intr_info)) 4829 WARN_ON(!skip_emulated_instruction(vcpu)); 4830 4831 kvm_queue_exception_p(vcpu, DB_VECTOR, dr6); 4832 return 1; 4833 } 4834 kvm_run->debug.arch.dr6 = dr6 | DR6_FIXED_1 | DR6_RTM; 4835 kvm_run->debug.arch.dr7 = vmcs_readl(GUEST_DR7); 4836 fallthrough; 4837 case BP_VECTOR: 4838 /* 4839 * Update instruction length as we may reinject #BP from 4840 * user space while in guest debugging mode. Reading it for 4841 * #DB as well causes no harm, it is not used in that case. 4842 */ 4843 vmx->vcpu.arch.event_exit_inst_len = 4844 vmcs_read32(VM_EXIT_INSTRUCTION_LEN); 4845 kvm_run->exit_reason = KVM_EXIT_DEBUG; 4846 rip = kvm_rip_read(vcpu); 4847 kvm_run->debug.arch.pc = vmcs_readl(GUEST_CS_BASE) + rip; 4848 kvm_run->debug.arch.exception = ex_no; 4849 break; 4850 case AC_VECTOR: 4851 if (guest_inject_ac(vcpu)) { 4852 kvm_queue_exception_e(vcpu, AC_VECTOR, error_code); 4853 return 1; 4854 } 4855 4856 /* 4857 * Handle split lock. Depending on detection mode this will 4858 * either warn and disable split lock detection for this 4859 * task or force SIGBUS on it. 4860 */ 4861 if (handle_guest_split_lock(kvm_rip_read(vcpu))) 4862 return 1; 4863 fallthrough; 4864 default: 4865 kvm_run->exit_reason = KVM_EXIT_EXCEPTION; 4866 kvm_run->ex.exception = ex_no; 4867 kvm_run->ex.error_code = error_code; 4868 break; 4869 } 4870 return 0; 4871 } 4872 4873 static __always_inline int handle_external_interrupt(struct kvm_vcpu *vcpu) 4874 { 4875 ++vcpu->stat.irq_exits; 4876 return 1; 4877 } 4878 4879 static int handle_triple_fault(struct kvm_vcpu *vcpu) 4880 { 4881 vcpu->run->exit_reason = KVM_EXIT_SHUTDOWN; 4882 vcpu->mmio_needed = 0; 4883 return 0; 4884 } 4885 4886 static int handle_io(struct kvm_vcpu *vcpu) 4887 { 4888 unsigned long exit_qualification; 4889 int size, in, string; 4890 unsigned port; 4891 4892 exit_qualification = vmx_get_exit_qual(vcpu); 4893 string = (exit_qualification & 16) != 0; 4894 4895 ++vcpu->stat.io_exits; 4896 4897 if (string) 4898 return kvm_emulate_instruction(vcpu, 0); 4899 4900 port = exit_qualification >> 16; 4901 size = (exit_qualification & 7) + 1; 4902 in = (exit_qualification & 8) != 0; 4903 4904 return kvm_fast_pio(vcpu, size, port, in); 4905 } 4906 4907 static void 4908 vmx_patch_hypercall(struct kvm_vcpu *vcpu, unsigned char *hypercall) 4909 { 4910 /* 4911 * Patch in the VMCALL instruction: 4912 */ 4913 hypercall[0] = 0x0f; 4914 hypercall[1] = 0x01; 4915 hypercall[2] = 0xc1; 4916 } 4917 4918 /* called to set cr0 as appropriate for a mov-to-cr0 exit. */ 4919 static int handle_set_cr0(struct kvm_vcpu *vcpu, unsigned long val) 4920 { 4921 if (is_guest_mode(vcpu)) { 4922 struct vmcs12 *vmcs12 = get_vmcs12(vcpu); 4923 unsigned long orig_val = val; 4924 4925 /* 4926 * We get here when L2 changed cr0 in a way that did not change 4927 * any of L1's shadowed bits (see nested_vmx_exit_handled_cr), 4928 * but did change L0 shadowed bits. So we first calculate the 4929 * effective cr0 value that L1 would like to write into the 4930 * hardware. It consists of the L2-owned bits from the new 4931 * value combined with the L1-owned bits from L1's guest_cr0. 4932 */ 4933 val = (val & ~vmcs12->cr0_guest_host_mask) | 4934 (vmcs12->guest_cr0 & vmcs12->cr0_guest_host_mask); 4935 4936 if (!nested_guest_cr0_valid(vcpu, val)) 4937 return 1; 4938 4939 if (kvm_set_cr0(vcpu, val)) 4940 return 1; 4941 vmcs_writel(CR0_READ_SHADOW, orig_val); 4942 return 0; 4943 } else { 4944 if (to_vmx(vcpu)->nested.vmxon && 4945 !nested_host_cr0_valid(vcpu, val)) 4946 return 1; 4947 4948 return kvm_set_cr0(vcpu, val); 4949 } 4950 } 4951 4952 static int handle_set_cr4(struct kvm_vcpu *vcpu, unsigned long val) 4953 { 4954 if (is_guest_mode(vcpu)) { 4955 struct vmcs12 *vmcs12 = get_vmcs12(vcpu); 4956 unsigned long orig_val = val; 4957 4958 /* analogously to handle_set_cr0 */ 4959 val = (val & ~vmcs12->cr4_guest_host_mask) | 4960 (vmcs12->guest_cr4 & vmcs12->cr4_guest_host_mask); 4961 if (kvm_set_cr4(vcpu, val)) 4962 return 1; 4963 vmcs_writel(CR4_READ_SHADOW, orig_val); 4964 return 0; 4965 } else 4966 return kvm_set_cr4(vcpu, val); 4967 } 4968 4969 static int handle_desc(struct kvm_vcpu *vcpu) 4970 { 4971 WARN_ON(!(vcpu->arch.cr4 & X86_CR4_UMIP)); 4972 return kvm_emulate_instruction(vcpu, 0); 4973 } 4974 4975 static int handle_cr(struct kvm_vcpu *vcpu) 4976 { 4977 unsigned long exit_qualification, val; 4978 int cr; 4979 int reg; 4980 int err; 4981 int ret; 4982 4983 exit_qualification = vmx_get_exit_qual(vcpu); 4984 cr = exit_qualification & 15; 4985 reg = (exit_qualification >> 8) & 15; 4986 switch ((exit_qualification >> 4) & 3) { 4987 case 0: /* mov to cr */ 4988 val = kvm_register_readl(vcpu, reg); 4989 trace_kvm_cr_write(cr, val); 4990 switch (cr) { 4991 case 0: 4992 err = handle_set_cr0(vcpu, val); 4993 return kvm_complete_insn_gp(vcpu, err); 4994 case 3: 4995 WARN_ON_ONCE(enable_unrestricted_guest); 4996 err = kvm_set_cr3(vcpu, val); 4997 return kvm_complete_insn_gp(vcpu, err); 4998 case 4: 4999 err = handle_set_cr4(vcpu, val); 5000 return kvm_complete_insn_gp(vcpu, err); 5001 case 8: { 5002 u8 cr8_prev = kvm_get_cr8(vcpu); 5003 u8 cr8 = (u8)val; 5004 err = kvm_set_cr8(vcpu, cr8); 5005 ret = kvm_complete_insn_gp(vcpu, err); 5006 if (lapic_in_kernel(vcpu)) 5007 return ret; 5008 if (cr8_prev <= cr8) 5009 return ret; 5010 /* 5011 * TODO: we might be squashing a 5012 * KVM_GUESTDBG_SINGLESTEP-triggered 5013 * KVM_EXIT_DEBUG here. 5014 */ 5015 vcpu->run->exit_reason = KVM_EXIT_SET_TPR; 5016 return 0; 5017 } 5018 } 5019 break; 5020 case 2: /* clts */ 5021 WARN_ONCE(1, "Guest should always own CR0.TS"); 5022 vmx_set_cr0(vcpu, kvm_read_cr0_bits(vcpu, ~X86_CR0_TS)); 5023 trace_kvm_cr_write(0, kvm_read_cr0(vcpu)); 5024 return kvm_skip_emulated_instruction(vcpu); 5025 case 1: /*mov from cr*/ 5026 switch (cr) { 5027 case 3: 5028 WARN_ON_ONCE(enable_unrestricted_guest); 5029 val = kvm_read_cr3(vcpu); 5030 kvm_register_write(vcpu, reg, val); 5031 trace_kvm_cr_read(cr, val); 5032 return kvm_skip_emulated_instruction(vcpu); 5033 case 8: 5034 val = kvm_get_cr8(vcpu); 5035 kvm_register_write(vcpu, reg, val); 5036 trace_kvm_cr_read(cr, val); 5037 return kvm_skip_emulated_instruction(vcpu); 5038 } 5039 break; 5040 case 3: /* lmsw */ 5041 val = (exit_qualification >> LMSW_SOURCE_DATA_SHIFT) & 0x0f; 5042 trace_kvm_cr_write(0, (kvm_read_cr0(vcpu) & ~0xful) | val); 5043 kvm_lmsw(vcpu, val); 5044 5045 return kvm_skip_emulated_instruction(vcpu); 5046 default: 5047 break; 5048 } 5049 vcpu->run->exit_reason = 0; 5050 vcpu_unimpl(vcpu, "unhandled control register: op %d cr %d\n", 5051 (int)(exit_qualification >> 4) & 3, cr); 5052 return 0; 5053 } 5054 5055 static int handle_dr(struct kvm_vcpu *vcpu) 5056 { 5057 unsigned long exit_qualification; 5058 int dr, dr7, reg; 5059 5060 exit_qualification = vmx_get_exit_qual(vcpu); 5061 dr = exit_qualification & DEBUG_REG_ACCESS_NUM; 5062 5063 /* First, if DR does not exist, trigger UD */ 5064 if (!kvm_require_dr(vcpu, dr)) 5065 return 1; 5066 5067 /* Do not handle if the CPL > 0, will trigger GP on re-entry */ 5068 if (!kvm_require_cpl(vcpu, 0)) 5069 return 1; 5070 dr7 = vmcs_readl(GUEST_DR7); 5071 if (dr7 & DR7_GD) { 5072 /* 5073 * As the vm-exit takes precedence over the debug trap, we 5074 * need to emulate the latter, either for the host or the 5075 * guest debugging itself. 5076 */ 5077 if (vcpu->guest_debug & KVM_GUESTDBG_USE_HW_BP) { 5078 vcpu->run->debug.arch.dr6 = DR6_BD | DR6_RTM | DR6_FIXED_1; 5079 vcpu->run->debug.arch.dr7 = dr7; 5080 vcpu->run->debug.arch.pc = kvm_get_linear_rip(vcpu); 5081 vcpu->run->debug.arch.exception = DB_VECTOR; 5082 vcpu->run->exit_reason = KVM_EXIT_DEBUG; 5083 return 0; 5084 } else { 5085 kvm_queue_exception_p(vcpu, DB_VECTOR, DR6_BD); 5086 return 1; 5087 } 5088 } 5089 5090 if (vcpu->guest_debug == 0) { 5091 exec_controls_clearbit(to_vmx(vcpu), CPU_BASED_MOV_DR_EXITING); 5092 5093 /* 5094 * No more DR vmexits; force a reload of the debug registers 5095 * and reenter on this instruction. The next vmexit will 5096 * retrieve the full state of the debug registers. 5097 */ 5098 vcpu->arch.switch_db_regs |= KVM_DEBUGREG_WONT_EXIT; 5099 return 1; 5100 } 5101 5102 reg = DEBUG_REG_ACCESS_REG(exit_qualification); 5103 if (exit_qualification & TYPE_MOV_FROM_DR) { 5104 unsigned long val; 5105 5106 if (kvm_get_dr(vcpu, dr, &val)) 5107 return 1; 5108 kvm_register_write(vcpu, reg, val); 5109 } else 5110 if (kvm_set_dr(vcpu, dr, kvm_register_readl(vcpu, reg))) 5111 return 1; 5112 5113 return kvm_skip_emulated_instruction(vcpu); 5114 } 5115 5116 static void vmx_sync_dirty_debug_regs(struct kvm_vcpu *vcpu) 5117 { 5118 get_debugreg(vcpu->arch.db[0], 0); 5119 get_debugreg(vcpu->arch.db[1], 1); 5120 get_debugreg(vcpu->arch.db[2], 2); 5121 get_debugreg(vcpu->arch.db[3], 3); 5122 get_debugreg(vcpu->arch.dr6, 6); 5123 vcpu->arch.dr7 = vmcs_readl(GUEST_DR7); 5124 5125 vcpu->arch.switch_db_regs &= ~KVM_DEBUGREG_WONT_EXIT; 5126 exec_controls_setbit(to_vmx(vcpu), CPU_BASED_MOV_DR_EXITING); 5127 } 5128 5129 static void vmx_set_dr7(struct kvm_vcpu *vcpu, unsigned long val) 5130 { 5131 vmcs_writel(GUEST_DR7, val); 5132 } 5133 5134 static int handle_tpr_below_threshold(struct kvm_vcpu *vcpu) 5135 { 5136 kvm_apic_update_ppr(vcpu); 5137 return 1; 5138 } 5139 5140 static int handle_interrupt_window(struct kvm_vcpu *vcpu) 5141 { 5142 exec_controls_clearbit(to_vmx(vcpu), CPU_BASED_INTR_WINDOW_EXITING); 5143 5144 kvm_make_request(KVM_REQ_EVENT, vcpu); 5145 5146 ++vcpu->stat.irq_window_exits; 5147 return 1; 5148 } 5149 5150 static int handle_vmcall(struct kvm_vcpu *vcpu) 5151 { 5152 return kvm_emulate_hypercall(vcpu); 5153 } 5154 5155 static int handle_invd(struct kvm_vcpu *vcpu) 5156 { 5157 return kvm_emulate_instruction(vcpu, 0); 5158 } 5159 5160 static int handle_invlpg(struct kvm_vcpu *vcpu) 5161 { 5162 unsigned long exit_qualification = vmx_get_exit_qual(vcpu); 5163 5164 kvm_mmu_invlpg(vcpu, exit_qualification); 5165 return kvm_skip_emulated_instruction(vcpu); 5166 } 5167 5168 static int handle_rdpmc(struct kvm_vcpu *vcpu) 5169 { 5170 int err; 5171 5172 err = kvm_rdpmc(vcpu); 5173 return kvm_complete_insn_gp(vcpu, err); 5174 } 5175 5176 static int handle_wbinvd(struct kvm_vcpu *vcpu) 5177 { 5178 return kvm_emulate_wbinvd(vcpu); 5179 } 5180 5181 static int handle_xsetbv(struct kvm_vcpu *vcpu) 5182 { 5183 u64 new_bv = kvm_read_edx_eax(vcpu); 5184 u32 index = kvm_rcx_read(vcpu); 5185 5186 if (kvm_set_xcr(vcpu, index, new_bv) == 0) 5187 return kvm_skip_emulated_instruction(vcpu); 5188 return 1; 5189 } 5190 5191 static int handle_apic_access(struct kvm_vcpu *vcpu) 5192 { 5193 if (likely(fasteoi)) { 5194 unsigned long exit_qualification = vmx_get_exit_qual(vcpu); 5195 int access_type, offset; 5196 5197 access_type = exit_qualification & APIC_ACCESS_TYPE; 5198 offset = exit_qualification & APIC_ACCESS_OFFSET; 5199 /* 5200 * Sane guest uses MOV to write EOI, with written value 5201 * not cared. So make a short-circuit here by avoiding 5202 * heavy instruction emulation. 5203 */ 5204 if ((access_type == TYPE_LINEAR_APIC_INST_WRITE) && 5205 (offset == APIC_EOI)) { 5206 kvm_lapic_set_eoi(vcpu); 5207 return kvm_skip_emulated_instruction(vcpu); 5208 } 5209 } 5210 return kvm_emulate_instruction(vcpu, 0); 5211 } 5212 5213 static int handle_apic_eoi_induced(struct kvm_vcpu *vcpu) 5214 { 5215 unsigned long exit_qualification = vmx_get_exit_qual(vcpu); 5216 int vector = exit_qualification & 0xff; 5217 5218 /* EOI-induced VM exit is trap-like and thus no need to adjust IP */ 5219 kvm_apic_set_eoi_accelerated(vcpu, vector); 5220 return 1; 5221 } 5222 5223 static int handle_apic_write(struct kvm_vcpu *vcpu) 5224 { 5225 unsigned long exit_qualification = vmx_get_exit_qual(vcpu); 5226 u32 offset = exit_qualification & 0xfff; 5227 5228 /* APIC-write VM exit is trap-like and thus no need to adjust IP */ 5229 kvm_apic_write_nodecode(vcpu, offset); 5230 return 1; 5231 } 5232 5233 static int handle_task_switch(struct kvm_vcpu *vcpu) 5234 { 5235 struct vcpu_vmx *vmx = to_vmx(vcpu); 5236 unsigned long exit_qualification; 5237 bool has_error_code = false; 5238 u32 error_code = 0; 5239 u16 tss_selector; 5240 int reason, type, idt_v, idt_index; 5241 5242 idt_v = (vmx->idt_vectoring_info & VECTORING_INFO_VALID_MASK); 5243 idt_index = (vmx->idt_vectoring_info & VECTORING_INFO_VECTOR_MASK); 5244 type = (vmx->idt_vectoring_info & VECTORING_INFO_TYPE_MASK); 5245 5246 exit_qualification = vmx_get_exit_qual(vcpu); 5247 5248 reason = (u32)exit_qualification >> 30; 5249 if (reason == TASK_SWITCH_GATE && idt_v) { 5250 switch (type) { 5251 case INTR_TYPE_NMI_INTR: 5252 vcpu->arch.nmi_injected = false; 5253 vmx_set_nmi_mask(vcpu, true); 5254 break; 5255 case INTR_TYPE_EXT_INTR: 5256 case INTR_TYPE_SOFT_INTR: 5257 kvm_clear_interrupt_queue(vcpu); 5258 break; 5259 case INTR_TYPE_HARD_EXCEPTION: 5260 if (vmx->idt_vectoring_info & 5261 VECTORING_INFO_DELIVER_CODE_MASK) { 5262 has_error_code = true; 5263 error_code = 5264 vmcs_read32(IDT_VECTORING_ERROR_CODE); 5265 } 5266 fallthrough; 5267 case INTR_TYPE_SOFT_EXCEPTION: 5268 kvm_clear_exception_queue(vcpu); 5269 break; 5270 default: 5271 break; 5272 } 5273 } 5274 tss_selector = exit_qualification; 5275 5276 if (!idt_v || (type != INTR_TYPE_HARD_EXCEPTION && 5277 type != INTR_TYPE_EXT_INTR && 5278 type != INTR_TYPE_NMI_INTR)) 5279 WARN_ON(!skip_emulated_instruction(vcpu)); 5280 5281 /* 5282 * TODO: What about debug traps on tss switch? 5283 * Are we supposed to inject them and update dr6? 5284 */ 5285 return kvm_task_switch(vcpu, tss_selector, 5286 type == INTR_TYPE_SOFT_INTR ? idt_index : -1, 5287 reason, has_error_code, error_code); 5288 } 5289 5290 static int handle_ept_violation(struct kvm_vcpu *vcpu) 5291 { 5292 unsigned long exit_qualification; 5293 gpa_t gpa; 5294 u64 error_code; 5295 5296 exit_qualification = vmx_get_exit_qual(vcpu); 5297 5298 /* 5299 * EPT violation happened while executing iret from NMI, 5300 * "blocked by NMI" bit has to be set before next VM entry. 5301 * There are errata that may cause this bit to not be set: 5302 * AAK134, BY25. 5303 */ 5304 if (!(to_vmx(vcpu)->idt_vectoring_info & VECTORING_INFO_VALID_MASK) && 5305 enable_vnmi && 5306 (exit_qualification & INTR_INFO_UNBLOCK_NMI)) 5307 vmcs_set_bits(GUEST_INTERRUPTIBILITY_INFO, GUEST_INTR_STATE_NMI); 5308 5309 gpa = vmcs_read64(GUEST_PHYSICAL_ADDRESS); 5310 trace_kvm_page_fault(gpa, exit_qualification); 5311 5312 /* Is it a read fault? */ 5313 error_code = (exit_qualification & EPT_VIOLATION_ACC_READ) 5314 ? PFERR_USER_MASK : 0; 5315 /* Is it a write fault? */ 5316 error_code |= (exit_qualification & EPT_VIOLATION_ACC_WRITE) 5317 ? PFERR_WRITE_MASK : 0; 5318 /* Is it a fetch fault? */ 5319 error_code |= (exit_qualification & EPT_VIOLATION_ACC_INSTR) 5320 ? PFERR_FETCH_MASK : 0; 5321 /* ept page table entry is present? */ 5322 error_code |= (exit_qualification & 5323 (EPT_VIOLATION_READABLE | EPT_VIOLATION_WRITABLE | 5324 EPT_VIOLATION_EXECUTABLE)) 5325 ? PFERR_PRESENT_MASK : 0; 5326 5327 error_code |= (exit_qualification & 0x100) != 0 ? 5328 PFERR_GUEST_FINAL_MASK : PFERR_GUEST_PAGE_MASK; 5329 5330 vcpu->arch.exit_qualification = exit_qualification; 5331 5332 /* 5333 * Check that the GPA doesn't exceed physical memory limits, as that is 5334 * a guest page fault. We have to emulate the instruction here, because 5335 * if the illegal address is that of a paging structure, then 5336 * EPT_VIOLATION_ACC_WRITE bit is set. Alternatively, if supported we 5337 * would also use advanced VM-exit information for EPT violations to 5338 * reconstruct the page fault error code. 5339 */ 5340 if (unlikely(allow_smaller_maxphyaddr && kvm_mmu_is_illegal_gpa(vcpu, gpa))) 5341 return kvm_emulate_instruction(vcpu, 0); 5342 5343 return kvm_mmu_page_fault(vcpu, gpa, error_code, NULL, 0); 5344 } 5345 5346 static int handle_ept_misconfig(struct kvm_vcpu *vcpu) 5347 { 5348 gpa_t gpa; 5349 5350 /* 5351 * A nested guest cannot optimize MMIO vmexits, because we have an 5352 * nGPA here instead of the required GPA. 5353 */ 5354 gpa = vmcs_read64(GUEST_PHYSICAL_ADDRESS); 5355 if (!is_guest_mode(vcpu) && 5356 !kvm_io_bus_write(vcpu, KVM_FAST_MMIO_BUS, gpa, 0, NULL)) { 5357 trace_kvm_fast_mmio(gpa); 5358 return kvm_skip_emulated_instruction(vcpu); 5359 } 5360 5361 return kvm_mmu_page_fault(vcpu, gpa, PFERR_RSVD_MASK, NULL, 0); 5362 } 5363 5364 static int handle_nmi_window(struct kvm_vcpu *vcpu) 5365 { 5366 WARN_ON_ONCE(!enable_vnmi); 5367 exec_controls_clearbit(to_vmx(vcpu), CPU_BASED_NMI_WINDOW_EXITING); 5368 ++vcpu->stat.nmi_window_exits; 5369 kvm_make_request(KVM_REQ_EVENT, vcpu); 5370 5371 return 1; 5372 } 5373 5374 static int handle_invalid_guest_state(struct kvm_vcpu *vcpu) 5375 { 5376 struct vcpu_vmx *vmx = to_vmx(vcpu); 5377 bool intr_window_requested; 5378 unsigned count = 130; 5379 5380 intr_window_requested = exec_controls_get(vmx) & 5381 CPU_BASED_INTR_WINDOW_EXITING; 5382 5383 while (vmx->emulation_required && count-- != 0) { 5384 if (intr_window_requested && !vmx_interrupt_blocked(vcpu)) 5385 return handle_interrupt_window(&vmx->vcpu); 5386 5387 if (kvm_test_request(KVM_REQ_EVENT, vcpu)) 5388 return 1; 5389 5390 if (!kvm_emulate_instruction(vcpu, 0)) 5391 return 0; 5392 5393 if (vmx->emulation_required && !vmx->rmode.vm86_active && 5394 vcpu->arch.exception.pending) { 5395 vcpu->run->exit_reason = KVM_EXIT_INTERNAL_ERROR; 5396 vcpu->run->internal.suberror = 5397 KVM_INTERNAL_ERROR_EMULATION; 5398 vcpu->run->internal.ndata = 0; 5399 return 0; 5400 } 5401 5402 if (vcpu->arch.halt_request) { 5403 vcpu->arch.halt_request = 0; 5404 return kvm_vcpu_halt(vcpu); 5405 } 5406 5407 /* 5408 * Note, return 1 and not 0, vcpu_run() will invoke 5409 * xfer_to_guest_mode() which will create a proper return 5410 * code. 5411 */ 5412 if (__xfer_to_guest_mode_work_pending()) 5413 return 1; 5414 } 5415 5416 return 1; 5417 } 5418 5419 static void grow_ple_window(struct kvm_vcpu *vcpu) 5420 { 5421 struct vcpu_vmx *vmx = to_vmx(vcpu); 5422 unsigned int old = vmx->ple_window; 5423 5424 vmx->ple_window = __grow_ple_window(old, ple_window, 5425 ple_window_grow, 5426 ple_window_max); 5427 5428 if (vmx->ple_window != old) { 5429 vmx->ple_window_dirty = true; 5430 trace_kvm_ple_window_update(vcpu->vcpu_id, 5431 vmx->ple_window, old); 5432 } 5433 } 5434 5435 static void shrink_ple_window(struct kvm_vcpu *vcpu) 5436 { 5437 struct vcpu_vmx *vmx = to_vmx(vcpu); 5438 unsigned int old = vmx->ple_window; 5439 5440 vmx->ple_window = __shrink_ple_window(old, ple_window, 5441 ple_window_shrink, 5442 ple_window); 5443 5444 if (vmx->ple_window != old) { 5445 vmx->ple_window_dirty = true; 5446 trace_kvm_ple_window_update(vcpu->vcpu_id, 5447 vmx->ple_window, old); 5448 } 5449 } 5450 5451 /* 5452 * Handler for POSTED_INTERRUPT_WAKEUP_VECTOR. 5453 */ 5454 static void wakeup_handler(void) 5455 { 5456 struct kvm_vcpu *vcpu; 5457 int cpu = smp_processor_id(); 5458 5459 spin_lock(&per_cpu(blocked_vcpu_on_cpu_lock, cpu)); 5460 list_for_each_entry(vcpu, &per_cpu(blocked_vcpu_on_cpu, cpu), 5461 blocked_vcpu_list) { 5462 struct pi_desc *pi_desc = vcpu_to_pi_desc(vcpu); 5463 5464 if (pi_test_on(pi_desc) == 1) 5465 kvm_vcpu_kick(vcpu); 5466 } 5467 spin_unlock(&per_cpu(blocked_vcpu_on_cpu_lock, cpu)); 5468 } 5469 5470 static void vmx_enable_tdp(void) 5471 { 5472 kvm_mmu_set_mask_ptes(VMX_EPT_READABLE_MASK, 5473 enable_ept_ad_bits ? VMX_EPT_ACCESS_BIT : 0ull, 5474 enable_ept_ad_bits ? VMX_EPT_DIRTY_BIT : 0ull, 5475 0ull, VMX_EPT_EXECUTABLE_MASK, 5476 cpu_has_vmx_ept_execute_only() ? 0ull : VMX_EPT_READABLE_MASK, 5477 VMX_EPT_RWX_MASK, 0ull); 5478 5479 ept_set_mmio_spte_mask(); 5480 } 5481 5482 /* 5483 * Indicate a busy-waiting vcpu in spinlock. We do not enable the PAUSE 5484 * exiting, so only get here on cpu with PAUSE-Loop-Exiting. 5485 */ 5486 static int handle_pause(struct kvm_vcpu *vcpu) 5487 { 5488 if (!kvm_pause_in_guest(vcpu->kvm)) 5489 grow_ple_window(vcpu); 5490 5491 /* 5492 * Intel sdm vol3 ch-25.1.3 says: The "PAUSE-loop exiting" 5493 * VM-execution control is ignored if CPL > 0. OTOH, KVM 5494 * never set PAUSE_EXITING and just set PLE if supported, 5495 * so the vcpu must be CPL=0 if it gets a PAUSE exit. 5496 */ 5497 kvm_vcpu_on_spin(vcpu, true); 5498 return kvm_skip_emulated_instruction(vcpu); 5499 } 5500 5501 static int handle_nop(struct kvm_vcpu *vcpu) 5502 { 5503 return kvm_skip_emulated_instruction(vcpu); 5504 } 5505 5506 static int handle_mwait(struct kvm_vcpu *vcpu) 5507 { 5508 printk_once(KERN_WARNING "kvm: MWAIT instruction emulated as NOP!\n"); 5509 return handle_nop(vcpu); 5510 } 5511 5512 static int handle_invalid_op(struct kvm_vcpu *vcpu) 5513 { 5514 kvm_queue_exception(vcpu, UD_VECTOR); 5515 return 1; 5516 } 5517 5518 static int handle_monitor_trap(struct kvm_vcpu *vcpu) 5519 { 5520 return 1; 5521 } 5522 5523 static int handle_monitor(struct kvm_vcpu *vcpu) 5524 { 5525 printk_once(KERN_WARNING "kvm: MONITOR instruction emulated as NOP!\n"); 5526 return handle_nop(vcpu); 5527 } 5528 5529 static int handle_invpcid(struct kvm_vcpu *vcpu) 5530 { 5531 u32 vmx_instruction_info; 5532 unsigned long type; 5533 bool pcid_enabled; 5534 gva_t gva; 5535 struct x86_exception e; 5536 unsigned i; 5537 unsigned long roots_to_free = 0; 5538 struct { 5539 u64 pcid; 5540 u64 gla; 5541 } operand; 5542 int r; 5543 5544 if (!guest_cpuid_has(vcpu, X86_FEATURE_INVPCID)) { 5545 kvm_queue_exception(vcpu, UD_VECTOR); 5546 return 1; 5547 } 5548 5549 vmx_instruction_info = vmcs_read32(VMX_INSTRUCTION_INFO); 5550 type = kvm_register_readl(vcpu, (vmx_instruction_info >> 28) & 0xf); 5551 5552 if (type > 3) { 5553 kvm_inject_gp(vcpu, 0); 5554 return 1; 5555 } 5556 5557 /* According to the Intel instruction reference, the memory operand 5558 * is read even if it isn't needed (e.g., for type==all) 5559 */ 5560 if (get_vmx_mem_address(vcpu, vmx_get_exit_qual(vcpu), 5561 vmx_instruction_info, false, 5562 sizeof(operand), &gva)) 5563 return 1; 5564 5565 r = kvm_read_guest_virt(vcpu, gva, &operand, sizeof(operand), &e); 5566 if (r != X86EMUL_CONTINUE) 5567 return vmx_handle_memory_failure(vcpu, r, &e); 5568 5569 if (operand.pcid >> 12 != 0) { 5570 kvm_inject_gp(vcpu, 0); 5571 return 1; 5572 } 5573 5574 pcid_enabled = kvm_read_cr4_bits(vcpu, X86_CR4_PCIDE); 5575 5576 switch (type) { 5577 case INVPCID_TYPE_INDIV_ADDR: 5578 if ((!pcid_enabled && (operand.pcid != 0)) || 5579 is_noncanonical_address(operand.gla, vcpu)) { 5580 kvm_inject_gp(vcpu, 0); 5581 return 1; 5582 } 5583 kvm_mmu_invpcid_gva(vcpu, operand.gla, operand.pcid); 5584 return kvm_skip_emulated_instruction(vcpu); 5585 5586 case INVPCID_TYPE_SINGLE_CTXT: 5587 if (!pcid_enabled && (operand.pcid != 0)) { 5588 kvm_inject_gp(vcpu, 0); 5589 return 1; 5590 } 5591 5592 if (kvm_get_active_pcid(vcpu) == operand.pcid) { 5593 kvm_mmu_sync_roots(vcpu); 5594 kvm_make_request(KVM_REQ_TLB_FLUSH_CURRENT, vcpu); 5595 } 5596 5597 for (i = 0; i < KVM_MMU_NUM_PREV_ROOTS; i++) 5598 if (kvm_get_pcid(vcpu, vcpu->arch.mmu->prev_roots[i].pgd) 5599 == operand.pcid) 5600 roots_to_free |= KVM_MMU_ROOT_PREVIOUS(i); 5601 5602 kvm_mmu_free_roots(vcpu, vcpu->arch.mmu, roots_to_free); 5603 /* 5604 * If neither the current cr3 nor any of the prev_roots use the 5605 * given PCID, then nothing needs to be done here because a 5606 * resync will happen anyway before switching to any other CR3. 5607 */ 5608 5609 return kvm_skip_emulated_instruction(vcpu); 5610 5611 case INVPCID_TYPE_ALL_NON_GLOBAL: 5612 /* 5613 * Currently, KVM doesn't mark global entries in the shadow 5614 * page tables, so a non-global flush just degenerates to a 5615 * global flush. If needed, we could optimize this later by 5616 * keeping track of global entries in shadow page tables. 5617 */ 5618 5619 fallthrough; 5620 case INVPCID_TYPE_ALL_INCL_GLOBAL: 5621 kvm_mmu_unload(vcpu); 5622 return kvm_skip_emulated_instruction(vcpu); 5623 5624 default: 5625 BUG(); /* We have already checked above that type <= 3 */ 5626 } 5627 } 5628 5629 static int handle_pml_full(struct kvm_vcpu *vcpu) 5630 { 5631 unsigned long exit_qualification; 5632 5633 trace_kvm_pml_full(vcpu->vcpu_id); 5634 5635 exit_qualification = vmx_get_exit_qual(vcpu); 5636 5637 /* 5638 * PML buffer FULL happened while executing iret from NMI, 5639 * "blocked by NMI" bit has to be set before next VM entry. 5640 */ 5641 if (!(to_vmx(vcpu)->idt_vectoring_info & VECTORING_INFO_VALID_MASK) && 5642 enable_vnmi && 5643 (exit_qualification & INTR_INFO_UNBLOCK_NMI)) 5644 vmcs_set_bits(GUEST_INTERRUPTIBILITY_INFO, 5645 GUEST_INTR_STATE_NMI); 5646 5647 /* 5648 * PML buffer already flushed at beginning of VMEXIT. Nothing to do 5649 * here.., and there's no userspace involvement needed for PML. 5650 */ 5651 return 1; 5652 } 5653 5654 static fastpath_t handle_fastpath_preemption_timer(struct kvm_vcpu *vcpu) 5655 { 5656 struct vcpu_vmx *vmx = to_vmx(vcpu); 5657 5658 if (!vmx->req_immediate_exit && 5659 !unlikely(vmx->loaded_vmcs->hv_timer_soft_disabled)) { 5660 kvm_lapic_expired_hv_timer(vcpu); 5661 return EXIT_FASTPATH_REENTER_GUEST; 5662 } 5663 5664 return EXIT_FASTPATH_NONE; 5665 } 5666 5667 static int handle_preemption_timer(struct kvm_vcpu *vcpu) 5668 { 5669 handle_fastpath_preemption_timer(vcpu); 5670 return 1; 5671 } 5672 5673 /* 5674 * When nested=0, all VMX instruction VM Exits filter here. The handlers 5675 * are overwritten by nested_vmx_setup() when nested=1. 5676 */ 5677 static int handle_vmx_instruction(struct kvm_vcpu *vcpu) 5678 { 5679 kvm_queue_exception(vcpu, UD_VECTOR); 5680 return 1; 5681 } 5682 5683 static int handle_encls(struct kvm_vcpu *vcpu) 5684 { 5685 /* 5686 * SGX virtualization is not yet supported. There is no software 5687 * enable bit for SGX, so we have to trap ENCLS and inject a #UD 5688 * to prevent the guest from executing ENCLS. 5689 */ 5690 kvm_queue_exception(vcpu, UD_VECTOR); 5691 return 1; 5692 } 5693 5694 /* 5695 * The exit handlers return 1 if the exit was handled fully and guest execution 5696 * may resume. Otherwise they set the kvm_run parameter to indicate what needs 5697 * to be done to userspace and return 0. 5698 */ 5699 static int (*kvm_vmx_exit_handlers[])(struct kvm_vcpu *vcpu) = { 5700 [EXIT_REASON_EXCEPTION_NMI] = handle_exception_nmi, 5701 [EXIT_REASON_EXTERNAL_INTERRUPT] = handle_external_interrupt, 5702 [EXIT_REASON_TRIPLE_FAULT] = handle_triple_fault, 5703 [EXIT_REASON_NMI_WINDOW] = handle_nmi_window, 5704 [EXIT_REASON_IO_INSTRUCTION] = handle_io, 5705 [EXIT_REASON_CR_ACCESS] = handle_cr, 5706 [EXIT_REASON_DR_ACCESS] = handle_dr, 5707 [EXIT_REASON_CPUID] = kvm_emulate_cpuid, 5708 [EXIT_REASON_MSR_READ] = kvm_emulate_rdmsr, 5709 [EXIT_REASON_MSR_WRITE] = kvm_emulate_wrmsr, 5710 [EXIT_REASON_INTERRUPT_WINDOW] = handle_interrupt_window, 5711 [EXIT_REASON_HLT] = kvm_emulate_halt, 5712 [EXIT_REASON_INVD] = handle_invd, 5713 [EXIT_REASON_INVLPG] = handle_invlpg, 5714 [EXIT_REASON_RDPMC] = handle_rdpmc, 5715 [EXIT_REASON_VMCALL] = handle_vmcall, 5716 [EXIT_REASON_VMCLEAR] = handle_vmx_instruction, 5717 [EXIT_REASON_VMLAUNCH] = handle_vmx_instruction, 5718 [EXIT_REASON_VMPTRLD] = handle_vmx_instruction, 5719 [EXIT_REASON_VMPTRST] = handle_vmx_instruction, 5720 [EXIT_REASON_VMREAD] = handle_vmx_instruction, 5721 [EXIT_REASON_VMRESUME] = handle_vmx_instruction, 5722 [EXIT_REASON_VMWRITE] = handle_vmx_instruction, 5723 [EXIT_REASON_VMOFF] = handle_vmx_instruction, 5724 [EXIT_REASON_VMON] = handle_vmx_instruction, 5725 [EXIT_REASON_TPR_BELOW_THRESHOLD] = handle_tpr_below_threshold, 5726 [EXIT_REASON_APIC_ACCESS] = handle_apic_access, 5727 [EXIT_REASON_APIC_WRITE] = handle_apic_write, 5728 [EXIT_REASON_EOI_INDUCED] = handle_apic_eoi_induced, 5729 [EXIT_REASON_WBINVD] = handle_wbinvd, 5730 [EXIT_REASON_XSETBV] = handle_xsetbv, 5731 [EXIT_REASON_TASK_SWITCH] = handle_task_switch, 5732 [EXIT_REASON_MCE_DURING_VMENTRY] = handle_machine_check, 5733 [EXIT_REASON_GDTR_IDTR] = handle_desc, 5734 [EXIT_REASON_LDTR_TR] = handle_desc, 5735 [EXIT_REASON_EPT_VIOLATION] = handle_ept_violation, 5736 [EXIT_REASON_EPT_MISCONFIG] = handle_ept_misconfig, 5737 [EXIT_REASON_PAUSE_INSTRUCTION] = handle_pause, 5738 [EXIT_REASON_MWAIT_INSTRUCTION] = handle_mwait, 5739 [EXIT_REASON_MONITOR_TRAP_FLAG] = handle_monitor_trap, 5740 [EXIT_REASON_MONITOR_INSTRUCTION] = handle_monitor, 5741 [EXIT_REASON_INVEPT] = handle_vmx_instruction, 5742 [EXIT_REASON_INVVPID] = handle_vmx_instruction, 5743 [EXIT_REASON_RDRAND] = handle_invalid_op, 5744 [EXIT_REASON_RDSEED] = handle_invalid_op, 5745 [EXIT_REASON_PML_FULL] = handle_pml_full, 5746 [EXIT_REASON_INVPCID] = handle_invpcid, 5747 [EXIT_REASON_VMFUNC] = handle_vmx_instruction, 5748 [EXIT_REASON_PREEMPTION_TIMER] = handle_preemption_timer, 5749 [EXIT_REASON_ENCLS] = handle_encls, 5750 }; 5751 5752 static const int kvm_vmx_max_exit_handlers = 5753 ARRAY_SIZE(kvm_vmx_exit_handlers); 5754 5755 static void vmx_get_exit_info(struct kvm_vcpu *vcpu, u64 *info1, u64 *info2) 5756 { 5757 *info1 = vmx_get_exit_qual(vcpu); 5758 *info2 = vmx_get_intr_info(vcpu); 5759 } 5760 5761 static void vmx_destroy_pml_buffer(struct vcpu_vmx *vmx) 5762 { 5763 if (vmx->pml_pg) { 5764 __free_page(vmx->pml_pg); 5765 vmx->pml_pg = NULL; 5766 } 5767 } 5768 5769 static void vmx_flush_pml_buffer(struct kvm_vcpu *vcpu) 5770 { 5771 struct vcpu_vmx *vmx = to_vmx(vcpu); 5772 u64 *pml_buf; 5773 u16 pml_idx; 5774 5775 pml_idx = vmcs_read16(GUEST_PML_INDEX); 5776 5777 /* Do nothing if PML buffer is empty */ 5778 if (pml_idx == (PML_ENTITY_NUM - 1)) 5779 return; 5780 5781 /* PML index always points to next available PML buffer entity */ 5782 if (pml_idx >= PML_ENTITY_NUM) 5783 pml_idx = 0; 5784 else 5785 pml_idx++; 5786 5787 pml_buf = page_address(vmx->pml_pg); 5788 for (; pml_idx < PML_ENTITY_NUM; pml_idx++) { 5789 u64 gpa; 5790 5791 gpa = pml_buf[pml_idx]; 5792 WARN_ON(gpa & (PAGE_SIZE - 1)); 5793 kvm_vcpu_mark_page_dirty(vcpu, gpa >> PAGE_SHIFT); 5794 } 5795 5796 /* reset PML index */ 5797 vmcs_write16(GUEST_PML_INDEX, PML_ENTITY_NUM - 1); 5798 } 5799 5800 /* 5801 * Flush all vcpus' PML buffer and update logged GPAs to dirty_bitmap. 5802 * Called before reporting dirty_bitmap to userspace. 5803 */ 5804 static void kvm_flush_pml_buffers(struct kvm *kvm) 5805 { 5806 int i; 5807 struct kvm_vcpu *vcpu; 5808 /* 5809 * We only need to kick vcpu out of guest mode here, as PML buffer 5810 * is flushed at beginning of all VMEXITs, and it's obvious that only 5811 * vcpus running in guest are possible to have unflushed GPAs in PML 5812 * buffer. 5813 */ 5814 kvm_for_each_vcpu(i, vcpu, kvm) 5815 kvm_vcpu_kick(vcpu); 5816 } 5817 5818 static void vmx_dump_sel(char *name, uint32_t sel) 5819 { 5820 pr_err("%s sel=0x%04x, attr=0x%05x, limit=0x%08x, base=0x%016lx\n", 5821 name, vmcs_read16(sel), 5822 vmcs_read32(sel + GUEST_ES_AR_BYTES - GUEST_ES_SELECTOR), 5823 vmcs_read32(sel + GUEST_ES_LIMIT - GUEST_ES_SELECTOR), 5824 vmcs_readl(sel + GUEST_ES_BASE - GUEST_ES_SELECTOR)); 5825 } 5826 5827 static void vmx_dump_dtsel(char *name, uint32_t limit) 5828 { 5829 pr_err("%s limit=0x%08x, base=0x%016lx\n", 5830 name, vmcs_read32(limit), 5831 vmcs_readl(limit + GUEST_GDTR_BASE - GUEST_GDTR_LIMIT)); 5832 } 5833 5834 void dump_vmcs(void) 5835 { 5836 u32 vmentry_ctl, vmexit_ctl; 5837 u32 cpu_based_exec_ctrl, pin_based_exec_ctrl, secondary_exec_control; 5838 unsigned long cr4; 5839 u64 efer; 5840 5841 if (!dump_invalid_vmcs) { 5842 pr_warn_ratelimited("set kvm_intel.dump_invalid_vmcs=1 to dump internal KVM state.\n"); 5843 return; 5844 } 5845 5846 vmentry_ctl = vmcs_read32(VM_ENTRY_CONTROLS); 5847 vmexit_ctl = vmcs_read32(VM_EXIT_CONTROLS); 5848 cpu_based_exec_ctrl = vmcs_read32(CPU_BASED_VM_EXEC_CONTROL); 5849 pin_based_exec_ctrl = vmcs_read32(PIN_BASED_VM_EXEC_CONTROL); 5850 cr4 = vmcs_readl(GUEST_CR4); 5851 efer = vmcs_read64(GUEST_IA32_EFER); 5852 secondary_exec_control = 0; 5853 if (cpu_has_secondary_exec_ctrls()) 5854 secondary_exec_control = vmcs_read32(SECONDARY_VM_EXEC_CONTROL); 5855 5856 pr_err("*** Guest State ***\n"); 5857 pr_err("CR0: actual=0x%016lx, shadow=0x%016lx, gh_mask=%016lx\n", 5858 vmcs_readl(GUEST_CR0), vmcs_readl(CR0_READ_SHADOW), 5859 vmcs_readl(CR0_GUEST_HOST_MASK)); 5860 pr_err("CR4: actual=0x%016lx, shadow=0x%016lx, gh_mask=%016lx\n", 5861 cr4, vmcs_readl(CR4_READ_SHADOW), vmcs_readl(CR4_GUEST_HOST_MASK)); 5862 pr_err("CR3 = 0x%016lx\n", vmcs_readl(GUEST_CR3)); 5863 if ((secondary_exec_control & SECONDARY_EXEC_ENABLE_EPT) && 5864 (cr4 & X86_CR4_PAE) && !(efer & EFER_LMA)) 5865 { 5866 pr_err("PDPTR0 = 0x%016llx PDPTR1 = 0x%016llx\n", 5867 vmcs_read64(GUEST_PDPTR0), vmcs_read64(GUEST_PDPTR1)); 5868 pr_err("PDPTR2 = 0x%016llx PDPTR3 = 0x%016llx\n", 5869 vmcs_read64(GUEST_PDPTR2), vmcs_read64(GUEST_PDPTR3)); 5870 } 5871 pr_err("RSP = 0x%016lx RIP = 0x%016lx\n", 5872 vmcs_readl(GUEST_RSP), vmcs_readl(GUEST_RIP)); 5873 pr_err("RFLAGS=0x%08lx DR7 = 0x%016lx\n", 5874 vmcs_readl(GUEST_RFLAGS), vmcs_readl(GUEST_DR7)); 5875 pr_err("Sysenter RSP=%016lx CS:RIP=%04x:%016lx\n", 5876 vmcs_readl(GUEST_SYSENTER_ESP), 5877 vmcs_read32(GUEST_SYSENTER_CS), vmcs_readl(GUEST_SYSENTER_EIP)); 5878 vmx_dump_sel("CS: ", GUEST_CS_SELECTOR); 5879 vmx_dump_sel("DS: ", GUEST_DS_SELECTOR); 5880 vmx_dump_sel("SS: ", GUEST_SS_SELECTOR); 5881 vmx_dump_sel("ES: ", GUEST_ES_SELECTOR); 5882 vmx_dump_sel("FS: ", GUEST_FS_SELECTOR); 5883 vmx_dump_sel("GS: ", GUEST_GS_SELECTOR); 5884 vmx_dump_dtsel("GDTR:", GUEST_GDTR_LIMIT); 5885 vmx_dump_sel("LDTR:", GUEST_LDTR_SELECTOR); 5886 vmx_dump_dtsel("IDTR:", GUEST_IDTR_LIMIT); 5887 vmx_dump_sel("TR: ", GUEST_TR_SELECTOR); 5888 if ((vmexit_ctl & (VM_EXIT_SAVE_IA32_PAT | VM_EXIT_SAVE_IA32_EFER)) || 5889 (vmentry_ctl & (VM_ENTRY_LOAD_IA32_PAT | VM_ENTRY_LOAD_IA32_EFER))) 5890 pr_err("EFER = 0x%016llx PAT = 0x%016llx\n", 5891 efer, vmcs_read64(GUEST_IA32_PAT)); 5892 pr_err("DebugCtl = 0x%016llx DebugExceptions = 0x%016lx\n", 5893 vmcs_read64(GUEST_IA32_DEBUGCTL), 5894 vmcs_readl(GUEST_PENDING_DBG_EXCEPTIONS)); 5895 if (cpu_has_load_perf_global_ctrl() && 5896 vmentry_ctl & VM_ENTRY_LOAD_IA32_PERF_GLOBAL_CTRL) 5897 pr_err("PerfGlobCtl = 0x%016llx\n", 5898 vmcs_read64(GUEST_IA32_PERF_GLOBAL_CTRL)); 5899 if (vmentry_ctl & VM_ENTRY_LOAD_BNDCFGS) 5900 pr_err("BndCfgS = 0x%016llx\n", vmcs_read64(GUEST_BNDCFGS)); 5901 pr_err("Interruptibility = %08x ActivityState = %08x\n", 5902 vmcs_read32(GUEST_INTERRUPTIBILITY_INFO), 5903 vmcs_read32(GUEST_ACTIVITY_STATE)); 5904 if (secondary_exec_control & SECONDARY_EXEC_VIRTUAL_INTR_DELIVERY) 5905 pr_err("InterruptStatus = %04x\n", 5906 vmcs_read16(GUEST_INTR_STATUS)); 5907 5908 pr_err("*** Host State ***\n"); 5909 pr_err("RIP = 0x%016lx RSP = 0x%016lx\n", 5910 vmcs_readl(HOST_RIP), vmcs_readl(HOST_RSP)); 5911 pr_err("CS=%04x SS=%04x DS=%04x ES=%04x FS=%04x GS=%04x TR=%04x\n", 5912 vmcs_read16(HOST_CS_SELECTOR), vmcs_read16(HOST_SS_SELECTOR), 5913 vmcs_read16(HOST_DS_SELECTOR), vmcs_read16(HOST_ES_SELECTOR), 5914 vmcs_read16(HOST_FS_SELECTOR), vmcs_read16(HOST_GS_SELECTOR), 5915 vmcs_read16(HOST_TR_SELECTOR)); 5916 pr_err("FSBase=%016lx GSBase=%016lx TRBase=%016lx\n", 5917 vmcs_readl(HOST_FS_BASE), vmcs_readl(HOST_GS_BASE), 5918 vmcs_readl(HOST_TR_BASE)); 5919 pr_err("GDTBase=%016lx IDTBase=%016lx\n", 5920 vmcs_readl(HOST_GDTR_BASE), vmcs_readl(HOST_IDTR_BASE)); 5921 pr_err("CR0=%016lx CR3=%016lx CR4=%016lx\n", 5922 vmcs_readl(HOST_CR0), vmcs_readl(HOST_CR3), 5923 vmcs_readl(HOST_CR4)); 5924 pr_err("Sysenter RSP=%016lx CS:RIP=%04x:%016lx\n", 5925 vmcs_readl(HOST_IA32_SYSENTER_ESP), 5926 vmcs_read32(HOST_IA32_SYSENTER_CS), 5927 vmcs_readl(HOST_IA32_SYSENTER_EIP)); 5928 if (vmexit_ctl & (VM_EXIT_LOAD_IA32_PAT | VM_EXIT_LOAD_IA32_EFER)) 5929 pr_err("EFER = 0x%016llx PAT = 0x%016llx\n", 5930 vmcs_read64(HOST_IA32_EFER), 5931 vmcs_read64(HOST_IA32_PAT)); 5932 if (cpu_has_load_perf_global_ctrl() && 5933 vmexit_ctl & VM_EXIT_LOAD_IA32_PERF_GLOBAL_CTRL) 5934 pr_err("PerfGlobCtl = 0x%016llx\n", 5935 vmcs_read64(HOST_IA32_PERF_GLOBAL_CTRL)); 5936 5937 pr_err("*** Control State ***\n"); 5938 pr_err("PinBased=%08x CPUBased=%08x SecondaryExec=%08x\n", 5939 pin_based_exec_ctrl, cpu_based_exec_ctrl, secondary_exec_control); 5940 pr_err("EntryControls=%08x ExitControls=%08x\n", vmentry_ctl, vmexit_ctl); 5941 pr_err("ExceptionBitmap=%08x PFECmask=%08x PFECmatch=%08x\n", 5942 vmcs_read32(EXCEPTION_BITMAP), 5943 vmcs_read32(PAGE_FAULT_ERROR_CODE_MASK), 5944 vmcs_read32(PAGE_FAULT_ERROR_CODE_MATCH)); 5945 pr_err("VMEntry: intr_info=%08x errcode=%08x ilen=%08x\n", 5946 vmcs_read32(VM_ENTRY_INTR_INFO_FIELD), 5947 vmcs_read32(VM_ENTRY_EXCEPTION_ERROR_CODE), 5948 vmcs_read32(VM_ENTRY_INSTRUCTION_LEN)); 5949 pr_err("VMExit: intr_info=%08x errcode=%08x ilen=%08x\n", 5950 vmcs_read32(VM_EXIT_INTR_INFO), 5951 vmcs_read32(VM_EXIT_INTR_ERROR_CODE), 5952 vmcs_read32(VM_EXIT_INSTRUCTION_LEN)); 5953 pr_err(" reason=%08x qualification=%016lx\n", 5954 vmcs_read32(VM_EXIT_REASON), vmcs_readl(EXIT_QUALIFICATION)); 5955 pr_err("IDTVectoring: info=%08x errcode=%08x\n", 5956 vmcs_read32(IDT_VECTORING_INFO_FIELD), 5957 vmcs_read32(IDT_VECTORING_ERROR_CODE)); 5958 pr_err("TSC Offset = 0x%016llx\n", vmcs_read64(TSC_OFFSET)); 5959 if (secondary_exec_control & SECONDARY_EXEC_TSC_SCALING) 5960 pr_err("TSC Multiplier = 0x%016llx\n", 5961 vmcs_read64(TSC_MULTIPLIER)); 5962 if (cpu_based_exec_ctrl & CPU_BASED_TPR_SHADOW) { 5963 if (secondary_exec_control & SECONDARY_EXEC_VIRTUAL_INTR_DELIVERY) { 5964 u16 status = vmcs_read16(GUEST_INTR_STATUS); 5965 pr_err("SVI|RVI = %02x|%02x ", status >> 8, status & 0xff); 5966 } 5967 pr_cont("TPR Threshold = 0x%02x\n", vmcs_read32(TPR_THRESHOLD)); 5968 if (secondary_exec_control & SECONDARY_EXEC_VIRTUALIZE_APIC_ACCESSES) 5969 pr_err("APIC-access addr = 0x%016llx ", vmcs_read64(APIC_ACCESS_ADDR)); 5970 pr_cont("virt-APIC addr = 0x%016llx\n", vmcs_read64(VIRTUAL_APIC_PAGE_ADDR)); 5971 } 5972 if (pin_based_exec_ctrl & PIN_BASED_POSTED_INTR) 5973 pr_err("PostedIntrVec = 0x%02x\n", vmcs_read16(POSTED_INTR_NV)); 5974 if ((secondary_exec_control & SECONDARY_EXEC_ENABLE_EPT)) 5975 pr_err("EPT pointer = 0x%016llx\n", vmcs_read64(EPT_POINTER)); 5976 if (secondary_exec_control & SECONDARY_EXEC_PAUSE_LOOP_EXITING) 5977 pr_err("PLE Gap=%08x Window=%08x\n", 5978 vmcs_read32(PLE_GAP), vmcs_read32(PLE_WINDOW)); 5979 if (secondary_exec_control & SECONDARY_EXEC_ENABLE_VPID) 5980 pr_err("Virtual processor ID = 0x%04x\n", 5981 vmcs_read16(VIRTUAL_PROCESSOR_ID)); 5982 } 5983 5984 /* 5985 * The guest has exited. See if we can fix it or if we need userspace 5986 * assistance. 5987 */ 5988 static int vmx_handle_exit(struct kvm_vcpu *vcpu, fastpath_t exit_fastpath) 5989 { 5990 struct vcpu_vmx *vmx = to_vmx(vcpu); 5991 u32 exit_reason = vmx->exit_reason; 5992 u32 vectoring_info = vmx->idt_vectoring_info; 5993 5994 /* 5995 * Flush logged GPAs PML buffer, this will make dirty_bitmap more 5996 * updated. Another good is, in kvm_vm_ioctl_get_dirty_log, before 5997 * querying dirty_bitmap, we only need to kick all vcpus out of guest 5998 * mode as if vcpus is in root mode, the PML buffer must has been 5999 * flushed already. 6000 */ 6001 if (enable_pml) 6002 vmx_flush_pml_buffer(vcpu); 6003 6004 /* 6005 * We should never reach this point with a pending nested VM-Enter, and 6006 * more specifically emulation of L2 due to invalid guest state (see 6007 * below) should never happen as that means we incorrectly allowed a 6008 * nested VM-Enter with an invalid vmcs12. 6009 */ 6010 WARN_ON_ONCE(vmx->nested.nested_run_pending); 6011 6012 /* If guest state is invalid, start emulating */ 6013 if (vmx->emulation_required) 6014 return handle_invalid_guest_state(vcpu); 6015 6016 if (is_guest_mode(vcpu)) { 6017 /* 6018 * The host physical addresses of some pages of guest memory 6019 * are loaded into the vmcs02 (e.g. vmcs12's Virtual APIC 6020 * Page). The CPU may write to these pages via their host 6021 * physical address while L2 is running, bypassing any 6022 * address-translation-based dirty tracking (e.g. EPT write 6023 * protection). 6024 * 6025 * Mark them dirty on every exit from L2 to prevent them from 6026 * getting out of sync with dirty tracking. 6027 */ 6028 nested_mark_vmcs12_pages_dirty(vcpu); 6029 6030 if (nested_vmx_reflect_vmexit(vcpu)) 6031 return 1; 6032 } 6033 6034 if (exit_reason & VMX_EXIT_REASONS_FAILED_VMENTRY) { 6035 dump_vmcs(); 6036 vcpu->run->exit_reason = KVM_EXIT_FAIL_ENTRY; 6037 vcpu->run->fail_entry.hardware_entry_failure_reason 6038 = exit_reason; 6039 vcpu->run->fail_entry.cpu = vcpu->arch.last_vmentry_cpu; 6040 return 0; 6041 } 6042 6043 if (unlikely(vmx->fail)) { 6044 dump_vmcs(); 6045 vcpu->run->exit_reason = KVM_EXIT_FAIL_ENTRY; 6046 vcpu->run->fail_entry.hardware_entry_failure_reason 6047 = vmcs_read32(VM_INSTRUCTION_ERROR); 6048 vcpu->run->fail_entry.cpu = vcpu->arch.last_vmentry_cpu; 6049 return 0; 6050 } 6051 6052 /* 6053 * Note: 6054 * Do not try to fix EXIT_REASON_EPT_MISCONFIG if it caused by 6055 * delivery event since it indicates guest is accessing MMIO. 6056 * The vm-exit can be triggered again after return to guest that 6057 * will cause infinite loop. 6058 */ 6059 if ((vectoring_info & VECTORING_INFO_VALID_MASK) && 6060 (exit_reason != EXIT_REASON_EXCEPTION_NMI && 6061 exit_reason != EXIT_REASON_EPT_VIOLATION && 6062 exit_reason != EXIT_REASON_PML_FULL && 6063 exit_reason != EXIT_REASON_APIC_ACCESS && 6064 exit_reason != EXIT_REASON_TASK_SWITCH)) { 6065 vcpu->run->exit_reason = KVM_EXIT_INTERNAL_ERROR; 6066 vcpu->run->internal.suberror = KVM_INTERNAL_ERROR_DELIVERY_EV; 6067 vcpu->run->internal.ndata = 3; 6068 vcpu->run->internal.data[0] = vectoring_info; 6069 vcpu->run->internal.data[1] = exit_reason; 6070 vcpu->run->internal.data[2] = vcpu->arch.exit_qualification; 6071 if (exit_reason == EXIT_REASON_EPT_MISCONFIG) { 6072 vcpu->run->internal.ndata++; 6073 vcpu->run->internal.data[3] = 6074 vmcs_read64(GUEST_PHYSICAL_ADDRESS); 6075 } 6076 vcpu->run->internal.data[vcpu->run->internal.ndata++] = 6077 vcpu->arch.last_vmentry_cpu; 6078 return 0; 6079 } 6080 6081 if (unlikely(!enable_vnmi && 6082 vmx->loaded_vmcs->soft_vnmi_blocked)) { 6083 if (!vmx_interrupt_blocked(vcpu)) { 6084 vmx->loaded_vmcs->soft_vnmi_blocked = 0; 6085 } else if (vmx->loaded_vmcs->vnmi_blocked_time > 1000000000LL && 6086 vcpu->arch.nmi_pending) { 6087 /* 6088 * This CPU don't support us in finding the end of an 6089 * NMI-blocked window if the guest runs with IRQs 6090 * disabled. So we pull the trigger after 1 s of 6091 * futile waiting, but inform the user about this. 6092 */ 6093 printk(KERN_WARNING "%s: Breaking out of NMI-blocked " 6094 "state on VCPU %d after 1 s timeout\n", 6095 __func__, vcpu->vcpu_id); 6096 vmx->loaded_vmcs->soft_vnmi_blocked = 0; 6097 } 6098 } 6099 6100 if (exit_fastpath != EXIT_FASTPATH_NONE) 6101 return 1; 6102 6103 if (exit_reason >= kvm_vmx_max_exit_handlers) 6104 goto unexpected_vmexit; 6105 #ifdef CONFIG_RETPOLINE 6106 if (exit_reason == EXIT_REASON_MSR_WRITE) 6107 return kvm_emulate_wrmsr(vcpu); 6108 else if (exit_reason == EXIT_REASON_PREEMPTION_TIMER) 6109 return handle_preemption_timer(vcpu); 6110 else if (exit_reason == EXIT_REASON_INTERRUPT_WINDOW) 6111 return handle_interrupt_window(vcpu); 6112 else if (exit_reason == EXIT_REASON_EXTERNAL_INTERRUPT) 6113 return handle_external_interrupt(vcpu); 6114 else if (exit_reason == EXIT_REASON_HLT) 6115 return kvm_emulate_halt(vcpu); 6116 else if (exit_reason == EXIT_REASON_EPT_MISCONFIG) 6117 return handle_ept_misconfig(vcpu); 6118 #endif 6119 6120 exit_reason = array_index_nospec(exit_reason, 6121 kvm_vmx_max_exit_handlers); 6122 if (!kvm_vmx_exit_handlers[exit_reason]) 6123 goto unexpected_vmexit; 6124 6125 return kvm_vmx_exit_handlers[exit_reason](vcpu); 6126 6127 unexpected_vmexit: 6128 vcpu_unimpl(vcpu, "vmx: unexpected exit reason 0x%x\n", exit_reason); 6129 dump_vmcs(); 6130 vcpu->run->exit_reason = KVM_EXIT_INTERNAL_ERROR; 6131 vcpu->run->internal.suberror = 6132 KVM_INTERNAL_ERROR_UNEXPECTED_EXIT_REASON; 6133 vcpu->run->internal.ndata = 2; 6134 vcpu->run->internal.data[0] = exit_reason; 6135 vcpu->run->internal.data[1] = vcpu->arch.last_vmentry_cpu; 6136 return 0; 6137 } 6138 6139 /* 6140 * Software based L1D cache flush which is used when microcode providing 6141 * the cache control MSR is not loaded. 6142 * 6143 * The L1D cache is 32 KiB on Nehalem and later microarchitectures, but to 6144 * flush it is required to read in 64 KiB because the replacement algorithm 6145 * is not exactly LRU. This could be sized at runtime via topology 6146 * information but as all relevant affected CPUs have 32KiB L1D cache size 6147 * there is no point in doing so. 6148 */ 6149 static noinstr void vmx_l1d_flush(struct kvm_vcpu *vcpu) 6150 { 6151 int size = PAGE_SIZE << L1D_CACHE_ORDER; 6152 6153 /* 6154 * This code is only executed when the the flush mode is 'cond' or 6155 * 'always' 6156 */ 6157 if (static_branch_likely(&vmx_l1d_flush_cond)) { 6158 bool flush_l1d; 6159 6160 /* 6161 * Clear the per-vcpu flush bit, it gets set again 6162 * either from vcpu_run() or from one of the unsafe 6163 * VMEXIT handlers. 6164 */ 6165 flush_l1d = vcpu->arch.l1tf_flush_l1d; 6166 vcpu->arch.l1tf_flush_l1d = false; 6167 6168 /* 6169 * Clear the per-cpu flush bit, it gets set again from 6170 * the interrupt handlers. 6171 */ 6172 flush_l1d |= kvm_get_cpu_l1tf_flush_l1d(); 6173 kvm_clear_cpu_l1tf_flush_l1d(); 6174 6175 if (!flush_l1d) 6176 return; 6177 } 6178 6179 vcpu->stat.l1d_flush++; 6180 6181 if (static_cpu_has(X86_FEATURE_FLUSH_L1D)) { 6182 native_wrmsrl(MSR_IA32_FLUSH_CMD, L1D_FLUSH); 6183 return; 6184 } 6185 6186 asm volatile( 6187 /* First ensure the pages are in the TLB */ 6188 "xorl %%eax, %%eax\n" 6189 ".Lpopulate_tlb:\n\t" 6190 "movzbl (%[flush_pages], %%" _ASM_AX "), %%ecx\n\t" 6191 "addl $4096, %%eax\n\t" 6192 "cmpl %%eax, %[size]\n\t" 6193 "jne .Lpopulate_tlb\n\t" 6194 "xorl %%eax, %%eax\n\t" 6195 "cpuid\n\t" 6196 /* Now fill the cache */ 6197 "xorl %%eax, %%eax\n" 6198 ".Lfill_cache:\n" 6199 "movzbl (%[flush_pages], %%" _ASM_AX "), %%ecx\n\t" 6200 "addl $64, %%eax\n\t" 6201 "cmpl %%eax, %[size]\n\t" 6202 "jne .Lfill_cache\n\t" 6203 "lfence\n" 6204 :: [flush_pages] "r" (vmx_l1d_flush_pages), 6205 [size] "r" (size) 6206 : "eax", "ebx", "ecx", "edx"); 6207 } 6208 6209 static void update_cr8_intercept(struct kvm_vcpu *vcpu, int tpr, int irr) 6210 { 6211 struct vmcs12 *vmcs12 = get_vmcs12(vcpu); 6212 int tpr_threshold; 6213 6214 if (is_guest_mode(vcpu) && 6215 nested_cpu_has(vmcs12, CPU_BASED_TPR_SHADOW)) 6216 return; 6217 6218 tpr_threshold = (irr == -1 || tpr < irr) ? 0 : irr; 6219 if (is_guest_mode(vcpu)) 6220 to_vmx(vcpu)->nested.l1_tpr_threshold = tpr_threshold; 6221 else 6222 vmcs_write32(TPR_THRESHOLD, tpr_threshold); 6223 } 6224 6225 void vmx_set_virtual_apic_mode(struct kvm_vcpu *vcpu) 6226 { 6227 struct vcpu_vmx *vmx = to_vmx(vcpu); 6228 u32 sec_exec_control; 6229 6230 if (!lapic_in_kernel(vcpu)) 6231 return; 6232 6233 if (!flexpriority_enabled && 6234 !cpu_has_vmx_virtualize_x2apic_mode()) 6235 return; 6236 6237 /* Postpone execution until vmcs01 is the current VMCS. */ 6238 if (is_guest_mode(vcpu)) { 6239 vmx->nested.change_vmcs01_virtual_apic_mode = true; 6240 return; 6241 } 6242 6243 sec_exec_control = secondary_exec_controls_get(vmx); 6244 sec_exec_control &= ~(SECONDARY_EXEC_VIRTUALIZE_APIC_ACCESSES | 6245 SECONDARY_EXEC_VIRTUALIZE_X2APIC_MODE); 6246 6247 switch (kvm_get_apic_mode(vcpu)) { 6248 case LAPIC_MODE_INVALID: 6249 WARN_ONCE(true, "Invalid local APIC state"); 6250 case LAPIC_MODE_DISABLED: 6251 break; 6252 case LAPIC_MODE_XAPIC: 6253 if (flexpriority_enabled) { 6254 sec_exec_control |= 6255 SECONDARY_EXEC_VIRTUALIZE_APIC_ACCESSES; 6256 kvm_make_request(KVM_REQ_APIC_PAGE_RELOAD, vcpu); 6257 6258 /* 6259 * Flush the TLB, reloading the APIC access page will 6260 * only do so if its physical address has changed, but 6261 * the guest may have inserted a non-APIC mapping into 6262 * the TLB while the APIC access page was disabled. 6263 */ 6264 kvm_make_request(KVM_REQ_TLB_FLUSH_CURRENT, vcpu); 6265 } 6266 break; 6267 case LAPIC_MODE_X2APIC: 6268 if (cpu_has_vmx_virtualize_x2apic_mode()) 6269 sec_exec_control |= 6270 SECONDARY_EXEC_VIRTUALIZE_X2APIC_MODE; 6271 break; 6272 } 6273 secondary_exec_controls_set(vmx, sec_exec_control); 6274 6275 vmx_update_msr_bitmap(vcpu); 6276 } 6277 6278 static void vmx_set_apic_access_page_addr(struct kvm_vcpu *vcpu) 6279 { 6280 struct page *page; 6281 6282 /* Defer reload until vmcs01 is the current VMCS. */ 6283 if (is_guest_mode(vcpu)) { 6284 to_vmx(vcpu)->nested.reload_vmcs01_apic_access_page = true; 6285 return; 6286 } 6287 6288 if (!(secondary_exec_controls_get(to_vmx(vcpu)) & 6289 SECONDARY_EXEC_VIRTUALIZE_APIC_ACCESSES)) 6290 return; 6291 6292 page = gfn_to_page(vcpu->kvm, APIC_DEFAULT_PHYS_BASE >> PAGE_SHIFT); 6293 if (is_error_page(page)) 6294 return; 6295 6296 vmcs_write64(APIC_ACCESS_ADDR, page_to_phys(page)); 6297 vmx_flush_tlb_current(vcpu); 6298 6299 /* 6300 * Do not pin apic access page in memory, the MMU notifier 6301 * will call us again if it is migrated or swapped out. 6302 */ 6303 put_page(page); 6304 } 6305 6306 static void vmx_hwapic_isr_update(struct kvm_vcpu *vcpu, int max_isr) 6307 { 6308 u16 status; 6309 u8 old; 6310 6311 if (max_isr == -1) 6312 max_isr = 0; 6313 6314 status = vmcs_read16(GUEST_INTR_STATUS); 6315 old = status >> 8; 6316 if (max_isr != old) { 6317 status &= 0xff; 6318 status |= max_isr << 8; 6319 vmcs_write16(GUEST_INTR_STATUS, status); 6320 } 6321 } 6322 6323 static void vmx_set_rvi(int vector) 6324 { 6325 u16 status; 6326 u8 old; 6327 6328 if (vector == -1) 6329 vector = 0; 6330 6331 status = vmcs_read16(GUEST_INTR_STATUS); 6332 old = (u8)status & 0xff; 6333 if ((u8)vector != old) { 6334 status &= ~0xff; 6335 status |= (u8)vector; 6336 vmcs_write16(GUEST_INTR_STATUS, status); 6337 } 6338 } 6339 6340 static void vmx_hwapic_irr_update(struct kvm_vcpu *vcpu, int max_irr) 6341 { 6342 /* 6343 * When running L2, updating RVI is only relevant when 6344 * vmcs12 virtual-interrupt-delivery enabled. 6345 * However, it can be enabled only when L1 also 6346 * intercepts external-interrupts and in that case 6347 * we should not update vmcs02 RVI but instead intercept 6348 * interrupt. Therefore, do nothing when running L2. 6349 */ 6350 if (!is_guest_mode(vcpu)) 6351 vmx_set_rvi(max_irr); 6352 } 6353 6354 static int vmx_sync_pir_to_irr(struct kvm_vcpu *vcpu) 6355 { 6356 struct vcpu_vmx *vmx = to_vmx(vcpu); 6357 int max_irr; 6358 bool max_irr_updated; 6359 6360 WARN_ON(!vcpu->arch.apicv_active); 6361 if (pi_test_on(&vmx->pi_desc)) { 6362 pi_clear_on(&vmx->pi_desc); 6363 /* 6364 * IOMMU can write to PID.ON, so the barrier matters even on UP. 6365 * But on x86 this is just a compiler barrier anyway. 6366 */ 6367 smp_mb__after_atomic(); 6368 max_irr_updated = 6369 kvm_apic_update_irr(vcpu, vmx->pi_desc.pir, &max_irr); 6370 6371 /* 6372 * If we are running L2 and L1 has a new pending interrupt 6373 * which can be injected, we should re-evaluate 6374 * what should be done with this new L1 interrupt. 6375 * If L1 intercepts external-interrupts, we should 6376 * exit from L2 to L1. Otherwise, interrupt should be 6377 * delivered directly to L2. 6378 */ 6379 if (is_guest_mode(vcpu) && max_irr_updated) { 6380 if (nested_exit_on_intr(vcpu)) 6381 kvm_vcpu_exiting_guest_mode(vcpu); 6382 else 6383 kvm_make_request(KVM_REQ_EVENT, vcpu); 6384 } 6385 } else { 6386 max_irr = kvm_lapic_find_highest_irr(vcpu); 6387 } 6388 vmx_hwapic_irr_update(vcpu, max_irr); 6389 return max_irr; 6390 } 6391 6392 static bool vmx_dy_apicv_has_pending_interrupt(struct kvm_vcpu *vcpu) 6393 { 6394 struct pi_desc *pi_desc = vcpu_to_pi_desc(vcpu); 6395 6396 return pi_test_on(pi_desc) || 6397 (pi_test_sn(pi_desc) && !pi_is_pir_empty(pi_desc)); 6398 } 6399 6400 static void vmx_load_eoi_exitmap(struct kvm_vcpu *vcpu, u64 *eoi_exit_bitmap) 6401 { 6402 if (!kvm_vcpu_apicv_active(vcpu)) 6403 return; 6404 6405 vmcs_write64(EOI_EXIT_BITMAP0, eoi_exit_bitmap[0]); 6406 vmcs_write64(EOI_EXIT_BITMAP1, eoi_exit_bitmap[1]); 6407 vmcs_write64(EOI_EXIT_BITMAP2, eoi_exit_bitmap[2]); 6408 vmcs_write64(EOI_EXIT_BITMAP3, eoi_exit_bitmap[3]); 6409 } 6410 6411 static void vmx_apicv_post_state_restore(struct kvm_vcpu *vcpu) 6412 { 6413 struct vcpu_vmx *vmx = to_vmx(vcpu); 6414 6415 pi_clear_on(&vmx->pi_desc); 6416 memset(vmx->pi_desc.pir, 0, sizeof(vmx->pi_desc.pir)); 6417 } 6418 6419 static void handle_exception_nmi_irqoff(struct vcpu_vmx *vmx) 6420 { 6421 u32 intr_info = vmx_get_intr_info(&vmx->vcpu); 6422 6423 /* if exit due to PF check for async PF */ 6424 if (is_page_fault(intr_info)) { 6425 vmx->vcpu.arch.apf.host_apf_flags = kvm_read_and_reset_apf_flags(); 6426 /* Handle machine checks before interrupts are enabled */ 6427 } else if (is_machine_check(intr_info)) { 6428 kvm_machine_check(); 6429 /* We need to handle NMIs before interrupts are enabled */ 6430 } else if (is_nmi(intr_info)) { 6431 kvm_before_interrupt(&vmx->vcpu); 6432 asm("int $2"); 6433 kvm_after_interrupt(&vmx->vcpu); 6434 } 6435 } 6436 6437 static void handle_external_interrupt_irqoff(struct kvm_vcpu *vcpu) 6438 { 6439 unsigned int vector; 6440 unsigned long entry; 6441 #ifdef CONFIG_X86_64 6442 unsigned long tmp; 6443 #endif 6444 gate_desc *desc; 6445 u32 intr_info = vmx_get_intr_info(vcpu); 6446 6447 if (WARN_ONCE(!is_external_intr(intr_info), 6448 "KVM: unexpected VM-Exit interrupt info: 0x%x", intr_info)) 6449 return; 6450 6451 vector = intr_info & INTR_INFO_VECTOR_MASK; 6452 desc = (gate_desc *)host_idt_base + vector; 6453 entry = gate_offset(desc); 6454 6455 kvm_before_interrupt(vcpu); 6456 6457 asm volatile( 6458 #ifdef CONFIG_X86_64 6459 "mov %%rsp, %[sp]\n\t" 6460 "and $-16, %%rsp\n\t" 6461 "push %[ss]\n\t" 6462 "push %[sp]\n\t" 6463 #endif 6464 "pushf\n\t" 6465 "push %[cs]\n\t" 6466 CALL_NOSPEC 6467 : 6468 #ifdef CONFIG_X86_64 6469 [sp]"=&r"(tmp), 6470 #endif 6471 ASM_CALL_CONSTRAINT 6472 : 6473 [thunk_target]"r"(entry), 6474 #ifdef CONFIG_X86_64 6475 [ss]"i"(__KERNEL_DS), 6476 #endif 6477 [cs]"i"(__KERNEL_CS) 6478 ); 6479 6480 kvm_after_interrupt(vcpu); 6481 } 6482 STACK_FRAME_NON_STANDARD(handle_external_interrupt_irqoff); 6483 6484 static void vmx_handle_exit_irqoff(struct kvm_vcpu *vcpu) 6485 { 6486 struct vcpu_vmx *vmx = to_vmx(vcpu); 6487 6488 if (vmx->exit_reason == EXIT_REASON_EXTERNAL_INTERRUPT) 6489 handle_external_interrupt_irqoff(vcpu); 6490 else if (vmx->exit_reason == EXIT_REASON_EXCEPTION_NMI) 6491 handle_exception_nmi_irqoff(vmx); 6492 } 6493 6494 static bool vmx_has_emulated_msr(u32 index) 6495 { 6496 switch (index) { 6497 case MSR_IA32_SMBASE: 6498 /* 6499 * We cannot do SMM unless we can run the guest in big 6500 * real mode. 6501 */ 6502 return enable_unrestricted_guest || emulate_invalid_guest_state; 6503 case MSR_IA32_VMX_BASIC ... MSR_IA32_VMX_VMFUNC: 6504 return nested; 6505 case MSR_AMD64_VIRT_SPEC_CTRL: 6506 /* This is AMD only. */ 6507 return false; 6508 default: 6509 return true; 6510 } 6511 } 6512 6513 static void vmx_recover_nmi_blocking(struct vcpu_vmx *vmx) 6514 { 6515 u32 exit_intr_info; 6516 bool unblock_nmi; 6517 u8 vector; 6518 bool idtv_info_valid; 6519 6520 idtv_info_valid = vmx->idt_vectoring_info & VECTORING_INFO_VALID_MASK; 6521 6522 if (enable_vnmi) { 6523 if (vmx->loaded_vmcs->nmi_known_unmasked) 6524 return; 6525 6526 exit_intr_info = vmx_get_intr_info(&vmx->vcpu); 6527 unblock_nmi = (exit_intr_info & INTR_INFO_UNBLOCK_NMI) != 0; 6528 vector = exit_intr_info & INTR_INFO_VECTOR_MASK; 6529 /* 6530 * SDM 3: 27.7.1.2 (September 2008) 6531 * Re-set bit "block by NMI" before VM entry if vmexit caused by 6532 * a guest IRET fault. 6533 * SDM 3: 23.2.2 (September 2008) 6534 * Bit 12 is undefined in any of the following cases: 6535 * If the VM exit sets the valid bit in the IDT-vectoring 6536 * information field. 6537 * If the VM exit is due to a double fault. 6538 */ 6539 if ((exit_intr_info & INTR_INFO_VALID_MASK) && unblock_nmi && 6540 vector != DF_VECTOR && !idtv_info_valid) 6541 vmcs_set_bits(GUEST_INTERRUPTIBILITY_INFO, 6542 GUEST_INTR_STATE_NMI); 6543 else 6544 vmx->loaded_vmcs->nmi_known_unmasked = 6545 !(vmcs_read32(GUEST_INTERRUPTIBILITY_INFO) 6546 & GUEST_INTR_STATE_NMI); 6547 } else if (unlikely(vmx->loaded_vmcs->soft_vnmi_blocked)) 6548 vmx->loaded_vmcs->vnmi_blocked_time += 6549 ktime_to_ns(ktime_sub(ktime_get(), 6550 vmx->loaded_vmcs->entry_time)); 6551 } 6552 6553 static void __vmx_complete_interrupts(struct kvm_vcpu *vcpu, 6554 u32 idt_vectoring_info, 6555 int instr_len_field, 6556 int error_code_field) 6557 { 6558 u8 vector; 6559 int type; 6560 bool idtv_info_valid; 6561 6562 idtv_info_valid = idt_vectoring_info & VECTORING_INFO_VALID_MASK; 6563 6564 vcpu->arch.nmi_injected = false; 6565 kvm_clear_exception_queue(vcpu); 6566 kvm_clear_interrupt_queue(vcpu); 6567 6568 if (!idtv_info_valid) 6569 return; 6570 6571 kvm_make_request(KVM_REQ_EVENT, vcpu); 6572 6573 vector = idt_vectoring_info & VECTORING_INFO_VECTOR_MASK; 6574 type = idt_vectoring_info & VECTORING_INFO_TYPE_MASK; 6575 6576 switch (type) { 6577 case INTR_TYPE_NMI_INTR: 6578 vcpu->arch.nmi_injected = true; 6579 /* 6580 * SDM 3: 27.7.1.2 (September 2008) 6581 * Clear bit "block by NMI" before VM entry if a NMI 6582 * delivery faulted. 6583 */ 6584 vmx_set_nmi_mask(vcpu, false); 6585 break; 6586 case INTR_TYPE_SOFT_EXCEPTION: 6587 vcpu->arch.event_exit_inst_len = vmcs_read32(instr_len_field); 6588 fallthrough; 6589 case INTR_TYPE_HARD_EXCEPTION: 6590 if (idt_vectoring_info & VECTORING_INFO_DELIVER_CODE_MASK) { 6591 u32 err = vmcs_read32(error_code_field); 6592 kvm_requeue_exception_e(vcpu, vector, err); 6593 } else 6594 kvm_requeue_exception(vcpu, vector); 6595 break; 6596 case INTR_TYPE_SOFT_INTR: 6597 vcpu->arch.event_exit_inst_len = vmcs_read32(instr_len_field); 6598 fallthrough; 6599 case INTR_TYPE_EXT_INTR: 6600 kvm_queue_interrupt(vcpu, vector, type == INTR_TYPE_SOFT_INTR); 6601 break; 6602 default: 6603 break; 6604 } 6605 } 6606 6607 static void vmx_complete_interrupts(struct vcpu_vmx *vmx) 6608 { 6609 __vmx_complete_interrupts(&vmx->vcpu, vmx->idt_vectoring_info, 6610 VM_EXIT_INSTRUCTION_LEN, 6611 IDT_VECTORING_ERROR_CODE); 6612 } 6613 6614 static void vmx_cancel_injection(struct kvm_vcpu *vcpu) 6615 { 6616 __vmx_complete_interrupts(vcpu, 6617 vmcs_read32(VM_ENTRY_INTR_INFO_FIELD), 6618 VM_ENTRY_INSTRUCTION_LEN, 6619 VM_ENTRY_EXCEPTION_ERROR_CODE); 6620 6621 vmcs_write32(VM_ENTRY_INTR_INFO_FIELD, 0); 6622 } 6623 6624 static void atomic_switch_perf_msrs(struct vcpu_vmx *vmx) 6625 { 6626 int i, nr_msrs; 6627 struct perf_guest_switch_msr *msrs; 6628 6629 msrs = perf_guest_get_msrs(&nr_msrs); 6630 6631 if (!msrs) 6632 return; 6633 6634 for (i = 0; i < nr_msrs; i++) 6635 if (msrs[i].host == msrs[i].guest) 6636 clear_atomic_switch_msr(vmx, msrs[i].msr); 6637 else 6638 add_atomic_switch_msr(vmx, msrs[i].msr, msrs[i].guest, 6639 msrs[i].host, false); 6640 } 6641 6642 static void vmx_update_hv_timer(struct kvm_vcpu *vcpu) 6643 { 6644 struct vcpu_vmx *vmx = to_vmx(vcpu); 6645 u64 tscl; 6646 u32 delta_tsc; 6647 6648 if (vmx->req_immediate_exit) { 6649 vmcs_write32(VMX_PREEMPTION_TIMER_VALUE, 0); 6650 vmx->loaded_vmcs->hv_timer_soft_disabled = false; 6651 } else if (vmx->hv_deadline_tsc != -1) { 6652 tscl = rdtsc(); 6653 if (vmx->hv_deadline_tsc > tscl) 6654 /* set_hv_timer ensures the delta fits in 32-bits */ 6655 delta_tsc = (u32)((vmx->hv_deadline_tsc - tscl) >> 6656 cpu_preemption_timer_multi); 6657 else 6658 delta_tsc = 0; 6659 6660 vmcs_write32(VMX_PREEMPTION_TIMER_VALUE, delta_tsc); 6661 vmx->loaded_vmcs->hv_timer_soft_disabled = false; 6662 } else if (!vmx->loaded_vmcs->hv_timer_soft_disabled) { 6663 vmcs_write32(VMX_PREEMPTION_TIMER_VALUE, -1); 6664 vmx->loaded_vmcs->hv_timer_soft_disabled = true; 6665 } 6666 } 6667 6668 void noinstr vmx_update_host_rsp(struct vcpu_vmx *vmx, unsigned long host_rsp) 6669 { 6670 if (unlikely(host_rsp != vmx->loaded_vmcs->host_state.rsp)) { 6671 vmx->loaded_vmcs->host_state.rsp = host_rsp; 6672 vmcs_writel(HOST_RSP, host_rsp); 6673 } 6674 } 6675 6676 static fastpath_t vmx_exit_handlers_fastpath(struct kvm_vcpu *vcpu) 6677 { 6678 switch (to_vmx(vcpu)->exit_reason) { 6679 case EXIT_REASON_MSR_WRITE: 6680 return handle_fastpath_set_msr_irqoff(vcpu); 6681 case EXIT_REASON_PREEMPTION_TIMER: 6682 return handle_fastpath_preemption_timer(vcpu); 6683 default: 6684 return EXIT_FASTPATH_NONE; 6685 } 6686 } 6687 6688 bool __vmx_vcpu_run(struct vcpu_vmx *vmx, unsigned long *regs, bool launched); 6689 6690 static noinstr void vmx_vcpu_enter_exit(struct kvm_vcpu *vcpu, 6691 struct vcpu_vmx *vmx) 6692 { 6693 /* 6694 * VMENTER enables interrupts (host state), but the kernel state is 6695 * interrupts disabled when this is invoked. Also tell RCU about 6696 * it. This is the same logic as for exit_to_user_mode(). 6697 * 6698 * This ensures that e.g. latency analysis on the host observes 6699 * guest mode as interrupt enabled. 6700 * 6701 * guest_enter_irqoff() informs context tracking about the 6702 * transition to guest mode and if enabled adjusts RCU state 6703 * accordingly. 6704 */ 6705 instrumentation_begin(); 6706 trace_hardirqs_on_prepare(); 6707 lockdep_hardirqs_on_prepare(CALLER_ADDR0); 6708 instrumentation_end(); 6709 6710 guest_enter_irqoff(); 6711 lockdep_hardirqs_on(CALLER_ADDR0); 6712 6713 /* L1D Flush includes CPU buffer clear to mitigate MDS */ 6714 if (static_branch_unlikely(&vmx_l1d_should_flush)) 6715 vmx_l1d_flush(vcpu); 6716 else if (static_branch_unlikely(&mds_user_clear)) 6717 mds_clear_cpu_buffers(); 6718 6719 if (vcpu->arch.cr2 != native_read_cr2()) 6720 native_write_cr2(vcpu->arch.cr2); 6721 6722 vmx->fail = __vmx_vcpu_run(vmx, (unsigned long *)&vcpu->arch.regs, 6723 vmx->loaded_vmcs->launched); 6724 6725 vcpu->arch.cr2 = native_read_cr2(); 6726 6727 /* 6728 * VMEXIT disables interrupts (host state), but tracing and lockdep 6729 * have them in state 'on' as recorded before entering guest mode. 6730 * Same as enter_from_user_mode(). 6731 * 6732 * guest_exit_irqoff() restores host context and reinstates RCU if 6733 * enabled and required. 6734 * 6735 * This needs to be done before the below as native_read_msr() 6736 * contains a tracepoint and x86_spec_ctrl_restore_host() calls 6737 * into world and some more. 6738 */ 6739 lockdep_hardirqs_off(CALLER_ADDR0); 6740 guest_exit_irqoff(); 6741 6742 instrumentation_begin(); 6743 trace_hardirqs_off_finish(); 6744 instrumentation_end(); 6745 } 6746 6747 static fastpath_t vmx_vcpu_run(struct kvm_vcpu *vcpu) 6748 { 6749 fastpath_t exit_fastpath; 6750 struct vcpu_vmx *vmx = to_vmx(vcpu); 6751 unsigned long cr3, cr4; 6752 6753 reenter_guest: 6754 /* Record the guest's net vcpu time for enforced NMI injections. */ 6755 if (unlikely(!enable_vnmi && 6756 vmx->loaded_vmcs->soft_vnmi_blocked)) 6757 vmx->loaded_vmcs->entry_time = ktime_get(); 6758 6759 /* Don't enter VMX if guest state is invalid, let the exit handler 6760 start emulation until we arrive back to a valid state */ 6761 if (vmx->emulation_required) 6762 return EXIT_FASTPATH_NONE; 6763 6764 if (vmx->ple_window_dirty) { 6765 vmx->ple_window_dirty = false; 6766 vmcs_write32(PLE_WINDOW, vmx->ple_window); 6767 } 6768 6769 /* 6770 * We did this in prepare_switch_to_guest, because it needs to 6771 * be within srcu_read_lock. 6772 */ 6773 WARN_ON_ONCE(vmx->nested.need_vmcs12_to_shadow_sync); 6774 6775 if (kvm_register_is_dirty(vcpu, VCPU_REGS_RSP)) 6776 vmcs_writel(GUEST_RSP, vcpu->arch.regs[VCPU_REGS_RSP]); 6777 if (kvm_register_is_dirty(vcpu, VCPU_REGS_RIP)) 6778 vmcs_writel(GUEST_RIP, vcpu->arch.regs[VCPU_REGS_RIP]); 6779 6780 cr3 = __get_current_cr3_fast(); 6781 if (unlikely(cr3 != vmx->loaded_vmcs->host_state.cr3)) { 6782 vmcs_writel(HOST_CR3, cr3); 6783 vmx->loaded_vmcs->host_state.cr3 = cr3; 6784 } 6785 6786 cr4 = cr4_read_shadow(); 6787 if (unlikely(cr4 != vmx->loaded_vmcs->host_state.cr4)) { 6788 vmcs_writel(HOST_CR4, cr4); 6789 vmx->loaded_vmcs->host_state.cr4 = cr4; 6790 } 6791 6792 /* When single-stepping over STI and MOV SS, we must clear the 6793 * corresponding interruptibility bits in the guest state. Otherwise 6794 * vmentry fails as it then expects bit 14 (BS) in pending debug 6795 * exceptions being set, but that's not correct for the guest debugging 6796 * case. */ 6797 if (vcpu->guest_debug & KVM_GUESTDBG_SINGLESTEP) 6798 vmx_set_interrupt_shadow(vcpu, 0); 6799 6800 kvm_load_guest_xsave_state(vcpu); 6801 6802 pt_guest_enter(vmx); 6803 6804 atomic_switch_perf_msrs(vmx); 6805 6806 if (enable_preemption_timer) 6807 vmx_update_hv_timer(vcpu); 6808 6809 if (lapic_in_kernel(vcpu) && 6810 vcpu->arch.apic->lapic_timer.timer_advance_ns) 6811 kvm_wait_lapic_expire(vcpu); 6812 6813 /* 6814 * If this vCPU has touched SPEC_CTRL, restore the guest's value if 6815 * it's non-zero. Since vmentry is serialising on affected CPUs, there 6816 * is no need to worry about the conditional branch over the wrmsr 6817 * being speculatively taken. 6818 */ 6819 x86_spec_ctrl_set_guest(vmx->spec_ctrl, 0); 6820 6821 /* The actual VMENTER/EXIT is in the .noinstr.text section. */ 6822 vmx_vcpu_enter_exit(vcpu, vmx); 6823 6824 /* 6825 * We do not use IBRS in the kernel. If this vCPU has used the 6826 * SPEC_CTRL MSR it may have left it on; save the value and 6827 * turn it off. This is much more efficient than blindly adding 6828 * it to the atomic save/restore list. Especially as the former 6829 * (Saving guest MSRs on vmexit) doesn't even exist in KVM. 6830 * 6831 * For non-nested case: 6832 * If the L01 MSR bitmap does not intercept the MSR, then we need to 6833 * save it. 6834 * 6835 * For nested case: 6836 * If the L02 MSR bitmap does not intercept the MSR, then we need to 6837 * save it. 6838 */ 6839 if (unlikely(!msr_write_intercepted(vcpu, MSR_IA32_SPEC_CTRL))) 6840 vmx->spec_ctrl = native_read_msr(MSR_IA32_SPEC_CTRL); 6841 6842 x86_spec_ctrl_restore_host(vmx->spec_ctrl, 0); 6843 6844 /* All fields are clean at this point */ 6845 if (static_branch_unlikely(&enable_evmcs)) 6846 current_evmcs->hv_clean_fields |= 6847 HV_VMX_ENLIGHTENED_CLEAN_FIELD_ALL; 6848 6849 if (static_branch_unlikely(&enable_evmcs)) 6850 current_evmcs->hv_vp_id = vcpu->arch.hyperv.vp_index; 6851 6852 /* MSR_IA32_DEBUGCTLMSR is zeroed on vmexit. Restore it if needed */ 6853 if (vmx->host_debugctlmsr) 6854 update_debugctlmsr(vmx->host_debugctlmsr); 6855 6856 #ifndef CONFIG_X86_64 6857 /* 6858 * The sysexit path does not restore ds/es, so we must set them to 6859 * a reasonable value ourselves. 6860 * 6861 * We can't defer this to vmx_prepare_switch_to_host() since that 6862 * function may be executed in interrupt context, which saves and 6863 * restore segments around it, nullifying its effect. 6864 */ 6865 loadsegment(ds, __USER_DS); 6866 loadsegment(es, __USER_DS); 6867 #endif 6868 6869 vmx_register_cache_reset(vcpu); 6870 6871 pt_guest_exit(vmx); 6872 6873 kvm_load_host_xsave_state(vcpu); 6874 6875 vmx->nested.nested_run_pending = 0; 6876 vmx->idt_vectoring_info = 0; 6877 6878 if (unlikely(vmx->fail)) { 6879 vmx->exit_reason = 0xdead; 6880 return EXIT_FASTPATH_NONE; 6881 } 6882 6883 vmx->exit_reason = vmcs_read32(VM_EXIT_REASON); 6884 if (unlikely((u16)vmx->exit_reason == EXIT_REASON_MCE_DURING_VMENTRY)) 6885 kvm_machine_check(); 6886 6887 trace_kvm_exit(vmx->exit_reason, vcpu, KVM_ISA_VMX); 6888 6889 if (unlikely(vmx->exit_reason & VMX_EXIT_REASONS_FAILED_VMENTRY)) 6890 return EXIT_FASTPATH_NONE; 6891 6892 vmx->loaded_vmcs->launched = 1; 6893 vmx->idt_vectoring_info = vmcs_read32(IDT_VECTORING_INFO_FIELD); 6894 6895 vmx_recover_nmi_blocking(vmx); 6896 vmx_complete_interrupts(vmx); 6897 6898 if (is_guest_mode(vcpu)) 6899 return EXIT_FASTPATH_NONE; 6900 6901 exit_fastpath = vmx_exit_handlers_fastpath(vcpu); 6902 if (exit_fastpath == EXIT_FASTPATH_REENTER_GUEST) { 6903 if (!kvm_vcpu_exit_request(vcpu)) { 6904 /* 6905 * FIXME: this goto should be a loop in vcpu_enter_guest, 6906 * but it would incur the cost of a retpoline for now. 6907 * Revisit once static calls are available. 6908 */ 6909 if (vcpu->arch.apicv_active) 6910 vmx_sync_pir_to_irr(vcpu); 6911 goto reenter_guest; 6912 } 6913 exit_fastpath = EXIT_FASTPATH_EXIT_HANDLED; 6914 } 6915 6916 return exit_fastpath; 6917 } 6918 6919 static void vmx_free_vcpu(struct kvm_vcpu *vcpu) 6920 { 6921 struct vcpu_vmx *vmx = to_vmx(vcpu); 6922 6923 if (enable_pml) 6924 vmx_destroy_pml_buffer(vmx); 6925 free_vpid(vmx->vpid); 6926 nested_vmx_free_vcpu(vcpu); 6927 free_loaded_vmcs(vmx->loaded_vmcs); 6928 } 6929 6930 static int vmx_create_vcpu(struct kvm_vcpu *vcpu) 6931 { 6932 struct vcpu_vmx *vmx; 6933 unsigned long *msr_bitmap; 6934 int i, cpu, err; 6935 6936 BUILD_BUG_ON(offsetof(struct vcpu_vmx, vcpu) != 0); 6937 vmx = to_vmx(vcpu); 6938 6939 err = -ENOMEM; 6940 6941 vmx->vpid = allocate_vpid(); 6942 6943 /* 6944 * If PML is turned on, failure on enabling PML just results in failure 6945 * of creating the vcpu, therefore we can simplify PML logic (by 6946 * avoiding dealing with cases, such as enabling PML partially on vcpus 6947 * for the guest), etc. 6948 */ 6949 if (enable_pml) { 6950 vmx->pml_pg = alloc_page(GFP_KERNEL_ACCOUNT | __GFP_ZERO); 6951 if (!vmx->pml_pg) 6952 goto free_vpid; 6953 } 6954 6955 BUILD_BUG_ON(ARRAY_SIZE(vmx_msr_index) != NR_SHARED_MSRS); 6956 6957 for (i = 0; i < ARRAY_SIZE(vmx_msr_index); ++i) { 6958 u32 index = vmx_msr_index[i]; 6959 u32 data_low, data_high; 6960 int j = vmx->nmsrs; 6961 6962 if (rdmsr_safe(index, &data_low, &data_high) < 0) 6963 continue; 6964 if (wrmsr_safe(index, data_low, data_high) < 0) 6965 continue; 6966 6967 vmx->guest_msrs[j].index = i; 6968 vmx->guest_msrs[j].data = 0; 6969 switch (index) { 6970 case MSR_IA32_TSX_CTRL: 6971 /* 6972 * No need to pass TSX_CTRL_CPUID_CLEAR through, so 6973 * let's avoid changing CPUID bits under the host 6974 * kernel's feet. 6975 */ 6976 vmx->guest_msrs[j].mask = ~(u64)TSX_CTRL_CPUID_CLEAR; 6977 break; 6978 default: 6979 vmx->guest_msrs[j].mask = -1ull; 6980 break; 6981 } 6982 ++vmx->nmsrs; 6983 } 6984 6985 err = alloc_loaded_vmcs(&vmx->vmcs01); 6986 if (err < 0) 6987 goto free_pml; 6988 6989 msr_bitmap = vmx->vmcs01.msr_bitmap; 6990 vmx_disable_intercept_for_msr(msr_bitmap, MSR_IA32_TSC, MSR_TYPE_R); 6991 vmx_disable_intercept_for_msr(msr_bitmap, MSR_FS_BASE, MSR_TYPE_RW); 6992 vmx_disable_intercept_for_msr(msr_bitmap, MSR_GS_BASE, MSR_TYPE_RW); 6993 vmx_disable_intercept_for_msr(msr_bitmap, MSR_KERNEL_GS_BASE, MSR_TYPE_RW); 6994 vmx_disable_intercept_for_msr(msr_bitmap, MSR_IA32_SYSENTER_CS, MSR_TYPE_RW); 6995 vmx_disable_intercept_for_msr(msr_bitmap, MSR_IA32_SYSENTER_ESP, MSR_TYPE_RW); 6996 vmx_disable_intercept_for_msr(msr_bitmap, MSR_IA32_SYSENTER_EIP, MSR_TYPE_RW); 6997 if (kvm_cstate_in_guest(vcpu->kvm)) { 6998 vmx_disable_intercept_for_msr(msr_bitmap, MSR_CORE_C1_RES, MSR_TYPE_R); 6999 vmx_disable_intercept_for_msr(msr_bitmap, MSR_CORE_C3_RESIDENCY, MSR_TYPE_R); 7000 vmx_disable_intercept_for_msr(msr_bitmap, MSR_CORE_C6_RESIDENCY, MSR_TYPE_R); 7001 vmx_disable_intercept_for_msr(msr_bitmap, MSR_CORE_C7_RESIDENCY, MSR_TYPE_R); 7002 } 7003 vmx->msr_bitmap_mode = 0; 7004 7005 vmx->loaded_vmcs = &vmx->vmcs01; 7006 cpu = get_cpu(); 7007 vmx_vcpu_load(vcpu, cpu); 7008 vcpu->cpu = cpu; 7009 init_vmcs(vmx); 7010 vmx_vcpu_put(vcpu); 7011 put_cpu(); 7012 if (cpu_need_virtualize_apic_accesses(vcpu)) { 7013 err = alloc_apic_access_page(vcpu->kvm); 7014 if (err) 7015 goto free_vmcs; 7016 } 7017 7018 if (enable_ept && !enable_unrestricted_guest) { 7019 err = init_rmode_identity_map(vcpu->kvm); 7020 if (err) 7021 goto free_vmcs; 7022 } 7023 7024 if (nested) 7025 nested_vmx_setup_ctls_msrs(&vmx->nested.msrs, 7026 vmx_capability.ept); 7027 else 7028 memset(&vmx->nested.msrs, 0, sizeof(vmx->nested.msrs)); 7029 7030 vmx->nested.posted_intr_nv = -1; 7031 vmx->nested.current_vmptr = -1ull; 7032 7033 vcpu->arch.microcode_version = 0x100000000ULL; 7034 vmx->msr_ia32_feature_control_valid_bits = FEAT_CTL_LOCKED; 7035 7036 /* 7037 * Enforce invariant: pi_desc.nv is always either POSTED_INTR_VECTOR 7038 * or POSTED_INTR_WAKEUP_VECTOR. 7039 */ 7040 vmx->pi_desc.nv = POSTED_INTR_VECTOR; 7041 vmx->pi_desc.sn = 1; 7042 7043 vmx->ept_pointer = INVALID_PAGE; 7044 7045 return 0; 7046 7047 free_vmcs: 7048 free_loaded_vmcs(vmx->loaded_vmcs); 7049 free_pml: 7050 vmx_destroy_pml_buffer(vmx); 7051 free_vpid: 7052 free_vpid(vmx->vpid); 7053 return err; 7054 } 7055 7056 #define L1TF_MSG_SMT "L1TF CPU bug present and SMT on, data leak possible. See CVE-2018-3646 and https://www.kernel.org/doc/html/latest/admin-guide/hw-vuln/l1tf.html for details.\n" 7057 #define L1TF_MSG_L1D "L1TF CPU bug present and virtualization mitigation disabled, data leak possible. See CVE-2018-3646 and https://www.kernel.org/doc/html/latest/admin-guide/hw-vuln/l1tf.html for details.\n" 7058 7059 static int vmx_vm_init(struct kvm *kvm) 7060 { 7061 spin_lock_init(&to_kvm_vmx(kvm)->ept_pointer_lock); 7062 7063 if (!ple_gap) 7064 kvm->arch.pause_in_guest = true; 7065 7066 if (boot_cpu_has(X86_BUG_L1TF) && enable_ept) { 7067 switch (l1tf_mitigation) { 7068 case L1TF_MITIGATION_OFF: 7069 case L1TF_MITIGATION_FLUSH_NOWARN: 7070 /* 'I explicitly don't care' is set */ 7071 break; 7072 case L1TF_MITIGATION_FLUSH: 7073 case L1TF_MITIGATION_FLUSH_NOSMT: 7074 case L1TF_MITIGATION_FULL: 7075 /* 7076 * Warn upon starting the first VM in a potentially 7077 * insecure environment. 7078 */ 7079 if (sched_smt_active()) 7080 pr_warn_once(L1TF_MSG_SMT); 7081 if (l1tf_vmx_mitigation == VMENTER_L1D_FLUSH_NEVER) 7082 pr_warn_once(L1TF_MSG_L1D); 7083 break; 7084 case L1TF_MITIGATION_FULL_FORCE: 7085 /* Flush is enforced */ 7086 break; 7087 } 7088 } 7089 kvm_apicv_init(kvm, enable_apicv); 7090 return 0; 7091 } 7092 7093 static int __init vmx_check_processor_compat(void) 7094 { 7095 struct vmcs_config vmcs_conf; 7096 struct vmx_capability vmx_cap; 7097 7098 if (!this_cpu_has(X86_FEATURE_MSR_IA32_FEAT_CTL) || 7099 !this_cpu_has(X86_FEATURE_VMX)) { 7100 pr_err("kvm: VMX is disabled on CPU %d\n", smp_processor_id()); 7101 return -EIO; 7102 } 7103 7104 if (setup_vmcs_config(&vmcs_conf, &vmx_cap) < 0) 7105 return -EIO; 7106 if (nested) 7107 nested_vmx_setup_ctls_msrs(&vmcs_conf.nested, vmx_cap.ept); 7108 if (memcmp(&vmcs_config, &vmcs_conf, sizeof(struct vmcs_config)) != 0) { 7109 printk(KERN_ERR "kvm: CPU %d feature inconsistency!\n", 7110 smp_processor_id()); 7111 return -EIO; 7112 } 7113 return 0; 7114 } 7115 7116 static u64 vmx_get_mt_mask(struct kvm_vcpu *vcpu, gfn_t gfn, bool is_mmio) 7117 { 7118 u8 cache; 7119 u64 ipat = 0; 7120 7121 /* We wanted to honor guest CD/MTRR/PAT, but doing so could result in 7122 * memory aliases with conflicting memory types and sometimes MCEs. 7123 * We have to be careful as to what are honored and when. 7124 * 7125 * For MMIO, guest CD/MTRR are ignored. The EPT memory type is set to 7126 * UC. The effective memory type is UC or WC depending on guest PAT. 7127 * This was historically the source of MCEs and we want to be 7128 * conservative. 7129 * 7130 * When there is no need to deal with noncoherent DMA (e.g., no VT-d 7131 * or VT-d has snoop control), guest CD/MTRR/PAT are all ignored. The 7132 * EPT memory type is set to WB. The effective memory type is forced 7133 * WB. 7134 * 7135 * Otherwise, we trust guest. Guest CD/MTRR/PAT are all honored. The 7136 * EPT memory type is used to emulate guest CD/MTRR. 7137 */ 7138 7139 if (is_mmio) { 7140 cache = MTRR_TYPE_UNCACHABLE; 7141 goto exit; 7142 } 7143 7144 if (!kvm_arch_has_noncoherent_dma(vcpu->kvm)) { 7145 ipat = VMX_EPT_IPAT_BIT; 7146 cache = MTRR_TYPE_WRBACK; 7147 goto exit; 7148 } 7149 7150 if (kvm_read_cr0(vcpu) & X86_CR0_CD) { 7151 ipat = VMX_EPT_IPAT_BIT; 7152 if (kvm_check_has_quirk(vcpu->kvm, KVM_X86_QUIRK_CD_NW_CLEARED)) 7153 cache = MTRR_TYPE_WRBACK; 7154 else 7155 cache = MTRR_TYPE_UNCACHABLE; 7156 goto exit; 7157 } 7158 7159 cache = kvm_mtrr_get_guest_memory_type(vcpu, gfn); 7160 7161 exit: 7162 return (cache << VMX_EPT_MT_EPTE_SHIFT) | ipat; 7163 } 7164 7165 static void vmcs_set_secondary_exec_control(struct vcpu_vmx *vmx) 7166 { 7167 /* 7168 * These bits in the secondary execution controls field 7169 * are dynamic, the others are mostly based on the hypervisor 7170 * architecture and the guest's CPUID. Do not touch the 7171 * dynamic bits. 7172 */ 7173 u32 mask = 7174 SECONDARY_EXEC_SHADOW_VMCS | 7175 SECONDARY_EXEC_VIRTUALIZE_X2APIC_MODE | 7176 SECONDARY_EXEC_VIRTUALIZE_APIC_ACCESSES | 7177 SECONDARY_EXEC_DESC; 7178 7179 u32 new_ctl = vmx->secondary_exec_control; 7180 u32 cur_ctl = secondary_exec_controls_get(vmx); 7181 7182 secondary_exec_controls_set(vmx, (new_ctl & ~mask) | (cur_ctl & mask)); 7183 } 7184 7185 /* 7186 * Generate MSR_IA32_VMX_CR{0,4}_FIXED1 according to CPUID. Only set bits 7187 * (indicating "allowed-1") if they are supported in the guest's CPUID. 7188 */ 7189 static void nested_vmx_cr_fixed1_bits_update(struct kvm_vcpu *vcpu) 7190 { 7191 struct vcpu_vmx *vmx = to_vmx(vcpu); 7192 struct kvm_cpuid_entry2 *entry; 7193 7194 vmx->nested.msrs.cr0_fixed1 = 0xffffffff; 7195 vmx->nested.msrs.cr4_fixed1 = X86_CR4_PCE; 7196 7197 #define cr4_fixed1_update(_cr4_mask, _reg, _cpuid_mask) do { \ 7198 if (entry && (entry->_reg & (_cpuid_mask))) \ 7199 vmx->nested.msrs.cr4_fixed1 |= (_cr4_mask); \ 7200 } while (0) 7201 7202 entry = kvm_find_cpuid_entry(vcpu, 0x1, 0); 7203 cr4_fixed1_update(X86_CR4_VME, edx, feature_bit(VME)); 7204 cr4_fixed1_update(X86_CR4_PVI, edx, feature_bit(VME)); 7205 cr4_fixed1_update(X86_CR4_TSD, edx, feature_bit(TSC)); 7206 cr4_fixed1_update(X86_CR4_DE, edx, feature_bit(DE)); 7207 cr4_fixed1_update(X86_CR4_PSE, edx, feature_bit(PSE)); 7208 cr4_fixed1_update(X86_CR4_PAE, edx, feature_bit(PAE)); 7209 cr4_fixed1_update(X86_CR4_MCE, edx, feature_bit(MCE)); 7210 cr4_fixed1_update(X86_CR4_PGE, edx, feature_bit(PGE)); 7211 cr4_fixed1_update(X86_CR4_OSFXSR, edx, feature_bit(FXSR)); 7212 cr4_fixed1_update(X86_CR4_OSXMMEXCPT, edx, feature_bit(XMM)); 7213 cr4_fixed1_update(X86_CR4_VMXE, ecx, feature_bit(VMX)); 7214 cr4_fixed1_update(X86_CR4_SMXE, ecx, feature_bit(SMX)); 7215 cr4_fixed1_update(X86_CR4_PCIDE, ecx, feature_bit(PCID)); 7216 cr4_fixed1_update(X86_CR4_OSXSAVE, ecx, feature_bit(XSAVE)); 7217 7218 entry = kvm_find_cpuid_entry(vcpu, 0x7, 0); 7219 cr4_fixed1_update(X86_CR4_FSGSBASE, ebx, feature_bit(FSGSBASE)); 7220 cr4_fixed1_update(X86_CR4_SMEP, ebx, feature_bit(SMEP)); 7221 cr4_fixed1_update(X86_CR4_SMAP, ebx, feature_bit(SMAP)); 7222 cr4_fixed1_update(X86_CR4_PKE, ecx, feature_bit(PKU)); 7223 cr4_fixed1_update(X86_CR4_UMIP, ecx, feature_bit(UMIP)); 7224 cr4_fixed1_update(X86_CR4_LA57, ecx, feature_bit(LA57)); 7225 7226 #undef cr4_fixed1_update 7227 } 7228 7229 static void nested_vmx_entry_exit_ctls_update(struct kvm_vcpu *vcpu) 7230 { 7231 struct vcpu_vmx *vmx = to_vmx(vcpu); 7232 7233 if (kvm_mpx_supported()) { 7234 bool mpx_enabled = guest_cpuid_has(vcpu, X86_FEATURE_MPX); 7235 7236 if (mpx_enabled) { 7237 vmx->nested.msrs.entry_ctls_high |= VM_ENTRY_LOAD_BNDCFGS; 7238 vmx->nested.msrs.exit_ctls_high |= VM_EXIT_CLEAR_BNDCFGS; 7239 } else { 7240 vmx->nested.msrs.entry_ctls_high &= ~VM_ENTRY_LOAD_BNDCFGS; 7241 vmx->nested.msrs.exit_ctls_high &= ~VM_EXIT_CLEAR_BNDCFGS; 7242 } 7243 } 7244 } 7245 7246 static void update_intel_pt_cfg(struct kvm_vcpu *vcpu) 7247 { 7248 struct vcpu_vmx *vmx = to_vmx(vcpu); 7249 struct kvm_cpuid_entry2 *best = NULL; 7250 int i; 7251 7252 for (i = 0; i < PT_CPUID_LEAVES; i++) { 7253 best = kvm_find_cpuid_entry(vcpu, 0x14, i); 7254 if (!best) 7255 return; 7256 vmx->pt_desc.caps[CPUID_EAX + i*PT_CPUID_REGS_NUM] = best->eax; 7257 vmx->pt_desc.caps[CPUID_EBX + i*PT_CPUID_REGS_NUM] = best->ebx; 7258 vmx->pt_desc.caps[CPUID_ECX + i*PT_CPUID_REGS_NUM] = best->ecx; 7259 vmx->pt_desc.caps[CPUID_EDX + i*PT_CPUID_REGS_NUM] = best->edx; 7260 } 7261 7262 /* Get the number of configurable Address Ranges for filtering */ 7263 vmx->pt_desc.addr_range = intel_pt_validate_cap(vmx->pt_desc.caps, 7264 PT_CAP_num_address_ranges); 7265 7266 /* Initialize and clear the no dependency bits */ 7267 vmx->pt_desc.ctl_bitmask = ~(RTIT_CTL_TRACEEN | RTIT_CTL_OS | 7268 RTIT_CTL_USR | RTIT_CTL_TSC_EN | RTIT_CTL_DISRETC); 7269 7270 /* 7271 * If CPUID.(EAX=14H,ECX=0):EBX[0]=1 CR3Filter can be set otherwise 7272 * will inject an #GP 7273 */ 7274 if (intel_pt_validate_cap(vmx->pt_desc.caps, PT_CAP_cr3_filtering)) 7275 vmx->pt_desc.ctl_bitmask &= ~RTIT_CTL_CR3EN; 7276 7277 /* 7278 * If CPUID.(EAX=14H,ECX=0):EBX[1]=1 CYCEn, CycThresh and 7279 * PSBFreq can be set 7280 */ 7281 if (intel_pt_validate_cap(vmx->pt_desc.caps, PT_CAP_psb_cyc)) 7282 vmx->pt_desc.ctl_bitmask &= ~(RTIT_CTL_CYCLEACC | 7283 RTIT_CTL_CYC_THRESH | RTIT_CTL_PSB_FREQ); 7284 7285 /* 7286 * If CPUID.(EAX=14H,ECX=0):EBX[3]=1 MTCEn BranchEn and 7287 * MTCFreq can be set 7288 */ 7289 if (intel_pt_validate_cap(vmx->pt_desc.caps, PT_CAP_mtc)) 7290 vmx->pt_desc.ctl_bitmask &= ~(RTIT_CTL_MTC_EN | 7291 RTIT_CTL_BRANCH_EN | RTIT_CTL_MTC_RANGE); 7292 7293 /* If CPUID.(EAX=14H,ECX=0):EBX[4]=1 FUPonPTW and PTWEn can be set */ 7294 if (intel_pt_validate_cap(vmx->pt_desc.caps, PT_CAP_ptwrite)) 7295 vmx->pt_desc.ctl_bitmask &= ~(RTIT_CTL_FUP_ON_PTW | 7296 RTIT_CTL_PTW_EN); 7297 7298 /* If CPUID.(EAX=14H,ECX=0):EBX[5]=1 PwrEvEn can be set */ 7299 if (intel_pt_validate_cap(vmx->pt_desc.caps, PT_CAP_power_event_trace)) 7300 vmx->pt_desc.ctl_bitmask &= ~RTIT_CTL_PWR_EVT_EN; 7301 7302 /* If CPUID.(EAX=14H,ECX=0):ECX[0]=1 ToPA can be set */ 7303 if (intel_pt_validate_cap(vmx->pt_desc.caps, PT_CAP_topa_output)) 7304 vmx->pt_desc.ctl_bitmask &= ~RTIT_CTL_TOPA; 7305 7306 /* If CPUID.(EAX=14H,ECX=0):ECX[3]=1 FabircEn can be set */ 7307 if (intel_pt_validate_cap(vmx->pt_desc.caps, PT_CAP_output_subsys)) 7308 vmx->pt_desc.ctl_bitmask &= ~RTIT_CTL_FABRIC_EN; 7309 7310 /* unmask address range configure area */ 7311 for (i = 0; i < vmx->pt_desc.addr_range; i++) 7312 vmx->pt_desc.ctl_bitmask &= ~(0xfULL << (32 + i * 4)); 7313 } 7314 7315 static void vmx_vcpu_after_set_cpuid(struct kvm_vcpu *vcpu) 7316 { 7317 struct vcpu_vmx *vmx = to_vmx(vcpu); 7318 7319 /* xsaves_enabled is recomputed in vmx_compute_secondary_exec_control(). */ 7320 vcpu->arch.xsaves_enabled = false; 7321 7322 if (cpu_has_secondary_exec_ctrls()) { 7323 vmx_compute_secondary_exec_control(vmx); 7324 vmcs_set_secondary_exec_control(vmx); 7325 } 7326 7327 if (nested_vmx_allowed(vcpu)) 7328 to_vmx(vcpu)->msr_ia32_feature_control_valid_bits |= 7329 FEAT_CTL_VMX_ENABLED_INSIDE_SMX | 7330 FEAT_CTL_VMX_ENABLED_OUTSIDE_SMX; 7331 else 7332 to_vmx(vcpu)->msr_ia32_feature_control_valid_bits &= 7333 ~(FEAT_CTL_VMX_ENABLED_INSIDE_SMX | 7334 FEAT_CTL_VMX_ENABLED_OUTSIDE_SMX); 7335 7336 if (nested_vmx_allowed(vcpu)) { 7337 nested_vmx_cr_fixed1_bits_update(vcpu); 7338 nested_vmx_entry_exit_ctls_update(vcpu); 7339 } 7340 7341 if (boot_cpu_has(X86_FEATURE_INTEL_PT) && 7342 guest_cpuid_has(vcpu, X86_FEATURE_INTEL_PT)) 7343 update_intel_pt_cfg(vcpu); 7344 7345 if (boot_cpu_has(X86_FEATURE_RTM)) { 7346 struct shared_msr_entry *msr; 7347 msr = find_msr_entry(vmx, MSR_IA32_TSX_CTRL); 7348 if (msr) { 7349 bool enabled = guest_cpuid_has(vcpu, X86_FEATURE_RTM); 7350 vmx_set_guest_msr(vmx, msr, enabled ? 0 : TSX_CTRL_RTM_DISABLE); 7351 } 7352 } 7353 } 7354 7355 static __init void vmx_set_cpu_caps(void) 7356 { 7357 kvm_set_cpu_caps(); 7358 7359 /* CPUID 0x1 */ 7360 if (nested) 7361 kvm_cpu_cap_set(X86_FEATURE_VMX); 7362 7363 /* CPUID 0x7 */ 7364 if (kvm_mpx_supported()) 7365 kvm_cpu_cap_check_and_set(X86_FEATURE_MPX); 7366 if (cpu_has_vmx_invpcid()) 7367 kvm_cpu_cap_check_and_set(X86_FEATURE_INVPCID); 7368 if (vmx_pt_mode_is_host_guest()) 7369 kvm_cpu_cap_check_and_set(X86_FEATURE_INTEL_PT); 7370 7371 if (vmx_umip_emulated()) 7372 kvm_cpu_cap_set(X86_FEATURE_UMIP); 7373 7374 /* CPUID 0xD.1 */ 7375 supported_xss = 0; 7376 if (!vmx_xsaves_supported()) 7377 kvm_cpu_cap_clear(X86_FEATURE_XSAVES); 7378 7379 /* CPUID 0x80000001 */ 7380 if (!cpu_has_vmx_rdtscp()) 7381 kvm_cpu_cap_clear(X86_FEATURE_RDTSCP); 7382 7383 if (vmx_waitpkg_supported()) 7384 kvm_cpu_cap_check_and_set(X86_FEATURE_WAITPKG); 7385 } 7386 7387 static void vmx_request_immediate_exit(struct kvm_vcpu *vcpu) 7388 { 7389 to_vmx(vcpu)->req_immediate_exit = true; 7390 } 7391 7392 static int vmx_check_intercept_io(struct kvm_vcpu *vcpu, 7393 struct x86_instruction_info *info) 7394 { 7395 struct vmcs12 *vmcs12 = get_vmcs12(vcpu); 7396 unsigned short port; 7397 bool intercept; 7398 int size; 7399 7400 if (info->intercept == x86_intercept_in || 7401 info->intercept == x86_intercept_ins) { 7402 port = info->src_val; 7403 size = info->dst_bytes; 7404 } else { 7405 port = info->dst_val; 7406 size = info->src_bytes; 7407 } 7408 7409 /* 7410 * If the 'use IO bitmaps' VM-execution control is 0, IO instruction 7411 * VM-exits depend on the 'unconditional IO exiting' VM-execution 7412 * control. 7413 * 7414 * Otherwise, IO instruction VM-exits are controlled by the IO bitmaps. 7415 */ 7416 if (!nested_cpu_has(vmcs12, CPU_BASED_USE_IO_BITMAPS)) 7417 intercept = nested_cpu_has(vmcs12, 7418 CPU_BASED_UNCOND_IO_EXITING); 7419 else 7420 intercept = nested_vmx_check_io_bitmaps(vcpu, port, size); 7421 7422 /* FIXME: produce nested vmexit and return X86EMUL_INTERCEPTED. */ 7423 return intercept ? X86EMUL_UNHANDLEABLE : X86EMUL_CONTINUE; 7424 } 7425 7426 static int vmx_check_intercept(struct kvm_vcpu *vcpu, 7427 struct x86_instruction_info *info, 7428 enum x86_intercept_stage stage, 7429 struct x86_exception *exception) 7430 { 7431 struct vmcs12 *vmcs12 = get_vmcs12(vcpu); 7432 7433 switch (info->intercept) { 7434 /* 7435 * RDPID causes #UD if disabled through secondary execution controls. 7436 * Because it is marked as EmulateOnUD, we need to intercept it here. 7437 */ 7438 case x86_intercept_rdtscp: 7439 if (!nested_cpu_has2(vmcs12, SECONDARY_EXEC_RDTSCP)) { 7440 exception->vector = UD_VECTOR; 7441 exception->error_code_valid = false; 7442 return X86EMUL_PROPAGATE_FAULT; 7443 } 7444 break; 7445 7446 case x86_intercept_in: 7447 case x86_intercept_ins: 7448 case x86_intercept_out: 7449 case x86_intercept_outs: 7450 return vmx_check_intercept_io(vcpu, info); 7451 7452 case x86_intercept_lgdt: 7453 case x86_intercept_lidt: 7454 case x86_intercept_lldt: 7455 case x86_intercept_ltr: 7456 case x86_intercept_sgdt: 7457 case x86_intercept_sidt: 7458 case x86_intercept_sldt: 7459 case x86_intercept_str: 7460 if (!nested_cpu_has2(vmcs12, SECONDARY_EXEC_DESC)) 7461 return X86EMUL_CONTINUE; 7462 7463 /* FIXME: produce nested vmexit and return X86EMUL_INTERCEPTED. */ 7464 break; 7465 7466 /* TODO: check more intercepts... */ 7467 default: 7468 break; 7469 } 7470 7471 return X86EMUL_UNHANDLEABLE; 7472 } 7473 7474 #ifdef CONFIG_X86_64 7475 /* (a << shift) / divisor, return 1 if overflow otherwise 0 */ 7476 static inline int u64_shl_div_u64(u64 a, unsigned int shift, 7477 u64 divisor, u64 *result) 7478 { 7479 u64 low = a << shift, high = a >> (64 - shift); 7480 7481 /* To avoid the overflow on divq */ 7482 if (high >= divisor) 7483 return 1; 7484 7485 /* Low hold the result, high hold rem which is discarded */ 7486 asm("divq %2\n\t" : "=a" (low), "=d" (high) : 7487 "rm" (divisor), "0" (low), "1" (high)); 7488 *result = low; 7489 7490 return 0; 7491 } 7492 7493 static int vmx_set_hv_timer(struct kvm_vcpu *vcpu, u64 guest_deadline_tsc, 7494 bool *expired) 7495 { 7496 struct vcpu_vmx *vmx; 7497 u64 tscl, guest_tscl, delta_tsc, lapic_timer_advance_cycles; 7498 struct kvm_timer *ktimer = &vcpu->arch.apic->lapic_timer; 7499 7500 vmx = to_vmx(vcpu); 7501 tscl = rdtsc(); 7502 guest_tscl = kvm_read_l1_tsc(vcpu, tscl); 7503 delta_tsc = max(guest_deadline_tsc, guest_tscl) - guest_tscl; 7504 lapic_timer_advance_cycles = nsec_to_cycles(vcpu, 7505 ktimer->timer_advance_ns); 7506 7507 if (delta_tsc > lapic_timer_advance_cycles) 7508 delta_tsc -= lapic_timer_advance_cycles; 7509 else 7510 delta_tsc = 0; 7511 7512 /* Convert to host delta tsc if tsc scaling is enabled */ 7513 if (vcpu->arch.tsc_scaling_ratio != kvm_default_tsc_scaling_ratio && 7514 delta_tsc && u64_shl_div_u64(delta_tsc, 7515 kvm_tsc_scaling_ratio_frac_bits, 7516 vcpu->arch.tsc_scaling_ratio, &delta_tsc)) 7517 return -ERANGE; 7518 7519 /* 7520 * If the delta tsc can't fit in the 32 bit after the multi shift, 7521 * we can't use the preemption timer. 7522 * It's possible that it fits on later vmentries, but checking 7523 * on every vmentry is costly so we just use an hrtimer. 7524 */ 7525 if (delta_tsc >> (cpu_preemption_timer_multi + 32)) 7526 return -ERANGE; 7527 7528 vmx->hv_deadline_tsc = tscl + delta_tsc; 7529 *expired = !delta_tsc; 7530 return 0; 7531 } 7532 7533 static void vmx_cancel_hv_timer(struct kvm_vcpu *vcpu) 7534 { 7535 to_vmx(vcpu)->hv_deadline_tsc = -1; 7536 } 7537 #endif 7538 7539 static void vmx_sched_in(struct kvm_vcpu *vcpu, int cpu) 7540 { 7541 if (!kvm_pause_in_guest(vcpu->kvm)) 7542 shrink_ple_window(vcpu); 7543 } 7544 7545 static void vmx_slot_enable_log_dirty(struct kvm *kvm, 7546 struct kvm_memory_slot *slot) 7547 { 7548 if (!kvm_dirty_log_manual_protect_and_init_set(kvm)) 7549 kvm_mmu_slot_leaf_clear_dirty(kvm, slot); 7550 kvm_mmu_slot_largepage_remove_write_access(kvm, slot); 7551 } 7552 7553 static void vmx_slot_disable_log_dirty(struct kvm *kvm, 7554 struct kvm_memory_slot *slot) 7555 { 7556 kvm_mmu_slot_set_dirty(kvm, slot); 7557 } 7558 7559 static void vmx_flush_log_dirty(struct kvm *kvm) 7560 { 7561 kvm_flush_pml_buffers(kvm); 7562 } 7563 7564 static void vmx_enable_log_dirty_pt_masked(struct kvm *kvm, 7565 struct kvm_memory_slot *memslot, 7566 gfn_t offset, unsigned long mask) 7567 { 7568 kvm_mmu_clear_dirty_pt_masked(kvm, memslot, offset, mask); 7569 } 7570 7571 static void __pi_post_block(struct kvm_vcpu *vcpu) 7572 { 7573 struct pi_desc *pi_desc = vcpu_to_pi_desc(vcpu); 7574 struct pi_desc old, new; 7575 unsigned int dest; 7576 7577 do { 7578 old.control = new.control = pi_desc->control; 7579 WARN(old.nv != POSTED_INTR_WAKEUP_VECTOR, 7580 "Wakeup handler not enabled while the VCPU is blocked\n"); 7581 7582 dest = cpu_physical_id(vcpu->cpu); 7583 7584 if (x2apic_enabled()) 7585 new.ndst = dest; 7586 else 7587 new.ndst = (dest << 8) & 0xFF00; 7588 7589 /* set 'NV' to 'notification vector' */ 7590 new.nv = POSTED_INTR_VECTOR; 7591 } while (cmpxchg64(&pi_desc->control, old.control, 7592 new.control) != old.control); 7593 7594 if (!WARN_ON_ONCE(vcpu->pre_pcpu == -1)) { 7595 spin_lock(&per_cpu(blocked_vcpu_on_cpu_lock, vcpu->pre_pcpu)); 7596 list_del(&vcpu->blocked_vcpu_list); 7597 spin_unlock(&per_cpu(blocked_vcpu_on_cpu_lock, vcpu->pre_pcpu)); 7598 vcpu->pre_pcpu = -1; 7599 } 7600 } 7601 7602 /* 7603 * This routine does the following things for vCPU which is going 7604 * to be blocked if VT-d PI is enabled. 7605 * - Store the vCPU to the wakeup list, so when interrupts happen 7606 * we can find the right vCPU to wake up. 7607 * - Change the Posted-interrupt descriptor as below: 7608 * 'NDST' <-- vcpu->pre_pcpu 7609 * 'NV' <-- POSTED_INTR_WAKEUP_VECTOR 7610 * - If 'ON' is set during this process, which means at least one 7611 * interrupt is posted for this vCPU, we cannot block it, in 7612 * this case, return 1, otherwise, return 0. 7613 * 7614 */ 7615 static int pi_pre_block(struct kvm_vcpu *vcpu) 7616 { 7617 unsigned int dest; 7618 struct pi_desc old, new; 7619 struct pi_desc *pi_desc = vcpu_to_pi_desc(vcpu); 7620 7621 if (!kvm_arch_has_assigned_device(vcpu->kvm) || 7622 !irq_remapping_cap(IRQ_POSTING_CAP) || 7623 !kvm_vcpu_apicv_active(vcpu)) 7624 return 0; 7625 7626 WARN_ON(irqs_disabled()); 7627 local_irq_disable(); 7628 if (!WARN_ON_ONCE(vcpu->pre_pcpu != -1)) { 7629 vcpu->pre_pcpu = vcpu->cpu; 7630 spin_lock(&per_cpu(blocked_vcpu_on_cpu_lock, vcpu->pre_pcpu)); 7631 list_add_tail(&vcpu->blocked_vcpu_list, 7632 &per_cpu(blocked_vcpu_on_cpu, 7633 vcpu->pre_pcpu)); 7634 spin_unlock(&per_cpu(blocked_vcpu_on_cpu_lock, vcpu->pre_pcpu)); 7635 } 7636 7637 do { 7638 old.control = new.control = pi_desc->control; 7639 7640 WARN((pi_desc->sn == 1), 7641 "Warning: SN field of posted-interrupts " 7642 "is set before blocking\n"); 7643 7644 /* 7645 * Since vCPU can be preempted during this process, 7646 * vcpu->cpu could be different with pre_pcpu, we 7647 * need to set pre_pcpu as the destination of wakeup 7648 * notification event, then we can find the right vCPU 7649 * to wakeup in wakeup handler if interrupts happen 7650 * when the vCPU is in blocked state. 7651 */ 7652 dest = cpu_physical_id(vcpu->pre_pcpu); 7653 7654 if (x2apic_enabled()) 7655 new.ndst = dest; 7656 else 7657 new.ndst = (dest << 8) & 0xFF00; 7658 7659 /* set 'NV' to 'wakeup vector' */ 7660 new.nv = POSTED_INTR_WAKEUP_VECTOR; 7661 } while (cmpxchg64(&pi_desc->control, old.control, 7662 new.control) != old.control); 7663 7664 /* We should not block the vCPU if an interrupt is posted for it. */ 7665 if (pi_test_on(pi_desc) == 1) 7666 __pi_post_block(vcpu); 7667 7668 local_irq_enable(); 7669 return (vcpu->pre_pcpu == -1); 7670 } 7671 7672 static int vmx_pre_block(struct kvm_vcpu *vcpu) 7673 { 7674 if (pi_pre_block(vcpu)) 7675 return 1; 7676 7677 if (kvm_lapic_hv_timer_in_use(vcpu)) 7678 kvm_lapic_switch_to_sw_timer(vcpu); 7679 7680 return 0; 7681 } 7682 7683 static void pi_post_block(struct kvm_vcpu *vcpu) 7684 { 7685 if (vcpu->pre_pcpu == -1) 7686 return; 7687 7688 WARN_ON(irqs_disabled()); 7689 local_irq_disable(); 7690 __pi_post_block(vcpu); 7691 local_irq_enable(); 7692 } 7693 7694 static void vmx_post_block(struct kvm_vcpu *vcpu) 7695 { 7696 if (kvm_x86_ops.set_hv_timer) 7697 kvm_lapic_switch_to_hv_timer(vcpu); 7698 7699 pi_post_block(vcpu); 7700 } 7701 7702 /* 7703 * vmx_update_pi_irte - set IRTE for Posted-Interrupts 7704 * 7705 * @kvm: kvm 7706 * @host_irq: host irq of the interrupt 7707 * @guest_irq: gsi of the interrupt 7708 * @set: set or unset PI 7709 * returns 0 on success, < 0 on failure 7710 */ 7711 static int vmx_update_pi_irte(struct kvm *kvm, unsigned int host_irq, 7712 uint32_t guest_irq, bool set) 7713 { 7714 struct kvm_kernel_irq_routing_entry *e; 7715 struct kvm_irq_routing_table *irq_rt; 7716 struct kvm_lapic_irq irq; 7717 struct kvm_vcpu *vcpu; 7718 struct vcpu_data vcpu_info; 7719 int idx, ret = 0; 7720 7721 if (!kvm_arch_has_assigned_device(kvm) || 7722 !irq_remapping_cap(IRQ_POSTING_CAP) || 7723 !kvm_vcpu_apicv_active(kvm->vcpus[0])) 7724 return 0; 7725 7726 idx = srcu_read_lock(&kvm->irq_srcu); 7727 irq_rt = srcu_dereference(kvm->irq_routing, &kvm->irq_srcu); 7728 if (guest_irq >= irq_rt->nr_rt_entries || 7729 hlist_empty(&irq_rt->map[guest_irq])) { 7730 pr_warn_once("no route for guest_irq %u/%u (broken user space?)\n", 7731 guest_irq, irq_rt->nr_rt_entries); 7732 goto out; 7733 } 7734 7735 hlist_for_each_entry(e, &irq_rt->map[guest_irq], link) { 7736 if (e->type != KVM_IRQ_ROUTING_MSI) 7737 continue; 7738 /* 7739 * VT-d PI cannot support posting multicast/broadcast 7740 * interrupts to a vCPU, we still use interrupt remapping 7741 * for these kind of interrupts. 7742 * 7743 * For lowest-priority interrupts, we only support 7744 * those with single CPU as the destination, e.g. user 7745 * configures the interrupts via /proc/irq or uses 7746 * irqbalance to make the interrupts single-CPU. 7747 * 7748 * We will support full lowest-priority interrupt later. 7749 * 7750 * In addition, we can only inject generic interrupts using 7751 * the PI mechanism, refuse to route others through it. 7752 */ 7753 7754 kvm_set_msi_irq(kvm, e, &irq); 7755 if (!kvm_intr_is_single_vcpu(kvm, &irq, &vcpu) || 7756 !kvm_irq_is_postable(&irq)) { 7757 /* 7758 * Make sure the IRTE is in remapped mode if 7759 * we don't handle it in posted mode. 7760 */ 7761 ret = irq_set_vcpu_affinity(host_irq, NULL); 7762 if (ret < 0) { 7763 printk(KERN_INFO 7764 "failed to back to remapped mode, irq: %u\n", 7765 host_irq); 7766 goto out; 7767 } 7768 7769 continue; 7770 } 7771 7772 vcpu_info.pi_desc_addr = __pa(vcpu_to_pi_desc(vcpu)); 7773 vcpu_info.vector = irq.vector; 7774 7775 trace_kvm_pi_irte_update(host_irq, vcpu->vcpu_id, e->gsi, 7776 vcpu_info.vector, vcpu_info.pi_desc_addr, set); 7777 7778 if (set) 7779 ret = irq_set_vcpu_affinity(host_irq, &vcpu_info); 7780 else 7781 ret = irq_set_vcpu_affinity(host_irq, NULL); 7782 7783 if (ret < 0) { 7784 printk(KERN_INFO "%s: failed to update PI IRTE\n", 7785 __func__); 7786 goto out; 7787 } 7788 } 7789 7790 ret = 0; 7791 out: 7792 srcu_read_unlock(&kvm->irq_srcu, idx); 7793 return ret; 7794 } 7795 7796 static void vmx_setup_mce(struct kvm_vcpu *vcpu) 7797 { 7798 if (vcpu->arch.mcg_cap & MCG_LMCE_P) 7799 to_vmx(vcpu)->msr_ia32_feature_control_valid_bits |= 7800 FEAT_CTL_LMCE_ENABLED; 7801 else 7802 to_vmx(vcpu)->msr_ia32_feature_control_valid_bits &= 7803 ~FEAT_CTL_LMCE_ENABLED; 7804 } 7805 7806 static int vmx_smi_allowed(struct kvm_vcpu *vcpu, bool for_injection) 7807 { 7808 /* we need a nested vmexit to enter SMM, postpone if run is pending */ 7809 if (to_vmx(vcpu)->nested.nested_run_pending) 7810 return -EBUSY; 7811 return !is_smm(vcpu); 7812 } 7813 7814 static int vmx_pre_enter_smm(struct kvm_vcpu *vcpu, char *smstate) 7815 { 7816 struct vcpu_vmx *vmx = to_vmx(vcpu); 7817 7818 vmx->nested.smm.guest_mode = is_guest_mode(vcpu); 7819 if (vmx->nested.smm.guest_mode) 7820 nested_vmx_vmexit(vcpu, -1, 0, 0); 7821 7822 vmx->nested.smm.vmxon = vmx->nested.vmxon; 7823 vmx->nested.vmxon = false; 7824 vmx_clear_hlt(vcpu); 7825 return 0; 7826 } 7827 7828 static int vmx_pre_leave_smm(struct kvm_vcpu *vcpu, const char *smstate) 7829 { 7830 struct vcpu_vmx *vmx = to_vmx(vcpu); 7831 int ret; 7832 7833 if (vmx->nested.smm.vmxon) { 7834 vmx->nested.vmxon = true; 7835 vmx->nested.smm.vmxon = false; 7836 } 7837 7838 if (vmx->nested.smm.guest_mode) { 7839 ret = nested_vmx_enter_non_root_mode(vcpu, false); 7840 if (ret) 7841 return ret; 7842 7843 vmx->nested.smm.guest_mode = false; 7844 } 7845 return 0; 7846 } 7847 7848 static void enable_smi_window(struct kvm_vcpu *vcpu) 7849 { 7850 /* RSM will cause a vmexit anyway. */ 7851 } 7852 7853 static bool vmx_need_emulation_on_page_fault(struct kvm_vcpu *vcpu) 7854 { 7855 return false; 7856 } 7857 7858 static bool vmx_apic_init_signal_blocked(struct kvm_vcpu *vcpu) 7859 { 7860 return to_vmx(vcpu)->nested.vmxon; 7861 } 7862 7863 static void vmx_migrate_timers(struct kvm_vcpu *vcpu) 7864 { 7865 if (is_guest_mode(vcpu)) { 7866 struct hrtimer *timer = &to_vmx(vcpu)->nested.preemption_timer; 7867 7868 if (hrtimer_try_to_cancel(timer) == 1) 7869 hrtimer_start_expires(timer, HRTIMER_MODE_ABS_PINNED); 7870 } 7871 } 7872 7873 static void hardware_unsetup(void) 7874 { 7875 if (nested) 7876 nested_vmx_hardware_unsetup(); 7877 7878 free_kvm_area(); 7879 } 7880 7881 static bool vmx_check_apicv_inhibit_reasons(ulong bit) 7882 { 7883 ulong supported = BIT(APICV_INHIBIT_REASON_DISABLE) | 7884 BIT(APICV_INHIBIT_REASON_HYPERV); 7885 7886 return supported & BIT(bit); 7887 } 7888 7889 static struct kvm_x86_ops vmx_x86_ops __initdata = { 7890 .hardware_unsetup = hardware_unsetup, 7891 7892 .hardware_enable = hardware_enable, 7893 .hardware_disable = hardware_disable, 7894 .cpu_has_accelerated_tpr = report_flexpriority, 7895 .has_emulated_msr = vmx_has_emulated_msr, 7896 7897 .vm_size = sizeof(struct kvm_vmx), 7898 .vm_init = vmx_vm_init, 7899 7900 .vcpu_create = vmx_create_vcpu, 7901 .vcpu_free = vmx_free_vcpu, 7902 .vcpu_reset = vmx_vcpu_reset, 7903 7904 .prepare_guest_switch = vmx_prepare_switch_to_guest, 7905 .vcpu_load = vmx_vcpu_load, 7906 .vcpu_put = vmx_vcpu_put, 7907 7908 .update_exception_bitmap = update_exception_bitmap, 7909 .get_msr_feature = vmx_get_msr_feature, 7910 .get_msr = vmx_get_msr, 7911 .set_msr = vmx_set_msr, 7912 .get_segment_base = vmx_get_segment_base, 7913 .get_segment = vmx_get_segment, 7914 .set_segment = vmx_set_segment, 7915 .get_cpl = vmx_get_cpl, 7916 .get_cs_db_l_bits = vmx_get_cs_db_l_bits, 7917 .set_cr0 = vmx_set_cr0, 7918 .set_cr4 = vmx_set_cr4, 7919 .set_efer = vmx_set_efer, 7920 .get_idt = vmx_get_idt, 7921 .set_idt = vmx_set_idt, 7922 .get_gdt = vmx_get_gdt, 7923 .set_gdt = vmx_set_gdt, 7924 .set_dr7 = vmx_set_dr7, 7925 .sync_dirty_debug_regs = vmx_sync_dirty_debug_regs, 7926 .cache_reg = vmx_cache_reg, 7927 .get_rflags = vmx_get_rflags, 7928 .set_rflags = vmx_set_rflags, 7929 7930 .tlb_flush_all = vmx_flush_tlb_all, 7931 .tlb_flush_current = vmx_flush_tlb_current, 7932 .tlb_flush_gva = vmx_flush_tlb_gva, 7933 .tlb_flush_guest = vmx_flush_tlb_guest, 7934 7935 .run = vmx_vcpu_run, 7936 .handle_exit = vmx_handle_exit, 7937 .skip_emulated_instruction = vmx_skip_emulated_instruction, 7938 .update_emulated_instruction = vmx_update_emulated_instruction, 7939 .set_interrupt_shadow = vmx_set_interrupt_shadow, 7940 .get_interrupt_shadow = vmx_get_interrupt_shadow, 7941 .patch_hypercall = vmx_patch_hypercall, 7942 .set_irq = vmx_inject_irq, 7943 .set_nmi = vmx_inject_nmi, 7944 .queue_exception = vmx_queue_exception, 7945 .cancel_injection = vmx_cancel_injection, 7946 .interrupt_allowed = vmx_interrupt_allowed, 7947 .nmi_allowed = vmx_nmi_allowed, 7948 .get_nmi_mask = vmx_get_nmi_mask, 7949 .set_nmi_mask = vmx_set_nmi_mask, 7950 .enable_nmi_window = enable_nmi_window, 7951 .enable_irq_window = enable_irq_window, 7952 .update_cr8_intercept = update_cr8_intercept, 7953 .set_virtual_apic_mode = vmx_set_virtual_apic_mode, 7954 .set_apic_access_page_addr = vmx_set_apic_access_page_addr, 7955 .refresh_apicv_exec_ctrl = vmx_refresh_apicv_exec_ctrl, 7956 .load_eoi_exitmap = vmx_load_eoi_exitmap, 7957 .apicv_post_state_restore = vmx_apicv_post_state_restore, 7958 .check_apicv_inhibit_reasons = vmx_check_apicv_inhibit_reasons, 7959 .hwapic_irr_update = vmx_hwapic_irr_update, 7960 .hwapic_isr_update = vmx_hwapic_isr_update, 7961 .guest_apic_has_interrupt = vmx_guest_apic_has_interrupt, 7962 .sync_pir_to_irr = vmx_sync_pir_to_irr, 7963 .deliver_posted_interrupt = vmx_deliver_posted_interrupt, 7964 .dy_apicv_has_pending_interrupt = vmx_dy_apicv_has_pending_interrupt, 7965 7966 .set_tss_addr = vmx_set_tss_addr, 7967 .set_identity_map_addr = vmx_set_identity_map_addr, 7968 .get_mt_mask = vmx_get_mt_mask, 7969 7970 .get_exit_info = vmx_get_exit_info, 7971 7972 .vcpu_after_set_cpuid = vmx_vcpu_after_set_cpuid, 7973 7974 .has_wbinvd_exit = cpu_has_vmx_wbinvd_exit, 7975 7976 .write_l1_tsc_offset = vmx_write_l1_tsc_offset, 7977 7978 .load_mmu_pgd = vmx_load_mmu_pgd, 7979 7980 .check_intercept = vmx_check_intercept, 7981 .handle_exit_irqoff = vmx_handle_exit_irqoff, 7982 7983 .request_immediate_exit = vmx_request_immediate_exit, 7984 7985 .sched_in = vmx_sched_in, 7986 7987 .slot_enable_log_dirty = vmx_slot_enable_log_dirty, 7988 .slot_disable_log_dirty = vmx_slot_disable_log_dirty, 7989 .flush_log_dirty = vmx_flush_log_dirty, 7990 .enable_log_dirty_pt_masked = vmx_enable_log_dirty_pt_masked, 7991 7992 .pre_block = vmx_pre_block, 7993 .post_block = vmx_post_block, 7994 7995 .pmu_ops = &intel_pmu_ops, 7996 .nested_ops = &vmx_nested_ops, 7997 7998 .update_pi_irte = vmx_update_pi_irte, 7999 8000 #ifdef CONFIG_X86_64 8001 .set_hv_timer = vmx_set_hv_timer, 8002 .cancel_hv_timer = vmx_cancel_hv_timer, 8003 #endif 8004 8005 .setup_mce = vmx_setup_mce, 8006 8007 .smi_allowed = vmx_smi_allowed, 8008 .pre_enter_smm = vmx_pre_enter_smm, 8009 .pre_leave_smm = vmx_pre_leave_smm, 8010 .enable_smi_window = enable_smi_window, 8011 8012 .need_emulation_on_page_fault = vmx_need_emulation_on_page_fault, 8013 .apic_init_signal_blocked = vmx_apic_init_signal_blocked, 8014 .migrate_timers = vmx_migrate_timers, 8015 }; 8016 8017 static __init int hardware_setup(void) 8018 { 8019 unsigned long host_bndcfgs; 8020 struct desc_ptr dt; 8021 int r, i, ept_lpage_level; 8022 8023 store_idt(&dt); 8024 host_idt_base = dt.address; 8025 8026 for (i = 0; i < ARRAY_SIZE(vmx_msr_index); ++i) 8027 kvm_define_shared_msr(i, vmx_msr_index[i]); 8028 8029 if (setup_vmcs_config(&vmcs_config, &vmx_capability) < 0) 8030 return -EIO; 8031 8032 if (boot_cpu_has(X86_FEATURE_NX)) 8033 kvm_enable_efer_bits(EFER_NX); 8034 8035 if (boot_cpu_has(X86_FEATURE_MPX)) { 8036 rdmsrl(MSR_IA32_BNDCFGS, host_bndcfgs); 8037 WARN_ONCE(host_bndcfgs, "KVM: BNDCFGS in host will be lost"); 8038 } 8039 8040 if (!cpu_has_vmx_mpx()) 8041 supported_xcr0 &= ~(XFEATURE_MASK_BNDREGS | 8042 XFEATURE_MASK_BNDCSR); 8043 8044 if (!cpu_has_vmx_vpid() || !cpu_has_vmx_invvpid() || 8045 !(cpu_has_vmx_invvpid_single() || cpu_has_vmx_invvpid_global())) 8046 enable_vpid = 0; 8047 8048 if (!cpu_has_vmx_ept() || 8049 !cpu_has_vmx_ept_4levels() || 8050 !cpu_has_vmx_ept_mt_wb() || 8051 !cpu_has_vmx_invept_global()) 8052 enable_ept = 0; 8053 8054 if (!cpu_has_vmx_ept_ad_bits() || !enable_ept) 8055 enable_ept_ad_bits = 0; 8056 8057 if (!cpu_has_vmx_unrestricted_guest() || !enable_ept) 8058 enable_unrestricted_guest = 0; 8059 8060 if (!cpu_has_vmx_flexpriority()) 8061 flexpriority_enabled = 0; 8062 8063 if (!cpu_has_virtual_nmis()) 8064 enable_vnmi = 0; 8065 8066 /* 8067 * set_apic_access_page_addr() is used to reload apic access 8068 * page upon invalidation. No need to do anything if not 8069 * using the APIC_ACCESS_ADDR VMCS field. 8070 */ 8071 if (!flexpriority_enabled) 8072 vmx_x86_ops.set_apic_access_page_addr = NULL; 8073 8074 if (!cpu_has_vmx_tpr_shadow()) 8075 vmx_x86_ops.update_cr8_intercept = NULL; 8076 8077 #if IS_ENABLED(CONFIG_HYPERV) 8078 if (ms_hyperv.nested_features & HV_X64_NESTED_GUEST_MAPPING_FLUSH 8079 && enable_ept) { 8080 vmx_x86_ops.tlb_remote_flush = hv_remote_flush_tlb; 8081 vmx_x86_ops.tlb_remote_flush_with_range = 8082 hv_remote_flush_tlb_with_range; 8083 } 8084 #endif 8085 8086 if (!cpu_has_vmx_ple()) { 8087 ple_gap = 0; 8088 ple_window = 0; 8089 ple_window_grow = 0; 8090 ple_window_max = 0; 8091 ple_window_shrink = 0; 8092 } 8093 8094 if (!cpu_has_vmx_apicv()) { 8095 enable_apicv = 0; 8096 vmx_x86_ops.sync_pir_to_irr = NULL; 8097 } 8098 8099 if (cpu_has_vmx_tsc_scaling()) { 8100 kvm_has_tsc_control = true; 8101 kvm_max_tsc_scaling_ratio = KVM_VMX_TSC_MULTIPLIER_MAX; 8102 kvm_tsc_scaling_ratio_frac_bits = 48; 8103 } 8104 8105 set_bit(0, vmx_vpid_bitmap); /* 0 is reserved for host */ 8106 8107 if (enable_ept) 8108 vmx_enable_tdp(); 8109 8110 if (!enable_ept) 8111 ept_lpage_level = 0; 8112 else if (cpu_has_vmx_ept_1g_page()) 8113 ept_lpage_level = PG_LEVEL_1G; 8114 else if (cpu_has_vmx_ept_2m_page()) 8115 ept_lpage_level = PG_LEVEL_2M; 8116 else 8117 ept_lpage_level = PG_LEVEL_4K; 8118 kvm_configure_mmu(enable_ept, vmx_get_max_tdp_level(), ept_lpage_level); 8119 8120 /* 8121 * Only enable PML when hardware supports PML feature, and both EPT 8122 * and EPT A/D bit features are enabled -- PML depends on them to work. 8123 */ 8124 if (!enable_ept || !enable_ept_ad_bits || !cpu_has_vmx_pml()) 8125 enable_pml = 0; 8126 8127 if (!enable_pml) { 8128 vmx_x86_ops.slot_enable_log_dirty = NULL; 8129 vmx_x86_ops.slot_disable_log_dirty = NULL; 8130 vmx_x86_ops.flush_log_dirty = NULL; 8131 vmx_x86_ops.enable_log_dirty_pt_masked = NULL; 8132 } 8133 8134 if (!cpu_has_vmx_preemption_timer()) 8135 enable_preemption_timer = false; 8136 8137 if (enable_preemption_timer) { 8138 u64 use_timer_freq = 5000ULL * 1000 * 1000; 8139 u64 vmx_msr; 8140 8141 rdmsrl(MSR_IA32_VMX_MISC, vmx_msr); 8142 cpu_preemption_timer_multi = 8143 vmx_msr & VMX_MISC_PREEMPTION_TIMER_RATE_MASK; 8144 8145 if (tsc_khz) 8146 use_timer_freq = (u64)tsc_khz * 1000; 8147 use_timer_freq >>= cpu_preemption_timer_multi; 8148 8149 /* 8150 * KVM "disables" the preemption timer by setting it to its max 8151 * value. Don't use the timer if it might cause spurious exits 8152 * at a rate faster than 0.1 Hz (of uninterrupted guest time). 8153 */ 8154 if (use_timer_freq > 0xffffffffu / 10) 8155 enable_preemption_timer = false; 8156 } 8157 8158 if (!enable_preemption_timer) { 8159 vmx_x86_ops.set_hv_timer = NULL; 8160 vmx_x86_ops.cancel_hv_timer = NULL; 8161 vmx_x86_ops.request_immediate_exit = __kvm_request_immediate_exit; 8162 } 8163 8164 kvm_set_posted_intr_wakeup_handler(wakeup_handler); 8165 8166 kvm_mce_cap_supported |= MCG_LMCE_P; 8167 8168 if (pt_mode != PT_MODE_SYSTEM && pt_mode != PT_MODE_HOST_GUEST) 8169 return -EINVAL; 8170 if (!enable_ept || !cpu_has_vmx_intel_pt()) 8171 pt_mode = PT_MODE_SYSTEM; 8172 8173 if (nested) { 8174 nested_vmx_setup_ctls_msrs(&vmcs_config.nested, 8175 vmx_capability.ept); 8176 8177 r = nested_vmx_hardware_setup(kvm_vmx_exit_handlers); 8178 if (r) 8179 return r; 8180 } 8181 8182 vmx_set_cpu_caps(); 8183 8184 r = alloc_kvm_area(); 8185 if (r) 8186 nested_vmx_hardware_unsetup(); 8187 return r; 8188 } 8189 8190 static struct kvm_x86_init_ops vmx_init_ops __initdata = { 8191 .cpu_has_kvm_support = cpu_has_kvm_support, 8192 .disabled_by_bios = vmx_disabled_by_bios, 8193 .check_processor_compatibility = vmx_check_processor_compat, 8194 .hardware_setup = hardware_setup, 8195 8196 .runtime_ops = &vmx_x86_ops, 8197 }; 8198 8199 static void vmx_cleanup_l1d_flush(void) 8200 { 8201 if (vmx_l1d_flush_pages) { 8202 free_pages((unsigned long)vmx_l1d_flush_pages, L1D_CACHE_ORDER); 8203 vmx_l1d_flush_pages = NULL; 8204 } 8205 /* Restore state so sysfs ignores VMX */ 8206 l1tf_vmx_mitigation = VMENTER_L1D_FLUSH_AUTO; 8207 } 8208 8209 static void vmx_exit(void) 8210 { 8211 #ifdef CONFIG_KEXEC_CORE 8212 RCU_INIT_POINTER(crash_vmclear_loaded_vmcss, NULL); 8213 synchronize_rcu(); 8214 #endif 8215 8216 kvm_exit(); 8217 8218 #if IS_ENABLED(CONFIG_HYPERV) 8219 if (static_branch_unlikely(&enable_evmcs)) { 8220 int cpu; 8221 struct hv_vp_assist_page *vp_ap; 8222 /* 8223 * Reset everything to support using non-enlightened VMCS 8224 * access later (e.g. when we reload the module with 8225 * enlightened_vmcs=0) 8226 */ 8227 for_each_online_cpu(cpu) { 8228 vp_ap = hv_get_vp_assist_page(cpu); 8229 8230 if (!vp_ap) 8231 continue; 8232 8233 vp_ap->nested_control.features.directhypercall = 0; 8234 vp_ap->current_nested_vmcs = 0; 8235 vp_ap->enlighten_vmentry = 0; 8236 } 8237 8238 static_branch_disable(&enable_evmcs); 8239 } 8240 #endif 8241 vmx_cleanup_l1d_flush(); 8242 } 8243 module_exit(vmx_exit); 8244 8245 static int __init vmx_init(void) 8246 { 8247 int r, cpu; 8248 8249 #if IS_ENABLED(CONFIG_HYPERV) 8250 /* 8251 * Enlightened VMCS usage should be recommended and the host needs 8252 * to support eVMCS v1 or above. We can also disable eVMCS support 8253 * with module parameter. 8254 */ 8255 if (enlightened_vmcs && 8256 ms_hyperv.hints & HV_X64_ENLIGHTENED_VMCS_RECOMMENDED && 8257 (ms_hyperv.nested_features & HV_X64_ENLIGHTENED_VMCS_VERSION) >= 8258 KVM_EVMCS_VERSION) { 8259 int cpu; 8260 8261 /* Check that we have assist pages on all online CPUs */ 8262 for_each_online_cpu(cpu) { 8263 if (!hv_get_vp_assist_page(cpu)) { 8264 enlightened_vmcs = false; 8265 break; 8266 } 8267 } 8268 8269 if (enlightened_vmcs) { 8270 pr_info("KVM: vmx: using Hyper-V Enlightened VMCS\n"); 8271 static_branch_enable(&enable_evmcs); 8272 } 8273 8274 if (ms_hyperv.nested_features & HV_X64_NESTED_DIRECT_FLUSH) 8275 vmx_x86_ops.enable_direct_tlbflush 8276 = hv_enable_direct_tlbflush; 8277 8278 } else { 8279 enlightened_vmcs = false; 8280 } 8281 #endif 8282 8283 r = kvm_init(&vmx_init_ops, sizeof(struct vcpu_vmx), 8284 __alignof__(struct vcpu_vmx), THIS_MODULE); 8285 if (r) 8286 return r; 8287 8288 /* 8289 * Must be called after kvm_init() so enable_ept is properly set 8290 * up. Hand the parameter mitigation value in which was stored in 8291 * the pre module init parser. If no parameter was given, it will 8292 * contain 'auto' which will be turned into the default 'cond' 8293 * mitigation mode. 8294 */ 8295 r = vmx_setup_l1d_flush(vmentry_l1d_flush_param); 8296 if (r) { 8297 vmx_exit(); 8298 return r; 8299 } 8300 8301 for_each_possible_cpu(cpu) { 8302 INIT_LIST_HEAD(&per_cpu(loaded_vmcss_on_cpu, cpu)); 8303 INIT_LIST_HEAD(&per_cpu(blocked_vcpu_on_cpu, cpu)); 8304 spin_lock_init(&per_cpu(blocked_vcpu_on_cpu_lock, cpu)); 8305 } 8306 8307 #ifdef CONFIG_KEXEC_CORE 8308 rcu_assign_pointer(crash_vmclear_loaded_vmcss, 8309 crash_vmclear_local_loaded_vmcss); 8310 #endif 8311 vmx_check_vmcs12_offsets(); 8312 8313 /* 8314 * Shadow paging doesn't have a (further) performance penalty 8315 * from GUEST_MAXPHYADDR < HOST_MAXPHYADDR so enable it 8316 * by default 8317 */ 8318 if (!enable_ept) 8319 allow_smaller_maxphyaddr = true; 8320 8321 return 0; 8322 } 8323 module_init(vmx_init); 8324