1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * Kernel-based Virtual Machine driver for Linux 4 * 5 * This module enables machines with Intel VT-x extensions to run virtual 6 * machines without emulation or binary translation. 7 * 8 * Copyright (C) 2006 Qumranet, Inc. 9 * Copyright 2010 Red Hat, Inc. and/or its affiliates. 10 * 11 * Authors: 12 * Avi Kivity <avi@qumranet.com> 13 * Yaniv Kamay <yaniv@qumranet.com> 14 */ 15 16 #include <linux/highmem.h> 17 #include <linux/hrtimer.h> 18 #include <linux/kernel.h> 19 #include <linux/kvm_host.h> 20 #include <linux/module.h> 21 #include <linux/moduleparam.h> 22 #include <linux/mod_devicetable.h> 23 #include <linux/mm.h> 24 #include <linux/objtool.h> 25 #include <linux/sched.h> 26 #include <linux/sched/smt.h> 27 #include <linux/slab.h> 28 #include <linux/tboot.h> 29 #include <linux/trace_events.h> 30 #include <linux/entry-kvm.h> 31 32 #include <asm/apic.h> 33 #include <asm/asm.h> 34 #include <asm/cpu.h> 35 #include <asm/cpu_device_id.h> 36 #include <asm/debugreg.h> 37 #include <asm/desc.h> 38 #include <asm/fpu/api.h> 39 #include <asm/fpu/xstate.h> 40 #include <asm/idtentry.h> 41 #include <asm/io.h> 42 #include <asm/irq_remapping.h> 43 #include <asm/kexec.h> 44 #include <asm/perf_event.h> 45 #include <asm/mmu_context.h> 46 #include <asm/mshyperv.h> 47 #include <asm/mwait.h> 48 #include <asm/spec-ctrl.h> 49 #include <asm/virtext.h> 50 #include <asm/vmx.h> 51 52 #include "capabilities.h" 53 #include "cpuid.h" 54 #include "evmcs.h" 55 #include "hyperv.h" 56 #include "kvm_onhyperv.h" 57 #include "irq.h" 58 #include "kvm_cache_regs.h" 59 #include "lapic.h" 60 #include "mmu.h" 61 #include "nested.h" 62 #include "pmu.h" 63 #include "sgx.h" 64 #include "trace.h" 65 #include "vmcs.h" 66 #include "vmcs12.h" 67 #include "vmx.h" 68 #include "x86.h" 69 70 MODULE_AUTHOR("Qumranet"); 71 MODULE_LICENSE("GPL"); 72 73 #ifdef MODULE 74 static const struct x86_cpu_id vmx_cpu_id[] = { 75 X86_MATCH_FEATURE(X86_FEATURE_VMX, NULL), 76 {} 77 }; 78 MODULE_DEVICE_TABLE(x86cpu, vmx_cpu_id); 79 #endif 80 81 bool __read_mostly enable_vpid = 1; 82 module_param_named(vpid, enable_vpid, bool, 0444); 83 84 static bool __read_mostly enable_vnmi = 1; 85 module_param_named(vnmi, enable_vnmi, bool, S_IRUGO); 86 87 bool __read_mostly flexpriority_enabled = 1; 88 module_param_named(flexpriority, flexpriority_enabled, bool, S_IRUGO); 89 90 bool __read_mostly enable_ept = 1; 91 module_param_named(ept, enable_ept, bool, S_IRUGO); 92 93 bool __read_mostly enable_unrestricted_guest = 1; 94 module_param_named(unrestricted_guest, 95 enable_unrestricted_guest, bool, S_IRUGO); 96 97 bool __read_mostly enable_ept_ad_bits = 1; 98 module_param_named(eptad, enable_ept_ad_bits, bool, S_IRUGO); 99 100 static bool __read_mostly emulate_invalid_guest_state = true; 101 module_param(emulate_invalid_guest_state, bool, S_IRUGO); 102 103 static bool __read_mostly fasteoi = 1; 104 module_param(fasteoi, bool, S_IRUGO); 105 106 module_param(enable_apicv, bool, S_IRUGO); 107 108 bool __read_mostly enable_ipiv = true; 109 module_param(enable_ipiv, bool, 0444); 110 111 /* 112 * If nested=1, nested virtualization is supported, i.e., guests may use 113 * VMX and be a hypervisor for its own guests. If nested=0, guests may not 114 * use VMX instructions. 115 */ 116 static bool __read_mostly nested = 1; 117 module_param(nested, bool, S_IRUGO); 118 119 bool __read_mostly enable_pml = 1; 120 module_param_named(pml, enable_pml, bool, S_IRUGO); 121 122 static bool __read_mostly error_on_inconsistent_vmcs_config = true; 123 module_param(error_on_inconsistent_vmcs_config, bool, 0444); 124 125 static bool __read_mostly dump_invalid_vmcs = 0; 126 module_param(dump_invalid_vmcs, bool, 0644); 127 128 #define MSR_BITMAP_MODE_X2APIC 1 129 #define MSR_BITMAP_MODE_X2APIC_APICV 2 130 131 #define KVM_VMX_TSC_MULTIPLIER_MAX 0xffffffffffffffffULL 132 133 /* Guest_tsc -> host_tsc conversion requires 64-bit division. */ 134 static int __read_mostly cpu_preemption_timer_multi; 135 static bool __read_mostly enable_preemption_timer = 1; 136 #ifdef CONFIG_X86_64 137 module_param_named(preemption_timer, enable_preemption_timer, bool, S_IRUGO); 138 #endif 139 140 extern bool __read_mostly allow_smaller_maxphyaddr; 141 module_param(allow_smaller_maxphyaddr, bool, S_IRUGO); 142 143 #define KVM_VM_CR0_ALWAYS_OFF (X86_CR0_NW | X86_CR0_CD) 144 #define KVM_VM_CR0_ALWAYS_ON_UNRESTRICTED_GUEST X86_CR0_NE 145 #define KVM_VM_CR0_ALWAYS_ON \ 146 (KVM_VM_CR0_ALWAYS_ON_UNRESTRICTED_GUEST | X86_CR0_PG | X86_CR0_PE) 147 148 #define KVM_VM_CR4_ALWAYS_ON_UNRESTRICTED_GUEST X86_CR4_VMXE 149 #define KVM_PMODE_VM_CR4_ALWAYS_ON (X86_CR4_PAE | X86_CR4_VMXE) 150 #define KVM_RMODE_VM_CR4_ALWAYS_ON (X86_CR4_VME | X86_CR4_PAE | X86_CR4_VMXE) 151 152 #define RMODE_GUEST_OWNED_EFLAGS_BITS (~(X86_EFLAGS_IOPL | X86_EFLAGS_VM)) 153 154 #define MSR_IA32_RTIT_STATUS_MASK (~(RTIT_STATUS_FILTEREN | \ 155 RTIT_STATUS_CONTEXTEN | RTIT_STATUS_TRIGGEREN | \ 156 RTIT_STATUS_ERROR | RTIT_STATUS_STOPPED | \ 157 RTIT_STATUS_BYTECNT)) 158 159 /* 160 * List of MSRs that can be directly passed to the guest. 161 * In addition to these x2apic and PT MSRs are handled specially. 162 */ 163 static u32 vmx_possible_passthrough_msrs[MAX_POSSIBLE_PASSTHROUGH_MSRS] = { 164 MSR_IA32_SPEC_CTRL, 165 MSR_IA32_PRED_CMD, 166 MSR_IA32_TSC, 167 #ifdef CONFIG_X86_64 168 MSR_FS_BASE, 169 MSR_GS_BASE, 170 MSR_KERNEL_GS_BASE, 171 MSR_IA32_XFD, 172 MSR_IA32_XFD_ERR, 173 #endif 174 MSR_IA32_SYSENTER_CS, 175 MSR_IA32_SYSENTER_ESP, 176 MSR_IA32_SYSENTER_EIP, 177 MSR_CORE_C1_RES, 178 MSR_CORE_C3_RESIDENCY, 179 MSR_CORE_C6_RESIDENCY, 180 MSR_CORE_C7_RESIDENCY, 181 }; 182 183 /* 184 * These 2 parameters are used to config the controls for Pause-Loop Exiting: 185 * ple_gap: upper bound on the amount of time between two successive 186 * executions of PAUSE in a loop. Also indicate if ple enabled. 187 * According to test, this time is usually smaller than 128 cycles. 188 * ple_window: upper bound on the amount of time a guest is allowed to execute 189 * in a PAUSE loop. Tests indicate that most spinlocks are held for 190 * less than 2^12 cycles 191 * Time is measured based on a counter that runs at the same rate as the TSC, 192 * refer SDM volume 3b section 21.6.13 & 22.1.3. 193 */ 194 static unsigned int ple_gap = KVM_DEFAULT_PLE_GAP; 195 module_param(ple_gap, uint, 0444); 196 197 static unsigned int ple_window = KVM_VMX_DEFAULT_PLE_WINDOW; 198 module_param(ple_window, uint, 0444); 199 200 /* Default doubles per-vcpu window every exit. */ 201 static unsigned int ple_window_grow = KVM_DEFAULT_PLE_WINDOW_GROW; 202 module_param(ple_window_grow, uint, 0444); 203 204 /* Default resets per-vcpu window every exit to ple_window. */ 205 static unsigned int ple_window_shrink = KVM_DEFAULT_PLE_WINDOW_SHRINK; 206 module_param(ple_window_shrink, uint, 0444); 207 208 /* Default is to compute the maximum so we can never overflow. */ 209 static unsigned int ple_window_max = KVM_VMX_DEFAULT_PLE_WINDOW_MAX; 210 module_param(ple_window_max, uint, 0444); 211 212 /* Default is SYSTEM mode, 1 for host-guest mode */ 213 int __read_mostly pt_mode = PT_MODE_SYSTEM; 214 module_param(pt_mode, int, S_IRUGO); 215 216 static DEFINE_STATIC_KEY_FALSE(vmx_l1d_should_flush); 217 static DEFINE_STATIC_KEY_FALSE(vmx_l1d_flush_cond); 218 static DEFINE_MUTEX(vmx_l1d_flush_mutex); 219 220 /* Storage for pre module init parameter parsing */ 221 static enum vmx_l1d_flush_state __read_mostly vmentry_l1d_flush_param = VMENTER_L1D_FLUSH_AUTO; 222 223 static const struct { 224 const char *option; 225 bool for_parse; 226 } vmentry_l1d_param[] = { 227 [VMENTER_L1D_FLUSH_AUTO] = {"auto", true}, 228 [VMENTER_L1D_FLUSH_NEVER] = {"never", true}, 229 [VMENTER_L1D_FLUSH_COND] = {"cond", true}, 230 [VMENTER_L1D_FLUSH_ALWAYS] = {"always", true}, 231 [VMENTER_L1D_FLUSH_EPT_DISABLED] = {"EPT disabled", false}, 232 [VMENTER_L1D_FLUSH_NOT_REQUIRED] = {"not required", false}, 233 }; 234 235 #define L1D_CACHE_ORDER 4 236 static void *vmx_l1d_flush_pages; 237 238 /* Control for disabling CPU Fill buffer clear */ 239 static bool __read_mostly vmx_fb_clear_ctrl_available; 240 241 static int vmx_setup_l1d_flush(enum vmx_l1d_flush_state l1tf) 242 { 243 struct page *page; 244 unsigned int i; 245 246 if (!boot_cpu_has_bug(X86_BUG_L1TF)) { 247 l1tf_vmx_mitigation = VMENTER_L1D_FLUSH_NOT_REQUIRED; 248 return 0; 249 } 250 251 if (!enable_ept) { 252 l1tf_vmx_mitigation = VMENTER_L1D_FLUSH_EPT_DISABLED; 253 return 0; 254 } 255 256 if (boot_cpu_has(X86_FEATURE_ARCH_CAPABILITIES)) { 257 u64 msr; 258 259 rdmsrl(MSR_IA32_ARCH_CAPABILITIES, msr); 260 if (msr & ARCH_CAP_SKIP_VMENTRY_L1DFLUSH) { 261 l1tf_vmx_mitigation = VMENTER_L1D_FLUSH_NOT_REQUIRED; 262 return 0; 263 } 264 } 265 266 /* If set to auto use the default l1tf mitigation method */ 267 if (l1tf == VMENTER_L1D_FLUSH_AUTO) { 268 switch (l1tf_mitigation) { 269 case L1TF_MITIGATION_OFF: 270 l1tf = VMENTER_L1D_FLUSH_NEVER; 271 break; 272 case L1TF_MITIGATION_FLUSH_NOWARN: 273 case L1TF_MITIGATION_FLUSH: 274 case L1TF_MITIGATION_FLUSH_NOSMT: 275 l1tf = VMENTER_L1D_FLUSH_COND; 276 break; 277 case L1TF_MITIGATION_FULL: 278 case L1TF_MITIGATION_FULL_FORCE: 279 l1tf = VMENTER_L1D_FLUSH_ALWAYS; 280 break; 281 } 282 } else if (l1tf_mitigation == L1TF_MITIGATION_FULL_FORCE) { 283 l1tf = VMENTER_L1D_FLUSH_ALWAYS; 284 } 285 286 if (l1tf != VMENTER_L1D_FLUSH_NEVER && !vmx_l1d_flush_pages && 287 !boot_cpu_has(X86_FEATURE_FLUSH_L1D)) { 288 /* 289 * This allocation for vmx_l1d_flush_pages is not tied to a VM 290 * lifetime and so should not be charged to a memcg. 291 */ 292 page = alloc_pages(GFP_KERNEL, L1D_CACHE_ORDER); 293 if (!page) 294 return -ENOMEM; 295 vmx_l1d_flush_pages = page_address(page); 296 297 /* 298 * Initialize each page with a different pattern in 299 * order to protect against KSM in the nested 300 * virtualization case. 301 */ 302 for (i = 0; i < 1u << L1D_CACHE_ORDER; ++i) { 303 memset(vmx_l1d_flush_pages + i * PAGE_SIZE, i + 1, 304 PAGE_SIZE); 305 } 306 } 307 308 l1tf_vmx_mitigation = l1tf; 309 310 if (l1tf != VMENTER_L1D_FLUSH_NEVER) 311 static_branch_enable(&vmx_l1d_should_flush); 312 else 313 static_branch_disable(&vmx_l1d_should_flush); 314 315 if (l1tf == VMENTER_L1D_FLUSH_COND) 316 static_branch_enable(&vmx_l1d_flush_cond); 317 else 318 static_branch_disable(&vmx_l1d_flush_cond); 319 return 0; 320 } 321 322 static int vmentry_l1d_flush_parse(const char *s) 323 { 324 unsigned int i; 325 326 if (s) { 327 for (i = 0; i < ARRAY_SIZE(vmentry_l1d_param); i++) { 328 if (vmentry_l1d_param[i].for_parse && 329 sysfs_streq(s, vmentry_l1d_param[i].option)) 330 return i; 331 } 332 } 333 return -EINVAL; 334 } 335 336 static int vmentry_l1d_flush_set(const char *s, const struct kernel_param *kp) 337 { 338 int l1tf, ret; 339 340 l1tf = vmentry_l1d_flush_parse(s); 341 if (l1tf < 0) 342 return l1tf; 343 344 if (!boot_cpu_has(X86_BUG_L1TF)) 345 return 0; 346 347 /* 348 * Has vmx_init() run already? If not then this is the pre init 349 * parameter parsing. In that case just store the value and let 350 * vmx_init() do the proper setup after enable_ept has been 351 * established. 352 */ 353 if (l1tf_vmx_mitigation == VMENTER_L1D_FLUSH_AUTO) { 354 vmentry_l1d_flush_param = l1tf; 355 return 0; 356 } 357 358 mutex_lock(&vmx_l1d_flush_mutex); 359 ret = vmx_setup_l1d_flush(l1tf); 360 mutex_unlock(&vmx_l1d_flush_mutex); 361 return ret; 362 } 363 364 static int vmentry_l1d_flush_get(char *s, const struct kernel_param *kp) 365 { 366 if (WARN_ON_ONCE(l1tf_vmx_mitigation >= ARRAY_SIZE(vmentry_l1d_param))) 367 return sprintf(s, "???\n"); 368 369 return sprintf(s, "%s\n", vmentry_l1d_param[l1tf_vmx_mitigation].option); 370 } 371 372 static void vmx_setup_fb_clear_ctrl(void) 373 { 374 u64 msr; 375 376 if (boot_cpu_has(X86_FEATURE_ARCH_CAPABILITIES) && 377 !boot_cpu_has_bug(X86_BUG_MDS) && 378 !boot_cpu_has_bug(X86_BUG_TAA)) { 379 rdmsrl(MSR_IA32_ARCH_CAPABILITIES, msr); 380 if (msr & ARCH_CAP_FB_CLEAR_CTRL) 381 vmx_fb_clear_ctrl_available = true; 382 } 383 } 384 385 static __always_inline void vmx_disable_fb_clear(struct vcpu_vmx *vmx) 386 { 387 u64 msr; 388 389 if (!vmx->disable_fb_clear) 390 return; 391 392 msr = __rdmsr(MSR_IA32_MCU_OPT_CTRL); 393 msr |= FB_CLEAR_DIS; 394 native_wrmsrl(MSR_IA32_MCU_OPT_CTRL, msr); 395 /* Cache the MSR value to avoid reading it later */ 396 vmx->msr_ia32_mcu_opt_ctrl = msr; 397 } 398 399 static __always_inline void vmx_enable_fb_clear(struct vcpu_vmx *vmx) 400 { 401 if (!vmx->disable_fb_clear) 402 return; 403 404 vmx->msr_ia32_mcu_opt_ctrl &= ~FB_CLEAR_DIS; 405 native_wrmsrl(MSR_IA32_MCU_OPT_CTRL, vmx->msr_ia32_mcu_opt_ctrl); 406 } 407 408 static void vmx_update_fb_clear_dis(struct kvm_vcpu *vcpu, struct vcpu_vmx *vmx) 409 { 410 vmx->disable_fb_clear = vmx_fb_clear_ctrl_available; 411 412 /* 413 * If guest will not execute VERW, there is no need to set FB_CLEAR_DIS 414 * at VMEntry. Skip the MSR read/write when a guest has no use case to 415 * execute VERW. 416 */ 417 if ((vcpu->arch.arch_capabilities & ARCH_CAP_FB_CLEAR) || 418 ((vcpu->arch.arch_capabilities & ARCH_CAP_MDS_NO) && 419 (vcpu->arch.arch_capabilities & ARCH_CAP_TAA_NO) && 420 (vcpu->arch.arch_capabilities & ARCH_CAP_PSDP_NO) && 421 (vcpu->arch.arch_capabilities & ARCH_CAP_FBSDP_NO) && 422 (vcpu->arch.arch_capabilities & ARCH_CAP_SBDR_SSDP_NO))) 423 vmx->disable_fb_clear = false; 424 } 425 426 static const struct kernel_param_ops vmentry_l1d_flush_ops = { 427 .set = vmentry_l1d_flush_set, 428 .get = vmentry_l1d_flush_get, 429 }; 430 module_param_cb(vmentry_l1d_flush, &vmentry_l1d_flush_ops, NULL, 0644); 431 432 static u32 vmx_segment_access_rights(struct kvm_segment *var); 433 434 void vmx_vmexit(void); 435 436 #define vmx_insn_failed(fmt...) \ 437 do { \ 438 WARN_ONCE(1, fmt); \ 439 pr_warn_ratelimited(fmt); \ 440 } while (0) 441 442 void vmread_error(unsigned long field, bool fault) 443 { 444 if (fault) 445 kvm_spurious_fault(); 446 else 447 vmx_insn_failed("kvm: vmread failed: field=%lx\n", field); 448 } 449 450 noinline void vmwrite_error(unsigned long field, unsigned long value) 451 { 452 vmx_insn_failed("kvm: vmwrite failed: field=%lx val=%lx err=%u\n", 453 field, value, vmcs_read32(VM_INSTRUCTION_ERROR)); 454 } 455 456 noinline void vmclear_error(struct vmcs *vmcs, u64 phys_addr) 457 { 458 vmx_insn_failed("kvm: vmclear failed: %p/%llx err=%u\n", 459 vmcs, phys_addr, vmcs_read32(VM_INSTRUCTION_ERROR)); 460 } 461 462 noinline void vmptrld_error(struct vmcs *vmcs, u64 phys_addr) 463 { 464 vmx_insn_failed("kvm: vmptrld failed: %p/%llx err=%u\n", 465 vmcs, phys_addr, vmcs_read32(VM_INSTRUCTION_ERROR)); 466 } 467 468 noinline void invvpid_error(unsigned long ext, u16 vpid, gva_t gva) 469 { 470 vmx_insn_failed("kvm: invvpid failed: ext=0x%lx vpid=%u gva=0x%lx\n", 471 ext, vpid, gva); 472 } 473 474 noinline void invept_error(unsigned long ext, u64 eptp, gpa_t gpa) 475 { 476 vmx_insn_failed("kvm: invept failed: ext=0x%lx eptp=%llx gpa=0x%llx\n", 477 ext, eptp, gpa); 478 } 479 480 static DEFINE_PER_CPU(struct vmcs *, vmxarea); 481 DEFINE_PER_CPU(struct vmcs *, current_vmcs); 482 /* 483 * We maintain a per-CPU linked-list of VMCS loaded on that CPU. This is needed 484 * when a CPU is brought down, and we need to VMCLEAR all VMCSs loaded on it. 485 */ 486 static DEFINE_PER_CPU(struct list_head, loaded_vmcss_on_cpu); 487 488 static DECLARE_BITMAP(vmx_vpid_bitmap, VMX_NR_VPIDS); 489 static DEFINE_SPINLOCK(vmx_vpid_lock); 490 491 struct vmcs_config vmcs_config; 492 struct vmx_capability vmx_capability; 493 494 #define VMX_SEGMENT_FIELD(seg) \ 495 [VCPU_SREG_##seg] = { \ 496 .selector = GUEST_##seg##_SELECTOR, \ 497 .base = GUEST_##seg##_BASE, \ 498 .limit = GUEST_##seg##_LIMIT, \ 499 .ar_bytes = GUEST_##seg##_AR_BYTES, \ 500 } 501 502 static const struct kvm_vmx_segment_field { 503 unsigned selector; 504 unsigned base; 505 unsigned limit; 506 unsigned ar_bytes; 507 } kvm_vmx_segment_fields[] = { 508 VMX_SEGMENT_FIELD(CS), 509 VMX_SEGMENT_FIELD(DS), 510 VMX_SEGMENT_FIELD(ES), 511 VMX_SEGMENT_FIELD(FS), 512 VMX_SEGMENT_FIELD(GS), 513 VMX_SEGMENT_FIELD(SS), 514 VMX_SEGMENT_FIELD(TR), 515 VMX_SEGMENT_FIELD(LDTR), 516 }; 517 518 static inline void vmx_segment_cache_clear(struct vcpu_vmx *vmx) 519 { 520 vmx->segment_cache.bitmask = 0; 521 } 522 523 static unsigned long host_idt_base; 524 525 #if IS_ENABLED(CONFIG_HYPERV) 526 static bool __read_mostly enlightened_vmcs = true; 527 module_param(enlightened_vmcs, bool, 0444); 528 529 static int hv_enable_direct_tlbflush(struct kvm_vcpu *vcpu) 530 { 531 struct hv_enlightened_vmcs *evmcs; 532 struct hv_partition_assist_pg **p_hv_pa_pg = 533 &to_kvm_hv(vcpu->kvm)->hv_pa_pg; 534 /* 535 * Synthetic VM-Exit is not enabled in current code and so All 536 * evmcs in singe VM shares same assist page. 537 */ 538 if (!*p_hv_pa_pg) 539 *p_hv_pa_pg = kzalloc(PAGE_SIZE, GFP_KERNEL_ACCOUNT); 540 541 if (!*p_hv_pa_pg) 542 return -ENOMEM; 543 544 evmcs = (struct hv_enlightened_vmcs *)to_vmx(vcpu)->loaded_vmcs->vmcs; 545 546 evmcs->partition_assist_page = 547 __pa(*p_hv_pa_pg); 548 evmcs->hv_vm_id = (unsigned long)vcpu->kvm; 549 evmcs->hv_enlightenments_control.nested_flush_hypercall = 1; 550 551 return 0; 552 } 553 554 #endif /* IS_ENABLED(CONFIG_HYPERV) */ 555 556 /* 557 * Comment's format: document - errata name - stepping - processor name. 558 * Refer from 559 * https://www.virtualbox.org/svn/vbox/trunk/src/VBox/VMM/VMMR0/HMR0.cpp 560 */ 561 static u32 vmx_preemption_cpu_tfms[] = { 562 /* 323344.pdf - BA86 - D0 - Xeon 7500 Series */ 563 0x000206E6, 564 /* 323056.pdf - AAX65 - C2 - Xeon L3406 */ 565 /* 322814.pdf - AAT59 - C2 - i7-600, i5-500, i5-400 and i3-300 Mobile */ 566 /* 322911.pdf - AAU65 - C2 - i5-600, i3-500 Desktop and Pentium G6950 */ 567 0x00020652, 568 /* 322911.pdf - AAU65 - K0 - i5-600, i3-500 Desktop and Pentium G6950 */ 569 0x00020655, 570 /* 322373.pdf - AAO95 - B1 - Xeon 3400 Series */ 571 /* 322166.pdf - AAN92 - B1 - i7-800 and i5-700 Desktop */ 572 /* 573 * 320767.pdf - AAP86 - B1 - 574 * i7-900 Mobile Extreme, i7-800 and i7-700 Mobile 575 */ 576 0x000106E5, 577 /* 321333.pdf - AAM126 - C0 - Xeon 3500 */ 578 0x000106A0, 579 /* 321333.pdf - AAM126 - C1 - Xeon 3500 */ 580 0x000106A1, 581 /* 320836.pdf - AAJ124 - C0 - i7-900 Desktop Extreme and i7-900 Desktop */ 582 0x000106A4, 583 /* 321333.pdf - AAM126 - D0 - Xeon 3500 */ 584 /* 321324.pdf - AAK139 - D0 - Xeon 5500 */ 585 /* 320836.pdf - AAJ124 - D0 - i7-900 Extreme and i7-900 Desktop */ 586 0x000106A5, 587 /* Xeon E3-1220 V2 */ 588 0x000306A8, 589 }; 590 591 static inline bool cpu_has_broken_vmx_preemption_timer(void) 592 { 593 u32 eax = cpuid_eax(0x00000001), i; 594 595 /* Clear the reserved bits */ 596 eax &= ~(0x3U << 14 | 0xfU << 28); 597 for (i = 0; i < ARRAY_SIZE(vmx_preemption_cpu_tfms); i++) 598 if (eax == vmx_preemption_cpu_tfms[i]) 599 return true; 600 601 return false; 602 } 603 604 static inline bool cpu_need_virtualize_apic_accesses(struct kvm_vcpu *vcpu) 605 { 606 return flexpriority_enabled && lapic_in_kernel(vcpu); 607 } 608 609 static int possible_passthrough_msr_slot(u32 msr) 610 { 611 u32 i; 612 613 for (i = 0; i < ARRAY_SIZE(vmx_possible_passthrough_msrs); i++) 614 if (vmx_possible_passthrough_msrs[i] == msr) 615 return i; 616 617 return -ENOENT; 618 } 619 620 static bool is_valid_passthrough_msr(u32 msr) 621 { 622 bool r; 623 624 switch (msr) { 625 case 0x800 ... 0x8ff: 626 /* x2APIC MSRs. These are handled in vmx_update_msr_bitmap_x2apic() */ 627 return true; 628 case MSR_IA32_RTIT_STATUS: 629 case MSR_IA32_RTIT_OUTPUT_BASE: 630 case MSR_IA32_RTIT_OUTPUT_MASK: 631 case MSR_IA32_RTIT_CR3_MATCH: 632 case MSR_IA32_RTIT_ADDR0_A ... MSR_IA32_RTIT_ADDR3_B: 633 /* PT MSRs. These are handled in pt_update_intercept_for_msr() */ 634 case MSR_LBR_SELECT: 635 case MSR_LBR_TOS: 636 case MSR_LBR_INFO_0 ... MSR_LBR_INFO_0 + 31: 637 case MSR_LBR_NHM_FROM ... MSR_LBR_NHM_FROM + 31: 638 case MSR_LBR_NHM_TO ... MSR_LBR_NHM_TO + 31: 639 case MSR_LBR_CORE_FROM ... MSR_LBR_CORE_FROM + 8: 640 case MSR_LBR_CORE_TO ... MSR_LBR_CORE_TO + 8: 641 /* LBR MSRs. These are handled in vmx_update_intercept_for_lbr_msrs() */ 642 return true; 643 } 644 645 r = possible_passthrough_msr_slot(msr) != -ENOENT; 646 647 WARN(!r, "Invalid MSR %x, please adapt vmx_possible_passthrough_msrs[]", msr); 648 649 return r; 650 } 651 652 struct vmx_uret_msr *vmx_find_uret_msr(struct vcpu_vmx *vmx, u32 msr) 653 { 654 int i; 655 656 i = kvm_find_user_return_msr(msr); 657 if (i >= 0) 658 return &vmx->guest_uret_msrs[i]; 659 return NULL; 660 } 661 662 static int vmx_set_guest_uret_msr(struct vcpu_vmx *vmx, 663 struct vmx_uret_msr *msr, u64 data) 664 { 665 unsigned int slot = msr - vmx->guest_uret_msrs; 666 int ret = 0; 667 668 if (msr->load_into_hardware) { 669 preempt_disable(); 670 ret = kvm_set_user_return_msr(slot, data, msr->mask); 671 preempt_enable(); 672 } 673 if (!ret) 674 msr->data = data; 675 return ret; 676 } 677 678 #ifdef CONFIG_KEXEC_CORE 679 static void crash_vmclear_local_loaded_vmcss(void) 680 { 681 int cpu = raw_smp_processor_id(); 682 struct loaded_vmcs *v; 683 684 list_for_each_entry(v, &per_cpu(loaded_vmcss_on_cpu, cpu), 685 loaded_vmcss_on_cpu_link) 686 vmcs_clear(v->vmcs); 687 } 688 #endif /* CONFIG_KEXEC_CORE */ 689 690 static void __loaded_vmcs_clear(void *arg) 691 { 692 struct loaded_vmcs *loaded_vmcs = arg; 693 int cpu = raw_smp_processor_id(); 694 695 if (loaded_vmcs->cpu != cpu) 696 return; /* vcpu migration can race with cpu offline */ 697 if (per_cpu(current_vmcs, cpu) == loaded_vmcs->vmcs) 698 per_cpu(current_vmcs, cpu) = NULL; 699 700 vmcs_clear(loaded_vmcs->vmcs); 701 if (loaded_vmcs->shadow_vmcs && loaded_vmcs->launched) 702 vmcs_clear(loaded_vmcs->shadow_vmcs); 703 704 list_del(&loaded_vmcs->loaded_vmcss_on_cpu_link); 705 706 /* 707 * Ensure all writes to loaded_vmcs, including deleting it from its 708 * current percpu list, complete before setting loaded_vmcs->cpu to 709 * -1, otherwise a different cpu can see loaded_vmcs->cpu == -1 first 710 * and add loaded_vmcs to its percpu list before it's deleted from this 711 * cpu's list. Pairs with the smp_rmb() in vmx_vcpu_load_vmcs(). 712 */ 713 smp_wmb(); 714 715 loaded_vmcs->cpu = -1; 716 loaded_vmcs->launched = 0; 717 } 718 719 void loaded_vmcs_clear(struct loaded_vmcs *loaded_vmcs) 720 { 721 int cpu = loaded_vmcs->cpu; 722 723 if (cpu != -1) 724 smp_call_function_single(cpu, 725 __loaded_vmcs_clear, loaded_vmcs, 1); 726 } 727 728 static bool vmx_segment_cache_test_set(struct vcpu_vmx *vmx, unsigned seg, 729 unsigned field) 730 { 731 bool ret; 732 u32 mask = 1 << (seg * SEG_FIELD_NR + field); 733 734 if (!kvm_register_is_available(&vmx->vcpu, VCPU_EXREG_SEGMENTS)) { 735 kvm_register_mark_available(&vmx->vcpu, VCPU_EXREG_SEGMENTS); 736 vmx->segment_cache.bitmask = 0; 737 } 738 ret = vmx->segment_cache.bitmask & mask; 739 vmx->segment_cache.bitmask |= mask; 740 return ret; 741 } 742 743 static u16 vmx_read_guest_seg_selector(struct vcpu_vmx *vmx, unsigned seg) 744 { 745 u16 *p = &vmx->segment_cache.seg[seg].selector; 746 747 if (!vmx_segment_cache_test_set(vmx, seg, SEG_FIELD_SEL)) 748 *p = vmcs_read16(kvm_vmx_segment_fields[seg].selector); 749 return *p; 750 } 751 752 static ulong vmx_read_guest_seg_base(struct vcpu_vmx *vmx, unsigned seg) 753 { 754 ulong *p = &vmx->segment_cache.seg[seg].base; 755 756 if (!vmx_segment_cache_test_set(vmx, seg, SEG_FIELD_BASE)) 757 *p = vmcs_readl(kvm_vmx_segment_fields[seg].base); 758 return *p; 759 } 760 761 static u32 vmx_read_guest_seg_limit(struct vcpu_vmx *vmx, unsigned seg) 762 { 763 u32 *p = &vmx->segment_cache.seg[seg].limit; 764 765 if (!vmx_segment_cache_test_set(vmx, seg, SEG_FIELD_LIMIT)) 766 *p = vmcs_read32(kvm_vmx_segment_fields[seg].limit); 767 return *p; 768 } 769 770 static u32 vmx_read_guest_seg_ar(struct vcpu_vmx *vmx, unsigned seg) 771 { 772 u32 *p = &vmx->segment_cache.seg[seg].ar; 773 774 if (!vmx_segment_cache_test_set(vmx, seg, SEG_FIELD_AR)) 775 *p = vmcs_read32(kvm_vmx_segment_fields[seg].ar_bytes); 776 return *p; 777 } 778 779 void vmx_update_exception_bitmap(struct kvm_vcpu *vcpu) 780 { 781 u32 eb; 782 783 eb = (1u << PF_VECTOR) | (1u << UD_VECTOR) | (1u << MC_VECTOR) | 784 (1u << DB_VECTOR) | (1u << AC_VECTOR); 785 /* 786 * Guest access to VMware backdoor ports could legitimately 787 * trigger #GP because of TSS I/O permission bitmap. 788 * We intercept those #GP and allow access to them anyway 789 * as VMware does. 790 */ 791 if (enable_vmware_backdoor) 792 eb |= (1u << GP_VECTOR); 793 if ((vcpu->guest_debug & 794 (KVM_GUESTDBG_ENABLE | KVM_GUESTDBG_USE_SW_BP)) == 795 (KVM_GUESTDBG_ENABLE | KVM_GUESTDBG_USE_SW_BP)) 796 eb |= 1u << BP_VECTOR; 797 if (to_vmx(vcpu)->rmode.vm86_active) 798 eb = ~0; 799 if (!vmx_need_pf_intercept(vcpu)) 800 eb &= ~(1u << PF_VECTOR); 801 802 /* When we are running a nested L2 guest and L1 specified for it a 803 * certain exception bitmap, we must trap the same exceptions and pass 804 * them to L1. When running L2, we will only handle the exceptions 805 * specified above if L1 did not want them. 806 */ 807 if (is_guest_mode(vcpu)) 808 eb |= get_vmcs12(vcpu)->exception_bitmap; 809 else { 810 int mask = 0, match = 0; 811 812 if (enable_ept && (eb & (1u << PF_VECTOR))) { 813 /* 814 * If EPT is enabled, #PF is currently only intercepted 815 * if MAXPHYADDR is smaller on the guest than on the 816 * host. In that case we only care about present, 817 * non-reserved faults. For vmcs02, however, PFEC_MASK 818 * and PFEC_MATCH are set in prepare_vmcs02_rare. 819 */ 820 mask = PFERR_PRESENT_MASK | PFERR_RSVD_MASK; 821 match = PFERR_PRESENT_MASK; 822 } 823 vmcs_write32(PAGE_FAULT_ERROR_CODE_MASK, mask); 824 vmcs_write32(PAGE_FAULT_ERROR_CODE_MATCH, match); 825 } 826 827 /* 828 * Disabling xfd interception indicates that dynamic xfeatures 829 * might be used in the guest. Always trap #NM in this case 830 * to save guest xfd_err timely. 831 */ 832 if (vcpu->arch.xfd_no_write_intercept) 833 eb |= (1u << NM_VECTOR); 834 835 vmcs_write32(EXCEPTION_BITMAP, eb); 836 } 837 838 /* 839 * Check if MSR is intercepted for currently loaded MSR bitmap. 840 */ 841 static bool msr_write_intercepted(struct vcpu_vmx *vmx, u32 msr) 842 { 843 if (!(exec_controls_get(vmx) & CPU_BASED_USE_MSR_BITMAPS)) 844 return true; 845 846 return vmx_test_msr_bitmap_write(vmx->loaded_vmcs->msr_bitmap, msr); 847 } 848 849 unsigned int __vmx_vcpu_run_flags(struct vcpu_vmx *vmx) 850 { 851 unsigned int flags = 0; 852 853 if (vmx->loaded_vmcs->launched) 854 flags |= VMX_RUN_VMRESUME; 855 856 /* 857 * If writes to the SPEC_CTRL MSR aren't intercepted, the guest is free 858 * to change it directly without causing a vmexit. In that case read 859 * it after vmexit and store it in vmx->spec_ctrl. 860 */ 861 if (unlikely(!msr_write_intercepted(vmx, MSR_IA32_SPEC_CTRL))) 862 flags |= VMX_RUN_SAVE_SPEC_CTRL; 863 864 return flags; 865 } 866 867 static __always_inline void clear_atomic_switch_msr_special(struct vcpu_vmx *vmx, 868 unsigned long entry, unsigned long exit) 869 { 870 vm_entry_controls_clearbit(vmx, entry); 871 vm_exit_controls_clearbit(vmx, exit); 872 } 873 874 int vmx_find_loadstore_msr_slot(struct vmx_msrs *m, u32 msr) 875 { 876 unsigned int i; 877 878 for (i = 0; i < m->nr; ++i) { 879 if (m->val[i].index == msr) 880 return i; 881 } 882 return -ENOENT; 883 } 884 885 static void clear_atomic_switch_msr(struct vcpu_vmx *vmx, unsigned msr) 886 { 887 int i; 888 struct msr_autoload *m = &vmx->msr_autoload; 889 890 switch (msr) { 891 case MSR_EFER: 892 if (cpu_has_load_ia32_efer()) { 893 clear_atomic_switch_msr_special(vmx, 894 VM_ENTRY_LOAD_IA32_EFER, 895 VM_EXIT_LOAD_IA32_EFER); 896 return; 897 } 898 break; 899 case MSR_CORE_PERF_GLOBAL_CTRL: 900 if (cpu_has_load_perf_global_ctrl()) { 901 clear_atomic_switch_msr_special(vmx, 902 VM_ENTRY_LOAD_IA32_PERF_GLOBAL_CTRL, 903 VM_EXIT_LOAD_IA32_PERF_GLOBAL_CTRL); 904 return; 905 } 906 break; 907 } 908 i = vmx_find_loadstore_msr_slot(&m->guest, msr); 909 if (i < 0) 910 goto skip_guest; 911 --m->guest.nr; 912 m->guest.val[i] = m->guest.val[m->guest.nr]; 913 vmcs_write32(VM_ENTRY_MSR_LOAD_COUNT, m->guest.nr); 914 915 skip_guest: 916 i = vmx_find_loadstore_msr_slot(&m->host, msr); 917 if (i < 0) 918 return; 919 920 --m->host.nr; 921 m->host.val[i] = m->host.val[m->host.nr]; 922 vmcs_write32(VM_EXIT_MSR_LOAD_COUNT, m->host.nr); 923 } 924 925 static __always_inline void add_atomic_switch_msr_special(struct vcpu_vmx *vmx, 926 unsigned long entry, unsigned long exit, 927 unsigned long guest_val_vmcs, unsigned long host_val_vmcs, 928 u64 guest_val, u64 host_val) 929 { 930 vmcs_write64(guest_val_vmcs, guest_val); 931 if (host_val_vmcs != HOST_IA32_EFER) 932 vmcs_write64(host_val_vmcs, host_val); 933 vm_entry_controls_setbit(vmx, entry); 934 vm_exit_controls_setbit(vmx, exit); 935 } 936 937 static void add_atomic_switch_msr(struct vcpu_vmx *vmx, unsigned msr, 938 u64 guest_val, u64 host_val, bool entry_only) 939 { 940 int i, j = 0; 941 struct msr_autoload *m = &vmx->msr_autoload; 942 943 switch (msr) { 944 case MSR_EFER: 945 if (cpu_has_load_ia32_efer()) { 946 add_atomic_switch_msr_special(vmx, 947 VM_ENTRY_LOAD_IA32_EFER, 948 VM_EXIT_LOAD_IA32_EFER, 949 GUEST_IA32_EFER, 950 HOST_IA32_EFER, 951 guest_val, host_val); 952 return; 953 } 954 break; 955 case MSR_CORE_PERF_GLOBAL_CTRL: 956 if (cpu_has_load_perf_global_ctrl()) { 957 add_atomic_switch_msr_special(vmx, 958 VM_ENTRY_LOAD_IA32_PERF_GLOBAL_CTRL, 959 VM_EXIT_LOAD_IA32_PERF_GLOBAL_CTRL, 960 GUEST_IA32_PERF_GLOBAL_CTRL, 961 HOST_IA32_PERF_GLOBAL_CTRL, 962 guest_val, host_val); 963 return; 964 } 965 break; 966 case MSR_IA32_PEBS_ENABLE: 967 /* PEBS needs a quiescent period after being disabled (to write 968 * a record). Disabling PEBS through VMX MSR swapping doesn't 969 * provide that period, so a CPU could write host's record into 970 * guest's memory. 971 */ 972 wrmsrl(MSR_IA32_PEBS_ENABLE, 0); 973 } 974 975 i = vmx_find_loadstore_msr_slot(&m->guest, msr); 976 if (!entry_only) 977 j = vmx_find_loadstore_msr_slot(&m->host, msr); 978 979 if ((i < 0 && m->guest.nr == MAX_NR_LOADSTORE_MSRS) || 980 (j < 0 && m->host.nr == MAX_NR_LOADSTORE_MSRS)) { 981 printk_once(KERN_WARNING "Not enough msr switch entries. " 982 "Can't add msr %x\n", msr); 983 return; 984 } 985 if (i < 0) { 986 i = m->guest.nr++; 987 vmcs_write32(VM_ENTRY_MSR_LOAD_COUNT, m->guest.nr); 988 } 989 m->guest.val[i].index = msr; 990 m->guest.val[i].value = guest_val; 991 992 if (entry_only) 993 return; 994 995 if (j < 0) { 996 j = m->host.nr++; 997 vmcs_write32(VM_EXIT_MSR_LOAD_COUNT, m->host.nr); 998 } 999 m->host.val[j].index = msr; 1000 m->host.val[j].value = host_val; 1001 } 1002 1003 static bool update_transition_efer(struct vcpu_vmx *vmx) 1004 { 1005 u64 guest_efer = vmx->vcpu.arch.efer; 1006 u64 ignore_bits = 0; 1007 int i; 1008 1009 /* Shadow paging assumes NX to be available. */ 1010 if (!enable_ept) 1011 guest_efer |= EFER_NX; 1012 1013 /* 1014 * LMA and LME handled by hardware; SCE meaningless outside long mode. 1015 */ 1016 ignore_bits |= EFER_SCE; 1017 #ifdef CONFIG_X86_64 1018 ignore_bits |= EFER_LMA | EFER_LME; 1019 /* SCE is meaningful only in long mode on Intel */ 1020 if (guest_efer & EFER_LMA) 1021 ignore_bits &= ~(u64)EFER_SCE; 1022 #endif 1023 1024 /* 1025 * On EPT, we can't emulate NX, so we must switch EFER atomically. 1026 * On CPUs that support "load IA32_EFER", always switch EFER 1027 * atomically, since it's faster than switching it manually. 1028 */ 1029 if (cpu_has_load_ia32_efer() || 1030 (enable_ept && ((vmx->vcpu.arch.efer ^ host_efer) & EFER_NX))) { 1031 if (!(guest_efer & EFER_LMA)) 1032 guest_efer &= ~EFER_LME; 1033 if (guest_efer != host_efer) 1034 add_atomic_switch_msr(vmx, MSR_EFER, 1035 guest_efer, host_efer, false); 1036 else 1037 clear_atomic_switch_msr(vmx, MSR_EFER); 1038 return false; 1039 } 1040 1041 i = kvm_find_user_return_msr(MSR_EFER); 1042 if (i < 0) 1043 return false; 1044 1045 clear_atomic_switch_msr(vmx, MSR_EFER); 1046 1047 guest_efer &= ~ignore_bits; 1048 guest_efer |= host_efer & ignore_bits; 1049 1050 vmx->guest_uret_msrs[i].data = guest_efer; 1051 vmx->guest_uret_msrs[i].mask = ~ignore_bits; 1052 1053 return true; 1054 } 1055 1056 #ifdef CONFIG_X86_32 1057 /* 1058 * On 32-bit kernels, VM exits still load the FS and GS bases from the 1059 * VMCS rather than the segment table. KVM uses this helper to figure 1060 * out the current bases to poke them into the VMCS before entry. 1061 */ 1062 static unsigned long segment_base(u16 selector) 1063 { 1064 struct desc_struct *table; 1065 unsigned long v; 1066 1067 if (!(selector & ~SEGMENT_RPL_MASK)) 1068 return 0; 1069 1070 table = get_current_gdt_ro(); 1071 1072 if ((selector & SEGMENT_TI_MASK) == SEGMENT_LDT) { 1073 u16 ldt_selector = kvm_read_ldt(); 1074 1075 if (!(ldt_selector & ~SEGMENT_RPL_MASK)) 1076 return 0; 1077 1078 table = (struct desc_struct *)segment_base(ldt_selector); 1079 } 1080 v = get_desc_base(&table[selector >> 3]); 1081 return v; 1082 } 1083 #endif 1084 1085 static inline bool pt_can_write_msr(struct vcpu_vmx *vmx) 1086 { 1087 return vmx_pt_mode_is_host_guest() && 1088 !(vmx->pt_desc.guest.ctl & RTIT_CTL_TRACEEN); 1089 } 1090 1091 static inline bool pt_output_base_valid(struct kvm_vcpu *vcpu, u64 base) 1092 { 1093 /* The base must be 128-byte aligned and a legal physical address. */ 1094 return kvm_vcpu_is_legal_aligned_gpa(vcpu, base, 128); 1095 } 1096 1097 static inline void pt_load_msr(struct pt_ctx *ctx, u32 addr_range) 1098 { 1099 u32 i; 1100 1101 wrmsrl(MSR_IA32_RTIT_STATUS, ctx->status); 1102 wrmsrl(MSR_IA32_RTIT_OUTPUT_BASE, ctx->output_base); 1103 wrmsrl(MSR_IA32_RTIT_OUTPUT_MASK, ctx->output_mask); 1104 wrmsrl(MSR_IA32_RTIT_CR3_MATCH, ctx->cr3_match); 1105 for (i = 0; i < addr_range; i++) { 1106 wrmsrl(MSR_IA32_RTIT_ADDR0_A + i * 2, ctx->addr_a[i]); 1107 wrmsrl(MSR_IA32_RTIT_ADDR0_B + i * 2, ctx->addr_b[i]); 1108 } 1109 } 1110 1111 static inline void pt_save_msr(struct pt_ctx *ctx, u32 addr_range) 1112 { 1113 u32 i; 1114 1115 rdmsrl(MSR_IA32_RTIT_STATUS, ctx->status); 1116 rdmsrl(MSR_IA32_RTIT_OUTPUT_BASE, ctx->output_base); 1117 rdmsrl(MSR_IA32_RTIT_OUTPUT_MASK, ctx->output_mask); 1118 rdmsrl(MSR_IA32_RTIT_CR3_MATCH, ctx->cr3_match); 1119 for (i = 0; i < addr_range; i++) { 1120 rdmsrl(MSR_IA32_RTIT_ADDR0_A + i * 2, ctx->addr_a[i]); 1121 rdmsrl(MSR_IA32_RTIT_ADDR0_B + i * 2, ctx->addr_b[i]); 1122 } 1123 } 1124 1125 static void pt_guest_enter(struct vcpu_vmx *vmx) 1126 { 1127 if (vmx_pt_mode_is_system()) 1128 return; 1129 1130 /* 1131 * GUEST_IA32_RTIT_CTL is already set in the VMCS. 1132 * Save host state before VM entry. 1133 */ 1134 rdmsrl(MSR_IA32_RTIT_CTL, vmx->pt_desc.host.ctl); 1135 if (vmx->pt_desc.guest.ctl & RTIT_CTL_TRACEEN) { 1136 wrmsrl(MSR_IA32_RTIT_CTL, 0); 1137 pt_save_msr(&vmx->pt_desc.host, vmx->pt_desc.num_address_ranges); 1138 pt_load_msr(&vmx->pt_desc.guest, vmx->pt_desc.num_address_ranges); 1139 } 1140 } 1141 1142 static void pt_guest_exit(struct vcpu_vmx *vmx) 1143 { 1144 if (vmx_pt_mode_is_system()) 1145 return; 1146 1147 if (vmx->pt_desc.guest.ctl & RTIT_CTL_TRACEEN) { 1148 pt_save_msr(&vmx->pt_desc.guest, vmx->pt_desc.num_address_ranges); 1149 pt_load_msr(&vmx->pt_desc.host, vmx->pt_desc.num_address_ranges); 1150 } 1151 1152 /* 1153 * KVM requires VM_EXIT_CLEAR_IA32_RTIT_CTL to expose PT to the guest, 1154 * i.e. RTIT_CTL is always cleared on VM-Exit. Restore it if necessary. 1155 */ 1156 if (vmx->pt_desc.host.ctl) 1157 wrmsrl(MSR_IA32_RTIT_CTL, vmx->pt_desc.host.ctl); 1158 } 1159 1160 void vmx_set_host_fs_gs(struct vmcs_host_state *host, u16 fs_sel, u16 gs_sel, 1161 unsigned long fs_base, unsigned long gs_base) 1162 { 1163 if (unlikely(fs_sel != host->fs_sel)) { 1164 if (!(fs_sel & 7)) 1165 vmcs_write16(HOST_FS_SELECTOR, fs_sel); 1166 else 1167 vmcs_write16(HOST_FS_SELECTOR, 0); 1168 host->fs_sel = fs_sel; 1169 } 1170 if (unlikely(gs_sel != host->gs_sel)) { 1171 if (!(gs_sel & 7)) 1172 vmcs_write16(HOST_GS_SELECTOR, gs_sel); 1173 else 1174 vmcs_write16(HOST_GS_SELECTOR, 0); 1175 host->gs_sel = gs_sel; 1176 } 1177 if (unlikely(fs_base != host->fs_base)) { 1178 vmcs_writel(HOST_FS_BASE, fs_base); 1179 host->fs_base = fs_base; 1180 } 1181 if (unlikely(gs_base != host->gs_base)) { 1182 vmcs_writel(HOST_GS_BASE, gs_base); 1183 host->gs_base = gs_base; 1184 } 1185 } 1186 1187 void vmx_prepare_switch_to_guest(struct kvm_vcpu *vcpu) 1188 { 1189 struct vcpu_vmx *vmx = to_vmx(vcpu); 1190 struct vmcs_host_state *host_state; 1191 #ifdef CONFIG_X86_64 1192 int cpu = raw_smp_processor_id(); 1193 #endif 1194 unsigned long fs_base, gs_base; 1195 u16 fs_sel, gs_sel; 1196 int i; 1197 1198 vmx->req_immediate_exit = false; 1199 1200 /* 1201 * Note that guest MSRs to be saved/restored can also be changed 1202 * when guest state is loaded. This happens when guest transitions 1203 * to/from long-mode by setting MSR_EFER.LMA. 1204 */ 1205 if (!vmx->guest_uret_msrs_loaded) { 1206 vmx->guest_uret_msrs_loaded = true; 1207 for (i = 0; i < kvm_nr_uret_msrs; ++i) { 1208 if (!vmx->guest_uret_msrs[i].load_into_hardware) 1209 continue; 1210 1211 kvm_set_user_return_msr(i, 1212 vmx->guest_uret_msrs[i].data, 1213 vmx->guest_uret_msrs[i].mask); 1214 } 1215 } 1216 1217 if (vmx->nested.need_vmcs12_to_shadow_sync) 1218 nested_sync_vmcs12_to_shadow(vcpu); 1219 1220 if (vmx->guest_state_loaded) 1221 return; 1222 1223 host_state = &vmx->loaded_vmcs->host_state; 1224 1225 /* 1226 * Set host fs and gs selectors. Unfortunately, 22.2.3 does not 1227 * allow segment selectors with cpl > 0 or ti == 1. 1228 */ 1229 host_state->ldt_sel = kvm_read_ldt(); 1230 1231 #ifdef CONFIG_X86_64 1232 savesegment(ds, host_state->ds_sel); 1233 savesegment(es, host_state->es_sel); 1234 1235 gs_base = cpu_kernelmode_gs_base(cpu); 1236 if (likely(is_64bit_mm(current->mm))) { 1237 current_save_fsgs(); 1238 fs_sel = current->thread.fsindex; 1239 gs_sel = current->thread.gsindex; 1240 fs_base = current->thread.fsbase; 1241 vmx->msr_host_kernel_gs_base = current->thread.gsbase; 1242 } else { 1243 savesegment(fs, fs_sel); 1244 savesegment(gs, gs_sel); 1245 fs_base = read_msr(MSR_FS_BASE); 1246 vmx->msr_host_kernel_gs_base = read_msr(MSR_KERNEL_GS_BASE); 1247 } 1248 1249 wrmsrl(MSR_KERNEL_GS_BASE, vmx->msr_guest_kernel_gs_base); 1250 #else 1251 savesegment(fs, fs_sel); 1252 savesegment(gs, gs_sel); 1253 fs_base = segment_base(fs_sel); 1254 gs_base = segment_base(gs_sel); 1255 #endif 1256 1257 vmx_set_host_fs_gs(host_state, fs_sel, gs_sel, fs_base, gs_base); 1258 vmx->guest_state_loaded = true; 1259 } 1260 1261 static void vmx_prepare_switch_to_host(struct vcpu_vmx *vmx) 1262 { 1263 struct vmcs_host_state *host_state; 1264 1265 if (!vmx->guest_state_loaded) 1266 return; 1267 1268 host_state = &vmx->loaded_vmcs->host_state; 1269 1270 ++vmx->vcpu.stat.host_state_reload; 1271 1272 #ifdef CONFIG_X86_64 1273 rdmsrl(MSR_KERNEL_GS_BASE, vmx->msr_guest_kernel_gs_base); 1274 #endif 1275 if (host_state->ldt_sel || (host_state->gs_sel & 7)) { 1276 kvm_load_ldt(host_state->ldt_sel); 1277 #ifdef CONFIG_X86_64 1278 load_gs_index(host_state->gs_sel); 1279 #else 1280 loadsegment(gs, host_state->gs_sel); 1281 #endif 1282 } 1283 if (host_state->fs_sel & 7) 1284 loadsegment(fs, host_state->fs_sel); 1285 #ifdef CONFIG_X86_64 1286 if (unlikely(host_state->ds_sel | host_state->es_sel)) { 1287 loadsegment(ds, host_state->ds_sel); 1288 loadsegment(es, host_state->es_sel); 1289 } 1290 #endif 1291 invalidate_tss_limit(); 1292 #ifdef CONFIG_X86_64 1293 wrmsrl(MSR_KERNEL_GS_BASE, vmx->msr_host_kernel_gs_base); 1294 #endif 1295 load_fixmap_gdt(raw_smp_processor_id()); 1296 vmx->guest_state_loaded = false; 1297 vmx->guest_uret_msrs_loaded = false; 1298 } 1299 1300 #ifdef CONFIG_X86_64 1301 static u64 vmx_read_guest_kernel_gs_base(struct vcpu_vmx *vmx) 1302 { 1303 preempt_disable(); 1304 if (vmx->guest_state_loaded) 1305 rdmsrl(MSR_KERNEL_GS_BASE, vmx->msr_guest_kernel_gs_base); 1306 preempt_enable(); 1307 return vmx->msr_guest_kernel_gs_base; 1308 } 1309 1310 static void vmx_write_guest_kernel_gs_base(struct vcpu_vmx *vmx, u64 data) 1311 { 1312 preempt_disable(); 1313 if (vmx->guest_state_loaded) 1314 wrmsrl(MSR_KERNEL_GS_BASE, data); 1315 preempt_enable(); 1316 vmx->msr_guest_kernel_gs_base = data; 1317 } 1318 #endif 1319 1320 void vmx_vcpu_load_vmcs(struct kvm_vcpu *vcpu, int cpu, 1321 struct loaded_vmcs *buddy) 1322 { 1323 struct vcpu_vmx *vmx = to_vmx(vcpu); 1324 bool already_loaded = vmx->loaded_vmcs->cpu == cpu; 1325 struct vmcs *prev; 1326 1327 if (!already_loaded) { 1328 loaded_vmcs_clear(vmx->loaded_vmcs); 1329 local_irq_disable(); 1330 1331 /* 1332 * Ensure loaded_vmcs->cpu is read before adding loaded_vmcs to 1333 * this cpu's percpu list, otherwise it may not yet be deleted 1334 * from its previous cpu's percpu list. Pairs with the 1335 * smb_wmb() in __loaded_vmcs_clear(). 1336 */ 1337 smp_rmb(); 1338 1339 list_add(&vmx->loaded_vmcs->loaded_vmcss_on_cpu_link, 1340 &per_cpu(loaded_vmcss_on_cpu, cpu)); 1341 local_irq_enable(); 1342 } 1343 1344 prev = per_cpu(current_vmcs, cpu); 1345 if (prev != vmx->loaded_vmcs->vmcs) { 1346 per_cpu(current_vmcs, cpu) = vmx->loaded_vmcs->vmcs; 1347 vmcs_load(vmx->loaded_vmcs->vmcs); 1348 1349 /* 1350 * No indirect branch prediction barrier needed when switching 1351 * the active VMCS within a guest, e.g. on nested VM-Enter. 1352 * The L1 VMM can protect itself with retpolines, IBPB or IBRS. 1353 */ 1354 if (!buddy || WARN_ON_ONCE(buddy->vmcs != prev)) 1355 indirect_branch_prediction_barrier(); 1356 } 1357 1358 if (!already_loaded) { 1359 void *gdt = get_current_gdt_ro(); 1360 1361 /* 1362 * Flush all EPTP/VPID contexts, the new pCPU may have stale 1363 * TLB entries from its previous association with the vCPU. 1364 */ 1365 kvm_make_request(KVM_REQ_TLB_FLUSH, vcpu); 1366 1367 /* 1368 * Linux uses per-cpu TSS and GDT, so set these when switching 1369 * processors. See 22.2.4. 1370 */ 1371 vmcs_writel(HOST_TR_BASE, 1372 (unsigned long)&get_cpu_entry_area(cpu)->tss.x86_tss); 1373 vmcs_writel(HOST_GDTR_BASE, (unsigned long)gdt); /* 22.2.4 */ 1374 1375 if (IS_ENABLED(CONFIG_IA32_EMULATION) || IS_ENABLED(CONFIG_X86_32)) { 1376 /* 22.2.3 */ 1377 vmcs_writel(HOST_IA32_SYSENTER_ESP, 1378 (unsigned long)(cpu_entry_stack(cpu) + 1)); 1379 } 1380 1381 vmx->loaded_vmcs->cpu = cpu; 1382 } 1383 } 1384 1385 /* 1386 * Switches to specified vcpu, until a matching vcpu_put(), but assumes 1387 * vcpu mutex is already taken. 1388 */ 1389 static void vmx_vcpu_load(struct kvm_vcpu *vcpu, int cpu) 1390 { 1391 struct vcpu_vmx *vmx = to_vmx(vcpu); 1392 1393 vmx_vcpu_load_vmcs(vcpu, cpu, NULL); 1394 1395 vmx_vcpu_pi_load(vcpu, cpu); 1396 1397 vmx->host_debugctlmsr = get_debugctlmsr(); 1398 } 1399 1400 static void vmx_vcpu_put(struct kvm_vcpu *vcpu) 1401 { 1402 vmx_vcpu_pi_put(vcpu); 1403 1404 vmx_prepare_switch_to_host(to_vmx(vcpu)); 1405 } 1406 1407 bool vmx_emulation_required(struct kvm_vcpu *vcpu) 1408 { 1409 return emulate_invalid_guest_state && !vmx_guest_state_valid(vcpu); 1410 } 1411 1412 unsigned long vmx_get_rflags(struct kvm_vcpu *vcpu) 1413 { 1414 struct vcpu_vmx *vmx = to_vmx(vcpu); 1415 unsigned long rflags, save_rflags; 1416 1417 if (!kvm_register_is_available(vcpu, VCPU_EXREG_RFLAGS)) { 1418 kvm_register_mark_available(vcpu, VCPU_EXREG_RFLAGS); 1419 rflags = vmcs_readl(GUEST_RFLAGS); 1420 if (vmx->rmode.vm86_active) { 1421 rflags &= RMODE_GUEST_OWNED_EFLAGS_BITS; 1422 save_rflags = vmx->rmode.save_rflags; 1423 rflags |= save_rflags & ~RMODE_GUEST_OWNED_EFLAGS_BITS; 1424 } 1425 vmx->rflags = rflags; 1426 } 1427 return vmx->rflags; 1428 } 1429 1430 void vmx_set_rflags(struct kvm_vcpu *vcpu, unsigned long rflags) 1431 { 1432 struct vcpu_vmx *vmx = to_vmx(vcpu); 1433 unsigned long old_rflags; 1434 1435 if (is_unrestricted_guest(vcpu)) { 1436 kvm_register_mark_available(vcpu, VCPU_EXREG_RFLAGS); 1437 vmx->rflags = rflags; 1438 vmcs_writel(GUEST_RFLAGS, rflags); 1439 return; 1440 } 1441 1442 old_rflags = vmx_get_rflags(vcpu); 1443 vmx->rflags = rflags; 1444 if (vmx->rmode.vm86_active) { 1445 vmx->rmode.save_rflags = rflags; 1446 rflags |= X86_EFLAGS_IOPL | X86_EFLAGS_VM; 1447 } 1448 vmcs_writel(GUEST_RFLAGS, rflags); 1449 1450 if ((old_rflags ^ vmx->rflags) & X86_EFLAGS_VM) 1451 vmx->emulation_required = vmx_emulation_required(vcpu); 1452 } 1453 1454 static bool vmx_get_if_flag(struct kvm_vcpu *vcpu) 1455 { 1456 return vmx_get_rflags(vcpu) & X86_EFLAGS_IF; 1457 } 1458 1459 u32 vmx_get_interrupt_shadow(struct kvm_vcpu *vcpu) 1460 { 1461 u32 interruptibility = vmcs_read32(GUEST_INTERRUPTIBILITY_INFO); 1462 int ret = 0; 1463 1464 if (interruptibility & GUEST_INTR_STATE_STI) 1465 ret |= KVM_X86_SHADOW_INT_STI; 1466 if (interruptibility & GUEST_INTR_STATE_MOV_SS) 1467 ret |= KVM_X86_SHADOW_INT_MOV_SS; 1468 1469 return ret; 1470 } 1471 1472 void vmx_set_interrupt_shadow(struct kvm_vcpu *vcpu, int mask) 1473 { 1474 u32 interruptibility_old = vmcs_read32(GUEST_INTERRUPTIBILITY_INFO); 1475 u32 interruptibility = interruptibility_old; 1476 1477 interruptibility &= ~(GUEST_INTR_STATE_STI | GUEST_INTR_STATE_MOV_SS); 1478 1479 if (mask & KVM_X86_SHADOW_INT_MOV_SS) 1480 interruptibility |= GUEST_INTR_STATE_MOV_SS; 1481 else if (mask & KVM_X86_SHADOW_INT_STI) 1482 interruptibility |= GUEST_INTR_STATE_STI; 1483 1484 if ((interruptibility != interruptibility_old)) 1485 vmcs_write32(GUEST_INTERRUPTIBILITY_INFO, interruptibility); 1486 } 1487 1488 static int vmx_rtit_ctl_check(struct kvm_vcpu *vcpu, u64 data) 1489 { 1490 struct vcpu_vmx *vmx = to_vmx(vcpu); 1491 unsigned long value; 1492 1493 /* 1494 * Any MSR write that attempts to change bits marked reserved will 1495 * case a #GP fault. 1496 */ 1497 if (data & vmx->pt_desc.ctl_bitmask) 1498 return 1; 1499 1500 /* 1501 * Any attempt to modify IA32_RTIT_CTL while TraceEn is set will 1502 * result in a #GP unless the same write also clears TraceEn. 1503 */ 1504 if ((vmx->pt_desc.guest.ctl & RTIT_CTL_TRACEEN) && 1505 ((vmx->pt_desc.guest.ctl ^ data) & ~RTIT_CTL_TRACEEN)) 1506 return 1; 1507 1508 /* 1509 * WRMSR to IA32_RTIT_CTL that sets TraceEn but clears this bit 1510 * and FabricEn would cause #GP, if 1511 * CPUID.(EAX=14H, ECX=0):ECX.SNGLRGNOUT[bit 2] = 0 1512 */ 1513 if ((data & RTIT_CTL_TRACEEN) && !(data & RTIT_CTL_TOPA) && 1514 !(data & RTIT_CTL_FABRIC_EN) && 1515 !intel_pt_validate_cap(vmx->pt_desc.caps, 1516 PT_CAP_single_range_output)) 1517 return 1; 1518 1519 /* 1520 * MTCFreq, CycThresh and PSBFreq encodings check, any MSR write that 1521 * utilize encodings marked reserved will cause a #GP fault. 1522 */ 1523 value = intel_pt_validate_cap(vmx->pt_desc.caps, PT_CAP_mtc_periods); 1524 if (intel_pt_validate_cap(vmx->pt_desc.caps, PT_CAP_mtc) && 1525 !test_bit((data & RTIT_CTL_MTC_RANGE) >> 1526 RTIT_CTL_MTC_RANGE_OFFSET, &value)) 1527 return 1; 1528 value = intel_pt_validate_cap(vmx->pt_desc.caps, 1529 PT_CAP_cycle_thresholds); 1530 if (intel_pt_validate_cap(vmx->pt_desc.caps, PT_CAP_psb_cyc) && 1531 !test_bit((data & RTIT_CTL_CYC_THRESH) >> 1532 RTIT_CTL_CYC_THRESH_OFFSET, &value)) 1533 return 1; 1534 value = intel_pt_validate_cap(vmx->pt_desc.caps, PT_CAP_psb_periods); 1535 if (intel_pt_validate_cap(vmx->pt_desc.caps, PT_CAP_psb_cyc) && 1536 !test_bit((data & RTIT_CTL_PSB_FREQ) >> 1537 RTIT_CTL_PSB_FREQ_OFFSET, &value)) 1538 return 1; 1539 1540 /* 1541 * If ADDRx_CFG is reserved or the encodings is >2 will 1542 * cause a #GP fault. 1543 */ 1544 value = (data & RTIT_CTL_ADDR0) >> RTIT_CTL_ADDR0_OFFSET; 1545 if ((value && (vmx->pt_desc.num_address_ranges < 1)) || (value > 2)) 1546 return 1; 1547 value = (data & RTIT_CTL_ADDR1) >> RTIT_CTL_ADDR1_OFFSET; 1548 if ((value && (vmx->pt_desc.num_address_ranges < 2)) || (value > 2)) 1549 return 1; 1550 value = (data & RTIT_CTL_ADDR2) >> RTIT_CTL_ADDR2_OFFSET; 1551 if ((value && (vmx->pt_desc.num_address_ranges < 3)) || (value > 2)) 1552 return 1; 1553 value = (data & RTIT_CTL_ADDR3) >> RTIT_CTL_ADDR3_OFFSET; 1554 if ((value && (vmx->pt_desc.num_address_ranges < 4)) || (value > 2)) 1555 return 1; 1556 1557 return 0; 1558 } 1559 1560 static bool vmx_can_emulate_instruction(struct kvm_vcpu *vcpu, int emul_type, 1561 void *insn, int insn_len) 1562 { 1563 /* 1564 * Emulation of instructions in SGX enclaves is impossible as RIP does 1565 * not point at the failing instruction, and even if it did, the code 1566 * stream is inaccessible. Inject #UD instead of exiting to userspace 1567 * so that guest userspace can't DoS the guest simply by triggering 1568 * emulation (enclaves are CPL3 only). 1569 */ 1570 if (to_vmx(vcpu)->exit_reason.enclave_mode) { 1571 kvm_queue_exception(vcpu, UD_VECTOR); 1572 return false; 1573 } 1574 return true; 1575 } 1576 1577 static int skip_emulated_instruction(struct kvm_vcpu *vcpu) 1578 { 1579 union vmx_exit_reason exit_reason = to_vmx(vcpu)->exit_reason; 1580 unsigned long rip, orig_rip; 1581 u32 instr_len; 1582 1583 /* 1584 * Using VMCS.VM_EXIT_INSTRUCTION_LEN on EPT misconfig depends on 1585 * undefined behavior: Intel's SDM doesn't mandate the VMCS field be 1586 * set when EPT misconfig occurs. In practice, real hardware updates 1587 * VM_EXIT_INSTRUCTION_LEN on EPT misconfig, but other hypervisors 1588 * (namely Hyper-V) don't set it due to it being undefined behavior, 1589 * i.e. we end up advancing IP with some random value. 1590 */ 1591 if (!static_cpu_has(X86_FEATURE_HYPERVISOR) || 1592 exit_reason.basic != EXIT_REASON_EPT_MISCONFIG) { 1593 instr_len = vmcs_read32(VM_EXIT_INSTRUCTION_LEN); 1594 1595 /* 1596 * Emulating an enclave's instructions isn't supported as KVM 1597 * cannot access the enclave's memory or its true RIP, e.g. the 1598 * vmcs.GUEST_RIP points at the exit point of the enclave, not 1599 * the RIP that actually triggered the VM-Exit. But, because 1600 * most instructions that cause VM-Exit will #UD in an enclave, 1601 * most instruction-based VM-Exits simply do not occur. 1602 * 1603 * There are a few exceptions, notably the debug instructions 1604 * INT1ICEBRK and INT3, as they are allowed in debug enclaves 1605 * and generate #DB/#BP as expected, which KVM might intercept. 1606 * But again, the CPU does the dirty work and saves an instr 1607 * length of zero so VMMs don't shoot themselves in the foot. 1608 * WARN if KVM tries to skip a non-zero length instruction on 1609 * a VM-Exit from an enclave. 1610 */ 1611 if (!instr_len) 1612 goto rip_updated; 1613 1614 WARN(exit_reason.enclave_mode, 1615 "KVM: skipping instruction after SGX enclave VM-Exit"); 1616 1617 orig_rip = kvm_rip_read(vcpu); 1618 rip = orig_rip + instr_len; 1619 #ifdef CONFIG_X86_64 1620 /* 1621 * We need to mask out the high 32 bits of RIP if not in 64-bit 1622 * mode, but just finding out that we are in 64-bit mode is 1623 * quite expensive. Only do it if there was a carry. 1624 */ 1625 if (unlikely(((rip ^ orig_rip) >> 31) == 3) && !is_64_bit_mode(vcpu)) 1626 rip = (u32)rip; 1627 #endif 1628 kvm_rip_write(vcpu, rip); 1629 } else { 1630 if (!kvm_emulate_instruction(vcpu, EMULTYPE_SKIP)) 1631 return 0; 1632 } 1633 1634 rip_updated: 1635 /* skipping an emulated instruction also counts */ 1636 vmx_set_interrupt_shadow(vcpu, 0); 1637 1638 return 1; 1639 } 1640 1641 /* 1642 * Recognizes a pending MTF VM-exit and records the nested state for later 1643 * delivery. 1644 */ 1645 static void vmx_update_emulated_instruction(struct kvm_vcpu *vcpu) 1646 { 1647 struct vmcs12 *vmcs12 = get_vmcs12(vcpu); 1648 struct vcpu_vmx *vmx = to_vmx(vcpu); 1649 1650 if (!is_guest_mode(vcpu)) 1651 return; 1652 1653 /* 1654 * Per the SDM, MTF takes priority over debug-trap exceptions besides 1655 * TSS T-bit traps and ICEBP (INT1). KVM doesn't emulate T-bit traps 1656 * or ICEBP (in the emulator proper), and skipping of ICEBP after an 1657 * intercepted #DB deliberately avoids single-step #DB and MTF updates 1658 * as ICEBP is higher priority than both. As instruction emulation is 1659 * completed at this point (i.e. KVM is at the instruction boundary), 1660 * any #DB exception pending delivery must be a debug-trap of lower 1661 * priority than MTF. Record the pending MTF state to be delivered in 1662 * vmx_check_nested_events(). 1663 */ 1664 if (nested_cpu_has_mtf(vmcs12) && 1665 (!vcpu->arch.exception.pending || 1666 vcpu->arch.exception.vector == DB_VECTOR) && 1667 (!vcpu->arch.exception_vmexit.pending || 1668 vcpu->arch.exception_vmexit.vector == DB_VECTOR)) { 1669 vmx->nested.mtf_pending = true; 1670 kvm_make_request(KVM_REQ_EVENT, vcpu); 1671 } else { 1672 vmx->nested.mtf_pending = false; 1673 } 1674 } 1675 1676 static int vmx_skip_emulated_instruction(struct kvm_vcpu *vcpu) 1677 { 1678 vmx_update_emulated_instruction(vcpu); 1679 return skip_emulated_instruction(vcpu); 1680 } 1681 1682 static void vmx_clear_hlt(struct kvm_vcpu *vcpu) 1683 { 1684 /* 1685 * Ensure that we clear the HLT state in the VMCS. We don't need to 1686 * explicitly skip the instruction because if the HLT state is set, 1687 * then the instruction is already executing and RIP has already been 1688 * advanced. 1689 */ 1690 if (kvm_hlt_in_guest(vcpu->kvm) && 1691 vmcs_read32(GUEST_ACTIVITY_STATE) == GUEST_ACTIVITY_HLT) 1692 vmcs_write32(GUEST_ACTIVITY_STATE, GUEST_ACTIVITY_ACTIVE); 1693 } 1694 1695 static void vmx_inject_exception(struct kvm_vcpu *vcpu) 1696 { 1697 struct kvm_queued_exception *ex = &vcpu->arch.exception; 1698 u32 intr_info = ex->vector | INTR_INFO_VALID_MASK; 1699 struct vcpu_vmx *vmx = to_vmx(vcpu); 1700 1701 kvm_deliver_exception_payload(vcpu, ex); 1702 1703 if (ex->has_error_code) { 1704 /* 1705 * Despite the error code being architecturally defined as 32 1706 * bits, and the VMCS field being 32 bits, Intel CPUs and thus 1707 * VMX don't actually supporting setting bits 31:16. Hardware 1708 * will (should) never provide a bogus error code, but AMD CPUs 1709 * do generate error codes with bits 31:16 set, and so KVM's 1710 * ABI lets userspace shove in arbitrary 32-bit values. Drop 1711 * the upper bits to avoid VM-Fail, losing information that 1712 * does't really exist is preferable to killing the VM. 1713 */ 1714 vmcs_write32(VM_ENTRY_EXCEPTION_ERROR_CODE, (u16)ex->error_code); 1715 intr_info |= INTR_INFO_DELIVER_CODE_MASK; 1716 } 1717 1718 if (vmx->rmode.vm86_active) { 1719 int inc_eip = 0; 1720 if (kvm_exception_is_soft(ex->vector)) 1721 inc_eip = vcpu->arch.event_exit_inst_len; 1722 kvm_inject_realmode_interrupt(vcpu, ex->vector, inc_eip); 1723 return; 1724 } 1725 1726 WARN_ON_ONCE(vmx->emulation_required); 1727 1728 if (kvm_exception_is_soft(ex->vector)) { 1729 vmcs_write32(VM_ENTRY_INSTRUCTION_LEN, 1730 vmx->vcpu.arch.event_exit_inst_len); 1731 intr_info |= INTR_TYPE_SOFT_EXCEPTION; 1732 } else 1733 intr_info |= INTR_TYPE_HARD_EXCEPTION; 1734 1735 vmcs_write32(VM_ENTRY_INTR_INFO_FIELD, intr_info); 1736 1737 vmx_clear_hlt(vcpu); 1738 } 1739 1740 static void vmx_setup_uret_msr(struct vcpu_vmx *vmx, unsigned int msr, 1741 bool load_into_hardware) 1742 { 1743 struct vmx_uret_msr *uret_msr; 1744 1745 uret_msr = vmx_find_uret_msr(vmx, msr); 1746 if (!uret_msr) 1747 return; 1748 1749 uret_msr->load_into_hardware = load_into_hardware; 1750 } 1751 1752 /* 1753 * Configuring user return MSRs to automatically save, load, and restore MSRs 1754 * that need to be shoved into hardware when running the guest. Note, omitting 1755 * an MSR here does _NOT_ mean it's not emulated, only that it will not be 1756 * loaded into hardware when running the guest. 1757 */ 1758 static void vmx_setup_uret_msrs(struct vcpu_vmx *vmx) 1759 { 1760 #ifdef CONFIG_X86_64 1761 bool load_syscall_msrs; 1762 1763 /* 1764 * The SYSCALL MSRs are only needed on long mode guests, and only 1765 * when EFER.SCE is set. 1766 */ 1767 load_syscall_msrs = is_long_mode(&vmx->vcpu) && 1768 (vmx->vcpu.arch.efer & EFER_SCE); 1769 1770 vmx_setup_uret_msr(vmx, MSR_STAR, load_syscall_msrs); 1771 vmx_setup_uret_msr(vmx, MSR_LSTAR, load_syscall_msrs); 1772 vmx_setup_uret_msr(vmx, MSR_SYSCALL_MASK, load_syscall_msrs); 1773 #endif 1774 vmx_setup_uret_msr(vmx, MSR_EFER, update_transition_efer(vmx)); 1775 1776 vmx_setup_uret_msr(vmx, MSR_TSC_AUX, 1777 guest_cpuid_has(&vmx->vcpu, X86_FEATURE_RDTSCP) || 1778 guest_cpuid_has(&vmx->vcpu, X86_FEATURE_RDPID)); 1779 1780 /* 1781 * hle=0, rtm=0, tsx_ctrl=1 can be found with some combinations of new 1782 * kernel and old userspace. If those guests run on a tsx=off host, do 1783 * allow guests to use TSX_CTRL, but don't change the value in hardware 1784 * so that TSX remains always disabled. 1785 */ 1786 vmx_setup_uret_msr(vmx, MSR_IA32_TSX_CTRL, boot_cpu_has(X86_FEATURE_RTM)); 1787 1788 /* 1789 * The set of MSRs to load may have changed, reload MSRs before the 1790 * next VM-Enter. 1791 */ 1792 vmx->guest_uret_msrs_loaded = false; 1793 } 1794 1795 u64 vmx_get_l2_tsc_offset(struct kvm_vcpu *vcpu) 1796 { 1797 struct vmcs12 *vmcs12 = get_vmcs12(vcpu); 1798 1799 if (nested_cpu_has(vmcs12, CPU_BASED_USE_TSC_OFFSETTING)) 1800 return vmcs12->tsc_offset; 1801 1802 return 0; 1803 } 1804 1805 u64 vmx_get_l2_tsc_multiplier(struct kvm_vcpu *vcpu) 1806 { 1807 struct vmcs12 *vmcs12 = get_vmcs12(vcpu); 1808 1809 if (nested_cpu_has(vmcs12, CPU_BASED_USE_TSC_OFFSETTING) && 1810 nested_cpu_has2(vmcs12, SECONDARY_EXEC_TSC_SCALING)) 1811 return vmcs12->tsc_multiplier; 1812 1813 return kvm_caps.default_tsc_scaling_ratio; 1814 } 1815 1816 static void vmx_write_tsc_offset(struct kvm_vcpu *vcpu, u64 offset) 1817 { 1818 vmcs_write64(TSC_OFFSET, offset); 1819 } 1820 1821 static void vmx_write_tsc_multiplier(struct kvm_vcpu *vcpu, u64 multiplier) 1822 { 1823 vmcs_write64(TSC_MULTIPLIER, multiplier); 1824 } 1825 1826 /* 1827 * nested_vmx_allowed() checks whether a guest should be allowed to use VMX 1828 * instructions and MSRs (i.e., nested VMX). Nested VMX is disabled for 1829 * all guests if the "nested" module option is off, and can also be disabled 1830 * for a single guest by disabling its VMX cpuid bit. 1831 */ 1832 bool nested_vmx_allowed(struct kvm_vcpu *vcpu) 1833 { 1834 return nested && guest_cpuid_has(vcpu, X86_FEATURE_VMX); 1835 } 1836 1837 static inline bool vmx_feature_control_msr_valid(struct kvm_vcpu *vcpu, 1838 uint64_t val) 1839 { 1840 uint64_t valid_bits = to_vmx(vcpu)->msr_ia32_feature_control_valid_bits; 1841 1842 return !(val & ~valid_bits); 1843 } 1844 1845 static int vmx_get_msr_feature(struct kvm_msr_entry *msr) 1846 { 1847 switch (msr->index) { 1848 case MSR_IA32_VMX_BASIC ... MSR_IA32_VMX_VMFUNC: 1849 if (!nested) 1850 return 1; 1851 return vmx_get_vmx_msr(&vmcs_config.nested, msr->index, &msr->data); 1852 case MSR_IA32_PERF_CAPABILITIES: 1853 msr->data = vmx_get_perf_capabilities(); 1854 return 0; 1855 default: 1856 return KVM_MSR_RET_INVALID; 1857 } 1858 } 1859 1860 /* 1861 * Reads an msr value (of 'msr_info->index') into 'msr_info->data'. 1862 * Returns 0 on success, non-0 otherwise. 1863 * Assumes vcpu_load() was already called. 1864 */ 1865 static int vmx_get_msr(struct kvm_vcpu *vcpu, struct msr_data *msr_info) 1866 { 1867 struct vcpu_vmx *vmx = to_vmx(vcpu); 1868 struct vmx_uret_msr *msr; 1869 u32 index; 1870 1871 switch (msr_info->index) { 1872 #ifdef CONFIG_X86_64 1873 case MSR_FS_BASE: 1874 msr_info->data = vmcs_readl(GUEST_FS_BASE); 1875 break; 1876 case MSR_GS_BASE: 1877 msr_info->data = vmcs_readl(GUEST_GS_BASE); 1878 break; 1879 case MSR_KERNEL_GS_BASE: 1880 msr_info->data = vmx_read_guest_kernel_gs_base(vmx); 1881 break; 1882 #endif 1883 case MSR_EFER: 1884 return kvm_get_msr_common(vcpu, msr_info); 1885 case MSR_IA32_TSX_CTRL: 1886 if (!msr_info->host_initiated && 1887 !(vcpu->arch.arch_capabilities & ARCH_CAP_TSX_CTRL_MSR)) 1888 return 1; 1889 goto find_uret_msr; 1890 case MSR_IA32_UMWAIT_CONTROL: 1891 if (!msr_info->host_initiated && !vmx_has_waitpkg(vmx)) 1892 return 1; 1893 1894 msr_info->data = vmx->msr_ia32_umwait_control; 1895 break; 1896 case MSR_IA32_SPEC_CTRL: 1897 if (!msr_info->host_initiated && 1898 !guest_has_spec_ctrl_msr(vcpu)) 1899 return 1; 1900 1901 msr_info->data = to_vmx(vcpu)->spec_ctrl; 1902 break; 1903 case MSR_IA32_SYSENTER_CS: 1904 msr_info->data = vmcs_read32(GUEST_SYSENTER_CS); 1905 break; 1906 case MSR_IA32_SYSENTER_EIP: 1907 msr_info->data = vmcs_readl(GUEST_SYSENTER_EIP); 1908 break; 1909 case MSR_IA32_SYSENTER_ESP: 1910 msr_info->data = vmcs_readl(GUEST_SYSENTER_ESP); 1911 break; 1912 case MSR_IA32_BNDCFGS: 1913 if (!kvm_mpx_supported() || 1914 (!msr_info->host_initiated && 1915 !guest_cpuid_has(vcpu, X86_FEATURE_MPX))) 1916 return 1; 1917 msr_info->data = vmcs_read64(GUEST_BNDCFGS); 1918 break; 1919 case MSR_IA32_MCG_EXT_CTL: 1920 if (!msr_info->host_initiated && 1921 !(vmx->msr_ia32_feature_control & 1922 FEAT_CTL_LMCE_ENABLED)) 1923 return 1; 1924 msr_info->data = vcpu->arch.mcg_ext_ctl; 1925 break; 1926 case MSR_IA32_FEAT_CTL: 1927 msr_info->data = vmx->msr_ia32_feature_control; 1928 break; 1929 case MSR_IA32_SGXLEPUBKEYHASH0 ... MSR_IA32_SGXLEPUBKEYHASH3: 1930 if (!msr_info->host_initiated && 1931 !guest_cpuid_has(vcpu, X86_FEATURE_SGX_LC)) 1932 return 1; 1933 msr_info->data = to_vmx(vcpu)->msr_ia32_sgxlepubkeyhash 1934 [msr_info->index - MSR_IA32_SGXLEPUBKEYHASH0]; 1935 break; 1936 case MSR_IA32_VMX_BASIC ... MSR_IA32_VMX_VMFUNC: 1937 if (!nested_vmx_allowed(vcpu)) 1938 return 1; 1939 if (vmx_get_vmx_msr(&vmx->nested.msrs, msr_info->index, 1940 &msr_info->data)) 1941 return 1; 1942 /* 1943 * Enlightened VMCS v1 doesn't have certain VMCS fields but 1944 * instead of just ignoring the features, different Hyper-V 1945 * versions are either trying to use them and fail or do some 1946 * sanity checking and refuse to boot. Filter all unsupported 1947 * features out. 1948 */ 1949 if (!msr_info->host_initiated && guest_cpuid_has_evmcs(vcpu)) 1950 nested_evmcs_filter_control_msr(vcpu, msr_info->index, 1951 &msr_info->data); 1952 break; 1953 case MSR_IA32_RTIT_CTL: 1954 if (!vmx_pt_mode_is_host_guest()) 1955 return 1; 1956 msr_info->data = vmx->pt_desc.guest.ctl; 1957 break; 1958 case MSR_IA32_RTIT_STATUS: 1959 if (!vmx_pt_mode_is_host_guest()) 1960 return 1; 1961 msr_info->data = vmx->pt_desc.guest.status; 1962 break; 1963 case MSR_IA32_RTIT_CR3_MATCH: 1964 if (!vmx_pt_mode_is_host_guest() || 1965 !intel_pt_validate_cap(vmx->pt_desc.caps, 1966 PT_CAP_cr3_filtering)) 1967 return 1; 1968 msr_info->data = vmx->pt_desc.guest.cr3_match; 1969 break; 1970 case MSR_IA32_RTIT_OUTPUT_BASE: 1971 if (!vmx_pt_mode_is_host_guest() || 1972 (!intel_pt_validate_cap(vmx->pt_desc.caps, 1973 PT_CAP_topa_output) && 1974 !intel_pt_validate_cap(vmx->pt_desc.caps, 1975 PT_CAP_single_range_output))) 1976 return 1; 1977 msr_info->data = vmx->pt_desc.guest.output_base; 1978 break; 1979 case MSR_IA32_RTIT_OUTPUT_MASK: 1980 if (!vmx_pt_mode_is_host_guest() || 1981 (!intel_pt_validate_cap(vmx->pt_desc.caps, 1982 PT_CAP_topa_output) && 1983 !intel_pt_validate_cap(vmx->pt_desc.caps, 1984 PT_CAP_single_range_output))) 1985 return 1; 1986 msr_info->data = vmx->pt_desc.guest.output_mask; 1987 break; 1988 case MSR_IA32_RTIT_ADDR0_A ... MSR_IA32_RTIT_ADDR3_B: 1989 index = msr_info->index - MSR_IA32_RTIT_ADDR0_A; 1990 if (!vmx_pt_mode_is_host_guest() || 1991 (index >= 2 * vmx->pt_desc.num_address_ranges)) 1992 return 1; 1993 if (index % 2) 1994 msr_info->data = vmx->pt_desc.guest.addr_b[index / 2]; 1995 else 1996 msr_info->data = vmx->pt_desc.guest.addr_a[index / 2]; 1997 break; 1998 case MSR_IA32_DEBUGCTLMSR: 1999 msr_info->data = vmcs_read64(GUEST_IA32_DEBUGCTL); 2000 break; 2001 default: 2002 find_uret_msr: 2003 msr = vmx_find_uret_msr(vmx, msr_info->index); 2004 if (msr) { 2005 msr_info->data = msr->data; 2006 break; 2007 } 2008 return kvm_get_msr_common(vcpu, msr_info); 2009 } 2010 2011 return 0; 2012 } 2013 2014 static u64 nested_vmx_truncate_sysenter_addr(struct kvm_vcpu *vcpu, 2015 u64 data) 2016 { 2017 #ifdef CONFIG_X86_64 2018 if (!guest_cpuid_has(vcpu, X86_FEATURE_LM)) 2019 return (u32)data; 2020 #endif 2021 return (unsigned long)data; 2022 } 2023 2024 static u64 vmx_get_supported_debugctl(struct kvm_vcpu *vcpu, bool host_initiated) 2025 { 2026 u64 debugctl = 0; 2027 2028 if (boot_cpu_has(X86_FEATURE_BUS_LOCK_DETECT) && 2029 (host_initiated || guest_cpuid_has(vcpu, X86_FEATURE_BUS_LOCK_DETECT))) 2030 debugctl |= DEBUGCTLMSR_BUS_LOCK_DETECT; 2031 2032 if ((vmx_get_perf_capabilities() & PMU_CAP_LBR_FMT) && 2033 (host_initiated || intel_pmu_lbr_is_enabled(vcpu))) 2034 debugctl |= DEBUGCTLMSR_LBR | DEBUGCTLMSR_FREEZE_LBRS_ON_PMI; 2035 2036 return debugctl; 2037 } 2038 2039 /* 2040 * Writes msr value into the appropriate "register". 2041 * Returns 0 on success, non-0 otherwise. 2042 * Assumes vcpu_load() was already called. 2043 */ 2044 static int vmx_set_msr(struct kvm_vcpu *vcpu, struct msr_data *msr_info) 2045 { 2046 struct vcpu_vmx *vmx = to_vmx(vcpu); 2047 struct vmx_uret_msr *msr; 2048 int ret = 0; 2049 u32 msr_index = msr_info->index; 2050 u64 data = msr_info->data; 2051 u32 index; 2052 2053 switch (msr_index) { 2054 case MSR_EFER: 2055 ret = kvm_set_msr_common(vcpu, msr_info); 2056 break; 2057 #ifdef CONFIG_X86_64 2058 case MSR_FS_BASE: 2059 vmx_segment_cache_clear(vmx); 2060 vmcs_writel(GUEST_FS_BASE, data); 2061 break; 2062 case MSR_GS_BASE: 2063 vmx_segment_cache_clear(vmx); 2064 vmcs_writel(GUEST_GS_BASE, data); 2065 break; 2066 case MSR_KERNEL_GS_BASE: 2067 vmx_write_guest_kernel_gs_base(vmx, data); 2068 break; 2069 case MSR_IA32_XFD: 2070 ret = kvm_set_msr_common(vcpu, msr_info); 2071 /* 2072 * Always intercepting WRMSR could incur non-negligible 2073 * overhead given xfd might be changed frequently in 2074 * guest context switch. Disable write interception 2075 * upon the first write with a non-zero value (indicating 2076 * potential usage on dynamic xfeatures). Also update 2077 * exception bitmap to trap #NM for proper virtualization 2078 * of guest xfd_err. 2079 */ 2080 if (!ret && data) { 2081 vmx_disable_intercept_for_msr(vcpu, MSR_IA32_XFD, 2082 MSR_TYPE_RW); 2083 vcpu->arch.xfd_no_write_intercept = true; 2084 vmx_update_exception_bitmap(vcpu); 2085 } 2086 break; 2087 #endif 2088 case MSR_IA32_SYSENTER_CS: 2089 if (is_guest_mode(vcpu)) 2090 get_vmcs12(vcpu)->guest_sysenter_cs = data; 2091 vmcs_write32(GUEST_SYSENTER_CS, data); 2092 break; 2093 case MSR_IA32_SYSENTER_EIP: 2094 if (is_guest_mode(vcpu)) { 2095 data = nested_vmx_truncate_sysenter_addr(vcpu, data); 2096 get_vmcs12(vcpu)->guest_sysenter_eip = data; 2097 } 2098 vmcs_writel(GUEST_SYSENTER_EIP, data); 2099 break; 2100 case MSR_IA32_SYSENTER_ESP: 2101 if (is_guest_mode(vcpu)) { 2102 data = nested_vmx_truncate_sysenter_addr(vcpu, data); 2103 get_vmcs12(vcpu)->guest_sysenter_esp = data; 2104 } 2105 vmcs_writel(GUEST_SYSENTER_ESP, data); 2106 break; 2107 case MSR_IA32_DEBUGCTLMSR: { 2108 u64 invalid; 2109 2110 invalid = data & ~vmx_get_supported_debugctl(vcpu, msr_info->host_initiated); 2111 if (invalid & (DEBUGCTLMSR_BTF|DEBUGCTLMSR_LBR)) { 2112 if (report_ignored_msrs) 2113 vcpu_unimpl(vcpu, "%s: BTF|LBR in IA32_DEBUGCTLMSR 0x%llx, nop\n", 2114 __func__, data); 2115 data &= ~(DEBUGCTLMSR_BTF|DEBUGCTLMSR_LBR); 2116 invalid &= ~(DEBUGCTLMSR_BTF|DEBUGCTLMSR_LBR); 2117 } 2118 2119 if (invalid) 2120 return 1; 2121 2122 if (is_guest_mode(vcpu) && get_vmcs12(vcpu)->vm_exit_controls & 2123 VM_EXIT_SAVE_DEBUG_CONTROLS) 2124 get_vmcs12(vcpu)->guest_ia32_debugctl = data; 2125 2126 vmcs_write64(GUEST_IA32_DEBUGCTL, data); 2127 if (intel_pmu_lbr_is_enabled(vcpu) && !to_vmx(vcpu)->lbr_desc.event && 2128 (data & DEBUGCTLMSR_LBR)) 2129 intel_pmu_create_guest_lbr_event(vcpu); 2130 return 0; 2131 } 2132 case MSR_IA32_BNDCFGS: 2133 if (!kvm_mpx_supported() || 2134 (!msr_info->host_initiated && 2135 !guest_cpuid_has(vcpu, X86_FEATURE_MPX))) 2136 return 1; 2137 if (is_noncanonical_address(data & PAGE_MASK, vcpu) || 2138 (data & MSR_IA32_BNDCFGS_RSVD)) 2139 return 1; 2140 2141 if (is_guest_mode(vcpu) && 2142 ((vmx->nested.msrs.entry_ctls_high & VM_ENTRY_LOAD_BNDCFGS) || 2143 (vmx->nested.msrs.exit_ctls_high & VM_EXIT_CLEAR_BNDCFGS))) 2144 get_vmcs12(vcpu)->guest_bndcfgs = data; 2145 2146 vmcs_write64(GUEST_BNDCFGS, data); 2147 break; 2148 case MSR_IA32_UMWAIT_CONTROL: 2149 if (!msr_info->host_initiated && !vmx_has_waitpkg(vmx)) 2150 return 1; 2151 2152 /* The reserved bit 1 and non-32 bit [63:32] should be zero */ 2153 if (data & (BIT_ULL(1) | GENMASK_ULL(63, 32))) 2154 return 1; 2155 2156 vmx->msr_ia32_umwait_control = data; 2157 break; 2158 case MSR_IA32_SPEC_CTRL: 2159 if (!msr_info->host_initiated && 2160 !guest_has_spec_ctrl_msr(vcpu)) 2161 return 1; 2162 2163 if (kvm_spec_ctrl_test_value(data)) 2164 return 1; 2165 2166 vmx->spec_ctrl = data; 2167 if (!data) 2168 break; 2169 2170 /* 2171 * For non-nested: 2172 * When it's written (to non-zero) for the first time, pass 2173 * it through. 2174 * 2175 * For nested: 2176 * The handling of the MSR bitmap for L2 guests is done in 2177 * nested_vmx_prepare_msr_bitmap. We should not touch the 2178 * vmcs02.msr_bitmap here since it gets completely overwritten 2179 * in the merging. We update the vmcs01 here for L1 as well 2180 * since it will end up touching the MSR anyway now. 2181 */ 2182 vmx_disable_intercept_for_msr(vcpu, 2183 MSR_IA32_SPEC_CTRL, 2184 MSR_TYPE_RW); 2185 break; 2186 case MSR_IA32_TSX_CTRL: 2187 if (!msr_info->host_initiated && 2188 !(vcpu->arch.arch_capabilities & ARCH_CAP_TSX_CTRL_MSR)) 2189 return 1; 2190 if (data & ~(TSX_CTRL_RTM_DISABLE | TSX_CTRL_CPUID_CLEAR)) 2191 return 1; 2192 goto find_uret_msr; 2193 case MSR_IA32_PRED_CMD: 2194 if (!msr_info->host_initiated && 2195 !guest_has_pred_cmd_msr(vcpu)) 2196 return 1; 2197 2198 if (data & ~PRED_CMD_IBPB) 2199 return 1; 2200 if (!boot_cpu_has(X86_FEATURE_IBPB)) 2201 return 1; 2202 if (!data) 2203 break; 2204 2205 wrmsrl(MSR_IA32_PRED_CMD, PRED_CMD_IBPB); 2206 2207 /* 2208 * For non-nested: 2209 * When it's written (to non-zero) for the first time, pass 2210 * it through. 2211 * 2212 * For nested: 2213 * The handling of the MSR bitmap for L2 guests is done in 2214 * nested_vmx_prepare_msr_bitmap. We should not touch the 2215 * vmcs02.msr_bitmap here since it gets completely overwritten 2216 * in the merging. 2217 */ 2218 vmx_disable_intercept_for_msr(vcpu, MSR_IA32_PRED_CMD, MSR_TYPE_W); 2219 break; 2220 case MSR_IA32_CR_PAT: 2221 if (!kvm_pat_valid(data)) 2222 return 1; 2223 2224 if (is_guest_mode(vcpu) && 2225 get_vmcs12(vcpu)->vm_exit_controls & VM_EXIT_SAVE_IA32_PAT) 2226 get_vmcs12(vcpu)->guest_ia32_pat = data; 2227 2228 if (vmcs_config.vmentry_ctrl & VM_ENTRY_LOAD_IA32_PAT) { 2229 vmcs_write64(GUEST_IA32_PAT, data); 2230 vcpu->arch.pat = data; 2231 break; 2232 } 2233 ret = kvm_set_msr_common(vcpu, msr_info); 2234 break; 2235 case MSR_IA32_MCG_EXT_CTL: 2236 if ((!msr_info->host_initiated && 2237 !(to_vmx(vcpu)->msr_ia32_feature_control & 2238 FEAT_CTL_LMCE_ENABLED)) || 2239 (data & ~MCG_EXT_CTL_LMCE_EN)) 2240 return 1; 2241 vcpu->arch.mcg_ext_ctl = data; 2242 break; 2243 case MSR_IA32_FEAT_CTL: 2244 if (!vmx_feature_control_msr_valid(vcpu, data) || 2245 (to_vmx(vcpu)->msr_ia32_feature_control & 2246 FEAT_CTL_LOCKED && !msr_info->host_initiated)) 2247 return 1; 2248 vmx->msr_ia32_feature_control = data; 2249 if (msr_info->host_initiated && data == 0) 2250 vmx_leave_nested(vcpu); 2251 2252 /* SGX may be enabled/disabled by guest's firmware */ 2253 vmx_write_encls_bitmap(vcpu, NULL); 2254 break; 2255 case MSR_IA32_SGXLEPUBKEYHASH0 ... MSR_IA32_SGXLEPUBKEYHASH3: 2256 /* 2257 * On real hardware, the LE hash MSRs are writable before 2258 * the firmware sets bit 0 in MSR 0x7a ("activating" SGX), 2259 * at which point SGX related bits in IA32_FEATURE_CONTROL 2260 * become writable. 2261 * 2262 * KVM does not emulate SGX activation for simplicity, so 2263 * allow writes to the LE hash MSRs if IA32_FEATURE_CONTROL 2264 * is unlocked. This is technically not architectural 2265 * behavior, but it's close enough. 2266 */ 2267 if (!msr_info->host_initiated && 2268 (!guest_cpuid_has(vcpu, X86_FEATURE_SGX_LC) || 2269 ((vmx->msr_ia32_feature_control & FEAT_CTL_LOCKED) && 2270 !(vmx->msr_ia32_feature_control & FEAT_CTL_SGX_LC_ENABLED)))) 2271 return 1; 2272 vmx->msr_ia32_sgxlepubkeyhash 2273 [msr_index - MSR_IA32_SGXLEPUBKEYHASH0] = data; 2274 break; 2275 case MSR_IA32_VMX_BASIC ... MSR_IA32_VMX_VMFUNC: 2276 if (!msr_info->host_initiated) 2277 return 1; /* they are read-only */ 2278 if (!nested_vmx_allowed(vcpu)) 2279 return 1; 2280 return vmx_set_vmx_msr(vcpu, msr_index, data); 2281 case MSR_IA32_RTIT_CTL: 2282 if (!vmx_pt_mode_is_host_guest() || 2283 vmx_rtit_ctl_check(vcpu, data) || 2284 vmx->nested.vmxon) 2285 return 1; 2286 vmcs_write64(GUEST_IA32_RTIT_CTL, data); 2287 vmx->pt_desc.guest.ctl = data; 2288 pt_update_intercept_for_msr(vcpu); 2289 break; 2290 case MSR_IA32_RTIT_STATUS: 2291 if (!pt_can_write_msr(vmx)) 2292 return 1; 2293 if (data & MSR_IA32_RTIT_STATUS_MASK) 2294 return 1; 2295 vmx->pt_desc.guest.status = data; 2296 break; 2297 case MSR_IA32_RTIT_CR3_MATCH: 2298 if (!pt_can_write_msr(vmx)) 2299 return 1; 2300 if (!intel_pt_validate_cap(vmx->pt_desc.caps, 2301 PT_CAP_cr3_filtering)) 2302 return 1; 2303 vmx->pt_desc.guest.cr3_match = data; 2304 break; 2305 case MSR_IA32_RTIT_OUTPUT_BASE: 2306 if (!pt_can_write_msr(vmx)) 2307 return 1; 2308 if (!intel_pt_validate_cap(vmx->pt_desc.caps, 2309 PT_CAP_topa_output) && 2310 !intel_pt_validate_cap(vmx->pt_desc.caps, 2311 PT_CAP_single_range_output)) 2312 return 1; 2313 if (!pt_output_base_valid(vcpu, data)) 2314 return 1; 2315 vmx->pt_desc.guest.output_base = data; 2316 break; 2317 case MSR_IA32_RTIT_OUTPUT_MASK: 2318 if (!pt_can_write_msr(vmx)) 2319 return 1; 2320 if (!intel_pt_validate_cap(vmx->pt_desc.caps, 2321 PT_CAP_topa_output) && 2322 !intel_pt_validate_cap(vmx->pt_desc.caps, 2323 PT_CAP_single_range_output)) 2324 return 1; 2325 vmx->pt_desc.guest.output_mask = data; 2326 break; 2327 case MSR_IA32_RTIT_ADDR0_A ... MSR_IA32_RTIT_ADDR3_B: 2328 if (!pt_can_write_msr(vmx)) 2329 return 1; 2330 index = msr_info->index - MSR_IA32_RTIT_ADDR0_A; 2331 if (index >= 2 * vmx->pt_desc.num_address_ranges) 2332 return 1; 2333 if (is_noncanonical_address(data, vcpu)) 2334 return 1; 2335 if (index % 2) 2336 vmx->pt_desc.guest.addr_b[index / 2] = data; 2337 else 2338 vmx->pt_desc.guest.addr_a[index / 2] = data; 2339 break; 2340 case MSR_IA32_PERF_CAPABILITIES: 2341 if (data && !vcpu_to_pmu(vcpu)->version) 2342 return 1; 2343 if (data & PMU_CAP_LBR_FMT) { 2344 if ((data & PMU_CAP_LBR_FMT) != 2345 (vmx_get_perf_capabilities() & PMU_CAP_LBR_FMT)) 2346 return 1; 2347 if (!cpuid_model_is_consistent(vcpu)) 2348 return 1; 2349 } 2350 if (data & PERF_CAP_PEBS_FORMAT) { 2351 if ((data & PERF_CAP_PEBS_MASK) != 2352 (vmx_get_perf_capabilities() & PERF_CAP_PEBS_MASK)) 2353 return 1; 2354 if (!guest_cpuid_has(vcpu, X86_FEATURE_DS)) 2355 return 1; 2356 if (!guest_cpuid_has(vcpu, X86_FEATURE_DTES64)) 2357 return 1; 2358 if (!cpuid_model_is_consistent(vcpu)) 2359 return 1; 2360 } 2361 ret = kvm_set_msr_common(vcpu, msr_info); 2362 break; 2363 2364 default: 2365 find_uret_msr: 2366 msr = vmx_find_uret_msr(vmx, msr_index); 2367 if (msr) 2368 ret = vmx_set_guest_uret_msr(vmx, msr, data); 2369 else 2370 ret = kvm_set_msr_common(vcpu, msr_info); 2371 } 2372 2373 /* FB_CLEAR may have changed, also update the FB_CLEAR_DIS behavior */ 2374 if (msr_index == MSR_IA32_ARCH_CAPABILITIES) 2375 vmx_update_fb_clear_dis(vcpu, vmx); 2376 2377 return ret; 2378 } 2379 2380 static void vmx_cache_reg(struct kvm_vcpu *vcpu, enum kvm_reg reg) 2381 { 2382 unsigned long guest_owned_bits; 2383 2384 kvm_register_mark_available(vcpu, reg); 2385 2386 switch (reg) { 2387 case VCPU_REGS_RSP: 2388 vcpu->arch.regs[VCPU_REGS_RSP] = vmcs_readl(GUEST_RSP); 2389 break; 2390 case VCPU_REGS_RIP: 2391 vcpu->arch.regs[VCPU_REGS_RIP] = vmcs_readl(GUEST_RIP); 2392 break; 2393 case VCPU_EXREG_PDPTR: 2394 if (enable_ept) 2395 ept_save_pdptrs(vcpu); 2396 break; 2397 case VCPU_EXREG_CR0: 2398 guest_owned_bits = vcpu->arch.cr0_guest_owned_bits; 2399 2400 vcpu->arch.cr0 &= ~guest_owned_bits; 2401 vcpu->arch.cr0 |= vmcs_readl(GUEST_CR0) & guest_owned_bits; 2402 break; 2403 case VCPU_EXREG_CR3: 2404 /* 2405 * When intercepting CR3 loads, e.g. for shadowing paging, KVM's 2406 * CR3 is loaded into hardware, not the guest's CR3. 2407 */ 2408 if (!(exec_controls_get(to_vmx(vcpu)) & CPU_BASED_CR3_LOAD_EXITING)) 2409 vcpu->arch.cr3 = vmcs_readl(GUEST_CR3); 2410 break; 2411 case VCPU_EXREG_CR4: 2412 guest_owned_bits = vcpu->arch.cr4_guest_owned_bits; 2413 2414 vcpu->arch.cr4 &= ~guest_owned_bits; 2415 vcpu->arch.cr4 |= vmcs_readl(GUEST_CR4) & guest_owned_bits; 2416 break; 2417 default: 2418 KVM_BUG_ON(1, vcpu->kvm); 2419 break; 2420 } 2421 } 2422 2423 static __init int cpu_has_kvm_support(void) 2424 { 2425 return cpu_has_vmx(); 2426 } 2427 2428 static __init int vmx_disabled_by_bios(void) 2429 { 2430 return !boot_cpu_has(X86_FEATURE_MSR_IA32_FEAT_CTL) || 2431 !boot_cpu_has(X86_FEATURE_VMX); 2432 } 2433 2434 static int kvm_cpu_vmxon(u64 vmxon_pointer) 2435 { 2436 u64 msr; 2437 2438 cr4_set_bits(X86_CR4_VMXE); 2439 2440 asm_volatile_goto("1: vmxon %[vmxon_pointer]\n\t" 2441 _ASM_EXTABLE(1b, %l[fault]) 2442 : : [vmxon_pointer] "m"(vmxon_pointer) 2443 : : fault); 2444 return 0; 2445 2446 fault: 2447 WARN_ONCE(1, "VMXON faulted, MSR_IA32_FEAT_CTL (0x3a) = 0x%llx\n", 2448 rdmsrl_safe(MSR_IA32_FEAT_CTL, &msr) ? 0xdeadbeef : msr); 2449 cr4_clear_bits(X86_CR4_VMXE); 2450 2451 return -EFAULT; 2452 } 2453 2454 static int vmx_hardware_enable(void) 2455 { 2456 int cpu = raw_smp_processor_id(); 2457 u64 phys_addr = __pa(per_cpu(vmxarea, cpu)); 2458 int r; 2459 2460 if (cr4_read_shadow() & X86_CR4_VMXE) 2461 return -EBUSY; 2462 2463 /* 2464 * This can happen if we hot-added a CPU but failed to allocate 2465 * VP assist page for it. 2466 */ 2467 if (static_branch_unlikely(&enable_evmcs) && 2468 !hv_get_vp_assist_page(cpu)) 2469 return -EFAULT; 2470 2471 intel_pt_handle_vmx(1); 2472 2473 r = kvm_cpu_vmxon(phys_addr); 2474 if (r) { 2475 intel_pt_handle_vmx(0); 2476 return r; 2477 } 2478 2479 if (enable_ept) 2480 ept_sync_global(); 2481 2482 return 0; 2483 } 2484 2485 static void vmclear_local_loaded_vmcss(void) 2486 { 2487 int cpu = raw_smp_processor_id(); 2488 struct loaded_vmcs *v, *n; 2489 2490 list_for_each_entry_safe(v, n, &per_cpu(loaded_vmcss_on_cpu, cpu), 2491 loaded_vmcss_on_cpu_link) 2492 __loaded_vmcs_clear(v); 2493 } 2494 2495 static void vmx_hardware_disable(void) 2496 { 2497 vmclear_local_loaded_vmcss(); 2498 2499 if (cpu_vmxoff()) 2500 kvm_spurious_fault(); 2501 2502 intel_pt_handle_vmx(0); 2503 } 2504 2505 /* 2506 * There is no X86_FEATURE for SGX yet, but anyway we need to query CPUID 2507 * directly instead of going through cpu_has(), to ensure KVM is trapping 2508 * ENCLS whenever it's supported in hardware. It does not matter whether 2509 * the host OS supports or has enabled SGX. 2510 */ 2511 static bool cpu_has_sgx(void) 2512 { 2513 return cpuid_eax(0) >= 0x12 && (cpuid_eax(0x12) & BIT(0)); 2514 } 2515 2516 /* 2517 * Some cpus support VM_{ENTRY,EXIT}_IA32_PERF_GLOBAL_CTRL but they 2518 * can't be used due to errata where VM Exit may incorrectly clear 2519 * IA32_PERF_GLOBAL_CTRL[34:32]. Work around the errata by using the 2520 * MSR load mechanism to switch IA32_PERF_GLOBAL_CTRL. 2521 */ 2522 static bool cpu_has_perf_global_ctrl_bug(void) 2523 { 2524 if (boot_cpu_data.x86 == 0x6) { 2525 switch (boot_cpu_data.x86_model) { 2526 case INTEL_FAM6_NEHALEM_EP: /* AAK155 */ 2527 case INTEL_FAM6_NEHALEM: /* AAP115 */ 2528 case INTEL_FAM6_WESTMERE: /* AAT100 */ 2529 case INTEL_FAM6_WESTMERE_EP: /* BC86,AAY89,BD102 */ 2530 case INTEL_FAM6_NEHALEM_EX: /* BA97 */ 2531 return true; 2532 default: 2533 break; 2534 } 2535 } 2536 2537 return false; 2538 } 2539 2540 static __init int adjust_vmx_controls(u32 ctl_min, u32 ctl_opt, 2541 u32 msr, u32 *result) 2542 { 2543 u32 vmx_msr_low, vmx_msr_high; 2544 u32 ctl = ctl_min | ctl_opt; 2545 2546 rdmsr(msr, vmx_msr_low, vmx_msr_high); 2547 2548 ctl &= vmx_msr_high; /* bit == 0 in high word ==> must be zero */ 2549 ctl |= vmx_msr_low; /* bit == 1 in low word ==> must be one */ 2550 2551 /* Ensure minimum (required) set of control bits are supported. */ 2552 if (ctl_min & ~ctl) 2553 return -EIO; 2554 2555 *result = ctl; 2556 return 0; 2557 } 2558 2559 static __init u64 adjust_vmx_controls64(u64 ctl_opt, u32 msr) 2560 { 2561 u64 allowed; 2562 2563 rdmsrl(msr, allowed); 2564 2565 return ctl_opt & allowed; 2566 } 2567 2568 static __init int setup_vmcs_config(struct vmcs_config *vmcs_conf, 2569 struct vmx_capability *vmx_cap) 2570 { 2571 u32 vmx_msr_low, vmx_msr_high; 2572 u32 _pin_based_exec_control = 0; 2573 u32 _cpu_based_exec_control = 0; 2574 u32 _cpu_based_2nd_exec_control = 0; 2575 u64 _cpu_based_3rd_exec_control = 0; 2576 u32 _vmexit_control = 0; 2577 u32 _vmentry_control = 0; 2578 u64 misc_msr; 2579 int i; 2580 2581 /* 2582 * LOAD/SAVE_DEBUG_CONTROLS are absent because both are mandatory. 2583 * SAVE_IA32_PAT and SAVE_IA32_EFER are absent because KVM always 2584 * intercepts writes to PAT and EFER, i.e. never enables those controls. 2585 */ 2586 struct { 2587 u32 entry_control; 2588 u32 exit_control; 2589 } const vmcs_entry_exit_pairs[] = { 2590 { VM_ENTRY_LOAD_IA32_PERF_GLOBAL_CTRL, VM_EXIT_LOAD_IA32_PERF_GLOBAL_CTRL }, 2591 { VM_ENTRY_LOAD_IA32_PAT, VM_EXIT_LOAD_IA32_PAT }, 2592 { VM_ENTRY_LOAD_IA32_EFER, VM_EXIT_LOAD_IA32_EFER }, 2593 { VM_ENTRY_LOAD_BNDCFGS, VM_EXIT_CLEAR_BNDCFGS }, 2594 { VM_ENTRY_LOAD_IA32_RTIT_CTL, VM_EXIT_CLEAR_IA32_RTIT_CTL }, 2595 }; 2596 2597 memset(vmcs_conf, 0, sizeof(*vmcs_conf)); 2598 2599 if (adjust_vmx_controls(KVM_REQUIRED_VMX_CPU_BASED_VM_EXEC_CONTROL, 2600 KVM_OPTIONAL_VMX_CPU_BASED_VM_EXEC_CONTROL, 2601 MSR_IA32_VMX_PROCBASED_CTLS, 2602 &_cpu_based_exec_control)) 2603 return -EIO; 2604 if (_cpu_based_exec_control & CPU_BASED_ACTIVATE_SECONDARY_CONTROLS) { 2605 if (adjust_vmx_controls(KVM_REQUIRED_VMX_SECONDARY_VM_EXEC_CONTROL, 2606 KVM_OPTIONAL_VMX_SECONDARY_VM_EXEC_CONTROL, 2607 MSR_IA32_VMX_PROCBASED_CTLS2, 2608 &_cpu_based_2nd_exec_control)) 2609 return -EIO; 2610 } 2611 #ifndef CONFIG_X86_64 2612 if (!(_cpu_based_2nd_exec_control & 2613 SECONDARY_EXEC_VIRTUALIZE_APIC_ACCESSES)) 2614 _cpu_based_exec_control &= ~CPU_BASED_TPR_SHADOW; 2615 #endif 2616 2617 if (!(_cpu_based_exec_control & CPU_BASED_TPR_SHADOW)) 2618 _cpu_based_2nd_exec_control &= ~( 2619 SECONDARY_EXEC_APIC_REGISTER_VIRT | 2620 SECONDARY_EXEC_VIRTUALIZE_X2APIC_MODE | 2621 SECONDARY_EXEC_VIRTUAL_INTR_DELIVERY); 2622 2623 rdmsr_safe(MSR_IA32_VMX_EPT_VPID_CAP, 2624 &vmx_cap->ept, &vmx_cap->vpid); 2625 2626 if (!(_cpu_based_2nd_exec_control & SECONDARY_EXEC_ENABLE_EPT) && 2627 vmx_cap->ept) { 2628 pr_warn_once("EPT CAP should not exist if not support " 2629 "1-setting enable EPT VM-execution control\n"); 2630 2631 if (error_on_inconsistent_vmcs_config) 2632 return -EIO; 2633 2634 vmx_cap->ept = 0; 2635 } 2636 if (!(_cpu_based_2nd_exec_control & SECONDARY_EXEC_ENABLE_VPID) && 2637 vmx_cap->vpid) { 2638 pr_warn_once("VPID CAP should not exist if not support " 2639 "1-setting enable VPID VM-execution control\n"); 2640 2641 if (error_on_inconsistent_vmcs_config) 2642 return -EIO; 2643 2644 vmx_cap->vpid = 0; 2645 } 2646 2647 if (!cpu_has_sgx()) 2648 _cpu_based_2nd_exec_control &= ~SECONDARY_EXEC_ENCLS_EXITING; 2649 2650 if (_cpu_based_exec_control & CPU_BASED_ACTIVATE_TERTIARY_CONTROLS) 2651 _cpu_based_3rd_exec_control = 2652 adjust_vmx_controls64(KVM_OPTIONAL_VMX_TERTIARY_VM_EXEC_CONTROL, 2653 MSR_IA32_VMX_PROCBASED_CTLS3); 2654 2655 if (adjust_vmx_controls(KVM_REQUIRED_VMX_VM_EXIT_CONTROLS, 2656 KVM_OPTIONAL_VMX_VM_EXIT_CONTROLS, 2657 MSR_IA32_VMX_EXIT_CTLS, 2658 &_vmexit_control)) 2659 return -EIO; 2660 2661 if (adjust_vmx_controls(KVM_REQUIRED_VMX_PIN_BASED_VM_EXEC_CONTROL, 2662 KVM_OPTIONAL_VMX_PIN_BASED_VM_EXEC_CONTROL, 2663 MSR_IA32_VMX_PINBASED_CTLS, 2664 &_pin_based_exec_control)) 2665 return -EIO; 2666 2667 if (cpu_has_broken_vmx_preemption_timer()) 2668 _pin_based_exec_control &= ~PIN_BASED_VMX_PREEMPTION_TIMER; 2669 if (!(_cpu_based_2nd_exec_control & 2670 SECONDARY_EXEC_VIRTUAL_INTR_DELIVERY)) 2671 _pin_based_exec_control &= ~PIN_BASED_POSTED_INTR; 2672 2673 if (adjust_vmx_controls(KVM_REQUIRED_VMX_VM_ENTRY_CONTROLS, 2674 KVM_OPTIONAL_VMX_VM_ENTRY_CONTROLS, 2675 MSR_IA32_VMX_ENTRY_CTLS, 2676 &_vmentry_control)) 2677 return -EIO; 2678 2679 for (i = 0; i < ARRAY_SIZE(vmcs_entry_exit_pairs); i++) { 2680 u32 n_ctrl = vmcs_entry_exit_pairs[i].entry_control; 2681 u32 x_ctrl = vmcs_entry_exit_pairs[i].exit_control; 2682 2683 if (!(_vmentry_control & n_ctrl) == !(_vmexit_control & x_ctrl)) 2684 continue; 2685 2686 pr_warn_once("Inconsistent VM-Entry/VM-Exit pair, entry = %x, exit = %x\n", 2687 _vmentry_control & n_ctrl, _vmexit_control & x_ctrl); 2688 2689 if (error_on_inconsistent_vmcs_config) 2690 return -EIO; 2691 2692 _vmentry_control &= ~n_ctrl; 2693 _vmexit_control &= ~x_ctrl; 2694 } 2695 2696 rdmsr(MSR_IA32_VMX_BASIC, vmx_msr_low, vmx_msr_high); 2697 2698 /* IA-32 SDM Vol 3B: VMCS size is never greater than 4kB. */ 2699 if ((vmx_msr_high & 0x1fff) > PAGE_SIZE) 2700 return -EIO; 2701 2702 #ifdef CONFIG_X86_64 2703 /* IA-32 SDM Vol 3B: 64-bit CPUs always have VMX_BASIC_MSR[48]==0. */ 2704 if (vmx_msr_high & (1u<<16)) 2705 return -EIO; 2706 #endif 2707 2708 /* Require Write-Back (WB) memory type for VMCS accesses. */ 2709 if (((vmx_msr_high >> 18) & 15) != 6) 2710 return -EIO; 2711 2712 rdmsrl(MSR_IA32_VMX_MISC, misc_msr); 2713 2714 vmcs_conf->size = vmx_msr_high & 0x1fff; 2715 vmcs_conf->basic_cap = vmx_msr_high & ~0x1fff; 2716 2717 vmcs_conf->revision_id = vmx_msr_low; 2718 2719 vmcs_conf->pin_based_exec_ctrl = _pin_based_exec_control; 2720 vmcs_conf->cpu_based_exec_ctrl = _cpu_based_exec_control; 2721 vmcs_conf->cpu_based_2nd_exec_ctrl = _cpu_based_2nd_exec_control; 2722 vmcs_conf->cpu_based_3rd_exec_ctrl = _cpu_based_3rd_exec_control; 2723 vmcs_conf->vmexit_ctrl = _vmexit_control; 2724 vmcs_conf->vmentry_ctrl = _vmentry_control; 2725 vmcs_conf->misc = misc_msr; 2726 2727 return 0; 2728 } 2729 2730 struct vmcs *alloc_vmcs_cpu(bool shadow, int cpu, gfp_t flags) 2731 { 2732 int node = cpu_to_node(cpu); 2733 struct page *pages; 2734 struct vmcs *vmcs; 2735 2736 pages = __alloc_pages_node(node, flags, 0); 2737 if (!pages) 2738 return NULL; 2739 vmcs = page_address(pages); 2740 memset(vmcs, 0, vmcs_config.size); 2741 2742 /* KVM supports Enlightened VMCS v1 only */ 2743 if (static_branch_unlikely(&enable_evmcs)) 2744 vmcs->hdr.revision_id = KVM_EVMCS_VERSION; 2745 else 2746 vmcs->hdr.revision_id = vmcs_config.revision_id; 2747 2748 if (shadow) 2749 vmcs->hdr.shadow_vmcs = 1; 2750 return vmcs; 2751 } 2752 2753 void free_vmcs(struct vmcs *vmcs) 2754 { 2755 free_page((unsigned long)vmcs); 2756 } 2757 2758 /* 2759 * Free a VMCS, but before that VMCLEAR it on the CPU where it was last loaded 2760 */ 2761 void free_loaded_vmcs(struct loaded_vmcs *loaded_vmcs) 2762 { 2763 if (!loaded_vmcs->vmcs) 2764 return; 2765 loaded_vmcs_clear(loaded_vmcs); 2766 free_vmcs(loaded_vmcs->vmcs); 2767 loaded_vmcs->vmcs = NULL; 2768 if (loaded_vmcs->msr_bitmap) 2769 free_page((unsigned long)loaded_vmcs->msr_bitmap); 2770 WARN_ON(loaded_vmcs->shadow_vmcs != NULL); 2771 } 2772 2773 int alloc_loaded_vmcs(struct loaded_vmcs *loaded_vmcs) 2774 { 2775 loaded_vmcs->vmcs = alloc_vmcs(false); 2776 if (!loaded_vmcs->vmcs) 2777 return -ENOMEM; 2778 2779 vmcs_clear(loaded_vmcs->vmcs); 2780 2781 loaded_vmcs->shadow_vmcs = NULL; 2782 loaded_vmcs->hv_timer_soft_disabled = false; 2783 loaded_vmcs->cpu = -1; 2784 loaded_vmcs->launched = 0; 2785 2786 if (cpu_has_vmx_msr_bitmap()) { 2787 loaded_vmcs->msr_bitmap = (unsigned long *) 2788 __get_free_page(GFP_KERNEL_ACCOUNT); 2789 if (!loaded_vmcs->msr_bitmap) 2790 goto out_vmcs; 2791 memset(loaded_vmcs->msr_bitmap, 0xff, PAGE_SIZE); 2792 } 2793 2794 memset(&loaded_vmcs->host_state, 0, sizeof(struct vmcs_host_state)); 2795 memset(&loaded_vmcs->controls_shadow, 0, 2796 sizeof(struct vmcs_controls_shadow)); 2797 2798 return 0; 2799 2800 out_vmcs: 2801 free_loaded_vmcs(loaded_vmcs); 2802 return -ENOMEM; 2803 } 2804 2805 static void free_kvm_area(void) 2806 { 2807 int cpu; 2808 2809 for_each_possible_cpu(cpu) { 2810 free_vmcs(per_cpu(vmxarea, cpu)); 2811 per_cpu(vmxarea, cpu) = NULL; 2812 } 2813 } 2814 2815 static __init int alloc_kvm_area(void) 2816 { 2817 int cpu; 2818 2819 for_each_possible_cpu(cpu) { 2820 struct vmcs *vmcs; 2821 2822 vmcs = alloc_vmcs_cpu(false, cpu, GFP_KERNEL); 2823 if (!vmcs) { 2824 free_kvm_area(); 2825 return -ENOMEM; 2826 } 2827 2828 /* 2829 * When eVMCS is enabled, alloc_vmcs_cpu() sets 2830 * vmcs->revision_id to KVM_EVMCS_VERSION instead of 2831 * revision_id reported by MSR_IA32_VMX_BASIC. 2832 * 2833 * However, even though not explicitly documented by 2834 * TLFS, VMXArea passed as VMXON argument should 2835 * still be marked with revision_id reported by 2836 * physical CPU. 2837 */ 2838 if (static_branch_unlikely(&enable_evmcs)) 2839 vmcs->hdr.revision_id = vmcs_config.revision_id; 2840 2841 per_cpu(vmxarea, cpu) = vmcs; 2842 } 2843 return 0; 2844 } 2845 2846 static void fix_pmode_seg(struct kvm_vcpu *vcpu, int seg, 2847 struct kvm_segment *save) 2848 { 2849 if (!emulate_invalid_guest_state) { 2850 /* 2851 * CS and SS RPL should be equal during guest entry according 2852 * to VMX spec, but in reality it is not always so. Since vcpu 2853 * is in the middle of the transition from real mode to 2854 * protected mode it is safe to assume that RPL 0 is a good 2855 * default value. 2856 */ 2857 if (seg == VCPU_SREG_CS || seg == VCPU_SREG_SS) 2858 save->selector &= ~SEGMENT_RPL_MASK; 2859 save->dpl = save->selector & SEGMENT_RPL_MASK; 2860 save->s = 1; 2861 } 2862 __vmx_set_segment(vcpu, save, seg); 2863 } 2864 2865 static void enter_pmode(struct kvm_vcpu *vcpu) 2866 { 2867 unsigned long flags; 2868 struct vcpu_vmx *vmx = to_vmx(vcpu); 2869 2870 /* 2871 * Update real mode segment cache. It may be not up-to-date if segment 2872 * register was written while vcpu was in a guest mode. 2873 */ 2874 vmx_get_segment(vcpu, &vmx->rmode.segs[VCPU_SREG_ES], VCPU_SREG_ES); 2875 vmx_get_segment(vcpu, &vmx->rmode.segs[VCPU_SREG_DS], VCPU_SREG_DS); 2876 vmx_get_segment(vcpu, &vmx->rmode.segs[VCPU_SREG_FS], VCPU_SREG_FS); 2877 vmx_get_segment(vcpu, &vmx->rmode.segs[VCPU_SREG_GS], VCPU_SREG_GS); 2878 vmx_get_segment(vcpu, &vmx->rmode.segs[VCPU_SREG_SS], VCPU_SREG_SS); 2879 vmx_get_segment(vcpu, &vmx->rmode.segs[VCPU_SREG_CS], VCPU_SREG_CS); 2880 2881 vmx->rmode.vm86_active = 0; 2882 2883 __vmx_set_segment(vcpu, &vmx->rmode.segs[VCPU_SREG_TR], VCPU_SREG_TR); 2884 2885 flags = vmcs_readl(GUEST_RFLAGS); 2886 flags &= RMODE_GUEST_OWNED_EFLAGS_BITS; 2887 flags |= vmx->rmode.save_rflags & ~RMODE_GUEST_OWNED_EFLAGS_BITS; 2888 vmcs_writel(GUEST_RFLAGS, flags); 2889 2890 vmcs_writel(GUEST_CR4, (vmcs_readl(GUEST_CR4) & ~X86_CR4_VME) | 2891 (vmcs_readl(CR4_READ_SHADOW) & X86_CR4_VME)); 2892 2893 vmx_update_exception_bitmap(vcpu); 2894 2895 fix_pmode_seg(vcpu, VCPU_SREG_CS, &vmx->rmode.segs[VCPU_SREG_CS]); 2896 fix_pmode_seg(vcpu, VCPU_SREG_SS, &vmx->rmode.segs[VCPU_SREG_SS]); 2897 fix_pmode_seg(vcpu, VCPU_SREG_ES, &vmx->rmode.segs[VCPU_SREG_ES]); 2898 fix_pmode_seg(vcpu, VCPU_SREG_DS, &vmx->rmode.segs[VCPU_SREG_DS]); 2899 fix_pmode_seg(vcpu, VCPU_SREG_FS, &vmx->rmode.segs[VCPU_SREG_FS]); 2900 fix_pmode_seg(vcpu, VCPU_SREG_GS, &vmx->rmode.segs[VCPU_SREG_GS]); 2901 } 2902 2903 static void fix_rmode_seg(int seg, struct kvm_segment *save) 2904 { 2905 const struct kvm_vmx_segment_field *sf = &kvm_vmx_segment_fields[seg]; 2906 struct kvm_segment var = *save; 2907 2908 var.dpl = 0x3; 2909 if (seg == VCPU_SREG_CS) 2910 var.type = 0x3; 2911 2912 if (!emulate_invalid_guest_state) { 2913 var.selector = var.base >> 4; 2914 var.base = var.base & 0xffff0; 2915 var.limit = 0xffff; 2916 var.g = 0; 2917 var.db = 0; 2918 var.present = 1; 2919 var.s = 1; 2920 var.l = 0; 2921 var.unusable = 0; 2922 var.type = 0x3; 2923 var.avl = 0; 2924 if (save->base & 0xf) 2925 printk_once(KERN_WARNING "kvm: segment base is not " 2926 "paragraph aligned when entering " 2927 "protected mode (seg=%d)", seg); 2928 } 2929 2930 vmcs_write16(sf->selector, var.selector); 2931 vmcs_writel(sf->base, var.base); 2932 vmcs_write32(sf->limit, var.limit); 2933 vmcs_write32(sf->ar_bytes, vmx_segment_access_rights(&var)); 2934 } 2935 2936 static void enter_rmode(struct kvm_vcpu *vcpu) 2937 { 2938 unsigned long flags; 2939 struct vcpu_vmx *vmx = to_vmx(vcpu); 2940 struct kvm_vmx *kvm_vmx = to_kvm_vmx(vcpu->kvm); 2941 2942 vmx_get_segment(vcpu, &vmx->rmode.segs[VCPU_SREG_TR], VCPU_SREG_TR); 2943 vmx_get_segment(vcpu, &vmx->rmode.segs[VCPU_SREG_ES], VCPU_SREG_ES); 2944 vmx_get_segment(vcpu, &vmx->rmode.segs[VCPU_SREG_DS], VCPU_SREG_DS); 2945 vmx_get_segment(vcpu, &vmx->rmode.segs[VCPU_SREG_FS], VCPU_SREG_FS); 2946 vmx_get_segment(vcpu, &vmx->rmode.segs[VCPU_SREG_GS], VCPU_SREG_GS); 2947 vmx_get_segment(vcpu, &vmx->rmode.segs[VCPU_SREG_SS], VCPU_SREG_SS); 2948 vmx_get_segment(vcpu, &vmx->rmode.segs[VCPU_SREG_CS], VCPU_SREG_CS); 2949 2950 vmx->rmode.vm86_active = 1; 2951 2952 /* 2953 * Very old userspace does not call KVM_SET_TSS_ADDR before entering 2954 * vcpu. Warn the user that an update is overdue. 2955 */ 2956 if (!kvm_vmx->tss_addr) 2957 printk_once(KERN_WARNING "kvm: KVM_SET_TSS_ADDR need to be " 2958 "called before entering vcpu\n"); 2959 2960 vmx_segment_cache_clear(vmx); 2961 2962 vmcs_writel(GUEST_TR_BASE, kvm_vmx->tss_addr); 2963 vmcs_write32(GUEST_TR_LIMIT, RMODE_TSS_SIZE - 1); 2964 vmcs_write32(GUEST_TR_AR_BYTES, 0x008b); 2965 2966 flags = vmcs_readl(GUEST_RFLAGS); 2967 vmx->rmode.save_rflags = flags; 2968 2969 flags |= X86_EFLAGS_IOPL | X86_EFLAGS_VM; 2970 2971 vmcs_writel(GUEST_RFLAGS, flags); 2972 vmcs_writel(GUEST_CR4, vmcs_readl(GUEST_CR4) | X86_CR4_VME); 2973 vmx_update_exception_bitmap(vcpu); 2974 2975 fix_rmode_seg(VCPU_SREG_SS, &vmx->rmode.segs[VCPU_SREG_SS]); 2976 fix_rmode_seg(VCPU_SREG_CS, &vmx->rmode.segs[VCPU_SREG_CS]); 2977 fix_rmode_seg(VCPU_SREG_ES, &vmx->rmode.segs[VCPU_SREG_ES]); 2978 fix_rmode_seg(VCPU_SREG_DS, &vmx->rmode.segs[VCPU_SREG_DS]); 2979 fix_rmode_seg(VCPU_SREG_GS, &vmx->rmode.segs[VCPU_SREG_GS]); 2980 fix_rmode_seg(VCPU_SREG_FS, &vmx->rmode.segs[VCPU_SREG_FS]); 2981 } 2982 2983 int vmx_set_efer(struct kvm_vcpu *vcpu, u64 efer) 2984 { 2985 struct vcpu_vmx *vmx = to_vmx(vcpu); 2986 2987 /* Nothing to do if hardware doesn't support EFER. */ 2988 if (!vmx_find_uret_msr(vmx, MSR_EFER)) 2989 return 0; 2990 2991 vcpu->arch.efer = efer; 2992 #ifdef CONFIG_X86_64 2993 if (efer & EFER_LMA) 2994 vm_entry_controls_setbit(vmx, VM_ENTRY_IA32E_MODE); 2995 else 2996 vm_entry_controls_clearbit(vmx, VM_ENTRY_IA32E_MODE); 2997 #else 2998 if (KVM_BUG_ON(efer & EFER_LMA, vcpu->kvm)) 2999 return 1; 3000 #endif 3001 3002 vmx_setup_uret_msrs(vmx); 3003 return 0; 3004 } 3005 3006 #ifdef CONFIG_X86_64 3007 3008 static void enter_lmode(struct kvm_vcpu *vcpu) 3009 { 3010 u32 guest_tr_ar; 3011 3012 vmx_segment_cache_clear(to_vmx(vcpu)); 3013 3014 guest_tr_ar = vmcs_read32(GUEST_TR_AR_BYTES); 3015 if ((guest_tr_ar & VMX_AR_TYPE_MASK) != VMX_AR_TYPE_BUSY_64_TSS) { 3016 pr_debug_ratelimited("%s: tss fixup for long mode. \n", 3017 __func__); 3018 vmcs_write32(GUEST_TR_AR_BYTES, 3019 (guest_tr_ar & ~VMX_AR_TYPE_MASK) 3020 | VMX_AR_TYPE_BUSY_64_TSS); 3021 } 3022 vmx_set_efer(vcpu, vcpu->arch.efer | EFER_LMA); 3023 } 3024 3025 static void exit_lmode(struct kvm_vcpu *vcpu) 3026 { 3027 vmx_set_efer(vcpu, vcpu->arch.efer & ~EFER_LMA); 3028 } 3029 3030 #endif 3031 3032 static void vmx_flush_tlb_all(struct kvm_vcpu *vcpu) 3033 { 3034 struct vcpu_vmx *vmx = to_vmx(vcpu); 3035 3036 /* 3037 * INVEPT must be issued when EPT is enabled, irrespective of VPID, as 3038 * the CPU is not required to invalidate guest-physical mappings on 3039 * VM-Entry, even if VPID is disabled. Guest-physical mappings are 3040 * associated with the root EPT structure and not any particular VPID 3041 * (INVVPID also isn't required to invalidate guest-physical mappings). 3042 */ 3043 if (enable_ept) { 3044 ept_sync_global(); 3045 } else if (enable_vpid) { 3046 if (cpu_has_vmx_invvpid_global()) { 3047 vpid_sync_vcpu_global(); 3048 } else { 3049 vpid_sync_vcpu_single(vmx->vpid); 3050 vpid_sync_vcpu_single(vmx->nested.vpid02); 3051 } 3052 } 3053 } 3054 3055 static inline int vmx_get_current_vpid(struct kvm_vcpu *vcpu) 3056 { 3057 if (is_guest_mode(vcpu)) 3058 return nested_get_vpid02(vcpu); 3059 return to_vmx(vcpu)->vpid; 3060 } 3061 3062 static void vmx_flush_tlb_current(struct kvm_vcpu *vcpu) 3063 { 3064 struct kvm_mmu *mmu = vcpu->arch.mmu; 3065 u64 root_hpa = mmu->root.hpa; 3066 3067 /* No flush required if the current context is invalid. */ 3068 if (!VALID_PAGE(root_hpa)) 3069 return; 3070 3071 if (enable_ept) 3072 ept_sync_context(construct_eptp(vcpu, root_hpa, 3073 mmu->root_role.level)); 3074 else 3075 vpid_sync_context(vmx_get_current_vpid(vcpu)); 3076 } 3077 3078 static void vmx_flush_tlb_gva(struct kvm_vcpu *vcpu, gva_t addr) 3079 { 3080 /* 3081 * vpid_sync_vcpu_addr() is a nop if vpid==0, see the comment in 3082 * vmx_flush_tlb_guest() for an explanation of why this is ok. 3083 */ 3084 vpid_sync_vcpu_addr(vmx_get_current_vpid(vcpu), addr); 3085 } 3086 3087 static void vmx_flush_tlb_guest(struct kvm_vcpu *vcpu) 3088 { 3089 /* 3090 * vpid_sync_context() is a nop if vpid==0, e.g. if enable_vpid==0 or a 3091 * vpid couldn't be allocated for this vCPU. VM-Enter and VM-Exit are 3092 * required to flush GVA->{G,H}PA mappings from the TLB if vpid is 3093 * disabled (VM-Enter with vpid enabled and vpid==0 is disallowed), 3094 * i.e. no explicit INVVPID is necessary. 3095 */ 3096 vpid_sync_context(vmx_get_current_vpid(vcpu)); 3097 } 3098 3099 void vmx_ept_load_pdptrs(struct kvm_vcpu *vcpu) 3100 { 3101 struct kvm_mmu *mmu = vcpu->arch.walk_mmu; 3102 3103 if (!kvm_register_is_dirty(vcpu, VCPU_EXREG_PDPTR)) 3104 return; 3105 3106 if (is_pae_paging(vcpu)) { 3107 vmcs_write64(GUEST_PDPTR0, mmu->pdptrs[0]); 3108 vmcs_write64(GUEST_PDPTR1, mmu->pdptrs[1]); 3109 vmcs_write64(GUEST_PDPTR2, mmu->pdptrs[2]); 3110 vmcs_write64(GUEST_PDPTR3, mmu->pdptrs[3]); 3111 } 3112 } 3113 3114 void ept_save_pdptrs(struct kvm_vcpu *vcpu) 3115 { 3116 struct kvm_mmu *mmu = vcpu->arch.walk_mmu; 3117 3118 if (WARN_ON_ONCE(!is_pae_paging(vcpu))) 3119 return; 3120 3121 mmu->pdptrs[0] = vmcs_read64(GUEST_PDPTR0); 3122 mmu->pdptrs[1] = vmcs_read64(GUEST_PDPTR1); 3123 mmu->pdptrs[2] = vmcs_read64(GUEST_PDPTR2); 3124 mmu->pdptrs[3] = vmcs_read64(GUEST_PDPTR3); 3125 3126 kvm_register_mark_available(vcpu, VCPU_EXREG_PDPTR); 3127 } 3128 3129 #define CR3_EXITING_BITS (CPU_BASED_CR3_LOAD_EXITING | \ 3130 CPU_BASED_CR3_STORE_EXITING) 3131 3132 void vmx_set_cr0(struct kvm_vcpu *vcpu, unsigned long cr0) 3133 { 3134 struct vcpu_vmx *vmx = to_vmx(vcpu); 3135 unsigned long hw_cr0, old_cr0_pg; 3136 u32 tmp; 3137 3138 old_cr0_pg = kvm_read_cr0_bits(vcpu, X86_CR0_PG); 3139 3140 hw_cr0 = (cr0 & ~KVM_VM_CR0_ALWAYS_OFF); 3141 if (is_unrestricted_guest(vcpu)) 3142 hw_cr0 |= KVM_VM_CR0_ALWAYS_ON_UNRESTRICTED_GUEST; 3143 else { 3144 hw_cr0 |= KVM_VM_CR0_ALWAYS_ON; 3145 if (!enable_ept) 3146 hw_cr0 |= X86_CR0_WP; 3147 3148 if (vmx->rmode.vm86_active && (cr0 & X86_CR0_PE)) 3149 enter_pmode(vcpu); 3150 3151 if (!vmx->rmode.vm86_active && !(cr0 & X86_CR0_PE)) 3152 enter_rmode(vcpu); 3153 } 3154 3155 vmcs_writel(CR0_READ_SHADOW, cr0); 3156 vmcs_writel(GUEST_CR0, hw_cr0); 3157 vcpu->arch.cr0 = cr0; 3158 kvm_register_mark_available(vcpu, VCPU_EXREG_CR0); 3159 3160 #ifdef CONFIG_X86_64 3161 if (vcpu->arch.efer & EFER_LME) { 3162 if (!old_cr0_pg && (cr0 & X86_CR0_PG)) 3163 enter_lmode(vcpu); 3164 else if (old_cr0_pg && !(cr0 & X86_CR0_PG)) 3165 exit_lmode(vcpu); 3166 } 3167 #endif 3168 3169 if (enable_ept && !is_unrestricted_guest(vcpu)) { 3170 /* 3171 * Ensure KVM has an up-to-date snapshot of the guest's CR3. If 3172 * the below code _enables_ CR3 exiting, vmx_cache_reg() will 3173 * (correctly) stop reading vmcs.GUEST_CR3 because it thinks 3174 * KVM's CR3 is installed. 3175 */ 3176 if (!kvm_register_is_available(vcpu, VCPU_EXREG_CR3)) 3177 vmx_cache_reg(vcpu, VCPU_EXREG_CR3); 3178 3179 /* 3180 * When running with EPT but not unrestricted guest, KVM must 3181 * intercept CR3 accesses when paging is _disabled_. This is 3182 * necessary because restricted guests can't actually run with 3183 * paging disabled, and so KVM stuffs its own CR3 in order to 3184 * run the guest when identity mapped page tables. 3185 * 3186 * Do _NOT_ check the old CR0.PG, e.g. to optimize away the 3187 * update, it may be stale with respect to CR3 interception, 3188 * e.g. after nested VM-Enter. 3189 * 3190 * Lastly, honor L1's desires, i.e. intercept CR3 loads and/or 3191 * stores to forward them to L1, even if KVM does not need to 3192 * intercept them to preserve its identity mapped page tables. 3193 */ 3194 if (!(cr0 & X86_CR0_PG)) { 3195 exec_controls_setbit(vmx, CR3_EXITING_BITS); 3196 } else if (!is_guest_mode(vcpu)) { 3197 exec_controls_clearbit(vmx, CR3_EXITING_BITS); 3198 } else { 3199 tmp = exec_controls_get(vmx); 3200 tmp &= ~CR3_EXITING_BITS; 3201 tmp |= get_vmcs12(vcpu)->cpu_based_vm_exec_control & CR3_EXITING_BITS; 3202 exec_controls_set(vmx, tmp); 3203 } 3204 3205 /* Note, vmx_set_cr4() consumes the new vcpu->arch.cr0. */ 3206 if ((old_cr0_pg ^ cr0) & X86_CR0_PG) 3207 vmx_set_cr4(vcpu, kvm_read_cr4(vcpu)); 3208 3209 /* 3210 * When !CR0_PG -> CR0_PG, vcpu->arch.cr3 becomes active, but 3211 * GUEST_CR3 is still vmx->ept_identity_map_addr if EPT + !URG. 3212 */ 3213 if (!(old_cr0_pg & X86_CR0_PG) && (cr0 & X86_CR0_PG)) 3214 kvm_register_mark_dirty(vcpu, VCPU_EXREG_CR3); 3215 } 3216 3217 /* depends on vcpu->arch.cr0 to be set to a new value */ 3218 vmx->emulation_required = vmx_emulation_required(vcpu); 3219 } 3220 3221 static int vmx_get_max_tdp_level(void) 3222 { 3223 if (cpu_has_vmx_ept_5levels()) 3224 return 5; 3225 return 4; 3226 } 3227 3228 u64 construct_eptp(struct kvm_vcpu *vcpu, hpa_t root_hpa, int root_level) 3229 { 3230 u64 eptp = VMX_EPTP_MT_WB; 3231 3232 eptp |= (root_level == 5) ? VMX_EPTP_PWL_5 : VMX_EPTP_PWL_4; 3233 3234 if (enable_ept_ad_bits && 3235 (!is_guest_mode(vcpu) || nested_ept_ad_enabled(vcpu))) 3236 eptp |= VMX_EPTP_AD_ENABLE_BIT; 3237 eptp |= root_hpa; 3238 3239 return eptp; 3240 } 3241 3242 static void vmx_load_mmu_pgd(struct kvm_vcpu *vcpu, hpa_t root_hpa, 3243 int root_level) 3244 { 3245 struct kvm *kvm = vcpu->kvm; 3246 bool update_guest_cr3 = true; 3247 unsigned long guest_cr3; 3248 u64 eptp; 3249 3250 if (enable_ept) { 3251 eptp = construct_eptp(vcpu, root_hpa, root_level); 3252 vmcs_write64(EPT_POINTER, eptp); 3253 3254 hv_track_root_tdp(vcpu, root_hpa); 3255 3256 if (!enable_unrestricted_guest && !is_paging(vcpu)) 3257 guest_cr3 = to_kvm_vmx(kvm)->ept_identity_map_addr; 3258 else if (kvm_register_is_dirty(vcpu, VCPU_EXREG_CR3)) 3259 guest_cr3 = vcpu->arch.cr3; 3260 else /* vmcs.GUEST_CR3 is already up-to-date. */ 3261 update_guest_cr3 = false; 3262 vmx_ept_load_pdptrs(vcpu); 3263 } else { 3264 guest_cr3 = root_hpa | kvm_get_active_pcid(vcpu); 3265 } 3266 3267 if (update_guest_cr3) 3268 vmcs_writel(GUEST_CR3, guest_cr3); 3269 } 3270 3271 3272 static bool vmx_is_valid_cr4(struct kvm_vcpu *vcpu, unsigned long cr4) 3273 { 3274 /* 3275 * We operate under the default treatment of SMM, so VMX cannot be 3276 * enabled under SMM. Note, whether or not VMXE is allowed at all, 3277 * i.e. is a reserved bit, is handled by common x86 code. 3278 */ 3279 if ((cr4 & X86_CR4_VMXE) && is_smm(vcpu)) 3280 return false; 3281 3282 if (to_vmx(vcpu)->nested.vmxon && !nested_cr4_valid(vcpu, cr4)) 3283 return false; 3284 3285 return true; 3286 } 3287 3288 void vmx_set_cr4(struct kvm_vcpu *vcpu, unsigned long cr4) 3289 { 3290 unsigned long old_cr4 = vcpu->arch.cr4; 3291 struct vcpu_vmx *vmx = to_vmx(vcpu); 3292 /* 3293 * Pass through host's Machine Check Enable value to hw_cr4, which 3294 * is in force while we are in guest mode. Do not let guests control 3295 * this bit, even if host CR4.MCE == 0. 3296 */ 3297 unsigned long hw_cr4; 3298 3299 hw_cr4 = (cr4_read_shadow() & X86_CR4_MCE) | (cr4 & ~X86_CR4_MCE); 3300 if (is_unrestricted_guest(vcpu)) 3301 hw_cr4 |= KVM_VM_CR4_ALWAYS_ON_UNRESTRICTED_GUEST; 3302 else if (vmx->rmode.vm86_active) 3303 hw_cr4 |= KVM_RMODE_VM_CR4_ALWAYS_ON; 3304 else 3305 hw_cr4 |= KVM_PMODE_VM_CR4_ALWAYS_ON; 3306 3307 if (!boot_cpu_has(X86_FEATURE_UMIP) && vmx_umip_emulated()) { 3308 if (cr4 & X86_CR4_UMIP) { 3309 secondary_exec_controls_setbit(vmx, SECONDARY_EXEC_DESC); 3310 hw_cr4 &= ~X86_CR4_UMIP; 3311 } else if (!is_guest_mode(vcpu) || 3312 !nested_cpu_has2(get_vmcs12(vcpu), SECONDARY_EXEC_DESC)) { 3313 secondary_exec_controls_clearbit(vmx, SECONDARY_EXEC_DESC); 3314 } 3315 } 3316 3317 vcpu->arch.cr4 = cr4; 3318 kvm_register_mark_available(vcpu, VCPU_EXREG_CR4); 3319 3320 if (!is_unrestricted_guest(vcpu)) { 3321 if (enable_ept) { 3322 if (!is_paging(vcpu)) { 3323 hw_cr4 &= ~X86_CR4_PAE; 3324 hw_cr4 |= X86_CR4_PSE; 3325 } else if (!(cr4 & X86_CR4_PAE)) { 3326 hw_cr4 &= ~X86_CR4_PAE; 3327 } 3328 } 3329 3330 /* 3331 * SMEP/SMAP/PKU is disabled if CPU is in non-paging mode in 3332 * hardware. To emulate this behavior, SMEP/SMAP/PKU needs 3333 * to be manually disabled when guest switches to non-paging 3334 * mode. 3335 * 3336 * If !enable_unrestricted_guest, the CPU is always running 3337 * with CR0.PG=1 and CR4 needs to be modified. 3338 * If enable_unrestricted_guest, the CPU automatically 3339 * disables SMEP/SMAP/PKU when the guest sets CR0.PG=0. 3340 */ 3341 if (!is_paging(vcpu)) 3342 hw_cr4 &= ~(X86_CR4_SMEP | X86_CR4_SMAP | X86_CR4_PKE); 3343 } 3344 3345 vmcs_writel(CR4_READ_SHADOW, cr4); 3346 vmcs_writel(GUEST_CR4, hw_cr4); 3347 3348 if ((cr4 ^ old_cr4) & (X86_CR4_OSXSAVE | X86_CR4_PKE)) 3349 kvm_update_cpuid_runtime(vcpu); 3350 } 3351 3352 void vmx_get_segment(struct kvm_vcpu *vcpu, struct kvm_segment *var, int seg) 3353 { 3354 struct vcpu_vmx *vmx = to_vmx(vcpu); 3355 u32 ar; 3356 3357 if (vmx->rmode.vm86_active && seg != VCPU_SREG_LDTR) { 3358 *var = vmx->rmode.segs[seg]; 3359 if (seg == VCPU_SREG_TR 3360 || var->selector == vmx_read_guest_seg_selector(vmx, seg)) 3361 return; 3362 var->base = vmx_read_guest_seg_base(vmx, seg); 3363 var->selector = vmx_read_guest_seg_selector(vmx, seg); 3364 return; 3365 } 3366 var->base = vmx_read_guest_seg_base(vmx, seg); 3367 var->limit = vmx_read_guest_seg_limit(vmx, seg); 3368 var->selector = vmx_read_guest_seg_selector(vmx, seg); 3369 ar = vmx_read_guest_seg_ar(vmx, seg); 3370 var->unusable = (ar >> 16) & 1; 3371 var->type = ar & 15; 3372 var->s = (ar >> 4) & 1; 3373 var->dpl = (ar >> 5) & 3; 3374 /* 3375 * Some userspaces do not preserve unusable property. Since usable 3376 * segment has to be present according to VMX spec we can use present 3377 * property to amend userspace bug by making unusable segment always 3378 * nonpresent. vmx_segment_access_rights() already marks nonpresent 3379 * segment as unusable. 3380 */ 3381 var->present = !var->unusable; 3382 var->avl = (ar >> 12) & 1; 3383 var->l = (ar >> 13) & 1; 3384 var->db = (ar >> 14) & 1; 3385 var->g = (ar >> 15) & 1; 3386 } 3387 3388 static u64 vmx_get_segment_base(struct kvm_vcpu *vcpu, int seg) 3389 { 3390 struct kvm_segment s; 3391 3392 if (to_vmx(vcpu)->rmode.vm86_active) { 3393 vmx_get_segment(vcpu, &s, seg); 3394 return s.base; 3395 } 3396 return vmx_read_guest_seg_base(to_vmx(vcpu), seg); 3397 } 3398 3399 int vmx_get_cpl(struct kvm_vcpu *vcpu) 3400 { 3401 struct vcpu_vmx *vmx = to_vmx(vcpu); 3402 3403 if (unlikely(vmx->rmode.vm86_active)) 3404 return 0; 3405 else { 3406 int ar = vmx_read_guest_seg_ar(vmx, VCPU_SREG_SS); 3407 return VMX_AR_DPL(ar); 3408 } 3409 } 3410 3411 static u32 vmx_segment_access_rights(struct kvm_segment *var) 3412 { 3413 u32 ar; 3414 3415 if (var->unusable || !var->present) 3416 ar = 1 << 16; 3417 else { 3418 ar = var->type & 15; 3419 ar |= (var->s & 1) << 4; 3420 ar |= (var->dpl & 3) << 5; 3421 ar |= (var->present & 1) << 7; 3422 ar |= (var->avl & 1) << 12; 3423 ar |= (var->l & 1) << 13; 3424 ar |= (var->db & 1) << 14; 3425 ar |= (var->g & 1) << 15; 3426 } 3427 3428 return ar; 3429 } 3430 3431 void __vmx_set_segment(struct kvm_vcpu *vcpu, struct kvm_segment *var, int seg) 3432 { 3433 struct vcpu_vmx *vmx = to_vmx(vcpu); 3434 const struct kvm_vmx_segment_field *sf = &kvm_vmx_segment_fields[seg]; 3435 3436 vmx_segment_cache_clear(vmx); 3437 3438 if (vmx->rmode.vm86_active && seg != VCPU_SREG_LDTR) { 3439 vmx->rmode.segs[seg] = *var; 3440 if (seg == VCPU_SREG_TR) 3441 vmcs_write16(sf->selector, var->selector); 3442 else if (var->s) 3443 fix_rmode_seg(seg, &vmx->rmode.segs[seg]); 3444 return; 3445 } 3446 3447 vmcs_writel(sf->base, var->base); 3448 vmcs_write32(sf->limit, var->limit); 3449 vmcs_write16(sf->selector, var->selector); 3450 3451 /* 3452 * Fix the "Accessed" bit in AR field of segment registers for older 3453 * qemu binaries. 3454 * IA32 arch specifies that at the time of processor reset the 3455 * "Accessed" bit in the AR field of segment registers is 1. And qemu 3456 * is setting it to 0 in the userland code. This causes invalid guest 3457 * state vmexit when "unrestricted guest" mode is turned on. 3458 * Fix for this setup issue in cpu_reset is being pushed in the qemu 3459 * tree. Newer qemu binaries with that qemu fix would not need this 3460 * kvm hack. 3461 */ 3462 if (is_unrestricted_guest(vcpu) && (seg != VCPU_SREG_LDTR)) 3463 var->type |= 0x1; /* Accessed */ 3464 3465 vmcs_write32(sf->ar_bytes, vmx_segment_access_rights(var)); 3466 } 3467 3468 static void vmx_set_segment(struct kvm_vcpu *vcpu, struct kvm_segment *var, int seg) 3469 { 3470 __vmx_set_segment(vcpu, var, seg); 3471 3472 to_vmx(vcpu)->emulation_required = vmx_emulation_required(vcpu); 3473 } 3474 3475 static void vmx_get_cs_db_l_bits(struct kvm_vcpu *vcpu, int *db, int *l) 3476 { 3477 u32 ar = vmx_read_guest_seg_ar(to_vmx(vcpu), VCPU_SREG_CS); 3478 3479 *db = (ar >> 14) & 1; 3480 *l = (ar >> 13) & 1; 3481 } 3482 3483 static void vmx_get_idt(struct kvm_vcpu *vcpu, struct desc_ptr *dt) 3484 { 3485 dt->size = vmcs_read32(GUEST_IDTR_LIMIT); 3486 dt->address = vmcs_readl(GUEST_IDTR_BASE); 3487 } 3488 3489 static void vmx_set_idt(struct kvm_vcpu *vcpu, struct desc_ptr *dt) 3490 { 3491 vmcs_write32(GUEST_IDTR_LIMIT, dt->size); 3492 vmcs_writel(GUEST_IDTR_BASE, dt->address); 3493 } 3494 3495 static void vmx_get_gdt(struct kvm_vcpu *vcpu, struct desc_ptr *dt) 3496 { 3497 dt->size = vmcs_read32(GUEST_GDTR_LIMIT); 3498 dt->address = vmcs_readl(GUEST_GDTR_BASE); 3499 } 3500 3501 static void vmx_set_gdt(struct kvm_vcpu *vcpu, struct desc_ptr *dt) 3502 { 3503 vmcs_write32(GUEST_GDTR_LIMIT, dt->size); 3504 vmcs_writel(GUEST_GDTR_BASE, dt->address); 3505 } 3506 3507 static bool rmode_segment_valid(struct kvm_vcpu *vcpu, int seg) 3508 { 3509 struct kvm_segment var; 3510 u32 ar; 3511 3512 vmx_get_segment(vcpu, &var, seg); 3513 var.dpl = 0x3; 3514 if (seg == VCPU_SREG_CS) 3515 var.type = 0x3; 3516 ar = vmx_segment_access_rights(&var); 3517 3518 if (var.base != (var.selector << 4)) 3519 return false; 3520 if (var.limit != 0xffff) 3521 return false; 3522 if (ar != 0xf3) 3523 return false; 3524 3525 return true; 3526 } 3527 3528 static bool code_segment_valid(struct kvm_vcpu *vcpu) 3529 { 3530 struct kvm_segment cs; 3531 unsigned int cs_rpl; 3532 3533 vmx_get_segment(vcpu, &cs, VCPU_SREG_CS); 3534 cs_rpl = cs.selector & SEGMENT_RPL_MASK; 3535 3536 if (cs.unusable) 3537 return false; 3538 if (~cs.type & (VMX_AR_TYPE_CODE_MASK|VMX_AR_TYPE_ACCESSES_MASK)) 3539 return false; 3540 if (!cs.s) 3541 return false; 3542 if (cs.type & VMX_AR_TYPE_WRITEABLE_MASK) { 3543 if (cs.dpl > cs_rpl) 3544 return false; 3545 } else { 3546 if (cs.dpl != cs_rpl) 3547 return false; 3548 } 3549 if (!cs.present) 3550 return false; 3551 3552 /* TODO: Add Reserved field check, this'll require a new member in the kvm_segment_field structure */ 3553 return true; 3554 } 3555 3556 static bool stack_segment_valid(struct kvm_vcpu *vcpu) 3557 { 3558 struct kvm_segment ss; 3559 unsigned int ss_rpl; 3560 3561 vmx_get_segment(vcpu, &ss, VCPU_SREG_SS); 3562 ss_rpl = ss.selector & SEGMENT_RPL_MASK; 3563 3564 if (ss.unusable) 3565 return true; 3566 if (ss.type != 3 && ss.type != 7) 3567 return false; 3568 if (!ss.s) 3569 return false; 3570 if (ss.dpl != ss_rpl) /* DPL != RPL */ 3571 return false; 3572 if (!ss.present) 3573 return false; 3574 3575 return true; 3576 } 3577 3578 static bool data_segment_valid(struct kvm_vcpu *vcpu, int seg) 3579 { 3580 struct kvm_segment var; 3581 unsigned int rpl; 3582 3583 vmx_get_segment(vcpu, &var, seg); 3584 rpl = var.selector & SEGMENT_RPL_MASK; 3585 3586 if (var.unusable) 3587 return true; 3588 if (!var.s) 3589 return false; 3590 if (!var.present) 3591 return false; 3592 if (~var.type & (VMX_AR_TYPE_CODE_MASK|VMX_AR_TYPE_WRITEABLE_MASK)) { 3593 if (var.dpl < rpl) /* DPL < RPL */ 3594 return false; 3595 } 3596 3597 /* TODO: Add other members to kvm_segment_field to allow checking for other access 3598 * rights flags 3599 */ 3600 return true; 3601 } 3602 3603 static bool tr_valid(struct kvm_vcpu *vcpu) 3604 { 3605 struct kvm_segment tr; 3606 3607 vmx_get_segment(vcpu, &tr, VCPU_SREG_TR); 3608 3609 if (tr.unusable) 3610 return false; 3611 if (tr.selector & SEGMENT_TI_MASK) /* TI = 1 */ 3612 return false; 3613 if (tr.type != 3 && tr.type != 11) /* TODO: Check if guest is in IA32e mode */ 3614 return false; 3615 if (!tr.present) 3616 return false; 3617 3618 return true; 3619 } 3620 3621 static bool ldtr_valid(struct kvm_vcpu *vcpu) 3622 { 3623 struct kvm_segment ldtr; 3624 3625 vmx_get_segment(vcpu, &ldtr, VCPU_SREG_LDTR); 3626 3627 if (ldtr.unusable) 3628 return true; 3629 if (ldtr.selector & SEGMENT_TI_MASK) /* TI = 1 */ 3630 return false; 3631 if (ldtr.type != 2) 3632 return false; 3633 if (!ldtr.present) 3634 return false; 3635 3636 return true; 3637 } 3638 3639 static bool cs_ss_rpl_check(struct kvm_vcpu *vcpu) 3640 { 3641 struct kvm_segment cs, ss; 3642 3643 vmx_get_segment(vcpu, &cs, VCPU_SREG_CS); 3644 vmx_get_segment(vcpu, &ss, VCPU_SREG_SS); 3645 3646 return ((cs.selector & SEGMENT_RPL_MASK) == 3647 (ss.selector & SEGMENT_RPL_MASK)); 3648 } 3649 3650 /* 3651 * Check if guest state is valid. Returns true if valid, false if 3652 * not. 3653 * We assume that registers are always usable 3654 */ 3655 bool __vmx_guest_state_valid(struct kvm_vcpu *vcpu) 3656 { 3657 /* real mode guest state checks */ 3658 if (!is_protmode(vcpu) || (vmx_get_rflags(vcpu) & X86_EFLAGS_VM)) { 3659 if (!rmode_segment_valid(vcpu, VCPU_SREG_CS)) 3660 return false; 3661 if (!rmode_segment_valid(vcpu, VCPU_SREG_SS)) 3662 return false; 3663 if (!rmode_segment_valid(vcpu, VCPU_SREG_DS)) 3664 return false; 3665 if (!rmode_segment_valid(vcpu, VCPU_SREG_ES)) 3666 return false; 3667 if (!rmode_segment_valid(vcpu, VCPU_SREG_FS)) 3668 return false; 3669 if (!rmode_segment_valid(vcpu, VCPU_SREG_GS)) 3670 return false; 3671 } else { 3672 /* protected mode guest state checks */ 3673 if (!cs_ss_rpl_check(vcpu)) 3674 return false; 3675 if (!code_segment_valid(vcpu)) 3676 return false; 3677 if (!stack_segment_valid(vcpu)) 3678 return false; 3679 if (!data_segment_valid(vcpu, VCPU_SREG_DS)) 3680 return false; 3681 if (!data_segment_valid(vcpu, VCPU_SREG_ES)) 3682 return false; 3683 if (!data_segment_valid(vcpu, VCPU_SREG_FS)) 3684 return false; 3685 if (!data_segment_valid(vcpu, VCPU_SREG_GS)) 3686 return false; 3687 if (!tr_valid(vcpu)) 3688 return false; 3689 if (!ldtr_valid(vcpu)) 3690 return false; 3691 } 3692 /* TODO: 3693 * - Add checks on RIP 3694 * - Add checks on RFLAGS 3695 */ 3696 3697 return true; 3698 } 3699 3700 static int init_rmode_tss(struct kvm *kvm, void __user *ua) 3701 { 3702 const void *zero_page = (const void *) __va(page_to_phys(ZERO_PAGE(0))); 3703 u16 data; 3704 int i; 3705 3706 for (i = 0; i < 3; i++) { 3707 if (__copy_to_user(ua + PAGE_SIZE * i, zero_page, PAGE_SIZE)) 3708 return -EFAULT; 3709 } 3710 3711 data = TSS_BASE_SIZE + TSS_REDIRECTION_SIZE; 3712 if (__copy_to_user(ua + TSS_IOPB_BASE_OFFSET, &data, sizeof(u16))) 3713 return -EFAULT; 3714 3715 data = ~0; 3716 if (__copy_to_user(ua + RMODE_TSS_SIZE - 1, &data, sizeof(u8))) 3717 return -EFAULT; 3718 3719 return 0; 3720 } 3721 3722 static int init_rmode_identity_map(struct kvm *kvm) 3723 { 3724 struct kvm_vmx *kvm_vmx = to_kvm_vmx(kvm); 3725 int i, r = 0; 3726 void __user *uaddr; 3727 u32 tmp; 3728 3729 /* Protect kvm_vmx->ept_identity_pagetable_done. */ 3730 mutex_lock(&kvm->slots_lock); 3731 3732 if (likely(kvm_vmx->ept_identity_pagetable_done)) 3733 goto out; 3734 3735 if (!kvm_vmx->ept_identity_map_addr) 3736 kvm_vmx->ept_identity_map_addr = VMX_EPT_IDENTITY_PAGETABLE_ADDR; 3737 3738 uaddr = __x86_set_memory_region(kvm, 3739 IDENTITY_PAGETABLE_PRIVATE_MEMSLOT, 3740 kvm_vmx->ept_identity_map_addr, 3741 PAGE_SIZE); 3742 if (IS_ERR(uaddr)) { 3743 r = PTR_ERR(uaddr); 3744 goto out; 3745 } 3746 3747 /* Set up identity-mapping pagetable for EPT in real mode */ 3748 for (i = 0; i < (PAGE_SIZE / sizeof(tmp)); i++) { 3749 tmp = (i << 22) + (_PAGE_PRESENT | _PAGE_RW | _PAGE_USER | 3750 _PAGE_ACCESSED | _PAGE_DIRTY | _PAGE_PSE); 3751 if (__copy_to_user(uaddr + i * sizeof(tmp), &tmp, sizeof(tmp))) { 3752 r = -EFAULT; 3753 goto out; 3754 } 3755 } 3756 kvm_vmx->ept_identity_pagetable_done = true; 3757 3758 out: 3759 mutex_unlock(&kvm->slots_lock); 3760 return r; 3761 } 3762 3763 static void seg_setup(int seg) 3764 { 3765 const struct kvm_vmx_segment_field *sf = &kvm_vmx_segment_fields[seg]; 3766 unsigned int ar; 3767 3768 vmcs_write16(sf->selector, 0); 3769 vmcs_writel(sf->base, 0); 3770 vmcs_write32(sf->limit, 0xffff); 3771 ar = 0x93; 3772 if (seg == VCPU_SREG_CS) 3773 ar |= 0x08; /* code segment */ 3774 3775 vmcs_write32(sf->ar_bytes, ar); 3776 } 3777 3778 static int alloc_apic_access_page(struct kvm *kvm) 3779 { 3780 struct page *page; 3781 void __user *hva; 3782 int ret = 0; 3783 3784 mutex_lock(&kvm->slots_lock); 3785 if (kvm->arch.apic_access_memslot_enabled) 3786 goto out; 3787 hva = __x86_set_memory_region(kvm, APIC_ACCESS_PAGE_PRIVATE_MEMSLOT, 3788 APIC_DEFAULT_PHYS_BASE, PAGE_SIZE); 3789 if (IS_ERR(hva)) { 3790 ret = PTR_ERR(hva); 3791 goto out; 3792 } 3793 3794 page = gfn_to_page(kvm, APIC_DEFAULT_PHYS_BASE >> PAGE_SHIFT); 3795 if (is_error_page(page)) { 3796 ret = -EFAULT; 3797 goto out; 3798 } 3799 3800 /* 3801 * Do not pin the page in memory, so that memory hot-unplug 3802 * is able to migrate it. 3803 */ 3804 put_page(page); 3805 kvm->arch.apic_access_memslot_enabled = true; 3806 out: 3807 mutex_unlock(&kvm->slots_lock); 3808 return ret; 3809 } 3810 3811 int allocate_vpid(void) 3812 { 3813 int vpid; 3814 3815 if (!enable_vpid) 3816 return 0; 3817 spin_lock(&vmx_vpid_lock); 3818 vpid = find_first_zero_bit(vmx_vpid_bitmap, VMX_NR_VPIDS); 3819 if (vpid < VMX_NR_VPIDS) 3820 __set_bit(vpid, vmx_vpid_bitmap); 3821 else 3822 vpid = 0; 3823 spin_unlock(&vmx_vpid_lock); 3824 return vpid; 3825 } 3826 3827 void free_vpid(int vpid) 3828 { 3829 if (!enable_vpid || vpid == 0) 3830 return; 3831 spin_lock(&vmx_vpid_lock); 3832 __clear_bit(vpid, vmx_vpid_bitmap); 3833 spin_unlock(&vmx_vpid_lock); 3834 } 3835 3836 static void vmx_msr_bitmap_l01_changed(struct vcpu_vmx *vmx) 3837 { 3838 /* 3839 * When KVM is a nested hypervisor on top of Hyper-V and uses 3840 * 'Enlightened MSR Bitmap' feature L0 needs to know that MSR 3841 * bitmap has changed. 3842 */ 3843 if (static_branch_unlikely(&enable_evmcs)) 3844 evmcs_touch_msr_bitmap(); 3845 3846 vmx->nested.force_msr_bitmap_recalc = true; 3847 } 3848 3849 void vmx_disable_intercept_for_msr(struct kvm_vcpu *vcpu, u32 msr, int type) 3850 { 3851 struct vcpu_vmx *vmx = to_vmx(vcpu); 3852 unsigned long *msr_bitmap = vmx->vmcs01.msr_bitmap; 3853 3854 if (!cpu_has_vmx_msr_bitmap()) 3855 return; 3856 3857 vmx_msr_bitmap_l01_changed(vmx); 3858 3859 /* 3860 * Mark the desired intercept state in shadow bitmap, this is needed 3861 * for resync when the MSR filters change. 3862 */ 3863 if (is_valid_passthrough_msr(msr)) { 3864 int idx = possible_passthrough_msr_slot(msr); 3865 3866 if (idx != -ENOENT) { 3867 if (type & MSR_TYPE_R) 3868 clear_bit(idx, vmx->shadow_msr_intercept.read); 3869 if (type & MSR_TYPE_W) 3870 clear_bit(idx, vmx->shadow_msr_intercept.write); 3871 } 3872 } 3873 3874 if ((type & MSR_TYPE_R) && 3875 !kvm_msr_allowed(vcpu, msr, KVM_MSR_FILTER_READ)) { 3876 vmx_set_msr_bitmap_read(msr_bitmap, msr); 3877 type &= ~MSR_TYPE_R; 3878 } 3879 3880 if ((type & MSR_TYPE_W) && 3881 !kvm_msr_allowed(vcpu, msr, KVM_MSR_FILTER_WRITE)) { 3882 vmx_set_msr_bitmap_write(msr_bitmap, msr); 3883 type &= ~MSR_TYPE_W; 3884 } 3885 3886 if (type & MSR_TYPE_R) 3887 vmx_clear_msr_bitmap_read(msr_bitmap, msr); 3888 3889 if (type & MSR_TYPE_W) 3890 vmx_clear_msr_bitmap_write(msr_bitmap, msr); 3891 } 3892 3893 void vmx_enable_intercept_for_msr(struct kvm_vcpu *vcpu, u32 msr, int type) 3894 { 3895 struct vcpu_vmx *vmx = to_vmx(vcpu); 3896 unsigned long *msr_bitmap = vmx->vmcs01.msr_bitmap; 3897 3898 if (!cpu_has_vmx_msr_bitmap()) 3899 return; 3900 3901 vmx_msr_bitmap_l01_changed(vmx); 3902 3903 /* 3904 * Mark the desired intercept state in shadow bitmap, this is needed 3905 * for resync when the MSR filter changes. 3906 */ 3907 if (is_valid_passthrough_msr(msr)) { 3908 int idx = possible_passthrough_msr_slot(msr); 3909 3910 if (idx != -ENOENT) { 3911 if (type & MSR_TYPE_R) 3912 set_bit(idx, vmx->shadow_msr_intercept.read); 3913 if (type & MSR_TYPE_W) 3914 set_bit(idx, vmx->shadow_msr_intercept.write); 3915 } 3916 } 3917 3918 if (type & MSR_TYPE_R) 3919 vmx_set_msr_bitmap_read(msr_bitmap, msr); 3920 3921 if (type & MSR_TYPE_W) 3922 vmx_set_msr_bitmap_write(msr_bitmap, msr); 3923 } 3924 3925 static void vmx_reset_x2apic_msrs(struct kvm_vcpu *vcpu, u8 mode) 3926 { 3927 unsigned long *msr_bitmap = to_vmx(vcpu)->vmcs01.msr_bitmap; 3928 unsigned long read_intercept; 3929 int msr; 3930 3931 read_intercept = (mode & MSR_BITMAP_MODE_X2APIC_APICV) ? 0 : ~0; 3932 3933 for (msr = 0x800; msr <= 0x8ff; msr += BITS_PER_LONG) { 3934 unsigned int read_idx = msr / BITS_PER_LONG; 3935 unsigned int write_idx = read_idx + (0x800 / sizeof(long)); 3936 3937 msr_bitmap[read_idx] = read_intercept; 3938 msr_bitmap[write_idx] = ~0ul; 3939 } 3940 } 3941 3942 static void vmx_update_msr_bitmap_x2apic(struct kvm_vcpu *vcpu) 3943 { 3944 struct vcpu_vmx *vmx = to_vmx(vcpu); 3945 u8 mode; 3946 3947 if (!cpu_has_vmx_msr_bitmap()) 3948 return; 3949 3950 if (cpu_has_secondary_exec_ctrls() && 3951 (secondary_exec_controls_get(vmx) & 3952 SECONDARY_EXEC_VIRTUALIZE_X2APIC_MODE)) { 3953 mode = MSR_BITMAP_MODE_X2APIC; 3954 if (enable_apicv && kvm_vcpu_apicv_active(vcpu)) 3955 mode |= MSR_BITMAP_MODE_X2APIC_APICV; 3956 } else { 3957 mode = 0; 3958 } 3959 3960 if (mode == vmx->x2apic_msr_bitmap_mode) 3961 return; 3962 3963 vmx->x2apic_msr_bitmap_mode = mode; 3964 3965 vmx_reset_x2apic_msrs(vcpu, mode); 3966 3967 /* 3968 * TPR reads and writes can be virtualized even if virtual interrupt 3969 * delivery is not in use. 3970 */ 3971 vmx_set_intercept_for_msr(vcpu, X2APIC_MSR(APIC_TASKPRI), MSR_TYPE_RW, 3972 !(mode & MSR_BITMAP_MODE_X2APIC)); 3973 3974 if (mode & MSR_BITMAP_MODE_X2APIC_APICV) { 3975 vmx_enable_intercept_for_msr(vcpu, X2APIC_MSR(APIC_TMCCT), MSR_TYPE_RW); 3976 vmx_disable_intercept_for_msr(vcpu, X2APIC_MSR(APIC_EOI), MSR_TYPE_W); 3977 vmx_disable_intercept_for_msr(vcpu, X2APIC_MSR(APIC_SELF_IPI), MSR_TYPE_W); 3978 if (enable_ipiv) 3979 vmx_disable_intercept_for_msr(vcpu, X2APIC_MSR(APIC_ICR), MSR_TYPE_RW); 3980 } 3981 } 3982 3983 void pt_update_intercept_for_msr(struct kvm_vcpu *vcpu) 3984 { 3985 struct vcpu_vmx *vmx = to_vmx(vcpu); 3986 bool flag = !(vmx->pt_desc.guest.ctl & RTIT_CTL_TRACEEN); 3987 u32 i; 3988 3989 vmx_set_intercept_for_msr(vcpu, MSR_IA32_RTIT_STATUS, MSR_TYPE_RW, flag); 3990 vmx_set_intercept_for_msr(vcpu, MSR_IA32_RTIT_OUTPUT_BASE, MSR_TYPE_RW, flag); 3991 vmx_set_intercept_for_msr(vcpu, MSR_IA32_RTIT_OUTPUT_MASK, MSR_TYPE_RW, flag); 3992 vmx_set_intercept_for_msr(vcpu, MSR_IA32_RTIT_CR3_MATCH, MSR_TYPE_RW, flag); 3993 for (i = 0; i < vmx->pt_desc.num_address_ranges; i++) { 3994 vmx_set_intercept_for_msr(vcpu, MSR_IA32_RTIT_ADDR0_A + i * 2, MSR_TYPE_RW, flag); 3995 vmx_set_intercept_for_msr(vcpu, MSR_IA32_RTIT_ADDR0_B + i * 2, MSR_TYPE_RW, flag); 3996 } 3997 } 3998 3999 static bool vmx_guest_apic_has_interrupt(struct kvm_vcpu *vcpu) 4000 { 4001 struct vcpu_vmx *vmx = to_vmx(vcpu); 4002 void *vapic_page; 4003 u32 vppr; 4004 int rvi; 4005 4006 if (WARN_ON_ONCE(!is_guest_mode(vcpu)) || 4007 !nested_cpu_has_vid(get_vmcs12(vcpu)) || 4008 WARN_ON_ONCE(!vmx->nested.virtual_apic_map.gfn)) 4009 return false; 4010 4011 rvi = vmx_get_rvi(); 4012 4013 vapic_page = vmx->nested.virtual_apic_map.hva; 4014 vppr = *((u32 *)(vapic_page + APIC_PROCPRI)); 4015 4016 return ((rvi & 0xf0) > (vppr & 0xf0)); 4017 } 4018 4019 static void vmx_msr_filter_changed(struct kvm_vcpu *vcpu) 4020 { 4021 struct vcpu_vmx *vmx = to_vmx(vcpu); 4022 u32 i; 4023 4024 /* 4025 * Redo intercept permissions for MSRs that KVM is passing through to 4026 * the guest. Disabling interception will check the new MSR filter and 4027 * ensure that KVM enables interception if usersepace wants to filter 4028 * the MSR. MSRs that KVM is already intercepting don't need to be 4029 * refreshed since KVM is going to intercept them regardless of what 4030 * userspace wants. 4031 */ 4032 for (i = 0; i < ARRAY_SIZE(vmx_possible_passthrough_msrs); i++) { 4033 u32 msr = vmx_possible_passthrough_msrs[i]; 4034 4035 if (!test_bit(i, vmx->shadow_msr_intercept.read)) 4036 vmx_disable_intercept_for_msr(vcpu, msr, MSR_TYPE_R); 4037 4038 if (!test_bit(i, vmx->shadow_msr_intercept.write)) 4039 vmx_disable_intercept_for_msr(vcpu, msr, MSR_TYPE_W); 4040 } 4041 4042 /* PT MSRs can be passed through iff PT is exposed to the guest. */ 4043 if (vmx_pt_mode_is_host_guest()) 4044 pt_update_intercept_for_msr(vcpu); 4045 } 4046 4047 static inline void kvm_vcpu_trigger_posted_interrupt(struct kvm_vcpu *vcpu, 4048 int pi_vec) 4049 { 4050 #ifdef CONFIG_SMP 4051 if (vcpu->mode == IN_GUEST_MODE) { 4052 /* 4053 * The vector of the virtual has already been set in the PIR. 4054 * Send a notification event to deliver the virtual interrupt 4055 * unless the vCPU is the currently running vCPU, i.e. the 4056 * event is being sent from a fastpath VM-Exit handler, in 4057 * which case the PIR will be synced to the vIRR before 4058 * re-entering the guest. 4059 * 4060 * When the target is not the running vCPU, the following 4061 * possibilities emerge: 4062 * 4063 * Case 1: vCPU stays in non-root mode. Sending a notification 4064 * event posts the interrupt to the vCPU. 4065 * 4066 * Case 2: vCPU exits to root mode and is still runnable. The 4067 * PIR will be synced to the vIRR before re-entering the guest. 4068 * Sending a notification event is ok as the host IRQ handler 4069 * will ignore the spurious event. 4070 * 4071 * Case 3: vCPU exits to root mode and is blocked. vcpu_block() 4072 * has already synced PIR to vIRR and never blocks the vCPU if 4073 * the vIRR is not empty. Therefore, a blocked vCPU here does 4074 * not wait for any requested interrupts in PIR, and sending a 4075 * notification event also results in a benign, spurious event. 4076 */ 4077 4078 if (vcpu != kvm_get_running_vcpu()) 4079 apic->send_IPI_mask(get_cpu_mask(vcpu->cpu), pi_vec); 4080 return; 4081 } 4082 #endif 4083 /* 4084 * The vCPU isn't in the guest; wake the vCPU in case it is blocking, 4085 * otherwise do nothing as KVM will grab the highest priority pending 4086 * IRQ via ->sync_pir_to_irr() in vcpu_enter_guest(). 4087 */ 4088 kvm_vcpu_wake_up(vcpu); 4089 } 4090 4091 static int vmx_deliver_nested_posted_interrupt(struct kvm_vcpu *vcpu, 4092 int vector) 4093 { 4094 struct vcpu_vmx *vmx = to_vmx(vcpu); 4095 4096 if (is_guest_mode(vcpu) && 4097 vector == vmx->nested.posted_intr_nv) { 4098 /* 4099 * If a posted intr is not recognized by hardware, 4100 * we will accomplish it in the next vmentry. 4101 */ 4102 vmx->nested.pi_pending = true; 4103 kvm_make_request(KVM_REQ_EVENT, vcpu); 4104 4105 /* 4106 * This pairs with the smp_mb_*() after setting vcpu->mode in 4107 * vcpu_enter_guest() to guarantee the vCPU sees the event 4108 * request if triggering a posted interrupt "fails" because 4109 * vcpu->mode != IN_GUEST_MODE. The extra barrier is needed as 4110 * the smb_wmb() in kvm_make_request() only ensures everything 4111 * done before making the request is visible when the request 4112 * is visible, it doesn't ensure ordering between the store to 4113 * vcpu->requests and the load from vcpu->mode. 4114 */ 4115 smp_mb__after_atomic(); 4116 4117 /* the PIR and ON have been set by L1. */ 4118 kvm_vcpu_trigger_posted_interrupt(vcpu, POSTED_INTR_NESTED_VECTOR); 4119 return 0; 4120 } 4121 return -1; 4122 } 4123 /* 4124 * Send interrupt to vcpu via posted interrupt way. 4125 * 1. If target vcpu is running(non-root mode), send posted interrupt 4126 * notification to vcpu and hardware will sync PIR to vIRR atomically. 4127 * 2. If target vcpu isn't running(root mode), kick it to pick up the 4128 * interrupt from PIR in next vmentry. 4129 */ 4130 static int vmx_deliver_posted_interrupt(struct kvm_vcpu *vcpu, int vector) 4131 { 4132 struct vcpu_vmx *vmx = to_vmx(vcpu); 4133 int r; 4134 4135 r = vmx_deliver_nested_posted_interrupt(vcpu, vector); 4136 if (!r) 4137 return 0; 4138 4139 /* Note, this is called iff the local APIC is in-kernel. */ 4140 if (!vcpu->arch.apic->apicv_active) 4141 return -1; 4142 4143 if (pi_test_and_set_pir(vector, &vmx->pi_desc)) 4144 return 0; 4145 4146 /* If a previous notification has sent the IPI, nothing to do. */ 4147 if (pi_test_and_set_on(&vmx->pi_desc)) 4148 return 0; 4149 4150 /* 4151 * The implied barrier in pi_test_and_set_on() pairs with the smp_mb_*() 4152 * after setting vcpu->mode in vcpu_enter_guest(), thus the vCPU is 4153 * guaranteed to see PID.ON=1 and sync the PIR to IRR if triggering a 4154 * posted interrupt "fails" because vcpu->mode != IN_GUEST_MODE. 4155 */ 4156 kvm_vcpu_trigger_posted_interrupt(vcpu, POSTED_INTR_VECTOR); 4157 return 0; 4158 } 4159 4160 static void vmx_deliver_interrupt(struct kvm_lapic *apic, int delivery_mode, 4161 int trig_mode, int vector) 4162 { 4163 struct kvm_vcpu *vcpu = apic->vcpu; 4164 4165 if (vmx_deliver_posted_interrupt(vcpu, vector)) { 4166 kvm_lapic_set_irr(vector, apic); 4167 kvm_make_request(KVM_REQ_EVENT, vcpu); 4168 kvm_vcpu_kick(vcpu); 4169 } else { 4170 trace_kvm_apicv_accept_irq(vcpu->vcpu_id, delivery_mode, 4171 trig_mode, vector); 4172 } 4173 } 4174 4175 /* 4176 * Set up the vmcs's constant host-state fields, i.e., host-state fields that 4177 * will not change in the lifetime of the guest. 4178 * Note that host-state that does change is set elsewhere. E.g., host-state 4179 * that is set differently for each CPU is set in vmx_vcpu_load(), not here. 4180 */ 4181 void vmx_set_constant_host_state(struct vcpu_vmx *vmx) 4182 { 4183 u32 low32, high32; 4184 unsigned long tmpl; 4185 unsigned long cr0, cr3, cr4; 4186 4187 cr0 = read_cr0(); 4188 WARN_ON(cr0 & X86_CR0_TS); 4189 vmcs_writel(HOST_CR0, cr0); /* 22.2.3 */ 4190 4191 /* 4192 * Save the most likely value for this task's CR3 in the VMCS. 4193 * We can't use __get_current_cr3_fast() because we're not atomic. 4194 */ 4195 cr3 = __read_cr3(); 4196 vmcs_writel(HOST_CR3, cr3); /* 22.2.3 FIXME: shadow tables */ 4197 vmx->loaded_vmcs->host_state.cr3 = cr3; 4198 4199 /* Save the most likely value for this task's CR4 in the VMCS. */ 4200 cr4 = cr4_read_shadow(); 4201 vmcs_writel(HOST_CR4, cr4); /* 22.2.3, 22.2.5 */ 4202 vmx->loaded_vmcs->host_state.cr4 = cr4; 4203 4204 vmcs_write16(HOST_CS_SELECTOR, __KERNEL_CS); /* 22.2.4 */ 4205 #ifdef CONFIG_X86_64 4206 /* 4207 * Load null selectors, so we can avoid reloading them in 4208 * vmx_prepare_switch_to_host(), in case userspace uses 4209 * the null selectors too (the expected case). 4210 */ 4211 vmcs_write16(HOST_DS_SELECTOR, 0); 4212 vmcs_write16(HOST_ES_SELECTOR, 0); 4213 #else 4214 vmcs_write16(HOST_DS_SELECTOR, __KERNEL_DS); /* 22.2.4 */ 4215 vmcs_write16(HOST_ES_SELECTOR, __KERNEL_DS); /* 22.2.4 */ 4216 #endif 4217 vmcs_write16(HOST_SS_SELECTOR, __KERNEL_DS); /* 22.2.4 */ 4218 vmcs_write16(HOST_TR_SELECTOR, GDT_ENTRY_TSS*8); /* 22.2.4 */ 4219 4220 vmcs_writel(HOST_IDTR_BASE, host_idt_base); /* 22.2.4 */ 4221 4222 vmcs_writel(HOST_RIP, (unsigned long)vmx_vmexit); /* 22.2.5 */ 4223 4224 rdmsr(MSR_IA32_SYSENTER_CS, low32, high32); 4225 vmcs_write32(HOST_IA32_SYSENTER_CS, low32); 4226 4227 /* 4228 * SYSENTER is used for 32-bit system calls on either 32-bit or 4229 * 64-bit kernels. It is always zero If neither is allowed, otherwise 4230 * vmx_vcpu_load_vmcs loads it with the per-CPU entry stack (and may 4231 * have already done so!). 4232 */ 4233 if (!IS_ENABLED(CONFIG_IA32_EMULATION) && !IS_ENABLED(CONFIG_X86_32)) 4234 vmcs_writel(HOST_IA32_SYSENTER_ESP, 0); 4235 4236 rdmsrl(MSR_IA32_SYSENTER_EIP, tmpl); 4237 vmcs_writel(HOST_IA32_SYSENTER_EIP, tmpl); /* 22.2.3 */ 4238 4239 if (vmcs_config.vmexit_ctrl & VM_EXIT_LOAD_IA32_PAT) { 4240 rdmsr(MSR_IA32_CR_PAT, low32, high32); 4241 vmcs_write64(HOST_IA32_PAT, low32 | ((u64) high32 << 32)); 4242 } 4243 4244 if (cpu_has_load_ia32_efer()) 4245 vmcs_write64(HOST_IA32_EFER, host_efer); 4246 } 4247 4248 void set_cr4_guest_host_mask(struct vcpu_vmx *vmx) 4249 { 4250 struct kvm_vcpu *vcpu = &vmx->vcpu; 4251 4252 vcpu->arch.cr4_guest_owned_bits = KVM_POSSIBLE_CR4_GUEST_BITS & 4253 ~vcpu->arch.cr4_guest_rsvd_bits; 4254 if (!enable_ept) { 4255 vcpu->arch.cr4_guest_owned_bits &= ~X86_CR4_TLBFLUSH_BITS; 4256 vcpu->arch.cr4_guest_owned_bits &= ~X86_CR4_PDPTR_BITS; 4257 } 4258 if (is_guest_mode(&vmx->vcpu)) 4259 vcpu->arch.cr4_guest_owned_bits &= 4260 ~get_vmcs12(vcpu)->cr4_guest_host_mask; 4261 vmcs_writel(CR4_GUEST_HOST_MASK, ~vcpu->arch.cr4_guest_owned_bits); 4262 } 4263 4264 static u32 vmx_pin_based_exec_ctrl(struct vcpu_vmx *vmx) 4265 { 4266 u32 pin_based_exec_ctrl = vmcs_config.pin_based_exec_ctrl; 4267 4268 if (!kvm_vcpu_apicv_active(&vmx->vcpu)) 4269 pin_based_exec_ctrl &= ~PIN_BASED_POSTED_INTR; 4270 4271 if (!enable_vnmi) 4272 pin_based_exec_ctrl &= ~PIN_BASED_VIRTUAL_NMIS; 4273 4274 if (!enable_preemption_timer) 4275 pin_based_exec_ctrl &= ~PIN_BASED_VMX_PREEMPTION_TIMER; 4276 4277 return pin_based_exec_ctrl; 4278 } 4279 4280 static u32 vmx_vmentry_ctrl(void) 4281 { 4282 u32 vmentry_ctrl = vmcs_config.vmentry_ctrl; 4283 4284 if (vmx_pt_mode_is_system()) 4285 vmentry_ctrl &= ~(VM_ENTRY_PT_CONCEAL_PIP | 4286 VM_ENTRY_LOAD_IA32_RTIT_CTL); 4287 /* 4288 * IA32e mode, and loading of EFER and PERF_GLOBAL_CTRL are toggled dynamically. 4289 */ 4290 vmentry_ctrl &= ~(VM_ENTRY_LOAD_IA32_PERF_GLOBAL_CTRL | 4291 VM_ENTRY_LOAD_IA32_EFER | 4292 VM_ENTRY_IA32E_MODE); 4293 4294 if (cpu_has_perf_global_ctrl_bug()) 4295 vmentry_ctrl &= ~VM_ENTRY_LOAD_IA32_PERF_GLOBAL_CTRL; 4296 4297 return vmentry_ctrl; 4298 } 4299 4300 static u32 vmx_vmexit_ctrl(void) 4301 { 4302 u32 vmexit_ctrl = vmcs_config.vmexit_ctrl; 4303 4304 /* 4305 * Not used by KVM and never set in vmcs01 or vmcs02, but emulated for 4306 * nested virtualization and thus allowed to be set in vmcs12. 4307 */ 4308 vmexit_ctrl &= ~(VM_EXIT_SAVE_IA32_PAT | VM_EXIT_SAVE_IA32_EFER | 4309 VM_EXIT_SAVE_VMX_PREEMPTION_TIMER); 4310 4311 if (vmx_pt_mode_is_system()) 4312 vmexit_ctrl &= ~(VM_EXIT_PT_CONCEAL_PIP | 4313 VM_EXIT_CLEAR_IA32_RTIT_CTL); 4314 4315 if (cpu_has_perf_global_ctrl_bug()) 4316 vmexit_ctrl &= ~VM_EXIT_LOAD_IA32_PERF_GLOBAL_CTRL; 4317 4318 /* Loading of EFER and PERF_GLOBAL_CTRL are toggled dynamically */ 4319 return vmexit_ctrl & 4320 ~(VM_EXIT_LOAD_IA32_PERF_GLOBAL_CTRL | VM_EXIT_LOAD_IA32_EFER); 4321 } 4322 4323 static void vmx_refresh_apicv_exec_ctrl(struct kvm_vcpu *vcpu) 4324 { 4325 struct vcpu_vmx *vmx = to_vmx(vcpu); 4326 4327 if (is_guest_mode(vcpu)) { 4328 vmx->nested.update_vmcs01_apicv_status = true; 4329 return; 4330 } 4331 4332 pin_controls_set(vmx, vmx_pin_based_exec_ctrl(vmx)); 4333 4334 if (kvm_vcpu_apicv_active(vcpu)) { 4335 secondary_exec_controls_setbit(vmx, 4336 SECONDARY_EXEC_APIC_REGISTER_VIRT | 4337 SECONDARY_EXEC_VIRTUAL_INTR_DELIVERY); 4338 if (enable_ipiv) 4339 tertiary_exec_controls_setbit(vmx, TERTIARY_EXEC_IPI_VIRT); 4340 } else { 4341 secondary_exec_controls_clearbit(vmx, 4342 SECONDARY_EXEC_APIC_REGISTER_VIRT | 4343 SECONDARY_EXEC_VIRTUAL_INTR_DELIVERY); 4344 if (enable_ipiv) 4345 tertiary_exec_controls_clearbit(vmx, TERTIARY_EXEC_IPI_VIRT); 4346 } 4347 4348 vmx_update_msr_bitmap_x2apic(vcpu); 4349 } 4350 4351 static u32 vmx_exec_control(struct vcpu_vmx *vmx) 4352 { 4353 u32 exec_control = vmcs_config.cpu_based_exec_ctrl; 4354 4355 /* 4356 * Not used by KVM, but fully supported for nesting, i.e. are allowed in 4357 * vmcs12 and propagated to vmcs02 when set in vmcs12. 4358 */ 4359 exec_control &= ~(CPU_BASED_RDTSC_EXITING | 4360 CPU_BASED_USE_IO_BITMAPS | 4361 CPU_BASED_MONITOR_TRAP_FLAG | 4362 CPU_BASED_PAUSE_EXITING); 4363 4364 /* INTR_WINDOW_EXITING and NMI_WINDOW_EXITING are toggled dynamically */ 4365 exec_control &= ~(CPU_BASED_INTR_WINDOW_EXITING | 4366 CPU_BASED_NMI_WINDOW_EXITING); 4367 4368 if (vmx->vcpu.arch.switch_db_regs & KVM_DEBUGREG_WONT_EXIT) 4369 exec_control &= ~CPU_BASED_MOV_DR_EXITING; 4370 4371 if (!cpu_need_tpr_shadow(&vmx->vcpu)) 4372 exec_control &= ~CPU_BASED_TPR_SHADOW; 4373 4374 #ifdef CONFIG_X86_64 4375 if (exec_control & CPU_BASED_TPR_SHADOW) 4376 exec_control &= ~(CPU_BASED_CR8_LOAD_EXITING | 4377 CPU_BASED_CR8_STORE_EXITING); 4378 else 4379 exec_control |= CPU_BASED_CR8_STORE_EXITING | 4380 CPU_BASED_CR8_LOAD_EXITING; 4381 #endif 4382 /* No need to intercept CR3 access or INVPLG when using EPT. */ 4383 if (enable_ept) 4384 exec_control &= ~(CPU_BASED_CR3_LOAD_EXITING | 4385 CPU_BASED_CR3_STORE_EXITING | 4386 CPU_BASED_INVLPG_EXITING); 4387 if (kvm_mwait_in_guest(vmx->vcpu.kvm)) 4388 exec_control &= ~(CPU_BASED_MWAIT_EXITING | 4389 CPU_BASED_MONITOR_EXITING); 4390 if (kvm_hlt_in_guest(vmx->vcpu.kvm)) 4391 exec_control &= ~CPU_BASED_HLT_EXITING; 4392 return exec_control; 4393 } 4394 4395 static u64 vmx_tertiary_exec_control(struct vcpu_vmx *vmx) 4396 { 4397 u64 exec_control = vmcs_config.cpu_based_3rd_exec_ctrl; 4398 4399 /* 4400 * IPI virtualization relies on APICv. Disable IPI virtualization if 4401 * APICv is inhibited. 4402 */ 4403 if (!enable_ipiv || !kvm_vcpu_apicv_active(&vmx->vcpu)) 4404 exec_control &= ~TERTIARY_EXEC_IPI_VIRT; 4405 4406 return exec_control; 4407 } 4408 4409 /* 4410 * Adjust a single secondary execution control bit to intercept/allow an 4411 * instruction in the guest. This is usually done based on whether or not a 4412 * feature has been exposed to the guest in order to correctly emulate faults. 4413 */ 4414 static inline void 4415 vmx_adjust_secondary_exec_control(struct vcpu_vmx *vmx, u32 *exec_control, 4416 u32 control, bool enabled, bool exiting) 4417 { 4418 /* 4419 * If the control is for an opt-in feature, clear the control if the 4420 * feature is not exposed to the guest, i.e. not enabled. If the 4421 * control is opt-out, i.e. an exiting control, clear the control if 4422 * the feature _is_ exposed to the guest, i.e. exiting/interception is 4423 * disabled for the associated instruction. Note, the caller is 4424 * responsible presetting exec_control to set all supported bits. 4425 */ 4426 if (enabled == exiting) 4427 *exec_control &= ~control; 4428 4429 /* 4430 * Update the nested MSR settings so that a nested VMM can/can't set 4431 * controls for features that are/aren't exposed to the guest. 4432 */ 4433 if (nested) { 4434 if (enabled) 4435 vmx->nested.msrs.secondary_ctls_high |= control; 4436 else 4437 vmx->nested.msrs.secondary_ctls_high &= ~control; 4438 } 4439 } 4440 4441 /* 4442 * Wrapper macro for the common case of adjusting a secondary execution control 4443 * based on a single guest CPUID bit, with a dedicated feature bit. This also 4444 * verifies that the control is actually supported by KVM and hardware. 4445 */ 4446 #define vmx_adjust_sec_exec_control(vmx, exec_control, name, feat_name, ctrl_name, exiting) \ 4447 ({ \ 4448 bool __enabled; \ 4449 \ 4450 if (cpu_has_vmx_##name()) { \ 4451 __enabled = guest_cpuid_has(&(vmx)->vcpu, \ 4452 X86_FEATURE_##feat_name); \ 4453 vmx_adjust_secondary_exec_control(vmx, exec_control, \ 4454 SECONDARY_EXEC_##ctrl_name, __enabled, exiting); \ 4455 } \ 4456 }) 4457 4458 /* More macro magic for ENABLE_/opt-in versus _EXITING/opt-out controls. */ 4459 #define vmx_adjust_sec_exec_feature(vmx, exec_control, lname, uname) \ 4460 vmx_adjust_sec_exec_control(vmx, exec_control, lname, uname, ENABLE_##uname, false) 4461 4462 #define vmx_adjust_sec_exec_exiting(vmx, exec_control, lname, uname) \ 4463 vmx_adjust_sec_exec_control(vmx, exec_control, lname, uname, uname##_EXITING, true) 4464 4465 static u32 vmx_secondary_exec_control(struct vcpu_vmx *vmx) 4466 { 4467 struct kvm_vcpu *vcpu = &vmx->vcpu; 4468 4469 u32 exec_control = vmcs_config.cpu_based_2nd_exec_ctrl; 4470 4471 if (vmx_pt_mode_is_system()) 4472 exec_control &= ~(SECONDARY_EXEC_PT_USE_GPA | SECONDARY_EXEC_PT_CONCEAL_VMX); 4473 if (!cpu_need_virtualize_apic_accesses(vcpu)) 4474 exec_control &= ~SECONDARY_EXEC_VIRTUALIZE_APIC_ACCESSES; 4475 if (vmx->vpid == 0) 4476 exec_control &= ~SECONDARY_EXEC_ENABLE_VPID; 4477 if (!enable_ept) { 4478 exec_control &= ~SECONDARY_EXEC_ENABLE_EPT; 4479 enable_unrestricted_guest = 0; 4480 } 4481 if (!enable_unrestricted_guest) 4482 exec_control &= ~SECONDARY_EXEC_UNRESTRICTED_GUEST; 4483 if (kvm_pause_in_guest(vmx->vcpu.kvm)) 4484 exec_control &= ~SECONDARY_EXEC_PAUSE_LOOP_EXITING; 4485 if (!kvm_vcpu_apicv_active(vcpu)) 4486 exec_control &= ~(SECONDARY_EXEC_APIC_REGISTER_VIRT | 4487 SECONDARY_EXEC_VIRTUAL_INTR_DELIVERY); 4488 exec_control &= ~SECONDARY_EXEC_VIRTUALIZE_X2APIC_MODE; 4489 4490 /* SECONDARY_EXEC_DESC is enabled/disabled on writes to CR4.UMIP, 4491 * in vmx_set_cr4. */ 4492 exec_control &= ~SECONDARY_EXEC_DESC; 4493 4494 /* SECONDARY_EXEC_SHADOW_VMCS is enabled when L1 executes VMPTRLD 4495 (handle_vmptrld). 4496 We can NOT enable shadow_vmcs here because we don't have yet 4497 a current VMCS12 4498 */ 4499 exec_control &= ~SECONDARY_EXEC_SHADOW_VMCS; 4500 4501 /* 4502 * PML is enabled/disabled when dirty logging of memsmlots changes, but 4503 * it needs to be set here when dirty logging is already active, e.g. 4504 * if this vCPU was created after dirty logging was enabled. 4505 */ 4506 if (!vcpu->kvm->arch.cpu_dirty_logging_count) 4507 exec_control &= ~SECONDARY_EXEC_ENABLE_PML; 4508 4509 if (cpu_has_vmx_xsaves()) { 4510 /* Exposing XSAVES only when XSAVE is exposed */ 4511 bool xsaves_enabled = 4512 boot_cpu_has(X86_FEATURE_XSAVE) && 4513 guest_cpuid_has(vcpu, X86_FEATURE_XSAVE) && 4514 guest_cpuid_has(vcpu, X86_FEATURE_XSAVES); 4515 4516 vcpu->arch.xsaves_enabled = xsaves_enabled; 4517 4518 vmx_adjust_secondary_exec_control(vmx, &exec_control, 4519 SECONDARY_EXEC_XSAVES, 4520 xsaves_enabled, false); 4521 } 4522 4523 /* 4524 * RDPID is also gated by ENABLE_RDTSCP, turn on the control if either 4525 * feature is exposed to the guest. This creates a virtualization hole 4526 * if both are supported in hardware but only one is exposed to the 4527 * guest, but letting the guest execute RDTSCP or RDPID when either one 4528 * is advertised is preferable to emulating the advertised instruction 4529 * in KVM on #UD, and obviously better than incorrectly injecting #UD. 4530 */ 4531 if (cpu_has_vmx_rdtscp()) { 4532 bool rdpid_or_rdtscp_enabled = 4533 guest_cpuid_has(vcpu, X86_FEATURE_RDTSCP) || 4534 guest_cpuid_has(vcpu, X86_FEATURE_RDPID); 4535 4536 vmx_adjust_secondary_exec_control(vmx, &exec_control, 4537 SECONDARY_EXEC_ENABLE_RDTSCP, 4538 rdpid_or_rdtscp_enabled, false); 4539 } 4540 vmx_adjust_sec_exec_feature(vmx, &exec_control, invpcid, INVPCID); 4541 4542 vmx_adjust_sec_exec_exiting(vmx, &exec_control, rdrand, RDRAND); 4543 vmx_adjust_sec_exec_exiting(vmx, &exec_control, rdseed, RDSEED); 4544 4545 vmx_adjust_sec_exec_control(vmx, &exec_control, waitpkg, WAITPKG, 4546 ENABLE_USR_WAIT_PAUSE, false); 4547 4548 if (!vcpu->kvm->arch.bus_lock_detection_enabled) 4549 exec_control &= ~SECONDARY_EXEC_BUS_LOCK_DETECTION; 4550 4551 if (!kvm_notify_vmexit_enabled(vcpu->kvm)) 4552 exec_control &= ~SECONDARY_EXEC_NOTIFY_VM_EXITING; 4553 4554 return exec_control; 4555 } 4556 4557 static inline int vmx_get_pid_table_order(struct kvm *kvm) 4558 { 4559 return get_order(kvm->arch.max_vcpu_ids * sizeof(*to_kvm_vmx(kvm)->pid_table)); 4560 } 4561 4562 static int vmx_alloc_ipiv_pid_table(struct kvm *kvm) 4563 { 4564 struct page *pages; 4565 struct kvm_vmx *kvm_vmx = to_kvm_vmx(kvm); 4566 4567 if (!irqchip_in_kernel(kvm) || !enable_ipiv) 4568 return 0; 4569 4570 if (kvm_vmx->pid_table) 4571 return 0; 4572 4573 pages = alloc_pages(GFP_KERNEL | __GFP_ZERO, vmx_get_pid_table_order(kvm)); 4574 if (!pages) 4575 return -ENOMEM; 4576 4577 kvm_vmx->pid_table = (void *)page_address(pages); 4578 return 0; 4579 } 4580 4581 static int vmx_vcpu_precreate(struct kvm *kvm) 4582 { 4583 return vmx_alloc_ipiv_pid_table(kvm); 4584 } 4585 4586 #define VMX_XSS_EXIT_BITMAP 0 4587 4588 static void init_vmcs(struct vcpu_vmx *vmx) 4589 { 4590 struct kvm *kvm = vmx->vcpu.kvm; 4591 struct kvm_vmx *kvm_vmx = to_kvm_vmx(kvm); 4592 4593 if (nested) 4594 nested_vmx_set_vmcs_shadowing_bitmap(); 4595 4596 if (cpu_has_vmx_msr_bitmap()) 4597 vmcs_write64(MSR_BITMAP, __pa(vmx->vmcs01.msr_bitmap)); 4598 4599 vmcs_write64(VMCS_LINK_POINTER, INVALID_GPA); /* 22.3.1.5 */ 4600 4601 /* Control */ 4602 pin_controls_set(vmx, vmx_pin_based_exec_ctrl(vmx)); 4603 4604 exec_controls_set(vmx, vmx_exec_control(vmx)); 4605 4606 if (cpu_has_secondary_exec_ctrls()) 4607 secondary_exec_controls_set(vmx, vmx_secondary_exec_control(vmx)); 4608 4609 if (cpu_has_tertiary_exec_ctrls()) 4610 tertiary_exec_controls_set(vmx, vmx_tertiary_exec_control(vmx)); 4611 4612 if (enable_apicv && lapic_in_kernel(&vmx->vcpu)) { 4613 vmcs_write64(EOI_EXIT_BITMAP0, 0); 4614 vmcs_write64(EOI_EXIT_BITMAP1, 0); 4615 vmcs_write64(EOI_EXIT_BITMAP2, 0); 4616 vmcs_write64(EOI_EXIT_BITMAP3, 0); 4617 4618 vmcs_write16(GUEST_INTR_STATUS, 0); 4619 4620 vmcs_write16(POSTED_INTR_NV, POSTED_INTR_VECTOR); 4621 vmcs_write64(POSTED_INTR_DESC_ADDR, __pa((&vmx->pi_desc))); 4622 } 4623 4624 if (vmx_can_use_ipiv(&vmx->vcpu)) { 4625 vmcs_write64(PID_POINTER_TABLE, __pa(kvm_vmx->pid_table)); 4626 vmcs_write16(LAST_PID_POINTER_INDEX, kvm->arch.max_vcpu_ids - 1); 4627 } 4628 4629 if (!kvm_pause_in_guest(kvm)) { 4630 vmcs_write32(PLE_GAP, ple_gap); 4631 vmx->ple_window = ple_window; 4632 vmx->ple_window_dirty = true; 4633 } 4634 4635 if (kvm_notify_vmexit_enabled(kvm)) 4636 vmcs_write32(NOTIFY_WINDOW, kvm->arch.notify_window); 4637 4638 vmcs_write32(PAGE_FAULT_ERROR_CODE_MASK, 0); 4639 vmcs_write32(PAGE_FAULT_ERROR_CODE_MATCH, 0); 4640 vmcs_write32(CR3_TARGET_COUNT, 0); /* 22.2.1 */ 4641 4642 vmcs_write16(HOST_FS_SELECTOR, 0); /* 22.2.4 */ 4643 vmcs_write16(HOST_GS_SELECTOR, 0); /* 22.2.4 */ 4644 vmx_set_constant_host_state(vmx); 4645 vmcs_writel(HOST_FS_BASE, 0); /* 22.2.4 */ 4646 vmcs_writel(HOST_GS_BASE, 0); /* 22.2.4 */ 4647 4648 if (cpu_has_vmx_vmfunc()) 4649 vmcs_write64(VM_FUNCTION_CONTROL, 0); 4650 4651 vmcs_write32(VM_EXIT_MSR_STORE_COUNT, 0); 4652 vmcs_write32(VM_EXIT_MSR_LOAD_COUNT, 0); 4653 vmcs_write64(VM_EXIT_MSR_LOAD_ADDR, __pa(vmx->msr_autoload.host.val)); 4654 vmcs_write32(VM_ENTRY_MSR_LOAD_COUNT, 0); 4655 vmcs_write64(VM_ENTRY_MSR_LOAD_ADDR, __pa(vmx->msr_autoload.guest.val)); 4656 4657 if (vmcs_config.vmentry_ctrl & VM_ENTRY_LOAD_IA32_PAT) 4658 vmcs_write64(GUEST_IA32_PAT, vmx->vcpu.arch.pat); 4659 4660 vm_exit_controls_set(vmx, vmx_vmexit_ctrl()); 4661 4662 /* 22.2.1, 20.8.1 */ 4663 vm_entry_controls_set(vmx, vmx_vmentry_ctrl()); 4664 4665 vmx->vcpu.arch.cr0_guest_owned_bits = KVM_POSSIBLE_CR0_GUEST_BITS; 4666 vmcs_writel(CR0_GUEST_HOST_MASK, ~vmx->vcpu.arch.cr0_guest_owned_bits); 4667 4668 set_cr4_guest_host_mask(vmx); 4669 4670 if (vmx->vpid != 0) 4671 vmcs_write16(VIRTUAL_PROCESSOR_ID, vmx->vpid); 4672 4673 if (cpu_has_vmx_xsaves()) 4674 vmcs_write64(XSS_EXIT_BITMAP, VMX_XSS_EXIT_BITMAP); 4675 4676 if (enable_pml) { 4677 vmcs_write64(PML_ADDRESS, page_to_phys(vmx->pml_pg)); 4678 vmcs_write16(GUEST_PML_INDEX, PML_ENTITY_NUM - 1); 4679 } 4680 4681 vmx_write_encls_bitmap(&vmx->vcpu, NULL); 4682 4683 if (vmx_pt_mode_is_host_guest()) { 4684 memset(&vmx->pt_desc, 0, sizeof(vmx->pt_desc)); 4685 /* Bit[6~0] are forced to 1, writes are ignored. */ 4686 vmx->pt_desc.guest.output_mask = 0x7F; 4687 vmcs_write64(GUEST_IA32_RTIT_CTL, 0); 4688 } 4689 4690 vmcs_write32(GUEST_SYSENTER_CS, 0); 4691 vmcs_writel(GUEST_SYSENTER_ESP, 0); 4692 vmcs_writel(GUEST_SYSENTER_EIP, 0); 4693 vmcs_write64(GUEST_IA32_DEBUGCTL, 0); 4694 4695 if (cpu_has_vmx_tpr_shadow()) { 4696 vmcs_write64(VIRTUAL_APIC_PAGE_ADDR, 0); 4697 if (cpu_need_tpr_shadow(&vmx->vcpu)) 4698 vmcs_write64(VIRTUAL_APIC_PAGE_ADDR, 4699 __pa(vmx->vcpu.arch.apic->regs)); 4700 vmcs_write32(TPR_THRESHOLD, 0); 4701 } 4702 4703 vmx_setup_uret_msrs(vmx); 4704 } 4705 4706 static void __vmx_vcpu_reset(struct kvm_vcpu *vcpu) 4707 { 4708 struct vcpu_vmx *vmx = to_vmx(vcpu); 4709 4710 init_vmcs(vmx); 4711 4712 if (nested) 4713 memcpy(&vmx->nested.msrs, &vmcs_config.nested, sizeof(vmx->nested.msrs)); 4714 4715 vcpu_setup_sgx_lepubkeyhash(vcpu); 4716 4717 vmx->nested.posted_intr_nv = -1; 4718 vmx->nested.vmxon_ptr = INVALID_GPA; 4719 vmx->nested.current_vmptr = INVALID_GPA; 4720 vmx->nested.hv_evmcs_vmptr = EVMPTR_INVALID; 4721 4722 vcpu->arch.microcode_version = 0x100000000ULL; 4723 vmx->msr_ia32_feature_control_valid_bits = FEAT_CTL_LOCKED; 4724 4725 /* 4726 * Enforce invariant: pi_desc.nv is always either POSTED_INTR_VECTOR 4727 * or POSTED_INTR_WAKEUP_VECTOR. 4728 */ 4729 vmx->pi_desc.nv = POSTED_INTR_VECTOR; 4730 vmx->pi_desc.sn = 1; 4731 } 4732 4733 static void vmx_vcpu_reset(struct kvm_vcpu *vcpu, bool init_event) 4734 { 4735 struct vcpu_vmx *vmx = to_vmx(vcpu); 4736 4737 if (!init_event) 4738 __vmx_vcpu_reset(vcpu); 4739 4740 vmx->rmode.vm86_active = 0; 4741 vmx->spec_ctrl = 0; 4742 4743 vmx->msr_ia32_umwait_control = 0; 4744 4745 vmx->hv_deadline_tsc = -1; 4746 kvm_set_cr8(vcpu, 0); 4747 4748 vmx_segment_cache_clear(vmx); 4749 kvm_register_mark_available(vcpu, VCPU_EXREG_SEGMENTS); 4750 4751 seg_setup(VCPU_SREG_CS); 4752 vmcs_write16(GUEST_CS_SELECTOR, 0xf000); 4753 vmcs_writel(GUEST_CS_BASE, 0xffff0000ul); 4754 4755 seg_setup(VCPU_SREG_DS); 4756 seg_setup(VCPU_SREG_ES); 4757 seg_setup(VCPU_SREG_FS); 4758 seg_setup(VCPU_SREG_GS); 4759 seg_setup(VCPU_SREG_SS); 4760 4761 vmcs_write16(GUEST_TR_SELECTOR, 0); 4762 vmcs_writel(GUEST_TR_BASE, 0); 4763 vmcs_write32(GUEST_TR_LIMIT, 0xffff); 4764 vmcs_write32(GUEST_TR_AR_BYTES, 0x008b); 4765 4766 vmcs_write16(GUEST_LDTR_SELECTOR, 0); 4767 vmcs_writel(GUEST_LDTR_BASE, 0); 4768 vmcs_write32(GUEST_LDTR_LIMIT, 0xffff); 4769 vmcs_write32(GUEST_LDTR_AR_BYTES, 0x00082); 4770 4771 vmcs_writel(GUEST_GDTR_BASE, 0); 4772 vmcs_write32(GUEST_GDTR_LIMIT, 0xffff); 4773 4774 vmcs_writel(GUEST_IDTR_BASE, 0); 4775 vmcs_write32(GUEST_IDTR_LIMIT, 0xffff); 4776 4777 vmcs_write32(GUEST_ACTIVITY_STATE, GUEST_ACTIVITY_ACTIVE); 4778 vmcs_write32(GUEST_INTERRUPTIBILITY_INFO, 0); 4779 vmcs_writel(GUEST_PENDING_DBG_EXCEPTIONS, 0); 4780 if (kvm_mpx_supported()) 4781 vmcs_write64(GUEST_BNDCFGS, 0); 4782 4783 vmcs_write32(VM_ENTRY_INTR_INFO_FIELD, 0); /* 22.2.1 */ 4784 4785 kvm_make_request(KVM_REQ_APIC_PAGE_RELOAD, vcpu); 4786 4787 vpid_sync_context(vmx->vpid); 4788 4789 vmx_update_fb_clear_dis(vcpu, vmx); 4790 } 4791 4792 static void vmx_enable_irq_window(struct kvm_vcpu *vcpu) 4793 { 4794 exec_controls_setbit(to_vmx(vcpu), CPU_BASED_INTR_WINDOW_EXITING); 4795 } 4796 4797 static void vmx_enable_nmi_window(struct kvm_vcpu *vcpu) 4798 { 4799 if (!enable_vnmi || 4800 vmcs_read32(GUEST_INTERRUPTIBILITY_INFO) & GUEST_INTR_STATE_STI) { 4801 vmx_enable_irq_window(vcpu); 4802 return; 4803 } 4804 4805 exec_controls_setbit(to_vmx(vcpu), CPU_BASED_NMI_WINDOW_EXITING); 4806 } 4807 4808 static void vmx_inject_irq(struct kvm_vcpu *vcpu, bool reinjected) 4809 { 4810 struct vcpu_vmx *vmx = to_vmx(vcpu); 4811 uint32_t intr; 4812 int irq = vcpu->arch.interrupt.nr; 4813 4814 trace_kvm_inj_virq(irq, vcpu->arch.interrupt.soft, reinjected); 4815 4816 ++vcpu->stat.irq_injections; 4817 if (vmx->rmode.vm86_active) { 4818 int inc_eip = 0; 4819 if (vcpu->arch.interrupt.soft) 4820 inc_eip = vcpu->arch.event_exit_inst_len; 4821 kvm_inject_realmode_interrupt(vcpu, irq, inc_eip); 4822 return; 4823 } 4824 intr = irq | INTR_INFO_VALID_MASK; 4825 if (vcpu->arch.interrupt.soft) { 4826 intr |= INTR_TYPE_SOFT_INTR; 4827 vmcs_write32(VM_ENTRY_INSTRUCTION_LEN, 4828 vmx->vcpu.arch.event_exit_inst_len); 4829 } else 4830 intr |= INTR_TYPE_EXT_INTR; 4831 vmcs_write32(VM_ENTRY_INTR_INFO_FIELD, intr); 4832 4833 vmx_clear_hlt(vcpu); 4834 } 4835 4836 static void vmx_inject_nmi(struct kvm_vcpu *vcpu) 4837 { 4838 struct vcpu_vmx *vmx = to_vmx(vcpu); 4839 4840 if (!enable_vnmi) { 4841 /* 4842 * Tracking the NMI-blocked state in software is built upon 4843 * finding the next open IRQ window. This, in turn, depends on 4844 * well-behaving guests: They have to keep IRQs disabled at 4845 * least as long as the NMI handler runs. Otherwise we may 4846 * cause NMI nesting, maybe breaking the guest. But as this is 4847 * highly unlikely, we can live with the residual risk. 4848 */ 4849 vmx->loaded_vmcs->soft_vnmi_blocked = 1; 4850 vmx->loaded_vmcs->vnmi_blocked_time = 0; 4851 } 4852 4853 ++vcpu->stat.nmi_injections; 4854 vmx->loaded_vmcs->nmi_known_unmasked = false; 4855 4856 if (vmx->rmode.vm86_active) { 4857 kvm_inject_realmode_interrupt(vcpu, NMI_VECTOR, 0); 4858 return; 4859 } 4860 4861 vmcs_write32(VM_ENTRY_INTR_INFO_FIELD, 4862 INTR_TYPE_NMI_INTR | INTR_INFO_VALID_MASK | NMI_VECTOR); 4863 4864 vmx_clear_hlt(vcpu); 4865 } 4866 4867 bool vmx_get_nmi_mask(struct kvm_vcpu *vcpu) 4868 { 4869 struct vcpu_vmx *vmx = to_vmx(vcpu); 4870 bool masked; 4871 4872 if (!enable_vnmi) 4873 return vmx->loaded_vmcs->soft_vnmi_blocked; 4874 if (vmx->loaded_vmcs->nmi_known_unmasked) 4875 return false; 4876 masked = vmcs_read32(GUEST_INTERRUPTIBILITY_INFO) & GUEST_INTR_STATE_NMI; 4877 vmx->loaded_vmcs->nmi_known_unmasked = !masked; 4878 return masked; 4879 } 4880 4881 void vmx_set_nmi_mask(struct kvm_vcpu *vcpu, bool masked) 4882 { 4883 struct vcpu_vmx *vmx = to_vmx(vcpu); 4884 4885 if (!enable_vnmi) { 4886 if (vmx->loaded_vmcs->soft_vnmi_blocked != masked) { 4887 vmx->loaded_vmcs->soft_vnmi_blocked = masked; 4888 vmx->loaded_vmcs->vnmi_blocked_time = 0; 4889 } 4890 } else { 4891 vmx->loaded_vmcs->nmi_known_unmasked = !masked; 4892 if (masked) 4893 vmcs_set_bits(GUEST_INTERRUPTIBILITY_INFO, 4894 GUEST_INTR_STATE_NMI); 4895 else 4896 vmcs_clear_bits(GUEST_INTERRUPTIBILITY_INFO, 4897 GUEST_INTR_STATE_NMI); 4898 } 4899 } 4900 4901 bool vmx_nmi_blocked(struct kvm_vcpu *vcpu) 4902 { 4903 if (is_guest_mode(vcpu) && nested_exit_on_nmi(vcpu)) 4904 return false; 4905 4906 if (!enable_vnmi && to_vmx(vcpu)->loaded_vmcs->soft_vnmi_blocked) 4907 return true; 4908 4909 return (vmcs_read32(GUEST_INTERRUPTIBILITY_INFO) & 4910 (GUEST_INTR_STATE_MOV_SS | GUEST_INTR_STATE_STI | 4911 GUEST_INTR_STATE_NMI)); 4912 } 4913 4914 static int vmx_nmi_allowed(struct kvm_vcpu *vcpu, bool for_injection) 4915 { 4916 if (to_vmx(vcpu)->nested.nested_run_pending) 4917 return -EBUSY; 4918 4919 /* An NMI must not be injected into L2 if it's supposed to VM-Exit. */ 4920 if (for_injection && is_guest_mode(vcpu) && nested_exit_on_nmi(vcpu)) 4921 return -EBUSY; 4922 4923 return !vmx_nmi_blocked(vcpu); 4924 } 4925 4926 bool vmx_interrupt_blocked(struct kvm_vcpu *vcpu) 4927 { 4928 if (is_guest_mode(vcpu) && nested_exit_on_intr(vcpu)) 4929 return false; 4930 4931 return !(vmx_get_rflags(vcpu) & X86_EFLAGS_IF) || 4932 (vmcs_read32(GUEST_INTERRUPTIBILITY_INFO) & 4933 (GUEST_INTR_STATE_STI | GUEST_INTR_STATE_MOV_SS)); 4934 } 4935 4936 static int vmx_interrupt_allowed(struct kvm_vcpu *vcpu, bool for_injection) 4937 { 4938 if (to_vmx(vcpu)->nested.nested_run_pending) 4939 return -EBUSY; 4940 4941 /* 4942 * An IRQ must not be injected into L2 if it's supposed to VM-Exit, 4943 * e.g. if the IRQ arrived asynchronously after checking nested events. 4944 */ 4945 if (for_injection && is_guest_mode(vcpu) && nested_exit_on_intr(vcpu)) 4946 return -EBUSY; 4947 4948 return !vmx_interrupt_blocked(vcpu); 4949 } 4950 4951 static int vmx_set_tss_addr(struct kvm *kvm, unsigned int addr) 4952 { 4953 void __user *ret; 4954 4955 if (enable_unrestricted_guest) 4956 return 0; 4957 4958 mutex_lock(&kvm->slots_lock); 4959 ret = __x86_set_memory_region(kvm, TSS_PRIVATE_MEMSLOT, addr, 4960 PAGE_SIZE * 3); 4961 mutex_unlock(&kvm->slots_lock); 4962 4963 if (IS_ERR(ret)) 4964 return PTR_ERR(ret); 4965 4966 to_kvm_vmx(kvm)->tss_addr = addr; 4967 4968 return init_rmode_tss(kvm, ret); 4969 } 4970 4971 static int vmx_set_identity_map_addr(struct kvm *kvm, u64 ident_addr) 4972 { 4973 to_kvm_vmx(kvm)->ept_identity_map_addr = ident_addr; 4974 return 0; 4975 } 4976 4977 static bool rmode_exception(struct kvm_vcpu *vcpu, int vec) 4978 { 4979 switch (vec) { 4980 case BP_VECTOR: 4981 /* 4982 * Update instruction length as we may reinject the exception 4983 * from user space while in guest debugging mode. 4984 */ 4985 to_vmx(vcpu)->vcpu.arch.event_exit_inst_len = 4986 vmcs_read32(VM_EXIT_INSTRUCTION_LEN); 4987 if (vcpu->guest_debug & KVM_GUESTDBG_USE_SW_BP) 4988 return false; 4989 fallthrough; 4990 case DB_VECTOR: 4991 return !(vcpu->guest_debug & 4992 (KVM_GUESTDBG_SINGLESTEP | KVM_GUESTDBG_USE_HW_BP)); 4993 case DE_VECTOR: 4994 case OF_VECTOR: 4995 case BR_VECTOR: 4996 case UD_VECTOR: 4997 case DF_VECTOR: 4998 case SS_VECTOR: 4999 case GP_VECTOR: 5000 case MF_VECTOR: 5001 return true; 5002 } 5003 return false; 5004 } 5005 5006 static int handle_rmode_exception(struct kvm_vcpu *vcpu, 5007 int vec, u32 err_code) 5008 { 5009 /* 5010 * Instruction with address size override prefix opcode 0x67 5011 * Cause the #SS fault with 0 error code in VM86 mode. 5012 */ 5013 if (((vec == GP_VECTOR) || (vec == SS_VECTOR)) && err_code == 0) { 5014 if (kvm_emulate_instruction(vcpu, 0)) { 5015 if (vcpu->arch.halt_request) { 5016 vcpu->arch.halt_request = 0; 5017 return kvm_emulate_halt_noskip(vcpu); 5018 } 5019 return 1; 5020 } 5021 return 0; 5022 } 5023 5024 /* 5025 * Forward all other exceptions that are valid in real mode. 5026 * FIXME: Breaks guest debugging in real mode, needs to be fixed with 5027 * the required debugging infrastructure rework. 5028 */ 5029 kvm_queue_exception(vcpu, vec); 5030 return 1; 5031 } 5032 5033 static int handle_machine_check(struct kvm_vcpu *vcpu) 5034 { 5035 /* handled by vmx_vcpu_run() */ 5036 return 1; 5037 } 5038 5039 /* 5040 * If the host has split lock detection disabled, then #AC is 5041 * unconditionally injected into the guest, which is the pre split lock 5042 * detection behaviour. 5043 * 5044 * If the host has split lock detection enabled then #AC is 5045 * only injected into the guest when: 5046 * - Guest CPL == 3 (user mode) 5047 * - Guest has #AC detection enabled in CR0 5048 * - Guest EFLAGS has AC bit set 5049 */ 5050 bool vmx_guest_inject_ac(struct kvm_vcpu *vcpu) 5051 { 5052 if (!boot_cpu_has(X86_FEATURE_SPLIT_LOCK_DETECT)) 5053 return true; 5054 5055 return vmx_get_cpl(vcpu) == 3 && kvm_read_cr0_bits(vcpu, X86_CR0_AM) && 5056 (kvm_get_rflags(vcpu) & X86_EFLAGS_AC); 5057 } 5058 5059 static int handle_exception_nmi(struct kvm_vcpu *vcpu) 5060 { 5061 struct vcpu_vmx *vmx = to_vmx(vcpu); 5062 struct kvm_run *kvm_run = vcpu->run; 5063 u32 intr_info, ex_no, error_code; 5064 unsigned long cr2, dr6; 5065 u32 vect_info; 5066 5067 vect_info = vmx->idt_vectoring_info; 5068 intr_info = vmx_get_intr_info(vcpu); 5069 5070 if (is_machine_check(intr_info) || is_nmi(intr_info)) 5071 return 1; /* handled by handle_exception_nmi_irqoff() */ 5072 5073 /* 5074 * Queue the exception here instead of in handle_nm_fault_irqoff(). 5075 * This ensures the nested_vmx check is not skipped so vmexit can 5076 * be reflected to L1 (when it intercepts #NM) before reaching this 5077 * point. 5078 */ 5079 if (is_nm_fault(intr_info)) { 5080 kvm_queue_exception(vcpu, NM_VECTOR); 5081 return 1; 5082 } 5083 5084 if (is_invalid_opcode(intr_info)) 5085 return handle_ud(vcpu); 5086 5087 error_code = 0; 5088 if (intr_info & INTR_INFO_DELIVER_CODE_MASK) 5089 error_code = vmcs_read32(VM_EXIT_INTR_ERROR_CODE); 5090 5091 if (!vmx->rmode.vm86_active && is_gp_fault(intr_info)) { 5092 WARN_ON_ONCE(!enable_vmware_backdoor); 5093 5094 /* 5095 * VMware backdoor emulation on #GP interception only handles 5096 * IN{S}, OUT{S}, and RDPMC, none of which generate a non-zero 5097 * error code on #GP. 5098 */ 5099 if (error_code) { 5100 kvm_queue_exception_e(vcpu, GP_VECTOR, error_code); 5101 return 1; 5102 } 5103 return kvm_emulate_instruction(vcpu, EMULTYPE_VMWARE_GP); 5104 } 5105 5106 /* 5107 * The #PF with PFEC.RSVD = 1 indicates the guest is accessing 5108 * MMIO, it is better to report an internal error. 5109 * See the comments in vmx_handle_exit. 5110 */ 5111 if ((vect_info & VECTORING_INFO_VALID_MASK) && 5112 !(is_page_fault(intr_info) && !(error_code & PFERR_RSVD_MASK))) { 5113 vcpu->run->exit_reason = KVM_EXIT_INTERNAL_ERROR; 5114 vcpu->run->internal.suberror = KVM_INTERNAL_ERROR_SIMUL_EX; 5115 vcpu->run->internal.ndata = 4; 5116 vcpu->run->internal.data[0] = vect_info; 5117 vcpu->run->internal.data[1] = intr_info; 5118 vcpu->run->internal.data[2] = error_code; 5119 vcpu->run->internal.data[3] = vcpu->arch.last_vmentry_cpu; 5120 return 0; 5121 } 5122 5123 if (is_page_fault(intr_info)) { 5124 cr2 = vmx_get_exit_qual(vcpu); 5125 if (enable_ept && !vcpu->arch.apf.host_apf_flags) { 5126 /* 5127 * EPT will cause page fault only if we need to 5128 * detect illegal GPAs. 5129 */ 5130 WARN_ON_ONCE(!allow_smaller_maxphyaddr); 5131 kvm_fixup_and_inject_pf_error(vcpu, cr2, error_code); 5132 return 1; 5133 } else 5134 return kvm_handle_page_fault(vcpu, error_code, cr2, NULL, 0); 5135 } 5136 5137 ex_no = intr_info & INTR_INFO_VECTOR_MASK; 5138 5139 if (vmx->rmode.vm86_active && rmode_exception(vcpu, ex_no)) 5140 return handle_rmode_exception(vcpu, ex_no, error_code); 5141 5142 switch (ex_no) { 5143 case DB_VECTOR: 5144 dr6 = vmx_get_exit_qual(vcpu); 5145 if (!(vcpu->guest_debug & 5146 (KVM_GUESTDBG_SINGLESTEP | KVM_GUESTDBG_USE_HW_BP))) { 5147 /* 5148 * If the #DB was due to ICEBP, a.k.a. INT1, skip the 5149 * instruction. ICEBP generates a trap-like #DB, but 5150 * despite its interception control being tied to #DB, 5151 * is an instruction intercept, i.e. the VM-Exit occurs 5152 * on the ICEBP itself. Use the inner "skip" helper to 5153 * avoid single-step #DB and MTF updates, as ICEBP is 5154 * higher priority. Note, skipping ICEBP still clears 5155 * STI and MOVSS blocking. 5156 * 5157 * For all other #DBs, set vmcs.PENDING_DBG_EXCEPTIONS.BS 5158 * if single-step is enabled in RFLAGS and STI or MOVSS 5159 * blocking is active, as the CPU doesn't set the bit 5160 * on VM-Exit due to #DB interception. VM-Entry has a 5161 * consistency check that a single-step #DB is pending 5162 * in this scenario as the previous instruction cannot 5163 * have toggled RFLAGS.TF 0=>1 (because STI and POP/MOV 5164 * don't modify RFLAGS), therefore the one instruction 5165 * delay when activating single-step breakpoints must 5166 * have already expired. Note, the CPU sets/clears BS 5167 * as appropriate for all other VM-Exits types. 5168 */ 5169 if (is_icebp(intr_info)) 5170 WARN_ON(!skip_emulated_instruction(vcpu)); 5171 else if ((vmx_get_rflags(vcpu) & X86_EFLAGS_TF) && 5172 (vmcs_read32(GUEST_INTERRUPTIBILITY_INFO) & 5173 (GUEST_INTR_STATE_STI | GUEST_INTR_STATE_MOV_SS))) 5174 vmcs_writel(GUEST_PENDING_DBG_EXCEPTIONS, 5175 vmcs_readl(GUEST_PENDING_DBG_EXCEPTIONS) | DR6_BS); 5176 5177 kvm_queue_exception_p(vcpu, DB_VECTOR, dr6); 5178 return 1; 5179 } 5180 kvm_run->debug.arch.dr6 = dr6 | DR6_ACTIVE_LOW; 5181 kvm_run->debug.arch.dr7 = vmcs_readl(GUEST_DR7); 5182 fallthrough; 5183 case BP_VECTOR: 5184 /* 5185 * Update instruction length as we may reinject #BP from 5186 * user space while in guest debugging mode. Reading it for 5187 * #DB as well causes no harm, it is not used in that case. 5188 */ 5189 vmx->vcpu.arch.event_exit_inst_len = 5190 vmcs_read32(VM_EXIT_INSTRUCTION_LEN); 5191 kvm_run->exit_reason = KVM_EXIT_DEBUG; 5192 kvm_run->debug.arch.pc = kvm_get_linear_rip(vcpu); 5193 kvm_run->debug.arch.exception = ex_no; 5194 break; 5195 case AC_VECTOR: 5196 if (vmx_guest_inject_ac(vcpu)) { 5197 kvm_queue_exception_e(vcpu, AC_VECTOR, error_code); 5198 return 1; 5199 } 5200 5201 /* 5202 * Handle split lock. Depending on detection mode this will 5203 * either warn and disable split lock detection for this 5204 * task or force SIGBUS on it. 5205 */ 5206 if (handle_guest_split_lock(kvm_rip_read(vcpu))) 5207 return 1; 5208 fallthrough; 5209 default: 5210 kvm_run->exit_reason = KVM_EXIT_EXCEPTION; 5211 kvm_run->ex.exception = ex_no; 5212 kvm_run->ex.error_code = error_code; 5213 break; 5214 } 5215 return 0; 5216 } 5217 5218 static __always_inline int handle_external_interrupt(struct kvm_vcpu *vcpu) 5219 { 5220 ++vcpu->stat.irq_exits; 5221 return 1; 5222 } 5223 5224 static int handle_triple_fault(struct kvm_vcpu *vcpu) 5225 { 5226 vcpu->run->exit_reason = KVM_EXIT_SHUTDOWN; 5227 vcpu->mmio_needed = 0; 5228 return 0; 5229 } 5230 5231 static int handle_io(struct kvm_vcpu *vcpu) 5232 { 5233 unsigned long exit_qualification; 5234 int size, in, string; 5235 unsigned port; 5236 5237 exit_qualification = vmx_get_exit_qual(vcpu); 5238 string = (exit_qualification & 16) != 0; 5239 5240 ++vcpu->stat.io_exits; 5241 5242 if (string) 5243 return kvm_emulate_instruction(vcpu, 0); 5244 5245 port = exit_qualification >> 16; 5246 size = (exit_qualification & 7) + 1; 5247 in = (exit_qualification & 8) != 0; 5248 5249 return kvm_fast_pio(vcpu, size, port, in); 5250 } 5251 5252 static void 5253 vmx_patch_hypercall(struct kvm_vcpu *vcpu, unsigned char *hypercall) 5254 { 5255 /* 5256 * Patch in the VMCALL instruction: 5257 */ 5258 hypercall[0] = 0x0f; 5259 hypercall[1] = 0x01; 5260 hypercall[2] = 0xc1; 5261 } 5262 5263 /* called to set cr0 as appropriate for a mov-to-cr0 exit. */ 5264 static int handle_set_cr0(struct kvm_vcpu *vcpu, unsigned long val) 5265 { 5266 if (is_guest_mode(vcpu)) { 5267 struct vmcs12 *vmcs12 = get_vmcs12(vcpu); 5268 unsigned long orig_val = val; 5269 5270 /* 5271 * We get here when L2 changed cr0 in a way that did not change 5272 * any of L1's shadowed bits (see nested_vmx_exit_handled_cr), 5273 * but did change L0 shadowed bits. So we first calculate the 5274 * effective cr0 value that L1 would like to write into the 5275 * hardware. It consists of the L2-owned bits from the new 5276 * value combined with the L1-owned bits from L1's guest_cr0. 5277 */ 5278 val = (val & ~vmcs12->cr0_guest_host_mask) | 5279 (vmcs12->guest_cr0 & vmcs12->cr0_guest_host_mask); 5280 5281 if (!nested_guest_cr0_valid(vcpu, val)) 5282 return 1; 5283 5284 if (kvm_set_cr0(vcpu, val)) 5285 return 1; 5286 vmcs_writel(CR0_READ_SHADOW, orig_val); 5287 return 0; 5288 } else { 5289 if (to_vmx(vcpu)->nested.vmxon && 5290 !nested_host_cr0_valid(vcpu, val)) 5291 return 1; 5292 5293 return kvm_set_cr0(vcpu, val); 5294 } 5295 } 5296 5297 static int handle_set_cr4(struct kvm_vcpu *vcpu, unsigned long val) 5298 { 5299 if (is_guest_mode(vcpu)) { 5300 struct vmcs12 *vmcs12 = get_vmcs12(vcpu); 5301 unsigned long orig_val = val; 5302 5303 /* analogously to handle_set_cr0 */ 5304 val = (val & ~vmcs12->cr4_guest_host_mask) | 5305 (vmcs12->guest_cr4 & vmcs12->cr4_guest_host_mask); 5306 if (kvm_set_cr4(vcpu, val)) 5307 return 1; 5308 vmcs_writel(CR4_READ_SHADOW, orig_val); 5309 return 0; 5310 } else 5311 return kvm_set_cr4(vcpu, val); 5312 } 5313 5314 static int handle_desc(struct kvm_vcpu *vcpu) 5315 { 5316 WARN_ON(!(vcpu->arch.cr4 & X86_CR4_UMIP)); 5317 return kvm_emulate_instruction(vcpu, 0); 5318 } 5319 5320 static int handle_cr(struct kvm_vcpu *vcpu) 5321 { 5322 unsigned long exit_qualification, val; 5323 int cr; 5324 int reg; 5325 int err; 5326 int ret; 5327 5328 exit_qualification = vmx_get_exit_qual(vcpu); 5329 cr = exit_qualification & 15; 5330 reg = (exit_qualification >> 8) & 15; 5331 switch ((exit_qualification >> 4) & 3) { 5332 case 0: /* mov to cr */ 5333 val = kvm_register_read(vcpu, reg); 5334 trace_kvm_cr_write(cr, val); 5335 switch (cr) { 5336 case 0: 5337 err = handle_set_cr0(vcpu, val); 5338 return kvm_complete_insn_gp(vcpu, err); 5339 case 3: 5340 WARN_ON_ONCE(enable_unrestricted_guest); 5341 5342 err = kvm_set_cr3(vcpu, val); 5343 return kvm_complete_insn_gp(vcpu, err); 5344 case 4: 5345 err = handle_set_cr4(vcpu, val); 5346 return kvm_complete_insn_gp(vcpu, err); 5347 case 8: { 5348 u8 cr8_prev = kvm_get_cr8(vcpu); 5349 u8 cr8 = (u8)val; 5350 err = kvm_set_cr8(vcpu, cr8); 5351 ret = kvm_complete_insn_gp(vcpu, err); 5352 if (lapic_in_kernel(vcpu)) 5353 return ret; 5354 if (cr8_prev <= cr8) 5355 return ret; 5356 /* 5357 * TODO: we might be squashing a 5358 * KVM_GUESTDBG_SINGLESTEP-triggered 5359 * KVM_EXIT_DEBUG here. 5360 */ 5361 vcpu->run->exit_reason = KVM_EXIT_SET_TPR; 5362 return 0; 5363 } 5364 } 5365 break; 5366 case 2: /* clts */ 5367 KVM_BUG(1, vcpu->kvm, "Guest always owns CR0.TS"); 5368 return -EIO; 5369 case 1: /*mov from cr*/ 5370 switch (cr) { 5371 case 3: 5372 WARN_ON_ONCE(enable_unrestricted_guest); 5373 5374 val = kvm_read_cr3(vcpu); 5375 kvm_register_write(vcpu, reg, val); 5376 trace_kvm_cr_read(cr, val); 5377 return kvm_skip_emulated_instruction(vcpu); 5378 case 8: 5379 val = kvm_get_cr8(vcpu); 5380 kvm_register_write(vcpu, reg, val); 5381 trace_kvm_cr_read(cr, val); 5382 return kvm_skip_emulated_instruction(vcpu); 5383 } 5384 break; 5385 case 3: /* lmsw */ 5386 val = (exit_qualification >> LMSW_SOURCE_DATA_SHIFT) & 0x0f; 5387 trace_kvm_cr_write(0, (kvm_read_cr0(vcpu) & ~0xful) | val); 5388 kvm_lmsw(vcpu, val); 5389 5390 return kvm_skip_emulated_instruction(vcpu); 5391 default: 5392 break; 5393 } 5394 vcpu->run->exit_reason = 0; 5395 vcpu_unimpl(vcpu, "unhandled control register: op %d cr %d\n", 5396 (int)(exit_qualification >> 4) & 3, cr); 5397 return 0; 5398 } 5399 5400 static int handle_dr(struct kvm_vcpu *vcpu) 5401 { 5402 unsigned long exit_qualification; 5403 int dr, dr7, reg; 5404 int err = 1; 5405 5406 exit_qualification = vmx_get_exit_qual(vcpu); 5407 dr = exit_qualification & DEBUG_REG_ACCESS_NUM; 5408 5409 /* First, if DR does not exist, trigger UD */ 5410 if (!kvm_require_dr(vcpu, dr)) 5411 return 1; 5412 5413 if (vmx_get_cpl(vcpu) > 0) 5414 goto out; 5415 5416 dr7 = vmcs_readl(GUEST_DR7); 5417 if (dr7 & DR7_GD) { 5418 /* 5419 * As the vm-exit takes precedence over the debug trap, we 5420 * need to emulate the latter, either for the host or the 5421 * guest debugging itself. 5422 */ 5423 if (vcpu->guest_debug & KVM_GUESTDBG_USE_HW_BP) { 5424 vcpu->run->debug.arch.dr6 = DR6_BD | DR6_ACTIVE_LOW; 5425 vcpu->run->debug.arch.dr7 = dr7; 5426 vcpu->run->debug.arch.pc = kvm_get_linear_rip(vcpu); 5427 vcpu->run->debug.arch.exception = DB_VECTOR; 5428 vcpu->run->exit_reason = KVM_EXIT_DEBUG; 5429 return 0; 5430 } else { 5431 kvm_queue_exception_p(vcpu, DB_VECTOR, DR6_BD); 5432 return 1; 5433 } 5434 } 5435 5436 if (vcpu->guest_debug == 0) { 5437 exec_controls_clearbit(to_vmx(vcpu), CPU_BASED_MOV_DR_EXITING); 5438 5439 /* 5440 * No more DR vmexits; force a reload of the debug registers 5441 * and reenter on this instruction. The next vmexit will 5442 * retrieve the full state of the debug registers. 5443 */ 5444 vcpu->arch.switch_db_regs |= KVM_DEBUGREG_WONT_EXIT; 5445 return 1; 5446 } 5447 5448 reg = DEBUG_REG_ACCESS_REG(exit_qualification); 5449 if (exit_qualification & TYPE_MOV_FROM_DR) { 5450 unsigned long val; 5451 5452 kvm_get_dr(vcpu, dr, &val); 5453 kvm_register_write(vcpu, reg, val); 5454 err = 0; 5455 } else { 5456 err = kvm_set_dr(vcpu, dr, kvm_register_read(vcpu, reg)); 5457 } 5458 5459 out: 5460 return kvm_complete_insn_gp(vcpu, err); 5461 } 5462 5463 static void vmx_sync_dirty_debug_regs(struct kvm_vcpu *vcpu) 5464 { 5465 get_debugreg(vcpu->arch.db[0], 0); 5466 get_debugreg(vcpu->arch.db[1], 1); 5467 get_debugreg(vcpu->arch.db[2], 2); 5468 get_debugreg(vcpu->arch.db[3], 3); 5469 get_debugreg(vcpu->arch.dr6, 6); 5470 vcpu->arch.dr7 = vmcs_readl(GUEST_DR7); 5471 5472 vcpu->arch.switch_db_regs &= ~KVM_DEBUGREG_WONT_EXIT; 5473 exec_controls_setbit(to_vmx(vcpu), CPU_BASED_MOV_DR_EXITING); 5474 5475 /* 5476 * exc_debug expects dr6 to be cleared after it runs, avoid that it sees 5477 * a stale dr6 from the guest. 5478 */ 5479 set_debugreg(DR6_RESERVED, 6); 5480 } 5481 5482 static void vmx_set_dr7(struct kvm_vcpu *vcpu, unsigned long val) 5483 { 5484 vmcs_writel(GUEST_DR7, val); 5485 } 5486 5487 static int handle_tpr_below_threshold(struct kvm_vcpu *vcpu) 5488 { 5489 kvm_apic_update_ppr(vcpu); 5490 return 1; 5491 } 5492 5493 static int handle_interrupt_window(struct kvm_vcpu *vcpu) 5494 { 5495 exec_controls_clearbit(to_vmx(vcpu), CPU_BASED_INTR_WINDOW_EXITING); 5496 5497 kvm_make_request(KVM_REQ_EVENT, vcpu); 5498 5499 ++vcpu->stat.irq_window_exits; 5500 return 1; 5501 } 5502 5503 static int handle_invlpg(struct kvm_vcpu *vcpu) 5504 { 5505 unsigned long exit_qualification = vmx_get_exit_qual(vcpu); 5506 5507 kvm_mmu_invlpg(vcpu, exit_qualification); 5508 return kvm_skip_emulated_instruction(vcpu); 5509 } 5510 5511 static int handle_apic_access(struct kvm_vcpu *vcpu) 5512 { 5513 if (likely(fasteoi)) { 5514 unsigned long exit_qualification = vmx_get_exit_qual(vcpu); 5515 int access_type, offset; 5516 5517 access_type = exit_qualification & APIC_ACCESS_TYPE; 5518 offset = exit_qualification & APIC_ACCESS_OFFSET; 5519 /* 5520 * Sane guest uses MOV to write EOI, with written value 5521 * not cared. So make a short-circuit here by avoiding 5522 * heavy instruction emulation. 5523 */ 5524 if ((access_type == TYPE_LINEAR_APIC_INST_WRITE) && 5525 (offset == APIC_EOI)) { 5526 kvm_lapic_set_eoi(vcpu); 5527 return kvm_skip_emulated_instruction(vcpu); 5528 } 5529 } 5530 return kvm_emulate_instruction(vcpu, 0); 5531 } 5532 5533 static int handle_apic_eoi_induced(struct kvm_vcpu *vcpu) 5534 { 5535 unsigned long exit_qualification = vmx_get_exit_qual(vcpu); 5536 int vector = exit_qualification & 0xff; 5537 5538 /* EOI-induced VM exit is trap-like and thus no need to adjust IP */ 5539 kvm_apic_set_eoi_accelerated(vcpu, vector); 5540 return 1; 5541 } 5542 5543 static int handle_apic_write(struct kvm_vcpu *vcpu) 5544 { 5545 unsigned long exit_qualification = vmx_get_exit_qual(vcpu); 5546 5547 /* 5548 * APIC-write VM-Exit is trap-like, KVM doesn't need to advance RIP and 5549 * hardware has done any necessary aliasing, offset adjustments, etc... 5550 * for the access. I.e. the correct value has already been written to 5551 * the vAPIC page for the correct 16-byte chunk. KVM needs only to 5552 * retrieve the register value and emulate the access. 5553 */ 5554 u32 offset = exit_qualification & 0xff0; 5555 5556 kvm_apic_write_nodecode(vcpu, offset); 5557 return 1; 5558 } 5559 5560 static int handle_task_switch(struct kvm_vcpu *vcpu) 5561 { 5562 struct vcpu_vmx *vmx = to_vmx(vcpu); 5563 unsigned long exit_qualification; 5564 bool has_error_code = false; 5565 u32 error_code = 0; 5566 u16 tss_selector; 5567 int reason, type, idt_v, idt_index; 5568 5569 idt_v = (vmx->idt_vectoring_info & VECTORING_INFO_VALID_MASK); 5570 idt_index = (vmx->idt_vectoring_info & VECTORING_INFO_VECTOR_MASK); 5571 type = (vmx->idt_vectoring_info & VECTORING_INFO_TYPE_MASK); 5572 5573 exit_qualification = vmx_get_exit_qual(vcpu); 5574 5575 reason = (u32)exit_qualification >> 30; 5576 if (reason == TASK_SWITCH_GATE && idt_v) { 5577 switch (type) { 5578 case INTR_TYPE_NMI_INTR: 5579 vcpu->arch.nmi_injected = false; 5580 vmx_set_nmi_mask(vcpu, true); 5581 break; 5582 case INTR_TYPE_EXT_INTR: 5583 case INTR_TYPE_SOFT_INTR: 5584 kvm_clear_interrupt_queue(vcpu); 5585 break; 5586 case INTR_TYPE_HARD_EXCEPTION: 5587 if (vmx->idt_vectoring_info & 5588 VECTORING_INFO_DELIVER_CODE_MASK) { 5589 has_error_code = true; 5590 error_code = 5591 vmcs_read32(IDT_VECTORING_ERROR_CODE); 5592 } 5593 fallthrough; 5594 case INTR_TYPE_SOFT_EXCEPTION: 5595 kvm_clear_exception_queue(vcpu); 5596 break; 5597 default: 5598 break; 5599 } 5600 } 5601 tss_selector = exit_qualification; 5602 5603 if (!idt_v || (type != INTR_TYPE_HARD_EXCEPTION && 5604 type != INTR_TYPE_EXT_INTR && 5605 type != INTR_TYPE_NMI_INTR)) 5606 WARN_ON(!skip_emulated_instruction(vcpu)); 5607 5608 /* 5609 * TODO: What about debug traps on tss switch? 5610 * Are we supposed to inject them and update dr6? 5611 */ 5612 return kvm_task_switch(vcpu, tss_selector, 5613 type == INTR_TYPE_SOFT_INTR ? idt_index : -1, 5614 reason, has_error_code, error_code); 5615 } 5616 5617 static int handle_ept_violation(struct kvm_vcpu *vcpu) 5618 { 5619 unsigned long exit_qualification; 5620 gpa_t gpa; 5621 u64 error_code; 5622 5623 exit_qualification = vmx_get_exit_qual(vcpu); 5624 5625 /* 5626 * EPT violation happened while executing iret from NMI, 5627 * "blocked by NMI" bit has to be set before next VM entry. 5628 * There are errata that may cause this bit to not be set: 5629 * AAK134, BY25. 5630 */ 5631 if (!(to_vmx(vcpu)->idt_vectoring_info & VECTORING_INFO_VALID_MASK) && 5632 enable_vnmi && 5633 (exit_qualification & INTR_INFO_UNBLOCK_NMI)) 5634 vmcs_set_bits(GUEST_INTERRUPTIBILITY_INFO, GUEST_INTR_STATE_NMI); 5635 5636 gpa = vmcs_read64(GUEST_PHYSICAL_ADDRESS); 5637 trace_kvm_page_fault(vcpu, gpa, exit_qualification); 5638 5639 /* Is it a read fault? */ 5640 error_code = (exit_qualification & EPT_VIOLATION_ACC_READ) 5641 ? PFERR_USER_MASK : 0; 5642 /* Is it a write fault? */ 5643 error_code |= (exit_qualification & EPT_VIOLATION_ACC_WRITE) 5644 ? PFERR_WRITE_MASK : 0; 5645 /* Is it a fetch fault? */ 5646 error_code |= (exit_qualification & EPT_VIOLATION_ACC_INSTR) 5647 ? PFERR_FETCH_MASK : 0; 5648 /* ept page table entry is present? */ 5649 error_code |= (exit_qualification & EPT_VIOLATION_RWX_MASK) 5650 ? PFERR_PRESENT_MASK : 0; 5651 5652 error_code |= (exit_qualification & EPT_VIOLATION_GVA_TRANSLATED) != 0 ? 5653 PFERR_GUEST_FINAL_MASK : PFERR_GUEST_PAGE_MASK; 5654 5655 vcpu->arch.exit_qualification = exit_qualification; 5656 5657 /* 5658 * Check that the GPA doesn't exceed physical memory limits, as that is 5659 * a guest page fault. We have to emulate the instruction here, because 5660 * if the illegal address is that of a paging structure, then 5661 * EPT_VIOLATION_ACC_WRITE bit is set. Alternatively, if supported we 5662 * would also use advanced VM-exit information for EPT violations to 5663 * reconstruct the page fault error code. 5664 */ 5665 if (unlikely(allow_smaller_maxphyaddr && kvm_vcpu_is_illegal_gpa(vcpu, gpa))) 5666 return kvm_emulate_instruction(vcpu, 0); 5667 5668 return kvm_mmu_page_fault(vcpu, gpa, error_code, NULL, 0); 5669 } 5670 5671 static int handle_ept_misconfig(struct kvm_vcpu *vcpu) 5672 { 5673 gpa_t gpa; 5674 5675 if (!vmx_can_emulate_instruction(vcpu, EMULTYPE_PF, NULL, 0)) 5676 return 1; 5677 5678 /* 5679 * A nested guest cannot optimize MMIO vmexits, because we have an 5680 * nGPA here instead of the required GPA. 5681 */ 5682 gpa = vmcs_read64(GUEST_PHYSICAL_ADDRESS); 5683 if (!is_guest_mode(vcpu) && 5684 !kvm_io_bus_write(vcpu, KVM_FAST_MMIO_BUS, gpa, 0, NULL)) { 5685 trace_kvm_fast_mmio(gpa); 5686 return kvm_skip_emulated_instruction(vcpu); 5687 } 5688 5689 return kvm_mmu_page_fault(vcpu, gpa, PFERR_RSVD_MASK, NULL, 0); 5690 } 5691 5692 static int handle_nmi_window(struct kvm_vcpu *vcpu) 5693 { 5694 if (KVM_BUG_ON(!enable_vnmi, vcpu->kvm)) 5695 return -EIO; 5696 5697 exec_controls_clearbit(to_vmx(vcpu), CPU_BASED_NMI_WINDOW_EXITING); 5698 ++vcpu->stat.nmi_window_exits; 5699 kvm_make_request(KVM_REQ_EVENT, vcpu); 5700 5701 return 1; 5702 } 5703 5704 static bool vmx_emulation_required_with_pending_exception(struct kvm_vcpu *vcpu) 5705 { 5706 struct vcpu_vmx *vmx = to_vmx(vcpu); 5707 5708 return vmx->emulation_required && !vmx->rmode.vm86_active && 5709 (kvm_is_exception_pending(vcpu) || vcpu->arch.exception.injected); 5710 } 5711 5712 static int handle_invalid_guest_state(struct kvm_vcpu *vcpu) 5713 { 5714 struct vcpu_vmx *vmx = to_vmx(vcpu); 5715 bool intr_window_requested; 5716 unsigned count = 130; 5717 5718 intr_window_requested = exec_controls_get(vmx) & 5719 CPU_BASED_INTR_WINDOW_EXITING; 5720 5721 while (vmx->emulation_required && count-- != 0) { 5722 if (intr_window_requested && !vmx_interrupt_blocked(vcpu)) 5723 return handle_interrupt_window(&vmx->vcpu); 5724 5725 if (kvm_test_request(KVM_REQ_EVENT, vcpu)) 5726 return 1; 5727 5728 if (!kvm_emulate_instruction(vcpu, 0)) 5729 return 0; 5730 5731 if (vmx_emulation_required_with_pending_exception(vcpu)) { 5732 kvm_prepare_emulation_failure_exit(vcpu); 5733 return 0; 5734 } 5735 5736 if (vcpu->arch.halt_request) { 5737 vcpu->arch.halt_request = 0; 5738 return kvm_emulate_halt_noskip(vcpu); 5739 } 5740 5741 /* 5742 * Note, return 1 and not 0, vcpu_run() will invoke 5743 * xfer_to_guest_mode() which will create a proper return 5744 * code. 5745 */ 5746 if (__xfer_to_guest_mode_work_pending()) 5747 return 1; 5748 } 5749 5750 return 1; 5751 } 5752 5753 static int vmx_vcpu_pre_run(struct kvm_vcpu *vcpu) 5754 { 5755 if (vmx_emulation_required_with_pending_exception(vcpu)) { 5756 kvm_prepare_emulation_failure_exit(vcpu); 5757 return 0; 5758 } 5759 5760 return 1; 5761 } 5762 5763 static void grow_ple_window(struct kvm_vcpu *vcpu) 5764 { 5765 struct vcpu_vmx *vmx = to_vmx(vcpu); 5766 unsigned int old = vmx->ple_window; 5767 5768 vmx->ple_window = __grow_ple_window(old, ple_window, 5769 ple_window_grow, 5770 ple_window_max); 5771 5772 if (vmx->ple_window != old) { 5773 vmx->ple_window_dirty = true; 5774 trace_kvm_ple_window_update(vcpu->vcpu_id, 5775 vmx->ple_window, old); 5776 } 5777 } 5778 5779 static void shrink_ple_window(struct kvm_vcpu *vcpu) 5780 { 5781 struct vcpu_vmx *vmx = to_vmx(vcpu); 5782 unsigned int old = vmx->ple_window; 5783 5784 vmx->ple_window = __shrink_ple_window(old, ple_window, 5785 ple_window_shrink, 5786 ple_window); 5787 5788 if (vmx->ple_window != old) { 5789 vmx->ple_window_dirty = true; 5790 trace_kvm_ple_window_update(vcpu->vcpu_id, 5791 vmx->ple_window, old); 5792 } 5793 } 5794 5795 /* 5796 * Indicate a busy-waiting vcpu in spinlock. We do not enable the PAUSE 5797 * exiting, so only get here on cpu with PAUSE-Loop-Exiting. 5798 */ 5799 static int handle_pause(struct kvm_vcpu *vcpu) 5800 { 5801 if (!kvm_pause_in_guest(vcpu->kvm)) 5802 grow_ple_window(vcpu); 5803 5804 /* 5805 * Intel sdm vol3 ch-25.1.3 says: The "PAUSE-loop exiting" 5806 * VM-execution control is ignored if CPL > 0. OTOH, KVM 5807 * never set PAUSE_EXITING and just set PLE if supported, 5808 * so the vcpu must be CPL=0 if it gets a PAUSE exit. 5809 */ 5810 kvm_vcpu_on_spin(vcpu, true); 5811 return kvm_skip_emulated_instruction(vcpu); 5812 } 5813 5814 static int handle_monitor_trap(struct kvm_vcpu *vcpu) 5815 { 5816 return 1; 5817 } 5818 5819 static int handle_invpcid(struct kvm_vcpu *vcpu) 5820 { 5821 u32 vmx_instruction_info; 5822 unsigned long type; 5823 gva_t gva; 5824 struct { 5825 u64 pcid; 5826 u64 gla; 5827 } operand; 5828 int gpr_index; 5829 5830 if (!guest_cpuid_has(vcpu, X86_FEATURE_INVPCID)) { 5831 kvm_queue_exception(vcpu, UD_VECTOR); 5832 return 1; 5833 } 5834 5835 vmx_instruction_info = vmcs_read32(VMX_INSTRUCTION_INFO); 5836 gpr_index = vmx_get_instr_info_reg2(vmx_instruction_info); 5837 type = kvm_register_read(vcpu, gpr_index); 5838 5839 /* According to the Intel instruction reference, the memory operand 5840 * is read even if it isn't needed (e.g., for type==all) 5841 */ 5842 if (get_vmx_mem_address(vcpu, vmx_get_exit_qual(vcpu), 5843 vmx_instruction_info, false, 5844 sizeof(operand), &gva)) 5845 return 1; 5846 5847 return kvm_handle_invpcid(vcpu, type, gva); 5848 } 5849 5850 static int handle_pml_full(struct kvm_vcpu *vcpu) 5851 { 5852 unsigned long exit_qualification; 5853 5854 trace_kvm_pml_full(vcpu->vcpu_id); 5855 5856 exit_qualification = vmx_get_exit_qual(vcpu); 5857 5858 /* 5859 * PML buffer FULL happened while executing iret from NMI, 5860 * "blocked by NMI" bit has to be set before next VM entry. 5861 */ 5862 if (!(to_vmx(vcpu)->idt_vectoring_info & VECTORING_INFO_VALID_MASK) && 5863 enable_vnmi && 5864 (exit_qualification & INTR_INFO_UNBLOCK_NMI)) 5865 vmcs_set_bits(GUEST_INTERRUPTIBILITY_INFO, 5866 GUEST_INTR_STATE_NMI); 5867 5868 /* 5869 * PML buffer already flushed at beginning of VMEXIT. Nothing to do 5870 * here.., and there's no userspace involvement needed for PML. 5871 */ 5872 return 1; 5873 } 5874 5875 static fastpath_t handle_fastpath_preemption_timer(struct kvm_vcpu *vcpu) 5876 { 5877 struct vcpu_vmx *vmx = to_vmx(vcpu); 5878 5879 if (!vmx->req_immediate_exit && 5880 !unlikely(vmx->loaded_vmcs->hv_timer_soft_disabled)) { 5881 kvm_lapic_expired_hv_timer(vcpu); 5882 return EXIT_FASTPATH_REENTER_GUEST; 5883 } 5884 5885 return EXIT_FASTPATH_NONE; 5886 } 5887 5888 static int handle_preemption_timer(struct kvm_vcpu *vcpu) 5889 { 5890 handle_fastpath_preemption_timer(vcpu); 5891 return 1; 5892 } 5893 5894 /* 5895 * When nested=0, all VMX instruction VM Exits filter here. The handlers 5896 * are overwritten by nested_vmx_setup() when nested=1. 5897 */ 5898 static int handle_vmx_instruction(struct kvm_vcpu *vcpu) 5899 { 5900 kvm_queue_exception(vcpu, UD_VECTOR); 5901 return 1; 5902 } 5903 5904 #ifndef CONFIG_X86_SGX_KVM 5905 static int handle_encls(struct kvm_vcpu *vcpu) 5906 { 5907 /* 5908 * SGX virtualization is disabled. There is no software enable bit for 5909 * SGX, so KVM intercepts all ENCLS leafs and injects a #UD to prevent 5910 * the guest from executing ENCLS (when SGX is supported by hardware). 5911 */ 5912 kvm_queue_exception(vcpu, UD_VECTOR); 5913 return 1; 5914 } 5915 #endif /* CONFIG_X86_SGX_KVM */ 5916 5917 static int handle_bus_lock_vmexit(struct kvm_vcpu *vcpu) 5918 { 5919 /* 5920 * Hardware may or may not set the BUS_LOCK_DETECTED flag on BUS_LOCK 5921 * VM-Exits. Unconditionally set the flag here and leave the handling to 5922 * vmx_handle_exit(). 5923 */ 5924 to_vmx(vcpu)->exit_reason.bus_lock_detected = true; 5925 return 1; 5926 } 5927 5928 static int handle_notify(struct kvm_vcpu *vcpu) 5929 { 5930 unsigned long exit_qual = vmx_get_exit_qual(vcpu); 5931 bool context_invalid = exit_qual & NOTIFY_VM_CONTEXT_INVALID; 5932 5933 ++vcpu->stat.notify_window_exits; 5934 5935 /* 5936 * Notify VM exit happened while executing iret from NMI, 5937 * "blocked by NMI" bit has to be set before next VM entry. 5938 */ 5939 if (enable_vnmi && (exit_qual & INTR_INFO_UNBLOCK_NMI)) 5940 vmcs_set_bits(GUEST_INTERRUPTIBILITY_INFO, 5941 GUEST_INTR_STATE_NMI); 5942 5943 if (vcpu->kvm->arch.notify_vmexit_flags & KVM_X86_NOTIFY_VMEXIT_USER || 5944 context_invalid) { 5945 vcpu->run->exit_reason = KVM_EXIT_NOTIFY; 5946 vcpu->run->notify.flags = context_invalid ? 5947 KVM_NOTIFY_CONTEXT_INVALID : 0; 5948 return 0; 5949 } 5950 5951 return 1; 5952 } 5953 5954 /* 5955 * The exit handlers return 1 if the exit was handled fully and guest execution 5956 * may resume. Otherwise they set the kvm_run parameter to indicate what needs 5957 * to be done to userspace and return 0. 5958 */ 5959 static int (*kvm_vmx_exit_handlers[])(struct kvm_vcpu *vcpu) = { 5960 [EXIT_REASON_EXCEPTION_NMI] = handle_exception_nmi, 5961 [EXIT_REASON_EXTERNAL_INTERRUPT] = handle_external_interrupt, 5962 [EXIT_REASON_TRIPLE_FAULT] = handle_triple_fault, 5963 [EXIT_REASON_NMI_WINDOW] = handle_nmi_window, 5964 [EXIT_REASON_IO_INSTRUCTION] = handle_io, 5965 [EXIT_REASON_CR_ACCESS] = handle_cr, 5966 [EXIT_REASON_DR_ACCESS] = handle_dr, 5967 [EXIT_REASON_CPUID] = kvm_emulate_cpuid, 5968 [EXIT_REASON_MSR_READ] = kvm_emulate_rdmsr, 5969 [EXIT_REASON_MSR_WRITE] = kvm_emulate_wrmsr, 5970 [EXIT_REASON_INTERRUPT_WINDOW] = handle_interrupt_window, 5971 [EXIT_REASON_HLT] = kvm_emulate_halt, 5972 [EXIT_REASON_INVD] = kvm_emulate_invd, 5973 [EXIT_REASON_INVLPG] = handle_invlpg, 5974 [EXIT_REASON_RDPMC] = kvm_emulate_rdpmc, 5975 [EXIT_REASON_VMCALL] = kvm_emulate_hypercall, 5976 [EXIT_REASON_VMCLEAR] = handle_vmx_instruction, 5977 [EXIT_REASON_VMLAUNCH] = handle_vmx_instruction, 5978 [EXIT_REASON_VMPTRLD] = handle_vmx_instruction, 5979 [EXIT_REASON_VMPTRST] = handle_vmx_instruction, 5980 [EXIT_REASON_VMREAD] = handle_vmx_instruction, 5981 [EXIT_REASON_VMRESUME] = handle_vmx_instruction, 5982 [EXIT_REASON_VMWRITE] = handle_vmx_instruction, 5983 [EXIT_REASON_VMOFF] = handle_vmx_instruction, 5984 [EXIT_REASON_VMON] = handle_vmx_instruction, 5985 [EXIT_REASON_TPR_BELOW_THRESHOLD] = handle_tpr_below_threshold, 5986 [EXIT_REASON_APIC_ACCESS] = handle_apic_access, 5987 [EXIT_REASON_APIC_WRITE] = handle_apic_write, 5988 [EXIT_REASON_EOI_INDUCED] = handle_apic_eoi_induced, 5989 [EXIT_REASON_WBINVD] = kvm_emulate_wbinvd, 5990 [EXIT_REASON_XSETBV] = kvm_emulate_xsetbv, 5991 [EXIT_REASON_TASK_SWITCH] = handle_task_switch, 5992 [EXIT_REASON_MCE_DURING_VMENTRY] = handle_machine_check, 5993 [EXIT_REASON_GDTR_IDTR] = handle_desc, 5994 [EXIT_REASON_LDTR_TR] = handle_desc, 5995 [EXIT_REASON_EPT_VIOLATION] = handle_ept_violation, 5996 [EXIT_REASON_EPT_MISCONFIG] = handle_ept_misconfig, 5997 [EXIT_REASON_PAUSE_INSTRUCTION] = handle_pause, 5998 [EXIT_REASON_MWAIT_INSTRUCTION] = kvm_emulate_mwait, 5999 [EXIT_REASON_MONITOR_TRAP_FLAG] = handle_monitor_trap, 6000 [EXIT_REASON_MONITOR_INSTRUCTION] = kvm_emulate_monitor, 6001 [EXIT_REASON_INVEPT] = handle_vmx_instruction, 6002 [EXIT_REASON_INVVPID] = handle_vmx_instruction, 6003 [EXIT_REASON_RDRAND] = kvm_handle_invalid_op, 6004 [EXIT_REASON_RDSEED] = kvm_handle_invalid_op, 6005 [EXIT_REASON_PML_FULL] = handle_pml_full, 6006 [EXIT_REASON_INVPCID] = handle_invpcid, 6007 [EXIT_REASON_VMFUNC] = handle_vmx_instruction, 6008 [EXIT_REASON_PREEMPTION_TIMER] = handle_preemption_timer, 6009 [EXIT_REASON_ENCLS] = handle_encls, 6010 [EXIT_REASON_BUS_LOCK] = handle_bus_lock_vmexit, 6011 [EXIT_REASON_NOTIFY] = handle_notify, 6012 }; 6013 6014 static const int kvm_vmx_max_exit_handlers = 6015 ARRAY_SIZE(kvm_vmx_exit_handlers); 6016 6017 static void vmx_get_exit_info(struct kvm_vcpu *vcpu, u32 *reason, 6018 u64 *info1, u64 *info2, 6019 u32 *intr_info, u32 *error_code) 6020 { 6021 struct vcpu_vmx *vmx = to_vmx(vcpu); 6022 6023 *reason = vmx->exit_reason.full; 6024 *info1 = vmx_get_exit_qual(vcpu); 6025 if (!(vmx->exit_reason.failed_vmentry)) { 6026 *info2 = vmx->idt_vectoring_info; 6027 *intr_info = vmx_get_intr_info(vcpu); 6028 if (is_exception_with_error_code(*intr_info)) 6029 *error_code = vmcs_read32(VM_EXIT_INTR_ERROR_CODE); 6030 else 6031 *error_code = 0; 6032 } else { 6033 *info2 = 0; 6034 *intr_info = 0; 6035 *error_code = 0; 6036 } 6037 } 6038 6039 static void vmx_destroy_pml_buffer(struct vcpu_vmx *vmx) 6040 { 6041 if (vmx->pml_pg) { 6042 __free_page(vmx->pml_pg); 6043 vmx->pml_pg = NULL; 6044 } 6045 } 6046 6047 static void vmx_flush_pml_buffer(struct kvm_vcpu *vcpu) 6048 { 6049 struct vcpu_vmx *vmx = to_vmx(vcpu); 6050 u64 *pml_buf; 6051 u16 pml_idx; 6052 6053 pml_idx = vmcs_read16(GUEST_PML_INDEX); 6054 6055 /* Do nothing if PML buffer is empty */ 6056 if (pml_idx == (PML_ENTITY_NUM - 1)) 6057 return; 6058 6059 /* PML index always points to next available PML buffer entity */ 6060 if (pml_idx >= PML_ENTITY_NUM) 6061 pml_idx = 0; 6062 else 6063 pml_idx++; 6064 6065 pml_buf = page_address(vmx->pml_pg); 6066 for (; pml_idx < PML_ENTITY_NUM; pml_idx++) { 6067 u64 gpa; 6068 6069 gpa = pml_buf[pml_idx]; 6070 WARN_ON(gpa & (PAGE_SIZE - 1)); 6071 kvm_vcpu_mark_page_dirty(vcpu, gpa >> PAGE_SHIFT); 6072 } 6073 6074 /* reset PML index */ 6075 vmcs_write16(GUEST_PML_INDEX, PML_ENTITY_NUM - 1); 6076 } 6077 6078 static void vmx_dump_sel(char *name, uint32_t sel) 6079 { 6080 pr_err("%s sel=0x%04x, attr=0x%05x, limit=0x%08x, base=0x%016lx\n", 6081 name, vmcs_read16(sel), 6082 vmcs_read32(sel + GUEST_ES_AR_BYTES - GUEST_ES_SELECTOR), 6083 vmcs_read32(sel + GUEST_ES_LIMIT - GUEST_ES_SELECTOR), 6084 vmcs_readl(sel + GUEST_ES_BASE - GUEST_ES_SELECTOR)); 6085 } 6086 6087 static void vmx_dump_dtsel(char *name, uint32_t limit) 6088 { 6089 pr_err("%s limit=0x%08x, base=0x%016lx\n", 6090 name, vmcs_read32(limit), 6091 vmcs_readl(limit + GUEST_GDTR_BASE - GUEST_GDTR_LIMIT)); 6092 } 6093 6094 static void vmx_dump_msrs(char *name, struct vmx_msrs *m) 6095 { 6096 unsigned int i; 6097 struct vmx_msr_entry *e; 6098 6099 pr_err("MSR %s:\n", name); 6100 for (i = 0, e = m->val; i < m->nr; ++i, ++e) 6101 pr_err(" %2d: msr=0x%08x value=0x%016llx\n", i, e->index, e->value); 6102 } 6103 6104 void dump_vmcs(struct kvm_vcpu *vcpu) 6105 { 6106 struct vcpu_vmx *vmx = to_vmx(vcpu); 6107 u32 vmentry_ctl, vmexit_ctl; 6108 u32 cpu_based_exec_ctrl, pin_based_exec_ctrl, secondary_exec_control; 6109 u64 tertiary_exec_control; 6110 unsigned long cr4; 6111 int efer_slot; 6112 6113 if (!dump_invalid_vmcs) { 6114 pr_warn_ratelimited("set kvm_intel.dump_invalid_vmcs=1 to dump internal KVM state.\n"); 6115 return; 6116 } 6117 6118 vmentry_ctl = vmcs_read32(VM_ENTRY_CONTROLS); 6119 vmexit_ctl = vmcs_read32(VM_EXIT_CONTROLS); 6120 cpu_based_exec_ctrl = vmcs_read32(CPU_BASED_VM_EXEC_CONTROL); 6121 pin_based_exec_ctrl = vmcs_read32(PIN_BASED_VM_EXEC_CONTROL); 6122 cr4 = vmcs_readl(GUEST_CR4); 6123 6124 if (cpu_has_secondary_exec_ctrls()) 6125 secondary_exec_control = vmcs_read32(SECONDARY_VM_EXEC_CONTROL); 6126 else 6127 secondary_exec_control = 0; 6128 6129 if (cpu_has_tertiary_exec_ctrls()) 6130 tertiary_exec_control = vmcs_read64(TERTIARY_VM_EXEC_CONTROL); 6131 else 6132 tertiary_exec_control = 0; 6133 6134 pr_err("VMCS %p, last attempted VM-entry on CPU %d\n", 6135 vmx->loaded_vmcs->vmcs, vcpu->arch.last_vmentry_cpu); 6136 pr_err("*** Guest State ***\n"); 6137 pr_err("CR0: actual=0x%016lx, shadow=0x%016lx, gh_mask=%016lx\n", 6138 vmcs_readl(GUEST_CR0), vmcs_readl(CR0_READ_SHADOW), 6139 vmcs_readl(CR0_GUEST_HOST_MASK)); 6140 pr_err("CR4: actual=0x%016lx, shadow=0x%016lx, gh_mask=%016lx\n", 6141 cr4, vmcs_readl(CR4_READ_SHADOW), vmcs_readl(CR4_GUEST_HOST_MASK)); 6142 pr_err("CR3 = 0x%016lx\n", vmcs_readl(GUEST_CR3)); 6143 if (cpu_has_vmx_ept()) { 6144 pr_err("PDPTR0 = 0x%016llx PDPTR1 = 0x%016llx\n", 6145 vmcs_read64(GUEST_PDPTR0), vmcs_read64(GUEST_PDPTR1)); 6146 pr_err("PDPTR2 = 0x%016llx PDPTR3 = 0x%016llx\n", 6147 vmcs_read64(GUEST_PDPTR2), vmcs_read64(GUEST_PDPTR3)); 6148 } 6149 pr_err("RSP = 0x%016lx RIP = 0x%016lx\n", 6150 vmcs_readl(GUEST_RSP), vmcs_readl(GUEST_RIP)); 6151 pr_err("RFLAGS=0x%08lx DR7 = 0x%016lx\n", 6152 vmcs_readl(GUEST_RFLAGS), vmcs_readl(GUEST_DR7)); 6153 pr_err("Sysenter RSP=%016lx CS:RIP=%04x:%016lx\n", 6154 vmcs_readl(GUEST_SYSENTER_ESP), 6155 vmcs_read32(GUEST_SYSENTER_CS), vmcs_readl(GUEST_SYSENTER_EIP)); 6156 vmx_dump_sel("CS: ", GUEST_CS_SELECTOR); 6157 vmx_dump_sel("DS: ", GUEST_DS_SELECTOR); 6158 vmx_dump_sel("SS: ", GUEST_SS_SELECTOR); 6159 vmx_dump_sel("ES: ", GUEST_ES_SELECTOR); 6160 vmx_dump_sel("FS: ", GUEST_FS_SELECTOR); 6161 vmx_dump_sel("GS: ", GUEST_GS_SELECTOR); 6162 vmx_dump_dtsel("GDTR:", GUEST_GDTR_LIMIT); 6163 vmx_dump_sel("LDTR:", GUEST_LDTR_SELECTOR); 6164 vmx_dump_dtsel("IDTR:", GUEST_IDTR_LIMIT); 6165 vmx_dump_sel("TR: ", GUEST_TR_SELECTOR); 6166 efer_slot = vmx_find_loadstore_msr_slot(&vmx->msr_autoload.guest, MSR_EFER); 6167 if (vmentry_ctl & VM_ENTRY_LOAD_IA32_EFER) 6168 pr_err("EFER= 0x%016llx\n", vmcs_read64(GUEST_IA32_EFER)); 6169 else if (efer_slot >= 0) 6170 pr_err("EFER= 0x%016llx (autoload)\n", 6171 vmx->msr_autoload.guest.val[efer_slot].value); 6172 else if (vmentry_ctl & VM_ENTRY_IA32E_MODE) 6173 pr_err("EFER= 0x%016llx (effective)\n", 6174 vcpu->arch.efer | (EFER_LMA | EFER_LME)); 6175 else 6176 pr_err("EFER= 0x%016llx (effective)\n", 6177 vcpu->arch.efer & ~(EFER_LMA | EFER_LME)); 6178 if (vmentry_ctl & VM_ENTRY_LOAD_IA32_PAT) 6179 pr_err("PAT = 0x%016llx\n", vmcs_read64(GUEST_IA32_PAT)); 6180 pr_err("DebugCtl = 0x%016llx DebugExceptions = 0x%016lx\n", 6181 vmcs_read64(GUEST_IA32_DEBUGCTL), 6182 vmcs_readl(GUEST_PENDING_DBG_EXCEPTIONS)); 6183 if (cpu_has_load_perf_global_ctrl() && 6184 vmentry_ctl & VM_ENTRY_LOAD_IA32_PERF_GLOBAL_CTRL) 6185 pr_err("PerfGlobCtl = 0x%016llx\n", 6186 vmcs_read64(GUEST_IA32_PERF_GLOBAL_CTRL)); 6187 if (vmentry_ctl & VM_ENTRY_LOAD_BNDCFGS) 6188 pr_err("BndCfgS = 0x%016llx\n", vmcs_read64(GUEST_BNDCFGS)); 6189 pr_err("Interruptibility = %08x ActivityState = %08x\n", 6190 vmcs_read32(GUEST_INTERRUPTIBILITY_INFO), 6191 vmcs_read32(GUEST_ACTIVITY_STATE)); 6192 if (secondary_exec_control & SECONDARY_EXEC_VIRTUAL_INTR_DELIVERY) 6193 pr_err("InterruptStatus = %04x\n", 6194 vmcs_read16(GUEST_INTR_STATUS)); 6195 if (vmcs_read32(VM_ENTRY_MSR_LOAD_COUNT) > 0) 6196 vmx_dump_msrs("guest autoload", &vmx->msr_autoload.guest); 6197 if (vmcs_read32(VM_EXIT_MSR_STORE_COUNT) > 0) 6198 vmx_dump_msrs("guest autostore", &vmx->msr_autostore.guest); 6199 6200 pr_err("*** Host State ***\n"); 6201 pr_err("RIP = 0x%016lx RSP = 0x%016lx\n", 6202 vmcs_readl(HOST_RIP), vmcs_readl(HOST_RSP)); 6203 pr_err("CS=%04x SS=%04x DS=%04x ES=%04x FS=%04x GS=%04x TR=%04x\n", 6204 vmcs_read16(HOST_CS_SELECTOR), vmcs_read16(HOST_SS_SELECTOR), 6205 vmcs_read16(HOST_DS_SELECTOR), vmcs_read16(HOST_ES_SELECTOR), 6206 vmcs_read16(HOST_FS_SELECTOR), vmcs_read16(HOST_GS_SELECTOR), 6207 vmcs_read16(HOST_TR_SELECTOR)); 6208 pr_err("FSBase=%016lx GSBase=%016lx TRBase=%016lx\n", 6209 vmcs_readl(HOST_FS_BASE), vmcs_readl(HOST_GS_BASE), 6210 vmcs_readl(HOST_TR_BASE)); 6211 pr_err("GDTBase=%016lx IDTBase=%016lx\n", 6212 vmcs_readl(HOST_GDTR_BASE), vmcs_readl(HOST_IDTR_BASE)); 6213 pr_err("CR0=%016lx CR3=%016lx CR4=%016lx\n", 6214 vmcs_readl(HOST_CR0), vmcs_readl(HOST_CR3), 6215 vmcs_readl(HOST_CR4)); 6216 pr_err("Sysenter RSP=%016lx CS:RIP=%04x:%016lx\n", 6217 vmcs_readl(HOST_IA32_SYSENTER_ESP), 6218 vmcs_read32(HOST_IA32_SYSENTER_CS), 6219 vmcs_readl(HOST_IA32_SYSENTER_EIP)); 6220 if (vmexit_ctl & VM_EXIT_LOAD_IA32_EFER) 6221 pr_err("EFER= 0x%016llx\n", vmcs_read64(HOST_IA32_EFER)); 6222 if (vmexit_ctl & VM_EXIT_LOAD_IA32_PAT) 6223 pr_err("PAT = 0x%016llx\n", vmcs_read64(HOST_IA32_PAT)); 6224 if (cpu_has_load_perf_global_ctrl() && 6225 vmexit_ctl & VM_EXIT_LOAD_IA32_PERF_GLOBAL_CTRL) 6226 pr_err("PerfGlobCtl = 0x%016llx\n", 6227 vmcs_read64(HOST_IA32_PERF_GLOBAL_CTRL)); 6228 if (vmcs_read32(VM_EXIT_MSR_LOAD_COUNT) > 0) 6229 vmx_dump_msrs("host autoload", &vmx->msr_autoload.host); 6230 6231 pr_err("*** Control State ***\n"); 6232 pr_err("CPUBased=0x%08x SecondaryExec=0x%08x TertiaryExec=0x%016llx\n", 6233 cpu_based_exec_ctrl, secondary_exec_control, tertiary_exec_control); 6234 pr_err("PinBased=0x%08x EntryControls=%08x ExitControls=%08x\n", 6235 pin_based_exec_ctrl, vmentry_ctl, vmexit_ctl); 6236 pr_err("ExceptionBitmap=%08x PFECmask=%08x PFECmatch=%08x\n", 6237 vmcs_read32(EXCEPTION_BITMAP), 6238 vmcs_read32(PAGE_FAULT_ERROR_CODE_MASK), 6239 vmcs_read32(PAGE_FAULT_ERROR_CODE_MATCH)); 6240 pr_err("VMEntry: intr_info=%08x errcode=%08x ilen=%08x\n", 6241 vmcs_read32(VM_ENTRY_INTR_INFO_FIELD), 6242 vmcs_read32(VM_ENTRY_EXCEPTION_ERROR_CODE), 6243 vmcs_read32(VM_ENTRY_INSTRUCTION_LEN)); 6244 pr_err("VMExit: intr_info=%08x errcode=%08x ilen=%08x\n", 6245 vmcs_read32(VM_EXIT_INTR_INFO), 6246 vmcs_read32(VM_EXIT_INTR_ERROR_CODE), 6247 vmcs_read32(VM_EXIT_INSTRUCTION_LEN)); 6248 pr_err(" reason=%08x qualification=%016lx\n", 6249 vmcs_read32(VM_EXIT_REASON), vmcs_readl(EXIT_QUALIFICATION)); 6250 pr_err("IDTVectoring: info=%08x errcode=%08x\n", 6251 vmcs_read32(IDT_VECTORING_INFO_FIELD), 6252 vmcs_read32(IDT_VECTORING_ERROR_CODE)); 6253 pr_err("TSC Offset = 0x%016llx\n", vmcs_read64(TSC_OFFSET)); 6254 if (secondary_exec_control & SECONDARY_EXEC_TSC_SCALING) 6255 pr_err("TSC Multiplier = 0x%016llx\n", 6256 vmcs_read64(TSC_MULTIPLIER)); 6257 if (cpu_based_exec_ctrl & CPU_BASED_TPR_SHADOW) { 6258 if (secondary_exec_control & SECONDARY_EXEC_VIRTUAL_INTR_DELIVERY) { 6259 u16 status = vmcs_read16(GUEST_INTR_STATUS); 6260 pr_err("SVI|RVI = %02x|%02x ", status >> 8, status & 0xff); 6261 } 6262 pr_cont("TPR Threshold = 0x%02x\n", vmcs_read32(TPR_THRESHOLD)); 6263 if (secondary_exec_control & SECONDARY_EXEC_VIRTUALIZE_APIC_ACCESSES) 6264 pr_err("APIC-access addr = 0x%016llx ", vmcs_read64(APIC_ACCESS_ADDR)); 6265 pr_cont("virt-APIC addr = 0x%016llx\n", vmcs_read64(VIRTUAL_APIC_PAGE_ADDR)); 6266 } 6267 if (pin_based_exec_ctrl & PIN_BASED_POSTED_INTR) 6268 pr_err("PostedIntrVec = 0x%02x\n", vmcs_read16(POSTED_INTR_NV)); 6269 if ((secondary_exec_control & SECONDARY_EXEC_ENABLE_EPT)) 6270 pr_err("EPT pointer = 0x%016llx\n", vmcs_read64(EPT_POINTER)); 6271 if (secondary_exec_control & SECONDARY_EXEC_PAUSE_LOOP_EXITING) 6272 pr_err("PLE Gap=%08x Window=%08x\n", 6273 vmcs_read32(PLE_GAP), vmcs_read32(PLE_WINDOW)); 6274 if (secondary_exec_control & SECONDARY_EXEC_ENABLE_VPID) 6275 pr_err("Virtual processor ID = 0x%04x\n", 6276 vmcs_read16(VIRTUAL_PROCESSOR_ID)); 6277 } 6278 6279 /* 6280 * The guest has exited. See if we can fix it or if we need userspace 6281 * assistance. 6282 */ 6283 static int __vmx_handle_exit(struct kvm_vcpu *vcpu, fastpath_t exit_fastpath) 6284 { 6285 struct vcpu_vmx *vmx = to_vmx(vcpu); 6286 union vmx_exit_reason exit_reason = vmx->exit_reason; 6287 u32 vectoring_info = vmx->idt_vectoring_info; 6288 u16 exit_handler_index; 6289 6290 /* 6291 * Flush logged GPAs PML buffer, this will make dirty_bitmap more 6292 * updated. Another good is, in kvm_vm_ioctl_get_dirty_log, before 6293 * querying dirty_bitmap, we only need to kick all vcpus out of guest 6294 * mode as if vcpus is in root mode, the PML buffer must has been 6295 * flushed already. Note, PML is never enabled in hardware while 6296 * running L2. 6297 */ 6298 if (enable_pml && !is_guest_mode(vcpu)) 6299 vmx_flush_pml_buffer(vcpu); 6300 6301 /* 6302 * KVM should never reach this point with a pending nested VM-Enter. 6303 * More specifically, short-circuiting VM-Entry to emulate L2 due to 6304 * invalid guest state should never happen as that means KVM knowingly 6305 * allowed a nested VM-Enter with an invalid vmcs12. More below. 6306 */ 6307 if (KVM_BUG_ON(vmx->nested.nested_run_pending, vcpu->kvm)) 6308 return -EIO; 6309 6310 if (is_guest_mode(vcpu)) { 6311 /* 6312 * PML is never enabled when running L2, bail immediately if a 6313 * PML full exit occurs as something is horribly wrong. 6314 */ 6315 if (exit_reason.basic == EXIT_REASON_PML_FULL) 6316 goto unexpected_vmexit; 6317 6318 /* 6319 * The host physical addresses of some pages of guest memory 6320 * are loaded into the vmcs02 (e.g. vmcs12's Virtual APIC 6321 * Page). The CPU may write to these pages via their host 6322 * physical address while L2 is running, bypassing any 6323 * address-translation-based dirty tracking (e.g. EPT write 6324 * protection). 6325 * 6326 * Mark them dirty on every exit from L2 to prevent them from 6327 * getting out of sync with dirty tracking. 6328 */ 6329 nested_mark_vmcs12_pages_dirty(vcpu); 6330 6331 /* 6332 * Synthesize a triple fault if L2 state is invalid. In normal 6333 * operation, nested VM-Enter rejects any attempt to enter L2 6334 * with invalid state. However, those checks are skipped if 6335 * state is being stuffed via RSM or KVM_SET_NESTED_STATE. If 6336 * L2 state is invalid, it means either L1 modified SMRAM state 6337 * or userspace provided bad state. Synthesize TRIPLE_FAULT as 6338 * doing so is architecturally allowed in the RSM case, and is 6339 * the least awful solution for the userspace case without 6340 * risking false positives. 6341 */ 6342 if (vmx->emulation_required) { 6343 nested_vmx_vmexit(vcpu, EXIT_REASON_TRIPLE_FAULT, 0, 0); 6344 return 1; 6345 } 6346 6347 if (nested_vmx_reflect_vmexit(vcpu)) 6348 return 1; 6349 } 6350 6351 /* If guest state is invalid, start emulating. L2 is handled above. */ 6352 if (vmx->emulation_required) 6353 return handle_invalid_guest_state(vcpu); 6354 6355 if (exit_reason.failed_vmentry) { 6356 dump_vmcs(vcpu); 6357 vcpu->run->exit_reason = KVM_EXIT_FAIL_ENTRY; 6358 vcpu->run->fail_entry.hardware_entry_failure_reason 6359 = exit_reason.full; 6360 vcpu->run->fail_entry.cpu = vcpu->arch.last_vmentry_cpu; 6361 return 0; 6362 } 6363 6364 if (unlikely(vmx->fail)) { 6365 dump_vmcs(vcpu); 6366 vcpu->run->exit_reason = KVM_EXIT_FAIL_ENTRY; 6367 vcpu->run->fail_entry.hardware_entry_failure_reason 6368 = vmcs_read32(VM_INSTRUCTION_ERROR); 6369 vcpu->run->fail_entry.cpu = vcpu->arch.last_vmentry_cpu; 6370 return 0; 6371 } 6372 6373 /* 6374 * Note: 6375 * Do not try to fix EXIT_REASON_EPT_MISCONFIG if it caused by 6376 * delivery event since it indicates guest is accessing MMIO. 6377 * The vm-exit can be triggered again after return to guest that 6378 * will cause infinite loop. 6379 */ 6380 if ((vectoring_info & VECTORING_INFO_VALID_MASK) && 6381 (exit_reason.basic != EXIT_REASON_EXCEPTION_NMI && 6382 exit_reason.basic != EXIT_REASON_EPT_VIOLATION && 6383 exit_reason.basic != EXIT_REASON_PML_FULL && 6384 exit_reason.basic != EXIT_REASON_APIC_ACCESS && 6385 exit_reason.basic != EXIT_REASON_TASK_SWITCH && 6386 exit_reason.basic != EXIT_REASON_NOTIFY)) { 6387 int ndata = 3; 6388 6389 vcpu->run->exit_reason = KVM_EXIT_INTERNAL_ERROR; 6390 vcpu->run->internal.suberror = KVM_INTERNAL_ERROR_DELIVERY_EV; 6391 vcpu->run->internal.data[0] = vectoring_info; 6392 vcpu->run->internal.data[1] = exit_reason.full; 6393 vcpu->run->internal.data[2] = vcpu->arch.exit_qualification; 6394 if (exit_reason.basic == EXIT_REASON_EPT_MISCONFIG) { 6395 vcpu->run->internal.data[ndata++] = 6396 vmcs_read64(GUEST_PHYSICAL_ADDRESS); 6397 } 6398 vcpu->run->internal.data[ndata++] = vcpu->arch.last_vmentry_cpu; 6399 vcpu->run->internal.ndata = ndata; 6400 return 0; 6401 } 6402 6403 if (unlikely(!enable_vnmi && 6404 vmx->loaded_vmcs->soft_vnmi_blocked)) { 6405 if (!vmx_interrupt_blocked(vcpu)) { 6406 vmx->loaded_vmcs->soft_vnmi_blocked = 0; 6407 } else if (vmx->loaded_vmcs->vnmi_blocked_time > 1000000000LL && 6408 vcpu->arch.nmi_pending) { 6409 /* 6410 * This CPU don't support us in finding the end of an 6411 * NMI-blocked window if the guest runs with IRQs 6412 * disabled. So we pull the trigger after 1 s of 6413 * futile waiting, but inform the user about this. 6414 */ 6415 printk(KERN_WARNING "%s: Breaking out of NMI-blocked " 6416 "state on VCPU %d after 1 s timeout\n", 6417 __func__, vcpu->vcpu_id); 6418 vmx->loaded_vmcs->soft_vnmi_blocked = 0; 6419 } 6420 } 6421 6422 if (exit_fastpath != EXIT_FASTPATH_NONE) 6423 return 1; 6424 6425 if (exit_reason.basic >= kvm_vmx_max_exit_handlers) 6426 goto unexpected_vmexit; 6427 #ifdef CONFIG_RETPOLINE 6428 if (exit_reason.basic == EXIT_REASON_MSR_WRITE) 6429 return kvm_emulate_wrmsr(vcpu); 6430 else if (exit_reason.basic == EXIT_REASON_PREEMPTION_TIMER) 6431 return handle_preemption_timer(vcpu); 6432 else if (exit_reason.basic == EXIT_REASON_INTERRUPT_WINDOW) 6433 return handle_interrupt_window(vcpu); 6434 else if (exit_reason.basic == EXIT_REASON_EXTERNAL_INTERRUPT) 6435 return handle_external_interrupt(vcpu); 6436 else if (exit_reason.basic == EXIT_REASON_HLT) 6437 return kvm_emulate_halt(vcpu); 6438 else if (exit_reason.basic == EXIT_REASON_EPT_MISCONFIG) 6439 return handle_ept_misconfig(vcpu); 6440 #endif 6441 6442 exit_handler_index = array_index_nospec((u16)exit_reason.basic, 6443 kvm_vmx_max_exit_handlers); 6444 if (!kvm_vmx_exit_handlers[exit_handler_index]) 6445 goto unexpected_vmexit; 6446 6447 return kvm_vmx_exit_handlers[exit_handler_index](vcpu); 6448 6449 unexpected_vmexit: 6450 vcpu_unimpl(vcpu, "vmx: unexpected exit reason 0x%x\n", 6451 exit_reason.full); 6452 dump_vmcs(vcpu); 6453 vcpu->run->exit_reason = KVM_EXIT_INTERNAL_ERROR; 6454 vcpu->run->internal.suberror = 6455 KVM_INTERNAL_ERROR_UNEXPECTED_EXIT_REASON; 6456 vcpu->run->internal.ndata = 2; 6457 vcpu->run->internal.data[0] = exit_reason.full; 6458 vcpu->run->internal.data[1] = vcpu->arch.last_vmentry_cpu; 6459 return 0; 6460 } 6461 6462 static int vmx_handle_exit(struct kvm_vcpu *vcpu, fastpath_t exit_fastpath) 6463 { 6464 int ret = __vmx_handle_exit(vcpu, exit_fastpath); 6465 6466 /* 6467 * Exit to user space when bus lock detected to inform that there is 6468 * a bus lock in guest. 6469 */ 6470 if (to_vmx(vcpu)->exit_reason.bus_lock_detected) { 6471 if (ret > 0) 6472 vcpu->run->exit_reason = KVM_EXIT_X86_BUS_LOCK; 6473 6474 vcpu->run->flags |= KVM_RUN_X86_BUS_LOCK; 6475 return 0; 6476 } 6477 return ret; 6478 } 6479 6480 /* 6481 * Software based L1D cache flush which is used when microcode providing 6482 * the cache control MSR is not loaded. 6483 * 6484 * The L1D cache is 32 KiB on Nehalem and later microarchitectures, but to 6485 * flush it is required to read in 64 KiB because the replacement algorithm 6486 * is not exactly LRU. This could be sized at runtime via topology 6487 * information but as all relevant affected CPUs have 32KiB L1D cache size 6488 * there is no point in doing so. 6489 */ 6490 static noinstr void vmx_l1d_flush(struct kvm_vcpu *vcpu) 6491 { 6492 int size = PAGE_SIZE << L1D_CACHE_ORDER; 6493 6494 /* 6495 * This code is only executed when the flush mode is 'cond' or 6496 * 'always' 6497 */ 6498 if (static_branch_likely(&vmx_l1d_flush_cond)) { 6499 bool flush_l1d; 6500 6501 /* 6502 * Clear the per-vcpu flush bit, it gets set again 6503 * either from vcpu_run() or from one of the unsafe 6504 * VMEXIT handlers. 6505 */ 6506 flush_l1d = vcpu->arch.l1tf_flush_l1d; 6507 vcpu->arch.l1tf_flush_l1d = false; 6508 6509 /* 6510 * Clear the per-cpu flush bit, it gets set again from 6511 * the interrupt handlers. 6512 */ 6513 flush_l1d |= kvm_get_cpu_l1tf_flush_l1d(); 6514 kvm_clear_cpu_l1tf_flush_l1d(); 6515 6516 if (!flush_l1d) 6517 return; 6518 } 6519 6520 vcpu->stat.l1d_flush++; 6521 6522 if (static_cpu_has(X86_FEATURE_FLUSH_L1D)) { 6523 native_wrmsrl(MSR_IA32_FLUSH_CMD, L1D_FLUSH); 6524 return; 6525 } 6526 6527 asm volatile( 6528 /* First ensure the pages are in the TLB */ 6529 "xorl %%eax, %%eax\n" 6530 ".Lpopulate_tlb:\n\t" 6531 "movzbl (%[flush_pages], %%" _ASM_AX "), %%ecx\n\t" 6532 "addl $4096, %%eax\n\t" 6533 "cmpl %%eax, %[size]\n\t" 6534 "jne .Lpopulate_tlb\n\t" 6535 "xorl %%eax, %%eax\n\t" 6536 "cpuid\n\t" 6537 /* Now fill the cache */ 6538 "xorl %%eax, %%eax\n" 6539 ".Lfill_cache:\n" 6540 "movzbl (%[flush_pages], %%" _ASM_AX "), %%ecx\n\t" 6541 "addl $64, %%eax\n\t" 6542 "cmpl %%eax, %[size]\n\t" 6543 "jne .Lfill_cache\n\t" 6544 "lfence\n" 6545 :: [flush_pages] "r" (vmx_l1d_flush_pages), 6546 [size] "r" (size) 6547 : "eax", "ebx", "ecx", "edx"); 6548 } 6549 6550 static void vmx_update_cr8_intercept(struct kvm_vcpu *vcpu, int tpr, int irr) 6551 { 6552 struct vmcs12 *vmcs12 = get_vmcs12(vcpu); 6553 int tpr_threshold; 6554 6555 if (is_guest_mode(vcpu) && 6556 nested_cpu_has(vmcs12, CPU_BASED_TPR_SHADOW)) 6557 return; 6558 6559 tpr_threshold = (irr == -1 || tpr < irr) ? 0 : irr; 6560 if (is_guest_mode(vcpu)) 6561 to_vmx(vcpu)->nested.l1_tpr_threshold = tpr_threshold; 6562 else 6563 vmcs_write32(TPR_THRESHOLD, tpr_threshold); 6564 } 6565 6566 void vmx_set_virtual_apic_mode(struct kvm_vcpu *vcpu) 6567 { 6568 struct vcpu_vmx *vmx = to_vmx(vcpu); 6569 u32 sec_exec_control; 6570 6571 if (!lapic_in_kernel(vcpu)) 6572 return; 6573 6574 if (!flexpriority_enabled && 6575 !cpu_has_vmx_virtualize_x2apic_mode()) 6576 return; 6577 6578 /* Postpone execution until vmcs01 is the current VMCS. */ 6579 if (is_guest_mode(vcpu)) { 6580 vmx->nested.change_vmcs01_virtual_apic_mode = true; 6581 return; 6582 } 6583 6584 sec_exec_control = secondary_exec_controls_get(vmx); 6585 sec_exec_control &= ~(SECONDARY_EXEC_VIRTUALIZE_APIC_ACCESSES | 6586 SECONDARY_EXEC_VIRTUALIZE_X2APIC_MODE); 6587 6588 switch (kvm_get_apic_mode(vcpu)) { 6589 case LAPIC_MODE_INVALID: 6590 WARN_ONCE(true, "Invalid local APIC state"); 6591 break; 6592 case LAPIC_MODE_DISABLED: 6593 break; 6594 case LAPIC_MODE_XAPIC: 6595 if (flexpriority_enabled) { 6596 sec_exec_control |= 6597 SECONDARY_EXEC_VIRTUALIZE_APIC_ACCESSES; 6598 kvm_make_request(KVM_REQ_APIC_PAGE_RELOAD, vcpu); 6599 6600 /* 6601 * Flush the TLB, reloading the APIC access page will 6602 * only do so if its physical address has changed, but 6603 * the guest may have inserted a non-APIC mapping into 6604 * the TLB while the APIC access page was disabled. 6605 */ 6606 kvm_make_request(KVM_REQ_TLB_FLUSH_CURRENT, vcpu); 6607 } 6608 break; 6609 case LAPIC_MODE_X2APIC: 6610 if (cpu_has_vmx_virtualize_x2apic_mode()) 6611 sec_exec_control |= 6612 SECONDARY_EXEC_VIRTUALIZE_X2APIC_MODE; 6613 break; 6614 } 6615 secondary_exec_controls_set(vmx, sec_exec_control); 6616 6617 vmx_update_msr_bitmap_x2apic(vcpu); 6618 } 6619 6620 static void vmx_set_apic_access_page_addr(struct kvm_vcpu *vcpu) 6621 { 6622 struct page *page; 6623 6624 /* Defer reload until vmcs01 is the current VMCS. */ 6625 if (is_guest_mode(vcpu)) { 6626 to_vmx(vcpu)->nested.reload_vmcs01_apic_access_page = true; 6627 return; 6628 } 6629 6630 if (!(secondary_exec_controls_get(to_vmx(vcpu)) & 6631 SECONDARY_EXEC_VIRTUALIZE_APIC_ACCESSES)) 6632 return; 6633 6634 page = gfn_to_page(vcpu->kvm, APIC_DEFAULT_PHYS_BASE >> PAGE_SHIFT); 6635 if (is_error_page(page)) 6636 return; 6637 6638 vmcs_write64(APIC_ACCESS_ADDR, page_to_phys(page)); 6639 vmx_flush_tlb_current(vcpu); 6640 6641 /* 6642 * Do not pin apic access page in memory, the MMU notifier 6643 * will call us again if it is migrated or swapped out. 6644 */ 6645 put_page(page); 6646 } 6647 6648 static void vmx_hwapic_isr_update(int max_isr) 6649 { 6650 u16 status; 6651 u8 old; 6652 6653 if (max_isr == -1) 6654 max_isr = 0; 6655 6656 status = vmcs_read16(GUEST_INTR_STATUS); 6657 old = status >> 8; 6658 if (max_isr != old) { 6659 status &= 0xff; 6660 status |= max_isr << 8; 6661 vmcs_write16(GUEST_INTR_STATUS, status); 6662 } 6663 } 6664 6665 static void vmx_set_rvi(int vector) 6666 { 6667 u16 status; 6668 u8 old; 6669 6670 if (vector == -1) 6671 vector = 0; 6672 6673 status = vmcs_read16(GUEST_INTR_STATUS); 6674 old = (u8)status & 0xff; 6675 if ((u8)vector != old) { 6676 status &= ~0xff; 6677 status |= (u8)vector; 6678 vmcs_write16(GUEST_INTR_STATUS, status); 6679 } 6680 } 6681 6682 static void vmx_hwapic_irr_update(struct kvm_vcpu *vcpu, int max_irr) 6683 { 6684 /* 6685 * When running L2, updating RVI is only relevant when 6686 * vmcs12 virtual-interrupt-delivery enabled. 6687 * However, it can be enabled only when L1 also 6688 * intercepts external-interrupts and in that case 6689 * we should not update vmcs02 RVI but instead intercept 6690 * interrupt. Therefore, do nothing when running L2. 6691 */ 6692 if (!is_guest_mode(vcpu)) 6693 vmx_set_rvi(max_irr); 6694 } 6695 6696 static int vmx_sync_pir_to_irr(struct kvm_vcpu *vcpu) 6697 { 6698 struct vcpu_vmx *vmx = to_vmx(vcpu); 6699 int max_irr; 6700 bool got_posted_interrupt; 6701 6702 if (KVM_BUG_ON(!enable_apicv, vcpu->kvm)) 6703 return -EIO; 6704 6705 if (pi_test_on(&vmx->pi_desc)) { 6706 pi_clear_on(&vmx->pi_desc); 6707 /* 6708 * IOMMU can write to PID.ON, so the barrier matters even on UP. 6709 * But on x86 this is just a compiler barrier anyway. 6710 */ 6711 smp_mb__after_atomic(); 6712 got_posted_interrupt = 6713 kvm_apic_update_irr(vcpu, vmx->pi_desc.pir, &max_irr); 6714 } else { 6715 max_irr = kvm_lapic_find_highest_irr(vcpu); 6716 got_posted_interrupt = false; 6717 } 6718 6719 /* 6720 * Newly recognized interrupts are injected via either virtual interrupt 6721 * delivery (RVI) or KVM_REQ_EVENT. Virtual interrupt delivery is 6722 * disabled in two cases: 6723 * 6724 * 1) If L2 is running and the vCPU has a new pending interrupt. If L1 6725 * wants to exit on interrupts, KVM_REQ_EVENT is needed to synthesize a 6726 * VM-Exit to L1. If L1 doesn't want to exit, the interrupt is injected 6727 * into L2, but KVM doesn't use virtual interrupt delivery to inject 6728 * interrupts into L2, and so KVM_REQ_EVENT is again needed. 6729 * 6730 * 2) If APICv is disabled for this vCPU, assigned devices may still 6731 * attempt to post interrupts. The posted interrupt vector will cause 6732 * a VM-Exit and the subsequent entry will call sync_pir_to_irr. 6733 */ 6734 if (!is_guest_mode(vcpu) && kvm_vcpu_apicv_active(vcpu)) 6735 vmx_set_rvi(max_irr); 6736 else if (got_posted_interrupt) 6737 kvm_make_request(KVM_REQ_EVENT, vcpu); 6738 6739 return max_irr; 6740 } 6741 6742 static void vmx_load_eoi_exitmap(struct kvm_vcpu *vcpu, u64 *eoi_exit_bitmap) 6743 { 6744 if (!kvm_vcpu_apicv_active(vcpu)) 6745 return; 6746 6747 vmcs_write64(EOI_EXIT_BITMAP0, eoi_exit_bitmap[0]); 6748 vmcs_write64(EOI_EXIT_BITMAP1, eoi_exit_bitmap[1]); 6749 vmcs_write64(EOI_EXIT_BITMAP2, eoi_exit_bitmap[2]); 6750 vmcs_write64(EOI_EXIT_BITMAP3, eoi_exit_bitmap[3]); 6751 } 6752 6753 static void vmx_apicv_post_state_restore(struct kvm_vcpu *vcpu) 6754 { 6755 struct vcpu_vmx *vmx = to_vmx(vcpu); 6756 6757 pi_clear_on(&vmx->pi_desc); 6758 memset(vmx->pi_desc.pir, 0, sizeof(vmx->pi_desc.pir)); 6759 } 6760 6761 void vmx_do_interrupt_nmi_irqoff(unsigned long entry); 6762 6763 static void handle_interrupt_nmi_irqoff(struct kvm_vcpu *vcpu, 6764 unsigned long entry) 6765 { 6766 bool is_nmi = entry == (unsigned long)asm_exc_nmi_noist; 6767 6768 kvm_before_interrupt(vcpu, is_nmi ? KVM_HANDLING_NMI : KVM_HANDLING_IRQ); 6769 vmx_do_interrupt_nmi_irqoff(entry); 6770 kvm_after_interrupt(vcpu); 6771 } 6772 6773 static void handle_nm_fault_irqoff(struct kvm_vcpu *vcpu) 6774 { 6775 /* 6776 * Save xfd_err to guest_fpu before interrupt is enabled, so the 6777 * MSR value is not clobbered by the host activity before the guest 6778 * has chance to consume it. 6779 * 6780 * Do not blindly read xfd_err here, since this exception might 6781 * be caused by L1 interception on a platform which doesn't 6782 * support xfd at all. 6783 * 6784 * Do it conditionally upon guest_fpu::xfd. xfd_err matters 6785 * only when xfd contains a non-zero value. 6786 * 6787 * Queuing exception is done in vmx_handle_exit. See comment there. 6788 */ 6789 if (vcpu->arch.guest_fpu.fpstate->xfd) 6790 rdmsrl(MSR_IA32_XFD_ERR, vcpu->arch.guest_fpu.xfd_err); 6791 } 6792 6793 static void handle_exception_nmi_irqoff(struct vcpu_vmx *vmx) 6794 { 6795 const unsigned long nmi_entry = (unsigned long)asm_exc_nmi_noist; 6796 u32 intr_info = vmx_get_intr_info(&vmx->vcpu); 6797 6798 /* if exit due to PF check for async PF */ 6799 if (is_page_fault(intr_info)) 6800 vmx->vcpu.arch.apf.host_apf_flags = kvm_read_and_reset_apf_flags(); 6801 /* if exit due to NM, handle before interrupts are enabled */ 6802 else if (is_nm_fault(intr_info)) 6803 handle_nm_fault_irqoff(&vmx->vcpu); 6804 /* Handle machine checks before interrupts are enabled */ 6805 else if (is_machine_check(intr_info)) 6806 kvm_machine_check(); 6807 /* We need to handle NMIs before interrupts are enabled */ 6808 else if (is_nmi(intr_info)) 6809 handle_interrupt_nmi_irqoff(&vmx->vcpu, nmi_entry); 6810 } 6811 6812 static void handle_external_interrupt_irqoff(struct kvm_vcpu *vcpu) 6813 { 6814 u32 intr_info = vmx_get_intr_info(vcpu); 6815 unsigned int vector = intr_info & INTR_INFO_VECTOR_MASK; 6816 gate_desc *desc = (gate_desc *)host_idt_base + vector; 6817 6818 if (KVM_BUG(!is_external_intr(intr_info), vcpu->kvm, 6819 "KVM: unexpected VM-Exit interrupt info: 0x%x", intr_info)) 6820 return; 6821 6822 handle_interrupt_nmi_irqoff(vcpu, gate_offset(desc)); 6823 vcpu->arch.at_instruction_boundary = true; 6824 } 6825 6826 static void vmx_handle_exit_irqoff(struct kvm_vcpu *vcpu) 6827 { 6828 struct vcpu_vmx *vmx = to_vmx(vcpu); 6829 6830 if (vmx->emulation_required) 6831 return; 6832 6833 if (vmx->exit_reason.basic == EXIT_REASON_EXTERNAL_INTERRUPT) 6834 handle_external_interrupt_irqoff(vcpu); 6835 else if (vmx->exit_reason.basic == EXIT_REASON_EXCEPTION_NMI) 6836 handle_exception_nmi_irqoff(vmx); 6837 } 6838 6839 /* 6840 * The kvm parameter can be NULL (module initialization, or invocation before 6841 * VM creation). Be sure to check the kvm parameter before using it. 6842 */ 6843 static bool vmx_has_emulated_msr(struct kvm *kvm, u32 index) 6844 { 6845 switch (index) { 6846 case MSR_IA32_SMBASE: 6847 /* 6848 * We cannot do SMM unless we can run the guest in big 6849 * real mode. 6850 */ 6851 return enable_unrestricted_guest || emulate_invalid_guest_state; 6852 case MSR_IA32_VMX_BASIC ... MSR_IA32_VMX_VMFUNC: 6853 return nested; 6854 case MSR_AMD64_VIRT_SPEC_CTRL: 6855 case MSR_AMD64_TSC_RATIO: 6856 /* This is AMD only. */ 6857 return false; 6858 default: 6859 return true; 6860 } 6861 } 6862 6863 static void vmx_recover_nmi_blocking(struct vcpu_vmx *vmx) 6864 { 6865 u32 exit_intr_info; 6866 bool unblock_nmi; 6867 u8 vector; 6868 bool idtv_info_valid; 6869 6870 idtv_info_valid = vmx->idt_vectoring_info & VECTORING_INFO_VALID_MASK; 6871 6872 if (enable_vnmi) { 6873 if (vmx->loaded_vmcs->nmi_known_unmasked) 6874 return; 6875 6876 exit_intr_info = vmx_get_intr_info(&vmx->vcpu); 6877 unblock_nmi = (exit_intr_info & INTR_INFO_UNBLOCK_NMI) != 0; 6878 vector = exit_intr_info & INTR_INFO_VECTOR_MASK; 6879 /* 6880 * SDM 3: 27.7.1.2 (September 2008) 6881 * Re-set bit "block by NMI" before VM entry if vmexit caused by 6882 * a guest IRET fault. 6883 * SDM 3: 23.2.2 (September 2008) 6884 * Bit 12 is undefined in any of the following cases: 6885 * If the VM exit sets the valid bit in the IDT-vectoring 6886 * information field. 6887 * If the VM exit is due to a double fault. 6888 */ 6889 if ((exit_intr_info & INTR_INFO_VALID_MASK) && unblock_nmi && 6890 vector != DF_VECTOR && !idtv_info_valid) 6891 vmcs_set_bits(GUEST_INTERRUPTIBILITY_INFO, 6892 GUEST_INTR_STATE_NMI); 6893 else 6894 vmx->loaded_vmcs->nmi_known_unmasked = 6895 !(vmcs_read32(GUEST_INTERRUPTIBILITY_INFO) 6896 & GUEST_INTR_STATE_NMI); 6897 } else if (unlikely(vmx->loaded_vmcs->soft_vnmi_blocked)) 6898 vmx->loaded_vmcs->vnmi_blocked_time += 6899 ktime_to_ns(ktime_sub(ktime_get(), 6900 vmx->loaded_vmcs->entry_time)); 6901 } 6902 6903 static void __vmx_complete_interrupts(struct kvm_vcpu *vcpu, 6904 u32 idt_vectoring_info, 6905 int instr_len_field, 6906 int error_code_field) 6907 { 6908 u8 vector; 6909 int type; 6910 bool idtv_info_valid; 6911 6912 idtv_info_valid = idt_vectoring_info & VECTORING_INFO_VALID_MASK; 6913 6914 vcpu->arch.nmi_injected = false; 6915 kvm_clear_exception_queue(vcpu); 6916 kvm_clear_interrupt_queue(vcpu); 6917 6918 if (!idtv_info_valid) 6919 return; 6920 6921 kvm_make_request(KVM_REQ_EVENT, vcpu); 6922 6923 vector = idt_vectoring_info & VECTORING_INFO_VECTOR_MASK; 6924 type = idt_vectoring_info & VECTORING_INFO_TYPE_MASK; 6925 6926 switch (type) { 6927 case INTR_TYPE_NMI_INTR: 6928 vcpu->arch.nmi_injected = true; 6929 /* 6930 * SDM 3: 27.7.1.2 (September 2008) 6931 * Clear bit "block by NMI" before VM entry if a NMI 6932 * delivery faulted. 6933 */ 6934 vmx_set_nmi_mask(vcpu, false); 6935 break; 6936 case INTR_TYPE_SOFT_EXCEPTION: 6937 vcpu->arch.event_exit_inst_len = vmcs_read32(instr_len_field); 6938 fallthrough; 6939 case INTR_TYPE_HARD_EXCEPTION: 6940 if (idt_vectoring_info & VECTORING_INFO_DELIVER_CODE_MASK) { 6941 u32 err = vmcs_read32(error_code_field); 6942 kvm_requeue_exception_e(vcpu, vector, err); 6943 } else 6944 kvm_requeue_exception(vcpu, vector); 6945 break; 6946 case INTR_TYPE_SOFT_INTR: 6947 vcpu->arch.event_exit_inst_len = vmcs_read32(instr_len_field); 6948 fallthrough; 6949 case INTR_TYPE_EXT_INTR: 6950 kvm_queue_interrupt(vcpu, vector, type == INTR_TYPE_SOFT_INTR); 6951 break; 6952 default: 6953 break; 6954 } 6955 } 6956 6957 static void vmx_complete_interrupts(struct vcpu_vmx *vmx) 6958 { 6959 __vmx_complete_interrupts(&vmx->vcpu, vmx->idt_vectoring_info, 6960 VM_EXIT_INSTRUCTION_LEN, 6961 IDT_VECTORING_ERROR_CODE); 6962 } 6963 6964 static void vmx_cancel_injection(struct kvm_vcpu *vcpu) 6965 { 6966 __vmx_complete_interrupts(vcpu, 6967 vmcs_read32(VM_ENTRY_INTR_INFO_FIELD), 6968 VM_ENTRY_INSTRUCTION_LEN, 6969 VM_ENTRY_EXCEPTION_ERROR_CODE); 6970 6971 vmcs_write32(VM_ENTRY_INTR_INFO_FIELD, 0); 6972 } 6973 6974 static void atomic_switch_perf_msrs(struct vcpu_vmx *vmx) 6975 { 6976 int i, nr_msrs; 6977 struct perf_guest_switch_msr *msrs; 6978 struct kvm_pmu *pmu = vcpu_to_pmu(&vmx->vcpu); 6979 6980 pmu->host_cross_mapped_mask = 0; 6981 if (pmu->pebs_enable & pmu->global_ctrl) 6982 intel_pmu_cross_mapped_check(pmu); 6983 6984 /* Note, nr_msrs may be garbage if perf_guest_get_msrs() returns NULL. */ 6985 msrs = perf_guest_get_msrs(&nr_msrs, (void *)pmu); 6986 if (!msrs) 6987 return; 6988 6989 for (i = 0; i < nr_msrs; i++) 6990 if (msrs[i].host == msrs[i].guest) 6991 clear_atomic_switch_msr(vmx, msrs[i].msr); 6992 else 6993 add_atomic_switch_msr(vmx, msrs[i].msr, msrs[i].guest, 6994 msrs[i].host, false); 6995 } 6996 6997 static void vmx_update_hv_timer(struct kvm_vcpu *vcpu) 6998 { 6999 struct vcpu_vmx *vmx = to_vmx(vcpu); 7000 u64 tscl; 7001 u32 delta_tsc; 7002 7003 if (vmx->req_immediate_exit) { 7004 vmcs_write32(VMX_PREEMPTION_TIMER_VALUE, 0); 7005 vmx->loaded_vmcs->hv_timer_soft_disabled = false; 7006 } else if (vmx->hv_deadline_tsc != -1) { 7007 tscl = rdtsc(); 7008 if (vmx->hv_deadline_tsc > tscl) 7009 /* set_hv_timer ensures the delta fits in 32-bits */ 7010 delta_tsc = (u32)((vmx->hv_deadline_tsc - tscl) >> 7011 cpu_preemption_timer_multi); 7012 else 7013 delta_tsc = 0; 7014 7015 vmcs_write32(VMX_PREEMPTION_TIMER_VALUE, delta_tsc); 7016 vmx->loaded_vmcs->hv_timer_soft_disabled = false; 7017 } else if (!vmx->loaded_vmcs->hv_timer_soft_disabled) { 7018 vmcs_write32(VMX_PREEMPTION_TIMER_VALUE, -1); 7019 vmx->loaded_vmcs->hv_timer_soft_disabled = true; 7020 } 7021 } 7022 7023 void noinstr vmx_update_host_rsp(struct vcpu_vmx *vmx, unsigned long host_rsp) 7024 { 7025 if (unlikely(host_rsp != vmx->loaded_vmcs->host_state.rsp)) { 7026 vmx->loaded_vmcs->host_state.rsp = host_rsp; 7027 vmcs_writel(HOST_RSP, host_rsp); 7028 } 7029 } 7030 7031 void noinstr vmx_spec_ctrl_restore_host(struct vcpu_vmx *vmx, 7032 unsigned int flags) 7033 { 7034 u64 hostval = this_cpu_read(x86_spec_ctrl_current); 7035 7036 if (!cpu_feature_enabled(X86_FEATURE_MSR_SPEC_CTRL)) 7037 return; 7038 7039 if (flags & VMX_RUN_SAVE_SPEC_CTRL) 7040 vmx->spec_ctrl = __rdmsr(MSR_IA32_SPEC_CTRL); 7041 7042 /* 7043 * If the guest/host SPEC_CTRL values differ, restore the host value. 7044 * 7045 * For legacy IBRS, the IBRS bit always needs to be written after 7046 * transitioning from a less privileged predictor mode, regardless of 7047 * whether the guest/host values differ. 7048 */ 7049 if (cpu_feature_enabled(X86_FEATURE_KERNEL_IBRS) || 7050 vmx->spec_ctrl != hostval) 7051 native_wrmsrl(MSR_IA32_SPEC_CTRL, hostval); 7052 7053 barrier_nospec(); 7054 } 7055 7056 static fastpath_t vmx_exit_handlers_fastpath(struct kvm_vcpu *vcpu) 7057 { 7058 switch (to_vmx(vcpu)->exit_reason.basic) { 7059 case EXIT_REASON_MSR_WRITE: 7060 return handle_fastpath_set_msr_irqoff(vcpu); 7061 case EXIT_REASON_PREEMPTION_TIMER: 7062 return handle_fastpath_preemption_timer(vcpu); 7063 default: 7064 return EXIT_FASTPATH_NONE; 7065 } 7066 } 7067 7068 static noinstr void vmx_vcpu_enter_exit(struct kvm_vcpu *vcpu, 7069 struct vcpu_vmx *vmx, 7070 unsigned long flags) 7071 { 7072 guest_state_enter_irqoff(); 7073 7074 /* L1D Flush includes CPU buffer clear to mitigate MDS */ 7075 if (static_branch_unlikely(&vmx_l1d_should_flush)) 7076 vmx_l1d_flush(vcpu); 7077 else if (static_branch_unlikely(&mds_user_clear)) 7078 mds_clear_cpu_buffers(); 7079 else if (static_branch_unlikely(&mmio_stale_data_clear) && 7080 kvm_arch_has_assigned_device(vcpu->kvm)) 7081 mds_clear_cpu_buffers(); 7082 7083 vmx_disable_fb_clear(vmx); 7084 7085 if (vcpu->arch.cr2 != native_read_cr2()) 7086 native_write_cr2(vcpu->arch.cr2); 7087 7088 vmx->fail = __vmx_vcpu_run(vmx, (unsigned long *)&vcpu->arch.regs, 7089 flags); 7090 7091 vcpu->arch.cr2 = native_read_cr2(); 7092 7093 vmx_enable_fb_clear(vmx); 7094 7095 guest_state_exit_irqoff(); 7096 } 7097 7098 static fastpath_t vmx_vcpu_run(struct kvm_vcpu *vcpu) 7099 { 7100 struct vcpu_vmx *vmx = to_vmx(vcpu); 7101 unsigned long cr3, cr4; 7102 7103 /* Record the guest's net vcpu time for enforced NMI injections. */ 7104 if (unlikely(!enable_vnmi && 7105 vmx->loaded_vmcs->soft_vnmi_blocked)) 7106 vmx->loaded_vmcs->entry_time = ktime_get(); 7107 7108 /* 7109 * Don't enter VMX if guest state is invalid, let the exit handler 7110 * start emulation until we arrive back to a valid state. Synthesize a 7111 * consistency check VM-Exit due to invalid guest state and bail. 7112 */ 7113 if (unlikely(vmx->emulation_required)) { 7114 vmx->fail = 0; 7115 7116 vmx->exit_reason.full = EXIT_REASON_INVALID_STATE; 7117 vmx->exit_reason.failed_vmentry = 1; 7118 kvm_register_mark_available(vcpu, VCPU_EXREG_EXIT_INFO_1); 7119 vmx->exit_qualification = ENTRY_FAIL_DEFAULT; 7120 kvm_register_mark_available(vcpu, VCPU_EXREG_EXIT_INFO_2); 7121 vmx->exit_intr_info = 0; 7122 return EXIT_FASTPATH_NONE; 7123 } 7124 7125 trace_kvm_entry(vcpu); 7126 7127 if (vmx->ple_window_dirty) { 7128 vmx->ple_window_dirty = false; 7129 vmcs_write32(PLE_WINDOW, vmx->ple_window); 7130 } 7131 7132 /* 7133 * We did this in prepare_switch_to_guest, because it needs to 7134 * be within srcu_read_lock. 7135 */ 7136 WARN_ON_ONCE(vmx->nested.need_vmcs12_to_shadow_sync); 7137 7138 if (kvm_register_is_dirty(vcpu, VCPU_REGS_RSP)) 7139 vmcs_writel(GUEST_RSP, vcpu->arch.regs[VCPU_REGS_RSP]); 7140 if (kvm_register_is_dirty(vcpu, VCPU_REGS_RIP)) 7141 vmcs_writel(GUEST_RIP, vcpu->arch.regs[VCPU_REGS_RIP]); 7142 vcpu->arch.regs_dirty = 0; 7143 7144 /* 7145 * Refresh vmcs.HOST_CR3 if necessary. This must be done immediately 7146 * prior to VM-Enter, as the kernel may load a new ASID (PCID) any time 7147 * it switches back to the current->mm, which can occur in KVM context 7148 * when switching to a temporary mm to patch kernel code, e.g. if KVM 7149 * toggles a static key while handling a VM-Exit. 7150 */ 7151 cr3 = __get_current_cr3_fast(); 7152 if (unlikely(cr3 != vmx->loaded_vmcs->host_state.cr3)) { 7153 vmcs_writel(HOST_CR3, cr3); 7154 vmx->loaded_vmcs->host_state.cr3 = cr3; 7155 } 7156 7157 cr4 = cr4_read_shadow(); 7158 if (unlikely(cr4 != vmx->loaded_vmcs->host_state.cr4)) { 7159 vmcs_writel(HOST_CR4, cr4); 7160 vmx->loaded_vmcs->host_state.cr4 = cr4; 7161 } 7162 7163 /* When KVM_DEBUGREG_WONT_EXIT, dr6 is accessible in guest. */ 7164 if (unlikely(vcpu->arch.switch_db_regs & KVM_DEBUGREG_WONT_EXIT)) 7165 set_debugreg(vcpu->arch.dr6, 6); 7166 7167 /* When single-stepping over STI and MOV SS, we must clear the 7168 * corresponding interruptibility bits in the guest state. Otherwise 7169 * vmentry fails as it then expects bit 14 (BS) in pending debug 7170 * exceptions being set, but that's not correct for the guest debugging 7171 * case. */ 7172 if (vcpu->guest_debug & KVM_GUESTDBG_SINGLESTEP) 7173 vmx_set_interrupt_shadow(vcpu, 0); 7174 7175 kvm_load_guest_xsave_state(vcpu); 7176 7177 pt_guest_enter(vmx); 7178 7179 atomic_switch_perf_msrs(vmx); 7180 if (intel_pmu_lbr_is_enabled(vcpu)) 7181 vmx_passthrough_lbr_msrs(vcpu); 7182 7183 if (enable_preemption_timer) 7184 vmx_update_hv_timer(vcpu); 7185 7186 kvm_wait_lapic_expire(vcpu); 7187 7188 /* The actual VMENTER/EXIT is in the .noinstr.text section. */ 7189 vmx_vcpu_enter_exit(vcpu, vmx, __vmx_vcpu_run_flags(vmx)); 7190 7191 /* All fields are clean at this point */ 7192 if (static_branch_unlikely(&enable_evmcs)) { 7193 current_evmcs->hv_clean_fields |= 7194 HV_VMX_ENLIGHTENED_CLEAN_FIELD_ALL; 7195 7196 current_evmcs->hv_vp_id = kvm_hv_get_vpindex(vcpu); 7197 } 7198 7199 /* MSR_IA32_DEBUGCTLMSR is zeroed on vmexit. Restore it if needed */ 7200 if (vmx->host_debugctlmsr) 7201 update_debugctlmsr(vmx->host_debugctlmsr); 7202 7203 #ifndef CONFIG_X86_64 7204 /* 7205 * The sysexit path does not restore ds/es, so we must set them to 7206 * a reasonable value ourselves. 7207 * 7208 * We can't defer this to vmx_prepare_switch_to_host() since that 7209 * function may be executed in interrupt context, which saves and 7210 * restore segments around it, nullifying its effect. 7211 */ 7212 loadsegment(ds, __USER_DS); 7213 loadsegment(es, __USER_DS); 7214 #endif 7215 7216 vcpu->arch.regs_avail &= ~VMX_REGS_LAZY_LOAD_SET; 7217 7218 pt_guest_exit(vmx); 7219 7220 kvm_load_host_xsave_state(vcpu); 7221 7222 if (is_guest_mode(vcpu)) { 7223 /* 7224 * Track VMLAUNCH/VMRESUME that have made past guest state 7225 * checking. 7226 */ 7227 if (vmx->nested.nested_run_pending && 7228 !vmx->exit_reason.failed_vmentry) 7229 ++vcpu->stat.nested_run; 7230 7231 vmx->nested.nested_run_pending = 0; 7232 } 7233 7234 vmx->idt_vectoring_info = 0; 7235 7236 if (unlikely(vmx->fail)) { 7237 vmx->exit_reason.full = 0xdead; 7238 return EXIT_FASTPATH_NONE; 7239 } 7240 7241 vmx->exit_reason.full = vmcs_read32(VM_EXIT_REASON); 7242 if (unlikely((u16)vmx->exit_reason.basic == EXIT_REASON_MCE_DURING_VMENTRY)) 7243 kvm_machine_check(); 7244 7245 if (likely(!vmx->exit_reason.failed_vmentry)) 7246 vmx->idt_vectoring_info = vmcs_read32(IDT_VECTORING_INFO_FIELD); 7247 7248 trace_kvm_exit(vcpu, KVM_ISA_VMX); 7249 7250 if (unlikely(vmx->exit_reason.failed_vmentry)) 7251 return EXIT_FASTPATH_NONE; 7252 7253 vmx->loaded_vmcs->launched = 1; 7254 7255 vmx_recover_nmi_blocking(vmx); 7256 vmx_complete_interrupts(vmx); 7257 7258 if (is_guest_mode(vcpu)) 7259 return EXIT_FASTPATH_NONE; 7260 7261 return vmx_exit_handlers_fastpath(vcpu); 7262 } 7263 7264 static void vmx_vcpu_free(struct kvm_vcpu *vcpu) 7265 { 7266 struct vcpu_vmx *vmx = to_vmx(vcpu); 7267 7268 if (enable_pml) 7269 vmx_destroy_pml_buffer(vmx); 7270 free_vpid(vmx->vpid); 7271 nested_vmx_free_vcpu(vcpu); 7272 free_loaded_vmcs(vmx->loaded_vmcs); 7273 } 7274 7275 static int vmx_vcpu_create(struct kvm_vcpu *vcpu) 7276 { 7277 struct vmx_uret_msr *tsx_ctrl; 7278 struct vcpu_vmx *vmx; 7279 int i, err; 7280 7281 BUILD_BUG_ON(offsetof(struct vcpu_vmx, vcpu) != 0); 7282 vmx = to_vmx(vcpu); 7283 7284 INIT_LIST_HEAD(&vmx->pi_wakeup_list); 7285 7286 err = -ENOMEM; 7287 7288 vmx->vpid = allocate_vpid(); 7289 7290 /* 7291 * If PML is turned on, failure on enabling PML just results in failure 7292 * of creating the vcpu, therefore we can simplify PML logic (by 7293 * avoiding dealing with cases, such as enabling PML partially on vcpus 7294 * for the guest), etc. 7295 */ 7296 if (enable_pml) { 7297 vmx->pml_pg = alloc_page(GFP_KERNEL_ACCOUNT | __GFP_ZERO); 7298 if (!vmx->pml_pg) 7299 goto free_vpid; 7300 } 7301 7302 for (i = 0; i < kvm_nr_uret_msrs; ++i) 7303 vmx->guest_uret_msrs[i].mask = -1ull; 7304 if (boot_cpu_has(X86_FEATURE_RTM)) { 7305 /* 7306 * TSX_CTRL_CPUID_CLEAR is handled in the CPUID interception. 7307 * Keep the host value unchanged to avoid changing CPUID bits 7308 * under the host kernel's feet. 7309 */ 7310 tsx_ctrl = vmx_find_uret_msr(vmx, MSR_IA32_TSX_CTRL); 7311 if (tsx_ctrl) 7312 tsx_ctrl->mask = ~(u64)TSX_CTRL_CPUID_CLEAR; 7313 } 7314 7315 err = alloc_loaded_vmcs(&vmx->vmcs01); 7316 if (err < 0) 7317 goto free_pml; 7318 7319 /* 7320 * Use Hyper-V 'Enlightened MSR Bitmap' feature when KVM runs as a 7321 * nested (L1) hypervisor and Hyper-V in L0 supports it. Enable the 7322 * feature only for vmcs01, KVM currently isn't equipped to realize any 7323 * performance benefits from enabling it for vmcs02. 7324 */ 7325 if (IS_ENABLED(CONFIG_HYPERV) && static_branch_unlikely(&enable_evmcs) && 7326 (ms_hyperv.nested_features & HV_X64_NESTED_MSR_BITMAP)) { 7327 struct hv_enlightened_vmcs *evmcs = (void *)vmx->vmcs01.vmcs; 7328 7329 evmcs->hv_enlightenments_control.msr_bitmap = 1; 7330 } 7331 7332 /* The MSR bitmap starts with all ones */ 7333 bitmap_fill(vmx->shadow_msr_intercept.read, MAX_POSSIBLE_PASSTHROUGH_MSRS); 7334 bitmap_fill(vmx->shadow_msr_intercept.write, MAX_POSSIBLE_PASSTHROUGH_MSRS); 7335 7336 vmx_disable_intercept_for_msr(vcpu, MSR_IA32_TSC, MSR_TYPE_R); 7337 #ifdef CONFIG_X86_64 7338 vmx_disable_intercept_for_msr(vcpu, MSR_FS_BASE, MSR_TYPE_RW); 7339 vmx_disable_intercept_for_msr(vcpu, MSR_GS_BASE, MSR_TYPE_RW); 7340 vmx_disable_intercept_for_msr(vcpu, MSR_KERNEL_GS_BASE, MSR_TYPE_RW); 7341 #endif 7342 vmx_disable_intercept_for_msr(vcpu, MSR_IA32_SYSENTER_CS, MSR_TYPE_RW); 7343 vmx_disable_intercept_for_msr(vcpu, MSR_IA32_SYSENTER_ESP, MSR_TYPE_RW); 7344 vmx_disable_intercept_for_msr(vcpu, MSR_IA32_SYSENTER_EIP, MSR_TYPE_RW); 7345 if (kvm_cstate_in_guest(vcpu->kvm)) { 7346 vmx_disable_intercept_for_msr(vcpu, MSR_CORE_C1_RES, MSR_TYPE_R); 7347 vmx_disable_intercept_for_msr(vcpu, MSR_CORE_C3_RESIDENCY, MSR_TYPE_R); 7348 vmx_disable_intercept_for_msr(vcpu, MSR_CORE_C6_RESIDENCY, MSR_TYPE_R); 7349 vmx_disable_intercept_for_msr(vcpu, MSR_CORE_C7_RESIDENCY, MSR_TYPE_R); 7350 } 7351 7352 vmx->loaded_vmcs = &vmx->vmcs01; 7353 7354 if (cpu_need_virtualize_apic_accesses(vcpu)) { 7355 err = alloc_apic_access_page(vcpu->kvm); 7356 if (err) 7357 goto free_vmcs; 7358 } 7359 7360 if (enable_ept && !enable_unrestricted_guest) { 7361 err = init_rmode_identity_map(vcpu->kvm); 7362 if (err) 7363 goto free_vmcs; 7364 } 7365 7366 if (vmx_can_use_ipiv(vcpu)) 7367 WRITE_ONCE(to_kvm_vmx(vcpu->kvm)->pid_table[vcpu->vcpu_id], 7368 __pa(&vmx->pi_desc) | PID_TABLE_ENTRY_VALID); 7369 7370 return 0; 7371 7372 free_vmcs: 7373 free_loaded_vmcs(vmx->loaded_vmcs); 7374 free_pml: 7375 vmx_destroy_pml_buffer(vmx); 7376 free_vpid: 7377 free_vpid(vmx->vpid); 7378 return err; 7379 } 7380 7381 #define L1TF_MSG_SMT "L1TF CPU bug present and SMT on, data leak possible. See CVE-2018-3646 and https://www.kernel.org/doc/html/latest/admin-guide/hw-vuln/l1tf.html for details.\n" 7382 #define L1TF_MSG_L1D "L1TF CPU bug present and virtualization mitigation disabled, data leak possible. See CVE-2018-3646 and https://www.kernel.org/doc/html/latest/admin-guide/hw-vuln/l1tf.html for details.\n" 7383 7384 static int vmx_vm_init(struct kvm *kvm) 7385 { 7386 if (!ple_gap) 7387 kvm->arch.pause_in_guest = true; 7388 7389 if (boot_cpu_has(X86_BUG_L1TF) && enable_ept) { 7390 switch (l1tf_mitigation) { 7391 case L1TF_MITIGATION_OFF: 7392 case L1TF_MITIGATION_FLUSH_NOWARN: 7393 /* 'I explicitly don't care' is set */ 7394 break; 7395 case L1TF_MITIGATION_FLUSH: 7396 case L1TF_MITIGATION_FLUSH_NOSMT: 7397 case L1TF_MITIGATION_FULL: 7398 /* 7399 * Warn upon starting the first VM in a potentially 7400 * insecure environment. 7401 */ 7402 if (sched_smt_active()) 7403 pr_warn_once(L1TF_MSG_SMT); 7404 if (l1tf_vmx_mitigation == VMENTER_L1D_FLUSH_NEVER) 7405 pr_warn_once(L1TF_MSG_L1D); 7406 break; 7407 case L1TF_MITIGATION_FULL_FORCE: 7408 /* Flush is enforced */ 7409 break; 7410 } 7411 } 7412 return 0; 7413 } 7414 7415 static int __init vmx_check_processor_compat(void) 7416 { 7417 struct vmcs_config vmcs_conf; 7418 struct vmx_capability vmx_cap; 7419 7420 if (!this_cpu_has(X86_FEATURE_MSR_IA32_FEAT_CTL) || 7421 !this_cpu_has(X86_FEATURE_VMX)) { 7422 pr_err("kvm: VMX is disabled on CPU %d\n", smp_processor_id()); 7423 return -EIO; 7424 } 7425 7426 if (setup_vmcs_config(&vmcs_conf, &vmx_cap) < 0) 7427 return -EIO; 7428 if (nested) 7429 nested_vmx_setup_ctls_msrs(&vmcs_conf, vmx_cap.ept); 7430 if (memcmp(&vmcs_config, &vmcs_conf, sizeof(struct vmcs_config)) != 0) { 7431 printk(KERN_ERR "kvm: CPU %d feature inconsistency!\n", 7432 smp_processor_id()); 7433 return -EIO; 7434 } 7435 return 0; 7436 } 7437 7438 static u8 vmx_get_mt_mask(struct kvm_vcpu *vcpu, gfn_t gfn, bool is_mmio) 7439 { 7440 u8 cache; 7441 7442 /* We wanted to honor guest CD/MTRR/PAT, but doing so could result in 7443 * memory aliases with conflicting memory types and sometimes MCEs. 7444 * We have to be careful as to what are honored and when. 7445 * 7446 * For MMIO, guest CD/MTRR are ignored. The EPT memory type is set to 7447 * UC. The effective memory type is UC or WC depending on guest PAT. 7448 * This was historically the source of MCEs and we want to be 7449 * conservative. 7450 * 7451 * When there is no need to deal with noncoherent DMA (e.g., no VT-d 7452 * or VT-d has snoop control), guest CD/MTRR/PAT are all ignored. The 7453 * EPT memory type is set to WB. The effective memory type is forced 7454 * WB. 7455 * 7456 * Otherwise, we trust guest. Guest CD/MTRR/PAT are all honored. The 7457 * EPT memory type is used to emulate guest CD/MTRR. 7458 */ 7459 7460 if (is_mmio) 7461 return MTRR_TYPE_UNCACHABLE << VMX_EPT_MT_EPTE_SHIFT; 7462 7463 if (!kvm_arch_has_noncoherent_dma(vcpu->kvm)) 7464 return (MTRR_TYPE_WRBACK << VMX_EPT_MT_EPTE_SHIFT) | VMX_EPT_IPAT_BIT; 7465 7466 if (kvm_read_cr0(vcpu) & X86_CR0_CD) { 7467 if (kvm_check_has_quirk(vcpu->kvm, KVM_X86_QUIRK_CD_NW_CLEARED)) 7468 cache = MTRR_TYPE_WRBACK; 7469 else 7470 cache = MTRR_TYPE_UNCACHABLE; 7471 7472 return (cache << VMX_EPT_MT_EPTE_SHIFT) | VMX_EPT_IPAT_BIT; 7473 } 7474 7475 return kvm_mtrr_get_guest_memory_type(vcpu, gfn) << VMX_EPT_MT_EPTE_SHIFT; 7476 } 7477 7478 static void vmcs_set_secondary_exec_control(struct vcpu_vmx *vmx, u32 new_ctl) 7479 { 7480 /* 7481 * These bits in the secondary execution controls field 7482 * are dynamic, the others are mostly based on the hypervisor 7483 * architecture and the guest's CPUID. Do not touch the 7484 * dynamic bits. 7485 */ 7486 u32 mask = 7487 SECONDARY_EXEC_SHADOW_VMCS | 7488 SECONDARY_EXEC_VIRTUALIZE_X2APIC_MODE | 7489 SECONDARY_EXEC_VIRTUALIZE_APIC_ACCESSES | 7490 SECONDARY_EXEC_DESC; 7491 7492 u32 cur_ctl = secondary_exec_controls_get(vmx); 7493 7494 secondary_exec_controls_set(vmx, (new_ctl & ~mask) | (cur_ctl & mask)); 7495 } 7496 7497 /* 7498 * Generate MSR_IA32_VMX_CR{0,4}_FIXED1 according to CPUID. Only set bits 7499 * (indicating "allowed-1") if they are supported in the guest's CPUID. 7500 */ 7501 static void nested_vmx_cr_fixed1_bits_update(struct kvm_vcpu *vcpu) 7502 { 7503 struct vcpu_vmx *vmx = to_vmx(vcpu); 7504 struct kvm_cpuid_entry2 *entry; 7505 7506 vmx->nested.msrs.cr0_fixed1 = 0xffffffff; 7507 vmx->nested.msrs.cr4_fixed1 = X86_CR4_PCE; 7508 7509 #define cr4_fixed1_update(_cr4_mask, _reg, _cpuid_mask) do { \ 7510 if (entry && (entry->_reg & (_cpuid_mask))) \ 7511 vmx->nested.msrs.cr4_fixed1 |= (_cr4_mask); \ 7512 } while (0) 7513 7514 entry = kvm_find_cpuid_entry(vcpu, 0x1); 7515 cr4_fixed1_update(X86_CR4_VME, edx, feature_bit(VME)); 7516 cr4_fixed1_update(X86_CR4_PVI, edx, feature_bit(VME)); 7517 cr4_fixed1_update(X86_CR4_TSD, edx, feature_bit(TSC)); 7518 cr4_fixed1_update(X86_CR4_DE, edx, feature_bit(DE)); 7519 cr4_fixed1_update(X86_CR4_PSE, edx, feature_bit(PSE)); 7520 cr4_fixed1_update(X86_CR4_PAE, edx, feature_bit(PAE)); 7521 cr4_fixed1_update(X86_CR4_MCE, edx, feature_bit(MCE)); 7522 cr4_fixed1_update(X86_CR4_PGE, edx, feature_bit(PGE)); 7523 cr4_fixed1_update(X86_CR4_OSFXSR, edx, feature_bit(FXSR)); 7524 cr4_fixed1_update(X86_CR4_OSXMMEXCPT, edx, feature_bit(XMM)); 7525 cr4_fixed1_update(X86_CR4_VMXE, ecx, feature_bit(VMX)); 7526 cr4_fixed1_update(X86_CR4_SMXE, ecx, feature_bit(SMX)); 7527 cr4_fixed1_update(X86_CR4_PCIDE, ecx, feature_bit(PCID)); 7528 cr4_fixed1_update(X86_CR4_OSXSAVE, ecx, feature_bit(XSAVE)); 7529 7530 entry = kvm_find_cpuid_entry_index(vcpu, 0x7, 0); 7531 cr4_fixed1_update(X86_CR4_FSGSBASE, ebx, feature_bit(FSGSBASE)); 7532 cr4_fixed1_update(X86_CR4_SMEP, ebx, feature_bit(SMEP)); 7533 cr4_fixed1_update(X86_CR4_SMAP, ebx, feature_bit(SMAP)); 7534 cr4_fixed1_update(X86_CR4_PKE, ecx, feature_bit(PKU)); 7535 cr4_fixed1_update(X86_CR4_UMIP, ecx, feature_bit(UMIP)); 7536 cr4_fixed1_update(X86_CR4_LA57, ecx, feature_bit(LA57)); 7537 7538 #undef cr4_fixed1_update 7539 } 7540 7541 static void update_intel_pt_cfg(struct kvm_vcpu *vcpu) 7542 { 7543 struct vcpu_vmx *vmx = to_vmx(vcpu); 7544 struct kvm_cpuid_entry2 *best = NULL; 7545 int i; 7546 7547 for (i = 0; i < PT_CPUID_LEAVES; i++) { 7548 best = kvm_find_cpuid_entry_index(vcpu, 0x14, i); 7549 if (!best) 7550 return; 7551 vmx->pt_desc.caps[CPUID_EAX + i*PT_CPUID_REGS_NUM] = best->eax; 7552 vmx->pt_desc.caps[CPUID_EBX + i*PT_CPUID_REGS_NUM] = best->ebx; 7553 vmx->pt_desc.caps[CPUID_ECX + i*PT_CPUID_REGS_NUM] = best->ecx; 7554 vmx->pt_desc.caps[CPUID_EDX + i*PT_CPUID_REGS_NUM] = best->edx; 7555 } 7556 7557 /* Get the number of configurable Address Ranges for filtering */ 7558 vmx->pt_desc.num_address_ranges = intel_pt_validate_cap(vmx->pt_desc.caps, 7559 PT_CAP_num_address_ranges); 7560 7561 /* Initialize and clear the no dependency bits */ 7562 vmx->pt_desc.ctl_bitmask = ~(RTIT_CTL_TRACEEN | RTIT_CTL_OS | 7563 RTIT_CTL_USR | RTIT_CTL_TSC_EN | RTIT_CTL_DISRETC | 7564 RTIT_CTL_BRANCH_EN); 7565 7566 /* 7567 * If CPUID.(EAX=14H,ECX=0):EBX[0]=1 CR3Filter can be set otherwise 7568 * will inject an #GP 7569 */ 7570 if (intel_pt_validate_cap(vmx->pt_desc.caps, PT_CAP_cr3_filtering)) 7571 vmx->pt_desc.ctl_bitmask &= ~RTIT_CTL_CR3EN; 7572 7573 /* 7574 * If CPUID.(EAX=14H,ECX=0):EBX[1]=1 CYCEn, CycThresh and 7575 * PSBFreq can be set 7576 */ 7577 if (intel_pt_validate_cap(vmx->pt_desc.caps, PT_CAP_psb_cyc)) 7578 vmx->pt_desc.ctl_bitmask &= ~(RTIT_CTL_CYCLEACC | 7579 RTIT_CTL_CYC_THRESH | RTIT_CTL_PSB_FREQ); 7580 7581 /* 7582 * If CPUID.(EAX=14H,ECX=0):EBX[3]=1 MTCEn and MTCFreq can be set 7583 */ 7584 if (intel_pt_validate_cap(vmx->pt_desc.caps, PT_CAP_mtc)) 7585 vmx->pt_desc.ctl_bitmask &= ~(RTIT_CTL_MTC_EN | 7586 RTIT_CTL_MTC_RANGE); 7587 7588 /* If CPUID.(EAX=14H,ECX=0):EBX[4]=1 FUPonPTW and PTWEn can be set */ 7589 if (intel_pt_validate_cap(vmx->pt_desc.caps, PT_CAP_ptwrite)) 7590 vmx->pt_desc.ctl_bitmask &= ~(RTIT_CTL_FUP_ON_PTW | 7591 RTIT_CTL_PTW_EN); 7592 7593 /* If CPUID.(EAX=14H,ECX=0):EBX[5]=1 PwrEvEn can be set */ 7594 if (intel_pt_validate_cap(vmx->pt_desc.caps, PT_CAP_power_event_trace)) 7595 vmx->pt_desc.ctl_bitmask &= ~RTIT_CTL_PWR_EVT_EN; 7596 7597 /* If CPUID.(EAX=14H,ECX=0):ECX[0]=1 ToPA can be set */ 7598 if (intel_pt_validate_cap(vmx->pt_desc.caps, PT_CAP_topa_output)) 7599 vmx->pt_desc.ctl_bitmask &= ~RTIT_CTL_TOPA; 7600 7601 /* If CPUID.(EAX=14H,ECX=0):ECX[3]=1 FabricEn can be set */ 7602 if (intel_pt_validate_cap(vmx->pt_desc.caps, PT_CAP_output_subsys)) 7603 vmx->pt_desc.ctl_bitmask &= ~RTIT_CTL_FABRIC_EN; 7604 7605 /* unmask address range configure area */ 7606 for (i = 0; i < vmx->pt_desc.num_address_ranges; i++) 7607 vmx->pt_desc.ctl_bitmask &= ~(0xfULL << (32 + i * 4)); 7608 } 7609 7610 static void vmx_vcpu_after_set_cpuid(struct kvm_vcpu *vcpu) 7611 { 7612 struct vcpu_vmx *vmx = to_vmx(vcpu); 7613 7614 /* xsaves_enabled is recomputed in vmx_compute_secondary_exec_control(). */ 7615 vcpu->arch.xsaves_enabled = false; 7616 7617 vmx_setup_uret_msrs(vmx); 7618 7619 if (cpu_has_secondary_exec_ctrls()) 7620 vmcs_set_secondary_exec_control(vmx, 7621 vmx_secondary_exec_control(vmx)); 7622 7623 if (nested_vmx_allowed(vcpu)) 7624 vmx->msr_ia32_feature_control_valid_bits |= 7625 FEAT_CTL_VMX_ENABLED_INSIDE_SMX | 7626 FEAT_CTL_VMX_ENABLED_OUTSIDE_SMX; 7627 else 7628 vmx->msr_ia32_feature_control_valid_bits &= 7629 ~(FEAT_CTL_VMX_ENABLED_INSIDE_SMX | 7630 FEAT_CTL_VMX_ENABLED_OUTSIDE_SMX); 7631 7632 if (nested_vmx_allowed(vcpu)) 7633 nested_vmx_cr_fixed1_bits_update(vcpu); 7634 7635 if (boot_cpu_has(X86_FEATURE_INTEL_PT) && 7636 guest_cpuid_has(vcpu, X86_FEATURE_INTEL_PT)) 7637 update_intel_pt_cfg(vcpu); 7638 7639 if (boot_cpu_has(X86_FEATURE_RTM)) { 7640 struct vmx_uret_msr *msr; 7641 msr = vmx_find_uret_msr(vmx, MSR_IA32_TSX_CTRL); 7642 if (msr) { 7643 bool enabled = guest_cpuid_has(vcpu, X86_FEATURE_RTM); 7644 vmx_set_guest_uret_msr(vmx, msr, enabled ? 0 : TSX_CTRL_RTM_DISABLE); 7645 } 7646 } 7647 7648 if (kvm_cpu_cap_has(X86_FEATURE_XFD)) 7649 vmx_set_intercept_for_msr(vcpu, MSR_IA32_XFD_ERR, MSR_TYPE_R, 7650 !guest_cpuid_has(vcpu, X86_FEATURE_XFD)); 7651 7652 7653 set_cr4_guest_host_mask(vmx); 7654 7655 vmx_write_encls_bitmap(vcpu, NULL); 7656 if (guest_cpuid_has(vcpu, X86_FEATURE_SGX)) 7657 vmx->msr_ia32_feature_control_valid_bits |= FEAT_CTL_SGX_ENABLED; 7658 else 7659 vmx->msr_ia32_feature_control_valid_bits &= ~FEAT_CTL_SGX_ENABLED; 7660 7661 if (guest_cpuid_has(vcpu, X86_FEATURE_SGX_LC)) 7662 vmx->msr_ia32_feature_control_valid_bits |= 7663 FEAT_CTL_SGX_LC_ENABLED; 7664 else 7665 vmx->msr_ia32_feature_control_valid_bits &= 7666 ~FEAT_CTL_SGX_LC_ENABLED; 7667 7668 /* Refresh #PF interception to account for MAXPHYADDR changes. */ 7669 vmx_update_exception_bitmap(vcpu); 7670 } 7671 7672 static __init void vmx_set_cpu_caps(void) 7673 { 7674 kvm_set_cpu_caps(); 7675 7676 /* CPUID 0x1 */ 7677 if (nested) 7678 kvm_cpu_cap_set(X86_FEATURE_VMX); 7679 7680 /* CPUID 0x7 */ 7681 if (kvm_mpx_supported()) 7682 kvm_cpu_cap_check_and_set(X86_FEATURE_MPX); 7683 if (!cpu_has_vmx_invpcid()) 7684 kvm_cpu_cap_clear(X86_FEATURE_INVPCID); 7685 if (vmx_pt_mode_is_host_guest()) 7686 kvm_cpu_cap_check_and_set(X86_FEATURE_INTEL_PT); 7687 if (vmx_pebs_supported()) { 7688 kvm_cpu_cap_check_and_set(X86_FEATURE_DS); 7689 kvm_cpu_cap_check_and_set(X86_FEATURE_DTES64); 7690 } 7691 7692 if (!enable_pmu) 7693 kvm_cpu_cap_clear(X86_FEATURE_PDCM); 7694 7695 if (!enable_sgx) { 7696 kvm_cpu_cap_clear(X86_FEATURE_SGX); 7697 kvm_cpu_cap_clear(X86_FEATURE_SGX_LC); 7698 kvm_cpu_cap_clear(X86_FEATURE_SGX1); 7699 kvm_cpu_cap_clear(X86_FEATURE_SGX2); 7700 } 7701 7702 if (vmx_umip_emulated()) 7703 kvm_cpu_cap_set(X86_FEATURE_UMIP); 7704 7705 /* CPUID 0xD.1 */ 7706 kvm_caps.supported_xss = 0; 7707 if (!cpu_has_vmx_xsaves()) 7708 kvm_cpu_cap_clear(X86_FEATURE_XSAVES); 7709 7710 /* CPUID 0x80000001 and 0x7 (RDPID) */ 7711 if (!cpu_has_vmx_rdtscp()) { 7712 kvm_cpu_cap_clear(X86_FEATURE_RDTSCP); 7713 kvm_cpu_cap_clear(X86_FEATURE_RDPID); 7714 } 7715 7716 if (cpu_has_vmx_waitpkg()) 7717 kvm_cpu_cap_check_and_set(X86_FEATURE_WAITPKG); 7718 } 7719 7720 static void vmx_request_immediate_exit(struct kvm_vcpu *vcpu) 7721 { 7722 to_vmx(vcpu)->req_immediate_exit = true; 7723 } 7724 7725 static int vmx_check_intercept_io(struct kvm_vcpu *vcpu, 7726 struct x86_instruction_info *info) 7727 { 7728 struct vmcs12 *vmcs12 = get_vmcs12(vcpu); 7729 unsigned short port; 7730 bool intercept; 7731 int size; 7732 7733 if (info->intercept == x86_intercept_in || 7734 info->intercept == x86_intercept_ins) { 7735 port = info->src_val; 7736 size = info->dst_bytes; 7737 } else { 7738 port = info->dst_val; 7739 size = info->src_bytes; 7740 } 7741 7742 /* 7743 * If the 'use IO bitmaps' VM-execution control is 0, IO instruction 7744 * VM-exits depend on the 'unconditional IO exiting' VM-execution 7745 * control. 7746 * 7747 * Otherwise, IO instruction VM-exits are controlled by the IO bitmaps. 7748 */ 7749 if (!nested_cpu_has(vmcs12, CPU_BASED_USE_IO_BITMAPS)) 7750 intercept = nested_cpu_has(vmcs12, 7751 CPU_BASED_UNCOND_IO_EXITING); 7752 else 7753 intercept = nested_vmx_check_io_bitmaps(vcpu, port, size); 7754 7755 /* FIXME: produce nested vmexit and return X86EMUL_INTERCEPTED. */ 7756 return intercept ? X86EMUL_UNHANDLEABLE : X86EMUL_CONTINUE; 7757 } 7758 7759 static int vmx_check_intercept(struct kvm_vcpu *vcpu, 7760 struct x86_instruction_info *info, 7761 enum x86_intercept_stage stage, 7762 struct x86_exception *exception) 7763 { 7764 struct vmcs12 *vmcs12 = get_vmcs12(vcpu); 7765 7766 switch (info->intercept) { 7767 /* 7768 * RDPID causes #UD if disabled through secondary execution controls. 7769 * Because it is marked as EmulateOnUD, we need to intercept it here. 7770 * Note, RDPID is hidden behind ENABLE_RDTSCP. 7771 */ 7772 case x86_intercept_rdpid: 7773 if (!nested_cpu_has2(vmcs12, SECONDARY_EXEC_ENABLE_RDTSCP)) { 7774 exception->vector = UD_VECTOR; 7775 exception->error_code_valid = false; 7776 return X86EMUL_PROPAGATE_FAULT; 7777 } 7778 break; 7779 7780 case x86_intercept_in: 7781 case x86_intercept_ins: 7782 case x86_intercept_out: 7783 case x86_intercept_outs: 7784 return vmx_check_intercept_io(vcpu, info); 7785 7786 case x86_intercept_lgdt: 7787 case x86_intercept_lidt: 7788 case x86_intercept_lldt: 7789 case x86_intercept_ltr: 7790 case x86_intercept_sgdt: 7791 case x86_intercept_sidt: 7792 case x86_intercept_sldt: 7793 case x86_intercept_str: 7794 if (!nested_cpu_has2(vmcs12, SECONDARY_EXEC_DESC)) 7795 return X86EMUL_CONTINUE; 7796 7797 /* FIXME: produce nested vmexit and return X86EMUL_INTERCEPTED. */ 7798 break; 7799 7800 /* TODO: check more intercepts... */ 7801 default: 7802 break; 7803 } 7804 7805 return X86EMUL_UNHANDLEABLE; 7806 } 7807 7808 #ifdef CONFIG_X86_64 7809 /* (a << shift) / divisor, return 1 if overflow otherwise 0 */ 7810 static inline int u64_shl_div_u64(u64 a, unsigned int shift, 7811 u64 divisor, u64 *result) 7812 { 7813 u64 low = a << shift, high = a >> (64 - shift); 7814 7815 /* To avoid the overflow on divq */ 7816 if (high >= divisor) 7817 return 1; 7818 7819 /* Low hold the result, high hold rem which is discarded */ 7820 asm("divq %2\n\t" : "=a" (low), "=d" (high) : 7821 "rm" (divisor), "0" (low), "1" (high)); 7822 *result = low; 7823 7824 return 0; 7825 } 7826 7827 static int vmx_set_hv_timer(struct kvm_vcpu *vcpu, u64 guest_deadline_tsc, 7828 bool *expired) 7829 { 7830 struct vcpu_vmx *vmx; 7831 u64 tscl, guest_tscl, delta_tsc, lapic_timer_advance_cycles; 7832 struct kvm_timer *ktimer = &vcpu->arch.apic->lapic_timer; 7833 7834 vmx = to_vmx(vcpu); 7835 tscl = rdtsc(); 7836 guest_tscl = kvm_read_l1_tsc(vcpu, tscl); 7837 delta_tsc = max(guest_deadline_tsc, guest_tscl) - guest_tscl; 7838 lapic_timer_advance_cycles = nsec_to_cycles(vcpu, 7839 ktimer->timer_advance_ns); 7840 7841 if (delta_tsc > lapic_timer_advance_cycles) 7842 delta_tsc -= lapic_timer_advance_cycles; 7843 else 7844 delta_tsc = 0; 7845 7846 /* Convert to host delta tsc if tsc scaling is enabled */ 7847 if (vcpu->arch.l1_tsc_scaling_ratio != kvm_caps.default_tsc_scaling_ratio && 7848 delta_tsc && u64_shl_div_u64(delta_tsc, 7849 kvm_caps.tsc_scaling_ratio_frac_bits, 7850 vcpu->arch.l1_tsc_scaling_ratio, &delta_tsc)) 7851 return -ERANGE; 7852 7853 /* 7854 * If the delta tsc can't fit in the 32 bit after the multi shift, 7855 * we can't use the preemption timer. 7856 * It's possible that it fits on later vmentries, but checking 7857 * on every vmentry is costly so we just use an hrtimer. 7858 */ 7859 if (delta_tsc >> (cpu_preemption_timer_multi + 32)) 7860 return -ERANGE; 7861 7862 vmx->hv_deadline_tsc = tscl + delta_tsc; 7863 *expired = !delta_tsc; 7864 return 0; 7865 } 7866 7867 static void vmx_cancel_hv_timer(struct kvm_vcpu *vcpu) 7868 { 7869 to_vmx(vcpu)->hv_deadline_tsc = -1; 7870 } 7871 #endif 7872 7873 static void vmx_sched_in(struct kvm_vcpu *vcpu, int cpu) 7874 { 7875 if (!kvm_pause_in_guest(vcpu->kvm)) 7876 shrink_ple_window(vcpu); 7877 } 7878 7879 void vmx_update_cpu_dirty_logging(struct kvm_vcpu *vcpu) 7880 { 7881 struct vcpu_vmx *vmx = to_vmx(vcpu); 7882 7883 if (is_guest_mode(vcpu)) { 7884 vmx->nested.update_vmcs01_cpu_dirty_logging = true; 7885 return; 7886 } 7887 7888 /* 7889 * Note, cpu_dirty_logging_count can be changed concurrent with this 7890 * code, but in that case another update request will be made and so 7891 * the guest will never run with a stale PML value. 7892 */ 7893 if (vcpu->kvm->arch.cpu_dirty_logging_count) 7894 secondary_exec_controls_setbit(vmx, SECONDARY_EXEC_ENABLE_PML); 7895 else 7896 secondary_exec_controls_clearbit(vmx, SECONDARY_EXEC_ENABLE_PML); 7897 } 7898 7899 static void vmx_setup_mce(struct kvm_vcpu *vcpu) 7900 { 7901 if (vcpu->arch.mcg_cap & MCG_LMCE_P) 7902 to_vmx(vcpu)->msr_ia32_feature_control_valid_bits |= 7903 FEAT_CTL_LMCE_ENABLED; 7904 else 7905 to_vmx(vcpu)->msr_ia32_feature_control_valid_bits &= 7906 ~FEAT_CTL_LMCE_ENABLED; 7907 } 7908 7909 static int vmx_smi_allowed(struct kvm_vcpu *vcpu, bool for_injection) 7910 { 7911 /* we need a nested vmexit to enter SMM, postpone if run is pending */ 7912 if (to_vmx(vcpu)->nested.nested_run_pending) 7913 return -EBUSY; 7914 return !is_smm(vcpu); 7915 } 7916 7917 static int vmx_enter_smm(struct kvm_vcpu *vcpu, char *smstate) 7918 { 7919 struct vcpu_vmx *vmx = to_vmx(vcpu); 7920 7921 /* 7922 * TODO: Implement custom flows for forcing the vCPU out/in of L2 on 7923 * SMI and RSM. Using the common VM-Exit + VM-Enter routines is wrong 7924 * SMI and RSM only modify state that is saved and restored via SMRAM. 7925 * E.g. most MSRs are left untouched, but many are modified by VM-Exit 7926 * and VM-Enter, and thus L2's values may be corrupted on SMI+RSM. 7927 */ 7928 vmx->nested.smm.guest_mode = is_guest_mode(vcpu); 7929 if (vmx->nested.smm.guest_mode) 7930 nested_vmx_vmexit(vcpu, -1, 0, 0); 7931 7932 vmx->nested.smm.vmxon = vmx->nested.vmxon; 7933 vmx->nested.vmxon = false; 7934 vmx_clear_hlt(vcpu); 7935 return 0; 7936 } 7937 7938 static int vmx_leave_smm(struct kvm_vcpu *vcpu, const char *smstate) 7939 { 7940 struct vcpu_vmx *vmx = to_vmx(vcpu); 7941 int ret; 7942 7943 if (vmx->nested.smm.vmxon) { 7944 vmx->nested.vmxon = true; 7945 vmx->nested.smm.vmxon = false; 7946 } 7947 7948 if (vmx->nested.smm.guest_mode) { 7949 ret = nested_vmx_enter_non_root_mode(vcpu, false); 7950 if (ret) 7951 return ret; 7952 7953 vmx->nested.nested_run_pending = 1; 7954 vmx->nested.smm.guest_mode = false; 7955 } 7956 return 0; 7957 } 7958 7959 static void vmx_enable_smi_window(struct kvm_vcpu *vcpu) 7960 { 7961 /* RSM will cause a vmexit anyway. */ 7962 } 7963 7964 static bool vmx_apic_init_signal_blocked(struct kvm_vcpu *vcpu) 7965 { 7966 return to_vmx(vcpu)->nested.vmxon && !is_guest_mode(vcpu); 7967 } 7968 7969 static void vmx_migrate_timers(struct kvm_vcpu *vcpu) 7970 { 7971 if (is_guest_mode(vcpu)) { 7972 struct hrtimer *timer = &to_vmx(vcpu)->nested.preemption_timer; 7973 7974 if (hrtimer_try_to_cancel(timer) == 1) 7975 hrtimer_start_expires(timer, HRTIMER_MODE_ABS_PINNED); 7976 } 7977 } 7978 7979 static void vmx_hardware_unsetup(void) 7980 { 7981 kvm_set_posted_intr_wakeup_handler(NULL); 7982 7983 if (nested) 7984 nested_vmx_hardware_unsetup(); 7985 7986 free_kvm_area(); 7987 } 7988 7989 static bool vmx_check_apicv_inhibit_reasons(enum kvm_apicv_inhibit reason) 7990 { 7991 ulong supported = BIT(APICV_INHIBIT_REASON_DISABLE) | 7992 BIT(APICV_INHIBIT_REASON_ABSENT) | 7993 BIT(APICV_INHIBIT_REASON_HYPERV) | 7994 BIT(APICV_INHIBIT_REASON_BLOCKIRQ) | 7995 BIT(APICV_INHIBIT_REASON_APIC_ID_MODIFIED) | 7996 BIT(APICV_INHIBIT_REASON_APIC_BASE_MODIFIED); 7997 7998 return supported & BIT(reason); 7999 } 8000 8001 static void vmx_vm_destroy(struct kvm *kvm) 8002 { 8003 struct kvm_vmx *kvm_vmx = to_kvm_vmx(kvm); 8004 8005 free_pages((unsigned long)kvm_vmx->pid_table, vmx_get_pid_table_order(kvm)); 8006 } 8007 8008 static struct kvm_x86_ops vmx_x86_ops __initdata = { 8009 .name = "kvm_intel", 8010 8011 .hardware_unsetup = vmx_hardware_unsetup, 8012 8013 .hardware_enable = vmx_hardware_enable, 8014 .hardware_disable = vmx_hardware_disable, 8015 .has_emulated_msr = vmx_has_emulated_msr, 8016 8017 .vm_size = sizeof(struct kvm_vmx), 8018 .vm_init = vmx_vm_init, 8019 .vm_destroy = vmx_vm_destroy, 8020 8021 .vcpu_precreate = vmx_vcpu_precreate, 8022 .vcpu_create = vmx_vcpu_create, 8023 .vcpu_free = vmx_vcpu_free, 8024 .vcpu_reset = vmx_vcpu_reset, 8025 8026 .prepare_switch_to_guest = vmx_prepare_switch_to_guest, 8027 .vcpu_load = vmx_vcpu_load, 8028 .vcpu_put = vmx_vcpu_put, 8029 8030 .update_exception_bitmap = vmx_update_exception_bitmap, 8031 .get_msr_feature = vmx_get_msr_feature, 8032 .get_msr = vmx_get_msr, 8033 .set_msr = vmx_set_msr, 8034 .get_segment_base = vmx_get_segment_base, 8035 .get_segment = vmx_get_segment, 8036 .set_segment = vmx_set_segment, 8037 .get_cpl = vmx_get_cpl, 8038 .get_cs_db_l_bits = vmx_get_cs_db_l_bits, 8039 .set_cr0 = vmx_set_cr0, 8040 .is_valid_cr4 = vmx_is_valid_cr4, 8041 .set_cr4 = vmx_set_cr4, 8042 .set_efer = vmx_set_efer, 8043 .get_idt = vmx_get_idt, 8044 .set_idt = vmx_set_idt, 8045 .get_gdt = vmx_get_gdt, 8046 .set_gdt = vmx_set_gdt, 8047 .set_dr7 = vmx_set_dr7, 8048 .sync_dirty_debug_regs = vmx_sync_dirty_debug_regs, 8049 .cache_reg = vmx_cache_reg, 8050 .get_rflags = vmx_get_rflags, 8051 .set_rflags = vmx_set_rflags, 8052 .get_if_flag = vmx_get_if_flag, 8053 8054 .flush_tlb_all = vmx_flush_tlb_all, 8055 .flush_tlb_current = vmx_flush_tlb_current, 8056 .flush_tlb_gva = vmx_flush_tlb_gva, 8057 .flush_tlb_guest = vmx_flush_tlb_guest, 8058 8059 .vcpu_pre_run = vmx_vcpu_pre_run, 8060 .vcpu_run = vmx_vcpu_run, 8061 .handle_exit = vmx_handle_exit, 8062 .skip_emulated_instruction = vmx_skip_emulated_instruction, 8063 .update_emulated_instruction = vmx_update_emulated_instruction, 8064 .set_interrupt_shadow = vmx_set_interrupt_shadow, 8065 .get_interrupt_shadow = vmx_get_interrupt_shadow, 8066 .patch_hypercall = vmx_patch_hypercall, 8067 .inject_irq = vmx_inject_irq, 8068 .inject_nmi = vmx_inject_nmi, 8069 .inject_exception = vmx_inject_exception, 8070 .cancel_injection = vmx_cancel_injection, 8071 .interrupt_allowed = vmx_interrupt_allowed, 8072 .nmi_allowed = vmx_nmi_allowed, 8073 .get_nmi_mask = vmx_get_nmi_mask, 8074 .set_nmi_mask = vmx_set_nmi_mask, 8075 .enable_nmi_window = vmx_enable_nmi_window, 8076 .enable_irq_window = vmx_enable_irq_window, 8077 .update_cr8_intercept = vmx_update_cr8_intercept, 8078 .set_virtual_apic_mode = vmx_set_virtual_apic_mode, 8079 .set_apic_access_page_addr = vmx_set_apic_access_page_addr, 8080 .refresh_apicv_exec_ctrl = vmx_refresh_apicv_exec_ctrl, 8081 .load_eoi_exitmap = vmx_load_eoi_exitmap, 8082 .apicv_post_state_restore = vmx_apicv_post_state_restore, 8083 .check_apicv_inhibit_reasons = vmx_check_apicv_inhibit_reasons, 8084 .hwapic_irr_update = vmx_hwapic_irr_update, 8085 .hwapic_isr_update = vmx_hwapic_isr_update, 8086 .guest_apic_has_interrupt = vmx_guest_apic_has_interrupt, 8087 .sync_pir_to_irr = vmx_sync_pir_to_irr, 8088 .deliver_interrupt = vmx_deliver_interrupt, 8089 .dy_apicv_has_pending_interrupt = pi_has_pending_interrupt, 8090 8091 .set_tss_addr = vmx_set_tss_addr, 8092 .set_identity_map_addr = vmx_set_identity_map_addr, 8093 .get_mt_mask = vmx_get_mt_mask, 8094 8095 .get_exit_info = vmx_get_exit_info, 8096 8097 .vcpu_after_set_cpuid = vmx_vcpu_after_set_cpuid, 8098 8099 .has_wbinvd_exit = cpu_has_vmx_wbinvd_exit, 8100 8101 .get_l2_tsc_offset = vmx_get_l2_tsc_offset, 8102 .get_l2_tsc_multiplier = vmx_get_l2_tsc_multiplier, 8103 .write_tsc_offset = vmx_write_tsc_offset, 8104 .write_tsc_multiplier = vmx_write_tsc_multiplier, 8105 8106 .load_mmu_pgd = vmx_load_mmu_pgd, 8107 8108 .check_intercept = vmx_check_intercept, 8109 .handle_exit_irqoff = vmx_handle_exit_irqoff, 8110 8111 .request_immediate_exit = vmx_request_immediate_exit, 8112 8113 .sched_in = vmx_sched_in, 8114 8115 .cpu_dirty_log_size = PML_ENTITY_NUM, 8116 .update_cpu_dirty_logging = vmx_update_cpu_dirty_logging, 8117 8118 .nested_ops = &vmx_nested_ops, 8119 8120 .pi_update_irte = vmx_pi_update_irte, 8121 .pi_start_assignment = vmx_pi_start_assignment, 8122 8123 #ifdef CONFIG_X86_64 8124 .set_hv_timer = vmx_set_hv_timer, 8125 .cancel_hv_timer = vmx_cancel_hv_timer, 8126 #endif 8127 8128 .setup_mce = vmx_setup_mce, 8129 8130 .smi_allowed = vmx_smi_allowed, 8131 .enter_smm = vmx_enter_smm, 8132 .leave_smm = vmx_leave_smm, 8133 .enable_smi_window = vmx_enable_smi_window, 8134 8135 .can_emulate_instruction = vmx_can_emulate_instruction, 8136 .apic_init_signal_blocked = vmx_apic_init_signal_blocked, 8137 .migrate_timers = vmx_migrate_timers, 8138 8139 .msr_filter_changed = vmx_msr_filter_changed, 8140 .complete_emulated_msr = kvm_complete_insn_gp, 8141 8142 .vcpu_deliver_sipi_vector = kvm_vcpu_deliver_sipi_vector, 8143 }; 8144 8145 static unsigned int vmx_handle_intel_pt_intr(void) 8146 { 8147 struct kvm_vcpu *vcpu = kvm_get_running_vcpu(); 8148 8149 /* '0' on failure so that the !PT case can use a RET0 static call. */ 8150 if (!vcpu || !kvm_handling_nmi_from_guest(vcpu)) 8151 return 0; 8152 8153 kvm_make_request(KVM_REQ_PMI, vcpu); 8154 __set_bit(MSR_CORE_PERF_GLOBAL_OVF_CTRL_TRACE_TOPA_PMI_BIT, 8155 (unsigned long *)&vcpu->arch.pmu.global_status); 8156 return 1; 8157 } 8158 8159 static __init void vmx_setup_user_return_msrs(void) 8160 { 8161 8162 /* 8163 * Though SYSCALL is only supported in 64-bit mode on Intel CPUs, kvm 8164 * will emulate SYSCALL in legacy mode if the vendor string in guest 8165 * CPUID.0:{EBX,ECX,EDX} is "AuthenticAMD" or "AMDisbetter!" To 8166 * support this emulation, MSR_STAR is included in the list for i386, 8167 * but is never loaded into hardware. MSR_CSTAR is also never loaded 8168 * into hardware and is here purely for emulation purposes. 8169 */ 8170 const u32 vmx_uret_msrs_list[] = { 8171 #ifdef CONFIG_X86_64 8172 MSR_SYSCALL_MASK, MSR_LSTAR, MSR_CSTAR, 8173 #endif 8174 MSR_EFER, MSR_TSC_AUX, MSR_STAR, 8175 MSR_IA32_TSX_CTRL, 8176 }; 8177 int i; 8178 8179 BUILD_BUG_ON(ARRAY_SIZE(vmx_uret_msrs_list) != MAX_NR_USER_RETURN_MSRS); 8180 8181 for (i = 0; i < ARRAY_SIZE(vmx_uret_msrs_list); ++i) 8182 kvm_add_user_return_msr(vmx_uret_msrs_list[i]); 8183 } 8184 8185 static void __init vmx_setup_me_spte_mask(void) 8186 { 8187 u64 me_mask = 0; 8188 8189 /* 8190 * kvm_get_shadow_phys_bits() returns shadow_phys_bits. Use 8191 * the former to avoid exposing shadow_phys_bits. 8192 * 8193 * On pre-MKTME system, boot_cpu_data.x86_phys_bits equals to 8194 * shadow_phys_bits. On MKTME and/or TDX capable systems, 8195 * boot_cpu_data.x86_phys_bits holds the actual physical address 8196 * w/o the KeyID bits, and shadow_phys_bits equals to MAXPHYADDR 8197 * reported by CPUID. Those bits between are KeyID bits. 8198 */ 8199 if (boot_cpu_data.x86_phys_bits != kvm_get_shadow_phys_bits()) 8200 me_mask = rsvd_bits(boot_cpu_data.x86_phys_bits, 8201 kvm_get_shadow_phys_bits() - 1); 8202 /* 8203 * Unlike SME, host kernel doesn't support setting up any 8204 * MKTME KeyID on Intel platforms. No memory encryption 8205 * bits should be included into the SPTE. 8206 */ 8207 kvm_mmu_set_me_spte_mask(0, me_mask); 8208 } 8209 8210 static struct kvm_x86_init_ops vmx_init_ops __initdata; 8211 8212 static __init int hardware_setup(void) 8213 { 8214 unsigned long host_bndcfgs; 8215 struct desc_ptr dt; 8216 int r; 8217 8218 store_idt(&dt); 8219 host_idt_base = dt.address; 8220 8221 vmx_setup_user_return_msrs(); 8222 8223 if (setup_vmcs_config(&vmcs_config, &vmx_capability) < 0) 8224 return -EIO; 8225 8226 if (cpu_has_perf_global_ctrl_bug()) 8227 pr_warn_once("kvm: VM_EXIT_LOAD_IA32_PERF_GLOBAL_CTRL " 8228 "does not work properly. Using workaround\n"); 8229 8230 if (boot_cpu_has(X86_FEATURE_NX)) 8231 kvm_enable_efer_bits(EFER_NX); 8232 8233 if (boot_cpu_has(X86_FEATURE_MPX)) { 8234 rdmsrl(MSR_IA32_BNDCFGS, host_bndcfgs); 8235 WARN_ONCE(host_bndcfgs, "KVM: BNDCFGS in host will be lost"); 8236 } 8237 8238 if (!cpu_has_vmx_mpx()) 8239 kvm_caps.supported_xcr0 &= ~(XFEATURE_MASK_BNDREGS | 8240 XFEATURE_MASK_BNDCSR); 8241 8242 if (!cpu_has_vmx_vpid() || !cpu_has_vmx_invvpid() || 8243 !(cpu_has_vmx_invvpid_single() || cpu_has_vmx_invvpid_global())) 8244 enable_vpid = 0; 8245 8246 if (!cpu_has_vmx_ept() || 8247 !cpu_has_vmx_ept_4levels() || 8248 !cpu_has_vmx_ept_mt_wb() || 8249 !cpu_has_vmx_invept_global()) 8250 enable_ept = 0; 8251 8252 /* NX support is required for shadow paging. */ 8253 if (!enable_ept && !boot_cpu_has(X86_FEATURE_NX)) { 8254 pr_err_ratelimited("kvm: NX (Execute Disable) not supported\n"); 8255 return -EOPNOTSUPP; 8256 } 8257 8258 if (!cpu_has_vmx_ept_ad_bits() || !enable_ept) 8259 enable_ept_ad_bits = 0; 8260 8261 if (!cpu_has_vmx_unrestricted_guest() || !enable_ept) 8262 enable_unrestricted_guest = 0; 8263 8264 if (!cpu_has_vmx_flexpriority()) 8265 flexpriority_enabled = 0; 8266 8267 if (!cpu_has_virtual_nmis()) 8268 enable_vnmi = 0; 8269 8270 #ifdef CONFIG_X86_SGX_KVM 8271 if (!cpu_has_vmx_encls_vmexit()) 8272 enable_sgx = false; 8273 #endif 8274 8275 /* 8276 * set_apic_access_page_addr() is used to reload apic access 8277 * page upon invalidation. No need to do anything if not 8278 * using the APIC_ACCESS_ADDR VMCS field. 8279 */ 8280 if (!flexpriority_enabled) 8281 vmx_x86_ops.set_apic_access_page_addr = NULL; 8282 8283 if (!cpu_has_vmx_tpr_shadow()) 8284 vmx_x86_ops.update_cr8_intercept = NULL; 8285 8286 #if IS_ENABLED(CONFIG_HYPERV) 8287 if (ms_hyperv.nested_features & HV_X64_NESTED_GUEST_MAPPING_FLUSH 8288 && enable_ept) { 8289 vmx_x86_ops.tlb_remote_flush = hv_remote_flush_tlb; 8290 vmx_x86_ops.tlb_remote_flush_with_range = 8291 hv_remote_flush_tlb_with_range; 8292 } 8293 #endif 8294 8295 if (!cpu_has_vmx_ple()) { 8296 ple_gap = 0; 8297 ple_window = 0; 8298 ple_window_grow = 0; 8299 ple_window_max = 0; 8300 ple_window_shrink = 0; 8301 } 8302 8303 if (!cpu_has_vmx_apicv()) 8304 enable_apicv = 0; 8305 if (!enable_apicv) 8306 vmx_x86_ops.sync_pir_to_irr = NULL; 8307 8308 if (!enable_apicv || !cpu_has_vmx_ipiv()) 8309 enable_ipiv = false; 8310 8311 if (cpu_has_vmx_tsc_scaling()) 8312 kvm_caps.has_tsc_control = true; 8313 8314 kvm_caps.max_tsc_scaling_ratio = KVM_VMX_TSC_MULTIPLIER_MAX; 8315 kvm_caps.tsc_scaling_ratio_frac_bits = 48; 8316 kvm_caps.has_bus_lock_exit = cpu_has_vmx_bus_lock_detection(); 8317 kvm_caps.has_notify_vmexit = cpu_has_notify_vmexit(); 8318 8319 set_bit(0, vmx_vpid_bitmap); /* 0 is reserved for host */ 8320 8321 if (enable_ept) 8322 kvm_mmu_set_ept_masks(enable_ept_ad_bits, 8323 cpu_has_vmx_ept_execute_only()); 8324 8325 /* 8326 * Setup shadow_me_value/shadow_me_mask to include MKTME KeyID 8327 * bits to shadow_zero_check. 8328 */ 8329 vmx_setup_me_spte_mask(); 8330 8331 kvm_configure_mmu(enable_ept, 0, vmx_get_max_tdp_level(), 8332 ept_caps_to_lpage_level(vmx_capability.ept)); 8333 8334 /* 8335 * Only enable PML when hardware supports PML feature, and both EPT 8336 * and EPT A/D bit features are enabled -- PML depends on them to work. 8337 */ 8338 if (!enable_ept || !enable_ept_ad_bits || !cpu_has_vmx_pml()) 8339 enable_pml = 0; 8340 8341 if (!enable_pml) 8342 vmx_x86_ops.cpu_dirty_log_size = 0; 8343 8344 if (!cpu_has_vmx_preemption_timer()) 8345 enable_preemption_timer = false; 8346 8347 if (enable_preemption_timer) { 8348 u64 use_timer_freq = 5000ULL * 1000 * 1000; 8349 8350 cpu_preemption_timer_multi = 8351 vmcs_config.misc & VMX_MISC_PREEMPTION_TIMER_RATE_MASK; 8352 8353 if (tsc_khz) 8354 use_timer_freq = (u64)tsc_khz * 1000; 8355 use_timer_freq >>= cpu_preemption_timer_multi; 8356 8357 /* 8358 * KVM "disables" the preemption timer by setting it to its max 8359 * value. Don't use the timer if it might cause spurious exits 8360 * at a rate faster than 0.1 Hz (of uninterrupted guest time). 8361 */ 8362 if (use_timer_freq > 0xffffffffu / 10) 8363 enable_preemption_timer = false; 8364 } 8365 8366 if (!enable_preemption_timer) { 8367 vmx_x86_ops.set_hv_timer = NULL; 8368 vmx_x86_ops.cancel_hv_timer = NULL; 8369 vmx_x86_ops.request_immediate_exit = __kvm_request_immediate_exit; 8370 } 8371 8372 kvm_caps.supported_mce_cap |= MCG_LMCE_P; 8373 kvm_caps.supported_mce_cap |= MCG_CMCI_P; 8374 8375 if (pt_mode != PT_MODE_SYSTEM && pt_mode != PT_MODE_HOST_GUEST) 8376 return -EINVAL; 8377 if (!enable_ept || !enable_pmu || !cpu_has_vmx_intel_pt()) 8378 pt_mode = PT_MODE_SYSTEM; 8379 if (pt_mode == PT_MODE_HOST_GUEST) 8380 vmx_init_ops.handle_intel_pt_intr = vmx_handle_intel_pt_intr; 8381 else 8382 vmx_init_ops.handle_intel_pt_intr = NULL; 8383 8384 setup_default_sgx_lepubkeyhash(); 8385 8386 if (nested) { 8387 nested_vmx_setup_ctls_msrs(&vmcs_config, vmx_capability.ept); 8388 8389 r = nested_vmx_hardware_setup(kvm_vmx_exit_handlers); 8390 if (r) 8391 return r; 8392 } 8393 8394 vmx_set_cpu_caps(); 8395 8396 r = alloc_kvm_area(); 8397 if (r && nested) 8398 nested_vmx_hardware_unsetup(); 8399 8400 kvm_set_posted_intr_wakeup_handler(pi_wakeup_handler); 8401 8402 return r; 8403 } 8404 8405 static struct kvm_x86_init_ops vmx_init_ops __initdata = { 8406 .cpu_has_kvm_support = cpu_has_kvm_support, 8407 .disabled_by_bios = vmx_disabled_by_bios, 8408 .check_processor_compatibility = vmx_check_processor_compat, 8409 .hardware_setup = hardware_setup, 8410 .handle_intel_pt_intr = NULL, 8411 8412 .runtime_ops = &vmx_x86_ops, 8413 .pmu_ops = &intel_pmu_ops, 8414 }; 8415 8416 static void vmx_cleanup_l1d_flush(void) 8417 { 8418 if (vmx_l1d_flush_pages) { 8419 free_pages((unsigned long)vmx_l1d_flush_pages, L1D_CACHE_ORDER); 8420 vmx_l1d_flush_pages = NULL; 8421 } 8422 /* Restore state so sysfs ignores VMX */ 8423 l1tf_vmx_mitigation = VMENTER_L1D_FLUSH_AUTO; 8424 } 8425 8426 static void vmx_exit(void) 8427 { 8428 #ifdef CONFIG_KEXEC_CORE 8429 RCU_INIT_POINTER(crash_vmclear_loaded_vmcss, NULL); 8430 synchronize_rcu(); 8431 #endif 8432 8433 kvm_exit(); 8434 8435 #if IS_ENABLED(CONFIG_HYPERV) 8436 if (static_branch_unlikely(&enable_evmcs)) { 8437 int cpu; 8438 struct hv_vp_assist_page *vp_ap; 8439 /* 8440 * Reset everything to support using non-enlightened VMCS 8441 * access later (e.g. when we reload the module with 8442 * enlightened_vmcs=0) 8443 */ 8444 for_each_online_cpu(cpu) { 8445 vp_ap = hv_get_vp_assist_page(cpu); 8446 8447 if (!vp_ap) 8448 continue; 8449 8450 vp_ap->nested_control.features.directhypercall = 0; 8451 vp_ap->current_nested_vmcs = 0; 8452 vp_ap->enlighten_vmentry = 0; 8453 } 8454 8455 static_branch_disable(&enable_evmcs); 8456 } 8457 #endif 8458 vmx_cleanup_l1d_flush(); 8459 8460 allow_smaller_maxphyaddr = false; 8461 } 8462 module_exit(vmx_exit); 8463 8464 static int __init vmx_init(void) 8465 { 8466 int r, cpu; 8467 8468 #if IS_ENABLED(CONFIG_HYPERV) 8469 /* 8470 * Enlightened VMCS usage should be recommended and the host needs 8471 * to support eVMCS v1 or above. We can also disable eVMCS support 8472 * with module parameter. 8473 */ 8474 if (enlightened_vmcs && 8475 ms_hyperv.hints & HV_X64_ENLIGHTENED_VMCS_RECOMMENDED && 8476 (ms_hyperv.nested_features & HV_X64_ENLIGHTENED_VMCS_VERSION) >= 8477 KVM_EVMCS_VERSION) { 8478 8479 /* Check that we have assist pages on all online CPUs */ 8480 for_each_online_cpu(cpu) { 8481 if (!hv_get_vp_assist_page(cpu)) { 8482 enlightened_vmcs = false; 8483 break; 8484 } 8485 } 8486 8487 if (enlightened_vmcs) { 8488 pr_info("KVM: vmx: using Hyper-V Enlightened VMCS\n"); 8489 static_branch_enable(&enable_evmcs); 8490 } 8491 8492 if (ms_hyperv.nested_features & HV_X64_NESTED_DIRECT_FLUSH) 8493 vmx_x86_ops.enable_direct_tlbflush 8494 = hv_enable_direct_tlbflush; 8495 8496 } else { 8497 enlightened_vmcs = false; 8498 } 8499 #endif 8500 8501 r = kvm_init(&vmx_init_ops, sizeof(struct vcpu_vmx), 8502 __alignof__(struct vcpu_vmx), THIS_MODULE); 8503 if (r) 8504 return r; 8505 8506 /* 8507 * Must be called after kvm_init() so enable_ept is properly set 8508 * up. Hand the parameter mitigation value in which was stored in 8509 * the pre module init parser. If no parameter was given, it will 8510 * contain 'auto' which will be turned into the default 'cond' 8511 * mitigation mode. 8512 */ 8513 r = vmx_setup_l1d_flush(vmentry_l1d_flush_param); 8514 if (r) { 8515 vmx_exit(); 8516 return r; 8517 } 8518 8519 vmx_setup_fb_clear_ctrl(); 8520 8521 for_each_possible_cpu(cpu) { 8522 INIT_LIST_HEAD(&per_cpu(loaded_vmcss_on_cpu, cpu)); 8523 8524 pi_init_cpu(cpu); 8525 } 8526 8527 #ifdef CONFIG_KEXEC_CORE 8528 rcu_assign_pointer(crash_vmclear_loaded_vmcss, 8529 crash_vmclear_local_loaded_vmcss); 8530 #endif 8531 vmx_check_vmcs12_offsets(); 8532 8533 /* 8534 * Shadow paging doesn't have a (further) performance penalty 8535 * from GUEST_MAXPHYADDR < HOST_MAXPHYADDR so enable it 8536 * by default 8537 */ 8538 if (!enable_ept) 8539 allow_smaller_maxphyaddr = true; 8540 8541 return 0; 8542 } 8543 module_init(vmx_init); 8544