1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * Kernel-based Virtual Machine driver for Linux 4 * 5 * This module enables machines with Intel VT-x extensions to run virtual 6 * machines without emulation or binary translation. 7 * 8 * Copyright (C) 2006 Qumranet, Inc. 9 * Copyright 2010 Red Hat, Inc. and/or its affiliates. 10 * 11 * Authors: 12 * Avi Kivity <avi@qumranet.com> 13 * Yaniv Kamay <yaniv@qumranet.com> 14 */ 15 16 #include <linux/highmem.h> 17 #include <linux/hrtimer.h> 18 #include <linux/kernel.h> 19 #include <linux/kvm_host.h> 20 #include <linux/module.h> 21 #include <linux/moduleparam.h> 22 #include <linux/mod_devicetable.h> 23 #include <linux/mm.h> 24 #include <linux/objtool.h> 25 #include <linux/sched.h> 26 #include <linux/sched/smt.h> 27 #include <linux/slab.h> 28 #include <linux/tboot.h> 29 #include <linux/trace_events.h> 30 #include <linux/entry-kvm.h> 31 32 #include <asm/apic.h> 33 #include <asm/asm.h> 34 #include <asm/cpu.h> 35 #include <asm/cpu_device_id.h> 36 #include <asm/debugreg.h> 37 #include <asm/desc.h> 38 #include <asm/fpu/api.h> 39 #include <asm/fpu/xstate.h> 40 #include <asm/idtentry.h> 41 #include <asm/io.h> 42 #include <asm/irq_remapping.h> 43 #include <asm/kexec.h> 44 #include <asm/perf_event.h> 45 #include <asm/mmu_context.h> 46 #include <asm/mshyperv.h> 47 #include <asm/mwait.h> 48 #include <asm/spec-ctrl.h> 49 #include <asm/virtext.h> 50 #include <asm/vmx.h> 51 52 #include "capabilities.h" 53 #include "cpuid.h" 54 #include "evmcs.h" 55 #include "hyperv.h" 56 #include "kvm_onhyperv.h" 57 #include "irq.h" 58 #include "kvm_cache_regs.h" 59 #include "lapic.h" 60 #include "mmu.h" 61 #include "nested.h" 62 #include "pmu.h" 63 #include "sgx.h" 64 #include "trace.h" 65 #include "vmcs.h" 66 #include "vmcs12.h" 67 #include "vmx.h" 68 #include "x86.h" 69 70 MODULE_AUTHOR("Qumranet"); 71 MODULE_LICENSE("GPL"); 72 73 #ifdef MODULE 74 static const struct x86_cpu_id vmx_cpu_id[] = { 75 X86_MATCH_FEATURE(X86_FEATURE_VMX, NULL), 76 {} 77 }; 78 MODULE_DEVICE_TABLE(x86cpu, vmx_cpu_id); 79 #endif 80 81 bool __read_mostly enable_vpid = 1; 82 module_param_named(vpid, enable_vpid, bool, 0444); 83 84 static bool __read_mostly enable_vnmi = 1; 85 module_param_named(vnmi, enable_vnmi, bool, S_IRUGO); 86 87 bool __read_mostly flexpriority_enabled = 1; 88 module_param_named(flexpriority, flexpriority_enabled, bool, S_IRUGO); 89 90 bool __read_mostly enable_ept = 1; 91 module_param_named(ept, enable_ept, bool, S_IRUGO); 92 93 bool __read_mostly enable_unrestricted_guest = 1; 94 module_param_named(unrestricted_guest, 95 enable_unrestricted_guest, bool, S_IRUGO); 96 97 bool __read_mostly enable_ept_ad_bits = 1; 98 module_param_named(eptad, enable_ept_ad_bits, bool, S_IRUGO); 99 100 static bool __read_mostly emulate_invalid_guest_state = true; 101 module_param(emulate_invalid_guest_state, bool, S_IRUGO); 102 103 static bool __read_mostly fasteoi = 1; 104 module_param(fasteoi, bool, S_IRUGO); 105 106 module_param(enable_apicv, bool, S_IRUGO); 107 108 /* 109 * If nested=1, nested virtualization is supported, i.e., guests may use 110 * VMX and be a hypervisor for its own guests. If nested=0, guests may not 111 * use VMX instructions. 112 */ 113 static bool __read_mostly nested = 1; 114 module_param(nested, bool, S_IRUGO); 115 116 bool __read_mostly enable_pml = 1; 117 module_param_named(pml, enable_pml, bool, S_IRUGO); 118 119 static bool __read_mostly dump_invalid_vmcs = 0; 120 module_param(dump_invalid_vmcs, bool, 0644); 121 122 #define MSR_BITMAP_MODE_X2APIC 1 123 #define MSR_BITMAP_MODE_X2APIC_APICV 2 124 125 #define KVM_VMX_TSC_MULTIPLIER_MAX 0xffffffffffffffffULL 126 127 /* Guest_tsc -> host_tsc conversion requires 64-bit division. */ 128 static int __read_mostly cpu_preemption_timer_multi; 129 static bool __read_mostly enable_preemption_timer = 1; 130 #ifdef CONFIG_X86_64 131 module_param_named(preemption_timer, enable_preemption_timer, bool, S_IRUGO); 132 #endif 133 134 extern bool __read_mostly allow_smaller_maxphyaddr; 135 module_param(allow_smaller_maxphyaddr, bool, S_IRUGO); 136 137 #define KVM_VM_CR0_ALWAYS_OFF (X86_CR0_NW | X86_CR0_CD) 138 #define KVM_VM_CR0_ALWAYS_ON_UNRESTRICTED_GUEST X86_CR0_NE 139 #define KVM_VM_CR0_ALWAYS_ON \ 140 (KVM_VM_CR0_ALWAYS_ON_UNRESTRICTED_GUEST | X86_CR0_PG | X86_CR0_PE) 141 142 #define KVM_VM_CR4_ALWAYS_ON_UNRESTRICTED_GUEST X86_CR4_VMXE 143 #define KVM_PMODE_VM_CR4_ALWAYS_ON (X86_CR4_PAE | X86_CR4_VMXE) 144 #define KVM_RMODE_VM_CR4_ALWAYS_ON (X86_CR4_VME | X86_CR4_PAE | X86_CR4_VMXE) 145 146 #define RMODE_GUEST_OWNED_EFLAGS_BITS (~(X86_EFLAGS_IOPL | X86_EFLAGS_VM)) 147 148 #define MSR_IA32_RTIT_STATUS_MASK (~(RTIT_STATUS_FILTEREN | \ 149 RTIT_STATUS_CONTEXTEN | RTIT_STATUS_TRIGGEREN | \ 150 RTIT_STATUS_ERROR | RTIT_STATUS_STOPPED | \ 151 RTIT_STATUS_BYTECNT)) 152 153 /* 154 * List of MSRs that can be directly passed to the guest. 155 * In addition to these x2apic and PT MSRs are handled specially. 156 */ 157 static u32 vmx_possible_passthrough_msrs[MAX_POSSIBLE_PASSTHROUGH_MSRS] = { 158 MSR_IA32_SPEC_CTRL, 159 MSR_IA32_PRED_CMD, 160 MSR_IA32_TSC, 161 #ifdef CONFIG_X86_64 162 MSR_FS_BASE, 163 MSR_GS_BASE, 164 MSR_KERNEL_GS_BASE, 165 MSR_IA32_XFD, 166 MSR_IA32_XFD_ERR, 167 #endif 168 MSR_IA32_SYSENTER_CS, 169 MSR_IA32_SYSENTER_ESP, 170 MSR_IA32_SYSENTER_EIP, 171 MSR_CORE_C1_RES, 172 MSR_CORE_C3_RESIDENCY, 173 MSR_CORE_C6_RESIDENCY, 174 MSR_CORE_C7_RESIDENCY, 175 }; 176 177 /* 178 * These 2 parameters are used to config the controls for Pause-Loop Exiting: 179 * ple_gap: upper bound on the amount of time between two successive 180 * executions of PAUSE in a loop. Also indicate if ple enabled. 181 * According to test, this time is usually smaller than 128 cycles. 182 * ple_window: upper bound on the amount of time a guest is allowed to execute 183 * in a PAUSE loop. Tests indicate that most spinlocks are held for 184 * less than 2^12 cycles 185 * Time is measured based on a counter that runs at the same rate as the TSC, 186 * refer SDM volume 3b section 21.6.13 & 22.1.3. 187 */ 188 static unsigned int ple_gap = KVM_DEFAULT_PLE_GAP; 189 module_param(ple_gap, uint, 0444); 190 191 static unsigned int ple_window = KVM_VMX_DEFAULT_PLE_WINDOW; 192 module_param(ple_window, uint, 0444); 193 194 /* Default doubles per-vcpu window every exit. */ 195 static unsigned int ple_window_grow = KVM_DEFAULT_PLE_WINDOW_GROW; 196 module_param(ple_window_grow, uint, 0444); 197 198 /* Default resets per-vcpu window every exit to ple_window. */ 199 static unsigned int ple_window_shrink = KVM_DEFAULT_PLE_WINDOW_SHRINK; 200 module_param(ple_window_shrink, uint, 0444); 201 202 /* Default is to compute the maximum so we can never overflow. */ 203 static unsigned int ple_window_max = KVM_VMX_DEFAULT_PLE_WINDOW_MAX; 204 module_param(ple_window_max, uint, 0444); 205 206 /* Default is SYSTEM mode, 1 for host-guest mode */ 207 int __read_mostly pt_mode = PT_MODE_SYSTEM; 208 module_param(pt_mode, int, S_IRUGO); 209 210 static DEFINE_STATIC_KEY_FALSE(vmx_l1d_should_flush); 211 static DEFINE_STATIC_KEY_FALSE(vmx_l1d_flush_cond); 212 static DEFINE_MUTEX(vmx_l1d_flush_mutex); 213 214 /* Storage for pre module init parameter parsing */ 215 static enum vmx_l1d_flush_state __read_mostly vmentry_l1d_flush_param = VMENTER_L1D_FLUSH_AUTO; 216 217 static const struct { 218 const char *option; 219 bool for_parse; 220 } vmentry_l1d_param[] = { 221 [VMENTER_L1D_FLUSH_AUTO] = {"auto", true}, 222 [VMENTER_L1D_FLUSH_NEVER] = {"never", true}, 223 [VMENTER_L1D_FLUSH_COND] = {"cond", true}, 224 [VMENTER_L1D_FLUSH_ALWAYS] = {"always", true}, 225 [VMENTER_L1D_FLUSH_EPT_DISABLED] = {"EPT disabled", false}, 226 [VMENTER_L1D_FLUSH_NOT_REQUIRED] = {"not required", false}, 227 }; 228 229 #define L1D_CACHE_ORDER 4 230 static void *vmx_l1d_flush_pages; 231 232 static int vmx_setup_l1d_flush(enum vmx_l1d_flush_state l1tf) 233 { 234 struct page *page; 235 unsigned int i; 236 237 if (!boot_cpu_has_bug(X86_BUG_L1TF)) { 238 l1tf_vmx_mitigation = VMENTER_L1D_FLUSH_NOT_REQUIRED; 239 return 0; 240 } 241 242 if (!enable_ept) { 243 l1tf_vmx_mitigation = VMENTER_L1D_FLUSH_EPT_DISABLED; 244 return 0; 245 } 246 247 if (boot_cpu_has(X86_FEATURE_ARCH_CAPABILITIES)) { 248 u64 msr; 249 250 rdmsrl(MSR_IA32_ARCH_CAPABILITIES, msr); 251 if (msr & ARCH_CAP_SKIP_VMENTRY_L1DFLUSH) { 252 l1tf_vmx_mitigation = VMENTER_L1D_FLUSH_NOT_REQUIRED; 253 return 0; 254 } 255 } 256 257 /* If set to auto use the default l1tf mitigation method */ 258 if (l1tf == VMENTER_L1D_FLUSH_AUTO) { 259 switch (l1tf_mitigation) { 260 case L1TF_MITIGATION_OFF: 261 l1tf = VMENTER_L1D_FLUSH_NEVER; 262 break; 263 case L1TF_MITIGATION_FLUSH_NOWARN: 264 case L1TF_MITIGATION_FLUSH: 265 case L1TF_MITIGATION_FLUSH_NOSMT: 266 l1tf = VMENTER_L1D_FLUSH_COND; 267 break; 268 case L1TF_MITIGATION_FULL: 269 case L1TF_MITIGATION_FULL_FORCE: 270 l1tf = VMENTER_L1D_FLUSH_ALWAYS; 271 break; 272 } 273 } else if (l1tf_mitigation == L1TF_MITIGATION_FULL_FORCE) { 274 l1tf = VMENTER_L1D_FLUSH_ALWAYS; 275 } 276 277 if (l1tf != VMENTER_L1D_FLUSH_NEVER && !vmx_l1d_flush_pages && 278 !boot_cpu_has(X86_FEATURE_FLUSH_L1D)) { 279 /* 280 * This allocation for vmx_l1d_flush_pages is not tied to a VM 281 * lifetime and so should not be charged to a memcg. 282 */ 283 page = alloc_pages(GFP_KERNEL, L1D_CACHE_ORDER); 284 if (!page) 285 return -ENOMEM; 286 vmx_l1d_flush_pages = page_address(page); 287 288 /* 289 * Initialize each page with a different pattern in 290 * order to protect against KSM in the nested 291 * virtualization case. 292 */ 293 for (i = 0; i < 1u << L1D_CACHE_ORDER; ++i) { 294 memset(vmx_l1d_flush_pages + i * PAGE_SIZE, i + 1, 295 PAGE_SIZE); 296 } 297 } 298 299 l1tf_vmx_mitigation = l1tf; 300 301 if (l1tf != VMENTER_L1D_FLUSH_NEVER) 302 static_branch_enable(&vmx_l1d_should_flush); 303 else 304 static_branch_disable(&vmx_l1d_should_flush); 305 306 if (l1tf == VMENTER_L1D_FLUSH_COND) 307 static_branch_enable(&vmx_l1d_flush_cond); 308 else 309 static_branch_disable(&vmx_l1d_flush_cond); 310 return 0; 311 } 312 313 static int vmentry_l1d_flush_parse(const char *s) 314 { 315 unsigned int i; 316 317 if (s) { 318 for (i = 0; i < ARRAY_SIZE(vmentry_l1d_param); i++) { 319 if (vmentry_l1d_param[i].for_parse && 320 sysfs_streq(s, vmentry_l1d_param[i].option)) 321 return i; 322 } 323 } 324 return -EINVAL; 325 } 326 327 static int vmentry_l1d_flush_set(const char *s, const struct kernel_param *kp) 328 { 329 int l1tf, ret; 330 331 l1tf = vmentry_l1d_flush_parse(s); 332 if (l1tf < 0) 333 return l1tf; 334 335 if (!boot_cpu_has(X86_BUG_L1TF)) 336 return 0; 337 338 /* 339 * Has vmx_init() run already? If not then this is the pre init 340 * parameter parsing. In that case just store the value and let 341 * vmx_init() do the proper setup after enable_ept has been 342 * established. 343 */ 344 if (l1tf_vmx_mitigation == VMENTER_L1D_FLUSH_AUTO) { 345 vmentry_l1d_flush_param = l1tf; 346 return 0; 347 } 348 349 mutex_lock(&vmx_l1d_flush_mutex); 350 ret = vmx_setup_l1d_flush(l1tf); 351 mutex_unlock(&vmx_l1d_flush_mutex); 352 return ret; 353 } 354 355 static int vmentry_l1d_flush_get(char *s, const struct kernel_param *kp) 356 { 357 if (WARN_ON_ONCE(l1tf_vmx_mitigation >= ARRAY_SIZE(vmentry_l1d_param))) 358 return sprintf(s, "???\n"); 359 360 return sprintf(s, "%s\n", vmentry_l1d_param[l1tf_vmx_mitigation].option); 361 } 362 363 static const struct kernel_param_ops vmentry_l1d_flush_ops = { 364 .set = vmentry_l1d_flush_set, 365 .get = vmentry_l1d_flush_get, 366 }; 367 module_param_cb(vmentry_l1d_flush, &vmentry_l1d_flush_ops, NULL, 0644); 368 369 static u32 vmx_segment_access_rights(struct kvm_segment *var); 370 371 void vmx_vmexit(void); 372 373 #define vmx_insn_failed(fmt...) \ 374 do { \ 375 WARN_ONCE(1, fmt); \ 376 pr_warn_ratelimited(fmt); \ 377 } while (0) 378 379 asmlinkage void vmread_error(unsigned long field, bool fault) 380 { 381 if (fault) 382 kvm_spurious_fault(); 383 else 384 vmx_insn_failed("kvm: vmread failed: field=%lx\n", field); 385 } 386 387 noinline void vmwrite_error(unsigned long field, unsigned long value) 388 { 389 vmx_insn_failed("kvm: vmwrite failed: field=%lx val=%lx err=%d\n", 390 field, value, vmcs_read32(VM_INSTRUCTION_ERROR)); 391 } 392 393 noinline void vmclear_error(struct vmcs *vmcs, u64 phys_addr) 394 { 395 vmx_insn_failed("kvm: vmclear failed: %p/%llx\n", vmcs, phys_addr); 396 } 397 398 noinline void vmptrld_error(struct vmcs *vmcs, u64 phys_addr) 399 { 400 vmx_insn_failed("kvm: vmptrld failed: %p/%llx\n", vmcs, phys_addr); 401 } 402 403 noinline void invvpid_error(unsigned long ext, u16 vpid, gva_t gva) 404 { 405 vmx_insn_failed("kvm: invvpid failed: ext=0x%lx vpid=%u gva=0x%lx\n", 406 ext, vpid, gva); 407 } 408 409 noinline void invept_error(unsigned long ext, u64 eptp, gpa_t gpa) 410 { 411 vmx_insn_failed("kvm: invept failed: ext=0x%lx eptp=%llx gpa=0x%llx\n", 412 ext, eptp, gpa); 413 } 414 415 static DEFINE_PER_CPU(struct vmcs *, vmxarea); 416 DEFINE_PER_CPU(struct vmcs *, current_vmcs); 417 /* 418 * We maintain a per-CPU linked-list of VMCS loaded on that CPU. This is needed 419 * when a CPU is brought down, and we need to VMCLEAR all VMCSs loaded on it. 420 */ 421 static DEFINE_PER_CPU(struct list_head, loaded_vmcss_on_cpu); 422 423 static DECLARE_BITMAP(vmx_vpid_bitmap, VMX_NR_VPIDS); 424 static DEFINE_SPINLOCK(vmx_vpid_lock); 425 426 struct vmcs_config vmcs_config; 427 struct vmx_capability vmx_capability; 428 429 #define VMX_SEGMENT_FIELD(seg) \ 430 [VCPU_SREG_##seg] = { \ 431 .selector = GUEST_##seg##_SELECTOR, \ 432 .base = GUEST_##seg##_BASE, \ 433 .limit = GUEST_##seg##_LIMIT, \ 434 .ar_bytes = GUEST_##seg##_AR_BYTES, \ 435 } 436 437 static const struct kvm_vmx_segment_field { 438 unsigned selector; 439 unsigned base; 440 unsigned limit; 441 unsigned ar_bytes; 442 } kvm_vmx_segment_fields[] = { 443 VMX_SEGMENT_FIELD(CS), 444 VMX_SEGMENT_FIELD(DS), 445 VMX_SEGMENT_FIELD(ES), 446 VMX_SEGMENT_FIELD(FS), 447 VMX_SEGMENT_FIELD(GS), 448 VMX_SEGMENT_FIELD(SS), 449 VMX_SEGMENT_FIELD(TR), 450 VMX_SEGMENT_FIELD(LDTR), 451 }; 452 453 static inline void vmx_segment_cache_clear(struct vcpu_vmx *vmx) 454 { 455 vmx->segment_cache.bitmask = 0; 456 } 457 458 static unsigned long host_idt_base; 459 460 #if IS_ENABLED(CONFIG_HYPERV) 461 static bool __read_mostly enlightened_vmcs = true; 462 module_param(enlightened_vmcs, bool, 0444); 463 464 static int hv_enable_direct_tlbflush(struct kvm_vcpu *vcpu) 465 { 466 struct hv_enlightened_vmcs *evmcs; 467 struct hv_partition_assist_pg **p_hv_pa_pg = 468 &to_kvm_hv(vcpu->kvm)->hv_pa_pg; 469 /* 470 * Synthetic VM-Exit is not enabled in current code and so All 471 * evmcs in singe VM shares same assist page. 472 */ 473 if (!*p_hv_pa_pg) 474 *p_hv_pa_pg = kzalloc(PAGE_SIZE, GFP_KERNEL_ACCOUNT); 475 476 if (!*p_hv_pa_pg) 477 return -ENOMEM; 478 479 evmcs = (struct hv_enlightened_vmcs *)to_vmx(vcpu)->loaded_vmcs->vmcs; 480 481 evmcs->partition_assist_page = 482 __pa(*p_hv_pa_pg); 483 evmcs->hv_vm_id = (unsigned long)vcpu->kvm; 484 evmcs->hv_enlightenments_control.nested_flush_hypercall = 1; 485 486 return 0; 487 } 488 489 #endif /* IS_ENABLED(CONFIG_HYPERV) */ 490 491 /* 492 * Comment's format: document - errata name - stepping - processor name. 493 * Refer from 494 * https://www.virtualbox.org/svn/vbox/trunk/src/VBox/VMM/VMMR0/HMR0.cpp 495 */ 496 static u32 vmx_preemption_cpu_tfms[] = { 497 /* 323344.pdf - BA86 - D0 - Xeon 7500 Series */ 498 0x000206E6, 499 /* 323056.pdf - AAX65 - C2 - Xeon L3406 */ 500 /* 322814.pdf - AAT59 - C2 - i7-600, i5-500, i5-400 and i3-300 Mobile */ 501 /* 322911.pdf - AAU65 - C2 - i5-600, i3-500 Desktop and Pentium G6950 */ 502 0x00020652, 503 /* 322911.pdf - AAU65 - K0 - i5-600, i3-500 Desktop and Pentium G6950 */ 504 0x00020655, 505 /* 322373.pdf - AAO95 - B1 - Xeon 3400 Series */ 506 /* 322166.pdf - AAN92 - B1 - i7-800 and i5-700 Desktop */ 507 /* 508 * 320767.pdf - AAP86 - B1 - 509 * i7-900 Mobile Extreme, i7-800 and i7-700 Mobile 510 */ 511 0x000106E5, 512 /* 321333.pdf - AAM126 - C0 - Xeon 3500 */ 513 0x000106A0, 514 /* 321333.pdf - AAM126 - C1 - Xeon 3500 */ 515 0x000106A1, 516 /* 320836.pdf - AAJ124 - C0 - i7-900 Desktop Extreme and i7-900 Desktop */ 517 0x000106A4, 518 /* 321333.pdf - AAM126 - D0 - Xeon 3500 */ 519 /* 321324.pdf - AAK139 - D0 - Xeon 5500 */ 520 /* 320836.pdf - AAJ124 - D0 - i7-900 Extreme and i7-900 Desktop */ 521 0x000106A5, 522 /* Xeon E3-1220 V2 */ 523 0x000306A8, 524 }; 525 526 static inline bool cpu_has_broken_vmx_preemption_timer(void) 527 { 528 u32 eax = cpuid_eax(0x00000001), i; 529 530 /* Clear the reserved bits */ 531 eax &= ~(0x3U << 14 | 0xfU << 28); 532 for (i = 0; i < ARRAY_SIZE(vmx_preemption_cpu_tfms); i++) 533 if (eax == vmx_preemption_cpu_tfms[i]) 534 return true; 535 536 return false; 537 } 538 539 static inline bool cpu_need_virtualize_apic_accesses(struct kvm_vcpu *vcpu) 540 { 541 return flexpriority_enabled && lapic_in_kernel(vcpu); 542 } 543 544 static int possible_passthrough_msr_slot(u32 msr) 545 { 546 u32 i; 547 548 for (i = 0; i < ARRAY_SIZE(vmx_possible_passthrough_msrs); i++) 549 if (vmx_possible_passthrough_msrs[i] == msr) 550 return i; 551 552 return -ENOENT; 553 } 554 555 static bool is_valid_passthrough_msr(u32 msr) 556 { 557 bool r; 558 559 switch (msr) { 560 case 0x800 ... 0x8ff: 561 /* x2APIC MSRs. These are handled in vmx_update_msr_bitmap_x2apic() */ 562 return true; 563 case MSR_IA32_RTIT_STATUS: 564 case MSR_IA32_RTIT_OUTPUT_BASE: 565 case MSR_IA32_RTIT_OUTPUT_MASK: 566 case MSR_IA32_RTIT_CR3_MATCH: 567 case MSR_IA32_RTIT_ADDR0_A ... MSR_IA32_RTIT_ADDR3_B: 568 /* PT MSRs. These are handled in pt_update_intercept_for_msr() */ 569 case MSR_LBR_SELECT: 570 case MSR_LBR_TOS: 571 case MSR_LBR_INFO_0 ... MSR_LBR_INFO_0 + 31: 572 case MSR_LBR_NHM_FROM ... MSR_LBR_NHM_FROM + 31: 573 case MSR_LBR_NHM_TO ... MSR_LBR_NHM_TO + 31: 574 case MSR_LBR_CORE_FROM ... MSR_LBR_CORE_FROM + 8: 575 case MSR_LBR_CORE_TO ... MSR_LBR_CORE_TO + 8: 576 /* LBR MSRs. These are handled in vmx_update_intercept_for_lbr_msrs() */ 577 return true; 578 } 579 580 r = possible_passthrough_msr_slot(msr) != -ENOENT; 581 582 WARN(!r, "Invalid MSR %x, please adapt vmx_possible_passthrough_msrs[]", msr); 583 584 return r; 585 } 586 587 struct vmx_uret_msr *vmx_find_uret_msr(struct vcpu_vmx *vmx, u32 msr) 588 { 589 int i; 590 591 i = kvm_find_user_return_msr(msr); 592 if (i >= 0) 593 return &vmx->guest_uret_msrs[i]; 594 return NULL; 595 } 596 597 static int vmx_set_guest_uret_msr(struct vcpu_vmx *vmx, 598 struct vmx_uret_msr *msr, u64 data) 599 { 600 unsigned int slot = msr - vmx->guest_uret_msrs; 601 int ret = 0; 602 603 if (msr->load_into_hardware) { 604 preempt_disable(); 605 ret = kvm_set_user_return_msr(slot, data, msr->mask); 606 preempt_enable(); 607 } 608 if (!ret) 609 msr->data = data; 610 return ret; 611 } 612 613 #ifdef CONFIG_KEXEC_CORE 614 static void crash_vmclear_local_loaded_vmcss(void) 615 { 616 int cpu = raw_smp_processor_id(); 617 struct loaded_vmcs *v; 618 619 list_for_each_entry(v, &per_cpu(loaded_vmcss_on_cpu, cpu), 620 loaded_vmcss_on_cpu_link) 621 vmcs_clear(v->vmcs); 622 } 623 #endif /* CONFIG_KEXEC_CORE */ 624 625 static void __loaded_vmcs_clear(void *arg) 626 { 627 struct loaded_vmcs *loaded_vmcs = arg; 628 int cpu = raw_smp_processor_id(); 629 630 if (loaded_vmcs->cpu != cpu) 631 return; /* vcpu migration can race with cpu offline */ 632 if (per_cpu(current_vmcs, cpu) == loaded_vmcs->vmcs) 633 per_cpu(current_vmcs, cpu) = NULL; 634 635 vmcs_clear(loaded_vmcs->vmcs); 636 if (loaded_vmcs->shadow_vmcs && loaded_vmcs->launched) 637 vmcs_clear(loaded_vmcs->shadow_vmcs); 638 639 list_del(&loaded_vmcs->loaded_vmcss_on_cpu_link); 640 641 /* 642 * Ensure all writes to loaded_vmcs, including deleting it from its 643 * current percpu list, complete before setting loaded_vmcs->cpu to 644 * -1, otherwise a different cpu can see loaded_vmcs->cpu == -1 first 645 * and add loaded_vmcs to its percpu list before it's deleted from this 646 * cpu's list. Pairs with the smp_rmb() in vmx_vcpu_load_vmcs(). 647 */ 648 smp_wmb(); 649 650 loaded_vmcs->cpu = -1; 651 loaded_vmcs->launched = 0; 652 } 653 654 void loaded_vmcs_clear(struct loaded_vmcs *loaded_vmcs) 655 { 656 int cpu = loaded_vmcs->cpu; 657 658 if (cpu != -1) 659 smp_call_function_single(cpu, 660 __loaded_vmcs_clear, loaded_vmcs, 1); 661 } 662 663 static bool vmx_segment_cache_test_set(struct vcpu_vmx *vmx, unsigned seg, 664 unsigned field) 665 { 666 bool ret; 667 u32 mask = 1 << (seg * SEG_FIELD_NR + field); 668 669 if (!kvm_register_is_available(&vmx->vcpu, VCPU_EXREG_SEGMENTS)) { 670 kvm_register_mark_available(&vmx->vcpu, VCPU_EXREG_SEGMENTS); 671 vmx->segment_cache.bitmask = 0; 672 } 673 ret = vmx->segment_cache.bitmask & mask; 674 vmx->segment_cache.bitmask |= mask; 675 return ret; 676 } 677 678 static u16 vmx_read_guest_seg_selector(struct vcpu_vmx *vmx, unsigned seg) 679 { 680 u16 *p = &vmx->segment_cache.seg[seg].selector; 681 682 if (!vmx_segment_cache_test_set(vmx, seg, SEG_FIELD_SEL)) 683 *p = vmcs_read16(kvm_vmx_segment_fields[seg].selector); 684 return *p; 685 } 686 687 static ulong vmx_read_guest_seg_base(struct vcpu_vmx *vmx, unsigned seg) 688 { 689 ulong *p = &vmx->segment_cache.seg[seg].base; 690 691 if (!vmx_segment_cache_test_set(vmx, seg, SEG_FIELD_BASE)) 692 *p = vmcs_readl(kvm_vmx_segment_fields[seg].base); 693 return *p; 694 } 695 696 static u32 vmx_read_guest_seg_limit(struct vcpu_vmx *vmx, unsigned seg) 697 { 698 u32 *p = &vmx->segment_cache.seg[seg].limit; 699 700 if (!vmx_segment_cache_test_set(vmx, seg, SEG_FIELD_LIMIT)) 701 *p = vmcs_read32(kvm_vmx_segment_fields[seg].limit); 702 return *p; 703 } 704 705 static u32 vmx_read_guest_seg_ar(struct vcpu_vmx *vmx, unsigned seg) 706 { 707 u32 *p = &vmx->segment_cache.seg[seg].ar; 708 709 if (!vmx_segment_cache_test_set(vmx, seg, SEG_FIELD_AR)) 710 *p = vmcs_read32(kvm_vmx_segment_fields[seg].ar_bytes); 711 return *p; 712 } 713 714 void vmx_update_exception_bitmap(struct kvm_vcpu *vcpu) 715 { 716 u32 eb; 717 718 eb = (1u << PF_VECTOR) | (1u << UD_VECTOR) | (1u << MC_VECTOR) | 719 (1u << DB_VECTOR) | (1u << AC_VECTOR); 720 /* 721 * Guest access to VMware backdoor ports could legitimately 722 * trigger #GP because of TSS I/O permission bitmap. 723 * We intercept those #GP and allow access to them anyway 724 * as VMware does. 725 */ 726 if (enable_vmware_backdoor) 727 eb |= (1u << GP_VECTOR); 728 if ((vcpu->guest_debug & 729 (KVM_GUESTDBG_ENABLE | KVM_GUESTDBG_USE_SW_BP)) == 730 (KVM_GUESTDBG_ENABLE | KVM_GUESTDBG_USE_SW_BP)) 731 eb |= 1u << BP_VECTOR; 732 if (to_vmx(vcpu)->rmode.vm86_active) 733 eb = ~0; 734 if (!vmx_need_pf_intercept(vcpu)) 735 eb &= ~(1u << PF_VECTOR); 736 737 /* When we are running a nested L2 guest and L1 specified for it a 738 * certain exception bitmap, we must trap the same exceptions and pass 739 * them to L1. When running L2, we will only handle the exceptions 740 * specified above if L1 did not want them. 741 */ 742 if (is_guest_mode(vcpu)) 743 eb |= get_vmcs12(vcpu)->exception_bitmap; 744 else { 745 int mask = 0, match = 0; 746 747 if (enable_ept && (eb & (1u << PF_VECTOR))) { 748 /* 749 * If EPT is enabled, #PF is currently only intercepted 750 * if MAXPHYADDR is smaller on the guest than on the 751 * host. In that case we only care about present, 752 * non-reserved faults. For vmcs02, however, PFEC_MASK 753 * and PFEC_MATCH are set in prepare_vmcs02_rare. 754 */ 755 mask = PFERR_PRESENT_MASK | PFERR_RSVD_MASK; 756 match = PFERR_PRESENT_MASK; 757 } 758 vmcs_write32(PAGE_FAULT_ERROR_CODE_MASK, mask); 759 vmcs_write32(PAGE_FAULT_ERROR_CODE_MATCH, match); 760 } 761 762 /* 763 * Disabling xfd interception indicates that dynamic xfeatures 764 * might be used in the guest. Always trap #NM in this case 765 * to save guest xfd_err timely. 766 */ 767 if (vcpu->arch.xfd_no_write_intercept) 768 eb |= (1u << NM_VECTOR); 769 770 vmcs_write32(EXCEPTION_BITMAP, eb); 771 } 772 773 /* 774 * Check if MSR is intercepted for currently loaded MSR bitmap. 775 */ 776 static bool msr_write_intercepted(struct vcpu_vmx *vmx, u32 msr) 777 { 778 if (!(exec_controls_get(vmx) & CPU_BASED_USE_MSR_BITMAPS)) 779 return true; 780 781 return vmx_test_msr_bitmap_write(vmx->loaded_vmcs->msr_bitmap, 782 MSR_IA32_SPEC_CTRL); 783 } 784 785 static void clear_atomic_switch_msr_special(struct vcpu_vmx *vmx, 786 unsigned long entry, unsigned long exit) 787 { 788 vm_entry_controls_clearbit(vmx, entry); 789 vm_exit_controls_clearbit(vmx, exit); 790 } 791 792 int vmx_find_loadstore_msr_slot(struct vmx_msrs *m, u32 msr) 793 { 794 unsigned int i; 795 796 for (i = 0; i < m->nr; ++i) { 797 if (m->val[i].index == msr) 798 return i; 799 } 800 return -ENOENT; 801 } 802 803 static void clear_atomic_switch_msr(struct vcpu_vmx *vmx, unsigned msr) 804 { 805 int i; 806 struct msr_autoload *m = &vmx->msr_autoload; 807 808 switch (msr) { 809 case MSR_EFER: 810 if (cpu_has_load_ia32_efer()) { 811 clear_atomic_switch_msr_special(vmx, 812 VM_ENTRY_LOAD_IA32_EFER, 813 VM_EXIT_LOAD_IA32_EFER); 814 return; 815 } 816 break; 817 case MSR_CORE_PERF_GLOBAL_CTRL: 818 if (cpu_has_load_perf_global_ctrl()) { 819 clear_atomic_switch_msr_special(vmx, 820 VM_ENTRY_LOAD_IA32_PERF_GLOBAL_CTRL, 821 VM_EXIT_LOAD_IA32_PERF_GLOBAL_CTRL); 822 return; 823 } 824 break; 825 } 826 i = vmx_find_loadstore_msr_slot(&m->guest, msr); 827 if (i < 0) 828 goto skip_guest; 829 --m->guest.nr; 830 m->guest.val[i] = m->guest.val[m->guest.nr]; 831 vmcs_write32(VM_ENTRY_MSR_LOAD_COUNT, m->guest.nr); 832 833 skip_guest: 834 i = vmx_find_loadstore_msr_slot(&m->host, msr); 835 if (i < 0) 836 return; 837 838 --m->host.nr; 839 m->host.val[i] = m->host.val[m->host.nr]; 840 vmcs_write32(VM_EXIT_MSR_LOAD_COUNT, m->host.nr); 841 } 842 843 static void add_atomic_switch_msr_special(struct vcpu_vmx *vmx, 844 unsigned long entry, unsigned long exit, 845 unsigned long guest_val_vmcs, unsigned long host_val_vmcs, 846 u64 guest_val, u64 host_val) 847 { 848 vmcs_write64(guest_val_vmcs, guest_val); 849 if (host_val_vmcs != HOST_IA32_EFER) 850 vmcs_write64(host_val_vmcs, host_val); 851 vm_entry_controls_setbit(vmx, entry); 852 vm_exit_controls_setbit(vmx, exit); 853 } 854 855 static void add_atomic_switch_msr(struct vcpu_vmx *vmx, unsigned msr, 856 u64 guest_val, u64 host_val, bool entry_only) 857 { 858 int i, j = 0; 859 struct msr_autoload *m = &vmx->msr_autoload; 860 861 switch (msr) { 862 case MSR_EFER: 863 if (cpu_has_load_ia32_efer()) { 864 add_atomic_switch_msr_special(vmx, 865 VM_ENTRY_LOAD_IA32_EFER, 866 VM_EXIT_LOAD_IA32_EFER, 867 GUEST_IA32_EFER, 868 HOST_IA32_EFER, 869 guest_val, host_val); 870 return; 871 } 872 break; 873 case MSR_CORE_PERF_GLOBAL_CTRL: 874 if (cpu_has_load_perf_global_ctrl()) { 875 add_atomic_switch_msr_special(vmx, 876 VM_ENTRY_LOAD_IA32_PERF_GLOBAL_CTRL, 877 VM_EXIT_LOAD_IA32_PERF_GLOBAL_CTRL, 878 GUEST_IA32_PERF_GLOBAL_CTRL, 879 HOST_IA32_PERF_GLOBAL_CTRL, 880 guest_val, host_val); 881 return; 882 } 883 break; 884 case MSR_IA32_PEBS_ENABLE: 885 /* PEBS needs a quiescent period after being disabled (to write 886 * a record). Disabling PEBS through VMX MSR swapping doesn't 887 * provide that period, so a CPU could write host's record into 888 * guest's memory. 889 */ 890 wrmsrl(MSR_IA32_PEBS_ENABLE, 0); 891 } 892 893 i = vmx_find_loadstore_msr_slot(&m->guest, msr); 894 if (!entry_only) 895 j = vmx_find_loadstore_msr_slot(&m->host, msr); 896 897 if ((i < 0 && m->guest.nr == MAX_NR_LOADSTORE_MSRS) || 898 (j < 0 && m->host.nr == MAX_NR_LOADSTORE_MSRS)) { 899 printk_once(KERN_WARNING "Not enough msr switch entries. " 900 "Can't add msr %x\n", msr); 901 return; 902 } 903 if (i < 0) { 904 i = m->guest.nr++; 905 vmcs_write32(VM_ENTRY_MSR_LOAD_COUNT, m->guest.nr); 906 } 907 m->guest.val[i].index = msr; 908 m->guest.val[i].value = guest_val; 909 910 if (entry_only) 911 return; 912 913 if (j < 0) { 914 j = m->host.nr++; 915 vmcs_write32(VM_EXIT_MSR_LOAD_COUNT, m->host.nr); 916 } 917 m->host.val[j].index = msr; 918 m->host.val[j].value = host_val; 919 } 920 921 static bool update_transition_efer(struct vcpu_vmx *vmx) 922 { 923 u64 guest_efer = vmx->vcpu.arch.efer; 924 u64 ignore_bits = 0; 925 int i; 926 927 /* Shadow paging assumes NX to be available. */ 928 if (!enable_ept) 929 guest_efer |= EFER_NX; 930 931 /* 932 * LMA and LME handled by hardware; SCE meaningless outside long mode. 933 */ 934 ignore_bits |= EFER_SCE; 935 #ifdef CONFIG_X86_64 936 ignore_bits |= EFER_LMA | EFER_LME; 937 /* SCE is meaningful only in long mode on Intel */ 938 if (guest_efer & EFER_LMA) 939 ignore_bits &= ~(u64)EFER_SCE; 940 #endif 941 942 /* 943 * On EPT, we can't emulate NX, so we must switch EFER atomically. 944 * On CPUs that support "load IA32_EFER", always switch EFER 945 * atomically, since it's faster than switching it manually. 946 */ 947 if (cpu_has_load_ia32_efer() || 948 (enable_ept && ((vmx->vcpu.arch.efer ^ host_efer) & EFER_NX))) { 949 if (!(guest_efer & EFER_LMA)) 950 guest_efer &= ~EFER_LME; 951 if (guest_efer != host_efer) 952 add_atomic_switch_msr(vmx, MSR_EFER, 953 guest_efer, host_efer, false); 954 else 955 clear_atomic_switch_msr(vmx, MSR_EFER); 956 return false; 957 } 958 959 i = kvm_find_user_return_msr(MSR_EFER); 960 if (i < 0) 961 return false; 962 963 clear_atomic_switch_msr(vmx, MSR_EFER); 964 965 guest_efer &= ~ignore_bits; 966 guest_efer |= host_efer & ignore_bits; 967 968 vmx->guest_uret_msrs[i].data = guest_efer; 969 vmx->guest_uret_msrs[i].mask = ~ignore_bits; 970 971 return true; 972 } 973 974 #ifdef CONFIG_X86_32 975 /* 976 * On 32-bit kernels, VM exits still load the FS and GS bases from the 977 * VMCS rather than the segment table. KVM uses this helper to figure 978 * out the current bases to poke them into the VMCS before entry. 979 */ 980 static unsigned long segment_base(u16 selector) 981 { 982 struct desc_struct *table; 983 unsigned long v; 984 985 if (!(selector & ~SEGMENT_RPL_MASK)) 986 return 0; 987 988 table = get_current_gdt_ro(); 989 990 if ((selector & SEGMENT_TI_MASK) == SEGMENT_LDT) { 991 u16 ldt_selector = kvm_read_ldt(); 992 993 if (!(ldt_selector & ~SEGMENT_RPL_MASK)) 994 return 0; 995 996 table = (struct desc_struct *)segment_base(ldt_selector); 997 } 998 v = get_desc_base(&table[selector >> 3]); 999 return v; 1000 } 1001 #endif 1002 1003 static inline bool pt_can_write_msr(struct vcpu_vmx *vmx) 1004 { 1005 return vmx_pt_mode_is_host_guest() && 1006 !(vmx->pt_desc.guest.ctl & RTIT_CTL_TRACEEN); 1007 } 1008 1009 static inline bool pt_output_base_valid(struct kvm_vcpu *vcpu, u64 base) 1010 { 1011 /* The base must be 128-byte aligned and a legal physical address. */ 1012 return kvm_vcpu_is_legal_aligned_gpa(vcpu, base, 128); 1013 } 1014 1015 static inline void pt_load_msr(struct pt_ctx *ctx, u32 addr_range) 1016 { 1017 u32 i; 1018 1019 wrmsrl(MSR_IA32_RTIT_STATUS, ctx->status); 1020 wrmsrl(MSR_IA32_RTIT_OUTPUT_BASE, ctx->output_base); 1021 wrmsrl(MSR_IA32_RTIT_OUTPUT_MASK, ctx->output_mask); 1022 wrmsrl(MSR_IA32_RTIT_CR3_MATCH, ctx->cr3_match); 1023 for (i = 0; i < addr_range; i++) { 1024 wrmsrl(MSR_IA32_RTIT_ADDR0_A + i * 2, ctx->addr_a[i]); 1025 wrmsrl(MSR_IA32_RTIT_ADDR0_B + i * 2, ctx->addr_b[i]); 1026 } 1027 } 1028 1029 static inline void pt_save_msr(struct pt_ctx *ctx, u32 addr_range) 1030 { 1031 u32 i; 1032 1033 rdmsrl(MSR_IA32_RTIT_STATUS, ctx->status); 1034 rdmsrl(MSR_IA32_RTIT_OUTPUT_BASE, ctx->output_base); 1035 rdmsrl(MSR_IA32_RTIT_OUTPUT_MASK, ctx->output_mask); 1036 rdmsrl(MSR_IA32_RTIT_CR3_MATCH, ctx->cr3_match); 1037 for (i = 0; i < addr_range; i++) { 1038 rdmsrl(MSR_IA32_RTIT_ADDR0_A + i * 2, ctx->addr_a[i]); 1039 rdmsrl(MSR_IA32_RTIT_ADDR0_B + i * 2, ctx->addr_b[i]); 1040 } 1041 } 1042 1043 static void pt_guest_enter(struct vcpu_vmx *vmx) 1044 { 1045 if (vmx_pt_mode_is_system()) 1046 return; 1047 1048 /* 1049 * GUEST_IA32_RTIT_CTL is already set in the VMCS. 1050 * Save host state before VM entry. 1051 */ 1052 rdmsrl(MSR_IA32_RTIT_CTL, vmx->pt_desc.host.ctl); 1053 if (vmx->pt_desc.guest.ctl & RTIT_CTL_TRACEEN) { 1054 wrmsrl(MSR_IA32_RTIT_CTL, 0); 1055 pt_save_msr(&vmx->pt_desc.host, vmx->pt_desc.num_address_ranges); 1056 pt_load_msr(&vmx->pt_desc.guest, vmx->pt_desc.num_address_ranges); 1057 } 1058 } 1059 1060 static void pt_guest_exit(struct vcpu_vmx *vmx) 1061 { 1062 if (vmx_pt_mode_is_system()) 1063 return; 1064 1065 if (vmx->pt_desc.guest.ctl & RTIT_CTL_TRACEEN) { 1066 pt_save_msr(&vmx->pt_desc.guest, vmx->pt_desc.num_address_ranges); 1067 pt_load_msr(&vmx->pt_desc.host, vmx->pt_desc.num_address_ranges); 1068 } 1069 1070 /* 1071 * KVM requires VM_EXIT_CLEAR_IA32_RTIT_CTL to expose PT to the guest, 1072 * i.e. RTIT_CTL is always cleared on VM-Exit. Restore it if necessary. 1073 */ 1074 if (vmx->pt_desc.host.ctl) 1075 wrmsrl(MSR_IA32_RTIT_CTL, vmx->pt_desc.host.ctl); 1076 } 1077 1078 void vmx_set_host_fs_gs(struct vmcs_host_state *host, u16 fs_sel, u16 gs_sel, 1079 unsigned long fs_base, unsigned long gs_base) 1080 { 1081 if (unlikely(fs_sel != host->fs_sel)) { 1082 if (!(fs_sel & 7)) 1083 vmcs_write16(HOST_FS_SELECTOR, fs_sel); 1084 else 1085 vmcs_write16(HOST_FS_SELECTOR, 0); 1086 host->fs_sel = fs_sel; 1087 } 1088 if (unlikely(gs_sel != host->gs_sel)) { 1089 if (!(gs_sel & 7)) 1090 vmcs_write16(HOST_GS_SELECTOR, gs_sel); 1091 else 1092 vmcs_write16(HOST_GS_SELECTOR, 0); 1093 host->gs_sel = gs_sel; 1094 } 1095 if (unlikely(fs_base != host->fs_base)) { 1096 vmcs_writel(HOST_FS_BASE, fs_base); 1097 host->fs_base = fs_base; 1098 } 1099 if (unlikely(gs_base != host->gs_base)) { 1100 vmcs_writel(HOST_GS_BASE, gs_base); 1101 host->gs_base = gs_base; 1102 } 1103 } 1104 1105 void vmx_prepare_switch_to_guest(struct kvm_vcpu *vcpu) 1106 { 1107 struct vcpu_vmx *vmx = to_vmx(vcpu); 1108 struct vmcs_host_state *host_state; 1109 #ifdef CONFIG_X86_64 1110 int cpu = raw_smp_processor_id(); 1111 #endif 1112 unsigned long fs_base, gs_base; 1113 u16 fs_sel, gs_sel; 1114 int i; 1115 1116 vmx->req_immediate_exit = false; 1117 1118 /* 1119 * Note that guest MSRs to be saved/restored can also be changed 1120 * when guest state is loaded. This happens when guest transitions 1121 * to/from long-mode by setting MSR_EFER.LMA. 1122 */ 1123 if (!vmx->guest_uret_msrs_loaded) { 1124 vmx->guest_uret_msrs_loaded = true; 1125 for (i = 0; i < kvm_nr_uret_msrs; ++i) { 1126 if (!vmx->guest_uret_msrs[i].load_into_hardware) 1127 continue; 1128 1129 kvm_set_user_return_msr(i, 1130 vmx->guest_uret_msrs[i].data, 1131 vmx->guest_uret_msrs[i].mask); 1132 } 1133 } 1134 1135 if (vmx->nested.need_vmcs12_to_shadow_sync) 1136 nested_sync_vmcs12_to_shadow(vcpu); 1137 1138 if (vmx->guest_state_loaded) 1139 return; 1140 1141 host_state = &vmx->loaded_vmcs->host_state; 1142 1143 /* 1144 * Set host fs and gs selectors. Unfortunately, 22.2.3 does not 1145 * allow segment selectors with cpl > 0 or ti == 1. 1146 */ 1147 host_state->ldt_sel = kvm_read_ldt(); 1148 1149 #ifdef CONFIG_X86_64 1150 savesegment(ds, host_state->ds_sel); 1151 savesegment(es, host_state->es_sel); 1152 1153 gs_base = cpu_kernelmode_gs_base(cpu); 1154 if (likely(is_64bit_mm(current->mm))) { 1155 current_save_fsgs(); 1156 fs_sel = current->thread.fsindex; 1157 gs_sel = current->thread.gsindex; 1158 fs_base = current->thread.fsbase; 1159 vmx->msr_host_kernel_gs_base = current->thread.gsbase; 1160 } else { 1161 savesegment(fs, fs_sel); 1162 savesegment(gs, gs_sel); 1163 fs_base = read_msr(MSR_FS_BASE); 1164 vmx->msr_host_kernel_gs_base = read_msr(MSR_KERNEL_GS_BASE); 1165 } 1166 1167 wrmsrl(MSR_KERNEL_GS_BASE, vmx->msr_guest_kernel_gs_base); 1168 #else 1169 savesegment(fs, fs_sel); 1170 savesegment(gs, gs_sel); 1171 fs_base = segment_base(fs_sel); 1172 gs_base = segment_base(gs_sel); 1173 #endif 1174 1175 vmx_set_host_fs_gs(host_state, fs_sel, gs_sel, fs_base, gs_base); 1176 vmx->guest_state_loaded = true; 1177 } 1178 1179 static void vmx_prepare_switch_to_host(struct vcpu_vmx *vmx) 1180 { 1181 struct vmcs_host_state *host_state; 1182 1183 if (!vmx->guest_state_loaded) 1184 return; 1185 1186 host_state = &vmx->loaded_vmcs->host_state; 1187 1188 ++vmx->vcpu.stat.host_state_reload; 1189 1190 #ifdef CONFIG_X86_64 1191 rdmsrl(MSR_KERNEL_GS_BASE, vmx->msr_guest_kernel_gs_base); 1192 #endif 1193 if (host_state->ldt_sel || (host_state->gs_sel & 7)) { 1194 kvm_load_ldt(host_state->ldt_sel); 1195 #ifdef CONFIG_X86_64 1196 load_gs_index(host_state->gs_sel); 1197 #else 1198 loadsegment(gs, host_state->gs_sel); 1199 #endif 1200 } 1201 if (host_state->fs_sel & 7) 1202 loadsegment(fs, host_state->fs_sel); 1203 #ifdef CONFIG_X86_64 1204 if (unlikely(host_state->ds_sel | host_state->es_sel)) { 1205 loadsegment(ds, host_state->ds_sel); 1206 loadsegment(es, host_state->es_sel); 1207 } 1208 #endif 1209 invalidate_tss_limit(); 1210 #ifdef CONFIG_X86_64 1211 wrmsrl(MSR_KERNEL_GS_BASE, vmx->msr_host_kernel_gs_base); 1212 #endif 1213 load_fixmap_gdt(raw_smp_processor_id()); 1214 vmx->guest_state_loaded = false; 1215 vmx->guest_uret_msrs_loaded = false; 1216 } 1217 1218 #ifdef CONFIG_X86_64 1219 static u64 vmx_read_guest_kernel_gs_base(struct vcpu_vmx *vmx) 1220 { 1221 preempt_disable(); 1222 if (vmx->guest_state_loaded) 1223 rdmsrl(MSR_KERNEL_GS_BASE, vmx->msr_guest_kernel_gs_base); 1224 preempt_enable(); 1225 return vmx->msr_guest_kernel_gs_base; 1226 } 1227 1228 static void vmx_write_guest_kernel_gs_base(struct vcpu_vmx *vmx, u64 data) 1229 { 1230 preempt_disable(); 1231 if (vmx->guest_state_loaded) 1232 wrmsrl(MSR_KERNEL_GS_BASE, data); 1233 preempt_enable(); 1234 vmx->msr_guest_kernel_gs_base = data; 1235 } 1236 #endif 1237 1238 void vmx_vcpu_load_vmcs(struct kvm_vcpu *vcpu, int cpu, 1239 struct loaded_vmcs *buddy) 1240 { 1241 struct vcpu_vmx *vmx = to_vmx(vcpu); 1242 bool already_loaded = vmx->loaded_vmcs->cpu == cpu; 1243 struct vmcs *prev; 1244 1245 if (!already_loaded) { 1246 loaded_vmcs_clear(vmx->loaded_vmcs); 1247 local_irq_disable(); 1248 1249 /* 1250 * Ensure loaded_vmcs->cpu is read before adding loaded_vmcs to 1251 * this cpu's percpu list, otherwise it may not yet be deleted 1252 * from its previous cpu's percpu list. Pairs with the 1253 * smb_wmb() in __loaded_vmcs_clear(). 1254 */ 1255 smp_rmb(); 1256 1257 list_add(&vmx->loaded_vmcs->loaded_vmcss_on_cpu_link, 1258 &per_cpu(loaded_vmcss_on_cpu, cpu)); 1259 local_irq_enable(); 1260 } 1261 1262 prev = per_cpu(current_vmcs, cpu); 1263 if (prev != vmx->loaded_vmcs->vmcs) { 1264 per_cpu(current_vmcs, cpu) = vmx->loaded_vmcs->vmcs; 1265 vmcs_load(vmx->loaded_vmcs->vmcs); 1266 1267 /* 1268 * No indirect branch prediction barrier needed when switching 1269 * the active VMCS within a guest, e.g. on nested VM-Enter. 1270 * The L1 VMM can protect itself with retpolines, IBPB or IBRS. 1271 */ 1272 if (!buddy || WARN_ON_ONCE(buddy->vmcs != prev)) 1273 indirect_branch_prediction_barrier(); 1274 } 1275 1276 if (!already_loaded) { 1277 void *gdt = get_current_gdt_ro(); 1278 1279 /* 1280 * Flush all EPTP/VPID contexts, the new pCPU may have stale 1281 * TLB entries from its previous association with the vCPU. 1282 */ 1283 kvm_make_request(KVM_REQ_TLB_FLUSH, vcpu); 1284 1285 /* 1286 * Linux uses per-cpu TSS and GDT, so set these when switching 1287 * processors. See 22.2.4. 1288 */ 1289 vmcs_writel(HOST_TR_BASE, 1290 (unsigned long)&get_cpu_entry_area(cpu)->tss.x86_tss); 1291 vmcs_writel(HOST_GDTR_BASE, (unsigned long)gdt); /* 22.2.4 */ 1292 1293 if (IS_ENABLED(CONFIG_IA32_EMULATION) || IS_ENABLED(CONFIG_X86_32)) { 1294 /* 22.2.3 */ 1295 vmcs_writel(HOST_IA32_SYSENTER_ESP, 1296 (unsigned long)(cpu_entry_stack(cpu) + 1)); 1297 } 1298 1299 vmx->loaded_vmcs->cpu = cpu; 1300 } 1301 } 1302 1303 /* 1304 * Switches to specified vcpu, until a matching vcpu_put(), but assumes 1305 * vcpu mutex is already taken. 1306 */ 1307 static void vmx_vcpu_load(struct kvm_vcpu *vcpu, int cpu) 1308 { 1309 struct vcpu_vmx *vmx = to_vmx(vcpu); 1310 1311 vmx_vcpu_load_vmcs(vcpu, cpu, NULL); 1312 1313 vmx_vcpu_pi_load(vcpu, cpu); 1314 1315 vmx->host_debugctlmsr = get_debugctlmsr(); 1316 } 1317 1318 static void vmx_vcpu_put(struct kvm_vcpu *vcpu) 1319 { 1320 vmx_vcpu_pi_put(vcpu); 1321 1322 vmx_prepare_switch_to_host(to_vmx(vcpu)); 1323 } 1324 1325 bool vmx_emulation_required(struct kvm_vcpu *vcpu) 1326 { 1327 return emulate_invalid_guest_state && !vmx_guest_state_valid(vcpu); 1328 } 1329 1330 unsigned long vmx_get_rflags(struct kvm_vcpu *vcpu) 1331 { 1332 struct vcpu_vmx *vmx = to_vmx(vcpu); 1333 unsigned long rflags, save_rflags; 1334 1335 if (!kvm_register_is_available(vcpu, VCPU_EXREG_RFLAGS)) { 1336 kvm_register_mark_available(vcpu, VCPU_EXREG_RFLAGS); 1337 rflags = vmcs_readl(GUEST_RFLAGS); 1338 if (vmx->rmode.vm86_active) { 1339 rflags &= RMODE_GUEST_OWNED_EFLAGS_BITS; 1340 save_rflags = vmx->rmode.save_rflags; 1341 rflags |= save_rflags & ~RMODE_GUEST_OWNED_EFLAGS_BITS; 1342 } 1343 vmx->rflags = rflags; 1344 } 1345 return vmx->rflags; 1346 } 1347 1348 void vmx_set_rflags(struct kvm_vcpu *vcpu, unsigned long rflags) 1349 { 1350 struct vcpu_vmx *vmx = to_vmx(vcpu); 1351 unsigned long old_rflags; 1352 1353 if (is_unrestricted_guest(vcpu)) { 1354 kvm_register_mark_available(vcpu, VCPU_EXREG_RFLAGS); 1355 vmx->rflags = rflags; 1356 vmcs_writel(GUEST_RFLAGS, rflags); 1357 return; 1358 } 1359 1360 old_rflags = vmx_get_rflags(vcpu); 1361 vmx->rflags = rflags; 1362 if (vmx->rmode.vm86_active) { 1363 vmx->rmode.save_rflags = rflags; 1364 rflags |= X86_EFLAGS_IOPL | X86_EFLAGS_VM; 1365 } 1366 vmcs_writel(GUEST_RFLAGS, rflags); 1367 1368 if ((old_rflags ^ vmx->rflags) & X86_EFLAGS_VM) 1369 vmx->emulation_required = vmx_emulation_required(vcpu); 1370 } 1371 1372 static bool vmx_get_if_flag(struct kvm_vcpu *vcpu) 1373 { 1374 return vmx_get_rflags(vcpu) & X86_EFLAGS_IF; 1375 } 1376 1377 u32 vmx_get_interrupt_shadow(struct kvm_vcpu *vcpu) 1378 { 1379 u32 interruptibility = vmcs_read32(GUEST_INTERRUPTIBILITY_INFO); 1380 int ret = 0; 1381 1382 if (interruptibility & GUEST_INTR_STATE_STI) 1383 ret |= KVM_X86_SHADOW_INT_STI; 1384 if (interruptibility & GUEST_INTR_STATE_MOV_SS) 1385 ret |= KVM_X86_SHADOW_INT_MOV_SS; 1386 1387 return ret; 1388 } 1389 1390 void vmx_set_interrupt_shadow(struct kvm_vcpu *vcpu, int mask) 1391 { 1392 u32 interruptibility_old = vmcs_read32(GUEST_INTERRUPTIBILITY_INFO); 1393 u32 interruptibility = interruptibility_old; 1394 1395 interruptibility &= ~(GUEST_INTR_STATE_STI | GUEST_INTR_STATE_MOV_SS); 1396 1397 if (mask & KVM_X86_SHADOW_INT_MOV_SS) 1398 interruptibility |= GUEST_INTR_STATE_MOV_SS; 1399 else if (mask & KVM_X86_SHADOW_INT_STI) 1400 interruptibility |= GUEST_INTR_STATE_STI; 1401 1402 if ((interruptibility != interruptibility_old)) 1403 vmcs_write32(GUEST_INTERRUPTIBILITY_INFO, interruptibility); 1404 } 1405 1406 static int vmx_rtit_ctl_check(struct kvm_vcpu *vcpu, u64 data) 1407 { 1408 struct vcpu_vmx *vmx = to_vmx(vcpu); 1409 unsigned long value; 1410 1411 /* 1412 * Any MSR write that attempts to change bits marked reserved will 1413 * case a #GP fault. 1414 */ 1415 if (data & vmx->pt_desc.ctl_bitmask) 1416 return 1; 1417 1418 /* 1419 * Any attempt to modify IA32_RTIT_CTL while TraceEn is set will 1420 * result in a #GP unless the same write also clears TraceEn. 1421 */ 1422 if ((vmx->pt_desc.guest.ctl & RTIT_CTL_TRACEEN) && 1423 ((vmx->pt_desc.guest.ctl ^ data) & ~RTIT_CTL_TRACEEN)) 1424 return 1; 1425 1426 /* 1427 * WRMSR to IA32_RTIT_CTL that sets TraceEn but clears this bit 1428 * and FabricEn would cause #GP, if 1429 * CPUID.(EAX=14H, ECX=0):ECX.SNGLRGNOUT[bit 2] = 0 1430 */ 1431 if ((data & RTIT_CTL_TRACEEN) && !(data & RTIT_CTL_TOPA) && 1432 !(data & RTIT_CTL_FABRIC_EN) && 1433 !intel_pt_validate_cap(vmx->pt_desc.caps, 1434 PT_CAP_single_range_output)) 1435 return 1; 1436 1437 /* 1438 * MTCFreq, CycThresh and PSBFreq encodings check, any MSR write that 1439 * utilize encodings marked reserved will cause a #GP fault. 1440 */ 1441 value = intel_pt_validate_cap(vmx->pt_desc.caps, PT_CAP_mtc_periods); 1442 if (intel_pt_validate_cap(vmx->pt_desc.caps, PT_CAP_mtc) && 1443 !test_bit((data & RTIT_CTL_MTC_RANGE) >> 1444 RTIT_CTL_MTC_RANGE_OFFSET, &value)) 1445 return 1; 1446 value = intel_pt_validate_cap(vmx->pt_desc.caps, 1447 PT_CAP_cycle_thresholds); 1448 if (intel_pt_validate_cap(vmx->pt_desc.caps, PT_CAP_psb_cyc) && 1449 !test_bit((data & RTIT_CTL_CYC_THRESH) >> 1450 RTIT_CTL_CYC_THRESH_OFFSET, &value)) 1451 return 1; 1452 value = intel_pt_validate_cap(vmx->pt_desc.caps, PT_CAP_psb_periods); 1453 if (intel_pt_validate_cap(vmx->pt_desc.caps, PT_CAP_psb_cyc) && 1454 !test_bit((data & RTIT_CTL_PSB_FREQ) >> 1455 RTIT_CTL_PSB_FREQ_OFFSET, &value)) 1456 return 1; 1457 1458 /* 1459 * If ADDRx_CFG is reserved or the encodings is >2 will 1460 * cause a #GP fault. 1461 */ 1462 value = (data & RTIT_CTL_ADDR0) >> RTIT_CTL_ADDR0_OFFSET; 1463 if ((value && (vmx->pt_desc.num_address_ranges < 1)) || (value > 2)) 1464 return 1; 1465 value = (data & RTIT_CTL_ADDR1) >> RTIT_CTL_ADDR1_OFFSET; 1466 if ((value && (vmx->pt_desc.num_address_ranges < 2)) || (value > 2)) 1467 return 1; 1468 value = (data & RTIT_CTL_ADDR2) >> RTIT_CTL_ADDR2_OFFSET; 1469 if ((value && (vmx->pt_desc.num_address_ranges < 3)) || (value > 2)) 1470 return 1; 1471 value = (data & RTIT_CTL_ADDR3) >> RTIT_CTL_ADDR3_OFFSET; 1472 if ((value && (vmx->pt_desc.num_address_ranges < 4)) || (value > 2)) 1473 return 1; 1474 1475 return 0; 1476 } 1477 1478 static bool vmx_can_emulate_instruction(struct kvm_vcpu *vcpu, int emul_type, 1479 void *insn, int insn_len) 1480 { 1481 /* 1482 * Emulation of instructions in SGX enclaves is impossible as RIP does 1483 * not point at the failing instruction, and even if it did, the code 1484 * stream is inaccessible. Inject #UD instead of exiting to userspace 1485 * so that guest userspace can't DoS the guest simply by triggering 1486 * emulation (enclaves are CPL3 only). 1487 */ 1488 if (to_vmx(vcpu)->exit_reason.enclave_mode) { 1489 kvm_queue_exception(vcpu, UD_VECTOR); 1490 return false; 1491 } 1492 return true; 1493 } 1494 1495 static int skip_emulated_instruction(struct kvm_vcpu *vcpu) 1496 { 1497 union vmx_exit_reason exit_reason = to_vmx(vcpu)->exit_reason; 1498 unsigned long rip, orig_rip; 1499 u32 instr_len; 1500 1501 /* 1502 * Using VMCS.VM_EXIT_INSTRUCTION_LEN on EPT misconfig depends on 1503 * undefined behavior: Intel's SDM doesn't mandate the VMCS field be 1504 * set when EPT misconfig occurs. In practice, real hardware updates 1505 * VM_EXIT_INSTRUCTION_LEN on EPT misconfig, but other hypervisors 1506 * (namely Hyper-V) don't set it due to it being undefined behavior, 1507 * i.e. we end up advancing IP with some random value. 1508 */ 1509 if (!static_cpu_has(X86_FEATURE_HYPERVISOR) || 1510 exit_reason.basic != EXIT_REASON_EPT_MISCONFIG) { 1511 instr_len = vmcs_read32(VM_EXIT_INSTRUCTION_LEN); 1512 1513 /* 1514 * Emulating an enclave's instructions isn't supported as KVM 1515 * cannot access the enclave's memory or its true RIP, e.g. the 1516 * vmcs.GUEST_RIP points at the exit point of the enclave, not 1517 * the RIP that actually triggered the VM-Exit. But, because 1518 * most instructions that cause VM-Exit will #UD in an enclave, 1519 * most instruction-based VM-Exits simply do not occur. 1520 * 1521 * There are a few exceptions, notably the debug instructions 1522 * INT1ICEBRK and INT3, as they are allowed in debug enclaves 1523 * and generate #DB/#BP as expected, which KVM might intercept. 1524 * But again, the CPU does the dirty work and saves an instr 1525 * length of zero so VMMs don't shoot themselves in the foot. 1526 * WARN if KVM tries to skip a non-zero length instruction on 1527 * a VM-Exit from an enclave. 1528 */ 1529 if (!instr_len) 1530 goto rip_updated; 1531 1532 WARN(exit_reason.enclave_mode, 1533 "KVM: skipping instruction after SGX enclave VM-Exit"); 1534 1535 orig_rip = kvm_rip_read(vcpu); 1536 rip = orig_rip + instr_len; 1537 #ifdef CONFIG_X86_64 1538 /* 1539 * We need to mask out the high 32 bits of RIP if not in 64-bit 1540 * mode, but just finding out that we are in 64-bit mode is 1541 * quite expensive. Only do it if there was a carry. 1542 */ 1543 if (unlikely(((rip ^ orig_rip) >> 31) == 3) && !is_64_bit_mode(vcpu)) 1544 rip = (u32)rip; 1545 #endif 1546 kvm_rip_write(vcpu, rip); 1547 } else { 1548 if (!kvm_emulate_instruction(vcpu, EMULTYPE_SKIP)) 1549 return 0; 1550 } 1551 1552 rip_updated: 1553 /* skipping an emulated instruction also counts */ 1554 vmx_set_interrupt_shadow(vcpu, 0); 1555 1556 return 1; 1557 } 1558 1559 /* 1560 * Recognizes a pending MTF VM-exit and records the nested state for later 1561 * delivery. 1562 */ 1563 static void vmx_update_emulated_instruction(struct kvm_vcpu *vcpu) 1564 { 1565 struct vmcs12 *vmcs12 = get_vmcs12(vcpu); 1566 struct vcpu_vmx *vmx = to_vmx(vcpu); 1567 1568 if (!is_guest_mode(vcpu)) 1569 return; 1570 1571 /* 1572 * Per the SDM, MTF takes priority over debug-trap exceptions besides 1573 * T-bit traps. As instruction emulation is completed (i.e. at the 1574 * instruction boundary), any #DB exception pending delivery must be a 1575 * debug-trap. Record the pending MTF state to be delivered in 1576 * vmx_check_nested_events(). 1577 */ 1578 if (nested_cpu_has_mtf(vmcs12) && 1579 (!vcpu->arch.exception.pending || 1580 vcpu->arch.exception.nr == DB_VECTOR)) 1581 vmx->nested.mtf_pending = true; 1582 else 1583 vmx->nested.mtf_pending = false; 1584 } 1585 1586 static int vmx_skip_emulated_instruction(struct kvm_vcpu *vcpu) 1587 { 1588 vmx_update_emulated_instruction(vcpu); 1589 return skip_emulated_instruction(vcpu); 1590 } 1591 1592 static void vmx_clear_hlt(struct kvm_vcpu *vcpu) 1593 { 1594 /* 1595 * Ensure that we clear the HLT state in the VMCS. We don't need to 1596 * explicitly skip the instruction because if the HLT state is set, 1597 * then the instruction is already executing and RIP has already been 1598 * advanced. 1599 */ 1600 if (kvm_hlt_in_guest(vcpu->kvm) && 1601 vmcs_read32(GUEST_ACTIVITY_STATE) == GUEST_ACTIVITY_HLT) 1602 vmcs_write32(GUEST_ACTIVITY_STATE, GUEST_ACTIVITY_ACTIVE); 1603 } 1604 1605 static void vmx_queue_exception(struct kvm_vcpu *vcpu) 1606 { 1607 struct vcpu_vmx *vmx = to_vmx(vcpu); 1608 unsigned nr = vcpu->arch.exception.nr; 1609 bool has_error_code = vcpu->arch.exception.has_error_code; 1610 u32 error_code = vcpu->arch.exception.error_code; 1611 u32 intr_info = nr | INTR_INFO_VALID_MASK; 1612 1613 kvm_deliver_exception_payload(vcpu); 1614 1615 if (has_error_code) { 1616 vmcs_write32(VM_ENTRY_EXCEPTION_ERROR_CODE, error_code); 1617 intr_info |= INTR_INFO_DELIVER_CODE_MASK; 1618 } 1619 1620 if (vmx->rmode.vm86_active) { 1621 int inc_eip = 0; 1622 if (kvm_exception_is_soft(nr)) 1623 inc_eip = vcpu->arch.event_exit_inst_len; 1624 kvm_inject_realmode_interrupt(vcpu, nr, inc_eip); 1625 return; 1626 } 1627 1628 WARN_ON_ONCE(vmx->emulation_required); 1629 1630 if (kvm_exception_is_soft(nr)) { 1631 vmcs_write32(VM_ENTRY_INSTRUCTION_LEN, 1632 vmx->vcpu.arch.event_exit_inst_len); 1633 intr_info |= INTR_TYPE_SOFT_EXCEPTION; 1634 } else 1635 intr_info |= INTR_TYPE_HARD_EXCEPTION; 1636 1637 vmcs_write32(VM_ENTRY_INTR_INFO_FIELD, intr_info); 1638 1639 vmx_clear_hlt(vcpu); 1640 } 1641 1642 static void vmx_setup_uret_msr(struct vcpu_vmx *vmx, unsigned int msr, 1643 bool load_into_hardware) 1644 { 1645 struct vmx_uret_msr *uret_msr; 1646 1647 uret_msr = vmx_find_uret_msr(vmx, msr); 1648 if (!uret_msr) 1649 return; 1650 1651 uret_msr->load_into_hardware = load_into_hardware; 1652 } 1653 1654 /* 1655 * Configuring user return MSRs to automatically save, load, and restore MSRs 1656 * that need to be shoved into hardware when running the guest. Note, omitting 1657 * an MSR here does _NOT_ mean it's not emulated, only that it will not be 1658 * loaded into hardware when running the guest. 1659 */ 1660 static void vmx_setup_uret_msrs(struct vcpu_vmx *vmx) 1661 { 1662 #ifdef CONFIG_X86_64 1663 bool load_syscall_msrs; 1664 1665 /* 1666 * The SYSCALL MSRs are only needed on long mode guests, and only 1667 * when EFER.SCE is set. 1668 */ 1669 load_syscall_msrs = is_long_mode(&vmx->vcpu) && 1670 (vmx->vcpu.arch.efer & EFER_SCE); 1671 1672 vmx_setup_uret_msr(vmx, MSR_STAR, load_syscall_msrs); 1673 vmx_setup_uret_msr(vmx, MSR_LSTAR, load_syscall_msrs); 1674 vmx_setup_uret_msr(vmx, MSR_SYSCALL_MASK, load_syscall_msrs); 1675 #endif 1676 vmx_setup_uret_msr(vmx, MSR_EFER, update_transition_efer(vmx)); 1677 1678 vmx_setup_uret_msr(vmx, MSR_TSC_AUX, 1679 guest_cpuid_has(&vmx->vcpu, X86_FEATURE_RDTSCP) || 1680 guest_cpuid_has(&vmx->vcpu, X86_FEATURE_RDPID)); 1681 1682 /* 1683 * hle=0, rtm=0, tsx_ctrl=1 can be found with some combinations of new 1684 * kernel and old userspace. If those guests run on a tsx=off host, do 1685 * allow guests to use TSX_CTRL, but don't change the value in hardware 1686 * so that TSX remains always disabled. 1687 */ 1688 vmx_setup_uret_msr(vmx, MSR_IA32_TSX_CTRL, boot_cpu_has(X86_FEATURE_RTM)); 1689 1690 /* 1691 * The set of MSRs to load may have changed, reload MSRs before the 1692 * next VM-Enter. 1693 */ 1694 vmx->guest_uret_msrs_loaded = false; 1695 } 1696 1697 u64 vmx_get_l2_tsc_offset(struct kvm_vcpu *vcpu) 1698 { 1699 struct vmcs12 *vmcs12 = get_vmcs12(vcpu); 1700 1701 if (nested_cpu_has(vmcs12, CPU_BASED_USE_TSC_OFFSETTING)) 1702 return vmcs12->tsc_offset; 1703 1704 return 0; 1705 } 1706 1707 u64 vmx_get_l2_tsc_multiplier(struct kvm_vcpu *vcpu) 1708 { 1709 struct vmcs12 *vmcs12 = get_vmcs12(vcpu); 1710 1711 if (nested_cpu_has(vmcs12, CPU_BASED_USE_TSC_OFFSETTING) && 1712 nested_cpu_has2(vmcs12, SECONDARY_EXEC_TSC_SCALING)) 1713 return vmcs12->tsc_multiplier; 1714 1715 return kvm_default_tsc_scaling_ratio; 1716 } 1717 1718 static void vmx_write_tsc_offset(struct kvm_vcpu *vcpu, u64 offset) 1719 { 1720 vmcs_write64(TSC_OFFSET, offset); 1721 } 1722 1723 static void vmx_write_tsc_multiplier(struct kvm_vcpu *vcpu, u64 multiplier) 1724 { 1725 vmcs_write64(TSC_MULTIPLIER, multiplier); 1726 } 1727 1728 /* 1729 * nested_vmx_allowed() checks whether a guest should be allowed to use VMX 1730 * instructions and MSRs (i.e., nested VMX). Nested VMX is disabled for 1731 * all guests if the "nested" module option is off, and can also be disabled 1732 * for a single guest by disabling its VMX cpuid bit. 1733 */ 1734 bool nested_vmx_allowed(struct kvm_vcpu *vcpu) 1735 { 1736 return nested && guest_cpuid_has(vcpu, X86_FEATURE_VMX); 1737 } 1738 1739 static inline bool vmx_feature_control_msr_valid(struct kvm_vcpu *vcpu, 1740 uint64_t val) 1741 { 1742 uint64_t valid_bits = to_vmx(vcpu)->msr_ia32_feature_control_valid_bits; 1743 1744 return !(val & ~valid_bits); 1745 } 1746 1747 static int vmx_get_msr_feature(struct kvm_msr_entry *msr) 1748 { 1749 switch (msr->index) { 1750 case MSR_IA32_VMX_BASIC ... MSR_IA32_VMX_VMFUNC: 1751 if (!nested) 1752 return 1; 1753 return vmx_get_vmx_msr(&vmcs_config.nested, msr->index, &msr->data); 1754 case MSR_IA32_PERF_CAPABILITIES: 1755 msr->data = vmx_get_perf_capabilities(); 1756 return 0; 1757 default: 1758 return KVM_MSR_RET_INVALID; 1759 } 1760 } 1761 1762 /* 1763 * Reads an msr value (of 'msr_info->index') into 'msr_info->data'. 1764 * Returns 0 on success, non-0 otherwise. 1765 * Assumes vcpu_load() was already called. 1766 */ 1767 static int vmx_get_msr(struct kvm_vcpu *vcpu, struct msr_data *msr_info) 1768 { 1769 struct vcpu_vmx *vmx = to_vmx(vcpu); 1770 struct vmx_uret_msr *msr; 1771 u32 index; 1772 1773 switch (msr_info->index) { 1774 #ifdef CONFIG_X86_64 1775 case MSR_FS_BASE: 1776 msr_info->data = vmcs_readl(GUEST_FS_BASE); 1777 break; 1778 case MSR_GS_BASE: 1779 msr_info->data = vmcs_readl(GUEST_GS_BASE); 1780 break; 1781 case MSR_KERNEL_GS_BASE: 1782 msr_info->data = vmx_read_guest_kernel_gs_base(vmx); 1783 break; 1784 #endif 1785 case MSR_EFER: 1786 return kvm_get_msr_common(vcpu, msr_info); 1787 case MSR_IA32_TSX_CTRL: 1788 if (!msr_info->host_initiated && 1789 !(vcpu->arch.arch_capabilities & ARCH_CAP_TSX_CTRL_MSR)) 1790 return 1; 1791 goto find_uret_msr; 1792 case MSR_IA32_UMWAIT_CONTROL: 1793 if (!msr_info->host_initiated && !vmx_has_waitpkg(vmx)) 1794 return 1; 1795 1796 msr_info->data = vmx->msr_ia32_umwait_control; 1797 break; 1798 case MSR_IA32_SPEC_CTRL: 1799 if (!msr_info->host_initiated && 1800 !guest_has_spec_ctrl_msr(vcpu)) 1801 return 1; 1802 1803 msr_info->data = to_vmx(vcpu)->spec_ctrl; 1804 break; 1805 case MSR_IA32_SYSENTER_CS: 1806 msr_info->data = vmcs_read32(GUEST_SYSENTER_CS); 1807 break; 1808 case MSR_IA32_SYSENTER_EIP: 1809 msr_info->data = vmcs_readl(GUEST_SYSENTER_EIP); 1810 break; 1811 case MSR_IA32_SYSENTER_ESP: 1812 msr_info->data = vmcs_readl(GUEST_SYSENTER_ESP); 1813 break; 1814 case MSR_IA32_BNDCFGS: 1815 if (!kvm_mpx_supported() || 1816 (!msr_info->host_initiated && 1817 !guest_cpuid_has(vcpu, X86_FEATURE_MPX))) 1818 return 1; 1819 msr_info->data = vmcs_read64(GUEST_BNDCFGS); 1820 break; 1821 case MSR_IA32_MCG_EXT_CTL: 1822 if (!msr_info->host_initiated && 1823 !(vmx->msr_ia32_feature_control & 1824 FEAT_CTL_LMCE_ENABLED)) 1825 return 1; 1826 msr_info->data = vcpu->arch.mcg_ext_ctl; 1827 break; 1828 case MSR_IA32_FEAT_CTL: 1829 msr_info->data = vmx->msr_ia32_feature_control; 1830 break; 1831 case MSR_IA32_SGXLEPUBKEYHASH0 ... MSR_IA32_SGXLEPUBKEYHASH3: 1832 if (!msr_info->host_initiated && 1833 !guest_cpuid_has(vcpu, X86_FEATURE_SGX_LC)) 1834 return 1; 1835 msr_info->data = to_vmx(vcpu)->msr_ia32_sgxlepubkeyhash 1836 [msr_info->index - MSR_IA32_SGXLEPUBKEYHASH0]; 1837 break; 1838 case MSR_IA32_VMX_BASIC ... MSR_IA32_VMX_VMFUNC: 1839 if (!nested_vmx_allowed(vcpu)) 1840 return 1; 1841 if (vmx_get_vmx_msr(&vmx->nested.msrs, msr_info->index, 1842 &msr_info->data)) 1843 return 1; 1844 /* 1845 * Enlightened VMCS v1 doesn't have certain VMCS fields but 1846 * instead of just ignoring the features, different Hyper-V 1847 * versions are either trying to use them and fail or do some 1848 * sanity checking and refuse to boot. Filter all unsupported 1849 * features out. 1850 */ 1851 if (!msr_info->host_initiated && 1852 vmx->nested.enlightened_vmcs_enabled) 1853 nested_evmcs_filter_control_msr(msr_info->index, 1854 &msr_info->data); 1855 break; 1856 case MSR_IA32_RTIT_CTL: 1857 if (!vmx_pt_mode_is_host_guest()) 1858 return 1; 1859 msr_info->data = vmx->pt_desc.guest.ctl; 1860 break; 1861 case MSR_IA32_RTIT_STATUS: 1862 if (!vmx_pt_mode_is_host_guest()) 1863 return 1; 1864 msr_info->data = vmx->pt_desc.guest.status; 1865 break; 1866 case MSR_IA32_RTIT_CR3_MATCH: 1867 if (!vmx_pt_mode_is_host_guest() || 1868 !intel_pt_validate_cap(vmx->pt_desc.caps, 1869 PT_CAP_cr3_filtering)) 1870 return 1; 1871 msr_info->data = vmx->pt_desc.guest.cr3_match; 1872 break; 1873 case MSR_IA32_RTIT_OUTPUT_BASE: 1874 if (!vmx_pt_mode_is_host_guest() || 1875 (!intel_pt_validate_cap(vmx->pt_desc.caps, 1876 PT_CAP_topa_output) && 1877 !intel_pt_validate_cap(vmx->pt_desc.caps, 1878 PT_CAP_single_range_output))) 1879 return 1; 1880 msr_info->data = vmx->pt_desc.guest.output_base; 1881 break; 1882 case MSR_IA32_RTIT_OUTPUT_MASK: 1883 if (!vmx_pt_mode_is_host_guest() || 1884 (!intel_pt_validate_cap(vmx->pt_desc.caps, 1885 PT_CAP_topa_output) && 1886 !intel_pt_validate_cap(vmx->pt_desc.caps, 1887 PT_CAP_single_range_output))) 1888 return 1; 1889 msr_info->data = vmx->pt_desc.guest.output_mask; 1890 break; 1891 case MSR_IA32_RTIT_ADDR0_A ... MSR_IA32_RTIT_ADDR3_B: 1892 index = msr_info->index - MSR_IA32_RTIT_ADDR0_A; 1893 if (!vmx_pt_mode_is_host_guest() || 1894 (index >= 2 * vmx->pt_desc.num_address_ranges)) 1895 return 1; 1896 if (index % 2) 1897 msr_info->data = vmx->pt_desc.guest.addr_b[index / 2]; 1898 else 1899 msr_info->data = vmx->pt_desc.guest.addr_a[index / 2]; 1900 break; 1901 case MSR_IA32_DEBUGCTLMSR: 1902 msr_info->data = vmcs_read64(GUEST_IA32_DEBUGCTL); 1903 break; 1904 default: 1905 find_uret_msr: 1906 msr = vmx_find_uret_msr(vmx, msr_info->index); 1907 if (msr) { 1908 msr_info->data = msr->data; 1909 break; 1910 } 1911 return kvm_get_msr_common(vcpu, msr_info); 1912 } 1913 1914 return 0; 1915 } 1916 1917 static u64 nested_vmx_truncate_sysenter_addr(struct kvm_vcpu *vcpu, 1918 u64 data) 1919 { 1920 #ifdef CONFIG_X86_64 1921 if (!guest_cpuid_has(vcpu, X86_FEATURE_LM)) 1922 return (u32)data; 1923 #endif 1924 return (unsigned long)data; 1925 } 1926 1927 static u64 vcpu_supported_debugctl(struct kvm_vcpu *vcpu) 1928 { 1929 u64 debugctl = vmx_supported_debugctl(); 1930 1931 if (!intel_pmu_lbr_is_enabled(vcpu)) 1932 debugctl &= ~DEBUGCTLMSR_LBR_MASK; 1933 1934 if (!guest_cpuid_has(vcpu, X86_FEATURE_BUS_LOCK_DETECT)) 1935 debugctl &= ~DEBUGCTLMSR_BUS_LOCK_DETECT; 1936 1937 return debugctl; 1938 } 1939 1940 /* 1941 * Writes msr value into the appropriate "register". 1942 * Returns 0 on success, non-0 otherwise. 1943 * Assumes vcpu_load() was already called. 1944 */ 1945 static int vmx_set_msr(struct kvm_vcpu *vcpu, struct msr_data *msr_info) 1946 { 1947 struct vcpu_vmx *vmx = to_vmx(vcpu); 1948 struct vmx_uret_msr *msr; 1949 int ret = 0; 1950 u32 msr_index = msr_info->index; 1951 u64 data = msr_info->data; 1952 u32 index; 1953 1954 switch (msr_index) { 1955 case MSR_EFER: 1956 ret = kvm_set_msr_common(vcpu, msr_info); 1957 break; 1958 #ifdef CONFIG_X86_64 1959 case MSR_FS_BASE: 1960 vmx_segment_cache_clear(vmx); 1961 vmcs_writel(GUEST_FS_BASE, data); 1962 break; 1963 case MSR_GS_BASE: 1964 vmx_segment_cache_clear(vmx); 1965 vmcs_writel(GUEST_GS_BASE, data); 1966 break; 1967 case MSR_KERNEL_GS_BASE: 1968 vmx_write_guest_kernel_gs_base(vmx, data); 1969 break; 1970 case MSR_IA32_XFD: 1971 ret = kvm_set_msr_common(vcpu, msr_info); 1972 /* 1973 * Always intercepting WRMSR could incur non-negligible 1974 * overhead given xfd might be changed frequently in 1975 * guest context switch. Disable write interception 1976 * upon the first write with a non-zero value (indicating 1977 * potential usage on dynamic xfeatures). Also update 1978 * exception bitmap to trap #NM for proper virtualization 1979 * of guest xfd_err. 1980 */ 1981 if (!ret && data) { 1982 vmx_disable_intercept_for_msr(vcpu, MSR_IA32_XFD, 1983 MSR_TYPE_RW); 1984 vcpu->arch.xfd_no_write_intercept = true; 1985 vmx_update_exception_bitmap(vcpu); 1986 } 1987 break; 1988 #endif 1989 case MSR_IA32_SYSENTER_CS: 1990 if (is_guest_mode(vcpu)) 1991 get_vmcs12(vcpu)->guest_sysenter_cs = data; 1992 vmcs_write32(GUEST_SYSENTER_CS, data); 1993 break; 1994 case MSR_IA32_SYSENTER_EIP: 1995 if (is_guest_mode(vcpu)) { 1996 data = nested_vmx_truncate_sysenter_addr(vcpu, data); 1997 get_vmcs12(vcpu)->guest_sysenter_eip = data; 1998 } 1999 vmcs_writel(GUEST_SYSENTER_EIP, data); 2000 break; 2001 case MSR_IA32_SYSENTER_ESP: 2002 if (is_guest_mode(vcpu)) { 2003 data = nested_vmx_truncate_sysenter_addr(vcpu, data); 2004 get_vmcs12(vcpu)->guest_sysenter_esp = data; 2005 } 2006 vmcs_writel(GUEST_SYSENTER_ESP, data); 2007 break; 2008 case MSR_IA32_DEBUGCTLMSR: { 2009 u64 invalid = data & ~vcpu_supported_debugctl(vcpu); 2010 if (invalid & (DEBUGCTLMSR_BTF|DEBUGCTLMSR_LBR)) { 2011 if (report_ignored_msrs) 2012 vcpu_unimpl(vcpu, "%s: BTF|LBR in IA32_DEBUGCTLMSR 0x%llx, nop\n", 2013 __func__, data); 2014 data &= ~(DEBUGCTLMSR_BTF|DEBUGCTLMSR_LBR); 2015 invalid &= ~(DEBUGCTLMSR_BTF|DEBUGCTLMSR_LBR); 2016 } 2017 2018 if (invalid) 2019 return 1; 2020 2021 if (is_guest_mode(vcpu) && get_vmcs12(vcpu)->vm_exit_controls & 2022 VM_EXIT_SAVE_DEBUG_CONTROLS) 2023 get_vmcs12(vcpu)->guest_ia32_debugctl = data; 2024 2025 vmcs_write64(GUEST_IA32_DEBUGCTL, data); 2026 if (intel_pmu_lbr_is_enabled(vcpu) && !to_vmx(vcpu)->lbr_desc.event && 2027 (data & DEBUGCTLMSR_LBR)) 2028 intel_pmu_create_guest_lbr_event(vcpu); 2029 return 0; 2030 } 2031 case MSR_IA32_BNDCFGS: 2032 if (!kvm_mpx_supported() || 2033 (!msr_info->host_initiated && 2034 !guest_cpuid_has(vcpu, X86_FEATURE_MPX))) 2035 return 1; 2036 if (is_noncanonical_address(data & PAGE_MASK, vcpu) || 2037 (data & MSR_IA32_BNDCFGS_RSVD)) 2038 return 1; 2039 vmcs_write64(GUEST_BNDCFGS, data); 2040 break; 2041 case MSR_IA32_UMWAIT_CONTROL: 2042 if (!msr_info->host_initiated && !vmx_has_waitpkg(vmx)) 2043 return 1; 2044 2045 /* The reserved bit 1 and non-32 bit [63:32] should be zero */ 2046 if (data & (BIT_ULL(1) | GENMASK_ULL(63, 32))) 2047 return 1; 2048 2049 vmx->msr_ia32_umwait_control = data; 2050 break; 2051 case MSR_IA32_SPEC_CTRL: 2052 if (!msr_info->host_initiated && 2053 !guest_has_spec_ctrl_msr(vcpu)) 2054 return 1; 2055 2056 if (kvm_spec_ctrl_test_value(data)) 2057 return 1; 2058 2059 vmx->spec_ctrl = data; 2060 if (!data) 2061 break; 2062 2063 /* 2064 * For non-nested: 2065 * When it's written (to non-zero) for the first time, pass 2066 * it through. 2067 * 2068 * For nested: 2069 * The handling of the MSR bitmap for L2 guests is done in 2070 * nested_vmx_prepare_msr_bitmap. We should not touch the 2071 * vmcs02.msr_bitmap here since it gets completely overwritten 2072 * in the merging. We update the vmcs01 here for L1 as well 2073 * since it will end up touching the MSR anyway now. 2074 */ 2075 vmx_disable_intercept_for_msr(vcpu, 2076 MSR_IA32_SPEC_CTRL, 2077 MSR_TYPE_RW); 2078 break; 2079 case MSR_IA32_TSX_CTRL: 2080 if (!msr_info->host_initiated && 2081 !(vcpu->arch.arch_capabilities & ARCH_CAP_TSX_CTRL_MSR)) 2082 return 1; 2083 if (data & ~(TSX_CTRL_RTM_DISABLE | TSX_CTRL_CPUID_CLEAR)) 2084 return 1; 2085 goto find_uret_msr; 2086 case MSR_IA32_PRED_CMD: 2087 if (!msr_info->host_initiated && 2088 !guest_has_pred_cmd_msr(vcpu)) 2089 return 1; 2090 2091 if (data & ~PRED_CMD_IBPB) 2092 return 1; 2093 if (!boot_cpu_has(X86_FEATURE_IBPB)) 2094 return 1; 2095 if (!data) 2096 break; 2097 2098 wrmsrl(MSR_IA32_PRED_CMD, PRED_CMD_IBPB); 2099 2100 /* 2101 * For non-nested: 2102 * When it's written (to non-zero) for the first time, pass 2103 * it through. 2104 * 2105 * For nested: 2106 * The handling of the MSR bitmap for L2 guests is done in 2107 * nested_vmx_prepare_msr_bitmap. We should not touch the 2108 * vmcs02.msr_bitmap here since it gets completely overwritten 2109 * in the merging. 2110 */ 2111 vmx_disable_intercept_for_msr(vcpu, MSR_IA32_PRED_CMD, MSR_TYPE_W); 2112 break; 2113 case MSR_IA32_CR_PAT: 2114 if (!kvm_pat_valid(data)) 2115 return 1; 2116 2117 if (is_guest_mode(vcpu) && 2118 get_vmcs12(vcpu)->vm_exit_controls & VM_EXIT_SAVE_IA32_PAT) 2119 get_vmcs12(vcpu)->guest_ia32_pat = data; 2120 2121 if (vmcs_config.vmentry_ctrl & VM_ENTRY_LOAD_IA32_PAT) { 2122 vmcs_write64(GUEST_IA32_PAT, data); 2123 vcpu->arch.pat = data; 2124 break; 2125 } 2126 ret = kvm_set_msr_common(vcpu, msr_info); 2127 break; 2128 case MSR_IA32_MCG_EXT_CTL: 2129 if ((!msr_info->host_initiated && 2130 !(to_vmx(vcpu)->msr_ia32_feature_control & 2131 FEAT_CTL_LMCE_ENABLED)) || 2132 (data & ~MCG_EXT_CTL_LMCE_EN)) 2133 return 1; 2134 vcpu->arch.mcg_ext_ctl = data; 2135 break; 2136 case MSR_IA32_FEAT_CTL: 2137 if (!vmx_feature_control_msr_valid(vcpu, data) || 2138 (to_vmx(vcpu)->msr_ia32_feature_control & 2139 FEAT_CTL_LOCKED && !msr_info->host_initiated)) 2140 return 1; 2141 vmx->msr_ia32_feature_control = data; 2142 if (msr_info->host_initiated && data == 0) 2143 vmx_leave_nested(vcpu); 2144 2145 /* SGX may be enabled/disabled by guest's firmware */ 2146 vmx_write_encls_bitmap(vcpu, NULL); 2147 break; 2148 case MSR_IA32_SGXLEPUBKEYHASH0 ... MSR_IA32_SGXLEPUBKEYHASH3: 2149 /* 2150 * On real hardware, the LE hash MSRs are writable before 2151 * the firmware sets bit 0 in MSR 0x7a ("activating" SGX), 2152 * at which point SGX related bits in IA32_FEATURE_CONTROL 2153 * become writable. 2154 * 2155 * KVM does not emulate SGX activation for simplicity, so 2156 * allow writes to the LE hash MSRs if IA32_FEATURE_CONTROL 2157 * is unlocked. This is technically not architectural 2158 * behavior, but it's close enough. 2159 */ 2160 if (!msr_info->host_initiated && 2161 (!guest_cpuid_has(vcpu, X86_FEATURE_SGX_LC) || 2162 ((vmx->msr_ia32_feature_control & FEAT_CTL_LOCKED) && 2163 !(vmx->msr_ia32_feature_control & FEAT_CTL_SGX_LC_ENABLED)))) 2164 return 1; 2165 vmx->msr_ia32_sgxlepubkeyhash 2166 [msr_index - MSR_IA32_SGXLEPUBKEYHASH0] = data; 2167 break; 2168 case MSR_IA32_VMX_BASIC ... MSR_IA32_VMX_VMFUNC: 2169 if (!msr_info->host_initiated) 2170 return 1; /* they are read-only */ 2171 if (!nested_vmx_allowed(vcpu)) 2172 return 1; 2173 return vmx_set_vmx_msr(vcpu, msr_index, data); 2174 case MSR_IA32_RTIT_CTL: 2175 if (!vmx_pt_mode_is_host_guest() || 2176 vmx_rtit_ctl_check(vcpu, data) || 2177 vmx->nested.vmxon) 2178 return 1; 2179 vmcs_write64(GUEST_IA32_RTIT_CTL, data); 2180 vmx->pt_desc.guest.ctl = data; 2181 pt_update_intercept_for_msr(vcpu); 2182 break; 2183 case MSR_IA32_RTIT_STATUS: 2184 if (!pt_can_write_msr(vmx)) 2185 return 1; 2186 if (data & MSR_IA32_RTIT_STATUS_MASK) 2187 return 1; 2188 vmx->pt_desc.guest.status = data; 2189 break; 2190 case MSR_IA32_RTIT_CR3_MATCH: 2191 if (!pt_can_write_msr(vmx)) 2192 return 1; 2193 if (!intel_pt_validate_cap(vmx->pt_desc.caps, 2194 PT_CAP_cr3_filtering)) 2195 return 1; 2196 vmx->pt_desc.guest.cr3_match = data; 2197 break; 2198 case MSR_IA32_RTIT_OUTPUT_BASE: 2199 if (!pt_can_write_msr(vmx)) 2200 return 1; 2201 if (!intel_pt_validate_cap(vmx->pt_desc.caps, 2202 PT_CAP_topa_output) && 2203 !intel_pt_validate_cap(vmx->pt_desc.caps, 2204 PT_CAP_single_range_output)) 2205 return 1; 2206 if (!pt_output_base_valid(vcpu, data)) 2207 return 1; 2208 vmx->pt_desc.guest.output_base = data; 2209 break; 2210 case MSR_IA32_RTIT_OUTPUT_MASK: 2211 if (!pt_can_write_msr(vmx)) 2212 return 1; 2213 if (!intel_pt_validate_cap(vmx->pt_desc.caps, 2214 PT_CAP_topa_output) && 2215 !intel_pt_validate_cap(vmx->pt_desc.caps, 2216 PT_CAP_single_range_output)) 2217 return 1; 2218 vmx->pt_desc.guest.output_mask = data; 2219 break; 2220 case MSR_IA32_RTIT_ADDR0_A ... MSR_IA32_RTIT_ADDR3_B: 2221 if (!pt_can_write_msr(vmx)) 2222 return 1; 2223 index = msr_info->index - MSR_IA32_RTIT_ADDR0_A; 2224 if (index >= 2 * vmx->pt_desc.num_address_ranges) 2225 return 1; 2226 if (is_noncanonical_address(data, vcpu)) 2227 return 1; 2228 if (index % 2) 2229 vmx->pt_desc.guest.addr_b[index / 2] = data; 2230 else 2231 vmx->pt_desc.guest.addr_a[index / 2] = data; 2232 break; 2233 case MSR_IA32_PERF_CAPABILITIES: 2234 if (data && !vcpu_to_pmu(vcpu)->version) 2235 return 1; 2236 if (data & PMU_CAP_LBR_FMT) { 2237 if ((data & PMU_CAP_LBR_FMT) != 2238 (vmx_get_perf_capabilities() & PMU_CAP_LBR_FMT)) 2239 return 1; 2240 if (!intel_pmu_lbr_is_compatible(vcpu)) 2241 return 1; 2242 } 2243 ret = kvm_set_msr_common(vcpu, msr_info); 2244 break; 2245 2246 default: 2247 find_uret_msr: 2248 msr = vmx_find_uret_msr(vmx, msr_index); 2249 if (msr) 2250 ret = vmx_set_guest_uret_msr(vmx, msr, data); 2251 else 2252 ret = kvm_set_msr_common(vcpu, msr_info); 2253 } 2254 2255 return ret; 2256 } 2257 2258 static void vmx_cache_reg(struct kvm_vcpu *vcpu, enum kvm_reg reg) 2259 { 2260 unsigned long guest_owned_bits; 2261 2262 kvm_register_mark_available(vcpu, reg); 2263 2264 switch (reg) { 2265 case VCPU_REGS_RSP: 2266 vcpu->arch.regs[VCPU_REGS_RSP] = vmcs_readl(GUEST_RSP); 2267 break; 2268 case VCPU_REGS_RIP: 2269 vcpu->arch.regs[VCPU_REGS_RIP] = vmcs_readl(GUEST_RIP); 2270 break; 2271 case VCPU_EXREG_PDPTR: 2272 if (enable_ept) 2273 ept_save_pdptrs(vcpu); 2274 break; 2275 case VCPU_EXREG_CR0: 2276 guest_owned_bits = vcpu->arch.cr0_guest_owned_bits; 2277 2278 vcpu->arch.cr0 &= ~guest_owned_bits; 2279 vcpu->arch.cr0 |= vmcs_readl(GUEST_CR0) & guest_owned_bits; 2280 break; 2281 case VCPU_EXREG_CR3: 2282 /* 2283 * When intercepting CR3 loads, e.g. for shadowing paging, KVM's 2284 * CR3 is loaded into hardware, not the guest's CR3. 2285 */ 2286 if (!(exec_controls_get(to_vmx(vcpu)) & CPU_BASED_CR3_LOAD_EXITING)) 2287 vcpu->arch.cr3 = vmcs_readl(GUEST_CR3); 2288 break; 2289 case VCPU_EXREG_CR4: 2290 guest_owned_bits = vcpu->arch.cr4_guest_owned_bits; 2291 2292 vcpu->arch.cr4 &= ~guest_owned_bits; 2293 vcpu->arch.cr4 |= vmcs_readl(GUEST_CR4) & guest_owned_bits; 2294 break; 2295 default: 2296 KVM_BUG_ON(1, vcpu->kvm); 2297 break; 2298 } 2299 } 2300 2301 static __init int cpu_has_kvm_support(void) 2302 { 2303 return cpu_has_vmx(); 2304 } 2305 2306 static __init int vmx_disabled_by_bios(void) 2307 { 2308 return !boot_cpu_has(X86_FEATURE_MSR_IA32_FEAT_CTL) || 2309 !boot_cpu_has(X86_FEATURE_VMX); 2310 } 2311 2312 static int kvm_cpu_vmxon(u64 vmxon_pointer) 2313 { 2314 u64 msr; 2315 2316 cr4_set_bits(X86_CR4_VMXE); 2317 2318 asm_volatile_goto("1: vmxon %[vmxon_pointer]\n\t" 2319 _ASM_EXTABLE(1b, %l[fault]) 2320 : : [vmxon_pointer] "m"(vmxon_pointer) 2321 : : fault); 2322 return 0; 2323 2324 fault: 2325 WARN_ONCE(1, "VMXON faulted, MSR_IA32_FEAT_CTL (0x3a) = 0x%llx\n", 2326 rdmsrl_safe(MSR_IA32_FEAT_CTL, &msr) ? 0xdeadbeef : msr); 2327 cr4_clear_bits(X86_CR4_VMXE); 2328 2329 return -EFAULT; 2330 } 2331 2332 static int vmx_hardware_enable(void) 2333 { 2334 int cpu = raw_smp_processor_id(); 2335 u64 phys_addr = __pa(per_cpu(vmxarea, cpu)); 2336 int r; 2337 2338 if (cr4_read_shadow() & X86_CR4_VMXE) 2339 return -EBUSY; 2340 2341 /* 2342 * This can happen if we hot-added a CPU but failed to allocate 2343 * VP assist page for it. 2344 */ 2345 if (static_branch_unlikely(&enable_evmcs) && 2346 !hv_get_vp_assist_page(cpu)) 2347 return -EFAULT; 2348 2349 intel_pt_handle_vmx(1); 2350 2351 r = kvm_cpu_vmxon(phys_addr); 2352 if (r) { 2353 intel_pt_handle_vmx(0); 2354 return r; 2355 } 2356 2357 if (enable_ept) 2358 ept_sync_global(); 2359 2360 return 0; 2361 } 2362 2363 static void vmclear_local_loaded_vmcss(void) 2364 { 2365 int cpu = raw_smp_processor_id(); 2366 struct loaded_vmcs *v, *n; 2367 2368 list_for_each_entry_safe(v, n, &per_cpu(loaded_vmcss_on_cpu, cpu), 2369 loaded_vmcss_on_cpu_link) 2370 __loaded_vmcs_clear(v); 2371 } 2372 2373 static void vmx_hardware_disable(void) 2374 { 2375 vmclear_local_loaded_vmcss(); 2376 2377 if (cpu_vmxoff()) 2378 kvm_spurious_fault(); 2379 2380 intel_pt_handle_vmx(0); 2381 } 2382 2383 /* 2384 * There is no X86_FEATURE for SGX yet, but anyway we need to query CPUID 2385 * directly instead of going through cpu_has(), to ensure KVM is trapping 2386 * ENCLS whenever it's supported in hardware. It does not matter whether 2387 * the host OS supports or has enabled SGX. 2388 */ 2389 static bool cpu_has_sgx(void) 2390 { 2391 return cpuid_eax(0) >= 0x12 && (cpuid_eax(0x12) & BIT(0)); 2392 } 2393 2394 static __init int adjust_vmx_controls(u32 ctl_min, u32 ctl_opt, 2395 u32 msr, u32 *result) 2396 { 2397 u32 vmx_msr_low, vmx_msr_high; 2398 u32 ctl = ctl_min | ctl_opt; 2399 2400 rdmsr(msr, vmx_msr_low, vmx_msr_high); 2401 2402 ctl &= vmx_msr_high; /* bit == 0 in high word ==> must be zero */ 2403 ctl |= vmx_msr_low; /* bit == 1 in low word ==> must be one */ 2404 2405 /* Ensure minimum (required) set of control bits are supported. */ 2406 if (ctl_min & ~ctl) 2407 return -EIO; 2408 2409 *result = ctl; 2410 return 0; 2411 } 2412 2413 static __init int setup_vmcs_config(struct vmcs_config *vmcs_conf, 2414 struct vmx_capability *vmx_cap) 2415 { 2416 u32 vmx_msr_low, vmx_msr_high; 2417 u32 min, opt, min2, opt2; 2418 u32 _pin_based_exec_control = 0; 2419 u32 _cpu_based_exec_control = 0; 2420 u32 _cpu_based_2nd_exec_control = 0; 2421 u32 _vmexit_control = 0; 2422 u32 _vmentry_control = 0; 2423 2424 memset(vmcs_conf, 0, sizeof(*vmcs_conf)); 2425 min = CPU_BASED_HLT_EXITING | 2426 #ifdef CONFIG_X86_64 2427 CPU_BASED_CR8_LOAD_EXITING | 2428 CPU_BASED_CR8_STORE_EXITING | 2429 #endif 2430 CPU_BASED_CR3_LOAD_EXITING | 2431 CPU_BASED_CR3_STORE_EXITING | 2432 CPU_BASED_UNCOND_IO_EXITING | 2433 CPU_BASED_MOV_DR_EXITING | 2434 CPU_BASED_USE_TSC_OFFSETTING | 2435 CPU_BASED_MWAIT_EXITING | 2436 CPU_BASED_MONITOR_EXITING | 2437 CPU_BASED_INVLPG_EXITING | 2438 CPU_BASED_RDPMC_EXITING; 2439 2440 opt = CPU_BASED_TPR_SHADOW | 2441 CPU_BASED_USE_MSR_BITMAPS | 2442 CPU_BASED_ACTIVATE_SECONDARY_CONTROLS; 2443 if (adjust_vmx_controls(min, opt, MSR_IA32_VMX_PROCBASED_CTLS, 2444 &_cpu_based_exec_control) < 0) 2445 return -EIO; 2446 #ifdef CONFIG_X86_64 2447 if ((_cpu_based_exec_control & CPU_BASED_TPR_SHADOW)) 2448 _cpu_based_exec_control &= ~CPU_BASED_CR8_LOAD_EXITING & 2449 ~CPU_BASED_CR8_STORE_EXITING; 2450 #endif 2451 if (_cpu_based_exec_control & CPU_BASED_ACTIVATE_SECONDARY_CONTROLS) { 2452 min2 = 0; 2453 opt2 = SECONDARY_EXEC_VIRTUALIZE_APIC_ACCESSES | 2454 SECONDARY_EXEC_VIRTUALIZE_X2APIC_MODE | 2455 SECONDARY_EXEC_WBINVD_EXITING | 2456 SECONDARY_EXEC_ENABLE_VPID | 2457 SECONDARY_EXEC_ENABLE_EPT | 2458 SECONDARY_EXEC_UNRESTRICTED_GUEST | 2459 SECONDARY_EXEC_PAUSE_LOOP_EXITING | 2460 SECONDARY_EXEC_DESC | 2461 SECONDARY_EXEC_ENABLE_RDTSCP | 2462 SECONDARY_EXEC_ENABLE_INVPCID | 2463 SECONDARY_EXEC_APIC_REGISTER_VIRT | 2464 SECONDARY_EXEC_VIRTUAL_INTR_DELIVERY | 2465 SECONDARY_EXEC_SHADOW_VMCS | 2466 SECONDARY_EXEC_XSAVES | 2467 SECONDARY_EXEC_RDSEED_EXITING | 2468 SECONDARY_EXEC_RDRAND_EXITING | 2469 SECONDARY_EXEC_ENABLE_PML | 2470 SECONDARY_EXEC_TSC_SCALING | 2471 SECONDARY_EXEC_ENABLE_USR_WAIT_PAUSE | 2472 SECONDARY_EXEC_PT_USE_GPA | 2473 SECONDARY_EXEC_PT_CONCEAL_VMX | 2474 SECONDARY_EXEC_ENABLE_VMFUNC | 2475 SECONDARY_EXEC_BUS_LOCK_DETECTION; 2476 if (cpu_has_sgx()) 2477 opt2 |= SECONDARY_EXEC_ENCLS_EXITING; 2478 if (adjust_vmx_controls(min2, opt2, 2479 MSR_IA32_VMX_PROCBASED_CTLS2, 2480 &_cpu_based_2nd_exec_control) < 0) 2481 return -EIO; 2482 } 2483 #ifndef CONFIG_X86_64 2484 if (!(_cpu_based_2nd_exec_control & 2485 SECONDARY_EXEC_VIRTUALIZE_APIC_ACCESSES)) 2486 _cpu_based_exec_control &= ~CPU_BASED_TPR_SHADOW; 2487 #endif 2488 2489 if (!(_cpu_based_exec_control & CPU_BASED_TPR_SHADOW)) 2490 _cpu_based_2nd_exec_control &= ~( 2491 SECONDARY_EXEC_APIC_REGISTER_VIRT | 2492 SECONDARY_EXEC_VIRTUALIZE_X2APIC_MODE | 2493 SECONDARY_EXEC_VIRTUAL_INTR_DELIVERY); 2494 2495 rdmsr_safe(MSR_IA32_VMX_EPT_VPID_CAP, 2496 &vmx_cap->ept, &vmx_cap->vpid); 2497 2498 if (_cpu_based_2nd_exec_control & SECONDARY_EXEC_ENABLE_EPT) { 2499 /* CR3 accesses and invlpg don't need to cause VM Exits when EPT 2500 enabled */ 2501 _cpu_based_exec_control &= ~(CPU_BASED_CR3_LOAD_EXITING | 2502 CPU_BASED_CR3_STORE_EXITING | 2503 CPU_BASED_INVLPG_EXITING); 2504 } else if (vmx_cap->ept) { 2505 vmx_cap->ept = 0; 2506 pr_warn_once("EPT CAP should not exist if not support " 2507 "1-setting enable EPT VM-execution control\n"); 2508 } 2509 if (!(_cpu_based_2nd_exec_control & SECONDARY_EXEC_ENABLE_VPID) && 2510 vmx_cap->vpid) { 2511 vmx_cap->vpid = 0; 2512 pr_warn_once("VPID CAP should not exist if not support " 2513 "1-setting enable VPID VM-execution control\n"); 2514 } 2515 2516 min = VM_EXIT_SAVE_DEBUG_CONTROLS | VM_EXIT_ACK_INTR_ON_EXIT; 2517 #ifdef CONFIG_X86_64 2518 min |= VM_EXIT_HOST_ADDR_SPACE_SIZE; 2519 #endif 2520 opt = VM_EXIT_LOAD_IA32_PERF_GLOBAL_CTRL | 2521 VM_EXIT_LOAD_IA32_PAT | 2522 VM_EXIT_LOAD_IA32_EFER | 2523 VM_EXIT_CLEAR_BNDCFGS | 2524 VM_EXIT_PT_CONCEAL_PIP | 2525 VM_EXIT_CLEAR_IA32_RTIT_CTL; 2526 if (adjust_vmx_controls(min, opt, MSR_IA32_VMX_EXIT_CTLS, 2527 &_vmexit_control) < 0) 2528 return -EIO; 2529 2530 min = PIN_BASED_EXT_INTR_MASK | PIN_BASED_NMI_EXITING; 2531 opt = PIN_BASED_VIRTUAL_NMIS | PIN_BASED_POSTED_INTR | 2532 PIN_BASED_VMX_PREEMPTION_TIMER; 2533 if (adjust_vmx_controls(min, opt, MSR_IA32_VMX_PINBASED_CTLS, 2534 &_pin_based_exec_control) < 0) 2535 return -EIO; 2536 2537 if (cpu_has_broken_vmx_preemption_timer()) 2538 _pin_based_exec_control &= ~PIN_BASED_VMX_PREEMPTION_TIMER; 2539 if (!(_cpu_based_2nd_exec_control & 2540 SECONDARY_EXEC_VIRTUAL_INTR_DELIVERY)) 2541 _pin_based_exec_control &= ~PIN_BASED_POSTED_INTR; 2542 2543 min = VM_ENTRY_LOAD_DEBUG_CONTROLS; 2544 opt = VM_ENTRY_LOAD_IA32_PERF_GLOBAL_CTRL | 2545 VM_ENTRY_LOAD_IA32_PAT | 2546 VM_ENTRY_LOAD_IA32_EFER | 2547 VM_ENTRY_LOAD_BNDCFGS | 2548 VM_ENTRY_PT_CONCEAL_PIP | 2549 VM_ENTRY_LOAD_IA32_RTIT_CTL; 2550 if (adjust_vmx_controls(min, opt, MSR_IA32_VMX_ENTRY_CTLS, 2551 &_vmentry_control) < 0) 2552 return -EIO; 2553 2554 /* 2555 * Some cpus support VM_{ENTRY,EXIT}_IA32_PERF_GLOBAL_CTRL but they 2556 * can't be used due to an errata where VM Exit may incorrectly clear 2557 * IA32_PERF_GLOBAL_CTRL[34:32]. Workaround the errata by using the 2558 * MSR load mechanism to switch IA32_PERF_GLOBAL_CTRL. 2559 */ 2560 if (boot_cpu_data.x86 == 0x6) { 2561 switch (boot_cpu_data.x86_model) { 2562 case 26: /* AAK155 */ 2563 case 30: /* AAP115 */ 2564 case 37: /* AAT100 */ 2565 case 44: /* BC86,AAY89,BD102 */ 2566 case 46: /* BA97 */ 2567 _vmentry_control &= ~VM_ENTRY_LOAD_IA32_PERF_GLOBAL_CTRL; 2568 _vmexit_control &= ~VM_EXIT_LOAD_IA32_PERF_GLOBAL_CTRL; 2569 pr_warn_once("kvm: VM_EXIT_LOAD_IA32_PERF_GLOBAL_CTRL " 2570 "does not work properly. Using workaround\n"); 2571 break; 2572 default: 2573 break; 2574 } 2575 } 2576 2577 2578 rdmsr(MSR_IA32_VMX_BASIC, vmx_msr_low, vmx_msr_high); 2579 2580 /* IA-32 SDM Vol 3B: VMCS size is never greater than 4kB. */ 2581 if ((vmx_msr_high & 0x1fff) > PAGE_SIZE) 2582 return -EIO; 2583 2584 #ifdef CONFIG_X86_64 2585 /* IA-32 SDM Vol 3B: 64-bit CPUs always have VMX_BASIC_MSR[48]==0. */ 2586 if (vmx_msr_high & (1u<<16)) 2587 return -EIO; 2588 #endif 2589 2590 /* Require Write-Back (WB) memory type for VMCS accesses. */ 2591 if (((vmx_msr_high >> 18) & 15) != 6) 2592 return -EIO; 2593 2594 vmcs_conf->size = vmx_msr_high & 0x1fff; 2595 vmcs_conf->basic_cap = vmx_msr_high & ~0x1fff; 2596 2597 vmcs_conf->revision_id = vmx_msr_low; 2598 2599 vmcs_conf->pin_based_exec_ctrl = _pin_based_exec_control; 2600 vmcs_conf->cpu_based_exec_ctrl = _cpu_based_exec_control; 2601 vmcs_conf->cpu_based_2nd_exec_ctrl = _cpu_based_2nd_exec_control; 2602 vmcs_conf->vmexit_ctrl = _vmexit_control; 2603 vmcs_conf->vmentry_ctrl = _vmentry_control; 2604 2605 #if IS_ENABLED(CONFIG_HYPERV) 2606 if (enlightened_vmcs) 2607 evmcs_sanitize_exec_ctrls(vmcs_conf); 2608 #endif 2609 2610 return 0; 2611 } 2612 2613 struct vmcs *alloc_vmcs_cpu(bool shadow, int cpu, gfp_t flags) 2614 { 2615 int node = cpu_to_node(cpu); 2616 struct page *pages; 2617 struct vmcs *vmcs; 2618 2619 pages = __alloc_pages_node(node, flags, 0); 2620 if (!pages) 2621 return NULL; 2622 vmcs = page_address(pages); 2623 memset(vmcs, 0, vmcs_config.size); 2624 2625 /* KVM supports Enlightened VMCS v1 only */ 2626 if (static_branch_unlikely(&enable_evmcs)) 2627 vmcs->hdr.revision_id = KVM_EVMCS_VERSION; 2628 else 2629 vmcs->hdr.revision_id = vmcs_config.revision_id; 2630 2631 if (shadow) 2632 vmcs->hdr.shadow_vmcs = 1; 2633 return vmcs; 2634 } 2635 2636 void free_vmcs(struct vmcs *vmcs) 2637 { 2638 free_page((unsigned long)vmcs); 2639 } 2640 2641 /* 2642 * Free a VMCS, but before that VMCLEAR it on the CPU where it was last loaded 2643 */ 2644 void free_loaded_vmcs(struct loaded_vmcs *loaded_vmcs) 2645 { 2646 if (!loaded_vmcs->vmcs) 2647 return; 2648 loaded_vmcs_clear(loaded_vmcs); 2649 free_vmcs(loaded_vmcs->vmcs); 2650 loaded_vmcs->vmcs = NULL; 2651 if (loaded_vmcs->msr_bitmap) 2652 free_page((unsigned long)loaded_vmcs->msr_bitmap); 2653 WARN_ON(loaded_vmcs->shadow_vmcs != NULL); 2654 } 2655 2656 int alloc_loaded_vmcs(struct loaded_vmcs *loaded_vmcs) 2657 { 2658 loaded_vmcs->vmcs = alloc_vmcs(false); 2659 if (!loaded_vmcs->vmcs) 2660 return -ENOMEM; 2661 2662 vmcs_clear(loaded_vmcs->vmcs); 2663 2664 loaded_vmcs->shadow_vmcs = NULL; 2665 loaded_vmcs->hv_timer_soft_disabled = false; 2666 loaded_vmcs->cpu = -1; 2667 loaded_vmcs->launched = 0; 2668 2669 if (cpu_has_vmx_msr_bitmap()) { 2670 loaded_vmcs->msr_bitmap = (unsigned long *) 2671 __get_free_page(GFP_KERNEL_ACCOUNT); 2672 if (!loaded_vmcs->msr_bitmap) 2673 goto out_vmcs; 2674 memset(loaded_vmcs->msr_bitmap, 0xff, PAGE_SIZE); 2675 } 2676 2677 memset(&loaded_vmcs->host_state, 0, sizeof(struct vmcs_host_state)); 2678 memset(&loaded_vmcs->controls_shadow, 0, 2679 sizeof(struct vmcs_controls_shadow)); 2680 2681 return 0; 2682 2683 out_vmcs: 2684 free_loaded_vmcs(loaded_vmcs); 2685 return -ENOMEM; 2686 } 2687 2688 static void free_kvm_area(void) 2689 { 2690 int cpu; 2691 2692 for_each_possible_cpu(cpu) { 2693 free_vmcs(per_cpu(vmxarea, cpu)); 2694 per_cpu(vmxarea, cpu) = NULL; 2695 } 2696 } 2697 2698 static __init int alloc_kvm_area(void) 2699 { 2700 int cpu; 2701 2702 for_each_possible_cpu(cpu) { 2703 struct vmcs *vmcs; 2704 2705 vmcs = alloc_vmcs_cpu(false, cpu, GFP_KERNEL); 2706 if (!vmcs) { 2707 free_kvm_area(); 2708 return -ENOMEM; 2709 } 2710 2711 /* 2712 * When eVMCS is enabled, alloc_vmcs_cpu() sets 2713 * vmcs->revision_id to KVM_EVMCS_VERSION instead of 2714 * revision_id reported by MSR_IA32_VMX_BASIC. 2715 * 2716 * However, even though not explicitly documented by 2717 * TLFS, VMXArea passed as VMXON argument should 2718 * still be marked with revision_id reported by 2719 * physical CPU. 2720 */ 2721 if (static_branch_unlikely(&enable_evmcs)) 2722 vmcs->hdr.revision_id = vmcs_config.revision_id; 2723 2724 per_cpu(vmxarea, cpu) = vmcs; 2725 } 2726 return 0; 2727 } 2728 2729 static void fix_pmode_seg(struct kvm_vcpu *vcpu, int seg, 2730 struct kvm_segment *save) 2731 { 2732 if (!emulate_invalid_guest_state) { 2733 /* 2734 * CS and SS RPL should be equal during guest entry according 2735 * to VMX spec, but in reality it is not always so. Since vcpu 2736 * is in the middle of the transition from real mode to 2737 * protected mode it is safe to assume that RPL 0 is a good 2738 * default value. 2739 */ 2740 if (seg == VCPU_SREG_CS || seg == VCPU_SREG_SS) 2741 save->selector &= ~SEGMENT_RPL_MASK; 2742 save->dpl = save->selector & SEGMENT_RPL_MASK; 2743 save->s = 1; 2744 } 2745 __vmx_set_segment(vcpu, save, seg); 2746 } 2747 2748 static void enter_pmode(struct kvm_vcpu *vcpu) 2749 { 2750 unsigned long flags; 2751 struct vcpu_vmx *vmx = to_vmx(vcpu); 2752 2753 /* 2754 * Update real mode segment cache. It may be not up-to-date if segment 2755 * register was written while vcpu was in a guest mode. 2756 */ 2757 vmx_get_segment(vcpu, &vmx->rmode.segs[VCPU_SREG_ES], VCPU_SREG_ES); 2758 vmx_get_segment(vcpu, &vmx->rmode.segs[VCPU_SREG_DS], VCPU_SREG_DS); 2759 vmx_get_segment(vcpu, &vmx->rmode.segs[VCPU_SREG_FS], VCPU_SREG_FS); 2760 vmx_get_segment(vcpu, &vmx->rmode.segs[VCPU_SREG_GS], VCPU_SREG_GS); 2761 vmx_get_segment(vcpu, &vmx->rmode.segs[VCPU_SREG_SS], VCPU_SREG_SS); 2762 vmx_get_segment(vcpu, &vmx->rmode.segs[VCPU_SREG_CS], VCPU_SREG_CS); 2763 2764 vmx->rmode.vm86_active = 0; 2765 2766 __vmx_set_segment(vcpu, &vmx->rmode.segs[VCPU_SREG_TR], VCPU_SREG_TR); 2767 2768 flags = vmcs_readl(GUEST_RFLAGS); 2769 flags &= RMODE_GUEST_OWNED_EFLAGS_BITS; 2770 flags |= vmx->rmode.save_rflags & ~RMODE_GUEST_OWNED_EFLAGS_BITS; 2771 vmcs_writel(GUEST_RFLAGS, flags); 2772 2773 vmcs_writel(GUEST_CR4, (vmcs_readl(GUEST_CR4) & ~X86_CR4_VME) | 2774 (vmcs_readl(CR4_READ_SHADOW) & X86_CR4_VME)); 2775 2776 vmx_update_exception_bitmap(vcpu); 2777 2778 fix_pmode_seg(vcpu, VCPU_SREG_CS, &vmx->rmode.segs[VCPU_SREG_CS]); 2779 fix_pmode_seg(vcpu, VCPU_SREG_SS, &vmx->rmode.segs[VCPU_SREG_SS]); 2780 fix_pmode_seg(vcpu, VCPU_SREG_ES, &vmx->rmode.segs[VCPU_SREG_ES]); 2781 fix_pmode_seg(vcpu, VCPU_SREG_DS, &vmx->rmode.segs[VCPU_SREG_DS]); 2782 fix_pmode_seg(vcpu, VCPU_SREG_FS, &vmx->rmode.segs[VCPU_SREG_FS]); 2783 fix_pmode_seg(vcpu, VCPU_SREG_GS, &vmx->rmode.segs[VCPU_SREG_GS]); 2784 } 2785 2786 static void fix_rmode_seg(int seg, struct kvm_segment *save) 2787 { 2788 const struct kvm_vmx_segment_field *sf = &kvm_vmx_segment_fields[seg]; 2789 struct kvm_segment var = *save; 2790 2791 var.dpl = 0x3; 2792 if (seg == VCPU_SREG_CS) 2793 var.type = 0x3; 2794 2795 if (!emulate_invalid_guest_state) { 2796 var.selector = var.base >> 4; 2797 var.base = var.base & 0xffff0; 2798 var.limit = 0xffff; 2799 var.g = 0; 2800 var.db = 0; 2801 var.present = 1; 2802 var.s = 1; 2803 var.l = 0; 2804 var.unusable = 0; 2805 var.type = 0x3; 2806 var.avl = 0; 2807 if (save->base & 0xf) 2808 printk_once(KERN_WARNING "kvm: segment base is not " 2809 "paragraph aligned when entering " 2810 "protected mode (seg=%d)", seg); 2811 } 2812 2813 vmcs_write16(sf->selector, var.selector); 2814 vmcs_writel(sf->base, var.base); 2815 vmcs_write32(sf->limit, var.limit); 2816 vmcs_write32(sf->ar_bytes, vmx_segment_access_rights(&var)); 2817 } 2818 2819 static void enter_rmode(struct kvm_vcpu *vcpu) 2820 { 2821 unsigned long flags; 2822 struct vcpu_vmx *vmx = to_vmx(vcpu); 2823 struct kvm_vmx *kvm_vmx = to_kvm_vmx(vcpu->kvm); 2824 2825 vmx_get_segment(vcpu, &vmx->rmode.segs[VCPU_SREG_TR], VCPU_SREG_TR); 2826 vmx_get_segment(vcpu, &vmx->rmode.segs[VCPU_SREG_ES], VCPU_SREG_ES); 2827 vmx_get_segment(vcpu, &vmx->rmode.segs[VCPU_SREG_DS], VCPU_SREG_DS); 2828 vmx_get_segment(vcpu, &vmx->rmode.segs[VCPU_SREG_FS], VCPU_SREG_FS); 2829 vmx_get_segment(vcpu, &vmx->rmode.segs[VCPU_SREG_GS], VCPU_SREG_GS); 2830 vmx_get_segment(vcpu, &vmx->rmode.segs[VCPU_SREG_SS], VCPU_SREG_SS); 2831 vmx_get_segment(vcpu, &vmx->rmode.segs[VCPU_SREG_CS], VCPU_SREG_CS); 2832 2833 vmx->rmode.vm86_active = 1; 2834 2835 /* 2836 * Very old userspace does not call KVM_SET_TSS_ADDR before entering 2837 * vcpu. Warn the user that an update is overdue. 2838 */ 2839 if (!kvm_vmx->tss_addr) 2840 printk_once(KERN_WARNING "kvm: KVM_SET_TSS_ADDR need to be " 2841 "called before entering vcpu\n"); 2842 2843 vmx_segment_cache_clear(vmx); 2844 2845 vmcs_writel(GUEST_TR_BASE, kvm_vmx->tss_addr); 2846 vmcs_write32(GUEST_TR_LIMIT, RMODE_TSS_SIZE - 1); 2847 vmcs_write32(GUEST_TR_AR_BYTES, 0x008b); 2848 2849 flags = vmcs_readl(GUEST_RFLAGS); 2850 vmx->rmode.save_rflags = flags; 2851 2852 flags |= X86_EFLAGS_IOPL | X86_EFLAGS_VM; 2853 2854 vmcs_writel(GUEST_RFLAGS, flags); 2855 vmcs_writel(GUEST_CR4, vmcs_readl(GUEST_CR4) | X86_CR4_VME); 2856 vmx_update_exception_bitmap(vcpu); 2857 2858 fix_rmode_seg(VCPU_SREG_SS, &vmx->rmode.segs[VCPU_SREG_SS]); 2859 fix_rmode_seg(VCPU_SREG_CS, &vmx->rmode.segs[VCPU_SREG_CS]); 2860 fix_rmode_seg(VCPU_SREG_ES, &vmx->rmode.segs[VCPU_SREG_ES]); 2861 fix_rmode_seg(VCPU_SREG_DS, &vmx->rmode.segs[VCPU_SREG_DS]); 2862 fix_rmode_seg(VCPU_SREG_GS, &vmx->rmode.segs[VCPU_SREG_GS]); 2863 fix_rmode_seg(VCPU_SREG_FS, &vmx->rmode.segs[VCPU_SREG_FS]); 2864 } 2865 2866 int vmx_set_efer(struct kvm_vcpu *vcpu, u64 efer) 2867 { 2868 struct vcpu_vmx *vmx = to_vmx(vcpu); 2869 2870 /* Nothing to do if hardware doesn't support EFER. */ 2871 if (!vmx_find_uret_msr(vmx, MSR_EFER)) 2872 return 0; 2873 2874 vcpu->arch.efer = efer; 2875 if (efer & EFER_LMA) 2876 vm_entry_controls_setbit(vmx, VM_ENTRY_IA32E_MODE); 2877 else 2878 vm_entry_controls_clearbit(vmx, VM_ENTRY_IA32E_MODE); 2879 2880 vmx_setup_uret_msrs(vmx); 2881 return 0; 2882 } 2883 2884 #ifdef CONFIG_X86_64 2885 2886 static void enter_lmode(struct kvm_vcpu *vcpu) 2887 { 2888 u32 guest_tr_ar; 2889 2890 vmx_segment_cache_clear(to_vmx(vcpu)); 2891 2892 guest_tr_ar = vmcs_read32(GUEST_TR_AR_BYTES); 2893 if ((guest_tr_ar & VMX_AR_TYPE_MASK) != VMX_AR_TYPE_BUSY_64_TSS) { 2894 pr_debug_ratelimited("%s: tss fixup for long mode. \n", 2895 __func__); 2896 vmcs_write32(GUEST_TR_AR_BYTES, 2897 (guest_tr_ar & ~VMX_AR_TYPE_MASK) 2898 | VMX_AR_TYPE_BUSY_64_TSS); 2899 } 2900 vmx_set_efer(vcpu, vcpu->arch.efer | EFER_LMA); 2901 } 2902 2903 static void exit_lmode(struct kvm_vcpu *vcpu) 2904 { 2905 vmx_set_efer(vcpu, vcpu->arch.efer & ~EFER_LMA); 2906 } 2907 2908 #endif 2909 2910 static void vmx_flush_tlb_all(struct kvm_vcpu *vcpu) 2911 { 2912 struct vcpu_vmx *vmx = to_vmx(vcpu); 2913 2914 /* 2915 * INVEPT must be issued when EPT is enabled, irrespective of VPID, as 2916 * the CPU is not required to invalidate guest-physical mappings on 2917 * VM-Entry, even if VPID is disabled. Guest-physical mappings are 2918 * associated with the root EPT structure and not any particular VPID 2919 * (INVVPID also isn't required to invalidate guest-physical mappings). 2920 */ 2921 if (enable_ept) { 2922 ept_sync_global(); 2923 } else if (enable_vpid) { 2924 if (cpu_has_vmx_invvpid_global()) { 2925 vpid_sync_vcpu_global(); 2926 } else { 2927 vpid_sync_vcpu_single(vmx->vpid); 2928 vpid_sync_vcpu_single(vmx->nested.vpid02); 2929 } 2930 } 2931 } 2932 2933 static inline int vmx_get_current_vpid(struct kvm_vcpu *vcpu) 2934 { 2935 if (is_guest_mode(vcpu)) 2936 return nested_get_vpid02(vcpu); 2937 return to_vmx(vcpu)->vpid; 2938 } 2939 2940 static void vmx_flush_tlb_current(struct kvm_vcpu *vcpu) 2941 { 2942 struct kvm_mmu *mmu = vcpu->arch.mmu; 2943 u64 root_hpa = mmu->root.hpa; 2944 2945 /* No flush required if the current context is invalid. */ 2946 if (!VALID_PAGE(root_hpa)) 2947 return; 2948 2949 if (enable_ept) 2950 ept_sync_context(construct_eptp(vcpu, root_hpa, 2951 mmu->shadow_root_level)); 2952 else 2953 vpid_sync_context(vmx_get_current_vpid(vcpu)); 2954 } 2955 2956 static void vmx_flush_tlb_gva(struct kvm_vcpu *vcpu, gva_t addr) 2957 { 2958 /* 2959 * vpid_sync_vcpu_addr() is a nop if vpid==0, see the comment in 2960 * vmx_flush_tlb_guest() for an explanation of why this is ok. 2961 */ 2962 vpid_sync_vcpu_addr(vmx_get_current_vpid(vcpu), addr); 2963 } 2964 2965 static void vmx_flush_tlb_guest(struct kvm_vcpu *vcpu) 2966 { 2967 /* 2968 * vpid_sync_context() is a nop if vpid==0, e.g. if enable_vpid==0 or a 2969 * vpid couldn't be allocated for this vCPU. VM-Enter and VM-Exit are 2970 * required to flush GVA->{G,H}PA mappings from the TLB if vpid is 2971 * disabled (VM-Enter with vpid enabled and vpid==0 is disallowed), 2972 * i.e. no explicit INVVPID is necessary. 2973 */ 2974 vpid_sync_context(vmx_get_current_vpid(vcpu)); 2975 } 2976 2977 void vmx_ept_load_pdptrs(struct kvm_vcpu *vcpu) 2978 { 2979 struct kvm_mmu *mmu = vcpu->arch.walk_mmu; 2980 2981 if (!kvm_register_is_dirty(vcpu, VCPU_EXREG_PDPTR)) 2982 return; 2983 2984 if (is_pae_paging(vcpu)) { 2985 vmcs_write64(GUEST_PDPTR0, mmu->pdptrs[0]); 2986 vmcs_write64(GUEST_PDPTR1, mmu->pdptrs[1]); 2987 vmcs_write64(GUEST_PDPTR2, mmu->pdptrs[2]); 2988 vmcs_write64(GUEST_PDPTR3, mmu->pdptrs[3]); 2989 } 2990 } 2991 2992 void ept_save_pdptrs(struct kvm_vcpu *vcpu) 2993 { 2994 struct kvm_mmu *mmu = vcpu->arch.walk_mmu; 2995 2996 if (WARN_ON_ONCE(!is_pae_paging(vcpu))) 2997 return; 2998 2999 mmu->pdptrs[0] = vmcs_read64(GUEST_PDPTR0); 3000 mmu->pdptrs[1] = vmcs_read64(GUEST_PDPTR1); 3001 mmu->pdptrs[2] = vmcs_read64(GUEST_PDPTR2); 3002 mmu->pdptrs[3] = vmcs_read64(GUEST_PDPTR3); 3003 3004 kvm_register_mark_available(vcpu, VCPU_EXREG_PDPTR); 3005 } 3006 3007 #define CR3_EXITING_BITS (CPU_BASED_CR3_LOAD_EXITING | \ 3008 CPU_BASED_CR3_STORE_EXITING) 3009 3010 void vmx_set_cr0(struct kvm_vcpu *vcpu, unsigned long cr0) 3011 { 3012 struct vcpu_vmx *vmx = to_vmx(vcpu); 3013 unsigned long hw_cr0, old_cr0_pg; 3014 u32 tmp; 3015 3016 old_cr0_pg = kvm_read_cr0_bits(vcpu, X86_CR0_PG); 3017 3018 hw_cr0 = (cr0 & ~KVM_VM_CR0_ALWAYS_OFF); 3019 if (is_unrestricted_guest(vcpu)) 3020 hw_cr0 |= KVM_VM_CR0_ALWAYS_ON_UNRESTRICTED_GUEST; 3021 else { 3022 hw_cr0 |= KVM_VM_CR0_ALWAYS_ON; 3023 if (!enable_ept) 3024 hw_cr0 |= X86_CR0_WP; 3025 3026 if (vmx->rmode.vm86_active && (cr0 & X86_CR0_PE)) 3027 enter_pmode(vcpu); 3028 3029 if (!vmx->rmode.vm86_active && !(cr0 & X86_CR0_PE)) 3030 enter_rmode(vcpu); 3031 } 3032 3033 vmcs_writel(CR0_READ_SHADOW, cr0); 3034 vmcs_writel(GUEST_CR0, hw_cr0); 3035 vcpu->arch.cr0 = cr0; 3036 kvm_register_mark_available(vcpu, VCPU_EXREG_CR0); 3037 3038 #ifdef CONFIG_X86_64 3039 if (vcpu->arch.efer & EFER_LME) { 3040 if (!old_cr0_pg && (cr0 & X86_CR0_PG)) 3041 enter_lmode(vcpu); 3042 else if (old_cr0_pg && !(cr0 & X86_CR0_PG)) 3043 exit_lmode(vcpu); 3044 } 3045 #endif 3046 3047 if (enable_ept && !is_unrestricted_guest(vcpu)) { 3048 /* 3049 * Ensure KVM has an up-to-date snapshot of the guest's CR3. If 3050 * the below code _enables_ CR3 exiting, vmx_cache_reg() will 3051 * (correctly) stop reading vmcs.GUEST_CR3 because it thinks 3052 * KVM's CR3 is installed. 3053 */ 3054 if (!kvm_register_is_available(vcpu, VCPU_EXREG_CR3)) 3055 vmx_cache_reg(vcpu, VCPU_EXREG_CR3); 3056 3057 /* 3058 * When running with EPT but not unrestricted guest, KVM must 3059 * intercept CR3 accesses when paging is _disabled_. This is 3060 * necessary because restricted guests can't actually run with 3061 * paging disabled, and so KVM stuffs its own CR3 in order to 3062 * run the guest when identity mapped page tables. 3063 * 3064 * Do _NOT_ check the old CR0.PG, e.g. to optimize away the 3065 * update, it may be stale with respect to CR3 interception, 3066 * e.g. after nested VM-Enter. 3067 * 3068 * Lastly, honor L1's desires, i.e. intercept CR3 loads and/or 3069 * stores to forward them to L1, even if KVM does not need to 3070 * intercept them to preserve its identity mapped page tables. 3071 */ 3072 if (!(cr0 & X86_CR0_PG)) { 3073 exec_controls_setbit(vmx, CR3_EXITING_BITS); 3074 } else if (!is_guest_mode(vcpu)) { 3075 exec_controls_clearbit(vmx, CR3_EXITING_BITS); 3076 } else { 3077 tmp = exec_controls_get(vmx); 3078 tmp &= ~CR3_EXITING_BITS; 3079 tmp |= get_vmcs12(vcpu)->cpu_based_vm_exec_control & CR3_EXITING_BITS; 3080 exec_controls_set(vmx, tmp); 3081 } 3082 3083 /* Note, vmx_set_cr4() consumes the new vcpu->arch.cr0. */ 3084 if ((old_cr0_pg ^ cr0) & X86_CR0_PG) 3085 vmx_set_cr4(vcpu, kvm_read_cr4(vcpu)); 3086 3087 /* 3088 * When !CR0_PG -> CR0_PG, vcpu->arch.cr3 becomes active, but 3089 * GUEST_CR3 is still vmx->ept_identity_map_addr if EPT + !URG. 3090 */ 3091 if (!(old_cr0_pg & X86_CR0_PG) && (cr0 & X86_CR0_PG)) 3092 kvm_register_mark_dirty(vcpu, VCPU_EXREG_CR3); 3093 } 3094 3095 /* depends on vcpu->arch.cr0 to be set to a new value */ 3096 vmx->emulation_required = vmx_emulation_required(vcpu); 3097 } 3098 3099 static int vmx_get_max_tdp_level(void) 3100 { 3101 if (cpu_has_vmx_ept_5levels()) 3102 return 5; 3103 return 4; 3104 } 3105 3106 u64 construct_eptp(struct kvm_vcpu *vcpu, hpa_t root_hpa, int root_level) 3107 { 3108 u64 eptp = VMX_EPTP_MT_WB; 3109 3110 eptp |= (root_level == 5) ? VMX_EPTP_PWL_5 : VMX_EPTP_PWL_4; 3111 3112 if (enable_ept_ad_bits && 3113 (!is_guest_mode(vcpu) || nested_ept_ad_enabled(vcpu))) 3114 eptp |= VMX_EPTP_AD_ENABLE_BIT; 3115 eptp |= root_hpa; 3116 3117 return eptp; 3118 } 3119 3120 static void vmx_load_mmu_pgd(struct kvm_vcpu *vcpu, hpa_t root_hpa, 3121 int root_level) 3122 { 3123 struct kvm *kvm = vcpu->kvm; 3124 bool update_guest_cr3 = true; 3125 unsigned long guest_cr3; 3126 u64 eptp; 3127 3128 if (enable_ept) { 3129 eptp = construct_eptp(vcpu, root_hpa, root_level); 3130 vmcs_write64(EPT_POINTER, eptp); 3131 3132 hv_track_root_tdp(vcpu, root_hpa); 3133 3134 if (!enable_unrestricted_guest && !is_paging(vcpu)) 3135 guest_cr3 = to_kvm_vmx(kvm)->ept_identity_map_addr; 3136 else if (kvm_register_is_dirty(vcpu, VCPU_EXREG_CR3)) 3137 guest_cr3 = vcpu->arch.cr3; 3138 else /* vmcs.GUEST_CR3 is already up-to-date. */ 3139 update_guest_cr3 = false; 3140 vmx_ept_load_pdptrs(vcpu); 3141 } else { 3142 guest_cr3 = root_hpa | kvm_get_active_pcid(vcpu); 3143 } 3144 3145 if (update_guest_cr3) 3146 vmcs_writel(GUEST_CR3, guest_cr3); 3147 } 3148 3149 3150 static bool vmx_is_valid_cr4(struct kvm_vcpu *vcpu, unsigned long cr4) 3151 { 3152 /* 3153 * We operate under the default treatment of SMM, so VMX cannot be 3154 * enabled under SMM. Note, whether or not VMXE is allowed at all is 3155 * handled by kvm_is_valid_cr4(). 3156 */ 3157 if ((cr4 & X86_CR4_VMXE) && is_smm(vcpu)) 3158 return false; 3159 3160 if (to_vmx(vcpu)->nested.vmxon && !nested_cr4_valid(vcpu, cr4)) 3161 return false; 3162 3163 return true; 3164 } 3165 3166 void vmx_set_cr4(struct kvm_vcpu *vcpu, unsigned long cr4) 3167 { 3168 unsigned long old_cr4 = vcpu->arch.cr4; 3169 struct vcpu_vmx *vmx = to_vmx(vcpu); 3170 /* 3171 * Pass through host's Machine Check Enable value to hw_cr4, which 3172 * is in force while we are in guest mode. Do not let guests control 3173 * this bit, even if host CR4.MCE == 0. 3174 */ 3175 unsigned long hw_cr4; 3176 3177 hw_cr4 = (cr4_read_shadow() & X86_CR4_MCE) | (cr4 & ~X86_CR4_MCE); 3178 if (is_unrestricted_guest(vcpu)) 3179 hw_cr4 |= KVM_VM_CR4_ALWAYS_ON_UNRESTRICTED_GUEST; 3180 else if (vmx->rmode.vm86_active) 3181 hw_cr4 |= KVM_RMODE_VM_CR4_ALWAYS_ON; 3182 else 3183 hw_cr4 |= KVM_PMODE_VM_CR4_ALWAYS_ON; 3184 3185 if (!boot_cpu_has(X86_FEATURE_UMIP) && vmx_umip_emulated()) { 3186 if (cr4 & X86_CR4_UMIP) { 3187 secondary_exec_controls_setbit(vmx, SECONDARY_EXEC_DESC); 3188 hw_cr4 &= ~X86_CR4_UMIP; 3189 } else if (!is_guest_mode(vcpu) || 3190 !nested_cpu_has2(get_vmcs12(vcpu), SECONDARY_EXEC_DESC)) { 3191 secondary_exec_controls_clearbit(vmx, SECONDARY_EXEC_DESC); 3192 } 3193 } 3194 3195 vcpu->arch.cr4 = cr4; 3196 kvm_register_mark_available(vcpu, VCPU_EXREG_CR4); 3197 3198 if (!is_unrestricted_guest(vcpu)) { 3199 if (enable_ept) { 3200 if (!is_paging(vcpu)) { 3201 hw_cr4 &= ~X86_CR4_PAE; 3202 hw_cr4 |= X86_CR4_PSE; 3203 } else if (!(cr4 & X86_CR4_PAE)) { 3204 hw_cr4 &= ~X86_CR4_PAE; 3205 } 3206 } 3207 3208 /* 3209 * SMEP/SMAP/PKU is disabled if CPU is in non-paging mode in 3210 * hardware. To emulate this behavior, SMEP/SMAP/PKU needs 3211 * to be manually disabled when guest switches to non-paging 3212 * mode. 3213 * 3214 * If !enable_unrestricted_guest, the CPU is always running 3215 * with CR0.PG=1 and CR4 needs to be modified. 3216 * If enable_unrestricted_guest, the CPU automatically 3217 * disables SMEP/SMAP/PKU when the guest sets CR0.PG=0. 3218 */ 3219 if (!is_paging(vcpu)) 3220 hw_cr4 &= ~(X86_CR4_SMEP | X86_CR4_SMAP | X86_CR4_PKE); 3221 } 3222 3223 vmcs_writel(CR4_READ_SHADOW, cr4); 3224 vmcs_writel(GUEST_CR4, hw_cr4); 3225 3226 if ((cr4 ^ old_cr4) & (X86_CR4_OSXSAVE | X86_CR4_PKE)) 3227 kvm_update_cpuid_runtime(vcpu); 3228 } 3229 3230 void vmx_get_segment(struct kvm_vcpu *vcpu, struct kvm_segment *var, int seg) 3231 { 3232 struct vcpu_vmx *vmx = to_vmx(vcpu); 3233 u32 ar; 3234 3235 if (vmx->rmode.vm86_active && seg != VCPU_SREG_LDTR) { 3236 *var = vmx->rmode.segs[seg]; 3237 if (seg == VCPU_SREG_TR 3238 || var->selector == vmx_read_guest_seg_selector(vmx, seg)) 3239 return; 3240 var->base = vmx_read_guest_seg_base(vmx, seg); 3241 var->selector = vmx_read_guest_seg_selector(vmx, seg); 3242 return; 3243 } 3244 var->base = vmx_read_guest_seg_base(vmx, seg); 3245 var->limit = vmx_read_guest_seg_limit(vmx, seg); 3246 var->selector = vmx_read_guest_seg_selector(vmx, seg); 3247 ar = vmx_read_guest_seg_ar(vmx, seg); 3248 var->unusable = (ar >> 16) & 1; 3249 var->type = ar & 15; 3250 var->s = (ar >> 4) & 1; 3251 var->dpl = (ar >> 5) & 3; 3252 /* 3253 * Some userspaces do not preserve unusable property. Since usable 3254 * segment has to be present according to VMX spec we can use present 3255 * property to amend userspace bug by making unusable segment always 3256 * nonpresent. vmx_segment_access_rights() already marks nonpresent 3257 * segment as unusable. 3258 */ 3259 var->present = !var->unusable; 3260 var->avl = (ar >> 12) & 1; 3261 var->l = (ar >> 13) & 1; 3262 var->db = (ar >> 14) & 1; 3263 var->g = (ar >> 15) & 1; 3264 } 3265 3266 static u64 vmx_get_segment_base(struct kvm_vcpu *vcpu, int seg) 3267 { 3268 struct kvm_segment s; 3269 3270 if (to_vmx(vcpu)->rmode.vm86_active) { 3271 vmx_get_segment(vcpu, &s, seg); 3272 return s.base; 3273 } 3274 return vmx_read_guest_seg_base(to_vmx(vcpu), seg); 3275 } 3276 3277 int vmx_get_cpl(struct kvm_vcpu *vcpu) 3278 { 3279 struct vcpu_vmx *vmx = to_vmx(vcpu); 3280 3281 if (unlikely(vmx->rmode.vm86_active)) 3282 return 0; 3283 else { 3284 int ar = vmx_read_guest_seg_ar(vmx, VCPU_SREG_SS); 3285 return VMX_AR_DPL(ar); 3286 } 3287 } 3288 3289 static u32 vmx_segment_access_rights(struct kvm_segment *var) 3290 { 3291 u32 ar; 3292 3293 if (var->unusable || !var->present) 3294 ar = 1 << 16; 3295 else { 3296 ar = var->type & 15; 3297 ar |= (var->s & 1) << 4; 3298 ar |= (var->dpl & 3) << 5; 3299 ar |= (var->present & 1) << 7; 3300 ar |= (var->avl & 1) << 12; 3301 ar |= (var->l & 1) << 13; 3302 ar |= (var->db & 1) << 14; 3303 ar |= (var->g & 1) << 15; 3304 } 3305 3306 return ar; 3307 } 3308 3309 void __vmx_set_segment(struct kvm_vcpu *vcpu, struct kvm_segment *var, int seg) 3310 { 3311 struct vcpu_vmx *vmx = to_vmx(vcpu); 3312 const struct kvm_vmx_segment_field *sf = &kvm_vmx_segment_fields[seg]; 3313 3314 vmx_segment_cache_clear(vmx); 3315 3316 if (vmx->rmode.vm86_active && seg != VCPU_SREG_LDTR) { 3317 vmx->rmode.segs[seg] = *var; 3318 if (seg == VCPU_SREG_TR) 3319 vmcs_write16(sf->selector, var->selector); 3320 else if (var->s) 3321 fix_rmode_seg(seg, &vmx->rmode.segs[seg]); 3322 return; 3323 } 3324 3325 vmcs_writel(sf->base, var->base); 3326 vmcs_write32(sf->limit, var->limit); 3327 vmcs_write16(sf->selector, var->selector); 3328 3329 /* 3330 * Fix the "Accessed" bit in AR field of segment registers for older 3331 * qemu binaries. 3332 * IA32 arch specifies that at the time of processor reset the 3333 * "Accessed" bit in the AR field of segment registers is 1. And qemu 3334 * is setting it to 0 in the userland code. This causes invalid guest 3335 * state vmexit when "unrestricted guest" mode is turned on. 3336 * Fix for this setup issue in cpu_reset is being pushed in the qemu 3337 * tree. Newer qemu binaries with that qemu fix would not need this 3338 * kvm hack. 3339 */ 3340 if (is_unrestricted_guest(vcpu) && (seg != VCPU_SREG_LDTR)) 3341 var->type |= 0x1; /* Accessed */ 3342 3343 vmcs_write32(sf->ar_bytes, vmx_segment_access_rights(var)); 3344 } 3345 3346 static void vmx_set_segment(struct kvm_vcpu *vcpu, struct kvm_segment *var, int seg) 3347 { 3348 __vmx_set_segment(vcpu, var, seg); 3349 3350 to_vmx(vcpu)->emulation_required = vmx_emulation_required(vcpu); 3351 } 3352 3353 static void vmx_get_cs_db_l_bits(struct kvm_vcpu *vcpu, int *db, int *l) 3354 { 3355 u32 ar = vmx_read_guest_seg_ar(to_vmx(vcpu), VCPU_SREG_CS); 3356 3357 *db = (ar >> 14) & 1; 3358 *l = (ar >> 13) & 1; 3359 } 3360 3361 static void vmx_get_idt(struct kvm_vcpu *vcpu, struct desc_ptr *dt) 3362 { 3363 dt->size = vmcs_read32(GUEST_IDTR_LIMIT); 3364 dt->address = vmcs_readl(GUEST_IDTR_BASE); 3365 } 3366 3367 static void vmx_set_idt(struct kvm_vcpu *vcpu, struct desc_ptr *dt) 3368 { 3369 vmcs_write32(GUEST_IDTR_LIMIT, dt->size); 3370 vmcs_writel(GUEST_IDTR_BASE, dt->address); 3371 } 3372 3373 static void vmx_get_gdt(struct kvm_vcpu *vcpu, struct desc_ptr *dt) 3374 { 3375 dt->size = vmcs_read32(GUEST_GDTR_LIMIT); 3376 dt->address = vmcs_readl(GUEST_GDTR_BASE); 3377 } 3378 3379 static void vmx_set_gdt(struct kvm_vcpu *vcpu, struct desc_ptr *dt) 3380 { 3381 vmcs_write32(GUEST_GDTR_LIMIT, dt->size); 3382 vmcs_writel(GUEST_GDTR_BASE, dt->address); 3383 } 3384 3385 static bool rmode_segment_valid(struct kvm_vcpu *vcpu, int seg) 3386 { 3387 struct kvm_segment var; 3388 u32 ar; 3389 3390 vmx_get_segment(vcpu, &var, seg); 3391 var.dpl = 0x3; 3392 if (seg == VCPU_SREG_CS) 3393 var.type = 0x3; 3394 ar = vmx_segment_access_rights(&var); 3395 3396 if (var.base != (var.selector << 4)) 3397 return false; 3398 if (var.limit != 0xffff) 3399 return false; 3400 if (ar != 0xf3) 3401 return false; 3402 3403 return true; 3404 } 3405 3406 static bool code_segment_valid(struct kvm_vcpu *vcpu) 3407 { 3408 struct kvm_segment cs; 3409 unsigned int cs_rpl; 3410 3411 vmx_get_segment(vcpu, &cs, VCPU_SREG_CS); 3412 cs_rpl = cs.selector & SEGMENT_RPL_MASK; 3413 3414 if (cs.unusable) 3415 return false; 3416 if (~cs.type & (VMX_AR_TYPE_CODE_MASK|VMX_AR_TYPE_ACCESSES_MASK)) 3417 return false; 3418 if (!cs.s) 3419 return false; 3420 if (cs.type & VMX_AR_TYPE_WRITEABLE_MASK) { 3421 if (cs.dpl > cs_rpl) 3422 return false; 3423 } else { 3424 if (cs.dpl != cs_rpl) 3425 return false; 3426 } 3427 if (!cs.present) 3428 return false; 3429 3430 /* TODO: Add Reserved field check, this'll require a new member in the kvm_segment_field structure */ 3431 return true; 3432 } 3433 3434 static bool stack_segment_valid(struct kvm_vcpu *vcpu) 3435 { 3436 struct kvm_segment ss; 3437 unsigned int ss_rpl; 3438 3439 vmx_get_segment(vcpu, &ss, VCPU_SREG_SS); 3440 ss_rpl = ss.selector & SEGMENT_RPL_MASK; 3441 3442 if (ss.unusable) 3443 return true; 3444 if (ss.type != 3 && ss.type != 7) 3445 return false; 3446 if (!ss.s) 3447 return false; 3448 if (ss.dpl != ss_rpl) /* DPL != RPL */ 3449 return false; 3450 if (!ss.present) 3451 return false; 3452 3453 return true; 3454 } 3455 3456 static bool data_segment_valid(struct kvm_vcpu *vcpu, int seg) 3457 { 3458 struct kvm_segment var; 3459 unsigned int rpl; 3460 3461 vmx_get_segment(vcpu, &var, seg); 3462 rpl = var.selector & SEGMENT_RPL_MASK; 3463 3464 if (var.unusable) 3465 return true; 3466 if (!var.s) 3467 return false; 3468 if (!var.present) 3469 return false; 3470 if (~var.type & (VMX_AR_TYPE_CODE_MASK|VMX_AR_TYPE_WRITEABLE_MASK)) { 3471 if (var.dpl < rpl) /* DPL < RPL */ 3472 return false; 3473 } 3474 3475 /* TODO: Add other members to kvm_segment_field to allow checking for other access 3476 * rights flags 3477 */ 3478 return true; 3479 } 3480 3481 static bool tr_valid(struct kvm_vcpu *vcpu) 3482 { 3483 struct kvm_segment tr; 3484 3485 vmx_get_segment(vcpu, &tr, VCPU_SREG_TR); 3486 3487 if (tr.unusable) 3488 return false; 3489 if (tr.selector & SEGMENT_TI_MASK) /* TI = 1 */ 3490 return false; 3491 if (tr.type != 3 && tr.type != 11) /* TODO: Check if guest is in IA32e mode */ 3492 return false; 3493 if (!tr.present) 3494 return false; 3495 3496 return true; 3497 } 3498 3499 static bool ldtr_valid(struct kvm_vcpu *vcpu) 3500 { 3501 struct kvm_segment ldtr; 3502 3503 vmx_get_segment(vcpu, &ldtr, VCPU_SREG_LDTR); 3504 3505 if (ldtr.unusable) 3506 return true; 3507 if (ldtr.selector & SEGMENT_TI_MASK) /* TI = 1 */ 3508 return false; 3509 if (ldtr.type != 2) 3510 return false; 3511 if (!ldtr.present) 3512 return false; 3513 3514 return true; 3515 } 3516 3517 static bool cs_ss_rpl_check(struct kvm_vcpu *vcpu) 3518 { 3519 struct kvm_segment cs, ss; 3520 3521 vmx_get_segment(vcpu, &cs, VCPU_SREG_CS); 3522 vmx_get_segment(vcpu, &ss, VCPU_SREG_SS); 3523 3524 return ((cs.selector & SEGMENT_RPL_MASK) == 3525 (ss.selector & SEGMENT_RPL_MASK)); 3526 } 3527 3528 /* 3529 * Check if guest state is valid. Returns true if valid, false if 3530 * not. 3531 * We assume that registers are always usable 3532 */ 3533 bool __vmx_guest_state_valid(struct kvm_vcpu *vcpu) 3534 { 3535 /* real mode guest state checks */ 3536 if (!is_protmode(vcpu) || (vmx_get_rflags(vcpu) & X86_EFLAGS_VM)) { 3537 if (!rmode_segment_valid(vcpu, VCPU_SREG_CS)) 3538 return false; 3539 if (!rmode_segment_valid(vcpu, VCPU_SREG_SS)) 3540 return false; 3541 if (!rmode_segment_valid(vcpu, VCPU_SREG_DS)) 3542 return false; 3543 if (!rmode_segment_valid(vcpu, VCPU_SREG_ES)) 3544 return false; 3545 if (!rmode_segment_valid(vcpu, VCPU_SREG_FS)) 3546 return false; 3547 if (!rmode_segment_valid(vcpu, VCPU_SREG_GS)) 3548 return false; 3549 } else { 3550 /* protected mode guest state checks */ 3551 if (!cs_ss_rpl_check(vcpu)) 3552 return false; 3553 if (!code_segment_valid(vcpu)) 3554 return false; 3555 if (!stack_segment_valid(vcpu)) 3556 return false; 3557 if (!data_segment_valid(vcpu, VCPU_SREG_DS)) 3558 return false; 3559 if (!data_segment_valid(vcpu, VCPU_SREG_ES)) 3560 return false; 3561 if (!data_segment_valid(vcpu, VCPU_SREG_FS)) 3562 return false; 3563 if (!data_segment_valid(vcpu, VCPU_SREG_GS)) 3564 return false; 3565 if (!tr_valid(vcpu)) 3566 return false; 3567 if (!ldtr_valid(vcpu)) 3568 return false; 3569 } 3570 /* TODO: 3571 * - Add checks on RIP 3572 * - Add checks on RFLAGS 3573 */ 3574 3575 return true; 3576 } 3577 3578 static int init_rmode_tss(struct kvm *kvm, void __user *ua) 3579 { 3580 const void *zero_page = (const void *) __va(page_to_phys(ZERO_PAGE(0))); 3581 u16 data; 3582 int i; 3583 3584 for (i = 0; i < 3; i++) { 3585 if (__copy_to_user(ua + PAGE_SIZE * i, zero_page, PAGE_SIZE)) 3586 return -EFAULT; 3587 } 3588 3589 data = TSS_BASE_SIZE + TSS_REDIRECTION_SIZE; 3590 if (__copy_to_user(ua + TSS_IOPB_BASE_OFFSET, &data, sizeof(u16))) 3591 return -EFAULT; 3592 3593 data = ~0; 3594 if (__copy_to_user(ua + RMODE_TSS_SIZE - 1, &data, sizeof(u8))) 3595 return -EFAULT; 3596 3597 return 0; 3598 } 3599 3600 static int init_rmode_identity_map(struct kvm *kvm) 3601 { 3602 struct kvm_vmx *kvm_vmx = to_kvm_vmx(kvm); 3603 int i, r = 0; 3604 void __user *uaddr; 3605 u32 tmp; 3606 3607 /* Protect kvm_vmx->ept_identity_pagetable_done. */ 3608 mutex_lock(&kvm->slots_lock); 3609 3610 if (likely(kvm_vmx->ept_identity_pagetable_done)) 3611 goto out; 3612 3613 if (!kvm_vmx->ept_identity_map_addr) 3614 kvm_vmx->ept_identity_map_addr = VMX_EPT_IDENTITY_PAGETABLE_ADDR; 3615 3616 uaddr = __x86_set_memory_region(kvm, 3617 IDENTITY_PAGETABLE_PRIVATE_MEMSLOT, 3618 kvm_vmx->ept_identity_map_addr, 3619 PAGE_SIZE); 3620 if (IS_ERR(uaddr)) { 3621 r = PTR_ERR(uaddr); 3622 goto out; 3623 } 3624 3625 /* Set up identity-mapping pagetable for EPT in real mode */ 3626 for (i = 0; i < PT32_ENT_PER_PAGE; i++) { 3627 tmp = (i << 22) + (_PAGE_PRESENT | _PAGE_RW | _PAGE_USER | 3628 _PAGE_ACCESSED | _PAGE_DIRTY | _PAGE_PSE); 3629 if (__copy_to_user(uaddr + i * sizeof(tmp), &tmp, sizeof(tmp))) { 3630 r = -EFAULT; 3631 goto out; 3632 } 3633 } 3634 kvm_vmx->ept_identity_pagetable_done = true; 3635 3636 out: 3637 mutex_unlock(&kvm->slots_lock); 3638 return r; 3639 } 3640 3641 static void seg_setup(int seg) 3642 { 3643 const struct kvm_vmx_segment_field *sf = &kvm_vmx_segment_fields[seg]; 3644 unsigned int ar; 3645 3646 vmcs_write16(sf->selector, 0); 3647 vmcs_writel(sf->base, 0); 3648 vmcs_write32(sf->limit, 0xffff); 3649 ar = 0x93; 3650 if (seg == VCPU_SREG_CS) 3651 ar |= 0x08; /* code segment */ 3652 3653 vmcs_write32(sf->ar_bytes, ar); 3654 } 3655 3656 static int alloc_apic_access_page(struct kvm *kvm) 3657 { 3658 struct page *page; 3659 void __user *hva; 3660 int ret = 0; 3661 3662 mutex_lock(&kvm->slots_lock); 3663 if (kvm->arch.apic_access_memslot_enabled) 3664 goto out; 3665 hva = __x86_set_memory_region(kvm, APIC_ACCESS_PAGE_PRIVATE_MEMSLOT, 3666 APIC_DEFAULT_PHYS_BASE, PAGE_SIZE); 3667 if (IS_ERR(hva)) { 3668 ret = PTR_ERR(hva); 3669 goto out; 3670 } 3671 3672 page = gfn_to_page(kvm, APIC_DEFAULT_PHYS_BASE >> PAGE_SHIFT); 3673 if (is_error_page(page)) { 3674 ret = -EFAULT; 3675 goto out; 3676 } 3677 3678 /* 3679 * Do not pin the page in memory, so that memory hot-unplug 3680 * is able to migrate it. 3681 */ 3682 put_page(page); 3683 kvm->arch.apic_access_memslot_enabled = true; 3684 out: 3685 mutex_unlock(&kvm->slots_lock); 3686 return ret; 3687 } 3688 3689 int allocate_vpid(void) 3690 { 3691 int vpid; 3692 3693 if (!enable_vpid) 3694 return 0; 3695 spin_lock(&vmx_vpid_lock); 3696 vpid = find_first_zero_bit(vmx_vpid_bitmap, VMX_NR_VPIDS); 3697 if (vpid < VMX_NR_VPIDS) 3698 __set_bit(vpid, vmx_vpid_bitmap); 3699 else 3700 vpid = 0; 3701 spin_unlock(&vmx_vpid_lock); 3702 return vpid; 3703 } 3704 3705 void free_vpid(int vpid) 3706 { 3707 if (!enable_vpid || vpid == 0) 3708 return; 3709 spin_lock(&vmx_vpid_lock); 3710 __clear_bit(vpid, vmx_vpid_bitmap); 3711 spin_unlock(&vmx_vpid_lock); 3712 } 3713 3714 static void vmx_msr_bitmap_l01_changed(struct vcpu_vmx *vmx) 3715 { 3716 /* 3717 * When KVM is a nested hypervisor on top of Hyper-V and uses 3718 * 'Enlightened MSR Bitmap' feature L0 needs to know that MSR 3719 * bitmap has changed. 3720 */ 3721 if (static_branch_unlikely(&enable_evmcs)) 3722 evmcs_touch_msr_bitmap(); 3723 3724 vmx->nested.force_msr_bitmap_recalc = true; 3725 } 3726 3727 void vmx_disable_intercept_for_msr(struct kvm_vcpu *vcpu, u32 msr, int type) 3728 { 3729 struct vcpu_vmx *vmx = to_vmx(vcpu); 3730 unsigned long *msr_bitmap = vmx->vmcs01.msr_bitmap; 3731 3732 if (!cpu_has_vmx_msr_bitmap()) 3733 return; 3734 3735 vmx_msr_bitmap_l01_changed(vmx); 3736 3737 /* 3738 * Mark the desired intercept state in shadow bitmap, this is needed 3739 * for resync when the MSR filters change. 3740 */ 3741 if (is_valid_passthrough_msr(msr)) { 3742 int idx = possible_passthrough_msr_slot(msr); 3743 3744 if (idx != -ENOENT) { 3745 if (type & MSR_TYPE_R) 3746 clear_bit(idx, vmx->shadow_msr_intercept.read); 3747 if (type & MSR_TYPE_W) 3748 clear_bit(idx, vmx->shadow_msr_intercept.write); 3749 } 3750 } 3751 3752 if ((type & MSR_TYPE_R) && 3753 !kvm_msr_allowed(vcpu, msr, KVM_MSR_FILTER_READ)) { 3754 vmx_set_msr_bitmap_read(msr_bitmap, msr); 3755 type &= ~MSR_TYPE_R; 3756 } 3757 3758 if ((type & MSR_TYPE_W) && 3759 !kvm_msr_allowed(vcpu, msr, KVM_MSR_FILTER_WRITE)) { 3760 vmx_set_msr_bitmap_write(msr_bitmap, msr); 3761 type &= ~MSR_TYPE_W; 3762 } 3763 3764 if (type & MSR_TYPE_R) 3765 vmx_clear_msr_bitmap_read(msr_bitmap, msr); 3766 3767 if (type & MSR_TYPE_W) 3768 vmx_clear_msr_bitmap_write(msr_bitmap, msr); 3769 } 3770 3771 void vmx_enable_intercept_for_msr(struct kvm_vcpu *vcpu, u32 msr, int type) 3772 { 3773 struct vcpu_vmx *vmx = to_vmx(vcpu); 3774 unsigned long *msr_bitmap = vmx->vmcs01.msr_bitmap; 3775 3776 if (!cpu_has_vmx_msr_bitmap()) 3777 return; 3778 3779 vmx_msr_bitmap_l01_changed(vmx); 3780 3781 /* 3782 * Mark the desired intercept state in shadow bitmap, this is needed 3783 * for resync when the MSR filter changes. 3784 */ 3785 if (is_valid_passthrough_msr(msr)) { 3786 int idx = possible_passthrough_msr_slot(msr); 3787 3788 if (idx != -ENOENT) { 3789 if (type & MSR_TYPE_R) 3790 set_bit(idx, vmx->shadow_msr_intercept.read); 3791 if (type & MSR_TYPE_W) 3792 set_bit(idx, vmx->shadow_msr_intercept.write); 3793 } 3794 } 3795 3796 if (type & MSR_TYPE_R) 3797 vmx_set_msr_bitmap_read(msr_bitmap, msr); 3798 3799 if (type & MSR_TYPE_W) 3800 vmx_set_msr_bitmap_write(msr_bitmap, msr); 3801 } 3802 3803 static void vmx_reset_x2apic_msrs(struct kvm_vcpu *vcpu, u8 mode) 3804 { 3805 unsigned long *msr_bitmap = to_vmx(vcpu)->vmcs01.msr_bitmap; 3806 unsigned long read_intercept; 3807 int msr; 3808 3809 read_intercept = (mode & MSR_BITMAP_MODE_X2APIC_APICV) ? 0 : ~0; 3810 3811 for (msr = 0x800; msr <= 0x8ff; msr += BITS_PER_LONG) { 3812 unsigned int read_idx = msr / BITS_PER_LONG; 3813 unsigned int write_idx = read_idx + (0x800 / sizeof(long)); 3814 3815 msr_bitmap[read_idx] = read_intercept; 3816 msr_bitmap[write_idx] = ~0ul; 3817 } 3818 } 3819 3820 static void vmx_update_msr_bitmap_x2apic(struct kvm_vcpu *vcpu) 3821 { 3822 struct vcpu_vmx *vmx = to_vmx(vcpu); 3823 u8 mode; 3824 3825 if (!cpu_has_vmx_msr_bitmap()) 3826 return; 3827 3828 if (cpu_has_secondary_exec_ctrls() && 3829 (secondary_exec_controls_get(vmx) & 3830 SECONDARY_EXEC_VIRTUALIZE_X2APIC_MODE)) { 3831 mode = MSR_BITMAP_MODE_X2APIC; 3832 if (enable_apicv && kvm_vcpu_apicv_active(vcpu)) 3833 mode |= MSR_BITMAP_MODE_X2APIC_APICV; 3834 } else { 3835 mode = 0; 3836 } 3837 3838 if (mode == vmx->x2apic_msr_bitmap_mode) 3839 return; 3840 3841 vmx->x2apic_msr_bitmap_mode = mode; 3842 3843 vmx_reset_x2apic_msrs(vcpu, mode); 3844 3845 /* 3846 * TPR reads and writes can be virtualized even if virtual interrupt 3847 * delivery is not in use. 3848 */ 3849 vmx_set_intercept_for_msr(vcpu, X2APIC_MSR(APIC_TASKPRI), MSR_TYPE_RW, 3850 !(mode & MSR_BITMAP_MODE_X2APIC)); 3851 3852 if (mode & MSR_BITMAP_MODE_X2APIC_APICV) { 3853 vmx_enable_intercept_for_msr(vcpu, X2APIC_MSR(APIC_TMCCT), MSR_TYPE_RW); 3854 vmx_disable_intercept_for_msr(vcpu, X2APIC_MSR(APIC_EOI), MSR_TYPE_W); 3855 vmx_disable_intercept_for_msr(vcpu, X2APIC_MSR(APIC_SELF_IPI), MSR_TYPE_W); 3856 } 3857 } 3858 3859 void pt_update_intercept_for_msr(struct kvm_vcpu *vcpu) 3860 { 3861 struct vcpu_vmx *vmx = to_vmx(vcpu); 3862 bool flag = !(vmx->pt_desc.guest.ctl & RTIT_CTL_TRACEEN); 3863 u32 i; 3864 3865 vmx_set_intercept_for_msr(vcpu, MSR_IA32_RTIT_STATUS, MSR_TYPE_RW, flag); 3866 vmx_set_intercept_for_msr(vcpu, MSR_IA32_RTIT_OUTPUT_BASE, MSR_TYPE_RW, flag); 3867 vmx_set_intercept_for_msr(vcpu, MSR_IA32_RTIT_OUTPUT_MASK, MSR_TYPE_RW, flag); 3868 vmx_set_intercept_for_msr(vcpu, MSR_IA32_RTIT_CR3_MATCH, MSR_TYPE_RW, flag); 3869 for (i = 0; i < vmx->pt_desc.num_address_ranges; i++) { 3870 vmx_set_intercept_for_msr(vcpu, MSR_IA32_RTIT_ADDR0_A + i * 2, MSR_TYPE_RW, flag); 3871 vmx_set_intercept_for_msr(vcpu, MSR_IA32_RTIT_ADDR0_B + i * 2, MSR_TYPE_RW, flag); 3872 } 3873 } 3874 3875 static bool vmx_guest_apic_has_interrupt(struct kvm_vcpu *vcpu) 3876 { 3877 struct vcpu_vmx *vmx = to_vmx(vcpu); 3878 void *vapic_page; 3879 u32 vppr; 3880 int rvi; 3881 3882 if (WARN_ON_ONCE(!is_guest_mode(vcpu)) || 3883 !nested_cpu_has_vid(get_vmcs12(vcpu)) || 3884 WARN_ON_ONCE(!vmx->nested.virtual_apic_map.gfn)) 3885 return false; 3886 3887 rvi = vmx_get_rvi(); 3888 3889 vapic_page = vmx->nested.virtual_apic_map.hva; 3890 vppr = *((u32 *)(vapic_page + APIC_PROCPRI)); 3891 3892 return ((rvi & 0xf0) > (vppr & 0xf0)); 3893 } 3894 3895 static void vmx_msr_filter_changed(struct kvm_vcpu *vcpu) 3896 { 3897 struct vcpu_vmx *vmx = to_vmx(vcpu); 3898 u32 i; 3899 3900 /* 3901 * Set intercept permissions for all potentially passed through MSRs 3902 * again. They will automatically get filtered through the MSR filter, 3903 * so we are back in sync after this. 3904 */ 3905 for (i = 0; i < ARRAY_SIZE(vmx_possible_passthrough_msrs); i++) { 3906 u32 msr = vmx_possible_passthrough_msrs[i]; 3907 bool read = test_bit(i, vmx->shadow_msr_intercept.read); 3908 bool write = test_bit(i, vmx->shadow_msr_intercept.write); 3909 3910 vmx_set_intercept_for_msr(vcpu, msr, MSR_TYPE_R, read); 3911 vmx_set_intercept_for_msr(vcpu, msr, MSR_TYPE_W, write); 3912 } 3913 3914 pt_update_intercept_for_msr(vcpu); 3915 } 3916 3917 static inline void kvm_vcpu_trigger_posted_interrupt(struct kvm_vcpu *vcpu, 3918 int pi_vec) 3919 { 3920 #ifdef CONFIG_SMP 3921 if (vcpu->mode == IN_GUEST_MODE) { 3922 /* 3923 * The vector of the virtual has already been set in the PIR. 3924 * Send a notification event to deliver the virtual interrupt 3925 * unless the vCPU is the currently running vCPU, i.e. the 3926 * event is being sent from a fastpath VM-Exit handler, in 3927 * which case the PIR will be synced to the vIRR before 3928 * re-entering the guest. 3929 * 3930 * When the target is not the running vCPU, the following 3931 * possibilities emerge: 3932 * 3933 * Case 1: vCPU stays in non-root mode. Sending a notification 3934 * event posts the interrupt to the vCPU. 3935 * 3936 * Case 2: vCPU exits to root mode and is still runnable. The 3937 * PIR will be synced to the vIRR before re-entering the guest. 3938 * Sending a notification event is ok as the host IRQ handler 3939 * will ignore the spurious event. 3940 * 3941 * Case 3: vCPU exits to root mode and is blocked. vcpu_block() 3942 * has already synced PIR to vIRR and never blocks the vCPU if 3943 * the vIRR is not empty. Therefore, a blocked vCPU here does 3944 * not wait for any requested interrupts in PIR, and sending a 3945 * notification event also results in a benign, spurious event. 3946 */ 3947 3948 if (vcpu != kvm_get_running_vcpu()) 3949 apic->send_IPI_mask(get_cpu_mask(vcpu->cpu), pi_vec); 3950 return; 3951 } 3952 #endif 3953 /* 3954 * The vCPU isn't in the guest; wake the vCPU in case it is blocking, 3955 * otherwise do nothing as KVM will grab the highest priority pending 3956 * IRQ via ->sync_pir_to_irr() in vcpu_enter_guest(). 3957 */ 3958 kvm_vcpu_wake_up(vcpu); 3959 } 3960 3961 static int vmx_deliver_nested_posted_interrupt(struct kvm_vcpu *vcpu, 3962 int vector) 3963 { 3964 struct vcpu_vmx *vmx = to_vmx(vcpu); 3965 3966 if (is_guest_mode(vcpu) && 3967 vector == vmx->nested.posted_intr_nv) { 3968 /* 3969 * If a posted intr is not recognized by hardware, 3970 * we will accomplish it in the next vmentry. 3971 */ 3972 vmx->nested.pi_pending = true; 3973 kvm_make_request(KVM_REQ_EVENT, vcpu); 3974 3975 /* 3976 * This pairs with the smp_mb_*() after setting vcpu->mode in 3977 * vcpu_enter_guest() to guarantee the vCPU sees the event 3978 * request if triggering a posted interrupt "fails" because 3979 * vcpu->mode != IN_GUEST_MODE. The extra barrier is needed as 3980 * the smb_wmb() in kvm_make_request() only ensures everything 3981 * done before making the request is visible when the request 3982 * is visible, it doesn't ensure ordering between the store to 3983 * vcpu->requests and the load from vcpu->mode. 3984 */ 3985 smp_mb__after_atomic(); 3986 3987 /* the PIR and ON have been set by L1. */ 3988 kvm_vcpu_trigger_posted_interrupt(vcpu, POSTED_INTR_NESTED_VECTOR); 3989 return 0; 3990 } 3991 return -1; 3992 } 3993 /* 3994 * Send interrupt to vcpu via posted interrupt way. 3995 * 1. If target vcpu is running(non-root mode), send posted interrupt 3996 * notification to vcpu and hardware will sync PIR to vIRR atomically. 3997 * 2. If target vcpu isn't running(root mode), kick it to pick up the 3998 * interrupt from PIR in next vmentry. 3999 */ 4000 static int vmx_deliver_posted_interrupt(struct kvm_vcpu *vcpu, int vector) 4001 { 4002 struct vcpu_vmx *vmx = to_vmx(vcpu); 4003 int r; 4004 4005 r = vmx_deliver_nested_posted_interrupt(vcpu, vector); 4006 if (!r) 4007 return 0; 4008 4009 if (!vcpu->arch.apicv_active) 4010 return -1; 4011 4012 if (pi_test_and_set_pir(vector, &vmx->pi_desc)) 4013 return 0; 4014 4015 /* If a previous notification has sent the IPI, nothing to do. */ 4016 if (pi_test_and_set_on(&vmx->pi_desc)) 4017 return 0; 4018 4019 /* 4020 * The implied barrier in pi_test_and_set_on() pairs with the smp_mb_*() 4021 * after setting vcpu->mode in vcpu_enter_guest(), thus the vCPU is 4022 * guaranteed to see PID.ON=1 and sync the PIR to IRR if triggering a 4023 * posted interrupt "fails" because vcpu->mode != IN_GUEST_MODE. 4024 */ 4025 kvm_vcpu_trigger_posted_interrupt(vcpu, POSTED_INTR_VECTOR); 4026 return 0; 4027 } 4028 4029 static void vmx_deliver_interrupt(struct kvm_lapic *apic, int delivery_mode, 4030 int trig_mode, int vector) 4031 { 4032 struct kvm_vcpu *vcpu = apic->vcpu; 4033 4034 if (vmx_deliver_posted_interrupt(vcpu, vector)) { 4035 kvm_lapic_set_irr(vector, apic); 4036 kvm_make_request(KVM_REQ_EVENT, vcpu); 4037 kvm_vcpu_kick(vcpu); 4038 } else { 4039 trace_kvm_apicv_accept_irq(vcpu->vcpu_id, delivery_mode, 4040 trig_mode, vector); 4041 } 4042 } 4043 4044 /* 4045 * Set up the vmcs's constant host-state fields, i.e., host-state fields that 4046 * will not change in the lifetime of the guest. 4047 * Note that host-state that does change is set elsewhere. E.g., host-state 4048 * that is set differently for each CPU is set in vmx_vcpu_load(), not here. 4049 */ 4050 void vmx_set_constant_host_state(struct vcpu_vmx *vmx) 4051 { 4052 u32 low32, high32; 4053 unsigned long tmpl; 4054 unsigned long cr0, cr3, cr4; 4055 4056 cr0 = read_cr0(); 4057 WARN_ON(cr0 & X86_CR0_TS); 4058 vmcs_writel(HOST_CR0, cr0); /* 22.2.3 */ 4059 4060 /* 4061 * Save the most likely value for this task's CR3 in the VMCS. 4062 * We can't use __get_current_cr3_fast() because we're not atomic. 4063 */ 4064 cr3 = __read_cr3(); 4065 vmcs_writel(HOST_CR3, cr3); /* 22.2.3 FIXME: shadow tables */ 4066 vmx->loaded_vmcs->host_state.cr3 = cr3; 4067 4068 /* Save the most likely value for this task's CR4 in the VMCS. */ 4069 cr4 = cr4_read_shadow(); 4070 vmcs_writel(HOST_CR4, cr4); /* 22.2.3, 22.2.5 */ 4071 vmx->loaded_vmcs->host_state.cr4 = cr4; 4072 4073 vmcs_write16(HOST_CS_SELECTOR, __KERNEL_CS); /* 22.2.4 */ 4074 #ifdef CONFIG_X86_64 4075 /* 4076 * Load null selectors, so we can avoid reloading them in 4077 * vmx_prepare_switch_to_host(), in case userspace uses 4078 * the null selectors too (the expected case). 4079 */ 4080 vmcs_write16(HOST_DS_SELECTOR, 0); 4081 vmcs_write16(HOST_ES_SELECTOR, 0); 4082 #else 4083 vmcs_write16(HOST_DS_SELECTOR, __KERNEL_DS); /* 22.2.4 */ 4084 vmcs_write16(HOST_ES_SELECTOR, __KERNEL_DS); /* 22.2.4 */ 4085 #endif 4086 vmcs_write16(HOST_SS_SELECTOR, __KERNEL_DS); /* 22.2.4 */ 4087 vmcs_write16(HOST_TR_SELECTOR, GDT_ENTRY_TSS*8); /* 22.2.4 */ 4088 4089 vmcs_writel(HOST_IDTR_BASE, host_idt_base); /* 22.2.4 */ 4090 4091 vmcs_writel(HOST_RIP, (unsigned long)vmx_vmexit); /* 22.2.5 */ 4092 4093 rdmsr(MSR_IA32_SYSENTER_CS, low32, high32); 4094 vmcs_write32(HOST_IA32_SYSENTER_CS, low32); 4095 4096 /* 4097 * SYSENTER is used for 32-bit system calls on either 32-bit or 4098 * 64-bit kernels. It is always zero If neither is allowed, otherwise 4099 * vmx_vcpu_load_vmcs loads it with the per-CPU entry stack (and may 4100 * have already done so!). 4101 */ 4102 if (!IS_ENABLED(CONFIG_IA32_EMULATION) && !IS_ENABLED(CONFIG_X86_32)) 4103 vmcs_writel(HOST_IA32_SYSENTER_ESP, 0); 4104 4105 rdmsrl(MSR_IA32_SYSENTER_EIP, tmpl); 4106 vmcs_writel(HOST_IA32_SYSENTER_EIP, tmpl); /* 22.2.3 */ 4107 4108 if (vmcs_config.vmexit_ctrl & VM_EXIT_LOAD_IA32_PAT) { 4109 rdmsr(MSR_IA32_CR_PAT, low32, high32); 4110 vmcs_write64(HOST_IA32_PAT, low32 | ((u64) high32 << 32)); 4111 } 4112 4113 if (cpu_has_load_ia32_efer()) 4114 vmcs_write64(HOST_IA32_EFER, host_efer); 4115 } 4116 4117 void set_cr4_guest_host_mask(struct vcpu_vmx *vmx) 4118 { 4119 struct kvm_vcpu *vcpu = &vmx->vcpu; 4120 4121 vcpu->arch.cr4_guest_owned_bits = KVM_POSSIBLE_CR4_GUEST_BITS & 4122 ~vcpu->arch.cr4_guest_rsvd_bits; 4123 if (!enable_ept) { 4124 vcpu->arch.cr4_guest_owned_bits &= ~X86_CR4_TLBFLUSH_BITS; 4125 vcpu->arch.cr4_guest_owned_bits &= ~X86_CR4_PDPTR_BITS; 4126 } 4127 if (is_guest_mode(&vmx->vcpu)) 4128 vcpu->arch.cr4_guest_owned_bits &= 4129 ~get_vmcs12(vcpu)->cr4_guest_host_mask; 4130 vmcs_writel(CR4_GUEST_HOST_MASK, ~vcpu->arch.cr4_guest_owned_bits); 4131 } 4132 4133 static u32 vmx_pin_based_exec_ctrl(struct vcpu_vmx *vmx) 4134 { 4135 u32 pin_based_exec_ctrl = vmcs_config.pin_based_exec_ctrl; 4136 4137 if (!kvm_vcpu_apicv_active(&vmx->vcpu)) 4138 pin_based_exec_ctrl &= ~PIN_BASED_POSTED_INTR; 4139 4140 if (!enable_vnmi) 4141 pin_based_exec_ctrl &= ~PIN_BASED_VIRTUAL_NMIS; 4142 4143 if (!enable_preemption_timer) 4144 pin_based_exec_ctrl &= ~PIN_BASED_VMX_PREEMPTION_TIMER; 4145 4146 return pin_based_exec_ctrl; 4147 } 4148 4149 static u32 vmx_vmentry_ctrl(void) 4150 { 4151 u32 vmentry_ctrl = vmcs_config.vmentry_ctrl; 4152 4153 if (vmx_pt_mode_is_system()) 4154 vmentry_ctrl &= ~(VM_ENTRY_PT_CONCEAL_PIP | 4155 VM_ENTRY_LOAD_IA32_RTIT_CTL); 4156 /* Loading of EFER and PERF_GLOBAL_CTRL are toggled dynamically */ 4157 return vmentry_ctrl & 4158 ~(VM_ENTRY_LOAD_IA32_PERF_GLOBAL_CTRL | VM_ENTRY_LOAD_IA32_EFER); 4159 } 4160 4161 static u32 vmx_vmexit_ctrl(void) 4162 { 4163 u32 vmexit_ctrl = vmcs_config.vmexit_ctrl; 4164 4165 if (vmx_pt_mode_is_system()) 4166 vmexit_ctrl &= ~(VM_EXIT_PT_CONCEAL_PIP | 4167 VM_EXIT_CLEAR_IA32_RTIT_CTL); 4168 /* Loading of EFER and PERF_GLOBAL_CTRL are toggled dynamically */ 4169 return vmexit_ctrl & 4170 ~(VM_EXIT_LOAD_IA32_PERF_GLOBAL_CTRL | VM_EXIT_LOAD_IA32_EFER); 4171 } 4172 4173 static void vmx_refresh_apicv_exec_ctrl(struct kvm_vcpu *vcpu) 4174 { 4175 struct vcpu_vmx *vmx = to_vmx(vcpu); 4176 4177 if (is_guest_mode(vcpu)) { 4178 vmx->nested.update_vmcs01_apicv_status = true; 4179 return; 4180 } 4181 4182 pin_controls_set(vmx, vmx_pin_based_exec_ctrl(vmx)); 4183 if (cpu_has_secondary_exec_ctrls()) { 4184 if (kvm_vcpu_apicv_active(vcpu)) 4185 secondary_exec_controls_setbit(vmx, 4186 SECONDARY_EXEC_APIC_REGISTER_VIRT | 4187 SECONDARY_EXEC_VIRTUAL_INTR_DELIVERY); 4188 else 4189 secondary_exec_controls_clearbit(vmx, 4190 SECONDARY_EXEC_APIC_REGISTER_VIRT | 4191 SECONDARY_EXEC_VIRTUAL_INTR_DELIVERY); 4192 } 4193 4194 vmx_update_msr_bitmap_x2apic(vcpu); 4195 } 4196 4197 static u32 vmx_exec_control(struct vcpu_vmx *vmx) 4198 { 4199 u32 exec_control = vmcs_config.cpu_based_exec_ctrl; 4200 4201 if (vmx->vcpu.arch.switch_db_regs & KVM_DEBUGREG_WONT_EXIT) 4202 exec_control &= ~CPU_BASED_MOV_DR_EXITING; 4203 4204 if (!cpu_need_tpr_shadow(&vmx->vcpu)) { 4205 exec_control &= ~CPU_BASED_TPR_SHADOW; 4206 #ifdef CONFIG_X86_64 4207 exec_control |= CPU_BASED_CR8_STORE_EXITING | 4208 CPU_BASED_CR8_LOAD_EXITING; 4209 #endif 4210 } 4211 if (!enable_ept) 4212 exec_control |= CPU_BASED_CR3_STORE_EXITING | 4213 CPU_BASED_CR3_LOAD_EXITING | 4214 CPU_BASED_INVLPG_EXITING; 4215 if (kvm_mwait_in_guest(vmx->vcpu.kvm)) 4216 exec_control &= ~(CPU_BASED_MWAIT_EXITING | 4217 CPU_BASED_MONITOR_EXITING); 4218 if (kvm_hlt_in_guest(vmx->vcpu.kvm)) 4219 exec_control &= ~CPU_BASED_HLT_EXITING; 4220 return exec_control; 4221 } 4222 4223 /* 4224 * Adjust a single secondary execution control bit to intercept/allow an 4225 * instruction in the guest. This is usually done based on whether or not a 4226 * feature has been exposed to the guest in order to correctly emulate faults. 4227 */ 4228 static inline void 4229 vmx_adjust_secondary_exec_control(struct vcpu_vmx *vmx, u32 *exec_control, 4230 u32 control, bool enabled, bool exiting) 4231 { 4232 /* 4233 * If the control is for an opt-in feature, clear the control if the 4234 * feature is not exposed to the guest, i.e. not enabled. If the 4235 * control is opt-out, i.e. an exiting control, clear the control if 4236 * the feature _is_ exposed to the guest, i.e. exiting/interception is 4237 * disabled for the associated instruction. Note, the caller is 4238 * responsible presetting exec_control to set all supported bits. 4239 */ 4240 if (enabled == exiting) 4241 *exec_control &= ~control; 4242 4243 /* 4244 * Update the nested MSR settings so that a nested VMM can/can't set 4245 * controls for features that are/aren't exposed to the guest. 4246 */ 4247 if (nested) { 4248 if (enabled) 4249 vmx->nested.msrs.secondary_ctls_high |= control; 4250 else 4251 vmx->nested.msrs.secondary_ctls_high &= ~control; 4252 } 4253 } 4254 4255 /* 4256 * Wrapper macro for the common case of adjusting a secondary execution control 4257 * based on a single guest CPUID bit, with a dedicated feature bit. This also 4258 * verifies that the control is actually supported by KVM and hardware. 4259 */ 4260 #define vmx_adjust_sec_exec_control(vmx, exec_control, name, feat_name, ctrl_name, exiting) \ 4261 ({ \ 4262 bool __enabled; \ 4263 \ 4264 if (cpu_has_vmx_##name()) { \ 4265 __enabled = guest_cpuid_has(&(vmx)->vcpu, \ 4266 X86_FEATURE_##feat_name); \ 4267 vmx_adjust_secondary_exec_control(vmx, exec_control, \ 4268 SECONDARY_EXEC_##ctrl_name, __enabled, exiting); \ 4269 } \ 4270 }) 4271 4272 /* More macro magic for ENABLE_/opt-in versus _EXITING/opt-out controls. */ 4273 #define vmx_adjust_sec_exec_feature(vmx, exec_control, lname, uname) \ 4274 vmx_adjust_sec_exec_control(vmx, exec_control, lname, uname, ENABLE_##uname, false) 4275 4276 #define vmx_adjust_sec_exec_exiting(vmx, exec_control, lname, uname) \ 4277 vmx_adjust_sec_exec_control(vmx, exec_control, lname, uname, uname##_EXITING, true) 4278 4279 static u32 vmx_secondary_exec_control(struct vcpu_vmx *vmx) 4280 { 4281 struct kvm_vcpu *vcpu = &vmx->vcpu; 4282 4283 u32 exec_control = vmcs_config.cpu_based_2nd_exec_ctrl; 4284 4285 if (vmx_pt_mode_is_system()) 4286 exec_control &= ~(SECONDARY_EXEC_PT_USE_GPA | SECONDARY_EXEC_PT_CONCEAL_VMX); 4287 if (!cpu_need_virtualize_apic_accesses(vcpu)) 4288 exec_control &= ~SECONDARY_EXEC_VIRTUALIZE_APIC_ACCESSES; 4289 if (vmx->vpid == 0) 4290 exec_control &= ~SECONDARY_EXEC_ENABLE_VPID; 4291 if (!enable_ept) { 4292 exec_control &= ~SECONDARY_EXEC_ENABLE_EPT; 4293 enable_unrestricted_guest = 0; 4294 } 4295 if (!enable_unrestricted_guest) 4296 exec_control &= ~SECONDARY_EXEC_UNRESTRICTED_GUEST; 4297 if (kvm_pause_in_guest(vmx->vcpu.kvm)) 4298 exec_control &= ~SECONDARY_EXEC_PAUSE_LOOP_EXITING; 4299 if (!kvm_vcpu_apicv_active(vcpu)) 4300 exec_control &= ~(SECONDARY_EXEC_APIC_REGISTER_VIRT | 4301 SECONDARY_EXEC_VIRTUAL_INTR_DELIVERY); 4302 exec_control &= ~SECONDARY_EXEC_VIRTUALIZE_X2APIC_MODE; 4303 4304 /* SECONDARY_EXEC_DESC is enabled/disabled on writes to CR4.UMIP, 4305 * in vmx_set_cr4. */ 4306 exec_control &= ~SECONDARY_EXEC_DESC; 4307 4308 /* SECONDARY_EXEC_SHADOW_VMCS is enabled when L1 executes VMPTRLD 4309 (handle_vmptrld). 4310 We can NOT enable shadow_vmcs here because we don't have yet 4311 a current VMCS12 4312 */ 4313 exec_control &= ~SECONDARY_EXEC_SHADOW_VMCS; 4314 4315 /* 4316 * PML is enabled/disabled when dirty logging of memsmlots changes, but 4317 * it needs to be set here when dirty logging is already active, e.g. 4318 * if this vCPU was created after dirty logging was enabled. 4319 */ 4320 if (!vcpu->kvm->arch.cpu_dirty_logging_count) 4321 exec_control &= ~SECONDARY_EXEC_ENABLE_PML; 4322 4323 if (cpu_has_vmx_xsaves()) { 4324 /* Exposing XSAVES only when XSAVE is exposed */ 4325 bool xsaves_enabled = 4326 boot_cpu_has(X86_FEATURE_XSAVE) && 4327 guest_cpuid_has(vcpu, X86_FEATURE_XSAVE) && 4328 guest_cpuid_has(vcpu, X86_FEATURE_XSAVES); 4329 4330 vcpu->arch.xsaves_enabled = xsaves_enabled; 4331 4332 vmx_adjust_secondary_exec_control(vmx, &exec_control, 4333 SECONDARY_EXEC_XSAVES, 4334 xsaves_enabled, false); 4335 } 4336 4337 /* 4338 * RDPID is also gated by ENABLE_RDTSCP, turn on the control if either 4339 * feature is exposed to the guest. This creates a virtualization hole 4340 * if both are supported in hardware but only one is exposed to the 4341 * guest, but letting the guest execute RDTSCP or RDPID when either one 4342 * is advertised is preferable to emulating the advertised instruction 4343 * in KVM on #UD, and obviously better than incorrectly injecting #UD. 4344 */ 4345 if (cpu_has_vmx_rdtscp()) { 4346 bool rdpid_or_rdtscp_enabled = 4347 guest_cpuid_has(vcpu, X86_FEATURE_RDTSCP) || 4348 guest_cpuid_has(vcpu, X86_FEATURE_RDPID); 4349 4350 vmx_adjust_secondary_exec_control(vmx, &exec_control, 4351 SECONDARY_EXEC_ENABLE_RDTSCP, 4352 rdpid_or_rdtscp_enabled, false); 4353 } 4354 vmx_adjust_sec_exec_feature(vmx, &exec_control, invpcid, INVPCID); 4355 4356 vmx_adjust_sec_exec_exiting(vmx, &exec_control, rdrand, RDRAND); 4357 vmx_adjust_sec_exec_exiting(vmx, &exec_control, rdseed, RDSEED); 4358 4359 vmx_adjust_sec_exec_control(vmx, &exec_control, waitpkg, WAITPKG, 4360 ENABLE_USR_WAIT_PAUSE, false); 4361 4362 if (!vcpu->kvm->arch.bus_lock_detection_enabled) 4363 exec_control &= ~SECONDARY_EXEC_BUS_LOCK_DETECTION; 4364 4365 return exec_control; 4366 } 4367 4368 #define VMX_XSS_EXIT_BITMAP 0 4369 4370 static void init_vmcs(struct vcpu_vmx *vmx) 4371 { 4372 if (nested) 4373 nested_vmx_set_vmcs_shadowing_bitmap(); 4374 4375 if (cpu_has_vmx_msr_bitmap()) 4376 vmcs_write64(MSR_BITMAP, __pa(vmx->vmcs01.msr_bitmap)); 4377 4378 vmcs_write64(VMCS_LINK_POINTER, INVALID_GPA); /* 22.3.1.5 */ 4379 4380 /* Control */ 4381 pin_controls_set(vmx, vmx_pin_based_exec_ctrl(vmx)); 4382 4383 exec_controls_set(vmx, vmx_exec_control(vmx)); 4384 4385 if (cpu_has_secondary_exec_ctrls()) 4386 secondary_exec_controls_set(vmx, vmx_secondary_exec_control(vmx)); 4387 4388 if (kvm_vcpu_apicv_active(&vmx->vcpu)) { 4389 vmcs_write64(EOI_EXIT_BITMAP0, 0); 4390 vmcs_write64(EOI_EXIT_BITMAP1, 0); 4391 vmcs_write64(EOI_EXIT_BITMAP2, 0); 4392 vmcs_write64(EOI_EXIT_BITMAP3, 0); 4393 4394 vmcs_write16(GUEST_INTR_STATUS, 0); 4395 4396 vmcs_write16(POSTED_INTR_NV, POSTED_INTR_VECTOR); 4397 vmcs_write64(POSTED_INTR_DESC_ADDR, __pa((&vmx->pi_desc))); 4398 } 4399 4400 if (!kvm_pause_in_guest(vmx->vcpu.kvm)) { 4401 vmcs_write32(PLE_GAP, ple_gap); 4402 vmx->ple_window = ple_window; 4403 vmx->ple_window_dirty = true; 4404 } 4405 4406 vmcs_write32(PAGE_FAULT_ERROR_CODE_MASK, 0); 4407 vmcs_write32(PAGE_FAULT_ERROR_CODE_MATCH, 0); 4408 vmcs_write32(CR3_TARGET_COUNT, 0); /* 22.2.1 */ 4409 4410 vmcs_write16(HOST_FS_SELECTOR, 0); /* 22.2.4 */ 4411 vmcs_write16(HOST_GS_SELECTOR, 0); /* 22.2.4 */ 4412 vmx_set_constant_host_state(vmx); 4413 vmcs_writel(HOST_FS_BASE, 0); /* 22.2.4 */ 4414 vmcs_writel(HOST_GS_BASE, 0); /* 22.2.4 */ 4415 4416 if (cpu_has_vmx_vmfunc()) 4417 vmcs_write64(VM_FUNCTION_CONTROL, 0); 4418 4419 vmcs_write32(VM_EXIT_MSR_STORE_COUNT, 0); 4420 vmcs_write32(VM_EXIT_MSR_LOAD_COUNT, 0); 4421 vmcs_write64(VM_EXIT_MSR_LOAD_ADDR, __pa(vmx->msr_autoload.host.val)); 4422 vmcs_write32(VM_ENTRY_MSR_LOAD_COUNT, 0); 4423 vmcs_write64(VM_ENTRY_MSR_LOAD_ADDR, __pa(vmx->msr_autoload.guest.val)); 4424 4425 if (vmcs_config.vmentry_ctrl & VM_ENTRY_LOAD_IA32_PAT) 4426 vmcs_write64(GUEST_IA32_PAT, vmx->vcpu.arch.pat); 4427 4428 vm_exit_controls_set(vmx, vmx_vmexit_ctrl()); 4429 4430 /* 22.2.1, 20.8.1 */ 4431 vm_entry_controls_set(vmx, vmx_vmentry_ctrl()); 4432 4433 vmx->vcpu.arch.cr0_guest_owned_bits = KVM_POSSIBLE_CR0_GUEST_BITS; 4434 vmcs_writel(CR0_GUEST_HOST_MASK, ~vmx->vcpu.arch.cr0_guest_owned_bits); 4435 4436 set_cr4_guest_host_mask(vmx); 4437 4438 if (vmx->vpid != 0) 4439 vmcs_write16(VIRTUAL_PROCESSOR_ID, vmx->vpid); 4440 4441 if (cpu_has_vmx_xsaves()) 4442 vmcs_write64(XSS_EXIT_BITMAP, VMX_XSS_EXIT_BITMAP); 4443 4444 if (enable_pml) { 4445 vmcs_write64(PML_ADDRESS, page_to_phys(vmx->pml_pg)); 4446 vmcs_write16(GUEST_PML_INDEX, PML_ENTITY_NUM - 1); 4447 } 4448 4449 vmx_write_encls_bitmap(&vmx->vcpu, NULL); 4450 4451 if (vmx_pt_mode_is_host_guest()) { 4452 memset(&vmx->pt_desc, 0, sizeof(vmx->pt_desc)); 4453 /* Bit[6~0] are forced to 1, writes are ignored. */ 4454 vmx->pt_desc.guest.output_mask = 0x7F; 4455 vmcs_write64(GUEST_IA32_RTIT_CTL, 0); 4456 } 4457 4458 vmcs_write32(GUEST_SYSENTER_CS, 0); 4459 vmcs_writel(GUEST_SYSENTER_ESP, 0); 4460 vmcs_writel(GUEST_SYSENTER_EIP, 0); 4461 vmcs_write64(GUEST_IA32_DEBUGCTL, 0); 4462 4463 if (cpu_has_vmx_tpr_shadow()) { 4464 vmcs_write64(VIRTUAL_APIC_PAGE_ADDR, 0); 4465 if (cpu_need_tpr_shadow(&vmx->vcpu)) 4466 vmcs_write64(VIRTUAL_APIC_PAGE_ADDR, 4467 __pa(vmx->vcpu.arch.apic->regs)); 4468 vmcs_write32(TPR_THRESHOLD, 0); 4469 } 4470 4471 vmx_setup_uret_msrs(vmx); 4472 } 4473 4474 static void __vmx_vcpu_reset(struct kvm_vcpu *vcpu) 4475 { 4476 struct vcpu_vmx *vmx = to_vmx(vcpu); 4477 4478 init_vmcs(vmx); 4479 4480 if (nested) 4481 memcpy(&vmx->nested.msrs, &vmcs_config.nested, sizeof(vmx->nested.msrs)); 4482 4483 vcpu_setup_sgx_lepubkeyhash(vcpu); 4484 4485 vmx->nested.posted_intr_nv = -1; 4486 vmx->nested.vmxon_ptr = INVALID_GPA; 4487 vmx->nested.current_vmptr = INVALID_GPA; 4488 vmx->nested.hv_evmcs_vmptr = EVMPTR_INVALID; 4489 4490 vcpu->arch.microcode_version = 0x100000000ULL; 4491 vmx->msr_ia32_feature_control_valid_bits = FEAT_CTL_LOCKED; 4492 4493 /* 4494 * Enforce invariant: pi_desc.nv is always either POSTED_INTR_VECTOR 4495 * or POSTED_INTR_WAKEUP_VECTOR. 4496 */ 4497 vmx->pi_desc.nv = POSTED_INTR_VECTOR; 4498 vmx->pi_desc.sn = 1; 4499 } 4500 4501 static void vmx_vcpu_reset(struct kvm_vcpu *vcpu, bool init_event) 4502 { 4503 struct vcpu_vmx *vmx = to_vmx(vcpu); 4504 4505 if (!init_event) 4506 __vmx_vcpu_reset(vcpu); 4507 4508 vmx->rmode.vm86_active = 0; 4509 vmx->spec_ctrl = 0; 4510 4511 vmx->msr_ia32_umwait_control = 0; 4512 4513 vmx->hv_deadline_tsc = -1; 4514 kvm_set_cr8(vcpu, 0); 4515 4516 vmx_segment_cache_clear(vmx); 4517 kvm_register_mark_available(vcpu, VCPU_EXREG_SEGMENTS); 4518 4519 seg_setup(VCPU_SREG_CS); 4520 vmcs_write16(GUEST_CS_SELECTOR, 0xf000); 4521 vmcs_writel(GUEST_CS_BASE, 0xffff0000ul); 4522 4523 seg_setup(VCPU_SREG_DS); 4524 seg_setup(VCPU_SREG_ES); 4525 seg_setup(VCPU_SREG_FS); 4526 seg_setup(VCPU_SREG_GS); 4527 seg_setup(VCPU_SREG_SS); 4528 4529 vmcs_write16(GUEST_TR_SELECTOR, 0); 4530 vmcs_writel(GUEST_TR_BASE, 0); 4531 vmcs_write32(GUEST_TR_LIMIT, 0xffff); 4532 vmcs_write32(GUEST_TR_AR_BYTES, 0x008b); 4533 4534 vmcs_write16(GUEST_LDTR_SELECTOR, 0); 4535 vmcs_writel(GUEST_LDTR_BASE, 0); 4536 vmcs_write32(GUEST_LDTR_LIMIT, 0xffff); 4537 vmcs_write32(GUEST_LDTR_AR_BYTES, 0x00082); 4538 4539 vmcs_writel(GUEST_GDTR_BASE, 0); 4540 vmcs_write32(GUEST_GDTR_LIMIT, 0xffff); 4541 4542 vmcs_writel(GUEST_IDTR_BASE, 0); 4543 vmcs_write32(GUEST_IDTR_LIMIT, 0xffff); 4544 4545 vmcs_write32(GUEST_ACTIVITY_STATE, GUEST_ACTIVITY_ACTIVE); 4546 vmcs_write32(GUEST_INTERRUPTIBILITY_INFO, 0); 4547 vmcs_writel(GUEST_PENDING_DBG_EXCEPTIONS, 0); 4548 if (kvm_mpx_supported()) 4549 vmcs_write64(GUEST_BNDCFGS, 0); 4550 4551 vmcs_write32(VM_ENTRY_INTR_INFO_FIELD, 0); /* 22.2.1 */ 4552 4553 kvm_make_request(KVM_REQ_APIC_PAGE_RELOAD, vcpu); 4554 4555 vpid_sync_context(vmx->vpid); 4556 } 4557 4558 static void vmx_enable_irq_window(struct kvm_vcpu *vcpu) 4559 { 4560 exec_controls_setbit(to_vmx(vcpu), CPU_BASED_INTR_WINDOW_EXITING); 4561 } 4562 4563 static void vmx_enable_nmi_window(struct kvm_vcpu *vcpu) 4564 { 4565 if (!enable_vnmi || 4566 vmcs_read32(GUEST_INTERRUPTIBILITY_INFO) & GUEST_INTR_STATE_STI) { 4567 vmx_enable_irq_window(vcpu); 4568 return; 4569 } 4570 4571 exec_controls_setbit(to_vmx(vcpu), CPU_BASED_NMI_WINDOW_EXITING); 4572 } 4573 4574 static void vmx_inject_irq(struct kvm_vcpu *vcpu) 4575 { 4576 struct vcpu_vmx *vmx = to_vmx(vcpu); 4577 uint32_t intr; 4578 int irq = vcpu->arch.interrupt.nr; 4579 4580 trace_kvm_inj_virq(irq); 4581 4582 ++vcpu->stat.irq_injections; 4583 if (vmx->rmode.vm86_active) { 4584 int inc_eip = 0; 4585 if (vcpu->arch.interrupt.soft) 4586 inc_eip = vcpu->arch.event_exit_inst_len; 4587 kvm_inject_realmode_interrupt(vcpu, irq, inc_eip); 4588 return; 4589 } 4590 intr = irq | INTR_INFO_VALID_MASK; 4591 if (vcpu->arch.interrupt.soft) { 4592 intr |= INTR_TYPE_SOFT_INTR; 4593 vmcs_write32(VM_ENTRY_INSTRUCTION_LEN, 4594 vmx->vcpu.arch.event_exit_inst_len); 4595 } else 4596 intr |= INTR_TYPE_EXT_INTR; 4597 vmcs_write32(VM_ENTRY_INTR_INFO_FIELD, intr); 4598 4599 vmx_clear_hlt(vcpu); 4600 } 4601 4602 static void vmx_inject_nmi(struct kvm_vcpu *vcpu) 4603 { 4604 struct vcpu_vmx *vmx = to_vmx(vcpu); 4605 4606 if (!enable_vnmi) { 4607 /* 4608 * Tracking the NMI-blocked state in software is built upon 4609 * finding the next open IRQ window. This, in turn, depends on 4610 * well-behaving guests: They have to keep IRQs disabled at 4611 * least as long as the NMI handler runs. Otherwise we may 4612 * cause NMI nesting, maybe breaking the guest. But as this is 4613 * highly unlikely, we can live with the residual risk. 4614 */ 4615 vmx->loaded_vmcs->soft_vnmi_blocked = 1; 4616 vmx->loaded_vmcs->vnmi_blocked_time = 0; 4617 } 4618 4619 ++vcpu->stat.nmi_injections; 4620 vmx->loaded_vmcs->nmi_known_unmasked = false; 4621 4622 if (vmx->rmode.vm86_active) { 4623 kvm_inject_realmode_interrupt(vcpu, NMI_VECTOR, 0); 4624 return; 4625 } 4626 4627 vmcs_write32(VM_ENTRY_INTR_INFO_FIELD, 4628 INTR_TYPE_NMI_INTR | INTR_INFO_VALID_MASK | NMI_VECTOR); 4629 4630 vmx_clear_hlt(vcpu); 4631 } 4632 4633 bool vmx_get_nmi_mask(struct kvm_vcpu *vcpu) 4634 { 4635 struct vcpu_vmx *vmx = to_vmx(vcpu); 4636 bool masked; 4637 4638 if (!enable_vnmi) 4639 return vmx->loaded_vmcs->soft_vnmi_blocked; 4640 if (vmx->loaded_vmcs->nmi_known_unmasked) 4641 return false; 4642 masked = vmcs_read32(GUEST_INTERRUPTIBILITY_INFO) & GUEST_INTR_STATE_NMI; 4643 vmx->loaded_vmcs->nmi_known_unmasked = !masked; 4644 return masked; 4645 } 4646 4647 void vmx_set_nmi_mask(struct kvm_vcpu *vcpu, bool masked) 4648 { 4649 struct vcpu_vmx *vmx = to_vmx(vcpu); 4650 4651 if (!enable_vnmi) { 4652 if (vmx->loaded_vmcs->soft_vnmi_blocked != masked) { 4653 vmx->loaded_vmcs->soft_vnmi_blocked = masked; 4654 vmx->loaded_vmcs->vnmi_blocked_time = 0; 4655 } 4656 } else { 4657 vmx->loaded_vmcs->nmi_known_unmasked = !masked; 4658 if (masked) 4659 vmcs_set_bits(GUEST_INTERRUPTIBILITY_INFO, 4660 GUEST_INTR_STATE_NMI); 4661 else 4662 vmcs_clear_bits(GUEST_INTERRUPTIBILITY_INFO, 4663 GUEST_INTR_STATE_NMI); 4664 } 4665 } 4666 4667 bool vmx_nmi_blocked(struct kvm_vcpu *vcpu) 4668 { 4669 if (is_guest_mode(vcpu) && nested_exit_on_nmi(vcpu)) 4670 return false; 4671 4672 if (!enable_vnmi && to_vmx(vcpu)->loaded_vmcs->soft_vnmi_blocked) 4673 return true; 4674 4675 return (vmcs_read32(GUEST_INTERRUPTIBILITY_INFO) & 4676 (GUEST_INTR_STATE_MOV_SS | GUEST_INTR_STATE_STI | 4677 GUEST_INTR_STATE_NMI)); 4678 } 4679 4680 static int vmx_nmi_allowed(struct kvm_vcpu *vcpu, bool for_injection) 4681 { 4682 if (to_vmx(vcpu)->nested.nested_run_pending) 4683 return -EBUSY; 4684 4685 /* An NMI must not be injected into L2 if it's supposed to VM-Exit. */ 4686 if (for_injection && is_guest_mode(vcpu) && nested_exit_on_nmi(vcpu)) 4687 return -EBUSY; 4688 4689 return !vmx_nmi_blocked(vcpu); 4690 } 4691 4692 bool vmx_interrupt_blocked(struct kvm_vcpu *vcpu) 4693 { 4694 if (is_guest_mode(vcpu) && nested_exit_on_intr(vcpu)) 4695 return false; 4696 4697 return !(vmx_get_rflags(vcpu) & X86_EFLAGS_IF) || 4698 (vmcs_read32(GUEST_INTERRUPTIBILITY_INFO) & 4699 (GUEST_INTR_STATE_STI | GUEST_INTR_STATE_MOV_SS)); 4700 } 4701 4702 static int vmx_interrupt_allowed(struct kvm_vcpu *vcpu, bool for_injection) 4703 { 4704 if (to_vmx(vcpu)->nested.nested_run_pending) 4705 return -EBUSY; 4706 4707 /* 4708 * An IRQ must not be injected into L2 if it's supposed to VM-Exit, 4709 * e.g. if the IRQ arrived asynchronously after checking nested events. 4710 */ 4711 if (for_injection && is_guest_mode(vcpu) && nested_exit_on_intr(vcpu)) 4712 return -EBUSY; 4713 4714 return !vmx_interrupt_blocked(vcpu); 4715 } 4716 4717 static int vmx_set_tss_addr(struct kvm *kvm, unsigned int addr) 4718 { 4719 void __user *ret; 4720 4721 if (enable_unrestricted_guest) 4722 return 0; 4723 4724 mutex_lock(&kvm->slots_lock); 4725 ret = __x86_set_memory_region(kvm, TSS_PRIVATE_MEMSLOT, addr, 4726 PAGE_SIZE * 3); 4727 mutex_unlock(&kvm->slots_lock); 4728 4729 if (IS_ERR(ret)) 4730 return PTR_ERR(ret); 4731 4732 to_kvm_vmx(kvm)->tss_addr = addr; 4733 4734 return init_rmode_tss(kvm, ret); 4735 } 4736 4737 static int vmx_set_identity_map_addr(struct kvm *kvm, u64 ident_addr) 4738 { 4739 to_kvm_vmx(kvm)->ept_identity_map_addr = ident_addr; 4740 return 0; 4741 } 4742 4743 static bool rmode_exception(struct kvm_vcpu *vcpu, int vec) 4744 { 4745 switch (vec) { 4746 case BP_VECTOR: 4747 /* 4748 * Update instruction length as we may reinject the exception 4749 * from user space while in guest debugging mode. 4750 */ 4751 to_vmx(vcpu)->vcpu.arch.event_exit_inst_len = 4752 vmcs_read32(VM_EXIT_INSTRUCTION_LEN); 4753 if (vcpu->guest_debug & KVM_GUESTDBG_USE_SW_BP) 4754 return false; 4755 fallthrough; 4756 case DB_VECTOR: 4757 return !(vcpu->guest_debug & 4758 (KVM_GUESTDBG_SINGLESTEP | KVM_GUESTDBG_USE_HW_BP)); 4759 case DE_VECTOR: 4760 case OF_VECTOR: 4761 case BR_VECTOR: 4762 case UD_VECTOR: 4763 case DF_VECTOR: 4764 case SS_VECTOR: 4765 case GP_VECTOR: 4766 case MF_VECTOR: 4767 return true; 4768 } 4769 return false; 4770 } 4771 4772 static int handle_rmode_exception(struct kvm_vcpu *vcpu, 4773 int vec, u32 err_code) 4774 { 4775 /* 4776 * Instruction with address size override prefix opcode 0x67 4777 * Cause the #SS fault with 0 error code in VM86 mode. 4778 */ 4779 if (((vec == GP_VECTOR) || (vec == SS_VECTOR)) && err_code == 0) { 4780 if (kvm_emulate_instruction(vcpu, 0)) { 4781 if (vcpu->arch.halt_request) { 4782 vcpu->arch.halt_request = 0; 4783 return kvm_emulate_halt_noskip(vcpu); 4784 } 4785 return 1; 4786 } 4787 return 0; 4788 } 4789 4790 /* 4791 * Forward all other exceptions that are valid in real mode. 4792 * FIXME: Breaks guest debugging in real mode, needs to be fixed with 4793 * the required debugging infrastructure rework. 4794 */ 4795 kvm_queue_exception(vcpu, vec); 4796 return 1; 4797 } 4798 4799 static int handle_machine_check(struct kvm_vcpu *vcpu) 4800 { 4801 /* handled by vmx_vcpu_run() */ 4802 return 1; 4803 } 4804 4805 /* 4806 * If the host has split lock detection disabled, then #AC is 4807 * unconditionally injected into the guest, which is the pre split lock 4808 * detection behaviour. 4809 * 4810 * If the host has split lock detection enabled then #AC is 4811 * only injected into the guest when: 4812 * - Guest CPL == 3 (user mode) 4813 * - Guest has #AC detection enabled in CR0 4814 * - Guest EFLAGS has AC bit set 4815 */ 4816 bool vmx_guest_inject_ac(struct kvm_vcpu *vcpu) 4817 { 4818 if (!boot_cpu_has(X86_FEATURE_SPLIT_LOCK_DETECT)) 4819 return true; 4820 4821 return vmx_get_cpl(vcpu) == 3 && kvm_read_cr0_bits(vcpu, X86_CR0_AM) && 4822 (kvm_get_rflags(vcpu) & X86_EFLAGS_AC); 4823 } 4824 4825 static int handle_exception_nmi(struct kvm_vcpu *vcpu) 4826 { 4827 struct vcpu_vmx *vmx = to_vmx(vcpu); 4828 struct kvm_run *kvm_run = vcpu->run; 4829 u32 intr_info, ex_no, error_code; 4830 unsigned long cr2, dr6; 4831 u32 vect_info; 4832 4833 vect_info = vmx->idt_vectoring_info; 4834 intr_info = vmx_get_intr_info(vcpu); 4835 4836 if (is_machine_check(intr_info) || is_nmi(intr_info)) 4837 return 1; /* handled by handle_exception_nmi_irqoff() */ 4838 4839 /* 4840 * Queue the exception here instead of in handle_nm_fault_irqoff(). 4841 * This ensures the nested_vmx check is not skipped so vmexit can 4842 * be reflected to L1 (when it intercepts #NM) before reaching this 4843 * point. 4844 */ 4845 if (is_nm_fault(intr_info)) { 4846 kvm_queue_exception(vcpu, NM_VECTOR); 4847 return 1; 4848 } 4849 4850 if (is_invalid_opcode(intr_info)) 4851 return handle_ud(vcpu); 4852 4853 error_code = 0; 4854 if (intr_info & INTR_INFO_DELIVER_CODE_MASK) 4855 error_code = vmcs_read32(VM_EXIT_INTR_ERROR_CODE); 4856 4857 if (!vmx->rmode.vm86_active && is_gp_fault(intr_info)) { 4858 WARN_ON_ONCE(!enable_vmware_backdoor); 4859 4860 /* 4861 * VMware backdoor emulation on #GP interception only handles 4862 * IN{S}, OUT{S}, and RDPMC, none of which generate a non-zero 4863 * error code on #GP. 4864 */ 4865 if (error_code) { 4866 kvm_queue_exception_e(vcpu, GP_VECTOR, error_code); 4867 return 1; 4868 } 4869 return kvm_emulate_instruction(vcpu, EMULTYPE_VMWARE_GP); 4870 } 4871 4872 /* 4873 * The #PF with PFEC.RSVD = 1 indicates the guest is accessing 4874 * MMIO, it is better to report an internal error. 4875 * See the comments in vmx_handle_exit. 4876 */ 4877 if ((vect_info & VECTORING_INFO_VALID_MASK) && 4878 !(is_page_fault(intr_info) && !(error_code & PFERR_RSVD_MASK))) { 4879 vcpu->run->exit_reason = KVM_EXIT_INTERNAL_ERROR; 4880 vcpu->run->internal.suberror = KVM_INTERNAL_ERROR_SIMUL_EX; 4881 vcpu->run->internal.ndata = 4; 4882 vcpu->run->internal.data[0] = vect_info; 4883 vcpu->run->internal.data[1] = intr_info; 4884 vcpu->run->internal.data[2] = error_code; 4885 vcpu->run->internal.data[3] = vcpu->arch.last_vmentry_cpu; 4886 return 0; 4887 } 4888 4889 if (is_page_fault(intr_info)) { 4890 cr2 = vmx_get_exit_qual(vcpu); 4891 if (enable_ept && !vcpu->arch.apf.host_apf_flags) { 4892 /* 4893 * EPT will cause page fault only if we need to 4894 * detect illegal GPAs. 4895 */ 4896 WARN_ON_ONCE(!allow_smaller_maxphyaddr); 4897 kvm_fixup_and_inject_pf_error(vcpu, cr2, error_code); 4898 return 1; 4899 } else 4900 return kvm_handle_page_fault(vcpu, error_code, cr2, NULL, 0); 4901 } 4902 4903 ex_no = intr_info & INTR_INFO_VECTOR_MASK; 4904 4905 if (vmx->rmode.vm86_active && rmode_exception(vcpu, ex_no)) 4906 return handle_rmode_exception(vcpu, ex_no, error_code); 4907 4908 switch (ex_no) { 4909 case DB_VECTOR: 4910 dr6 = vmx_get_exit_qual(vcpu); 4911 if (!(vcpu->guest_debug & 4912 (KVM_GUESTDBG_SINGLESTEP | KVM_GUESTDBG_USE_HW_BP))) { 4913 /* 4914 * If the #DB was due to ICEBP, a.k.a. INT1, skip the 4915 * instruction. ICEBP generates a trap-like #DB, but 4916 * despite its interception control being tied to #DB, 4917 * is an instruction intercept, i.e. the VM-Exit occurs 4918 * on the ICEBP itself. Note, skipping ICEBP also 4919 * clears STI and MOVSS blocking. 4920 * 4921 * For all other #DBs, set vmcs.PENDING_DBG_EXCEPTIONS.BS 4922 * if single-step is enabled in RFLAGS and STI or MOVSS 4923 * blocking is active, as the CPU doesn't set the bit 4924 * on VM-Exit due to #DB interception. VM-Entry has a 4925 * consistency check that a single-step #DB is pending 4926 * in this scenario as the previous instruction cannot 4927 * have toggled RFLAGS.TF 0=>1 (because STI and POP/MOV 4928 * don't modify RFLAGS), therefore the one instruction 4929 * delay when activating single-step breakpoints must 4930 * have already expired. Note, the CPU sets/clears BS 4931 * as appropriate for all other VM-Exits types. 4932 */ 4933 if (is_icebp(intr_info)) 4934 WARN_ON(!skip_emulated_instruction(vcpu)); 4935 else if ((vmx_get_rflags(vcpu) & X86_EFLAGS_TF) && 4936 (vmcs_read32(GUEST_INTERRUPTIBILITY_INFO) & 4937 (GUEST_INTR_STATE_STI | GUEST_INTR_STATE_MOV_SS))) 4938 vmcs_writel(GUEST_PENDING_DBG_EXCEPTIONS, 4939 vmcs_readl(GUEST_PENDING_DBG_EXCEPTIONS) | DR6_BS); 4940 4941 kvm_queue_exception_p(vcpu, DB_VECTOR, dr6); 4942 return 1; 4943 } 4944 kvm_run->debug.arch.dr6 = dr6 | DR6_ACTIVE_LOW; 4945 kvm_run->debug.arch.dr7 = vmcs_readl(GUEST_DR7); 4946 fallthrough; 4947 case BP_VECTOR: 4948 /* 4949 * Update instruction length as we may reinject #BP from 4950 * user space while in guest debugging mode. Reading it for 4951 * #DB as well causes no harm, it is not used in that case. 4952 */ 4953 vmx->vcpu.arch.event_exit_inst_len = 4954 vmcs_read32(VM_EXIT_INSTRUCTION_LEN); 4955 kvm_run->exit_reason = KVM_EXIT_DEBUG; 4956 kvm_run->debug.arch.pc = kvm_get_linear_rip(vcpu); 4957 kvm_run->debug.arch.exception = ex_no; 4958 break; 4959 case AC_VECTOR: 4960 if (vmx_guest_inject_ac(vcpu)) { 4961 kvm_queue_exception_e(vcpu, AC_VECTOR, error_code); 4962 return 1; 4963 } 4964 4965 /* 4966 * Handle split lock. Depending on detection mode this will 4967 * either warn and disable split lock detection for this 4968 * task or force SIGBUS on it. 4969 */ 4970 if (handle_guest_split_lock(kvm_rip_read(vcpu))) 4971 return 1; 4972 fallthrough; 4973 default: 4974 kvm_run->exit_reason = KVM_EXIT_EXCEPTION; 4975 kvm_run->ex.exception = ex_no; 4976 kvm_run->ex.error_code = error_code; 4977 break; 4978 } 4979 return 0; 4980 } 4981 4982 static __always_inline int handle_external_interrupt(struct kvm_vcpu *vcpu) 4983 { 4984 ++vcpu->stat.irq_exits; 4985 return 1; 4986 } 4987 4988 static int handle_triple_fault(struct kvm_vcpu *vcpu) 4989 { 4990 vcpu->run->exit_reason = KVM_EXIT_SHUTDOWN; 4991 vcpu->mmio_needed = 0; 4992 return 0; 4993 } 4994 4995 static int handle_io(struct kvm_vcpu *vcpu) 4996 { 4997 unsigned long exit_qualification; 4998 int size, in, string; 4999 unsigned port; 5000 5001 exit_qualification = vmx_get_exit_qual(vcpu); 5002 string = (exit_qualification & 16) != 0; 5003 5004 ++vcpu->stat.io_exits; 5005 5006 if (string) 5007 return kvm_emulate_instruction(vcpu, 0); 5008 5009 port = exit_qualification >> 16; 5010 size = (exit_qualification & 7) + 1; 5011 in = (exit_qualification & 8) != 0; 5012 5013 return kvm_fast_pio(vcpu, size, port, in); 5014 } 5015 5016 static void 5017 vmx_patch_hypercall(struct kvm_vcpu *vcpu, unsigned char *hypercall) 5018 { 5019 /* 5020 * Patch in the VMCALL instruction: 5021 */ 5022 hypercall[0] = 0x0f; 5023 hypercall[1] = 0x01; 5024 hypercall[2] = 0xc1; 5025 } 5026 5027 /* called to set cr0 as appropriate for a mov-to-cr0 exit. */ 5028 static int handle_set_cr0(struct kvm_vcpu *vcpu, unsigned long val) 5029 { 5030 if (is_guest_mode(vcpu)) { 5031 struct vmcs12 *vmcs12 = get_vmcs12(vcpu); 5032 unsigned long orig_val = val; 5033 5034 /* 5035 * We get here when L2 changed cr0 in a way that did not change 5036 * any of L1's shadowed bits (see nested_vmx_exit_handled_cr), 5037 * but did change L0 shadowed bits. So we first calculate the 5038 * effective cr0 value that L1 would like to write into the 5039 * hardware. It consists of the L2-owned bits from the new 5040 * value combined with the L1-owned bits from L1's guest_cr0. 5041 */ 5042 val = (val & ~vmcs12->cr0_guest_host_mask) | 5043 (vmcs12->guest_cr0 & vmcs12->cr0_guest_host_mask); 5044 5045 if (!nested_guest_cr0_valid(vcpu, val)) 5046 return 1; 5047 5048 if (kvm_set_cr0(vcpu, val)) 5049 return 1; 5050 vmcs_writel(CR0_READ_SHADOW, orig_val); 5051 return 0; 5052 } else { 5053 if (to_vmx(vcpu)->nested.vmxon && 5054 !nested_host_cr0_valid(vcpu, val)) 5055 return 1; 5056 5057 return kvm_set_cr0(vcpu, val); 5058 } 5059 } 5060 5061 static int handle_set_cr4(struct kvm_vcpu *vcpu, unsigned long val) 5062 { 5063 if (is_guest_mode(vcpu)) { 5064 struct vmcs12 *vmcs12 = get_vmcs12(vcpu); 5065 unsigned long orig_val = val; 5066 5067 /* analogously to handle_set_cr0 */ 5068 val = (val & ~vmcs12->cr4_guest_host_mask) | 5069 (vmcs12->guest_cr4 & vmcs12->cr4_guest_host_mask); 5070 if (kvm_set_cr4(vcpu, val)) 5071 return 1; 5072 vmcs_writel(CR4_READ_SHADOW, orig_val); 5073 return 0; 5074 } else 5075 return kvm_set_cr4(vcpu, val); 5076 } 5077 5078 static int handle_desc(struct kvm_vcpu *vcpu) 5079 { 5080 WARN_ON(!(vcpu->arch.cr4 & X86_CR4_UMIP)); 5081 return kvm_emulate_instruction(vcpu, 0); 5082 } 5083 5084 static int handle_cr(struct kvm_vcpu *vcpu) 5085 { 5086 unsigned long exit_qualification, val; 5087 int cr; 5088 int reg; 5089 int err; 5090 int ret; 5091 5092 exit_qualification = vmx_get_exit_qual(vcpu); 5093 cr = exit_qualification & 15; 5094 reg = (exit_qualification >> 8) & 15; 5095 switch ((exit_qualification >> 4) & 3) { 5096 case 0: /* mov to cr */ 5097 val = kvm_register_read(vcpu, reg); 5098 trace_kvm_cr_write(cr, val); 5099 switch (cr) { 5100 case 0: 5101 err = handle_set_cr0(vcpu, val); 5102 return kvm_complete_insn_gp(vcpu, err); 5103 case 3: 5104 WARN_ON_ONCE(enable_unrestricted_guest); 5105 5106 err = kvm_set_cr3(vcpu, val); 5107 return kvm_complete_insn_gp(vcpu, err); 5108 case 4: 5109 err = handle_set_cr4(vcpu, val); 5110 return kvm_complete_insn_gp(vcpu, err); 5111 case 8: { 5112 u8 cr8_prev = kvm_get_cr8(vcpu); 5113 u8 cr8 = (u8)val; 5114 err = kvm_set_cr8(vcpu, cr8); 5115 ret = kvm_complete_insn_gp(vcpu, err); 5116 if (lapic_in_kernel(vcpu)) 5117 return ret; 5118 if (cr8_prev <= cr8) 5119 return ret; 5120 /* 5121 * TODO: we might be squashing a 5122 * KVM_GUESTDBG_SINGLESTEP-triggered 5123 * KVM_EXIT_DEBUG here. 5124 */ 5125 vcpu->run->exit_reason = KVM_EXIT_SET_TPR; 5126 return 0; 5127 } 5128 } 5129 break; 5130 case 2: /* clts */ 5131 KVM_BUG(1, vcpu->kvm, "Guest always owns CR0.TS"); 5132 return -EIO; 5133 case 1: /*mov from cr*/ 5134 switch (cr) { 5135 case 3: 5136 WARN_ON_ONCE(enable_unrestricted_guest); 5137 5138 val = kvm_read_cr3(vcpu); 5139 kvm_register_write(vcpu, reg, val); 5140 trace_kvm_cr_read(cr, val); 5141 return kvm_skip_emulated_instruction(vcpu); 5142 case 8: 5143 val = kvm_get_cr8(vcpu); 5144 kvm_register_write(vcpu, reg, val); 5145 trace_kvm_cr_read(cr, val); 5146 return kvm_skip_emulated_instruction(vcpu); 5147 } 5148 break; 5149 case 3: /* lmsw */ 5150 val = (exit_qualification >> LMSW_SOURCE_DATA_SHIFT) & 0x0f; 5151 trace_kvm_cr_write(0, (kvm_read_cr0(vcpu) & ~0xful) | val); 5152 kvm_lmsw(vcpu, val); 5153 5154 return kvm_skip_emulated_instruction(vcpu); 5155 default: 5156 break; 5157 } 5158 vcpu->run->exit_reason = 0; 5159 vcpu_unimpl(vcpu, "unhandled control register: op %d cr %d\n", 5160 (int)(exit_qualification >> 4) & 3, cr); 5161 return 0; 5162 } 5163 5164 static int handle_dr(struct kvm_vcpu *vcpu) 5165 { 5166 unsigned long exit_qualification; 5167 int dr, dr7, reg; 5168 int err = 1; 5169 5170 exit_qualification = vmx_get_exit_qual(vcpu); 5171 dr = exit_qualification & DEBUG_REG_ACCESS_NUM; 5172 5173 /* First, if DR does not exist, trigger UD */ 5174 if (!kvm_require_dr(vcpu, dr)) 5175 return 1; 5176 5177 if (vmx_get_cpl(vcpu) > 0) 5178 goto out; 5179 5180 dr7 = vmcs_readl(GUEST_DR7); 5181 if (dr7 & DR7_GD) { 5182 /* 5183 * As the vm-exit takes precedence over the debug trap, we 5184 * need to emulate the latter, either for the host or the 5185 * guest debugging itself. 5186 */ 5187 if (vcpu->guest_debug & KVM_GUESTDBG_USE_HW_BP) { 5188 vcpu->run->debug.arch.dr6 = DR6_BD | DR6_ACTIVE_LOW; 5189 vcpu->run->debug.arch.dr7 = dr7; 5190 vcpu->run->debug.arch.pc = kvm_get_linear_rip(vcpu); 5191 vcpu->run->debug.arch.exception = DB_VECTOR; 5192 vcpu->run->exit_reason = KVM_EXIT_DEBUG; 5193 return 0; 5194 } else { 5195 kvm_queue_exception_p(vcpu, DB_VECTOR, DR6_BD); 5196 return 1; 5197 } 5198 } 5199 5200 if (vcpu->guest_debug == 0) { 5201 exec_controls_clearbit(to_vmx(vcpu), CPU_BASED_MOV_DR_EXITING); 5202 5203 /* 5204 * No more DR vmexits; force a reload of the debug registers 5205 * and reenter on this instruction. The next vmexit will 5206 * retrieve the full state of the debug registers. 5207 */ 5208 vcpu->arch.switch_db_regs |= KVM_DEBUGREG_WONT_EXIT; 5209 return 1; 5210 } 5211 5212 reg = DEBUG_REG_ACCESS_REG(exit_qualification); 5213 if (exit_qualification & TYPE_MOV_FROM_DR) { 5214 unsigned long val; 5215 5216 kvm_get_dr(vcpu, dr, &val); 5217 kvm_register_write(vcpu, reg, val); 5218 err = 0; 5219 } else { 5220 err = kvm_set_dr(vcpu, dr, kvm_register_read(vcpu, reg)); 5221 } 5222 5223 out: 5224 return kvm_complete_insn_gp(vcpu, err); 5225 } 5226 5227 static void vmx_sync_dirty_debug_regs(struct kvm_vcpu *vcpu) 5228 { 5229 get_debugreg(vcpu->arch.db[0], 0); 5230 get_debugreg(vcpu->arch.db[1], 1); 5231 get_debugreg(vcpu->arch.db[2], 2); 5232 get_debugreg(vcpu->arch.db[3], 3); 5233 get_debugreg(vcpu->arch.dr6, 6); 5234 vcpu->arch.dr7 = vmcs_readl(GUEST_DR7); 5235 5236 vcpu->arch.switch_db_regs &= ~KVM_DEBUGREG_WONT_EXIT; 5237 exec_controls_setbit(to_vmx(vcpu), CPU_BASED_MOV_DR_EXITING); 5238 5239 /* 5240 * exc_debug expects dr6 to be cleared after it runs, avoid that it sees 5241 * a stale dr6 from the guest. 5242 */ 5243 set_debugreg(DR6_RESERVED, 6); 5244 } 5245 5246 static void vmx_set_dr7(struct kvm_vcpu *vcpu, unsigned long val) 5247 { 5248 vmcs_writel(GUEST_DR7, val); 5249 } 5250 5251 static int handle_tpr_below_threshold(struct kvm_vcpu *vcpu) 5252 { 5253 kvm_apic_update_ppr(vcpu); 5254 return 1; 5255 } 5256 5257 static int handle_interrupt_window(struct kvm_vcpu *vcpu) 5258 { 5259 exec_controls_clearbit(to_vmx(vcpu), CPU_BASED_INTR_WINDOW_EXITING); 5260 5261 kvm_make_request(KVM_REQ_EVENT, vcpu); 5262 5263 ++vcpu->stat.irq_window_exits; 5264 return 1; 5265 } 5266 5267 static int handle_invlpg(struct kvm_vcpu *vcpu) 5268 { 5269 unsigned long exit_qualification = vmx_get_exit_qual(vcpu); 5270 5271 kvm_mmu_invlpg(vcpu, exit_qualification); 5272 return kvm_skip_emulated_instruction(vcpu); 5273 } 5274 5275 static int handle_apic_access(struct kvm_vcpu *vcpu) 5276 { 5277 if (likely(fasteoi)) { 5278 unsigned long exit_qualification = vmx_get_exit_qual(vcpu); 5279 int access_type, offset; 5280 5281 access_type = exit_qualification & APIC_ACCESS_TYPE; 5282 offset = exit_qualification & APIC_ACCESS_OFFSET; 5283 /* 5284 * Sane guest uses MOV to write EOI, with written value 5285 * not cared. So make a short-circuit here by avoiding 5286 * heavy instruction emulation. 5287 */ 5288 if ((access_type == TYPE_LINEAR_APIC_INST_WRITE) && 5289 (offset == APIC_EOI)) { 5290 kvm_lapic_set_eoi(vcpu); 5291 return kvm_skip_emulated_instruction(vcpu); 5292 } 5293 } 5294 return kvm_emulate_instruction(vcpu, 0); 5295 } 5296 5297 static int handle_apic_eoi_induced(struct kvm_vcpu *vcpu) 5298 { 5299 unsigned long exit_qualification = vmx_get_exit_qual(vcpu); 5300 int vector = exit_qualification & 0xff; 5301 5302 /* EOI-induced VM exit is trap-like and thus no need to adjust IP */ 5303 kvm_apic_set_eoi_accelerated(vcpu, vector); 5304 return 1; 5305 } 5306 5307 static int handle_apic_write(struct kvm_vcpu *vcpu) 5308 { 5309 unsigned long exit_qualification = vmx_get_exit_qual(vcpu); 5310 5311 /* 5312 * APIC-write VM-Exit is trap-like, KVM doesn't need to advance RIP and 5313 * hardware has done any necessary aliasing, offset adjustments, etc... 5314 * for the access. I.e. the correct value has already been written to 5315 * the vAPIC page for the correct 16-byte chunk. KVM needs only to 5316 * retrieve the register value and emulate the access. 5317 */ 5318 u32 offset = exit_qualification & 0xff0; 5319 5320 kvm_apic_write_nodecode(vcpu, offset); 5321 return 1; 5322 } 5323 5324 static int handle_task_switch(struct kvm_vcpu *vcpu) 5325 { 5326 struct vcpu_vmx *vmx = to_vmx(vcpu); 5327 unsigned long exit_qualification; 5328 bool has_error_code = false; 5329 u32 error_code = 0; 5330 u16 tss_selector; 5331 int reason, type, idt_v, idt_index; 5332 5333 idt_v = (vmx->idt_vectoring_info & VECTORING_INFO_VALID_MASK); 5334 idt_index = (vmx->idt_vectoring_info & VECTORING_INFO_VECTOR_MASK); 5335 type = (vmx->idt_vectoring_info & VECTORING_INFO_TYPE_MASK); 5336 5337 exit_qualification = vmx_get_exit_qual(vcpu); 5338 5339 reason = (u32)exit_qualification >> 30; 5340 if (reason == TASK_SWITCH_GATE && idt_v) { 5341 switch (type) { 5342 case INTR_TYPE_NMI_INTR: 5343 vcpu->arch.nmi_injected = false; 5344 vmx_set_nmi_mask(vcpu, true); 5345 break; 5346 case INTR_TYPE_EXT_INTR: 5347 case INTR_TYPE_SOFT_INTR: 5348 kvm_clear_interrupt_queue(vcpu); 5349 break; 5350 case INTR_TYPE_HARD_EXCEPTION: 5351 if (vmx->idt_vectoring_info & 5352 VECTORING_INFO_DELIVER_CODE_MASK) { 5353 has_error_code = true; 5354 error_code = 5355 vmcs_read32(IDT_VECTORING_ERROR_CODE); 5356 } 5357 fallthrough; 5358 case INTR_TYPE_SOFT_EXCEPTION: 5359 kvm_clear_exception_queue(vcpu); 5360 break; 5361 default: 5362 break; 5363 } 5364 } 5365 tss_selector = exit_qualification; 5366 5367 if (!idt_v || (type != INTR_TYPE_HARD_EXCEPTION && 5368 type != INTR_TYPE_EXT_INTR && 5369 type != INTR_TYPE_NMI_INTR)) 5370 WARN_ON(!skip_emulated_instruction(vcpu)); 5371 5372 /* 5373 * TODO: What about debug traps on tss switch? 5374 * Are we supposed to inject them and update dr6? 5375 */ 5376 return kvm_task_switch(vcpu, tss_selector, 5377 type == INTR_TYPE_SOFT_INTR ? idt_index : -1, 5378 reason, has_error_code, error_code); 5379 } 5380 5381 static int handle_ept_violation(struct kvm_vcpu *vcpu) 5382 { 5383 unsigned long exit_qualification; 5384 gpa_t gpa; 5385 u64 error_code; 5386 5387 exit_qualification = vmx_get_exit_qual(vcpu); 5388 5389 /* 5390 * EPT violation happened while executing iret from NMI, 5391 * "blocked by NMI" bit has to be set before next VM entry. 5392 * There are errata that may cause this bit to not be set: 5393 * AAK134, BY25. 5394 */ 5395 if (!(to_vmx(vcpu)->idt_vectoring_info & VECTORING_INFO_VALID_MASK) && 5396 enable_vnmi && 5397 (exit_qualification & INTR_INFO_UNBLOCK_NMI)) 5398 vmcs_set_bits(GUEST_INTERRUPTIBILITY_INFO, GUEST_INTR_STATE_NMI); 5399 5400 gpa = vmcs_read64(GUEST_PHYSICAL_ADDRESS); 5401 trace_kvm_page_fault(gpa, exit_qualification); 5402 5403 /* Is it a read fault? */ 5404 error_code = (exit_qualification & EPT_VIOLATION_ACC_READ) 5405 ? PFERR_USER_MASK : 0; 5406 /* Is it a write fault? */ 5407 error_code |= (exit_qualification & EPT_VIOLATION_ACC_WRITE) 5408 ? PFERR_WRITE_MASK : 0; 5409 /* Is it a fetch fault? */ 5410 error_code |= (exit_qualification & EPT_VIOLATION_ACC_INSTR) 5411 ? PFERR_FETCH_MASK : 0; 5412 /* ept page table entry is present? */ 5413 error_code |= (exit_qualification & 5414 (EPT_VIOLATION_READABLE | EPT_VIOLATION_WRITABLE | 5415 EPT_VIOLATION_EXECUTABLE)) 5416 ? PFERR_PRESENT_MASK : 0; 5417 5418 error_code |= (exit_qualification & EPT_VIOLATION_GVA_TRANSLATED) != 0 ? 5419 PFERR_GUEST_FINAL_MASK : PFERR_GUEST_PAGE_MASK; 5420 5421 vcpu->arch.exit_qualification = exit_qualification; 5422 5423 /* 5424 * Check that the GPA doesn't exceed physical memory limits, as that is 5425 * a guest page fault. We have to emulate the instruction here, because 5426 * if the illegal address is that of a paging structure, then 5427 * EPT_VIOLATION_ACC_WRITE bit is set. Alternatively, if supported we 5428 * would also use advanced VM-exit information for EPT violations to 5429 * reconstruct the page fault error code. 5430 */ 5431 if (unlikely(allow_smaller_maxphyaddr && kvm_vcpu_is_illegal_gpa(vcpu, gpa))) 5432 return kvm_emulate_instruction(vcpu, 0); 5433 5434 return kvm_mmu_page_fault(vcpu, gpa, error_code, NULL, 0); 5435 } 5436 5437 static int handle_ept_misconfig(struct kvm_vcpu *vcpu) 5438 { 5439 gpa_t gpa; 5440 5441 if (!vmx_can_emulate_instruction(vcpu, EMULTYPE_PF, NULL, 0)) 5442 return 1; 5443 5444 /* 5445 * A nested guest cannot optimize MMIO vmexits, because we have an 5446 * nGPA here instead of the required GPA. 5447 */ 5448 gpa = vmcs_read64(GUEST_PHYSICAL_ADDRESS); 5449 if (!is_guest_mode(vcpu) && 5450 !kvm_io_bus_write(vcpu, KVM_FAST_MMIO_BUS, gpa, 0, NULL)) { 5451 trace_kvm_fast_mmio(gpa); 5452 return kvm_skip_emulated_instruction(vcpu); 5453 } 5454 5455 return kvm_mmu_page_fault(vcpu, gpa, PFERR_RSVD_MASK, NULL, 0); 5456 } 5457 5458 static int handle_nmi_window(struct kvm_vcpu *vcpu) 5459 { 5460 if (KVM_BUG_ON(!enable_vnmi, vcpu->kvm)) 5461 return -EIO; 5462 5463 exec_controls_clearbit(to_vmx(vcpu), CPU_BASED_NMI_WINDOW_EXITING); 5464 ++vcpu->stat.nmi_window_exits; 5465 kvm_make_request(KVM_REQ_EVENT, vcpu); 5466 5467 return 1; 5468 } 5469 5470 static bool vmx_emulation_required_with_pending_exception(struct kvm_vcpu *vcpu) 5471 { 5472 struct vcpu_vmx *vmx = to_vmx(vcpu); 5473 5474 return vmx->emulation_required && !vmx->rmode.vm86_active && 5475 vcpu->arch.exception.pending; 5476 } 5477 5478 static int handle_invalid_guest_state(struct kvm_vcpu *vcpu) 5479 { 5480 struct vcpu_vmx *vmx = to_vmx(vcpu); 5481 bool intr_window_requested; 5482 unsigned count = 130; 5483 5484 intr_window_requested = exec_controls_get(vmx) & 5485 CPU_BASED_INTR_WINDOW_EXITING; 5486 5487 while (vmx->emulation_required && count-- != 0) { 5488 if (intr_window_requested && !vmx_interrupt_blocked(vcpu)) 5489 return handle_interrupt_window(&vmx->vcpu); 5490 5491 if (kvm_test_request(KVM_REQ_EVENT, vcpu)) 5492 return 1; 5493 5494 if (!kvm_emulate_instruction(vcpu, 0)) 5495 return 0; 5496 5497 if (vmx_emulation_required_with_pending_exception(vcpu)) { 5498 kvm_prepare_emulation_failure_exit(vcpu); 5499 return 0; 5500 } 5501 5502 if (vcpu->arch.halt_request) { 5503 vcpu->arch.halt_request = 0; 5504 return kvm_emulate_halt_noskip(vcpu); 5505 } 5506 5507 /* 5508 * Note, return 1 and not 0, vcpu_run() will invoke 5509 * xfer_to_guest_mode() which will create a proper return 5510 * code. 5511 */ 5512 if (__xfer_to_guest_mode_work_pending()) 5513 return 1; 5514 } 5515 5516 return 1; 5517 } 5518 5519 static int vmx_vcpu_pre_run(struct kvm_vcpu *vcpu) 5520 { 5521 if (vmx_emulation_required_with_pending_exception(vcpu)) { 5522 kvm_prepare_emulation_failure_exit(vcpu); 5523 return 0; 5524 } 5525 5526 return 1; 5527 } 5528 5529 static void grow_ple_window(struct kvm_vcpu *vcpu) 5530 { 5531 struct vcpu_vmx *vmx = to_vmx(vcpu); 5532 unsigned int old = vmx->ple_window; 5533 5534 vmx->ple_window = __grow_ple_window(old, ple_window, 5535 ple_window_grow, 5536 ple_window_max); 5537 5538 if (vmx->ple_window != old) { 5539 vmx->ple_window_dirty = true; 5540 trace_kvm_ple_window_update(vcpu->vcpu_id, 5541 vmx->ple_window, old); 5542 } 5543 } 5544 5545 static void shrink_ple_window(struct kvm_vcpu *vcpu) 5546 { 5547 struct vcpu_vmx *vmx = to_vmx(vcpu); 5548 unsigned int old = vmx->ple_window; 5549 5550 vmx->ple_window = __shrink_ple_window(old, ple_window, 5551 ple_window_shrink, 5552 ple_window); 5553 5554 if (vmx->ple_window != old) { 5555 vmx->ple_window_dirty = true; 5556 trace_kvm_ple_window_update(vcpu->vcpu_id, 5557 vmx->ple_window, old); 5558 } 5559 } 5560 5561 /* 5562 * Indicate a busy-waiting vcpu in spinlock. We do not enable the PAUSE 5563 * exiting, so only get here on cpu with PAUSE-Loop-Exiting. 5564 */ 5565 static int handle_pause(struct kvm_vcpu *vcpu) 5566 { 5567 if (!kvm_pause_in_guest(vcpu->kvm)) 5568 grow_ple_window(vcpu); 5569 5570 /* 5571 * Intel sdm vol3 ch-25.1.3 says: The "PAUSE-loop exiting" 5572 * VM-execution control is ignored if CPL > 0. OTOH, KVM 5573 * never set PAUSE_EXITING and just set PLE if supported, 5574 * so the vcpu must be CPL=0 if it gets a PAUSE exit. 5575 */ 5576 kvm_vcpu_on_spin(vcpu, true); 5577 return kvm_skip_emulated_instruction(vcpu); 5578 } 5579 5580 static int handle_monitor_trap(struct kvm_vcpu *vcpu) 5581 { 5582 return 1; 5583 } 5584 5585 static int handle_invpcid(struct kvm_vcpu *vcpu) 5586 { 5587 u32 vmx_instruction_info; 5588 unsigned long type; 5589 gva_t gva; 5590 struct { 5591 u64 pcid; 5592 u64 gla; 5593 } operand; 5594 int gpr_index; 5595 5596 if (!guest_cpuid_has(vcpu, X86_FEATURE_INVPCID)) { 5597 kvm_queue_exception(vcpu, UD_VECTOR); 5598 return 1; 5599 } 5600 5601 vmx_instruction_info = vmcs_read32(VMX_INSTRUCTION_INFO); 5602 gpr_index = vmx_get_instr_info_reg2(vmx_instruction_info); 5603 type = kvm_register_read(vcpu, gpr_index); 5604 5605 /* According to the Intel instruction reference, the memory operand 5606 * is read even if it isn't needed (e.g., for type==all) 5607 */ 5608 if (get_vmx_mem_address(vcpu, vmx_get_exit_qual(vcpu), 5609 vmx_instruction_info, false, 5610 sizeof(operand), &gva)) 5611 return 1; 5612 5613 return kvm_handle_invpcid(vcpu, type, gva); 5614 } 5615 5616 static int handle_pml_full(struct kvm_vcpu *vcpu) 5617 { 5618 unsigned long exit_qualification; 5619 5620 trace_kvm_pml_full(vcpu->vcpu_id); 5621 5622 exit_qualification = vmx_get_exit_qual(vcpu); 5623 5624 /* 5625 * PML buffer FULL happened while executing iret from NMI, 5626 * "blocked by NMI" bit has to be set before next VM entry. 5627 */ 5628 if (!(to_vmx(vcpu)->idt_vectoring_info & VECTORING_INFO_VALID_MASK) && 5629 enable_vnmi && 5630 (exit_qualification & INTR_INFO_UNBLOCK_NMI)) 5631 vmcs_set_bits(GUEST_INTERRUPTIBILITY_INFO, 5632 GUEST_INTR_STATE_NMI); 5633 5634 /* 5635 * PML buffer already flushed at beginning of VMEXIT. Nothing to do 5636 * here.., and there's no userspace involvement needed for PML. 5637 */ 5638 return 1; 5639 } 5640 5641 static fastpath_t handle_fastpath_preemption_timer(struct kvm_vcpu *vcpu) 5642 { 5643 struct vcpu_vmx *vmx = to_vmx(vcpu); 5644 5645 if (!vmx->req_immediate_exit && 5646 !unlikely(vmx->loaded_vmcs->hv_timer_soft_disabled)) { 5647 kvm_lapic_expired_hv_timer(vcpu); 5648 return EXIT_FASTPATH_REENTER_GUEST; 5649 } 5650 5651 return EXIT_FASTPATH_NONE; 5652 } 5653 5654 static int handle_preemption_timer(struct kvm_vcpu *vcpu) 5655 { 5656 handle_fastpath_preemption_timer(vcpu); 5657 return 1; 5658 } 5659 5660 /* 5661 * When nested=0, all VMX instruction VM Exits filter here. The handlers 5662 * are overwritten by nested_vmx_setup() when nested=1. 5663 */ 5664 static int handle_vmx_instruction(struct kvm_vcpu *vcpu) 5665 { 5666 kvm_queue_exception(vcpu, UD_VECTOR); 5667 return 1; 5668 } 5669 5670 #ifndef CONFIG_X86_SGX_KVM 5671 static int handle_encls(struct kvm_vcpu *vcpu) 5672 { 5673 /* 5674 * SGX virtualization is disabled. There is no software enable bit for 5675 * SGX, so KVM intercepts all ENCLS leafs and injects a #UD to prevent 5676 * the guest from executing ENCLS (when SGX is supported by hardware). 5677 */ 5678 kvm_queue_exception(vcpu, UD_VECTOR); 5679 return 1; 5680 } 5681 #endif /* CONFIG_X86_SGX_KVM */ 5682 5683 static int handle_bus_lock_vmexit(struct kvm_vcpu *vcpu) 5684 { 5685 /* 5686 * Hardware may or may not set the BUS_LOCK_DETECTED flag on BUS_LOCK 5687 * VM-Exits. Unconditionally set the flag here and leave the handling to 5688 * vmx_handle_exit(). 5689 */ 5690 to_vmx(vcpu)->exit_reason.bus_lock_detected = true; 5691 return 1; 5692 } 5693 5694 /* 5695 * The exit handlers return 1 if the exit was handled fully and guest execution 5696 * may resume. Otherwise they set the kvm_run parameter to indicate what needs 5697 * to be done to userspace and return 0. 5698 */ 5699 static int (*kvm_vmx_exit_handlers[])(struct kvm_vcpu *vcpu) = { 5700 [EXIT_REASON_EXCEPTION_NMI] = handle_exception_nmi, 5701 [EXIT_REASON_EXTERNAL_INTERRUPT] = handle_external_interrupt, 5702 [EXIT_REASON_TRIPLE_FAULT] = handle_triple_fault, 5703 [EXIT_REASON_NMI_WINDOW] = handle_nmi_window, 5704 [EXIT_REASON_IO_INSTRUCTION] = handle_io, 5705 [EXIT_REASON_CR_ACCESS] = handle_cr, 5706 [EXIT_REASON_DR_ACCESS] = handle_dr, 5707 [EXIT_REASON_CPUID] = kvm_emulate_cpuid, 5708 [EXIT_REASON_MSR_READ] = kvm_emulate_rdmsr, 5709 [EXIT_REASON_MSR_WRITE] = kvm_emulate_wrmsr, 5710 [EXIT_REASON_INTERRUPT_WINDOW] = handle_interrupt_window, 5711 [EXIT_REASON_HLT] = kvm_emulate_halt, 5712 [EXIT_REASON_INVD] = kvm_emulate_invd, 5713 [EXIT_REASON_INVLPG] = handle_invlpg, 5714 [EXIT_REASON_RDPMC] = kvm_emulate_rdpmc, 5715 [EXIT_REASON_VMCALL] = kvm_emulate_hypercall, 5716 [EXIT_REASON_VMCLEAR] = handle_vmx_instruction, 5717 [EXIT_REASON_VMLAUNCH] = handle_vmx_instruction, 5718 [EXIT_REASON_VMPTRLD] = handle_vmx_instruction, 5719 [EXIT_REASON_VMPTRST] = handle_vmx_instruction, 5720 [EXIT_REASON_VMREAD] = handle_vmx_instruction, 5721 [EXIT_REASON_VMRESUME] = handle_vmx_instruction, 5722 [EXIT_REASON_VMWRITE] = handle_vmx_instruction, 5723 [EXIT_REASON_VMOFF] = handle_vmx_instruction, 5724 [EXIT_REASON_VMON] = handle_vmx_instruction, 5725 [EXIT_REASON_TPR_BELOW_THRESHOLD] = handle_tpr_below_threshold, 5726 [EXIT_REASON_APIC_ACCESS] = handle_apic_access, 5727 [EXIT_REASON_APIC_WRITE] = handle_apic_write, 5728 [EXIT_REASON_EOI_INDUCED] = handle_apic_eoi_induced, 5729 [EXIT_REASON_WBINVD] = kvm_emulate_wbinvd, 5730 [EXIT_REASON_XSETBV] = kvm_emulate_xsetbv, 5731 [EXIT_REASON_TASK_SWITCH] = handle_task_switch, 5732 [EXIT_REASON_MCE_DURING_VMENTRY] = handle_machine_check, 5733 [EXIT_REASON_GDTR_IDTR] = handle_desc, 5734 [EXIT_REASON_LDTR_TR] = handle_desc, 5735 [EXIT_REASON_EPT_VIOLATION] = handle_ept_violation, 5736 [EXIT_REASON_EPT_MISCONFIG] = handle_ept_misconfig, 5737 [EXIT_REASON_PAUSE_INSTRUCTION] = handle_pause, 5738 [EXIT_REASON_MWAIT_INSTRUCTION] = kvm_emulate_mwait, 5739 [EXIT_REASON_MONITOR_TRAP_FLAG] = handle_monitor_trap, 5740 [EXIT_REASON_MONITOR_INSTRUCTION] = kvm_emulate_monitor, 5741 [EXIT_REASON_INVEPT] = handle_vmx_instruction, 5742 [EXIT_REASON_INVVPID] = handle_vmx_instruction, 5743 [EXIT_REASON_RDRAND] = kvm_handle_invalid_op, 5744 [EXIT_REASON_RDSEED] = kvm_handle_invalid_op, 5745 [EXIT_REASON_PML_FULL] = handle_pml_full, 5746 [EXIT_REASON_INVPCID] = handle_invpcid, 5747 [EXIT_REASON_VMFUNC] = handle_vmx_instruction, 5748 [EXIT_REASON_PREEMPTION_TIMER] = handle_preemption_timer, 5749 [EXIT_REASON_ENCLS] = handle_encls, 5750 [EXIT_REASON_BUS_LOCK] = handle_bus_lock_vmexit, 5751 }; 5752 5753 static const int kvm_vmx_max_exit_handlers = 5754 ARRAY_SIZE(kvm_vmx_exit_handlers); 5755 5756 static void vmx_get_exit_info(struct kvm_vcpu *vcpu, u32 *reason, 5757 u64 *info1, u64 *info2, 5758 u32 *intr_info, u32 *error_code) 5759 { 5760 struct vcpu_vmx *vmx = to_vmx(vcpu); 5761 5762 *reason = vmx->exit_reason.full; 5763 *info1 = vmx_get_exit_qual(vcpu); 5764 if (!(vmx->exit_reason.failed_vmentry)) { 5765 *info2 = vmx->idt_vectoring_info; 5766 *intr_info = vmx_get_intr_info(vcpu); 5767 if (is_exception_with_error_code(*intr_info)) 5768 *error_code = vmcs_read32(VM_EXIT_INTR_ERROR_CODE); 5769 else 5770 *error_code = 0; 5771 } else { 5772 *info2 = 0; 5773 *intr_info = 0; 5774 *error_code = 0; 5775 } 5776 } 5777 5778 static void vmx_destroy_pml_buffer(struct vcpu_vmx *vmx) 5779 { 5780 if (vmx->pml_pg) { 5781 __free_page(vmx->pml_pg); 5782 vmx->pml_pg = NULL; 5783 } 5784 } 5785 5786 static void vmx_flush_pml_buffer(struct kvm_vcpu *vcpu) 5787 { 5788 struct vcpu_vmx *vmx = to_vmx(vcpu); 5789 u64 *pml_buf; 5790 u16 pml_idx; 5791 5792 pml_idx = vmcs_read16(GUEST_PML_INDEX); 5793 5794 /* Do nothing if PML buffer is empty */ 5795 if (pml_idx == (PML_ENTITY_NUM - 1)) 5796 return; 5797 5798 /* PML index always points to next available PML buffer entity */ 5799 if (pml_idx >= PML_ENTITY_NUM) 5800 pml_idx = 0; 5801 else 5802 pml_idx++; 5803 5804 pml_buf = page_address(vmx->pml_pg); 5805 for (; pml_idx < PML_ENTITY_NUM; pml_idx++) { 5806 u64 gpa; 5807 5808 gpa = pml_buf[pml_idx]; 5809 WARN_ON(gpa & (PAGE_SIZE - 1)); 5810 kvm_vcpu_mark_page_dirty(vcpu, gpa >> PAGE_SHIFT); 5811 } 5812 5813 /* reset PML index */ 5814 vmcs_write16(GUEST_PML_INDEX, PML_ENTITY_NUM - 1); 5815 } 5816 5817 static void vmx_dump_sel(char *name, uint32_t sel) 5818 { 5819 pr_err("%s sel=0x%04x, attr=0x%05x, limit=0x%08x, base=0x%016lx\n", 5820 name, vmcs_read16(sel), 5821 vmcs_read32(sel + GUEST_ES_AR_BYTES - GUEST_ES_SELECTOR), 5822 vmcs_read32(sel + GUEST_ES_LIMIT - GUEST_ES_SELECTOR), 5823 vmcs_readl(sel + GUEST_ES_BASE - GUEST_ES_SELECTOR)); 5824 } 5825 5826 static void vmx_dump_dtsel(char *name, uint32_t limit) 5827 { 5828 pr_err("%s limit=0x%08x, base=0x%016lx\n", 5829 name, vmcs_read32(limit), 5830 vmcs_readl(limit + GUEST_GDTR_BASE - GUEST_GDTR_LIMIT)); 5831 } 5832 5833 static void vmx_dump_msrs(char *name, struct vmx_msrs *m) 5834 { 5835 unsigned int i; 5836 struct vmx_msr_entry *e; 5837 5838 pr_err("MSR %s:\n", name); 5839 for (i = 0, e = m->val; i < m->nr; ++i, ++e) 5840 pr_err(" %2d: msr=0x%08x value=0x%016llx\n", i, e->index, e->value); 5841 } 5842 5843 void dump_vmcs(struct kvm_vcpu *vcpu) 5844 { 5845 struct vcpu_vmx *vmx = to_vmx(vcpu); 5846 u32 vmentry_ctl, vmexit_ctl; 5847 u32 cpu_based_exec_ctrl, pin_based_exec_ctrl, secondary_exec_control; 5848 unsigned long cr4; 5849 int efer_slot; 5850 5851 if (!dump_invalid_vmcs) { 5852 pr_warn_ratelimited("set kvm_intel.dump_invalid_vmcs=1 to dump internal KVM state.\n"); 5853 return; 5854 } 5855 5856 vmentry_ctl = vmcs_read32(VM_ENTRY_CONTROLS); 5857 vmexit_ctl = vmcs_read32(VM_EXIT_CONTROLS); 5858 cpu_based_exec_ctrl = vmcs_read32(CPU_BASED_VM_EXEC_CONTROL); 5859 pin_based_exec_ctrl = vmcs_read32(PIN_BASED_VM_EXEC_CONTROL); 5860 cr4 = vmcs_readl(GUEST_CR4); 5861 secondary_exec_control = 0; 5862 if (cpu_has_secondary_exec_ctrls()) 5863 secondary_exec_control = vmcs_read32(SECONDARY_VM_EXEC_CONTROL); 5864 5865 pr_err("VMCS %p, last attempted VM-entry on CPU %d\n", 5866 vmx->loaded_vmcs->vmcs, vcpu->arch.last_vmentry_cpu); 5867 pr_err("*** Guest State ***\n"); 5868 pr_err("CR0: actual=0x%016lx, shadow=0x%016lx, gh_mask=%016lx\n", 5869 vmcs_readl(GUEST_CR0), vmcs_readl(CR0_READ_SHADOW), 5870 vmcs_readl(CR0_GUEST_HOST_MASK)); 5871 pr_err("CR4: actual=0x%016lx, shadow=0x%016lx, gh_mask=%016lx\n", 5872 cr4, vmcs_readl(CR4_READ_SHADOW), vmcs_readl(CR4_GUEST_HOST_MASK)); 5873 pr_err("CR3 = 0x%016lx\n", vmcs_readl(GUEST_CR3)); 5874 if (cpu_has_vmx_ept()) { 5875 pr_err("PDPTR0 = 0x%016llx PDPTR1 = 0x%016llx\n", 5876 vmcs_read64(GUEST_PDPTR0), vmcs_read64(GUEST_PDPTR1)); 5877 pr_err("PDPTR2 = 0x%016llx PDPTR3 = 0x%016llx\n", 5878 vmcs_read64(GUEST_PDPTR2), vmcs_read64(GUEST_PDPTR3)); 5879 } 5880 pr_err("RSP = 0x%016lx RIP = 0x%016lx\n", 5881 vmcs_readl(GUEST_RSP), vmcs_readl(GUEST_RIP)); 5882 pr_err("RFLAGS=0x%08lx DR7 = 0x%016lx\n", 5883 vmcs_readl(GUEST_RFLAGS), vmcs_readl(GUEST_DR7)); 5884 pr_err("Sysenter RSP=%016lx CS:RIP=%04x:%016lx\n", 5885 vmcs_readl(GUEST_SYSENTER_ESP), 5886 vmcs_read32(GUEST_SYSENTER_CS), vmcs_readl(GUEST_SYSENTER_EIP)); 5887 vmx_dump_sel("CS: ", GUEST_CS_SELECTOR); 5888 vmx_dump_sel("DS: ", GUEST_DS_SELECTOR); 5889 vmx_dump_sel("SS: ", GUEST_SS_SELECTOR); 5890 vmx_dump_sel("ES: ", GUEST_ES_SELECTOR); 5891 vmx_dump_sel("FS: ", GUEST_FS_SELECTOR); 5892 vmx_dump_sel("GS: ", GUEST_GS_SELECTOR); 5893 vmx_dump_dtsel("GDTR:", GUEST_GDTR_LIMIT); 5894 vmx_dump_sel("LDTR:", GUEST_LDTR_SELECTOR); 5895 vmx_dump_dtsel("IDTR:", GUEST_IDTR_LIMIT); 5896 vmx_dump_sel("TR: ", GUEST_TR_SELECTOR); 5897 efer_slot = vmx_find_loadstore_msr_slot(&vmx->msr_autoload.guest, MSR_EFER); 5898 if (vmentry_ctl & VM_ENTRY_LOAD_IA32_EFER) 5899 pr_err("EFER= 0x%016llx\n", vmcs_read64(GUEST_IA32_EFER)); 5900 else if (efer_slot >= 0) 5901 pr_err("EFER= 0x%016llx (autoload)\n", 5902 vmx->msr_autoload.guest.val[efer_slot].value); 5903 else if (vmentry_ctl & VM_ENTRY_IA32E_MODE) 5904 pr_err("EFER= 0x%016llx (effective)\n", 5905 vcpu->arch.efer | (EFER_LMA | EFER_LME)); 5906 else 5907 pr_err("EFER= 0x%016llx (effective)\n", 5908 vcpu->arch.efer & ~(EFER_LMA | EFER_LME)); 5909 if (vmentry_ctl & VM_ENTRY_LOAD_IA32_PAT) 5910 pr_err("PAT = 0x%016llx\n", vmcs_read64(GUEST_IA32_PAT)); 5911 pr_err("DebugCtl = 0x%016llx DebugExceptions = 0x%016lx\n", 5912 vmcs_read64(GUEST_IA32_DEBUGCTL), 5913 vmcs_readl(GUEST_PENDING_DBG_EXCEPTIONS)); 5914 if (cpu_has_load_perf_global_ctrl() && 5915 vmentry_ctl & VM_ENTRY_LOAD_IA32_PERF_GLOBAL_CTRL) 5916 pr_err("PerfGlobCtl = 0x%016llx\n", 5917 vmcs_read64(GUEST_IA32_PERF_GLOBAL_CTRL)); 5918 if (vmentry_ctl & VM_ENTRY_LOAD_BNDCFGS) 5919 pr_err("BndCfgS = 0x%016llx\n", vmcs_read64(GUEST_BNDCFGS)); 5920 pr_err("Interruptibility = %08x ActivityState = %08x\n", 5921 vmcs_read32(GUEST_INTERRUPTIBILITY_INFO), 5922 vmcs_read32(GUEST_ACTIVITY_STATE)); 5923 if (secondary_exec_control & SECONDARY_EXEC_VIRTUAL_INTR_DELIVERY) 5924 pr_err("InterruptStatus = %04x\n", 5925 vmcs_read16(GUEST_INTR_STATUS)); 5926 if (vmcs_read32(VM_ENTRY_MSR_LOAD_COUNT) > 0) 5927 vmx_dump_msrs("guest autoload", &vmx->msr_autoload.guest); 5928 if (vmcs_read32(VM_EXIT_MSR_STORE_COUNT) > 0) 5929 vmx_dump_msrs("guest autostore", &vmx->msr_autostore.guest); 5930 5931 pr_err("*** Host State ***\n"); 5932 pr_err("RIP = 0x%016lx RSP = 0x%016lx\n", 5933 vmcs_readl(HOST_RIP), vmcs_readl(HOST_RSP)); 5934 pr_err("CS=%04x SS=%04x DS=%04x ES=%04x FS=%04x GS=%04x TR=%04x\n", 5935 vmcs_read16(HOST_CS_SELECTOR), vmcs_read16(HOST_SS_SELECTOR), 5936 vmcs_read16(HOST_DS_SELECTOR), vmcs_read16(HOST_ES_SELECTOR), 5937 vmcs_read16(HOST_FS_SELECTOR), vmcs_read16(HOST_GS_SELECTOR), 5938 vmcs_read16(HOST_TR_SELECTOR)); 5939 pr_err("FSBase=%016lx GSBase=%016lx TRBase=%016lx\n", 5940 vmcs_readl(HOST_FS_BASE), vmcs_readl(HOST_GS_BASE), 5941 vmcs_readl(HOST_TR_BASE)); 5942 pr_err("GDTBase=%016lx IDTBase=%016lx\n", 5943 vmcs_readl(HOST_GDTR_BASE), vmcs_readl(HOST_IDTR_BASE)); 5944 pr_err("CR0=%016lx CR3=%016lx CR4=%016lx\n", 5945 vmcs_readl(HOST_CR0), vmcs_readl(HOST_CR3), 5946 vmcs_readl(HOST_CR4)); 5947 pr_err("Sysenter RSP=%016lx CS:RIP=%04x:%016lx\n", 5948 vmcs_readl(HOST_IA32_SYSENTER_ESP), 5949 vmcs_read32(HOST_IA32_SYSENTER_CS), 5950 vmcs_readl(HOST_IA32_SYSENTER_EIP)); 5951 if (vmexit_ctl & VM_EXIT_LOAD_IA32_EFER) 5952 pr_err("EFER= 0x%016llx\n", vmcs_read64(HOST_IA32_EFER)); 5953 if (vmexit_ctl & VM_EXIT_LOAD_IA32_PAT) 5954 pr_err("PAT = 0x%016llx\n", vmcs_read64(HOST_IA32_PAT)); 5955 if (cpu_has_load_perf_global_ctrl() && 5956 vmexit_ctl & VM_EXIT_LOAD_IA32_PERF_GLOBAL_CTRL) 5957 pr_err("PerfGlobCtl = 0x%016llx\n", 5958 vmcs_read64(HOST_IA32_PERF_GLOBAL_CTRL)); 5959 if (vmcs_read32(VM_EXIT_MSR_LOAD_COUNT) > 0) 5960 vmx_dump_msrs("host autoload", &vmx->msr_autoload.host); 5961 5962 pr_err("*** Control State ***\n"); 5963 pr_err("PinBased=%08x CPUBased=%08x SecondaryExec=%08x\n", 5964 pin_based_exec_ctrl, cpu_based_exec_ctrl, secondary_exec_control); 5965 pr_err("EntryControls=%08x ExitControls=%08x\n", vmentry_ctl, vmexit_ctl); 5966 pr_err("ExceptionBitmap=%08x PFECmask=%08x PFECmatch=%08x\n", 5967 vmcs_read32(EXCEPTION_BITMAP), 5968 vmcs_read32(PAGE_FAULT_ERROR_CODE_MASK), 5969 vmcs_read32(PAGE_FAULT_ERROR_CODE_MATCH)); 5970 pr_err("VMEntry: intr_info=%08x errcode=%08x ilen=%08x\n", 5971 vmcs_read32(VM_ENTRY_INTR_INFO_FIELD), 5972 vmcs_read32(VM_ENTRY_EXCEPTION_ERROR_CODE), 5973 vmcs_read32(VM_ENTRY_INSTRUCTION_LEN)); 5974 pr_err("VMExit: intr_info=%08x errcode=%08x ilen=%08x\n", 5975 vmcs_read32(VM_EXIT_INTR_INFO), 5976 vmcs_read32(VM_EXIT_INTR_ERROR_CODE), 5977 vmcs_read32(VM_EXIT_INSTRUCTION_LEN)); 5978 pr_err(" reason=%08x qualification=%016lx\n", 5979 vmcs_read32(VM_EXIT_REASON), vmcs_readl(EXIT_QUALIFICATION)); 5980 pr_err("IDTVectoring: info=%08x errcode=%08x\n", 5981 vmcs_read32(IDT_VECTORING_INFO_FIELD), 5982 vmcs_read32(IDT_VECTORING_ERROR_CODE)); 5983 pr_err("TSC Offset = 0x%016llx\n", vmcs_read64(TSC_OFFSET)); 5984 if (secondary_exec_control & SECONDARY_EXEC_TSC_SCALING) 5985 pr_err("TSC Multiplier = 0x%016llx\n", 5986 vmcs_read64(TSC_MULTIPLIER)); 5987 if (cpu_based_exec_ctrl & CPU_BASED_TPR_SHADOW) { 5988 if (secondary_exec_control & SECONDARY_EXEC_VIRTUAL_INTR_DELIVERY) { 5989 u16 status = vmcs_read16(GUEST_INTR_STATUS); 5990 pr_err("SVI|RVI = %02x|%02x ", status >> 8, status & 0xff); 5991 } 5992 pr_cont("TPR Threshold = 0x%02x\n", vmcs_read32(TPR_THRESHOLD)); 5993 if (secondary_exec_control & SECONDARY_EXEC_VIRTUALIZE_APIC_ACCESSES) 5994 pr_err("APIC-access addr = 0x%016llx ", vmcs_read64(APIC_ACCESS_ADDR)); 5995 pr_cont("virt-APIC addr = 0x%016llx\n", vmcs_read64(VIRTUAL_APIC_PAGE_ADDR)); 5996 } 5997 if (pin_based_exec_ctrl & PIN_BASED_POSTED_INTR) 5998 pr_err("PostedIntrVec = 0x%02x\n", vmcs_read16(POSTED_INTR_NV)); 5999 if ((secondary_exec_control & SECONDARY_EXEC_ENABLE_EPT)) 6000 pr_err("EPT pointer = 0x%016llx\n", vmcs_read64(EPT_POINTER)); 6001 if (secondary_exec_control & SECONDARY_EXEC_PAUSE_LOOP_EXITING) 6002 pr_err("PLE Gap=%08x Window=%08x\n", 6003 vmcs_read32(PLE_GAP), vmcs_read32(PLE_WINDOW)); 6004 if (secondary_exec_control & SECONDARY_EXEC_ENABLE_VPID) 6005 pr_err("Virtual processor ID = 0x%04x\n", 6006 vmcs_read16(VIRTUAL_PROCESSOR_ID)); 6007 } 6008 6009 /* 6010 * The guest has exited. See if we can fix it or if we need userspace 6011 * assistance. 6012 */ 6013 static int __vmx_handle_exit(struct kvm_vcpu *vcpu, fastpath_t exit_fastpath) 6014 { 6015 struct vcpu_vmx *vmx = to_vmx(vcpu); 6016 union vmx_exit_reason exit_reason = vmx->exit_reason; 6017 u32 vectoring_info = vmx->idt_vectoring_info; 6018 u16 exit_handler_index; 6019 6020 /* 6021 * Flush logged GPAs PML buffer, this will make dirty_bitmap more 6022 * updated. Another good is, in kvm_vm_ioctl_get_dirty_log, before 6023 * querying dirty_bitmap, we only need to kick all vcpus out of guest 6024 * mode as if vcpus is in root mode, the PML buffer must has been 6025 * flushed already. Note, PML is never enabled in hardware while 6026 * running L2. 6027 */ 6028 if (enable_pml && !is_guest_mode(vcpu)) 6029 vmx_flush_pml_buffer(vcpu); 6030 6031 /* 6032 * KVM should never reach this point with a pending nested VM-Enter. 6033 * More specifically, short-circuiting VM-Entry to emulate L2 due to 6034 * invalid guest state should never happen as that means KVM knowingly 6035 * allowed a nested VM-Enter with an invalid vmcs12. More below. 6036 */ 6037 if (KVM_BUG_ON(vmx->nested.nested_run_pending, vcpu->kvm)) 6038 return -EIO; 6039 6040 if (is_guest_mode(vcpu)) { 6041 /* 6042 * PML is never enabled when running L2, bail immediately if a 6043 * PML full exit occurs as something is horribly wrong. 6044 */ 6045 if (exit_reason.basic == EXIT_REASON_PML_FULL) 6046 goto unexpected_vmexit; 6047 6048 /* 6049 * The host physical addresses of some pages of guest memory 6050 * are loaded into the vmcs02 (e.g. vmcs12's Virtual APIC 6051 * Page). The CPU may write to these pages via their host 6052 * physical address while L2 is running, bypassing any 6053 * address-translation-based dirty tracking (e.g. EPT write 6054 * protection). 6055 * 6056 * Mark them dirty on every exit from L2 to prevent them from 6057 * getting out of sync with dirty tracking. 6058 */ 6059 nested_mark_vmcs12_pages_dirty(vcpu); 6060 6061 /* 6062 * Synthesize a triple fault if L2 state is invalid. In normal 6063 * operation, nested VM-Enter rejects any attempt to enter L2 6064 * with invalid state. However, those checks are skipped if 6065 * state is being stuffed via RSM or KVM_SET_NESTED_STATE. If 6066 * L2 state is invalid, it means either L1 modified SMRAM state 6067 * or userspace provided bad state. Synthesize TRIPLE_FAULT as 6068 * doing so is architecturally allowed in the RSM case, and is 6069 * the least awful solution for the userspace case without 6070 * risking false positives. 6071 */ 6072 if (vmx->emulation_required) { 6073 nested_vmx_vmexit(vcpu, EXIT_REASON_TRIPLE_FAULT, 0, 0); 6074 return 1; 6075 } 6076 6077 if (nested_vmx_reflect_vmexit(vcpu)) 6078 return 1; 6079 } 6080 6081 /* If guest state is invalid, start emulating. L2 is handled above. */ 6082 if (vmx->emulation_required) 6083 return handle_invalid_guest_state(vcpu); 6084 6085 if (exit_reason.failed_vmentry) { 6086 dump_vmcs(vcpu); 6087 vcpu->run->exit_reason = KVM_EXIT_FAIL_ENTRY; 6088 vcpu->run->fail_entry.hardware_entry_failure_reason 6089 = exit_reason.full; 6090 vcpu->run->fail_entry.cpu = vcpu->arch.last_vmentry_cpu; 6091 return 0; 6092 } 6093 6094 if (unlikely(vmx->fail)) { 6095 dump_vmcs(vcpu); 6096 vcpu->run->exit_reason = KVM_EXIT_FAIL_ENTRY; 6097 vcpu->run->fail_entry.hardware_entry_failure_reason 6098 = vmcs_read32(VM_INSTRUCTION_ERROR); 6099 vcpu->run->fail_entry.cpu = vcpu->arch.last_vmentry_cpu; 6100 return 0; 6101 } 6102 6103 /* 6104 * Note: 6105 * Do not try to fix EXIT_REASON_EPT_MISCONFIG if it caused by 6106 * delivery event since it indicates guest is accessing MMIO. 6107 * The vm-exit can be triggered again after return to guest that 6108 * will cause infinite loop. 6109 */ 6110 if ((vectoring_info & VECTORING_INFO_VALID_MASK) && 6111 (exit_reason.basic != EXIT_REASON_EXCEPTION_NMI && 6112 exit_reason.basic != EXIT_REASON_EPT_VIOLATION && 6113 exit_reason.basic != EXIT_REASON_PML_FULL && 6114 exit_reason.basic != EXIT_REASON_APIC_ACCESS && 6115 exit_reason.basic != EXIT_REASON_TASK_SWITCH)) { 6116 int ndata = 3; 6117 6118 vcpu->run->exit_reason = KVM_EXIT_INTERNAL_ERROR; 6119 vcpu->run->internal.suberror = KVM_INTERNAL_ERROR_DELIVERY_EV; 6120 vcpu->run->internal.data[0] = vectoring_info; 6121 vcpu->run->internal.data[1] = exit_reason.full; 6122 vcpu->run->internal.data[2] = vcpu->arch.exit_qualification; 6123 if (exit_reason.basic == EXIT_REASON_EPT_MISCONFIG) { 6124 vcpu->run->internal.data[ndata++] = 6125 vmcs_read64(GUEST_PHYSICAL_ADDRESS); 6126 } 6127 vcpu->run->internal.data[ndata++] = vcpu->arch.last_vmentry_cpu; 6128 vcpu->run->internal.ndata = ndata; 6129 return 0; 6130 } 6131 6132 if (unlikely(!enable_vnmi && 6133 vmx->loaded_vmcs->soft_vnmi_blocked)) { 6134 if (!vmx_interrupt_blocked(vcpu)) { 6135 vmx->loaded_vmcs->soft_vnmi_blocked = 0; 6136 } else if (vmx->loaded_vmcs->vnmi_blocked_time > 1000000000LL && 6137 vcpu->arch.nmi_pending) { 6138 /* 6139 * This CPU don't support us in finding the end of an 6140 * NMI-blocked window if the guest runs with IRQs 6141 * disabled. So we pull the trigger after 1 s of 6142 * futile waiting, but inform the user about this. 6143 */ 6144 printk(KERN_WARNING "%s: Breaking out of NMI-blocked " 6145 "state on VCPU %d after 1 s timeout\n", 6146 __func__, vcpu->vcpu_id); 6147 vmx->loaded_vmcs->soft_vnmi_blocked = 0; 6148 } 6149 } 6150 6151 if (exit_fastpath != EXIT_FASTPATH_NONE) 6152 return 1; 6153 6154 if (exit_reason.basic >= kvm_vmx_max_exit_handlers) 6155 goto unexpected_vmexit; 6156 #ifdef CONFIG_RETPOLINE 6157 if (exit_reason.basic == EXIT_REASON_MSR_WRITE) 6158 return kvm_emulate_wrmsr(vcpu); 6159 else if (exit_reason.basic == EXIT_REASON_PREEMPTION_TIMER) 6160 return handle_preemption_timer(vcpu); 6161 else if (exit_reason.basic == EXIT_REASON_INTERRUPT_WINDOW) 6162 return handle_interrupt_window(vcpu); 6163 else if (exit_reason.basic == EXIT_REASON_EXTERNAL_INTERRUPT) 6164 return handle_external_interrupt(vcpu); 6165 else if (exit_reason.basic == EXIT_REASON_HLT) 6166 return kvm_emulate_halt(vcpu); 6167 else if (exit_reason.basic == EXIT_REASON_EPT_MISCONFIG) 6168 return handle_ept_misconfig(vcpu); 6169 #endif 6170 6171 exit_handler_index = array_index_nospec((u16)exit_reason.basic, 6172 kvm_vmx_max_exit_handlers); 6173 if (!kvm_vmx_exit_handlers[exit_handler_index]) 6174 goto unexpected_vmexit; 6175 6176 return kvm_vmx_exit_handlers[exit_handler_index](vcpu); 6177 6178 unexpected_vmexit: 6179 vcpu_unimpl(vcpu, "vmx: unexpected exit reason 0x%x\n", 6180 exit_reason.full); 6181 dump_vmcs(vcpu); 6182 vcpu->run->exit_reason = KVM_EXIT_INTERNAL_ERROR; 6183 vcpu->run->internal.suberror = 6184 KVM_INTERNAL_ERROR_UNEXPECTED_EXIT_REASON; 6185 vcpu->run->internal.ndata = 2; 6186 vcpu->run->internal.data[0] = exit_reason.full; 6187 vcpu->run->internal.data[1] = vcpu->arch.last_vmentry_cpu; 6188 return 0; 6189 } 6190 6191 static int vmx_handle_exit(struct kvm_vcpu *vcpu, fastpath_t exit_fastpath) 6192 { 6193 int ret = __vmx_handle_exit(vcpu, exit_fastpath); 6194 6195 /* 6196 * Exit to user space when bus lock detected to inform that there is 6197 * a bus lock in guest. 6198 */ 6199 if (to_vmx(vcpu)->exit_reason.bus_lock_detected) { 6200 if (ret > 0) 6201 vcpu->run->exit_reason = KVM_EXIT_X86_BUS_LOCK; 6202 6203 vcpu->run->flags |= KVM_RUN_X86_BUS_LOCK; 6204 return 0; 6205 } 6206 return ret; 6207 } 6208 6209 /* 6210 * Software based L1D cache flush which is used when microcode providing 6211 * the cache control MSR is not loaded. 6212 * 6213 * The L1D cache is 32 KiB on Nehalem and later microarchitectures, but to 6214 * flush it is required to read in 64 KiB because the replacement algorithm 6215 * is not exactly LRU. This could be sized at runtime via topology 6216 * information but as all relevant affected CPUs have 32KiB L1D cache size 6217 * there is no point in doing so. 6218 */ 6219 static noinstr void vmx_l1d_flush(struct kvm_vcpu *vcpu) 6220 { 6221 int size = PAGE_SIZE << L1D_CACHE_ORDER; 6222 6223 /* 6224 * This code is only executed when the the flush mode is 'cond' or 6225 * 'always' 6226 */ 6227 if (static_branch_likely(&vmx_l1d_flush_cond)) { 6228 bool flush_l1d; 6229 6230 /* 6231 * Clear the per-vcpu flush bit, it gets set again 6232 * either from vcpu_run() or from one of the unsafe 6233 * VMEXIT handlers. 6234 */ 6235 flush_l1d = vcpu->arch.l1tf_flush_l1d; 6236 vcpu->arch.l1tf_flush_l1d = false; 6237 6238 /* 6239 * Clear the per-cpu flush bit, it gets set again from 6240 * the interrupt handlers. 6241 */ 6242 flush_l1d |= kvm_get_cpu_l1tf_flush_l1d(); 6243 kvm_clear_cpu_l1tf_flush_l1d(); 6244 6245 if (!flush_l1d) 6246 return; 6247 } 6248 6249 vcpu->stat.l1d_flush++; 6250 6251 if (static_cpu_has(X86_FEATURE_FLUSH_L1D)) { 6252 native_wrmsrl(MSR_IA32_FLUSH_CMD, L1D_FLUSH); 6253 return; 6254 } 6255 6256 asm volatile( 6257 /* First ensure the pages are in the TLB */ 6258 "xorl %%eax, %%eax\n" 6259 ".Lpopulate_tlb:\n\t" 6260 "movzbl (%[flush_pages], %%" _ASM_AX "), %%ecx\n\t" 6261 "addl $4096, %%eax\n\t" 6262 "cmpl %%eax, %[size]\n\t" 6263 "jne .Lpopulate_tlb\n\t" 6264 "xorl %%eax, %%eax\n\t" 6265 "cpuid\n\t" 6266 /* Now fill the cache */ 6267 "xorl %%eax, %%eax\n" 6268 ".Lfill_cache:\n" 6269 "movzbl (%[flush_pages], %%" _ASM_AX "), %%ecx\n\t" 6270 "addl $64, %%eax\n\t" 6271 "cmpl %%eax, %[size]\n\t" 6272 "jne .Lfill_cache\n\t" 6273 "lfence\n" 6274 :: [flush_pages] "r" (vmx_l1d_flush_pages), 6275 [size] "r" (size) 6276 : "eax", "ebx", "ecx", "edx"); 6277 } 6278 6279 static void vmx_update_cr8_intercept(struct kvm_vcpu *vcpu, int tpr, int irr) 6280 { 6281 struct vmcs12 *vmcs12 = get_vmcs12(vcpu); 6282 int tpr_threshold; 6283 6284 if (is_guest_mode(vcpu) && 6285 nested_cpu_has(vmcs12, CPU_BASED_TPR_SHADOW)) 6286 return; 6287 6288 tpr_threshold = (irr == -1 || tpr < irr) ? 0 : irr; 6289 if (is_guest_mode(vcpu)) 6290 to_vmx(vcpu)->nested.l1_tpr_threshold = tpr_threshold; 6291 else 6292 vmcs_write32(TPR_THRESHOLD, tpr_threshold); 6293 } 6294 6295 void vmx_set_virtual_apic_mode(struct kvm_vcpu *vcpu) 6296 { 6297 struct vcpu_vmx *vmx = to_vmx(vcpu); 6298 u32 sec_exec_control; 6299 6300 if (!lapic_in_kernel(vcpu)) 6301 return; 6302 6303 if (!flexpriority_enabled && 6304 !cpu_has_vmx_virtualize_x2apic_mode()) 6305 return; 6306 6307 /* Postpone execution until vmcs01 is the current VMCS. */ 6308 if (is_guest_mode(vcpu)) { 6309 vmx->nested.change_vmcs01_virtual_apic_mode = true; 6310 return; 6311 } 6312 6313 sec_exec_control = secondary_exec_controls_get(vmx); 6314 sec_exec_control &= ~(SECONDARY_EXEC_VIRTUALIZE_APIC_ACCESSES | 6315 SECONDARY_EXEC_VIRTUALIZE_X2APIC_MODE); 6316 6317 switch (kvm_get_apic_mode(vcpu)) { 6318 case LAPIC_MODE_INVALID: 6319 WARN_ONCE(true, "Invalid local APIC state"); 6320 break; 6321 case LAPIC_MODE_DISABLED: 6322 break; 6323 case LAPIC_MODE_XAPIC: 6324 if (flexpriority_enabled) { 6325 sec_exec_control |= 6326 SECONDARY_EXEC_VIRTUALIZE_APIC_ACCESSES; 6327 kvm_make_request(KVM_REQ_APIC_PAGE_RELOAD, vcpu); 6328 6329 /* 6330 * Flush the TLB, reloading the APIC access page will 6331 * only do so if its physical address has changed, but 6332 * the guest may have inserted a non-APIC mapping into 6333 * the TLB while the APIC access page was disabled. 6334 */ 6335 kvm_make_request(KVM_REQ_TLB_FLUSH_CURRENT, vcpu); 6336 } 6337 break; 6338 case LAPIC_MODE_X2APIC: 6339 if (cpu_has_vmx_virtualize_x2apic_mode()) 6340 sec_exec_control |= 6341 SECONDARY_EXEC_VIRTUALIZE_X2APIC_MODE; 6342 break; 6343 } 6344 secondary_exec_controls_set(vmx, sec_exec_control); 6345 6346 vmx_update_msr_bitmap_x2apic(vcpu); 6347 } 6348 6349 static void vmx_set_apic_access_page_addr(struct kvm_vcpu *vcpu) 6350 { 6351 struct page *page; 6352 6353 /* Defer reload until vmcs01 is the current VMCS. */ 6354 if (is_guest_mode(vcpu)) { 6355 to_vmx(vcpu)->nested.reload_vmcs01_apic_access_page = true; 6356 return; 6357 } 6358 6359 if (!(secondary_exec_controls_get(to_vmx(vcpu)) & 6360 SECONDARY_EXEC_VIRTUALIZE_APIC_ACCESSES)) 6361 return; 6362 6363 page = gfn_to_page(vcpu->kvm, APIC_DEFAULT_PHYS_BASE >> PAGE_SHIFT); 6364 if (is_error_page(page)) 6365 return; 6366 6367 vmcs_write64(APIC_ACCESS_ADDR, page_to_phys(page)); 6368 vmx_flush_tlb_current(vcpu); 6369 6370 /* 6371 * Do not pin apic access page in memory, the MMU notifier 6372 * will call us again if it is migrated or swapped out. 6373 */ 6374 put_page(page); 6375 } 6376 6377 static void vmx_hwapic_isr_update(struct kvm_vcpu *vcpu, int max_isr) 6378 { 6379 u16 status; 6380 u8 old; 6381 6382 if (max_isr == -1) 6383 max_isr = 0; 6384 6385 status = vmcs_read16(GUEST_INTR_STATUS); 6386 old = status >> 8; 6387 if (max_isr != old) { 6388 status &= 0xff; 6389 status |= max_isr << 8; 6390 vmcs_write16(GUEST_INTR_STATUS, status); 6391 } 6392 } 6393 6394 static void vmx_set_rvi(int vector) 6395 { 6396 u16 status; 6397 u8 old; 6398 6399 if (vector == -1) 6400 vector = 0; 6401 6402 status = vmcs_read16(GUEST_INTR_STATUS); 6403 old = (u8)status & 0xff; 6404 if ((u8)vector != old) { 6405 status &= ~0xff; 6406 status |= (u8)vector; 6407 vmcs_write16(GUEST_INTR_STATUS, status); 6408 } 6409 } 6410 6411 static void vmx_hwapic_irr_update(struct kvm_vcpu *vcpu, int max_irr) 6412 { 6413 /* 6414 * When running L2, updating RVI is only relevant when 6415 * vmcs12 virtual-interrupt-delivery enabled. 6416 * However, it can be enabled only when L1 also 6417 * intercepts external-interrupts and in that case 6418 * we should not update vmcs02 RVI but instead intercept 6419 * interrupt. Therefore, do nothing when running L2. 6420 */ 6421 if (!is_guest_mode(vcpu)) 6422 vmx_set_rvi(max_irr); 6423 } 6424 6425 static int vmx_sync_pir_to_irr(struct kvm_vcpu *vcpu) 6426 { 6427 struct vcpu_vmx *vmx = to_vmx(vcpu); 6428 int max_irr; 6429 bool got_posted_interrupt; 6430 6431 if (KVM_BUG_ON(!enable_apicv, vcpu->kvm)) 6432 return -EIO; 6433 6434 if (pi_test_on(&vmx->pi_desc)) { 6435 pi_clear_on(&vmx->pi_desc); 6436 /* 6437 * IOMMU can write to PID.ON, so the barrier matters even on UP. 6438 * But on x86 this is just a compiler barrier anyway. 6439 */ 6440 smp_mb__after_atomic(); 6441 got_posted_interrupt = 6442 kvm_apic_update_irr(vcpu, vmx->pi_desc.pir, &max_irr); 6443 } else { 6444 max_irr = kvm_lapic_find_highest_irr(vcpu); 6445 got_posted_interrupt = false; 6446 } 6447 6448 /* 6449 * Newly recognized interrupts are injected via either virtual interrupt 6450 * delivery (RVI) or KVM_REQ_EVENT. Virtual interrupt delivery is 6451 * disabled in two cases: 6452 * 6453 * 1) If L2 is running and the vCPU has a new pending interrupt. If L1 6454 * wants to exit on interrupts, KVM_REQ_EVENT is needed to synthesize a 6455 * VM-Exit to L1. If L1 doesn't want to exit, the interrupt is injected 6456 * into L2, but KVM doesn't use virtual interrupt delivery to inject 6457 * interrupts into L2, and so KVM_REQ_EVENT is again needed. 6458 * 6459 * 2) If APICv is disabled for this vCPU, assigned devices may still 6460 * attempt to post interrupts. The posted interrupt vector will cause 6461 * a VM-Exit and the subsequent entry will call sync_pir_to_irr. 6462 */ 6463 if (!is_guest_mode(vcpu) && kvm_vcpu_apicv_active(vcpu)) 6464 vmx_set_rvi(max_irr); 6465 else if (got_posted_interrupt) 6466 kvm_make_request(KVM_REQ_EVENT, vcpu); 6467 6468 return max_irr; 6469 } 6470 6471 static void vmx_load_eoi_exitmap(struct kvm_vcpu *vcpu, u64 *eoi_exit_bitmap) 6472 { 6473 if (!kvm_vcpu_apicv_active(vcpu)) 6474 return; 6475 6476 vmcs_write64(EOI_EXIT_BITMAP0, eoi_exit_bitmap[0]); 6477 vmcs_write64(EOI_EXIT_BITMAP1, eoi_exit_bitmap[1]); 6478 vmcs_write64(EOI_EXIT_BITMAP2, eoi_exit_bitmap[2]); 6479 vmcs_write64(EOI_EXIT_BITMAP3, eoi_exit_bitmap[3]); 6480 } 6481 6482 static void vmx_apicv_post_state_restore(struct kvm_vcpu *vcpu) 6483 { 6484 struct vcpu_vmx *vmx = to_vmx(vcpu); 6485 6486 pi_clear_on(&vmx->pi_desc); 6487 memset(vmx->pi_desc.pir, 0, sizeof(vmx->pi_desc.pir)); 6488 } 6489 6490 void vmx_do_interrupt_nmi_irqoff(unsigned long entry); 6491 6492 static void handle_interrupt_nmi_irqoff(struct kvm_vcpu *vcpu, 6493 unsigned long entry) 6494 { 6495 bool is_nmi = entry == (unsigned long)asm_exc_nmi_noist; 6496 6497 kvm_before_interrupt(vcpu, is_nmi ? KVM_HANDLING_NMI : KVM_HANDLING_IRQ); 6498 vmx_do_interrupt_nmi_irqoff(entry); 6499 kvm_after_interrupt(vcpu); 6500 } 6501 6502 static void handle_nm_fault_irqoff(struct kvm_vcpu *vcpu) 6503 { 6504 /* 6505 * Save xfd_err to guest_fpu before interrupt is enabled, so the 6506 * MSR value is not clobbered by the host activity before the guest 6507 * has chance to consume it. 6508 * 6509 * Do not blindly read xfd_err here, since this exception might 6510 * be caused by L1 interception on a platform which doesn't 6511 * support xfd at all. 6512 * 6513 * Do it conditionally upon guest_fpu::xfd. xfd_err matters 6514 * only when xfd contains a non-zero value. 6515 * 6516 * Queuing exception is done in vmx_handle_exit. See comment there. 6517 */ 6518 if (vcpu->arch.guest_fpu.fpstate->xfd) 6519 rdmsrl(MSR_IA32_XFD_ERR, vcpu->arch.guest_fpu.xfd_err); 6520 } 6521 6522 static void handle_exception_nmi_irqoff(struct vcpu_vmx *vmx) 6523 { 6524 const unsigned long nmi_entry = (unsigned long)asm_exc_nmi_noist; 6525 u32 intr_info = vmx_get_intr_info(&vmx->vcpu); 6526 6527 /* if exit due to PF check for async PF */ 6528 if (is_page_fault(intr_info)) 6529 vmx->vcpu.arch.apf.host_apf_flags = kvm_read_and_reset_apf_flags(); 6530 /* if exit due to NM, handle before interrupts are enabled */ 6531 else if (is_nm_fault(intr_info)) 6532 handle_nm_fault_irqoff(&vmx->vcpu); 6533 /* Handle machine checks before interrupts are enabled */ 6534 else if (is_machine_check(intr_info)) 6535 kvm_machine_check(); 6536 /* We need to handle NMIs before interrupts are enabled */ 6537 else if (is_nmi(intr_info)) 6538 handle_interrupt_nmi_irqoff(&vmx->vcpu, nmi_entry); 6539 } 6540 6541 static void handle_external_interrupt_irqoff(struct kvm_vcpu *vcpu) 6542 { 6543 u32 intr_info = vmx_get_intr_info(vcpu); 6544 unsigned int vector = intr_info & INTR_INFO_VECTOR_MASK; 6545 gate_desc *desc = (gate_desc *)host_idt_base + vector; 6546 6547 if (KVM_BUG(!is_external_intr(intr_info), vcpu->kvm, 6548 "KVM: unexpected VM-Exit interrupt info: 0x%x", intr_info)) 6549 return; 6550 6551 handle_interrupt_nmi_irqoff(vcpu, gate_offset(desc)); 6552 } 6553 6554 static void vmx_handle_exit_irqoff(struct kvm_vcpu *vcpu) 6555 { 6556 struct vcpu_vmx *vmx = to_vmx(vcpu); 6557 6558 if (vmx->emulation_required) 6559 return; 6560 6561 if (vmx->exit_reason.basic == EXIT_REASON_EXTERNAL_INTERRUPT) 6562 handle_external_interrupt_irqoff(vcpu); 6563 else if (vmx->exit_reason.basic == EXIT_REASON_EXCEPTION_NMI) 6564 handle_exception_nmi_irqoff(vmx); 6565 } 6566 6567 /* 6568 * The kvm parameter can be NULL (module initialization, or invocation before 6569 * VM creation). Be sure to check the kvm parameter before using it. 6570 */ 6571 static bool vmx_has_emulated_msr(struct kvm *kvm, u32 index) 6572 { 6573 switch (index) { 6574 case MSR_IA32_SMBASE: 6575 /* 6576 * We cannot do SMM unless we can run the guest in big 6577 * real mode. 6578 */ 6579 return enable_unrestricted_guest || emulate_invalid_guest_state; 6580 case MSR_IA32_VMX_BASIC ... MSR_IA32_VMX_VMFUNC: 6581 return nested; 6582 case MSR_AMD64_VIRT_SPEC_CTRL: 6583 case MSR_AMD64_TSC_RATIO: 6584 /* This is AMD only. */ 6585 return false; 6586 default: 6587 return true; 6588 } 6589 } 6590 6591 static void vmx_recover_nmi_blocking(struct vcpu_vmx *vmx) 6592 { 6593 u32 exit_intr_info; 6594 bool unblock_nmi; 6595 u8 vector; 6596 bool idtv_info_valid; 6597 6598 idtv_info_valid = vmx->idt_vectoring_info & VECTORING_INFO_VALID_MASK; 6599 6600 if (enable_vnmi) { 6601 if (vmx->loaded_vmcs->nmi_known_unmasked) 6602 return; 6603 6604 exit_intr_info = vmx_get_intr_info(&vmx->vcpu); 6605 unblock_nmi = (exit_intr_info & INTR_INFO_UNBLOCK_NMI) != 0; 6606 vector = exit_intr_info & INTR_INFO_VECTOR_MASK; 6607 /* 6608 * SDM 3: 27.7.1.2 (September 2008) 6609 * Re-set bit "block by NMI" before VM entry if vmexit caused by 6610 * a guest IRET fault. 6611 * SDM 3: 23.2.2 (September 2008) 6612 * Bit 12 is undefined in any of the following cases: 6613 * If the VM exit sets the valid bit in the IDT-vectoring 6614 * information field. 6615 * If the VM exit is due to a double fault. 6616 */ 6617 if ((exit_intr_info & INTR_INFO_VALID_MASK) && unblock_nmi && 6618 vector != DF_VECTOR && !idtv_info_valid) 6619 vmcs_set_bits(GUEST_INTERRUPTIBILITY_INFO, 6620 GUEST_INTR_STATE_NMI); 6621 else 6622 vmx->loaded_vmcs->nmi_known_unmasked = 6623 !(vmcs_read32(GUEST_INTERRUPTIBILITY_INFO) 6624 & GUEST_INTR_STATE_NMI); 6625 } else if (unlikely(vmx->loaded_vmcs->soft_vnmi_blocked)) 6626 vmx->loaded_vmcs->vnmi_blocked_time += 6627 ktime_to_ns(ktime_sub(ktime_get(), 6628 vmx->loaded_vmcs->entry_time)); 6629 } 6630 6631 static void __vmx_complete_interrupts(struct kvm_vcpu *vcpu, 6632 u32 idt_vectoring_info, 6633 int instr_len_field, 6634 int error_code_field) 6635 { 6636 u8 vector; 6637 int type; 6638 bool idtv_info_valid; 6639 6640 idtv_info_valid = idt_vectoring_info & VECTORING_INFO_VALID_MASK; 6641 6642 vcpu->arch.nmi_injected = false; 6643 kvm_clear_exception_queue(vcpu); 6644 kvm_clear_interrupt_queue(vcpu); 6645 6646 if (!idtv_info_valid) 6647 return; 6648 6649 kvm_make_request(KVM_REQ_EVENT, vcpu); 6650 6651 vector = idt_vectoring_info & VECTORING_INFO_VECTOR_MASK; 6652 type = idt_vectoring_info & VECTORING_INFO_TYPE_MASK; 6653 6654 switch (type) { 6655 case INTR_TYPE_NMI_INTR: 6656 vcpu->arch.nmi_injected = true; 6657 /* 6658 * SDM 3: 27.7.1.2 (September 2008) 6659 * Clear bit "block by NMI" before VM entry if a NMI 6660 * delivery faulted. 6661 */ 6662 vmx_set_nmi_mask(vcpu, false); 6663 break; 6664 case INTR_TYPE_SOFT_EXCEPTION: 6665 vcpu->arch.event_exit_inst_len = vmcs_read32(instr_len_field); 6666 fallthrough; 6667 case INTR_TYPE_HARD_EXCEPTION: 6668 if (idt_vectoring_info & VECTORING_INFO_DELIVER_CODE_MASK) { 6669 u32 err = vmcs_read32(error_code_field); 6670 kvm_requeue_exception_e(vcpu, vector, err); 6671 } else 6672 kvm_requeue_exception(vcpu, vector); 6673 break; 6674 case INTR_TYPE_SOFT_INTR: 6675 vcpu->arch.event_exit_inst_len = vmcs_read32(instr_len_field); 6676 fallthrough; 6677 case INTR_TYPE_EXT_INTR: 6678 kvm_queue_interrupt(vcpu, vector, type == INTR_TYPE_SOFT_INTR); 6679 break; 6680 default: 6681 break; 6682 } 6683 } 6684 6685 static void vmx_complete_interrupts(struct vcpu_vmx *vmx) 6686 { 6687 __vmx_complete_interrupts(&vmx->vcpu, vmx->idt_vectoring_info, 6688 VM_EXIT_INSTRUCTION_LEN, 6689 IDT_VECTORING_ERROR_CODE); 6690 } 6691 6692 static void vmx_cancel_injection(struct kvm_vcpu *vcpu) 6693 { 6694 __vmx_complete_interrupts(vcpu, 6695 vmcs_read32(VM_ENTRY_INTR_INFO_FIELD), 6696 VM_ENTRY_INSTRUCTION_LEN, 6697 VM_ENTRY_EXCEPTION_ERROR_CODE); 6698 6699 vmcs_write32(VM_ENTRY_INTR_INFO_FIELD, 0); 6700 } 6701 6702 static void atomic_switch_perf_msrs(struct vcpu_vmx *vmx) 6703 { 6704 int i, nr_msrs; 6705 struct perf_guest_switch_msr *msrs; 6706 6707 /* Note, nr_msrs may be garbage if perf_guest_get_msrs() returns NULL. */ 6708 msrs = perf_guest_get_msrs(&nr_msrs); 6709 if (!msrs) 6710 return; 6711 6712 for (i = 0; i < nr_msrs; i++) 6713 if (msrs[i].host == msrs[i].guest) 6714 clear_atomic_switch_msr(vmx, msrs[i].msr); 6715 else 6716 add_atomic_switch_msr(vmx, msrs[i].msr, msrs[i].guest, 6717 msrs[i].host, false); 6718 } 6719 6720 static void vmx_update_hv_timer(struct kvm_vcpu *vcpu) 6721 { 6722 struct vcpu_vmx *vmx = to_vmx(vcpu); 6723 u64 tscl; 6724 u32 delta_tsc; 6725 6726 if (vmx->req_immediate_exit) { 6727 vmcs_write32(VMX_PREEMPTION_TIMER_VALUE, 0); 6728 vmx->loaded_vmcs->hv_timer_soft_disabled = false; 6729 } else if (vmx->hv_deadline_tsc != -1) { 6730 tscl = rdtsc(); 6731 if (vmx->hv_deadline_tsc > tscl) 6732 /* set_hv_timer ensures the delta fits in 32-bits */ 6733 delta_tsc = (u32)((vmx->hv_deadline_tsc - tscl) >> 6734 cpu_preemption_timer_multi); 6735 else 6736 delta_tsc = 0; 6737 6738 vmcs_write32(VMX_PREEMPTION_TIMER_VALUE, delta_tsc); 6739 vmx->loaded_vmcs->hv_timer_soft_disabled = false; 6740 } else if (!vmx->loaded_vmcs->hv_timer_soft_disabled) { 6741 vmcs_write32(VMX_PREEMPTION_TIMER_VALUE, -1); 6742 vmx->loaded_vmcs->hv_timer_soft_disabled = true; 6743 } 6744 } 6745 6746 void noinstr vmx_update_host_rsp(struct vcpu_vmx *vmx, unsigned long host_rsp) 6747 { 6748 if (unlikely(host_rsp != vmx->loaded_vmcs->host_state.rsp)) { 6749 vmx->loaded_vmcs->host_state.rsp = host_rsp; 6750 vmcs_writel(HOST_RSP, host_rsp); 6751 } 6752 } 6753 6754 static fastpath_t vmx_exit_handlers_fastpath(struct kvm_vcpu *vcpu) 6755 { 6756 switch (to_vmx(vcpu)->exit_reason.basic) { 6757 case EXIT_REASON_MSR_WRITE: 6758 return handle_fastpath_set_msr_irqoff(vcpu); 6759 case EXIT_REASON_PREEMPTION_TIMER: 6760 return handle_fastpath_preemption_timer(vcpu); 6761 default: 6762 return EXIT_FASTPATH_NONE; 6763 } 6764 } 6765 6766 static noinstr void vmx_vcpu_enter_exit(struct kvm_vcpu *vcpu, 6767 struct vcpu_vmx *vmx) 6768 { 6769 guest_state_enter_irqoff(); 6770 6771 /* L1D Flush includes CPU buffer clear to mitigate MDS */ 6772 if (static_branch_unlikely(&vmx_l1d_should_flush)) 6773 vmx_l1d_flush(vcpu); 6774 else if (static_branch_unlikely(&mds_user_clear)) 6775 mds_clear_cpu_buffers(); 6776 6777 if (vcpu->arch.cr2 != native_read_cr2()) 6778 native_write_cr2(vcpu->arch.cr2); 6779 6780 vmx->fail = __vmx_vcpu_run(vmx, (unsigned long *)&vcpu->arch.regs, 6781 vmx->loaded_vmcs->launched); 6782 6783 vcpu->arch.cr2 = native_read_cr2(); 6784 6785 guest_state_exit_irqoff(); 6786 } 6787 6788 static fastpath_t vmx_vcpu_run(struct kvm_vcpu *vcpu) 6789 { 6790 struct vcpu_vmx *vmx = to_vmx(vcpu); 6791 unsigned long cr3, cr4; 6792 6793 /* Record the guest's net vcpu time for enforced NMI injections. */ 6794 if (unlikely(!enable_vnmi && 6795 vmx->loaded_vmcs->soft_vnmi_blocked)) 6796 vmx->loaded_vmcs->entry_time = ktime_get(); 6797 6798 /* 6799 * Don't enter VMX if guest state is invalid, let the exit handler 6800 * start emulation until we arrive back to a valid state. Synthesize a 6801 * consistency check VM-Exit due to invalid guest state and bail. 6802 */ 6803 if (unlikely(vmx->emulation_required)) { 6804 vmx->fail = 0; 6805 6806 vmx->exit_reason.full = EXIT_REASON_INVALID_STATE; 6807 vmx->exit_reason.failed_vmentry = 1; 6808 kvm_register_mark_available(vcpu, VCPU_EXREG_EXIT_INFO_1); 6809 vmx->exit_qualification = ENTRY_FAIL_DEFAULT; 6810 kvm_register_mark_available(vcpu, VCPU_EXREG_EXIT_INFO_2); 6811 vmx->exit_intr_info = 0; 6812 return EXIT_FASTPATH_NONE; 6813 } 6814 6815 trace_kvm_entry(vcpu); 6816 6817 if (vmx->ple_window_dirty) { 6818 vmx->ple_window_dirty = false; 6819 vmcs_write32(PLE_WINDOW, vmx->ple_window); 6820 } 6821 6822 /* 6823 * We did this in prepare_switch_to_guest, because it needs to 6824 * be within srcu_read_lock. 6825 */ 6826 WARN_ON_ONCE(vmx->nested.need_vmcs12_to_shadow_sync); 6827 6828 if (kvm_register_is_dirty(vcpu, VCPU_REGS_RSP)) 6829 vmcs_writel(GUEST_RSP, vcpu->arch.regs[VCPU_REGS_RSP]); 6830 if (kvm_register_is_dirty(vcpu, VCPU_REGS_RIP)) 6831 vmcs_writel(GUEST_RIP, vcpu->arch.regs[VCPU_REGS_RIP]); 6832 vcpu->arch.regs_dirty = 0; 6833 6834 /* 6835 * Refresh vmcs.HOST_CR3 if necessary. This must be done immediately 6836 * prior to VM-Enter, as the kernel may load a new ASID (PCID) any time 6837 * it switches back to the current->mm, which can occur in KVM context 6838 * when switching to a temporary mm to patch kernel code, e.g. if KVM 6839 * toggles a static key while handling a VM-Exit. 6840 */ 6841 cr3 = __get_current_cr3_fast(); 6842 if (unlikely(cr3 != vmx->loaded_vmcs->host_state.cr3)) { 6843 vmcs_writel(HOST_CR3, cr3); 6844 vmx->loaded_vmcs->host_state.cr3 = cr3; 6845 } 6846 6847 cr4 = cr4_read_shadow(); 6848 if (unlikely(cr4 != vmx->loaded_vmcs->host_state.cr4)) { 6849 vmcs_writel(HOST_CR4, cr4); 6850 vmx->loaded_vmcs->host_state.cr4 = cr4; 6851 } 6852 6853 /* When KVM_DEBUGREG_WONT_EXIT, dr6 is accessible in guest. */ 6854 if (unlikely(vcpu->arch.switch_db_regs & KVM_DEBUGREG_WONT_EXIT)) 6855 set_debugreg(vcpu->arch.dr6, 6); 6856 6857 /* When single-stepping over STI and MOV SS, we must clear the 6858 * corresponding interruptibility bits in the guest state. Otherwise 6859 * vmentry fails as it then expects bit 14 (BS) in pending debug 6860 * exceptions being set, but that's not correct for the guest debugging 6861 * case. */ 6862 if (vcpu->guest_debug & KVM_GUESTDBG_SINGLESTEP) 6863 vmx_set_interrupt_shadow(vcpu, 0); 6864 6865 kvm_load_guest_xsave_state(vcpu); 6866 6867 pt_guest_enter(vmx); 6868 6869 atomic_switch_perf_msrs(vmx); 6870 if (intel_pmu_lbr_is_enabled(vcpu)) 6871 vmx_passthrough_lbr_msrs(vcpu); 6872 6873 if (enable_preemption_timer) 6874 vmx_update_hv_timer(vcpu); 6875 6876 kvm_wait_lapic_expire(vcpu); 6877 6878 /* 6879 * If this vCPU has touched SPEC_CTRL, restore the guest's value if 6880 * it's non-zero. Since vmentry is serialising on affected CPUs, there 6881 * is no need to worry about the conditional branch over the wrmsr 6882 * being speculatively taken. 6883 */ 6884 x86_spec_ctrl_set_guest(vmx->spec_ctrl, 0); 6885 6886 /* The actual VMENTER/EXIT is in the .noinstr.text section. */ 6887 vmx_vcpu_enter_exit(vcpu, vmx); 6888 6889 /* 6890 * We do not use IBRS in the kernel. If this vCPU has used the 6891 * SPEC_CTRL MSR it may have left it on; save the value and 6892 * turn it off. This is much more efficient than blindly adding 6893 * it to the atomic save/restore list. Especially as the former 6894 * (Saving guest MSRs on vmexit) doesn't even exist in KVM. 6895 * 6896 * For non-nested case: 6897 * If the L01 MSR bitmap does not intercept the MSR, then we need to 6898 * save it. 6899 * 6900 * For nested case: 6901 * If the L02 MSR bitmap does not intercept the MSR, then we need to 6902 * save it. 6903 */ 6904 if (unlikely(!msr_write_intercepted(vmx, MSR_IA32_SPEC_CTRL))) 6905 vmx->spec_ctrl = native_read_msr(MSR_IA32_SPEC_CTRL); 6906 6907 x86_spec_ctrl_restore_host(vmx->spec_ctrl, 0); 6908 6909 /* All fields are clean at this point */ 6910 if (static_branch_unlikely(&enable_evmcs)) { 6911 current_evmcs->hv_clean_fields |= 6912 HV_VMX_ENLIGHTENED_CLEAN_FIELD_ALL; 6913 6914 current_evmcs->hv_vp_id = kvm_hv_get_vpindex(vcpu); 6915 } 6916 6917 /* MSR_IA32_DEBUGCTLMSR is zeroed on vmexit. Restore it if needed */ 6918 if (vmx->host_debugctlmsr) 6919 update_debugctlmsr(vmx->host_debugctlmsr); 6920 6921 #ifndef CONFIG_X86_64 6922 /* 6923 * The sysexit path does not restore ds/es, so we must set them to 6924 * a reasonable value ourselves. 6925 * 6926 * We can't defer this to vmx_prepare_switch_to_host() since that 6927 * function may be executed in interrupt context, which saves and 6928 * restore segments around it, nullifying its effect. 6929 */ 6930 loadsegment(ds, __USER_DS); 6931 loadsegment(es, __USER_DS); 6932 #endif 6933 6934 vcpu->arch.regs_avail &= ~VMX_REGS_LAZY_LOAD_SET; 6935 6936 pt_guest_exit(vmx); 6937 6938 kvm_load_host_xsave_state(vcpu); 6939 6940 if (is_guest_mode(vcpu)) { 6941 /* 6942 * Track VMLAUNCH/VMRESUME that have made past guest state 6943 * checking. 6944 */ 6945 if (vmx->nested.nested_run_pending && 6946 !vmx->exit_reason.failed_vmentry) 6947 ++vcpu->stat.nested_run; 6948 6949 vmx->nested.nested_run_pending = 0; 6950 } 6951 6952 vmx->idt_vectoring_info = 0; 6953 6954 if (unlikely(vmx->fail)) { 6955 vmx->exit_reason.full = 0xdead; 6956 return EXIT_FASTPATH_NONE; 6957 } 6958 6959 vmx->exit_reason.full = vmcs_read32(VM_EXIT_REASON); 6960 if (unlikely((u16)vmx->exit_reason.basic == EXIT_REASON_MCE_DURING_VMENTRY)) 6961 kvm_machine_check(); 6962 6963 if (likely(!vmx->exit_reason.failed_vmentry)) 6964 vmx->idt_vectoring_info = vmcs_read32(IDT_VECTORING_INFO_FIELD); 6965 6966 trace_kvm_exit(vcpu, KVM_ISA_VMX); 6967 6968 if (unlikely(vmx->exit_reason.failed_vmentry)) 6969 return EXIT_FASTPATH_NONE; 6970 6971 vmx->loaded_vmcs->launched = 1; 6972 6973 vmx_recover_nmi_blocking(vmx); 6974 vmx_complete_interrupts(vmx); 6975 6976 if (is_guest_mode(vcpu)) 6977 return EXIT_FASTPATH_NONE; 6978 6979 return vmx_exit_handlers_fastpath(vcpu); 6980 } 6981 6982 static void vmx_vcpu_free(struct kvm_vcpu *vcpu) 6983 { 6984 struct vcpu_vmx *vmx = to_vmx(vcpu); 6985 6986 if (enable_pml) 6987 vmx_destroy_pml_buffer(vmx); 6988 free_vpid(vmx->vpid); 6989 nested_vmx_free_vcpu(vcpu); 6990 free_loaded_vmcs(vmx->loaded_vmcs); 6991 } 6992 6993 static int vmx_vcpu_create(struct kvm_vcpu *vcpu) 6994 { 6995 struct vmx_uret_msr *tsx_ctrl; 6996 struct vcpu_vmx *vmx; 6997 int i, err; 6998 6999 BUILD_BUG_ON(offsetof(struct vcpu_vmx, vcpu) != 0); 7000 vmx = to_vmx(vcpu); 7001 7002 INIT_LIST_HEAD(&vmx->pi_wakeup_list); 7003 7004 err = -ENOMEM; 7005 7006 vmx->vpid = allocate_vpid(); 7007 7008 /* 7009 * If PML is turned on, failure on enabling PML just results in failure 7010 * of creating the vcpu, therefore we can simplify PML logic (by 7011 * avoiding dealing with cases, such as enabling PML partially on vcpus 7012 * for the guest), etc. 7013 */ 7014 if (enable_pml) { 7015 vmx->pml_pg = alloc_page(GFP_KERNEL_ACCOUNT | __GFP_ZERO); 7016 if (!vmx->pml_pg) 7017 goto free_vpid; 7018 } 7019 7020 for (i = 0; i < kvm_nr_uret_msrs; ++i) 7021 vmx->guest_uret_msrs[i].mask = -1ull; 7022 if (boot_cpu_has(X86_FEATURE_RTM)) { 7023 /* 7024 * TSX_CTRL_CPUID_CLEAR is handled in the CPUID interception. 7025 * Keep the host value unchanged to avoid changing CPUID bits 7026 * under the host kernel's feet. 7027 */ 7028 tsx_ctrl = vmx_find_uret_msr(vmx, MSR_IA32_TSX_CTRL); 7029 if (tsx_ctrl) 7030 tsx_ctrl->mask = ~(u64)TSX_CTRL_CPUID_CLEAR; 7031 } 7032 7033 err = alloc_loaded_vmcs(&vmx->vmcs01); 7034 if (err < 0) 7035 goto free_pml; 7036 7037 /* 7038 * Use Hyper-V 'Enlightened MSR Bitmap' feature when KVM runs as a 7039 * nested (L1) hypervisor and Hyper-V in L0 supports it. Enable the 7040 * feature only for vmcs01, KVM currently isn't equipped to realize any 7041 * performance benefits from enabling it for vmcs02. 7042 */ 7043 if (IS_ENABLED(CONFIG_HYPERV) && static_branch_unlikely(&enable_evmcs) && 7044 (ms_hyperv.nested_features & HV_X64_NESTED_MSR_BITMAP)) { 7045 struct hv_enlightened_vmcs *evmcs = (void *)vmx->vmcs01.vmcs; 7046 7047 evmcs->hv_enlightenments_control.msr_bitmap = 1; 7048 } 7049 7050 /* The MSR bitmap starts with all ones */ 7051 bitmap_fill(vmx->shadow_msr_intercept.read, MAX_POSSIBLE_PASSTHROUGH_MSRS); 7052 bitmap_fill(vmx->shadow_msr_intercept.write, MAX_POSSIBLE_PASSTHROUGH_MSRS); 7053 7054 vmx_disable_intercept_for_msr(vcpu, MSR_IA32_TSC, MSR_TYPE_R); 7055 #ifdef CONFIG_X86_64 7056 vmx_disable_intercept_for_msr(vcpu, MSR_FS_BASE, MSR_TYPE_RW); 7057 vmx_disable_intercept_for_msr(vcpu, MSR_GS_BASE, MSR_TYPE_RW); 7058 vmx_disable_intercept_for_msr(vcpu, MSR_KERNEL_GS_BASE, MSR_TYPE_RW); 7059 #endif 7060 vmx_disable_intercept_for_msr(vcpu, MSR_IA32_SYSENTER_CS, MSR_TYPE_RW); 7061 vmx_disable_intercept_for_msr(vcpu, MSR_IA32_SYSENTER_ESP, MSR_TYPE_RW); 7062 vmx_disable_intercept_for_msr(vcpu, MSR_IA32_SYSENTER_EIP, MSR_TYPE_RW); 7063 if (kvm_cstate_in_guest(vcpu->kvm)) { 7064 vmx_disable_intercept_for_msr(vcpu, MSR_CORE_C1_RES, MSR_TYPE_R); 7065 vmx_disable_intercept_for_msr(vcpu, MSR_CORE_C3_RESIDENCY, MSR_TYPE_R); 7066 vmx_disable_intercept_for_msr(vcpu, MSR_CORE_C6_RESIDENCY, MSR_TYPE_R); 7067 vmx_disable_intercept_for_msr(vcpu, MSR_CORE_C7_RESIDENCY, MSR_TYPE_R); 7068 } 7069 7070 vmx->loaded_vmcs = &vmx->vmcs01; 7071 7072 if (cpu_need_virtualize_apic_accesses(vcpu)) { 7073 err = alloc_apic_access_page(vcpu->kvm); 7074 if (err) 7075 goto free_vmcs; 7076 } 7077 7078 if (enable_ept && !enable_unrestricted_guest) { 7079 err = init_rmode_identity_map(vcpu->kvm); 7080 if (err) 7081 goto free_vmcs; 7082 } 7083 7084 return 0; 7085 7086 free_vmcs: 7087 free_loaded_vmcs(vmx->loaded_vmcs); 7088 free_pml: 7089 vmx_destroy_pml_buffer(vmx); 7090 free_vpid: 7091 free_vpid(vmx->vpid); 7092 return err; 7093 } 7094 7095 #define L1TF_MSG_SMT "L1TF CPU bug present and SMT on, data leak possible. See CVE-2018-3646 and https://www.kernel.org/doc/html/latest/admin-guide/hw-vuln/l1tf.html for details.\n" 7096 #define L1TF_MSG_L1D "L1TF CPU bug present and virtualization mitigation disabled, data leak possible. See CVE-2018-3646 and https://www.kernel.org/doc/html/latest/admin-guide/hw-vuln/l1tf.html for details.\n" 7097 7098 static int vmx_vm_init(struct kvm *kvm) 7099 { 7100 if (!ple_gap) 7101 kvm->arch.pause_in_guest = true; 7102 7103 if (boot_cpu_has(X86_BUG_L1TF) && enable_ept) { 7104 switch (l1tf_mitigation) { 7105 case L1TF_MITIGATION_OFF: 7106 case L1TF_MITIGATION_FLUSH_NOWARN: 7107 /* 'I explicitly don't care' is set */ 7108 break; 7109 case L1TF_MITIGATION_FLUSH: 7110 case L1TF_MITIGATION_FLUSH_NOSMT: 7111 case L1TF_MITIGATION_FULL: 7112 /* 7113 * Warn upon starting the first VM in a potentially 7114 * insecure environment. 7115 */ 7116 if (sched_smt_active()) 7117 pr_warn_once(L1TF_MSG_SMT); 7118 if (l1tf_vmx_mitigation == VMENTER_L1D_FLUSH_NEVER) 7119 pr_warn_once(L1TF_MSG_L1D); 7120 break; 7121 case L1TF_MITIGATION_FULL_FORCE: 7122 /* Flush is enforced */ 7123 break; 7124 } 7125 } 7126 return 0; 7127 } 7128 7129 static int __init vmx_check_processor_compat(void) 7130 { 7131 struct vmcs_config vmcs_conf; 7132 struct vmx_capability vmx_cap; 7133 7134 if (!this_cpu_has(X86_FEATURE_MSR_IA32_FEAT_CTL) || 7135 !this_cpu_has(X86_FEATURE_VMX)) { 7136 pr_err("kvm: VMX is disabled on CPU %d\n", smp_processor_id()); 7137 return -EIO; 7138 } 7139 7140 if (setup_vmcs_config(&vmcs_conf, &vmx_cap) < 0) 7141 return -EIO; 7142 if (nested) 7143 nested_vmx_setup_ctls_msrs(&vmcs_conf.nested, vmx_cap.ept); 7144 if (memcmp(&vmcs_config, &vmcs_conf, sizeof(struct vmcs_config)) != 0) { 7145 printk(KERN_ERR "kvm: CPU %d feature inconsistency!\n", 7146 smp_processor_id()); 7147 return -EIO; 7148 } 7149 return 0; 7150 } 7151 7152 static u64 vmx_get_mt_mask(struct kvm_vcpu *vcpu, gfn_t gfn, bool is_mmio) 7153 { 7154 u8 cache; 7155 7156 /* We wanted to honor guest CD/MTRR/PAT, but doing so could result in 7157 * memory aliases with conflicting memory types and sometimes MCEs. 7158 * We have to be careful as to what are honored and when. 7159 * 7160 * For MMIO, guest CD/MTRR are ignored. The EPT memory type is set to 7161 * UC. The effective memory type is UC or WC depending on guest PAT. 7162 * This was historically the source of MCEs and we want to be 7163 * conservative. 7164 * 7165 * When there is no need to deal with noncoherent DMA (e.g., no VT-d 7166 * or VT-d has snoop control), guest CD/MTRR/PAT are all ignored. The 7167 * EPT memory type is set to WB. The effective memory type is forced 7168 * WB. 7169 * 7170 * Otherwise, we trust guest. Guest CD/MTRR/PAT are all honored. The 7171 * EPT memory type is used to emulate guest CD/MTRR. 7172 */ 7173 7174 if (is_mmio) 7175 return MTRR_TYPE_UNCACHABLE << VMX_EPT_MT_EPTE_SHIFT; 7176 7177 if (!kvm_arch_has_noncoherent_dma(vcpu->kvm)) 7178 return (MTRR_TYPE_WRBACK << VMX_EPT_MT_EPTE_SHIFT) | VMX_EPT_IPAT_BIT; 7179 7180 if (kvm_read_cr0(vcpu) & X86_CR0_CD) { 7181 if (kvm_check_has_quirk(vcpu->kvm, KVM_X86_QUIRK_CD_NW_CLEARED)) 7182 cache = MTRR_TYPE_WRBACK; 7183 else 7184 cache = MTRR_TYPE_UNCACHABLE; 7185 7186 return (cache << VMX_EPT_MT_EPTE_SHIFT) | VMX_EPT_IPAT_BIT; 7187 } 7188 7189 return kvm_mtrr_get_guest_memory_type(vcpu, gfn) << VMX_EPT_MT_EPTE_SHIFT; 7190 } 7191 7192 static void vmcs_set_secondary_exec_control(struct vcpu_vmx *vmx, u32 new_ctl) 7193 { 7194 /* 7195 * These bits in the secondary execution controls field 7196 * are dynamic, the others are mostly based on the hypervisor 7197 * architecture and the guest's CPUID. Do not touch the 7198 * dynamic bits. 7199 */ 7200 u32 mask = 7201 SECONDARY_EXEC_SHADOW_VMCS | 7202 SECONDARY_EXEC_VIRTUALIZE_X2APIC_MODE | 7203 SECONDARY_EXEC_VIRTUALIZE_APIC_ACCESSES | 7204 SECONDARY_EXEC_DESC; 7205 7206 u32 cur_ctl = secondary_exec_controls_get(vmx); 7207 7208 secondary_exec_controls_set(vmx, (new_ctl & ~mask) | (cur_ctl & mask)); 7209 } 7210 7211 /* 7212 * Generate MSR_IA32_VMX_CR{0,4}_FIXED1 according to CPUID. Only set bits 7213 * (indicating "allowed-1") if they are supported in the guest's CPUID. 7214 */ 7215 static void nested_vmx_cr_fixed1_bits_update(struct kvm_vcpu *vcpu) 7216 { 7217 struct vcpu_vmx *vmx = to_vmx(vcpu); 7218 struct kvm_cpuid_entry2 *entry; 7219 7220 vmx->nested.msrs.cr0_fixed1 = 0xffffffff; 7221 vmx->nested.msrs.cr4_fixed1 = X86_CR4_PCE; 7222 7223 #define cr4_fixed1_update(_cr4_mask, _reg, _cpuid_mask) do { \ 7224 if (entry && (entry->_reg & (_cpuid_mask))) \ 7225 vmx->nested.msrs.cr4_fixed1 |= (_cr4_mask); \ 7226 } while (0) 7227 7228 entry = kvm_find_cpuid_entry(vcpu, 0x1, 0); 7229 cr4_fixed1_update(X86_CR4_VME, edx, feature_bit(VME)); 7230 cr4_fixed1_update(X86_CR4_PVI, edx, feature_bit(VME)); 7231 cr4_fixed1_update(X86_CR4_TSD, edx, feature_bit(TSC)); 7232 cr4_fixed1_update(X86_CR4_DE, edx, feature_bit(DE)); 7233 cr4_fixed1_update(X86_CR4_PSE, edx, feature_bit(PSE)); 7234 cr4_fixed1_update(X86_CR4_PAE, edx, feature_bit(PAE)); 7235 cr4_fixed1_update(X86_CR4_MCE, edx, feature_bit(MCE)); 7236 cr4_fixed1_update(X86_CR4_PGE, edx, feature_bit(PGE)); 7237 cr4_fixed1_update(X86_CR4_OSFXSR, edx, feature_bit(FXSR)); 7238 cr4_fixed1_update(X86_CR4_OSXMMEXCPT, edx, feature_bit(XMM)); 7239 cr4_fixed1_update(X86_CR4_VMXE, ecx, feature_bit(VMX)); 7240 cr4_fixed1_update(X86_CR4_SMXE, ecx, feature_bit(SMX)); 7241 cr4_fixed1_update(X86_CR4_PCIDE, ecx, feature_bit(PCID)); 7242 cr4_fixed1_update(X86_CR4_OSXSAVE, ecx, feature_bit(XSAVE)); 7243 7244 entry = kvm_find_cpuid_entry(vcpu, 0x7, 0); 7245 cr4_fixed1_update(X86_CR4_FSGSBASE, ebx, feature_bit(FSGSBASE)); 7246 cr4_fixed1_update(X86_CR4_SMEP, ebx, feature_bit(SMEP)); 7247 cr4_fixed1_update(X86_CR4_SMAP, ebx, feature_bit(SMAP)); 7248 cr4_fixed1_update(X86_CR4_PKE, ecx, feature_bit(PKU)); 7249 cr4_fixed1_update(X86_CR4_UMIP, ecx, feature_bit(UMIP)); 7250 cr4_fixed1_update(X86_CR4_LA57, ecx, feature_bit(LA57)); 7251 7252 #undef cr4_fixed1_update 7253 } 7254 7255 static void nested_vmx_entry_exit_ctls_update(struct kvm_vcpu *vcpu) 7256 { 7257 struct vcpu_vmx *vmx = to_vmx(vcpu); 7258 7259 if (kvm_mpx_supported()) { 7260 bool mpx_enabled = guest_cpuid_has(vcpu, X86_FEATURE_MPX); 7261 7262 if (mpx_enabled) { 7263 vmx->nested.msrs.entry_ctls_high |= VM_ENTRY_LOAD_BNDCFGS; 7264 vmx->nested.msrs.exit_ctls_high |= VM_EXIT_CLEAR_BNDCFGS; 7265 } else { 7266 vmx->nested.msrs.entry_ctls_high &= ~VM_ENTRY_LOAD_BNDCFGS; 7267 vmx->nested.msrs.exit_ctls_high &= ~VM_EXIT_CLEAR_BNDCFGS; 7268 } 7269 } 7270 } 7271 7272 static void update_intel_pt_cfg(struct kvm_vcpu *vcpu) 7273 { 7274 struct vcpu_vmx *vmx = to_vmx(vcpu); 7275 struct kvm_cpuid_entry2 *best = NULL; 7276 int i; 7277 7278 for (i = 0; i < PT_CPUID_LEAVES; i++) { 7279 best = kvm_find_cpuid_entry(vcpu, 0x14, i); 7280 if (!best) 7281 return; 7282 vmx->pt_desc.caps[CPUID_EAX + i*PT_CPUID_REGS_NUM] = best->eax; 7283 vmx->pt_desc.caps[CPUID_EBX + i*PT_CPUID_REGS_NUM] = best->ebx; 7284 vmx->pt_desc.caps[CPUID_ECX + i*PT_CPUID_REGS_NUM] = best->ecx; 7285 vmx->pt_desc.caps[CPUID_EDX + i*PT_CPUID_REGS_NUM] = best->edx; 7286 } 7287 7288 /* Get the number of configurable Address Ranges for filtering */ 7289 vmx->pt_desc.num_address_ranges = intel_pt_validate_cap(vmx->pt_desc.caps, 7290 PT_CAP_num_address_ranges); 7291 7292 /* Initialize and clear the no dependency bits */ 7293 vmx->pt_desc.ctl_bitmask = ~(RTIT_CTL_TRACEEN | RTIT_CTL_OS | 7294 RTIT_CTL_USR | RTIT_CTL_TSC_EN | RTIT_CTL_DISRETC | 7295 RTIT_CTL_BRANCH_EN); 7296 7297 /* 7298 * If CPUID.(EAX=14H,ECX=0):EBX[0]=1 CR3Filter can be set otherwise 7299 * will inject an #GP 7300 */ 7301 if (intel_pt_validate_cap(vmx->pt_desc.caps, PT_CAP_cr3_filtering)) 7302 vmx->pt_desc.ctl_bitmask &= ~RTIT_CTL_CR3EN; 7303 7304 /* 7305 * If CPUID.(EAX=14H,ECX=0):EBX[1]=1 CYCEn, CycThresh and 7306 * PSBFreq can be set 7307 */ 7308 if (intel_pt_validate_cap(vmx->pt_desc.caps, PT_CAP_psb_cyc)) 7309 vmx->pt_desc.ctl_bitmask &= ~(RTIT_CTL_CYCLEACC | 7310 RTIT_CTL_CYC_THRESH | RTIT_CTL_PSB_FREQ); 7311 7312 /* 7313 * If CPUID.(EAX=14H,ECX=0):EBX[3]=1 MTCEn and MTCFreq can be set 7314 */ 7315 if (intel_pt_validate_cap(vmx->pt_desc.caps, PT_CAP_mtc)) 7316 vmx->pt_desc.ctl_bitmask &= ~(RTIT_CTL_MTC_EN | 7317 RTIT_CTL_MTC_RANGE); 7318 7319 /* If CPUID.(EAX=14H,ECX=0):EBX[4]=1 FUPonPTW and PTWEn can be set */ 7320 if (intel_pt_validate_cap(vmx->pt_desc.caps, PT_CAP_ptwrite)) 7321 vmx->pt_desc.ctl_bitmask &= ~(RTIT_CTL_FUP_ON_PTW | 7322 RTIT_CTL_PTW_EN); 7323 7324 /* If CPUID.(EAX=14H,ECX=0):EBX[5]=1 PwrEvEn can be set */ 7325 if (intel_pt_validate_cap(vmx->pt_desc.caps, PT_CAP_power_event_trace)) 7326 vmx->pt_desc.ctl_bitmask &= ~RTIT_CTL_PWR_EVT_EN; 7327 7328 /* If CPUID.(EAX=14H,ECX=0):ECX[0]=1 ToPA can be set */ 7329 if (intel_pt_validate_cap(vmx->pt_desc.caps, PT_CAP_topa_output)) 7330 vmx->pt_desc.ctl_bitmask &= ~RTIT_CTL_TOPA; 7331 7332 /* If CPUID.(EAX=14H,ECX=0):ECX[3]=1 FabricEn can be set */ 7333 if (intel_pt_validate_cap(vmx->pt_desc.caps, PT_CAP_output_subsys)) 7334 vmx->pt_desc.ctl_bitmask &= ~RTIT_CTL_FABRIC_EN; 7335 7336 /* unmask address range configure area */ 7337 for (i = 0; i < vmx->pt_desc.num_address_ranges; i++) 7338 vmx->pt_desc.ctl_bitmask &= ~(0xfULL << (32 + i * 4)); 7339 } 7340 7341 static void vmx_vcpu_after_set_cpuid(struct kvm_vcpu *vcpu) 7342 { 7343 struct vcpu_vmx *vmx = to_vmx(vcpu); 7344 7345 /* xsaves_enabled is recomputed in vmx_compute_secondary_exec_control(). */ 7346 vcpu->arch.xsaves_enabled = false; 7347 7348 vmx_setup_uret_msrs(vmx); 7349 7350 if (cpu_has_secondary_exec_ctrls()) 7351 vmcs_set_secondary_exec_control(vmx, 7352 vmx_secondary_exec_control(vmx)); 7353 7354 if (nested_vmx_allowed(vcpu)) 7355 vmx->msr_ia32_feature_control_valid_bits |= 7356 FEAT_CTL_VMX_ENABLED_INSIDE_SMX | 7357 FEAT_CTL_VMX_ENABLED_OUTSIDE_SMX; 7358 else 7359 vmx->msr_ia32_feature_control_valid_bits &= 7360 ~(FEAT_CTL_VMX_ENABLED_INSIDE_SMX | 7361 FEAT_CTL_VMX_ENABLED_OUTSIDE_SMX); 7362 7363 if (nested_vmx_allowed(vcpu)) { 7364 nested_vmx_cr_fixed1_bits_update(vcpu); 7365 nested_vmx_entry_exit_ctls_update(vcpu); 7366 } 7367 7368 if (boot_cpu_has(X86_FEATURE_INTEL_PT) && 7369 guest_cpuid_has(vcpu, X86_FEATURE_INTEL_PT)) 7370 update_intel_pt_cfg(vcpu); 7371 7372 if (boot_cpu_has(X86_FEATURE_RTM)) { 7373 struct vmx_uret_msr *msr; 7374 msr = vmx_find_uret_msr(vmx, MSR_IA32_TSX_CTRL); 7375 if (msr) { 7376 bool enabled = guest_cpuid_has(vcpu, X86_FEATURE_RTM); 7377 vmx_set_guest_uret_msr(vmx, msr, enabled ? 0 : TSX_CTRL_RTM_DISABLE); 7378 } 7379 } 7380 7381 if (kvm_cpu_cap_has(X86_FEATURE_XFD)) 7382 vmx_set_intercept_for_msr(vcpu, MSR_IA32_XFD_ERR, MSR_TYPE_R, 7383 !guest_cpuid_has(vcpu, X86_FEATURE_XFD)); 7384 7385 7386 set_cr4_guest_host_mask(vmx); 7387 7388 vmx_write_encls_bitmap(vcpu, NULL); 7389 if (guest_cpuid_has(vcpu, X86_FEATURE_SGX)) 7390 vmx->msr_ia32_feature_control_valid_bits |= FEAT_CTL_SGX_ENABLED; 7391 else 7392 vmx->msr_ia32_feature_control_valid_bits &= ~FEAT_CTL_SGX_ENABLED; 7393 7394 if (guest_cpuid_has(vcpu, X86_FEATURE_SGX_LC)) 7395 vmx->msr_ia32_feature_control_valid_bits |= 7396 FEAT_CTL_SGX_LC_ENABLED; 7397 else 7398 vmx->msr_ia32_feature_control_valid_bits &= 7399 ~FEAT_CTL_SGX_LC_ENABLED; 7400 7401 /* Refresh #PF interception to account for MAXPHYADDR changes. */ 7402 vmx_update_exception_bitmap(vcpu); 7403 } 7404 7405 static __init void vmx_set_cpu_caps(void) 7406 { 7407 kvm_set_cpu_caps(); 7408 7409 /* CPUID 0x1 */ 7410 if (nested) 7411 kvm_cpu_cap_set(X86_FEATURE_VMX); 7412 7413 /* CPUID 0x7 */ 7414 if (kvm_mpx_supported()) 7415 kvm_cpu_cap_check_and_set(X86_FEATURE_MPX); 7416 if (!cpu_has_vmx_invpcid()) 7417 kvm_cpu_cap_clear(X86_FEATURE_INVPCID); 7418 if (vmx_pt_mode_is_host_guest()) 7419 kvm_cpu_cap_check_and_set(X86_FEATURE_INTEL_PT); 7420 7421 if (!enable_sgx) { 7422 kvm_cpu_cap_clear(X86_FEATURE_SGX); 7423 kvm_cpu_cap_clear(X86_FEATURE_SGX_LC); 7424 kvm_cpu_cap_clear(X86_FEATURE_SGX1); 7425 kvm_cpu_cap_clear(X86_FEATURE_SGX2); 7426 } 7427 7428 if (vmx_umip_emulated()) 7429 kvm_cpu_cap_set(X86_FEATURE_UMIP); 7430 7431 /* CPUID 0xD.1 */ 7432 supported_xss = 0; 7433 if (!cpu_has_vmx_xsaves()) 7434 kvm_cpu_cap_clear(X86_FEATURE_XSAVES); 7435 7436 /* CPUID 0x80000001 and 0x7 (RDPID) */ 7437 if (!cpu_has_vmx_rdtscp()) { 7438 kvm_cpu_cap_clear(X86_FEATURE_RDTSCP); 7439 kvm_cpu_cap_clear(X86_FEATURE_RDPID); 7440 } 7441 7442 if (cpu_has_vmx_waitpkg()) 7443 kvm_cpu_cap_check_and_set(X86_FEATURE_WAITPKG); 7444 } 7445 7446 static void vmx_request_immediate_exit(struct kvm_vcpu *vcpu) 7447 { 7448 to_vmx(vcpu)->req_immediate_exit = true; 7449 } 7450 7451 static int vmx_check_intercept_io(struct kvm_vcpu *vcpu, 7452 struct x86_instruction_info *info) 7453 { 7454 struct vmcs12 *vmcs12 = get_vmcs12(vcpu); 7455 unsigned short port; 7456 bool intercept; 7457 int size; 7458 7459 if (info->intercept == x86_intercept_in || 7460 info->intercept == x86_intercept_ins) { 7461 port = info->src_val; 7462 size = info->dst_bytes; 7463 } else { 7464 port = info->dst_val; 7465 size = info->src_bytes; 7466 } 7467 7468 /* 7469 * If the 'use IO bitmaps' VM-execution control is 0, IO instruction 7470 * VM-exits depend on the 'unconditional IO exiting' VM-execution 7471 * control. 7472 * 7473 * Otherwise, IO instruction VM-exits are controlled by the IO bitmaps. 7474 */ 7475 if (!nested_cpu_has(vmcs12, CPU_BASED_USE_IO_BITMAPS)) 7476 intercept = nested_cpu_has(vmcs12, 7477 CPU_BASED_UNCOND_IO_EXITING); 7478 else 7479 intercept = nested_vmx_check_io_bitmaps(vcpu, port, size); 7480 7481 /* FIXME: produce nested vmexit and return X86EMUL_INTERCEPTED. */ 7482 return intercept ? X86EMUL_UNHANDLEABLE : X86EMUL_CONTINUE; 7483 } 7484 7485 static int vmx_check_intercept(struct kvm_vcpu *vcpu, 7486 struct x86_instruction_info *info, 7487 enum x86_intercept_stage stage, 7488 struct x86_exception *exception) 7489 { 7490 struct vmcs12 *vmcs12 = get_vmcs12(vcpu); 7491 7492 switch (info->intercept) { 7493 /* 7494 * RDPID causes #UD if disabled through secondary execution controls. 7495 * Because it is marked as EmulateOnUD, we need to intercept it here. 7496 * Note, RDPID is hidden behind ENABLE_RDTSCP. 7497 */ 7498 case x86_intercept_rdpid: 7499 if (!nested_cpu_has2(vmcs12, SECONDARY_EXEC_ENABLE_RDTSCP)) { 7500 exception->vector = UD_VECTOR; 7501 exception->error_code_valid = false; 7502 return X86EMUL_PROPAGATE_FAULT; 7503 } 7504 break; 7505 7506 case x86_intercept_in: 7507 case x86_intercept_ins: 7508 case x86_intercept_out: 7509 case x86_intercept_outs: 7510 return vmx_check_intercept_io(vcpu, info); 7511 7512 case x86_intercept_lgdt: 7513 case x86_intercept_lidt: 7514 case x86_intercept_lldt: 7515 case x86_intercept_ltr: 7516 case x86_intercept_sgdt: 7517 case x86_intercept_sidt: 7518 case x86_intercept_sldt: 7519 case x86_intercept_str: 7520 if (!nested_cpu_has2(vmcs12, SECONDARY_EXEC_DESC)) 7521 return X86EMUL_CONTINUE; 7522 7523 /* FIXME: produce nested vmexit and return X86EMUL_INTERCEPTED. */ 7524 break; 7525 7526 /* TODO: check more intercepts... */ 7527 default: 7528 break; 7529 } 7530 7531 return X86EMUL_UNHANDLEABLE; 7532 } 7533 7534 #ifdef CONFIG_X86_64 7535 /* (a << shift) / divisor, return 1 if overflow otherwise 0 */ 7536 static inline int u64_shl_div_u64(u64 a, unsigned int shift, 7537 u64 divisor, u64 *result) 7538 { 7539 u64 low = a << shift, high = a >> (64 - shift); 7540 7541 /* To avoid the overflow on divq */ 7542 if (high >= divisor) 7543 return 1; 7544 7545 /* Low hold the result, high hold rem which is discarded */ 7546 asm("divq %2\n\t" : "=a" (low), "=d" (high) : 7547 "rm" (divisor), "0" (low), "1" (high)); 7548 *result = low; 7549 7550 return 0; 7551 } 7552 7553 static int vmx_set_hv_timer(struct kvm_vcpu *vcpu, u64 guest_deadline_tsc, 7554 bool *expired) 7555 { 7556 struct vcpu_vmx *vmx; 7557 u64 tscl, guest_tscl, delta_tsc, lapic_timer_advance_cycles; 7558 struct kvm_timer *ktimer = &vcpu->arch.apic->lapic_timer; 7559 7560 vmx = to_vmx(vcpu); 7561 tscl = rdtsc(); 7562 guest_tscl = kvm_read_l1_tsc(vcpu, tscl); 7563 delta_tsc = max(guest_deadline_tsc, guest_tscl) - guest_tscl; 7564 lapic_timer_advance_cycles = nsec_to_cycles(vcpu, 7565 ktimer->timer_advance_ns); 7566 7567 if (delta_tsc > lapic_timer_advance_cycles) 7568 delta_tsc -= lapic_timer_advance_cycles; 7569 else 7570 delta_tsc = 0; 7571 7572 /* Convert to host delta tsc if tsc scaling is enabled */ 7573 if (vcpu->arch.l1_tsc_scaling_ratio != kvm_default_tsc_scaling_ratio && 7574 delta_tsc && u64_shl_div_u64(delta_tsc, 7575 kvm_tsc_scaling_ratio_frac_bits, 7576 vcpu->arch.l1_tsc_scaling_ratio, &delta_tsc)) 7577 return -ERANGE; 7578 7579 /* 7580 * If the delta tsc can't fit in the 32 bit after the multi shift, 7581 * we can't use the preemption timer. 7582 * It's possible that it fits on later vmentries, but checking 7583 * on every vmentry is costly so we just use an hrtimer. 7584 */ 7585 if (delta_tsc >> (cpu_preemption_timer_multi + 32)) 7586 return -ERANGE; 7587 7588 vmx->hv_deadline_tsc = tscl + delta_tsc; 7589 *expired = !delta_tsc; 7590 return 0; 7591 } 7592 7593 static void vmx_cancel_hv_timer(struct kvm_vcpu *vcpu) 7594 { 7595 to_vmx(vcpu)->hv_deadline_tsc = -1; 7596 } 7597 #endif 7598 7599 static void vmx_sched_in(struct kvm_vcpu *vcpu, int cpu) 7600 { 7601 if (!kvm_pause_in_guest(vcpu->kvm)) 7602 shrink_ple_window(vcpu); 7603 } 7604 7605 void vmx_update_cpu_dirty_logging(struct kvm_vcpu *vcpu) 7606 { 7607 struct vcpu_vmx *vmx = to_vmx(vcpu); 7608 7609 if (is_guest_mode(vcpu)) { 7610 vmx->nested.update_vmcs01_cpu_dirty_logging = true; 7611 return; 7612 } 7613 7614 /* 7615 * Note, cpu_dirty_logging_count can be changed concurrent with this 7616 * code, but in that case another update request will be made and so 7617 * the guest will never run with a stale PML value. 7618 */ 7619 if (vcpu->kvm->arch.cpu_dirty_logging_count) 7620 secondary_exec_controls_setbit(vmx, SECONDARY_EXEC_ENABLE_PML); 7621 else 7622 secondary_exec_controls_clearbit(vmx, SECONDARY_EXEC_ENABLE_PML); 7623 } 7624 7625 static void vmx_setup_mce(struct kvm_vcpu *vcpu) 7626 { 7627 if (vcpu->arch.mcg_cap & MCG_LMCE_P) 7628 to_vmx(vcpu)->msr_ia32_feature_control_valid_bits |= 7629 FEAT_CTL_LMCE_ENABLED; 7630 else 7631 to_vmx(vcpu)->msr_ia32_feature_control_valid_bits &= 7632 ~FEAT_CTL_LMCE_ENABLED; 7633 } 7634 7635 static int vmx_smi_allowed(struct kvm_vcpu *vcpu, bool for_injection) 7636 { 7637 /* we need a nested vmexit to enter SMM, postpone if run is pending */ 7638 if (to_vmx(vcpu)->nested.nested_run_pending) 7639 return -EBUSY; 7640 return !is_smm(vcpu); 7641 } 7642 7643 static int vmx_enter_smm(struct kvm_vcpu *vcpu, char *smstate) 7644 { 7645 struct vcpu_vmx *vmx = to_vmx(vcpu); 7646 7647 vmx->nested.smm.guest_mode = is_guest_mode(vcpu); 7648 if (vmx->nested.smm.guest_mode) 7649 nested_vmx_vmexit(vcpu, -1, 0, 0); 7650 7651 vmx->nested.smm.vmxon = vmx->nested.vmxon; 7652 vmx->nested.vmxon = false; 7653 vmx_clear_hlt(vcpu); 7654 return 0; 7655 } 7656 7657 static int vmx_leave_smm(struct kvm_vcpu *vcpu, const char *smstate) 7658 { 7659 struct vcpu_vmx *vmx = to_vmx(vcpu); 7660 int ret; 7661 7662 if (vmx->nested.smm.vmxon) { 7663 vmx->nested.vmxon = true; 7664 vmx->nested.smm.vmxon = false; 7665 } 7666 7667 if (vmx->nested.smm.guest_mode) { 7668 ret = nested_vmx_enter_non_root_mode(vcpu, false); 7669 if (ret) 7670 return ret; 7671 7672 vmx->nested.nested_run_pending = 1; 7673 vmx->nested.smm.guest_mode = false; 7674 } 7675 return 0; 7676 } 7677 7678 static void vmx_enable_smi_window(struct kvm_vcpu *vcpu) 7679 { 7680 /* RSM will cause a vmexit anyway. */ 7681 } 7682 7683 static bool vmx_apic_init_signal_blocked(struct kvm_vcpu *vcpu) 7684 { 7685 return to_vmx(vcpu)->nested.vmxon && !is_guest_mode(vcpu); 7686 } 7687 7688 static void vmx_migrate_timers(struct kvm_vcpu *vcpu) 7689 { 7690 if (is_guest_mode(vcpu)) { 7691 struct hrtimer *timer = &to_vmx(vcpu)->nested.preemption_timer; 7692 7693 if (hrtimer_try_to_cancel(timer) == 1) 7694 hrtimer_start_expires(timer, HRTIMER_MODE_ABS_PINNED); 7695 } 7696 } 7697 7698 static void vmx_hardware_unsetup(void) 7699 { 7700 kvm_set_posted_intr_wakeup_handler(NULL); 7701 7702 if (nested) 7703 nested_vmx_hardware_unsetup(); 7704 7705 free_kvm_area(); 7706 } 7707 7708 static bool vmx_check_apicv_inhibit_reasons(enum kvm_apicv_inhibit reason) 7709 { 7710 ulong supported = BIT(APICV_INHIBIT_REASON_DISABLE) | 7711 BIT(APICV_INHIBIT_REASON_ABSENT) | 7712 BIT(APICV_INHIBIT_REASON_HYPERV) | 7713 BIT(APICV_INHIBIT_REASON_BLOCKIRQ); 7714 7715 return supported & BIT(reason); 7716 } 7717 7718 static struct kvm_x86_ops vmx_x86_ops __initdata = { 7719 .name = "kvm_intel", 7720 7721 .hardware_unsetup = vmx_hardware_unsetup, 7722 7723 .hardware_enable = vmx_hardware_enable, 7724 .hardware_disable = vmx_hardware_disable, 7725 .has_emulated_msr = vmx_has_emulated_msr, 7726 7727 .vm_size = sizeof(struct kvm_vmx), 7728 .vm_init = vmx_vm_init, 7729 7730 .vcpu_create = vmx_vcpu_create, 7731 .vcpu_free = vmx_vcpu_free, 7732 .vcpu_reset = vmx_vcpu_reset, 7733 7734 .prepare_switch_to_guest = vmx_prepare_switch_to_guest, 7735 .vcpu_load = vmx_vcpu_load, 7736 .vcpu_put = vmx_vcpu_put, 7737 7738 .update_exception_bitmap = vmx_update_exception_bitmap, 7739 .get_msr_feature = vmx_get_msr_feature, 7740 .get_msr = vmx_get_msr, 7741 .set_msr = vmx_set_msr, 7742 .get_segment_base = vmx_get_segment_base, 7743 .get_segment = vmx_get_segment, 7744 .set_segment = vmx_set_segment, 7745 .get_cpl = vmx_get_cpl, 7746 .get_cs_db_l_bits = vmx_get_cs_db_l_bits, 7747 .set_cr0 = vmx_set_cr0, 7748 .is_valid_cr4 = vmx_is_valid_cr4, 7749 .set_cr4 = vmx_set_cr4, 7750 .set_efer = vmx_set_efer, 7751 .get_idt = vmx_get_idt, 7752 .set_idt = vmx_set_idt, 7753 .get_gdt = vmx_get_gdt, 7754 .set_gdt = vmx_set_gdt, 7755 .set_dr7 = vmx_set_dr7, 7756 .sync_dirty_debug_regs = vmx_sync_dirty_debug_regs, 7757 .cache_reg = vmx_cache_reg, 7758 .get_rflags = vmx_get_rflags, 7759 .set_rflags = vmx_set_rflags, 7760 .get_if_flag = vmx_get_if_flag, 7761 7762 .flush_tlb_all = vmx_flush_tlb_all, 7763 .flush_tlb_current = vmx_flush_tlb_current, 7764 .flush_tlb_gva = vmx_flush_tlb_gva, 7765 .flush_tlb_guest = vmx_flush_tlb_guest, 7766 7767 .vcpu_pre_run = vmx_vcpu_pre_run, 7768 .vcpu_run = vmx_vcpu_run, 7769 .handle_exit = vmx_handle_exit, 7770 .skip_emulated_instruction = vmx_skip_emulated_instruction, 7771 .update_emulated_instruction = vmx_update_emulated_instruction, 7772 .set_interrupt_shadow = vmx_set_interrupt_shadow, 7773 .get_interrupt_shadow = vmx_get_interrupt_shadow, 7774 .patch_hypercall = vmx_patch_hypercall, 7775 .inject_irq = vmx_inject_irq, 7776 .inject_nmi = vmx_inject_nmi, 7777 .queue_exception = vmx_queue_exception, 7778 .cancel_injection = vmx_cancel_injection, 7779 .interrupt_allowed = vmx_interrupt_allowed, 7780 .nmi_allowed = vmx_nmi_allowed, 7781 .get_nmi_mask = vmx_get_nmi_mask, 7782 .set_nmi_mask = vmx_set_nmi_mask, 7783 .enable_nmi_window = vmx_enable_nmi_window, 7784 .enable_irq_window = vmx_enable_irq_window, 7785 .update_cr8_intercept = vmx_update_cr8_intercept, 7786 .set_virtual_apic_mode = vmx_set_virtual_apic_mode, 7787 .set_apic_access_page_addr = vmx_set_apic_access_page_addr, 7788 .refresh_apicv_exec_ctrl = vmx_refresh_apicv_exec_ctrl, 7789 .load_eoi_exitmap = vmx_load_eoi_exitmap, 7790 .apicv_post_state_restore = vmx_apicv_post_state_restore, 7791 .check_apicv_inhibit_reasons = vmx_check_apicv_inhibit_reasons, 7792 .hwapic_irr_update = vmx_hwapic_irr_update, 7793 .hwapic_isr_update = vmx_hwapic_isr_update, 7794 .guest_apic_has_interrupt = vmx_guest_apic_has_interrupt, 7795 .sync_pir_to_irr = vmx_sync_pir_to_irr, 7796 .deliver_interrupt = vmx_deliver_interrupt, 7797 .dy_apicv_has_pending_interrupt = pi_has_pending_interrupt, 7798 7799 .set_tss_addr = vmx_set_tss_addr, 7800 .set_identity_map_addr = vmx_set_identity_map_addr, 7801 .get_mt_mask = vmx_get_mt_mask, 7802 7803 .get_exit_info = vmx_get_exit_info, 7804 7805 .vcpu_after_set_cpuid = vmx_vcpu_after_set_cpuid, 7806 7807 .has_wbinvd_exit = cpu_has_vmx_wbinvd_exit, 7808 7809 .get_l2_tsc_offset = vmx_get_l2_tsc_offset, 7810 .get_l2_tsc_multiplier = vmx_get_l2_tsc_multiplier, 7811 .write_tsc_offset = vmx_write_tsc_offset, 7812 .write_tsc_multiplier = vmx_write_tsc_multiplier, 7813 7814 .load_mmu_pgd = vmx_load_mmu_pgd, 7815 7816 .check_intercept = vmx_check_intercept, 7817 .handle_exit_irqoff = vmx_handle_exit_irqoff, 7818 7819 .request_immediate_exit = vmx_request_immediate_exit, 7820 7821 .sched_in = vmx_sched_in, 7822 7823 .cpu_dirty_log_size = PML_ENTITY_NUM, 7824 .update_cpu_dirty_logging = vmx_update_cpu_dirty_logging, 7825 7826 .pmu_ops = &intel_pmu_ops, 7827 .nested_ops = &vmx_nested_ops, 7828 7829 .pi_update_irte = vmx_pi_update_irte, 7830 .pi_start_assignment = vmx_pi_start_assignment, 7831 7832 #ifdef CONFIG_X86_64 7833 .set_hv_timer = vmx_set_hv_timer, 7834 .cancel_hv_timer = vmx_cancel_hv_timer, 7835 #endif 7836 7837 .setup_mce = vmx_setup_mce, 7838 7839 .smi_allowed = vmx_smi_allowed, 7840 .enter_smm = vmx_enter_smm, 7841 .leave_smm = vmx_leave_smm, 7842 .enable_smi_window = vmx_enable_smi_window, 7843 7844 .can_emulate_instruction = vmx_can_emulate_instruction, 7845 .apic_init_signal_blocked = vmx_apic_init_signal_blocked, 7846 .migrate_timers = vmx_migrate_timers, 7847 7848 .msr_filter_changed = vmx_msr_filter_changed, 7849 .complete_emulated_msr = kvm_complete_insn_gp, 7850 7851 .vcpu_deliver_sipi_vector = kvm_vcpu_deliver_sipi_vector, 7852 }; 7853 7854 static unsigned int vmx_handle_intel_pt_intr(void) 7855 { 7856 struct kvm_vcpu *vcpu = kvm_get_running_vcpu(); 7857 7858 /* '0' on failure so that the !PT case can use a RET0 static call. */ 7859 if (!kvm_arch_pmi_in_guest(vcpu)) 7860 return 0; 7861 7862 kvm_make_request(KVM_REQ_PMI, vcpu); 7863 __set_bit(MSR_CORE_PERF_GLOBAL_OVF_CTRL_TRACE_TOPA_PMI_BIT, 7864 (unsigned long *)&vcpu->arch.pmu.global_status); 7865 return 1; 7866 } 7867 7868 static __init void vmx_setup_user_return_msrs(void) 7869 { 7870 7871 /* 7872 * Though SYSCALL is only supported in 64-bit mode on Intel CPUs, kvm 7873 * will emulate SYSCALL in legacy mode if the vendor string in guest 7874 * CPUID.0:{EBX,ECX,EDX} is "AuthenticAMD" or "AMDisbetter!" To 7875 * support this emulation, MSR_STAR is included in the list for i386, 7876 * but is never loaded into hardware. MSR_CSTAR is also never loaded 7877 * into hardware and is here purely for emulation purposes. 7878 */ 7879 const u32 vmx_uret_msrs_list[] = { 7880 #ifdef CONFIG_X86_64 7881 MSR_SYSCALL_MASK, MSR_LSTAR, MSR_CSTAR, 7882 #endif 7883 MSR_EFER, MSR_TSC_AUX, MSR_STAR, 7884 MSR_IA32_TSX_CTRL, 7885 }; 7886 int i; 7887 7888 BUILD_BUG_ON(ARRAY_SIZE(vmx_uret_msrs_list) != MAX_NR_USER_RETURN_MSRS); 7889 7890 for (i = 0; i < ARRAY_SIZE(vmx_uret_msrs_list); ++i) 7891 kvm_add_user_return_msr(vmx_uret_msrs_list[i]); 7892 } 7893 7894 static struct kvm_x86_init_ops vmx_init_ops __initdata; 7895 7896 static __init int hardware_setup(void) 7897 { 7898 unsigned long host_bndcfgs; 7899 struct desc_ptr dt; 7900 int r; 7901 7902 store_idt(&dt); 7903 host_idt_base = dt.address; 7904 7905 vmx_setup_user_return_msrs(); 7906 7907 if (setup_vmcs_config(&vmcs_config, &vmx_capability) < 0) 7908 return -EIO; 7909 7910 if (boot_cpu_has(X86_FEATURE_NX)) 7911 kvm_enable_efer_bits(EFER_NX); 7912 7913 if (boot_cpu_has(X86_FEATURE_MPX)) { 7914 rdmsrl(MSR_IA32_BNDCFGS, host_bndcfgs); 7915 WARN_ONCE(host_bndcfgs, "KVM: BNDCFGS in host will be lost"); 7916 } 7917 7918 if (!cpu_has_vmx_mpx()) 7919 supported_xcr0 &= ~(XFEATURE_MASK_BNDREGS | 7920 XFEATURE_MASK_BNDCSR); 7921 7922 if (!cpu_has_vmx_vpid() || !cpu_has_vmx_invvpid() || 7923 !(cpu_has_vmx_invvpid_single() || cpu_has_vmx_invvpid_global())) 7924 enable_vpid = 0; 7925 7926 if (!cpu_has_vmx_ept() || 7927 !cpu_has_vmx_ept_4levels() || 7928 !cpu_has_vmx_ept_mt_wb() || 7929 !cpu_has_vmx_invept_global()) 7930 enable_ept = 0; 7931 7932 /* NX support is required for shadow paging. */ 7933 if (!enable_ept && !boot_cpu_has(X86_FEATURE_NX)) { 7934 pr_err_ratelimited("kvm: NX (Execute Disable) not supported\n"); 7935 return -EOPNOTSUPP; 7936 } 7937 7938 if (!cpu_has_vmx_ept_ad_bits() || !enable_ept) 7939 enable_ept_ad_bits = 0; 7940 7941 if (!cpu_has_vmx_unrestricted_guest() || !enable_ept) 7942 enable_unrestricted_guest = 0; 7943 7944 if (!cpu_has_vmx_flexpriority()) 7945 flexpriority_enabled = 0; 7946 7947 if (!cpu_has_virtual_nmis()) 7948 enable_vnmi = 0; 7949 7950 /* 7951 * set_apic_access_page_addr() is used to reload apic access 7952 * page upon invalidation. No need to do anything if not 7953 * using the APIC_ACCESS_ADDR VMCS field. 7954 */ 7955 if (!flexpriority_enabled) 7956 vmx_x86_ops.set_apic_access_page_addr = NULL; 7957 7958 if (!cpu_has_vmx_tpr_shadow()) 7959 vmx_x86_ops.update_cr8_intercept = NULL; 7960 7961 #if IS_ENABLED(CONFIG_HYPERV) 7962 if (ms_hyperv.nested_features & HV_X64_NESTED_GUEST_MAPPING_FLUSH 7963 && enable_ept) { 7964 vmx_x86_ops.tlb_remote_flush = hv_remote_flush_tlb; 7965 vmx_x86_ops.tlb_remote_flush_with_range = 7966 hv_remote_flush_tlb_with_range; 7967 } 7968 #endif 7969 7970 if (!cpu_has_vmx_ple()) { 7971 ple_gap = 0; 7972 ple_window = 0; 7973 ple_window_grow = 0; 7974 ple_window_max = 0; 7975 ple_window_shrink = 0; 7976 } 7977 7978 if (!cpu_has_vmx_apicv()) 7979 enable_apicv = 0; 7980 if (!enable_apicv) 7981 vmx_x86_ops.sync_pir_to_irr = NULL; 7982 7983 if (cpu_has_vmx_tsc_scaling()) 7984 kvm_has_tsc_control = true; 7985 7986 kvm_max_tsc_scaling_ratio = KVM_VMX_TSC_MULTIPLIER_MAX; 7987 kvm_tsc_scaling_ratio_frac_bits = 48; 7988 kvm_has_bus_lock_exit = cpu_has_vmx_bus_lock_detection(); 7989 7990 set_bit(0, vmx_vpid_bitmap); /* 0 is reserved for host */ 7991 7992 if (enable_ept) 7993 kvm_mmu_set_ept_masks(enable_ept_ad_bits, 7994 cpu_has_vmx_ept_execute_only()); 7995 7996 kvm_configure_mmu(enable_ept, 0, vmx_get_max_tdp_level(), 7997 ept_caps_to_lpage_level(vmx_capability.ept)); 7998 7999 /* 8000 * Only enable PML when hardware supports PML feature, and both EPT 8001 * and EPT A/D bit features are enabled -- PML depends on them to work. 8002 */ 8003 if (!enable_ept || !enable_ept_ad_bits || !cpu_has_vmx_pml()) 8004 enable_pml = 0; 8005 8006 if (!enable_pml) 8007 vmx_x86_ops.cpu_dirty_log_size = 0; 8008 8009 if (!cpu_has_vmx_preemption_timer()) 8010 enable_preemption_timer = false; 8011 8012 if (enable_preemption_timer) { 8013 u64 use_timer_freq = 5000ULL * 1000 * 1000; 8014 u64 vmx_msr; 8015 8016 rdmsrl(MSR_IA32_VMX_MISC, vmx_msr); 8017 cpu_preemption_timer_multi = 8018 vmx_msr & VMX_MISC_PREEMPTION_TIMER_RATE_MASK; 8019 8020 if (tsc_khz) 8021 use_timer_freq = (u64)tsc_khz * 1000; 8022 use_timer_freq >>= cpu_preemption_timer_multi; 8023 8024 /* 8025 * KVM "disables" the preemption timer by setting it to its max 8026 * value. Don't use the timer if it might cause spurious exits 8027 * at a rate faster than 0.1 Hz (of uninterrupted guest time). 8028 */ 8029 if (use_timer_freq > 0xffffffffu / 10) 8030 enable_preemption_timer = false; 8031 } 8032 8033 if (!enable_preemption_timer) { 8034 vmx_x86_ops.set_hv_timer = NULL; 8035 vmx_x86_ops.cancel_hv_timer = NULL; 8036 vmx_x86_ops.request_immediate_exit = __kvm_request_immediate_exit; 8037 } 8038 8039 kvm_mce_cap_supported |= MCG_LMCE_P; 8040 8041 if (pt_mode != PT_MODE_SYSTEM && pt_mode != PT_MODE_HOST_GUEST) 8042 return -EINVAL; 8043 if (!enable_ept || !cpu_has_vmx_intel_pt()) 8044 pt_mode = PT_MODE_SYSTEM; 8045 if (pt_mode == PT_MODE_HOST_GUEST) 8046 vmx_init_ops.handle_intel_pt_intr = vmx_handle_intel_pt_intr; 8047 else 8048 vmx_init_ops.handle_intel_pt_intr = NULL; 8049 8050 setup_default_sgx_lepubkeyhash(); 8051 8052 if (nested) { 8053 nested_vmx_setup_ctls_msrs(&vmcs_config.nested, 8054 vmx_capability.ept); 8055 8056 r = nested_vmx_hardware_setup(kvm_vmx_exit_handlers); 8057 if (r) 8058 return r; 8059 } 8060 8061 vmx_set_cpu_caps(); 8062 8063 r = alloc_kvm_area(); 8064 if (r && nested) 8065 nested_vmx_hardware_unsetup(); 8066 8067 kvm_set_posted_intr_wakeup_handler(pi_wakeup_handler); 8068 8069 return r; 8070 } 8071 8072 static struct kvm_x86_init_ops vmx_init_ops __initdata = { 8073 .cpu_has_kvm_support = cpu_has_kvm_support, 8074 .disabled_by_bios = vmx_disabled_by_bios, 8075 .check_processor_compatibility = vmx_check_processor_compat, 8076 .hardware_setup = hardware_setup, 8077 .handle_intel_pt_intr = NULL, 8078 8079 .runtime_ops = &vmx_x86_ops, 8080 }; 8081 8082 static void vmx_cleanup_l1d_flush(void) 8083 { 8084 if (vmx_l1d_flush_pages) { 8085 free_pages((unsigned long)vmx_l1d_flush_pages, L1D_CACHE_ORDER); 8086 vmx_l1d_flush_pages = NULL; 8087 } 8088 /* Restore state so sysfs ignores VMX */ 8089 l1tf_vmx_mitigation = VMENTER_L1D_FLUSH_AUTO; 8090 } 8091 8092 static void vmx_exit(void) 8093 { 8094 #ifdef CONFIG_KEXEC_CORE 8095 RCU_INIT_POINTER(crash_vmclear_loaded_vmcss, NULL); 8096 synchronize_rcu(); 8097 #endif 8098 8099 kvm_exit(); 8100 8101 #if IS_ENABLED(CONFIG_HYPERV) 8102 if (static_branch_unlikely(&enable_evmcs)) { 8103 int cpu; 8104 struct hv_vp_assist_page *vp_ap; 8105 /* 8106 * Reset everything to support using non-enlightened VMCS 8107 * access later (e.g. when we reload the module with 8108 * enlightened_vmcs=0) 8109 */ 8110 for_each_online_cpu(cpu) { 8111 vp_ap = hv_get_vp_assist_page(cpu); 8112 8113 if (!vp_ap) 8114 continue; 8115 8116 vp_ap->nested_control.features.directhypercall = 0; 8117 vp_ap->current_nested_vmcs = 0; 8118 vp_ap->enlighten_vmentry = 0; 8119 } 8120 8121 static_branch_disable(&enable_evmcs); 8122 } 8123 #endif 8124 vmx_cleanup_l1d_flush(); 8125 8126 allow_smaller_maxphyaddr = false; 8127 } 8128 module_exit(vmx_exit); 8129 8130 static int __init vmx_init(void) 8131 { 8132 int r, cpu; 8133 8134 #if IS_ENABLED(CONFIG_HYPERV) 8135 /* 8136 * Enlightened VMCS usage should be recommended and the host needs 8137 * to support eVMCS v1 or above. We can also disable eVMCS support 8138 * with module parameter. 8139 */ 8140 if (enlightened_vmcs && 8141 ms_hyperv.hints & HV_X64_ENLIGHTENED_VMCS_RECOMMENDED && 8142 (ms_hyperv.nested_features & HV_X64_ENLIGHTENED_VMCS_VERSION) >= 8143 KVM_EVMCS_VERSION) { 8144 8145 /* Check that we have assist pages on all online CPUs */ 8146 for_each_online_cpu(cpu) { 8147 if (!hv_get_vp_assist_page(cpu)) { 8148 enlightened_vmcs = false; 8149 break; 8150 } 8151 } 8152 8153 if (enlightened_vmcs) { 8154 pr_info("KVM: vmx: using Hyper-V Enlightened VMCS\n"); 8155 static_branch_enable(&enable_evmcs); 8156 } 8157 8158 if (ms_hyperv.nested_features & HV_X64_NESTED_DIRECT_FLUSH) 8159 vmx_x86_ops.enable_direct_tlbflush 8160 = hv_enable_direct_tlbflush; 8161 8162 } else { 8163 enlightened_vmcs = false; 8164 } 8165 #endif 8166 8167 r = kvm_init(&vmx_init_ops, sizeof(struct vcpu_vmx), 8168 __alignof__(struct vcpu_vmx), THIS_MODULE); 8169 if (r) 8170 return r; 8171 8172 /* 8173 * Must be called after kvm_init() so enable_ept is properly set 8174 * up. Hand the parameter mitigation value in which was stored in 8175 * the pre module init parser. If no parameter was given, it will 8176 * contain 'auto' which will be turned into the default 'cond' 8177 * mitigation mode. 8178 */ 8179 r = vmx_setup_l1d_flush(vmentry_l1d_flush_param); 8180 if (r) { 8181 vmx_exit(); 8182 return r; 8183 } 8184 8185 for_each_possible_cpu(cpu) { 8186 INIT_LIST_HEAD(&per_cpu(loaded_vmcss_on_cpu, cpu)); 8187 8188 pi_init_cpu(cpu); 8189 } 8190 8191 #ifdef CONFIG_KEXEC_CORE 8192 rcu_assign_pointer(crash_vmclear_loaded_vmcss, 8193 crash_vmclear_local_loaded_vmcss); 8194 #endif 8195 vmx_check_vmcs12_offsets(); 8196 8197 /* 8198 * Shadow paging doesn't have a (further) performance penalty 8199 * from GUEST_MAXPHYADDR < HOST_MAXPHYADDR so enable it 8200 * by default 8201 */ 8202 if (!enable_ept) 8203 allow_smaller_maxphyaddr = true; 8204 8205 return 0; 8206 } 8207 module_init(vmx_init); 8208