1 // SPDX-License-Identifier: GPL-2.0 2 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt 3 4 #include <linux/objtool.h> 5 #include <linux/percpu.h> 6 7 #include <asm/debugreg.h> 8 #include <asm/mmu_context.h> 9 10 #include "cpuid.h" 11 #include "hyperv.h" 12 #include "mmu.h" 13 #include "nested.h" 14 #include "pmu.h" 15 #include "sgx.h" 16 #include "trace.h" 17 #include "vmx.h" 18 #include "x86.h" 19 #include "smm.h" 20 21 static bool __read_mostly enable_shadow_vmcs = 1; 22 module_param_named(enable_shadow_vmcs, enable_shadow_vmcs, bool, S_IRUGO); 23 24 static bool __read_mostly nested_early_check = 0; 25 module_param(nested_early_check, bool, S_IRUGO); 26 27 #define CC KVM_NESTED_VMENTER_CONSISTENCY_CHECK 28 29 /* 30 * Hyper-V requires all of these, so mark them as supported even though 31 * they are just treated the same as all-context. 32 */ 33 #define VMX_VPID_EXTENT_SUPPORTED_MASK \ 34 (VMX_VPID_EXTENT_INDIVIDUAL_ADDR_BIT | \ 35 VMX_VPID_EXTENT_SINGLE_CONTEXT_BIT | \ 36 VMX_VPID_EXTENT_GLOBAL_CONTEXT_BIT | \ 37 VMX_VPID_EXTENT_SINGLE_NON_GLOBAL_BIT) 38 39 #define VMX_MISC_EMULATED_PREEMPTION_TIMER_RATE 5 40 41 enum { 42 VMX_VMREAD_BITMAP, 43 VMX_VMWRITE_BITMAP, 44 VMX_BITMAP_NR 45 }; 46 static unsigned long *vmx_bitmap[VMX_BITMAP_NR]; 47 48 #define vmx_vmread_bitmap (vmx_bitmap[VMX_VMREAD_BITMAP]) 49 #define vmx_vmwrite_bitmap (vmx_bitmap[VMX_VMWRITE_BITMAP]) 50 51 struct shadow_vmcs_field { 52 u16 encoding; 53 u16 offset; 54 }; 55 static struct shadow_vmcs_field shadow_read_only_fields[] = { 56 #define SHADOW_FIELD_RO(x, y) { x, offsetof(struct vmcs12, y) }, 57 #include "vmcs_shadow_fields.h" 58 }; 59 static int max_shadow_read_only_fields = 60 ARRAY_SIZE(shadow_read_only_fields); 61 62 static struct shadow_vmcs_field shadow_read_write_fields[] = { 63 #define SHADOW_FIELD_RW(x, y) { x, offsetof(struct vmcs12, y) }, 64 #include "vmcs_shadow_fields.h" 65 }; 66 static int max_shadow_read_write_fields = 67 ARRAY_SIZE(shadow_read_write_fields); 68 69 static void init_vmcs_shadow_fields(void) 70 { 71 int i, j; 72 73 memset(vmx_vmread_bitmap, 0xff, PAGE_SIZE); 74 memset(vmx_vmwrite_bitmap, 0xff, PAGE_SIZE); 75 76 for (i = j = 0; i < max_shadow_read_only_fields; i++) { 77 struct shadow_vmcs_field entry = shadow_read_only_fields[i]; 78 u16 field = entry.encoding; 79 80 if (vmcs_field_width(field) == VMCS_FIELD_WIDTH_U64 && 81 (i + 1 == max_shadow_read_only_fields || 82 shadow_read_only_fields[i + 1].encoding != field + 1)) 83 pr_err("Missing field from shadow_read_only_field %x\n", 84 field + 1); 85 86 clear_bit(field, vmx_vmread_bitmap); 87 if (field & 1) 88 #ifdef CONFIG_X86_64 89 continue; 90 #else 91 entry.offset += sizeof(u32); 92 #endif 93 shadow_read_only_fields[j++] = entry; 94 } 95 max_shadow_read_only_fields = j; 96 97 for (i = j = 0; i < max_shadow_read_write_fields; i++) { 98 struct shadow_vmcs_field entry = shadow_read_write_fields[i]; 99 u16 field = entry.encoding; 100 101 if (vmcs_field_width(field) == VMCS_FIELD_WIDTH_U64 && 102 (i + 1 == max_shadow_read_write_fields || 103 shadow_read_write_fields[i + 1].encoding != field + 1)) 104 pr_err("Missing field from shadow_read_write_field %x\n", 105 field + 1); 106 107 WARN_ONCE(field >= GUEST_ES_AR_BYTES && 108 field <= GUEST_TR_AR_BYTES, 109 "Update vmcs12_write_any() to drop reserved bits from AR_BYTES"); 110 111 /* 112 * PML and the preemption timer can be emulated, but the 113 * processor cannot vmwrite to fields that don't exist 114 * on bare metal. 115 */ 116 switch (field) { 117 case GUEST_PML_INDEX: 118 if (!cpu_has_vmx_pml()) 119 continue; 120 break; 121 case VMX_PREEMPTION_TIMER_VALUE: 122 if (!cpu_has_vmx_preemption_timer()) 123 continue; 124 break; 125 case GUEST_INTR_STATUS: 126 if (!cpu_has_vmx_apicv()) 127 continue; 128 break; 129 default: 130 break; 131 } 132 133 clear_bit(field, vmx_vmwrite_bitmap); 134 clear_bit(field, vmx_vmread_bitmap); 135 if (field & 1) 136 #ifdef CONFIG_X86_64 137 continue; 138 #else 139 entry.offset += sizeof(u32); 140 #endif 141 shadow_read_write_fields[j++] = entry; 142 } 143 max_shadow_read_write_fields = j; 144 } 145 146 /* 147 * The following 3 functions, nested_vmx_succeed()/failValid()/failInvalid(), 148 * set the success or error code of an emulated VMX instruction (as specified 149 * by Vol 2B, VMX Instruction Reference, "Conventions"), and skip the emulated 150 * instruction. 151 */ 152 static int nested_vmx_succeed(struct kvm_vcpu *vcpu) 153 { 154 vmx_set_rflags(vcpu, vmx_get_rflags(vcpu) 155 & ~(X86_EFLAGS_CF | X86_EFLAGS_PF | X86_EFLAGS_AF | 156 X86_EFLAGS_ZF | X86_EFLAGS_SF | X86_EFLAGS_OF)); 157 return kvm_skip_emulated_instruction(vcpu); 158 } 159 160 static int nested_vmx_failInvalid(struct kvm_vcpu *vcpu) 161 { 162 vmx_set_rflags(vcpu, (vmx_get_rflags(vcpu) 163 & ~(X86_EFLAGS_PF | X86_EFLAGS_AF | X86_EFLAGS_ZF | 164 X86_EFLAGS_SF | X86_EFLAGS_OF)) 165 | X86_EFLAGS_CF); 166 return kvm_skip_emulated_instruction(vcpu); 167 } 168 169 static int nested_vmx_failValid(struct kvm_vcpu *vcpu, 170 u32 vm_instruction_error) 171 { 172 vmx_set_rflags(vcpu, (vmx_get_rflags(vcpu) 173 & ~(X86_EFLAGS_CF | X86_EFLAGS_PF | X86_EFLAGS_AF | 174 X86_EFLAGS_SF | X86_EFLAGS_OF)) 175 | X86_EFLAGS_ZF); 176 get_vmcs12(vcpu)->vm_instruction_error = vm_instruction_error; 177 /* 178 * We don't need to force sync to shadow VMCS because 179 * VM_INSTRUCTION_ERROR is not shadowed. Enlightened VMCS 'shadows' all 180 * fields and thus must be synced. 181 */ 182 if (to_vmx(vcpu)->nested.hv_evmcs_vmptr != EVMPTR_INVALID) 183 to_vmx(vcpu)->nested.need_vmcs12_to_shadow_sync = true; 184 185 return kvm_skip_emulated_instruction(vcpu); 186 } 187 188 static int nested_vmx_fail(struct kvm_vcpu *vcpu, u32 vm_instruction_error) 189 { 190 struct vcpu_vmx *vmx = to_vmx(vcpu); 191 192 /* 193 * failValid writes the error number to the current VMCS, which 194 * can't be done if there isn't a current VMCS. 195 */ 196 if (vmx->nested.current_vmptr == INVALID_GPA && 197 !evmptr_is_valid(vmx->nested.hv_evmcs_vmptr)) 198 return nested_vmx_failInvalid(vcpu); 199 200 return nested_vmx_failValid(vcpu, vm_instruction_error); 201 } 202 203 static void nested_vmx_abort(struct kvm_vcpu *vcpu, u32 indicator) 204 { 205 /* TODO: not to reset guest simply here. */ 206 kvm_make_request(KVM_REQ_TRIPLE_FAULT, vcpu); 207 pr_debug_ratelimited("nested vmx abort, indicator %d\n", indicator); 208 } 209 210 static inline bool vmx_control_verify(u32 control, u32 low, u32 high) 211 { 212 return fixed_bits_valid(control, low, high); 213 } 214 215 static inline u64 vmx_control_msr(u32 low, u32 high) 216 { 217 return low | ((u64)high << 32); 218 } 219 220 static void vmx_disable_shadow_vmcs(struct vcpu_vmx *vmx) 221 { 222 secondary_exec_controls_clearbit(vmx, SECONDARY_EXEC_SHADOW_VMCS); 223 vmcs_write64(VMCS_LINK_POINTER, INVALID_GPA); 224 vmx->nested.need_vmcs12_to_shadow_sync = false; 225 } 226 227 static inline void nested_release_evmcs(struct kvm_vcpu *vcpu) 228 { 229 struct kvm_vcpu_hv *hv_vcpu = to_hv_vcpu(vcpu); 230 struct vcpu_vmx *vmx = to_vmx(vcpu); 231 232 if (evmptr_is_valid(vmx->nested.hv_evmcs_vmptr)) { 233 kvm_vcpu_unmap(vcpu, &vmx->nested.hv_evmcs_map, true); 234 vmx->nested.hv_evmcs = NULL; 235 } 236 237 vmx->nested.hv_evmcs_vmptr = EVMPTR_INVALID; 238 239 if (hv_vcpu) { 240 hv_vcpu->nested.pa_page_gpa = INVALID_GPA; 241 hv_vcpu->nested.vm_id = 0; 242 hv_vcpu->nested.vp_id = 0; 243 } 244 } 245 246 static void vmx_sync_vmcs_host_state(struct vcpu_vmx *vmx, 247 struct loaded_vmcs *prev) 248 { 249 struct vmcs_host_state *dest, *src; 250 251 if (unlikely(!vmx->guest_state_loaded)) 252 return; 253 254 src = &prev->host_state; 255 dest = &vmx->loaded_vmcs->host_state; 256 257 vmx_set_host_fs_gs(dest, src->fs_sel, src->gs_sel, src->fs_base, src->gs_base); 258 dest->ldt_sel = src->ldt_sel; 259 #ifdef CONFIG_X86_64 260 dest->ds_sel = src->ds_sel; 261 dest->es_sel = src->es_sel; 262 #endif 263 } 264 265 static void vmx_switch_vmcs(struct kvm_vcpu *vcpu, struct loaded_vmcs *vmcs) 266 { 267 struct vcpu_vmx *vmx = to_vmx(vcpu); 268 struct loaded_vmcs *prev; 269 int cpu; 270 271 if (WARN_ON_ONCE(vmx->loaded_vmcs == vmcs)) 272 return; 273 274 cpu = get_cpu(); 275 prev = vmx->loaded_vmcs; 276 vmx->loaded_vmcs = vmcs; 277 vmx_vcpu_load_vmcs(vcpu, cpu, prev); 278 vmx_sync_vmcs_host_state(vmx, prev); 279 put_cpu(); 280 281 vcpu->arch.regs_avail = ~VMX_REGS_LAZY_LOAD_SET; 282 283 /* 284 * All lazily updated registers will be reloaded from VMCS12 on both 285 * vmentry and vmexit. 286 */ 287 vcpu->arch.regs_dirty = 0; 288 } 289 290 /* 291 * Free whatever needs to be freed from vmx->nested when L1 goes down, or 292 * just stops using VMX. 293 */ 294 static void free_nested(struct kvm_vcpu *vcpu) 295 { 296 struct vcpu_vmx *vmx = to_vmx(vcpu); 297 298 if (WARN_ON_ONCE(vmx->loaded_vmcs != &vmx->vmcs01)) 299 vmx_switch_vmcs(vcpu, &vmx->vmcs01); 300 301 if (!vmx->nested.vmxon && !vmx->nested.smm.vmxon) 302 return; 303 304 kvm_clear_request(KVM_REQ_GET_NESTED_STATE_PAGES, vcpu); 305 306 vmx->nested.vmxon = false; 307 vmx->nested.smm.vmxon = false; 308 vmx->nested.vmxon_ptr = INVALID_GPA; 309 free_vpid(vmx->nested.vpid02); 310 vmx->nested.posted_intr_nv = -1; 311 vmx->nested.current_vmptr = INVALID_GPA; 312 if (enable_shadow_vmcs) { 313 vmx_disable_shadow_vmcs(vmx); 314 vmcs_clear(vmx->vmcs01.shadow_vmcs); 315 free_vmcs(vmx->vmcs01.shadow_vmcs); 316 vmx->vmcs01.shadow_vmcs = NULL; 317 } 318 kfree(vmx->nested.cached_vmcs12); 319 vmx->nested.cached_vmcs12 = NULL; 320 kfree(vmx->nested.cached_shadow_vmcs12); 321 vmx->nested.cached_shadow_vmcs12 = NULL; 322 /* 323 * Unpin physical memory we referred to in the vmcs02. The APIC access 324 * page's backing page (yeah, confusing) shouldn't actually be accessed, 325 * and if it is written, the contents are irrelevant. 326 */ 327 kvm_vcpu_unmap(vcpu, &vmx->nested.apic_access_page_map, false); 328 kvm_vcpu_unmap(vcpu, &vmx->nested.virtual_apic_map, true); 329 kvm_vcpu_unmap(vcpu, &vmx->nested.pi_desc_map, true); 330 vmx->nested.pi_desc = NULL; 331 332 kvm_mmu_free_roots(vcpu->kvm, &vcpu->arch.guest_mmu, KVM_MMU_ROOTS_ALL); 333 334 nested_release_evmcs(vcpu); 335 336 free_loaded_vmcs(&vmx->nested.vmcs02); 337 } 338 339 /* 340 * Ensure that the current vmcs of the logical processor is the 341 * vmcs01 of the vcpu before calling free_nested(). 342 */ 343 void nested_vmx_free_vcpu(struct kvm_vcpu *vcpu) 344 { 345 vcpu_load(vcpu); 346 vmx_leave_nested(vcpu); 347 vcpu_put(vcpu); 348 } 349 350 #define EPTP_PA_MASK GENMASK_ULL(51, 12) 351 352 static bool nested_ept_root_matches(hpa_t root_hpa, u64 root_eptp, u64 eptp) 353 { 354 return VALID_PAGE(root_hpa) && 355 ((root_eptp & EPTP_PA_MASK) == (eptp & EPTP_PA_MASK)); 356 } 357 358 static void nested_ept_invalidate_addr(struct kvm_vcpu *vcpu, gpa_t eptp, 359 gpa_t addr) 360 { 361 uint i; 362 struct kvm_mmu_root_info *cached_root; 363 364 WARN_ON_ONCE(!mmu_is_nested(vcpu)); 365 366 for (i = 0; i < KVM_MMU_NUM_PREV_ROOTS; i++) { 367 cached_root = &vcpu->arch.mmu->prev_roots[i]; 368 369 if (nested_ept_root_matches(cached_root->hpa, cached_root->pgd, 370 eptp)) 371 vcpu->arch.mmu->invlpg(vcpu, addr, cached_root->hpa); 372 } 373 } 374 375 static void nested_ept_inject_page_fault(struct kvm_vcpu *vcpu, 376 struct x86_exception *fault) 377 { 378 struct vmcs12 *vmcs12 = get_vmcs12(vcpu); 379 struct vcpu_vmx *vmx = to_vmx(vcpu); 380 u32 vm_exit_reason; 381 unsigned long exit_qualification = vcpu->arch.exit_qualification; 382 383 if (vmx->nested.pml_full) { 384 vm_exit_reason = EXIT_REASON_PML_FULL; 385 vmx->nested.pml_full = false; 386 exit_qualification &= INTR_INFO_UNBLOCK_NMI; 387 } else { 388 if (fault->error_code & PFERR_RSVD_MASK) 389 vm_exit_reason = EXIT_REASON_EPT_MISCONFIG; 390 else 391 vm_exit_reason = EXIT_REASON_EPT_VIOLATION; 392 393 /* 394 * Although the caller (kvm_inject_emulated_page_fault) would 395 * have already synced the faulting address in the shadow EPT 396 * tables for the current EPTP12, we also need to sync it for 397 * any other cached EPTP02s based on the same EP4TA, since the 398 * TLB associates mappings to the EP4TA rather than the full EPTP. 399 */ 400 nested_ept_invalidate_addr(vcpu, vmcs12->ept_pointer, 401 fault->address); 402 } 403 404 nested_vmx_vmexit(vcpu, vm_exit_reason, 0, exit_qualification); 405 vmcs12->guest_physical_address = fault->address; 406 } 407 408 static void nested_ept_new_eptp(struct kvm_vcpu *vcpu) 409 { 410 struct vcpu_vmx *vmx = to_vmx(vcpu); 411 bool execonly = vmx->nested.msrs.ept_caps & VMX_EPT_EXECUTE_ONLY_BIT; 412 int ept_lpage_level = ept_caps_to_lpage_level(vmx->nested.msrs.ept_caps); 413 414 kvm_init_shadow_ept_mmu(vcpu, execonly, ept_lpage_level, 415 nested_ept_ad_enabled(vcpu), 416 nested_ept_get_eptp(vcpu)); 417 } 418 419 static void nested_ept_init_mmu_context(struct kvm_vcpu *vcpu) 420 { 421 WARN_ON(mmu_is_nested(vcpu)); 422 423 vcpu->arch.mmu = &vcpu->arch.guest_mmu; 424 nested_ept_new_eptp(vcpu); 425 vcpu->arch.mmu->get_guest_pgd = nested_ept_get_eptp; 426 vcpu->arch.mmu->inject_page_fault = nested_ept_inject_page_fault; 427 vcpu->arch.mmu->get_pdptr = kvm_pdptr_read; 428 429 vcpu->arch.walk_mmu = &vcpu->arch.nested_mmu; 430 } 431 432 static void nested_ept_uninit_mmu_context(struct kvm_vcpu *vcpu) 433 { 434 vcpu->arch.mmu = &vcpu->arch.root_mmu; 435 vcpu->arch.walk_mmu = &vcpu->arch.root_mmu; 436 } 437 438 static bool nested_vmx_is_page_fault_vmexit(struct vmcs12 *vmcs12, 439 u16 error_code) 440 { 441 bool inequality, bit; 442 443 bit = (vmcs12->exception_bitmap & (1u << PF_VECTOR)) != 0; 444 inequality = 445 (error_code & vmcs12->page_fault_error_code_mask) != 446 vmcs12->page_fault_error_code_match; 447 return inequality ^ bit; 448 } 449 450 static bool nested_vmx_is_exception_vmexit(struct kvm_vcpu *vcpu, u8 vector, 451 u32 error_code) 452 { 453 struct vmcs12 *vmcs12 = get_vmcs12(vcpu); 454 455 /* 456 * Drop bits 31:16 of the error code when performing the #PF mask+match 457 * check. All VMCS fields involved are 32 bits, but Intel CPUs never 458 * set bits 31:16 and VMX disallows setting bits 31:16 in the injected 459 * error code. Including the to-be-dropped bits in the check might 460 * result in an "impossible" or missed exit from L1's perspective. 461 */ 462 if (vector == PF_VECTOR) 463 return nested_vmx_is_page_fault_vmexit(vmcs12, (u16)error_code); 464 465 return (vmcs12->exception_bitmap & (1u << vector)); 466 } 467 468 static int nested_vmx_check_io_bitmap_controls(struct kvm_vcpu *vcpu, 469 struct vmcs12 *vmcs12) 470 { 471 if (!nested_cpu_has(vmcs12, CPU_BASED_USE_IO_BITMAPS)) 472 return 0; 473 474 if (CC(!page_address_valid(vcpu, vmcs12->io_bitmap_a)) || 475 CC(!page_address_valid(vcpu, vmcs12->io_bitmap_b))) 476 return -EINVAL; 477 478 return 0; 479 } 480 481 static int nested_vmx_check_msr_bitmap_controls(struct kvm_vcpu *vcpu, 482 struct vmcs12 *vmcs12) 483 { 484 if (!nested_cpu_has(vmcs12, CPU_BASED_USE_MSR_BITMAPS)) 485 return 0; 486 487 if (CC(!page_address_valid(vcpu, vmcs12->msr_bitmap))) 488 return -EINVAL; 489 490 return 0; 491 } 492 493 static int nested_vmx_check_tpr_shadow_controls(struct kvm_vcpu *vcpu, 494 struct vmcs12 *vmcs12) 495 { 496 if (!nested_cpu_has(vmcs12, CPU_BASED_TPR_SHADOW)) 497 return 0; 498 499 if (CC(!page_address_valid(vcpu, vmcs12->virtual_apic_page_addr))) 500 return -EINVAL; 501 502 return 0; 503 } 504 505 /* 506 * For x2APIC MSRs, ignore the vmcs01 bitmap. L1 can enable x2APIC without L1 507 * itself utilizing x2APIC. All MSRs were previously set to be intercepted, 508 * only the "disable intercept" case needs to be handled. 509 */ 510 static void nested_vmx_disable_intercept_for_x2apic_msr(unsigned long *msr_bitmap_l1, 511 unsigned long *msr_bitmap_l0, 512 u32 msr, int type) 513 { 514 if (type & MSR_TYPE_R && !vmx_test_msr_bitmap_read(msr_bitmap_l1, msr)) 515 vmx_clear_msr_bitmap_read(msr_bitmap_l0, msr); 516 517 if (type & MSR_TYPE_W && !vmx_test_msr_bitmap_write(msr_bitmap_l1, msr)) 518 vmx_clear_msr_bitmap_write(msr_bitmap_l0, msr); 519 } 520 521 static inline void enable_x2apic_msr_intercepts(unsigned long *msr_bitmap) 522 { 523 int msr; 524 525 for (msr = 0x800; msr <= 0x8ff; msr += BITS_PER_LONG) { 526 unsigned word = msr / BITS_PER_LONG; 527 528 msr_bitmap[word] = ~0; 529 msr_bitmap[word + (0x800 / sizeof(long))] = ~0; 530 } 531 } 532 533 #define BUILD_NVMX_MSR_INTERCEPT_HELPER(rw) \ 534 static inline \ 535 void nested_vmx_set_msr_##rw##_intercept(struct vcpu_vmx *vmx, \ 536 unsigned long *msr_bitmap_l1, \ 537 unsigned long *msr_bitmap_l0, u32 msr) \ 538 { \ 539 if (vmx_test_msr_bitmap_##rw(vmx->vmcs01.msr_bitmap, msr) || \ 540 vmx_test_msr_bitmap_##rw(msr_bitmap_l1, msr)) \ 541 vmx_set_msr_bitmap_##rw(msr_bitmap_l0, msr); \ 542 else \ 543 vmx_clear_msr_bitmap_##rw(msr_bitmap_l0, msr); \ 544 } 545 BUILD_NVMX_MSR_INTERCEPT_HELPER(read) 546 BUILD_NVMX_MSR_INTERCEPT_HELPER(write) 547 548 static inline void nested_vmx_set_intercept_for_msr(struct vcpu_vmx *vmx, 549 unsigned long *msr_bitmap_l1, 550 unsigned long *msr_bitmap_l0, 551 u32 msr, int types) 552 { 553 if (types & MSR_TYPE_R) 554 nested_vmx_set_msr_read_intercept(vmx, msr_bitmap_l1, 555 msr_bitmap_l0, msr); 556 if (types & MSR_TYPE_W) 557 nested_vmx_set_msr_write_intercept(vmx, msr_bitmap_l1, 558 msr_bitmap_l0, msr); 559 } 560 561 /* 562 * Merge L0's and L1's MSR bitmap, return false to indicate that 563 * we do not use the hardware. 564 */ 565 static inline bool nested_vmx_prepare_msr_bitmap(struct kvm_vcpu *vcpu, 566 struct vmcs12 *vmcs12) 567 { 568 struct vcpu_vmx *vmx = to_vmx(vcpu); 569 int msr; 570 unsigned long *msr_bitmap_l1; 571 unsigned long *msr_bitmap_l0 = vmx->nested.vmcs02.msr_bitmap; 572 struct hv_enlightened_vmcs *evmcs = vmx->nested.hv_evmcs; 573 struct kvm_host_map *map = &vmx->nested.msr_bitmap_map; 574 575 /* Nothing to do if the MSR bitmap is not in use. */ 576 if (!cpu_has_vmx_msr_bitmap() || 577 !nested_cpu_has(vmcs12, CPU_BASED_USE_MSR_BITMAPS)) 578 return false; 579 580 /* 581 * MSR bitmap update can be skipped when: 582 * - MSR bitmap for L1 hasn't changed. 583 * - Nested hypervisor (L1) is attempting to launch the same L2 as 584 * before. 585 * - Nested hypervisor (L1) has enabled 'Enlightened MSR Bitmap' feature 586 * and tells KVM (L0) there were no changes in MSR bitmap for L2. 587 */ 588 if (!vmx->nested.force_msr_bitmap_recalc && evmcs && 589 evmcs->hv_enlightenments_control.msr_bitmap && 590 evmcs->hv_clean_fields & HV_VMX_ENLIGHTENED_CLEAN_FIELD_MSR_BITMAP) 591 return true; 592 593 if (kvm_vcpu_map(vcpu, gpa_to_gfn(vmcs12->msr_bitmap), map)) 594 return false; 595 596 msr_bitmap_l1 = (unsigned long *)map->hva; 597 598 /* 599 * To keep the control flow simple, pay eight 8-byte writes (sixteen 600 * 4-byte writes on 32-bit systems) up front to enable intercepts for 601 * the x2APIC MSR range and selectively toggle those relevant to L2. 602 */ 603 enable_x2apic_msr_intercepts(msr_bitmap_l0); 604 605 if (nested_cpu_has_virt_x2apic_mode(vmcs12)) { 606 if (nested_cpu_has_apic_reg_virt(vmcs12)) { 607 /* 608 * L0 need not intercept reads for MSRs between 0x800 609 * and 0x8ff, it just lets the processor take the value 610 * from the virtual-APIC page; take those 256 bits 611 * directly from the L1 bitmap. 612 */ 613 for (msr = 0x800; msr <= 0x8ff; msr += BITS_PER_LONG) { 614 unsigned word = msr / BITS_PER_LONG; 615 616 msr_bitmap_l0[word] = msr_bitmap_l1[word]; 617 } 618 } 619 620 nested_vmx_disable_intercept_for_x2apic_msr( 621 msr_bitmap_l1, msr_bitmap_l0, 622 X2APIC_MSR(APIC_TASKPRI), 623 MSR_TYPE_R | MSR_TYPE_W); 624 625 if (nested_cpu_has_vid(vmcs12)) { 626 nested_vmx_disable_intercept_for_x2apic_msr( 627 msr_bitmap_l1, msr_bitmap_l0, 628 X2APIC_MSR(APIC_EOI), 629 MSR_TYPE_W); 630 nested_vmx_disable_intercept_for_x2apic_msr( 631 msr_bitmap_l1, msr_bitmap_l0, 632 X2APIC_MSR(APIC_SELF_IPI), 633 MSR_TYPE_W); 634 } 635 } 636 637 /* 638 * Always check vmcs01's bitmap to honor userspace MSR filters and any 639 * other runtime changes to vmcs01's bitmap, e.g. dynamic pass-through. 640 */ 641 #ifdef CONFIG_X86_64 642 nested_vmx_set_intercept_for_msr(vmx, msr_bitmap_l1, msr_bitmap_l0, 643 MSR_FS_BASE, MSR_TYPE_RW); 644 645 nested_vmx_set_intercept_for_msr(vmx, msr_bitmap_l1, msr_bitmap_l0, 646 MSR_GS_BASE, MSR_TYPE_RW); 647 648 nested_vmx_set_intercept_for_msr(vmx, msr_bitmap_l1, msr_bitmap_l0, 649 MSR_KERNEL_GS_BASE, MSR_TYPE_RW); 650 #endif 651 nested_vmx_set_intercept_for_msr(vmx, msr_bitmap_l1, msr_bitmap_l0, 652 MSR_IA32_SPEC_CTRL, MSR_TYPE_RW); 653 654 nested_vmx_set_intercept_for_msr(vmx, msr_bitmap_l1, msr_bitmap_l0, 655 MSR_IA32_PRED_CMD, MSR_TYPE_W); 656 657 kvm_vcpu_unmap(vcpu, &vmx->nested.msr_bitmap_map, false); 658 659 vmx->nested.force_msr_bitmap_recalc = false; 660 661 return true; 662 } 663 664 static void nested_cache_shadow_vmcs12(struct kvm_vcpu *vcpu, 665 struct vmcs12 *vmcs12) 666 { 667 struct vcpu_vmx *vmx = to_vmx(vcpu); 668 struct gfn_to_hva_cache *ghc = &vmx->nested.shadow_vmcs12_cache; 669 670 if (!nested_cpu_has_shadow_vmcs(vmcs12) || 671 vmcs12->vmcs_link_pointer == INVALID_GPA) 672 return; 673 674 if (ghc->gpa != vmcs12->vmcs_link_pointer && 675 kvm_gfn_to_hva_cache_init(vcpu->kvm, ghc, 676 vmcs12->vmcs_link_pointer, VMCS12_SIZE)) 677 return; 678 679 kvm_read_guest_cached(vmx->vcpu.kvm, ghc, get_shadow_vmcs12(vcpu), 680 VMCS12_SIZE); 681 } 682 683 static void nested_flush_cached_shadow_vmcs12(struct kvm_vcpu *vcpu, 684 struct vmcs12 *vmcs12) 685 { 686 struct vcpu_vmx *vmx = to_vmx(vcpu); 687 struct gfn_to_hva_cache *ghc = &vmx->nested.shadow_vmcs12_cache; 688 689 if (!nested_cpu_has_shadow_vmcs(vmcs12) || 690 vmcs12->vmcs_link_pointer == INVALID_GPA) 691 return; 692 693 if (ghc->gpa != vmcs12->vmcs_link_pointer && 694 kvm_gfn_to_hva_cache_init(vcpu->kvm, ghc, 695 vmcs12->vmcs_link_pointer, VMCS12_SIZE)) 696 return; 697 698 kvm_write_guest_cached(vmx->vcpu.kvm, ghc, get_shadow_vmcs12(vcpu), 699 VMCS12_SIZE); 700 } 701 702 /* 703 * In nested virtualization, check if L1 has set 704 * VM_EXIT_ACK_INTR_ON_EXIT 705 */ 706 static bool nested_exit_intr_ack_set(struct kvm_vcpu *vcpu) 707 { 708 return get_vmcs12(vcpu)->vm_exit_controls & 709 VM_EXIT_ACK_INTR_ON_EXIT; 710 } 711 712 static int nested_vmx_check_apic_access_controls(struct kvm_vcpu *vcpu, 713 struct vmcs12 *vmcs12) 714 { 715 if (nested_cpu_has2(vmcs12, SECONDARY_EXEC_VIRTUALIZE_APIC_ACCESSES) && 716 CC(!page_address_valid(vcpu, vmcs12->apic_access_addr))) 717 return -EINVAL; 718 else 719 return 0; 720 } 721 722 static int nested_vmx_check_apicv_controls(struct kvm_vcpu *vcpu, 723 struct vmcs12 *vmcs12) 724 { 725 if (!nested_cpu_has_virt_x2apic_mode(vmcs12) && 726 !nested_cpu_has_apic_reg_virt(vmcs12) && 727 !nested_cpu_has_vid(vmcs12) && 728 !nested_cpu_has_posted_intr(vmcs12)) 729 return 0; 730 731 /* 732 * If virtualize x2apic mode is enabled, 733 * virtualize apic access must be disabled. 734 */ 735 if (CC(nested_cpu_has_virt_x2apic_mode(vmcs12) && 736 nested_cpu_has2(vmcs12, SECONDARY_EXEC_VIRTUALIZE_APIC_ACCESSES))) 737 return -EINVAL; 738 739 /* 740 * If virtual interrupt delivery is enabled, 741 * we must exit on external interrupts. 742 */ 743 if (CC(nested_cpu_has_vid(vmcs12) && !nested_exit_on_intr(vcpu))) 744 return -EINVAL; 745 746 /* 747 * bits 15:8 should be zero in posted_intr_nv, 748 * the descriptor address has been already checked 749 * in nested_get_vmcs12_pages. 750 * 751 * bits 5:0 of posted_intr_desc_addr should be zero. 752 */ 753 if (nested_cpu_has_posted_intr(vmcs12) && 754 (CC(!nested_cpu_has_vid(vmcs12)) || 755 CC(!nested_exit_intr_ack_set(vcpu)) || 756 CC((vmcs12->posted_intr_nv & 0xff00)) || 757 CC(!kvm_vcpu_is_legal_aligned_gpa(vcpu, vmcs12->posted_intr_desc_addr, 64)))) 758 return -EINVAL; 759 760 /* tpr shadow is needed by all apicv features. */ 761 if (CC(!nested_cpu_has(vmcs12, CPU_BASED_TPR_SHADOW))) 762 return -EINVAL; 763 764 return 0; 765 } 766 767 static int nested_vmx_check_msr_switch(struct kvm_vcpu *vcpu, 768 u32 count, u64 addr) 769 { 770 if (count == 0) 771 return 0; 772 773 if (!kvm_vcpu_is_legal_aligned_gpa(vcpu, addr, 16) || 774 !kvm_vcpu_is_legal_gpa(vcpu, (addr + count * sizeof(struct vmx_msr_entry) - 1))) 775 return -EINVAL; 776 777 return 0; 778 } 779 780 static int nested_vmx_check_exit_msr_switch_controls(struct kvm_vcpu *vcpu, 781 struct vmcs12 *vmcs12) 782 { 783 if (CC(nested_vmx_check_msr_switch(vcpu, 784 vmcs12->vm_exit_msr_load_count, 785 vmcs12->vm_exit_msr_load_addr)) || 786 CC(nested_vmx_check_msr_switch(vcpu, 787 vmcs12->vm_exit_msr_store_count, 788 vmcs12->vm_exit_msr_store_addr))) 789 return -EINVAL; 790 791 return 0; 792 } 793 794 static int nested_vmx_check_entry_msr_switch_controls(struct kvm_vcpu *vcpu, 795 struct vmcs12 *vmcs12) 796 { 797 if (CC(nested_vmx_check_msr_switch(vcpu, 798 vmcs12->vm_entry_msr_load_count, 799 vmcs12->vm_entry_msr_load_addr))) 800 return -EINVAL; 801 802 return 0; 803 } 804 805 static int nested_vmx_check_pml_controls(struct kvm_vcpu *vcpu, 806 struct vmcs12 *vmcs12) 807 { 808 if (!nested_cpu_has_pml(vmcs12)) 809 return 0; 810 811 if (CC(!nested_cpu_has_ept(vmcs12)) || 812 CC(!page_address_valid(vcpu, vmcs12->pml_address))) 813 return -EINVAL; 814 815 return 0; 816 } 817 818 static int nested_vmx_check_unrestricted_guest_controls(struct kvm_vcpu *vcpu, 819 struct vmcs12 *vmcs12) 820 { 821 if (CC(nested_cpu_has2(vmcs12, SECONDARY_EXEC_UNRESTRICTED_GUEST) && 822 !nested_cpu_has_ept(vmcs12))) 823 return -EINVAL; 824 return 0; 825 } 826 827 static int nested_vmx_check_mode_based_ept_exec_controls(struct kvm_vcpu *vcpu, 828 struct vmcs12 *vmcs12) 829 { 830 if (CC(nested_cpu_has2(vmcs12, SECONDARY_EXEC_MODE_BASED_EPT_EXEC) && 831 !nested_cpu_has_ept(vmcs12))) 832 return -EINVAL; 833 return 0; 834 } 835 836 static int nested_vmx_check_shadow_vmcs_controls(struct kvm_vcpu *vcpu, 837 struct vmcs12 *vmcs12) 838 { 839 if (!nested_cpu_has_shadow_vmcs(vmcs12)) 840 return 0; 841 842 if (CC(!page_address_valid(vcpu, vmcs12->vmread_bitmap)) || 843 CC(!page_address_valid(vcpu, vmcs12->vmwrite_bitmap))) 844 return -EINVAL; 845 846 return 0; 847 } 848 849 static int nested_vmx_msr_check_common(struct kvm_vcpu *vcpu, 850 struct vmx_msr_entry *e) 851 { 852 /* x2APIC MSR accesses are not allowed */ 853 if (CC(vcpu->arch.apic_base & X2APIC_ENABLE && e->index >> 8 == 0x8)) 854 return -EINVAL; 855 if (CC(e->index == MSR_IA32_UCODE_WRITE) || /* SDM Table 35-2 */ 856 CC(e->index == MSR_IA32_UCODE_REV)) 857 return -EINVAL; 858 if (CC(e->reserved != 0)) 859 return -EINVAL; 860 return 0; 861 } 862 863 static int nested_vmx_load_msr_check(struct kvm_vcpu *vcpu, 864 struct vmx_msr_entry *e) 865 { 866 if (CC(e->index == MSR_FS_BASE) || 867 CC(e->index == MSR_GS_BASE) || 868 CC(e->index == MSR_IA32_SMM_MONITOR_CTL) || /* SMM is not supported */ 869 nested_vmx_msr_check_common(vcpu, e)) 870 return -EINVAL; 871 return 0; 872 } 873 874 static int nested_vmx_store_msr_check(struct kvm_vcpu *vcpu, 875 struct vmx_msr_entry *e) 876 { 877 if (CC(e->index == MSR_IA32_SMBASE) || /* SMM is not supported */ 878 nested_vmx_msr_check_common(vcpu, e)) 879 return -EINVAL; 880 return 0; 881 } 882 883 static u32 nested_vmx_max_atomic_switch_msrs(struct kvm_vcpu *vcpu) 884 { 885 struct vcpu_vmx *vmx = to_vmx(vcpu); 886 u64 vmx_misc = vmx_control_msr(vmx->nested.msrs.misc_low, 887 vmx->nested.msrs.misc_high); 888 889 return (vmx_misc_max_msr(vmx_misc) + 1) * VMX_MISC_MSR_LIST_MULTIPLIER; 890 } 891 892 /* 893 * Load guest's/host's msr at nested entry/exit. 894 * return 0 for success, entry index for failure. 895 * 896 * One of the failure modes for MSR load/store is when a list exceeds the 897 * virtual hardware's capacity. To maintain compatibility with hardware inasmuch 898 * as possible, process all valid entries before failing rather than precheck 899 * for a capacity violation. 900 */ 901 static u32 nested_vmx_load_msr(struct kvm_vcpu *vcpu, u64 gpa, u32 count) 902 { 903 u32 i; 904 struct vmx_msr_entry e; 905 u32 max_msr_list_size = nested_vmx_max_atomic_switch_msrs(vcpu); 906 907 for (i = 0; i < count; i++) { 908 if (unlikely(i >= max_msr_list_size)) 909 goto fail; 910 911 if (kvm_vcpu_read_guest(vcpu, gpa + i * sizeof(e), 912 &e, sizeof(e))) { 913 pr_debug_ratelimited( 914 "%s cannot read MSR entry (%u, 0x%08llx)\n", 915 __func__, i, gpa + i * sizeof(e)); 916 goto fail; 917 } 918 if (nested_vmx_load_msr_check(vcpu, &e)) { 919 pr_debug_ratelimited( 920 "%s check failed (%u, 0x%x, 0x%x)\n", 921 __func__, i, e.index, e.reserved); 922 goto fail; 923 } 924 if (kvm_set_msr(vcpu, e.index, e.value)) { 925 pr_debug_ratelimited( 926 "%s cannot write MSR (%u, 0x%x, 0x%llx)\n", 927 __func__, i, e.index, e.value); 928 goto fail; 929 } 930 } 931 return 0; 932 fail: 933 /* Note, max_msr_list_size is at most 4096, i.e. this can't wrap. */ 934 return i + 1; 935 } 936 937 static bool nested_vmx_get_vmexit_msr_value(struct kvm_vcpu *vcpu, 938 u32 msr_index, 939 u64 *data) 940 { 941 struct vcpu_vmx *vmx = to_vmx(vcpu); 942 943 /* 944 * If the L0 hypervisor stored a more accurate value for the TSC that 945 * does not include the time taken for emulation of the L2->L1 946 * VM-exit in L0, use the more accurate value. 947 */ 948 if (msr_index == MSR_IA32_TSC) { 949 int i = vmx_find_loadstore_msr_slot(&vmx->msr_autostore.guest, 950 MSR_IA32_TSC); 951 952 if (i >= 0) { 953 u64 val = vmx->msr_autostore.guest.val[i].value; 954 955 *data = kvm_read_l1_tsc(vcpu, val); 956 return true; 957 } 958 } 959 960 if (kvm_get_msr(vcpu, msr_index, data)) { 961 pr_debug_ratelimited("%s cannot read MSR (0x%x)\n", __func__, 962 msr_index); 963 return false; 964 } 965 return true; 966 } 967 968 static bool read_and_check_msr_entry(struct kvm_vcpu *vcpu, u64 gpa, int i, 969 struct vmx_msr_entry *e) 970 { 971 if (kvm_vcpu_read_guest(vcpu, 972 gpa + i * sizeof(*e), 973 e, 2 * sizeof(u32))) { 974 pr_debug_ratelimited( 975 "%s cannot read MSR entry (%u, 0x%08llx)\n", 976 __func__, i, gpa + i * sizeof(*e)); 977 return false; 978 } 979 if (nested_vmx_store_msr_check(vcpu, e)) { 980 pr_debug_ratelimited( 981 "%s check failed (%u, 0x%x, 0x%x)\n", 982 __func__, i, e->index, e->reserved); 983 return false; 984 } 985 return true; 986 } 987 988 static int nested_vmx_store_msr(struct kvm_vcpu *vcpu, u64 gpa, u32 count) 989 { 990 u64 data; 991 u32 i; 992 struct vmx_msr_entry e; 993 u32 max_msr_list_size = nested_vmx_max_atomic_switch_msrs(vcpu); 994 995 for (i = 0; i < count; i++) { 996 if (unlikely(i >= max_msr_list_size)) 997 return -EINVAL; 998 999 if (!read_and_check_msr_entry(vcpu, gpa, i, &e)) 1000 return -EINVAL; 1001 1002 if (!nested_vmx_get_vmexit_msr_value(vcpu, e.index, &data)) 1003 return -EINVAL; 1004 1005 if (kvm_vcpu_write_guest(vcpu, 1006 gpa + i * sizeof(e) + 1007 offsetof(struct vmx_msr_entry, value), 1008 &data, sizeof(data))) { 1009 pr_debug_ratelimited( 1010 "%s cannot write MSR (%u, 0x%x, 0x%llx)\n", 1011 __func__, i, e.index, data); 1012 return -EINVAL; 1013 } 1014 } 1015 return 0; 1016 } 1017 1018 static bool nested_msr_store_list_has_msr(struct kvm_vcpu *vcpu, u32 msr_index) 1019 { 1020 struct vmcs12 *vmcs12 = get_vmcs12(vcpu); 1021 u32 count = vmcs12->vm_exit_msr_store_count; 1022 u64 gpa = vmcs12->vm_exit_msr_store_addr; 1023 struct vmx_msr_entry e; 1024 u32 i; 1025 1026 for (i = 0; i < count; i++) { 1027 if (!read_and_check_msr_entry(vcpu, gpa, i, &e)) 1028 return false; 1029 1030 if (e.index == msr_index) 1031 return true; 1032 } 1033 return false; 1034 } 1035 1036 static void prepare_vmx_msr_autostore_list(struct kvm_vcpu *vcpu, 1037 u32 msr_index) 1038 { 1039 struct vcpu_vmx *vmx = to_vmx(vcpu); 1040 struct vmx_msrs *autostore = &vmx->msr_autostore.guest; 1041 bool in_vmcs12_store_list; 1042 int msr_autostore_slot; 1043 bool in_autostore_list; 1044 int last; 1045 1046 msr_autostore_slot = vmx_find_loadstore_msr_slot(autostore, msr_index); 1047 in_autostore_list = msr_autostore_slot >= 0; 1048 in_vmcs12_store_list = nested_msr_store_list_has_msr(vcpu, msr_index); 1049 1050 if (in_vmcs12_store_list && !in_autostore_list) { 1051 if (autostore->nr == MAX_NR_LOADSTORE_MSRS) { 1052 /* 1053 * Emulated VMEntry does not fail here. Instead a less 1054 * accurate value will be returned by 1055 * nested_vmx_get_vmexit_msr_value() using kvm_get_msr() 1056 * instead of reading the value from the vmcs02 VMExit 1057 * MSR-store area. 1058 */ 1059 pr_warn_ratelimited( 1060 "Not enough msr entries in msr_autostore. Can't add msr %x\n", 1061 msr_index); 1062 return; 1063 } 1064 last = autostore->nr++; 1065 autostore->val[last].index = msr_index; 1066 } else if (!in_vmcs12_store_list && in_autostore_list) { 1067 last = --autostore->nr; 1068 autostore->val[msr_autostore_slot] = autostore->val[last]; 1069 } 1070 } 1071 1072 /* 1073 * Load guest's/host's cr3 at nested entry/exit. @nested_ept is true if we are 1074 * emulating VM-Entry into a guest with EPT enabled. On failure, the expected 1075 * Exit Qualification (for a VM-Entry consistency check VM-Exit) is assigned to 1076 * @entry_failure_code. 1077 */ 1078 static int nested_vmx_load_cr3(struct kvm_vcpu *vcpu, unsigned long cr3, 1079 bool nested_ept, bool reload_pdptrs, 1080 enum vm_entry_failure_code *entry_failure_code) 1081 { 1082 if (CC(kvm_vcpu_is_illegal_gpa(vcpu, cr3))) { 1083 *entry_failure_code = ENTRY_FAIL_DEFAULT; 1084 return -EINVAL; 1085 } 1086 1087 /* 1088 * If PAE paging and EPT are both on, CR3 is not used by the CPU and 1089 * must not be dereferenced. 1090 */ 1091 if (reload_pdptrs && !nested_ept && is_pae_paging(vcpu) && 1092 CC(!load_pdptrs(vcpu, cr3))) { 1093 *entry_failure_code = ENTRY_FAIL_PDPTE; 1094 return -EINVAL; 1095 } 1096 1097 vcpu->arch.cr3 = cr3; 1098 kvm_register_mark_dirty(vcpu, VCPU_EXREG_CR3); 1099 1100 /* Re-initialize the MMU, e.g. to pick up CR4 MMU role changes. */ 1101 kvm_init_mmu(vcpu); 1102 1103 if (!nested_ept) 1104 kvm_mmu_new_pgd(vcpu, cr3); 1105 1106 return 0; 1107 } 1108 1109 /* 1110 * Returns if KVM is able to config CPU to tag TLB entries 1111 * populated by L2 differently than TLB entries populated 1112 * by L1. 1113 * 1114 * If L0 uses EPT, L1 and L2 run with different EPTP because 1115 * guest_mode is part of kvm_mmu_page_role. Thus, TLB entries 1116 * are tagged with different EPTP. 1117 * 1118 * If L1 uses VPID and we allocated a vpid02, TLB entries are tagged 1119 * with different VPID (L1 entries are tagged with vmx->vpid 1120 * while L2 entries are tagged with vmx->nested.vpid02). 1121 */ 1122 static bool nested_has_guest_tlb_tag(struct kvm_vcpu *vcpu) 1123 { 1124 struct vmcs12 *vmcs12 = get_vmcs12(vcpu); 1125 1126 return enable_ept || 1127 (nested_cpu_has_vpid(vmcs12) && to_vmx(vcpu)->nested.vpid02); 1128 } 1129 1130 static void nested_vmx_transition_tlb_flush(struct kvm_vcpu *vcpu, 1131 struct vmcs12 *vmcs12, 1132 bool is_vmenter) 1133 { 1134 struct vcpu_vmx *vmx = to_vmx(vcpu); 1135 1136 /* 1137 * KVM_REQ_HV_TLB_FLUSH flushes entries from either L1's VP_ID or 1138 * L2's VP_ID upon request from the guest. Make sure we check for 1139 * pending entries in the right FIFO upon L1/L2 transition as these 1140 * requests are put by other vCPUs asynchronously. 1141 */ 1142 if (to_hv_vcpu(vcpu) && enable_ept) 1143 kvm_make_request(KVM_REQ_HV_TLB_FLUSH, vcpu); 1144 1145 /* 1146 * If vmcs12 doesn't use VPID, L1 expects linear and combined mappings 1147 * for *all* contexts to be flushed on VM-Enter/VM-Exit, i.e. it's a 1148 * full TLB flush from the guest's perspective. This is required even 1149 * if VPID is disabled in the host as KVM may need to synchronize the 1150 * MMU in response to the guest TLB flush. 1151 * 1152 * Note, using TLB_FLUSH_GUEST is correct even if nested EPT is in use. 1153 * EPT is a special snowflake, as guest-physical mappings aren't 1154 * flushed on VPID invalidations, including VM-Enter or VM-Exit with 1155 * VPID disabled. As a result, KVM _never_ needs to sync nEPT 1156 * entries on VM-Enter because L1 can't rely on VM-Enter to flush 1157 * those mappings. 1158 */ 1159 if (!nested_cpu_has_vpid(vmcs12)) { 1160 kvm_make_request(KVM_REQ_TLB_FLUSH_GUEST, vcpu); 1161 return; 1162 } 1163 1164 /* L2 should never have a VPID if VPID is disabled. */ 1165 WARN_ON(!enable_vpid); 1166 1167 /* 1168 * VPID is enabled and in use by vmcs12. If vpid12 is changing, then 1169 * emulate a guest TLB flush as KVM does not track vpid12 history nor 1170 * is the VPID incorporated into the MMU context. I.e. KVM must assume 1171 * that the new vpid12 has never been used and thus represents a new 1172 * guest ASID that cannot have entries in the TLB. 1173 */ 1174 if (is_vmenter && vmcs12->virtual_processor_id != vmx->nested.last_vpid) { 1175 vmx->nested.last_vpid = vmcs12->virtual_processor_id; 1176 kvm_make_request(KVM_REQ_TLB_FLUSH_GUEST, vcpu); 1177 return; 1178 } 1179 1180 /* 1181 * If VPID is enabled, used by vmc12, and vpid12 is not changing but 1182 * does not have a unique TLB tag (ASID), i.e. EPT is disabled and 1183 * KVM was unable to allocate a VPID for L2, flush the current context 1184 * as the effective ASID is common to both L1 and L2. 1185 */ 1186 if (!nested_has_guest_tlb_tag(vcpu)) 1187 kvm_make_request(KVM_REQ_TLB_FLUSH_CURRENT, vcpu); 1188 } 1189 1190 static bool is_bitwise_subset(u64 superset, u64 subset, u64 mask) 1191 { 1192 superset &= mask; 1193 subset &= mask; 1194 1195 return (superset | subset) == superset; 1196 } 1197 1198 static int vmx_restore_vmx_basic(struct vcpu_vmx *vmx, u64 data) 1199 { 1200 const u64 feature_and_reserved = 1201 /* feature (except bit 48; see below) */ 1202 BIT_ULL(49) | BIT_ULL(54) | BIT_ULL(55) | 1203 /* reserved */ 1204 BIT_ULL(31) | GENMASK_ULL(47, 45) | GENMASK_ULL(63, 56); 1205 u64 vmx_basic = vmcs_config.nested.basic; 1206 1207 if (!is_bitwise_subset(vmx_basic, data, feature_and_reserved)) 1208 return -EINVAL; 1209 1210 /* 1211 * KVM does not emulate a version of VMX that constrains physical 1212 * addresses of VMX structures (e.g. VMCS) to 32-bits. 1213 */ 1214 if (data & BIT_ULL(48)) 1215 return -EINVAL; 1216 1217 if (vmx_basic_vmcs_revision_id(vmx_basic) != 1218 vmx_basic_vmcs_revision_id(data)) 1219 return -EINVAL; 1220 1221 if (vmx_basic_vmcs_size(vmx_basic) > vmx_basic_vmcs_size(data)) 1222 return -EINVAL; 1223 1224 vmx->nested.msrs.basic = data; 1225 return 0; 1226 } 1227 1228 static void vmx_get_control_msr(struct nested_vmx_msrs *msrs, u32 msr_index, 1229 u32 **low, u32 **high) 1230 { 1231 switch (msr_index) { 1232 case MSR_IA32_VMX_TRUE_PINBASED_CTLS: 1233 *low = &msrs->pinbased_ctls_low; 1234 *high = &msrs->pinbased_ctls_high; 1235 break; 1236 case MSR_IA32_VMX_TRUE_PROCBASED_CTLS: 1237 *low = &msrs->procbased_ctls_low; 1238 *high = &msrs->procbased_ctls_high; 1239 break; 1240 case MSR_IA32_VMX_TRUE_EXIT_CTLS: 1241 *low = &msrs->exit_ctls_low; 1242 *high = &msrs->exit_ctls_high; 1243 break; 1244 case MSR_IA32_VMX_TRUE_ENTRY_CTLS: 1245 *low = &msrs->entry_ctls_low; 1246 *high = &msrs->entry_ctls_high; 1247 break; 1248 case MSR_IA32_VMX_PROCBASED_CTLS2: 1249 *low = &msrs->secondary_ctls_low; 1250 *high = &msrs->secondary_ctls_high; 1251 break; 1252 default: 1253 BUG(); 1254 } 1255 } 1256 1257 static int 1258 vmx_restore_control_msr(struct vcpu_vmx *vmx, u32 msr_index, u64 data) 1259 { 1260 u32 *lowp, *highp; 1261 u64 supported; 1262 1263 vmx_get_control_msr(&vmcs_config.nested, msr_index, &lowp, &highp); 1264 1265 supported = vmx_control_msr(*lowp, *highp); 1266 1267 /* Check must-be-1 bits are still 1. */ 1268 if (!is_bitwise_subset(data, supported, GENMASK_ULL(31, 0))) 1269 return -EINVAL; 1270 1271 /* Check must-be-0 bits are still 0. */ 1272 if (!is_bitwise_subset(supported, data, GENMASK_ULL(63, 32))) 1273 return -EINVAL; 1274 1275 vmx_get_control_msr(&vmx->nested.msrs, msr_index, &lowp, &highp); 1276 *lowp = data; 1277 *highp = data >> 32; 1278 return 0; 1279 } 1280 1281 static int vmx_restore_vmx_misc(struct vcpu_vmx *vmx, u64 data) 1282 { 1283 const u64 feature_and_reserved_bits = 1284 /* feature */ 1285 BIT_ULL(5) | GENMASK_ULL(8, 6) | BIT_ULL(14) | BIT_ULL(15) | 1286 BIT_ULL(28) | BIT_ULL(29) | BIT_ULL(30) | 1287 /* reserved */ 1288 GENMASK_ULL(13, 9) | BIT_ULL(31); 1289 u64 vmx_misc = vmx_control_msr(vmcs_config.nested.misc_low, 1290 vmcs_config.nested.misc_high); 1291 1292 if (!is_bitwise_subset(vmx_misc, data, feature_and_reserved_bits)) 1293 return -EINVAL; 1294 1295 if ((vmx->nested.msrs.pinbased_ctls_high & 1296 PIN_BASED_VMX_PREEMPTION_TIMER) && 1297 vmx_misc_preemption_timer_rate(data) != 1298 vmx_misc_preemption_timer_rate(vmx_misc)) 1299 return -EINVAL; 1300 1301 if (vmx_misc_cr3_count(data) > vmx_misc_cr3_count(vmx_misc)) 1302 return -EINVAL; 1303 1304 if (vmx_misc_max_msr(data) > vmx_misc_max_msr(vmx_misc)) 1305 return -EINVAL; 1306 1307 if (vmx_misc_mseg_revid(data) != vmx_misc_mseg_revid(vmx_misc)) 1308 return -EINVAL; 1309 1310 vmx->nested.msrs.misc_low = data; 1311 vmx->nested.msrs.misc_high = data >> 32; 1312 1313 return 0; 1314 } 1315 1316 static int vmx_restore_vmx_ept_vpid_cap(struct vcpu_vmx *vmx, u64 data) 1317 { 1318 u64 vmx_ept_vpid_cap = vmx_control_msr(vmcs_config.nested.ept_caps, 1319 vmcs_config.nested.vpid_caps); 1320 1321 /* Every bit is either reserved or a feature bit. */ 1322 if (!is_bitwise_subset(vmx_ept_vpid_cap, data, -1ULL)) 1323 return -EINVAL; 1324 1325 vmx->nested.msrs.ept_caps = data; 1326 vmx->nested.msrs.vpid_caps = data >> 32; 1327 return 0; 1328 } 1329 1330 static u64 *vmx_get_fixed0_msr(struct nested_vmx_msrs *msrs, u32 msr_index) 1331 { 1332 switch (msr_index) { 1333 case MSR_IA32_VMX_CR0_FIXED0: 1334 return &msrs->cr0_fixed0; 1335 case MSR_IA32_VMX_CR4_FIXED0: 1336 return &msrs->cr4_fixed0; 1337 default: 1338 BUG(); 1339 } 1340 } 1341 1342 static int vmx_restore_fixed0_msr(struct vcpu_vmx *vmx, u32 msr_index, u64 data) 1343 { 1344 const u64 *msr = vmx_get_fixed0_msr(&vmcs_config.nested, msr_index); 1345 1346 /* 1347 * 1 bits (which indicates bits which "must-be-1" during VMX operation) 1348 * must be 1 in the restored value. 1349 */ 1350 if (!is_bitwise_subset(data, *msr, -1ULL)) 1351 return -EINVAL; 1352 1353 *vmx_get_fixed0_msr(&vmx->nested.msrs, msr_index) = data; 1354 return 0; 1355 } 1356 1357 /* 1358 * Called when userspace is restoring VMX MSRs. 1359 * 1360 * Returns 0 on success, non-0 otherwise. 1361 */ 1362 int vmx_set_vmx_msr(struct kvm_vcpu *vcpu, u32 msr_index, u64 data) 1363 { 1364 struct vcpu_vmx *vmx = to_vmx(vcpu); 1365 1366 /* 1367 * Don't allow changes to the VMX capability MSRs while the vCPU 1368 * is in VMX operation. 1369 */ 1370 if (vmx->nested.vmxon) 1371 return -EBUSY; 1372 1373 switch (msr_index) { 1374 case MSR_IA32_VMX_BASIC: 1375 return vmx_restore_vmx_basic(vmx, data); 1376 case MSR_IA32_VMX_PINBASED_CTLS: 1377 case MSR_IA32_VMX_PROCBASED_CTLS: 1378 case MSR_IA32_VMX_EXIT_CTLS: 1379 case MSR_IA32_VMX_ENTRY_CTLS: 1380 /* 1381 * The "non-true" VMX capability MSRs are generated from the 1382 * "true" MSRs, so we do not support restoring them directly. 1383 * 1384 * If userspace wants to emulate VMX_BASIC[55]=0, userspace 1385 * should restore the "true" MSRs with the must-be-1 bits 1386 * set according to the SDM Vol 3. A.2 "RESERVED CONTROLS AND 1387 * DEFAULT SETTINGS". 1388 */ 1389 return -EINVAL; 1390 case MSR_IA32_VMX_TRUE_PINBASED_CTLS: 1391 case MSR_IA32_VMX_TRUE_PROCBASED_CTLS: 1392 case MSR_IA32_VMX_TRUE_EXIT_CTLS: 1393 case MSR_IA32_VMX_TRUE_ENTRY_CTLS: 1394 case MSR_IA32_VMX_PROCBASED_CTLS2: 1395 return vmx_restore_control_msr(vmx, msr_index, data); 1396 case MSR_IA32_VMX_MISC: 1397 return vmx_restore_vmx_misc(vmx, data); 1398 case MSR_IA32_VMX_CR0_FIXED0: 1399 case MSR_IA32_VMX_CR4_FIXED0: 1400 return vmx_restore_fixed0_msr(vmx, msr_index, data); 1401 case MSR_IA32_VMX_CR0_FIXED1: 1402 case MSR_IA32_VMX_CR4_FIXED1: 1403 /* 1404 * These MSRs are generated based on the vCPU's CPUID, so we 1405 * do not support restoring them directly. 1406 */ 1407 return -EINVAL; 1408 case MSR_IA32_VMX_EPT_VPID_CAP: 1409 return vmx_restore_vmx_ept_vpid_cap(vmx, data); 1410 case MSR_IA32_VMX_VMCS_ENUM: 1411 vmx->nested.msrs.vmcs_enum = data; 1412 return 0; 1413 case MSR_IA32_VMX_VMFUNC: 1414 if (data & ~vmcs_config.nested.vmfunc_controls) 1415 return -EINVAL; 1416 vmx->nested.msrs.vmfunc_controls = data; 1417 return 0; 1418 default: 1419 /* 1420 * The rest of the VMX capability MSRs do not support restore. 1421 */ 1422 return -EINVAL; 1423 } 1424 } 1425 1426 /* Returns 0 on success, non-0 otherwise. */ 1427 int vmx_get_vmx_msr(struct nested_vmx_msrs *msrs, u32 msr_index, u64 *pdata) 1428 { 1429 switch (msr_index) { 1430 case MSR_IA32_VMX_BASIC: 1431 *pdata = msrs->basic; 1432 break; 1433 case MSR_IA32_VMX_TRUE_PINBASED_CTLS: 1434 case MSR_IA32_VMX_PINBASED_CTLS: 1435 *pdata = vmx_control_msr( 1436 msrs->pinbased_ctls_low, 1437 msrs->pinbased_ctls_high); 1438 if (msr_index == MSR_IA32_VMX_PINBASED_CTLS) 1439 *pdata |= PIN_BASED_ALWAYSON_WITHOUT_TRUE_MSR; 1440 break; 1441 case MSR_IA32_VMX_TRUE_PROCBASED_CTLS: 1442 case MSR_IA32_VMX_PROCBASED_CTLS: 1443 *pdata = vmx_control_msr( 1444 msrs->procbased_ctls_low, 1445 msrs->procbased_ctls_high); 1446 if (msr_index == MSR_IA32_VMX_PROCBASED_CTLS) 1447 *pdata |= CPU_BASED_ALWAYSON_WITHOUT_TRUE_MSR; 1448 break; 1449 case MSR_IA32_VMX_TRUE_EXIT_CTLS: 1450 case MSR_IA32_VMX_EXIT_CTLS: 1451 *pdata = vmx_control_msr( 1452 msrs->exit_ctls_low, 1453 msrs->exit_ctls_high); 1454 if (msr_index == MSR_IA32_VMX_EXIT_CTLS) 1455 *pdata |= VM_EXIT_ALWAYSON_WITHOUT_TRUE_MSR; 1456 break; 1457 case MSR_IA32_VMX_TRUE_ENTRY_CTLS: 1458 case MSR_IA32_VMX_ENTRY_CTLS: 1459 *pdata = vmx_control_msr( 1460 msrs->entry_ctls_low, 1461 msrs->entry_ctls_high); 1462 if (msr_index == MSR_IA32_VMX_ENTRY_CTLS) 1463 *pdata |= VM_ENTRY_ALWAYSON_WITHOUT_TRUE_MSR; 1464 break; 1465 case MSR_IA32_VMX_MISC: 1466 *pdata = vmx_control_msr( 1467 msrs->misc_low, 1468 msrs->misc_high); 1469 break; 1470 case MSR_IA32_VMX_CR0_FIXED0: 1471 *pdata = msrs->cr0_fixed0; 1472 break; 1473 case MSR_IA32_VMX_CR0_FIXED1: 1474 *pdata = msrs->cr0_fixed1; 1475 break; 1476 case MSR_IA32_VMX_CR4_FIXED0: 1477 *pdata = msrs->cr4_fixed0; 1478 break; 1479 case MSR_IA32_VMX_CR4_FIXED1: 1480 *pdata = msrs->cr4_fixed1; 1481 break; 1482 case MSR_IA32_VMX_VMCS_ENUM: 1483 *pdata = msrs->vmcs_enum; 1484 break; 1485 case MSR_IA32_VMX_PROCBASED_CTLS2: 1486 *pdata = vmx_control_msr( 1487 msrs->secondary_ctls_low, 1488 msrs->secondary_ctls_high); 1489 break; 1490 case MSR_IA32_VMX_EPT_VPID_CAP: 1491 *pdata = msrs->ept_caps | 1492 ((u64)msrs->vpid_caps << 32); 1493 break; 1494 case MSR_IA32_VMX_VMFUNC: 1495 *pdata = msrs->vmfunc_controls; 1496 break; 1497 default: 1498 return 1; 1499 } 1500 1501 return 0; 1502 } 1503 1504 /* 1505 * Copy the writable VMCS shadow fields back to the VMCS12, in case they have 1506 * been modified by the L1 guest. Note, "writable" in this context means 1507 * "writable by the guest", i.e. tagged SHADOW_FIELD_RW; the set of 1508 * fields tagged SHADOW_FIELD_RO may or may not align with the "read-only" 1509 * VM-exit information fields (which are actually writable if the vCPU is 1510 * configured to support "VMWRITE to any supported field in the VMCS"). 1511 */ 1512 static void copy_shadow_to_vmcs12(struct vcpu_vmx *vmx) 1513 { 1514 struct vmcs *shadow_vmcs = vmx->vmcs01.shadow_vmcs; 1515 struct vmcs12 *vmcs12 = get_vmcs12(&vmx->vcpu); 1516 struct shadow_vmcs_field field; 1517 unsigned long val; 1518 int i; 1519 1520 if (WARN_ON(!shadow_vmcs)) 1521 return; 1522 1523 preempt_disable(); 1524 1525 vmcs_load(shadow_vmcs); 1526 1527 for (i = 0; i < max_shadow_read_write_fields; i++) { 1528 field = shadow_read_write_fields[i]; 1529 val = __vmcs_readl(field.encoding); 1530 vmcs12_write_any(vmcs12, field.encoding, field.offset, val); 1531 } 1532 1533 vmcs_clear(shadow_vmcs); 1534 vmcs_load(vmx->loaded_vmcs->vmcs); 1535 1536 preempt_enable(); 1537 } 1538 1539 static void copy_vmcs12_to_shadow(struct vcpu_vmx *vmx) 1540 { 1541 const struct shadow_vmcs_field *fields[] = { 1542 shadow_read_write_fields, 1543 shadow_read_only_fields 1544 }; 1545 const int max_fields[] = { 1546 max_shadow_read_write_fields, 1547 max_shadow_read_only_fields 1548 }; 1549 struct vmcs *shadow_vmcs = vmx->vmcs01.shadow_vmcs; 1550 struct vmcs12 *vmcs12 = get_vmcs12(&vmx->vcpu); 1551 struct shadow_vmcs_field field; 1552 unsigned long val; 1553 int i, q; 1554 1555 if (WARN_ON(!shadow_vmcs)) 1556 return; 1557 1558 vmcs_load(shadow_vmcs); 1559 1560 for (q = 0; q < ARRAY_SIZE(fields); q++) { 1561 for (i = 0; i < max_fields[q]; i++) { 1562 field = fields[q][i]; 1563 val = vmcs12_read_any(vmcs12, field.encoding, 1564 field.offset); 1565 __vmcs_writel(field.encoding, val); 1566 } 1567 } 1568 1569 vmcs_clear(shadow_vmcs); 1570 vmcs_load(vmx->loaded_vmcs->vmcs); 1571 } 1572 1573 static void copy_enlightened_to_vmcs12(struct vcpu_vmx *vmx, u32 hv_clean_fields) 1574 { 1575 struct vmcs12 *vmcs12 = vmx->nested.cached_vmcs12; 1576 struct hv_enlightened_vmcs *evmcs = vmx->nested.hv_evmcs; 1577 struct kvm_vcpu_hv *hv_vcpu = to_hv_vcpu(&vmx->vcpu); 1578 1579 /* HV_VMX_ENLIGHTENED_CLEAN_FIELD_NONE */ 1580 vmcs12->tpr_threshold = evmcs->tpr_threshold; 1581 vmcs12->guest_rip = evmcs->guest_rip; 1582 1583 if (unlikely(!(hv_clean_fields & 1584 HV_VMX_ENLIGHTENED_CLEAN_FIELD_ENLIGHTENMENTSCONTROL))) { 1585 hv_vcpu->nested.pa_page_gpa = evmcs->partition_assist_page; 1586 hv_vcpu->nested.vm_id = evmcs->hv_vm_id; 1587 hv_vcpu->nested.vp_id = evmcs->hv_vp_id; 1588 } 1589 1590 if (unlikely(!(hv_clean_fields & 1591 HV_VMX_ENLIGHTENED_CLEAN_FIELD_GUEST_BASIC))) { 1592 vmcs12->guest_rsp = evmcs->guest_rsp; 1593 vmcs12->guest_rflags = evmcs->guest_rflags; 1594 vmcs12->guest_interruptibility_info = 1595 evmcs->guest_interruptibility_info; 1596 /* 1597 * Not present in struct vmcs12: 1598 * vmcs12->guest_ssp = evmcs->guest_ssp; 1599 */ 1600 } 1601 1602 if (unlikely(!(hv_clean_fields & 1603 HV_VMX_ENLIGHTENED_CLEAN_FIELD_CONTROL_PROC))) { 1604 vmcs12->cpu_based_vm_exec_control = 1605 evmcs->cpu_based_vm_exec_control; 1606 } 1607 1608 if (unlikely(!(hv_clean_fields & 1609 HV_VMX_ENLIGHTENED_CLEAN_FIELD_CONTROL_EXCPN))) { 1610 vmcs12->exception_bitmap = evmcs->exception_bitmap; 1611 } 1612 1613 if (unlikely(!(hv_clean_fields & 1614 HV_VMX_ENLIGHTENED_CLEAN_FIELD_CONTROL_ENTRY))) { 1615 vmcs12->vm_entry_controls = evmcs->vm_entry_controls; 1616 } 1617 1618 if (unlikely(!(hv_clean_fields & 1619 HV_VMX_ENLIGHTENED_CLEAN_FIELD_CONTROL_EVENT))) { 1620 vmcs12->vm_entry_intr_info_field = 1621 evmcs->vm_entry_intr_info_field; 1622 vmcs12->vm_entry_exception_error_code = 1623 evmcs->vm_entry_exception_error_code; 1624 vmcs12->vm_entry_instruction_len = 1625 evmcs->vm_entry_instruction_len; 1626 } 1627 1628 if (unlikely(!(hv_clean_fields & 1629 HV_VMX_ENLIGHTENED_CLEAN_FIELD_HOST_GRP1))) { 1630 vmcs12->host_ia32_pat = evmcs->host_ia32_pat; 1631 vmcs12->host_ia32_efer = evmcs->host_ia32_efer; 1632 vmcs12->host_cr0 = evmcs->host_cr0; 1633 vmcs12->host_cr3 = evmcs->host_cr3; 1634 vmcs12->host_cr4 = evmcs->host_cr4; 1635 vmcs12->host_ia32_sysenter_esp = evmcs->host_ia32_sysenter_esp; 1636 vmcs12->host_ia32_sysenter_eip = evmcs->host_ia32_sysenter_eip; 1637 vmcs12->host_rip = evmcs->host_rip; 1638 vmcs12->host_ia32_sysenter_cs = evmcs->host_ia32_sysenter_cs; 1639 vmcs12->host_es_selector = evmcs->host_es_selector; 1640 vmcs12->host_cs_selector = evmcs->host_cs_selector; 1641 vmcs12->host_ss_selector = evmcs->host_ss_selector; 1642 vmcs12->host_ds_selector = evmcs->host_ds_selector; 1643 vmcs12->host_fs_selector = evmcs->host_fs_selector; 1644 vmcs12->host_gs_selector = evmcs->host_gs_selector; 1645 vmcs12->host_tr_selector = evmcs->host_tr_selector; 1646 vmcs12->host_ia32_perf_global_ctrl = evmcs->host_ia32_perf_global_ctrl; 1647 /* 1648 * Not present in struct vmcs12: 1649 * vmcs12->host_ia32_s_cet = evmcs->host_ia32_s_cet; 1650 * vmcs12->host_ssp = evmcs->host_ssp; 1651 * vmcs12->host_ia32_int_ssp_table_addr = evmcs->host_ia32_int_ssp_table_addr; 1652 */ 1653 } 1654 1655 if (unlikely(!(hv_clean_fields & 1656 HV_VMX_ENLIGHTENED_CLEAN_FIELD_CONTROL_GRP1))) { 1657 vmcs12->pin_based_vm_exec_control = 1658 evmcs->pin_based_vm_exec_control; 1659 vmcs12->vm_exit_controls = evmcs->vm_exit_controls; 1660 vmcs12->secondary_vm_exec_control = 1661 evmcs->secondary_vm_exec_control; 1662 } 1663 1664 if (unlikely(!(hv_clean_fields & 1665 HV_VMX_ENLIGHTENED_CLEAN_FIELD_IO_BITMAP))) { 1666 vmcs12->io_bitmap_a = evmcs->io_bitmap_a; 1667 vmcs12->io_bitmap_b = evmcs->io_bitmap_b; 1668 } 1669 1670 if (unlikely(!(hv_clean_fields & 1671 HV_VMX_ENLIGHTENED_CLEAN_FIELD_MSR_BITMAP))) { 1672 vmcs12->msr_bitmap = evmcs->msr_bitmap; 1673 } 1674 1675 if (unlikely(!(hv_clean_fields & 1676 HV_VMX_ENLIGHTENED_CLEAN_FIELD_GUEST_GRP2))) { 1677 vmcs12->guest_es_base = evmcs->guest_es_base; 1678 vmcs12->guest_cs_base = evmcs->guest_cs_base; 1679 vmcs12->guest_ss_base = evmcs->guest_ss_base; 1680 vmcs12->guest_ds_base = evmcs->guest_ds_base; 1681 vmcs12->guest_fs_base = evmcs->guest_fs_base; 1682 vmcs12->guest_gs_base = evmcs->guest_gs_base; 1683 vmcs12->guest_ldtr_base = evmcs->guest_ldtr_base; 1684 vmcs12->guest_tr_base = evmcs->guest_tr_base; 1685 vmcs12->guest_gdtr_base = evmcs->guest_gdtr_base; 1686 vmcs12->guest_idtr_base = evmcs->guest_idtr_base; 1687 vmcs12->guest_es_limit = evmcs->guest_es_limit; 1688 vmcs12->guest_cs_limit = evmcs->guest_cs_limit; 1689 vmcs12->guest_ss_limit = evmcs->guest_ss_limit; 1690 vmcs12->guest_ds_limit = evmcs->guest_ds_limit; 1691 vmcs12->guest_fs_limit = evmcs->guest_fs_limit; 1692 vmcs12->guest_gs_limit = evmcs->guest_gs_limit; 1693 vmcs12->guest_ldtr_limit = evmcs->guest_ldtr_limit; 1694 vmcs12->guest_tr_limit = evmcs->guest_tr_limit; 1695 vmcs12->guest_gdtr_limit = evmcs->guest_gdtr_limit; 1696 vmcs12->guest_idtr_limit = evmcs->guest_idtr_limit; 1697 vmcs12->guest_es_ar_bytes = evmcs->guest_es_ar_bytes; 1698 vmcs12->guest_cs_ar_bytes = evmcs->guest_cs_ar_bytes; 1699 vmcs12->guest_ss_ar_bytes = evmcs->guest_ss_ar_bytes; 1700 vmcs12->guest_ds_ar_bytes = evmcs->guest_ds_ar_bytes; 1701 vmcs12->guest_fs_ar_bytes = evmcs->guest_fs_ar_bytes; 1702 vmcs12->guest_gs_ar_bytes = evmcs->guest_gs_ar_bytes; 1703 vmcs12->guest_ldtr_ar_bytes = evmcs->guest_ldtr_ar_bytes; 1704 vmcs12->guest_tr_ar_bytes = evmcs->guest_tr_ar_bytes; 1705 vmcs12->guest_es_selector = evmcs->guest_es_selector; 1706 vmcs12->guest_cs_selector = evmcs->guest_cs_selector; 1707 vmcs12->guest_ss_selector = evmcs->guest_ss_selector; 1708 vmcs12->guest_ds_selector = evmcs->guest_ds_selector; 1709 vmcs12->guest_fs_selector = evmcs->guest_fs_selector; 1710 vmcs12->guest_gs_selector = evmcs->guest_gs_selector; 1711 vmcs12->guest_ldtr_selector = evmcs->guest_ldtr_selector; 1712 vmcs12->guest_tr_selector = evmcs->guest_tr_selector; 1713 } 1714 1715 if (unlikely(!(hv_clean_fields & 1716 HV_VMX_ENLIGHTENED_CLEAN_FIELD_CONTROL_GRP2))) { 1717 vmcs12->tsc_offset = evmcs->tsc_offset; 1718 vmcs12->virtual_apic_page_addr = evmcs->virtual_apic_page_addr; 1719 vmcs12->xss_exit_bitmap = evmcs->xss_exit_bitmap; 1720 vmcs12->encls_exiting_bitmap = evmcs->encls_exiting_bitmap; 1721 vmcs12->tsc_multiplier = evmcs->tsc_multiplier; 1722 } 1723 1724 if (unlikely(!(hv_clean_fields & 1725 HV_VMX_ENLIGHTENED_CLEAN_FIELD_CRDR))) { 1726 vmcs12->cr0_guest_host_mask = evmcs->cr0_guest_host_mask; 1727 vmcs12->cr4_guest_host_mask = evmcs->cr4_guest_host_mask; 1728 vmcs12->cr0_read_shadow = evmcs->cr0_read_shadow; 1729 vmcs12->cr4_read_shadow = evmcs->cr4_read_shadow; 1730 vmcs12->guest_cr0 = evmcs->guest_cr0; 1731 vmcs12->guest_cr3 = evmcs->guest_cr3; 1732 vmcs12->guest_cr4 = evmcs->guest_cr4; 1733 vmcs12->guest_dr7 = evmcs->guest_dr7; 1734 } 1735 1736 if (unlikely(!(hv_clean_fields & 1737 HV_VMX_ENLIGHTENED_CLEAN_FIELD_HOST_POINTER))) { 1738 vmcs12->host_fs_base = evmcs->host_fs_base; 1739 vmcs12->host_gs_base = evmcs->host_gs_base; 1740 vmcs12->host_tr_base = evmcs->host_tr_base; 1741 vmcs12->host_gdtr_base = evmcs->host_gdtr_base; 1742 vmcs12->host_idtr_base = evmcs->host_idtr_base; 1743 vmcs12->host_rsp = evmcs->host_rsp; 1744 } 1745 1746 if (unlikely(!(hv_clean_fields & 1747 HV_VMX_ENLIGHTENED_CLEAN_FIELD_CONTROL_XLAT))) { 1748 vmcs12->ept_pointer = evmcs->ept_pointer; 1749 vmcs12->virtual_processor_id = evmcs->virtual_processor_id; 1750 } 1751 1752 if (unlikely(!(hv_clean_fields & 1753 HV_VMX_ENLIGHTENED_CLEAN_FIELD_GUEST_GRP1))) { 1754 vmcs12->vmcs_link_pointer = evmcs->vmcs_link_pointer; 1755 vmcs12->guest_ia32_debugctl = evmcs->guest_ia32_debugctl; 1756 vmcs12->guest_ia32_pat = evmcs->guest_ia32_pat; 1757 vmcs12->guest_ia32_efer = evmcs->guest_ia32_efer; 1758 vmcs12->guest_pdptr0 = evmcs->guest_pdptr0; 1759 vmcs12->guest_pdptr1 = evmcs->guest_pdptr1; 1760 vmcs12->guest_pdptr2 = evmcs->guest_pdptr2; 1761 vmcs12->guest_pdptr3 = evmcs->guest_pdptr3; 1762 vmcs12->guest_pending_dbg_exceptions = 1763 evmcs->guest_pending_dbg_exceptions; 1764 vmcs12->guest_sysenter_esp = evmcs->guest_sysenter_esp; 1765 vmcs12->guest_sysenter_eip = evmcs->guest_sysenter_eip; 1766 vmcs12->guest_bndcfgs = evmcs->guest_bndcfgs; 1767 vmcs12->guest_activity_state = evmcs->guest_activity_state; 1768 vmcs12->guest_sysenter_cs = evmcs->guest_sysenter_cs; 1769 vmcs12->guest_ia32_perf_global_ctrl = evmcs->guest_ia32_perf_global_ctrl; 1770 /* 1771 * Not present in struct vmcs12: 1772 * vmcs12->guest_ia32_s_cet = evmcs->guest_ia32_s_cet; 1773 * vmcs12->guest_ia32_lbr_ctl = evmcs->guest_ia32_lbr_ctl; 1774 * vmcs12->guest_ia32_int_ssp_table_addr = evmcs->guest_ia32_int_ssp_table_addr; 1775 */ 1776 } 1777 1778 /* 1779 * Not used? 1780 * vmcs12->vm_exit_msr_store_addr = evmcs->vm_exit_msr_store_addr; 1781 * vmcs12->vm_exit_msr_load_addr = evmcs->vm_exit_msr_load_addr; 1782 * vmcs12->vm_entry_msr_load_addr = evmcs->vm_entry_msr_load_addr; 1783 * vmcs12->page_fault_error_code_mask = 1784 * evmcs->page_fault_error_code_mask; 1785 * vmcs12->page_fault_error_code_match = 1786 * evmcs->page_fault_error_code_match; 1787 * vmcs12->cr3_target_count = evmcs->cr3_target_count; 1788 * vmcs12->vm_exit_msr_store_count = evmcs->vm_exit_msr_store_count; 1789 * vmcs12->vm_exit_msr_load_count = evmcs->vm_exit_msr_load_count; 1790 * vmcs12->vm_entry_msr_load_count = evmcs->vm_entry_msr_load_count; 1791 */ 1792 1793 /* 1794 * Read only fields: 1795 * vmcs12->guest_physical_address = evmcs->guest_physical_address; 1796 * vmcs12->vm_instruction_error = evmcs->vm_instruction_error; 1797 * vmcs12->vm_exit_reason = evmcs->vm_exit_reason; 1798 * vmcs12->vm_exit_intr_info = evmcs->vm_exit_intr_info; 1799 * vmcs12->vm_exit_intr_error_code = evmcs->vm_exit_intr_error_code; 1800 * vmcs12->idt_vectoring_info_field = evmcs->idt_vectoring_info_field; 1801 * vmcs12->idt_vectoring_error_code = evmcs->idt_vectoring_error_code; 1802 * vmcs12->vm_exit_instruction_len = evmcs->vm_exit_instruction_len; 1803 * vmcs12->vmx_instruction_info = evmcs->vmx_instruction_info; 1804 * vmcs12->exit_qualification = evmcs->exit_qualification; 1805 * vmcs12->guest_linear_address = evmcs->guest_linear_address; 1806 * 1807 * Not present in struct vmcs12: 1808 * vmcs12->exit_io_instruction_ecx = evmcs->exit_io_instruction_ecx; 1809 * vmcs12->exit_io_instruction_esi = evmcs->exit_io_instruction_esi; 1810 * vmcs12->exit_io_instruction_edi = evmcs->exit_io_instruction_edi; 1811 * vmcs12->exit_io_instruction_eip = evmcs->exit_io_instruction_eip; 1812 */ 1813 1814 return; 1815 } 1816 1817 static void copy_vmcs12_to_enlightened(struct vcpu_vmx *vmx) 1818 { 1819 struct vmcs12 *vmcs12 = vmx->nested.cached_vmcs12; 1820 struct hv_enlightened_vmcs *evmcs = vmx->nested.hv_evmcs; 1821 1822 /* 1823 * Should not be changed by KVM: 1824 * 1825 * evmcs->host_es_selector = vmcs12->host_es_selector; 1826 * evmcs->host_cs_selector = vmcs12->host_cs_selector; 1827 * evmcs->host_ss_selector = vmcs12->host_ss_selector; 1828 * evmcs->host_ds_selector = vmcs12->host_ds_selector; 1829 * evmcs->host_fs_selector = vmcs12->host_fs_selector; 1830 * evmcs->host_gs_selector = vmcs12->host_gs_selector; 1831 * evmcs->host_tr_selector = vmcs12->host_tr_selector; 1832 * evmcs->host_ia32_pat = vmcs12->host_ia32_pat; 1833 * evmcs->host_ia32_efer = vmcs12->host_ia32_efer; 1834 * evmcs->host_cr0 = vmcs12->host_cr0; 1835 * evmcs->host_cr3 = vmcs12->host_cr3; 1836 * evmcs->host_cr4 = vmcs12->host_cr4; 1837 * evmcs->host_ia32_sysenter_esp = vmcs12->host_ia32_sysenter_esp; 1838 * evmcs->host_ia32_sysenter_eip = vmcs12->host_ia32_sysenter_eip; 1839 * evmcs->host_rip = vmcs12->host_rip; 1840 * evmcs->host_ia32_sysenter_cs = vmcs12->host_ia32_sysenter_cs; 1841 * evmcs->host_fs_base = vmcs12->host_fs_base; 1842 * evmcs->host_gs_base = vmcs12->host_gs_base; 1843 * evmcs->host_tr_base = vmcs12->host_tr_base; 1844 * evmcs->host_gdtr_base = vmcs12->host_gdtr_base; 1845 * evmcs->host_idtr_base = vmcs12->host_idtr_base; 1846 * evmcs->host_rsp = vmcs12->host_rsp; 1847 * sync_vmcs02_to_vmcs12() doesn't read these: 1848 * evmcs->io_bitmap_a = vmcs12->io_bitmap_a; 1849 * evmcs->io_bitmap_b = vmcs12->io_bitmap_b; 1850 * evmcs->msr_bitmap = vmcs12->msr_bitmap; 1851 * evmcs->ept_pointer = vmcs12->ept_pointer; 1852 * evmcs->xss_exit_bitmap = vmcs12->xss_exit_bitmap; 1853 * evmcs->vm_exit_msr_store_addr = vmcs12->vm_exit_msr_store_addr; 1854 * evmcs->vm_exit_msr_load_addr = vmcs12->vm_exit_msr_load_addr; 1855 * evmcs->vm_entry_msr_load_addr = vmcs12->vm_entry_msr_load_addr; 1856 * evmcs->tpr_threshold = vmcs12->tpr_threshold; 1857 * evmcs->virtual_processor_id = vmcs12->virtual_processor_id; 1858 * evmcs->exception_bitmap = vmcs12->exception_bitmap; 1859 * evmcs->vmcs_link_pointer = vmcs12->vmcs_link_pointer; 1860 * evmcs->pin_based_vm_exec_control = vmcs12->pin_based_vm_exec_control; 1861 * evmcs->vm_exit_controls = vmcs12->vm_exit_controls; 1862 * evmcs->secondary_vm_exec_control = vmcs12->secondary_vm_exec_control; 1863 * evmcs->page_fault_error_code_mask = 1864 * vmcs12->page_fault_error_code_mask; 1865 * evmcs->page_fault_error_code_match = 1866 * vmcs12->page_fault_error_code_match; 1867 * evmcs->cr3_target_count = vmcs12->cr3_target_count; 1868 * evmcs->virtual_apic_page_addr = vmcs12->virtual_apic_page_addr; 1869 * evmcs->tsc_offset = vmcs12->tsc_offset; 1870 * evmcs->guest_ia32_debugctl = vmcs12->guest_ia32_debugctl; 1871 * evmcs->cr0_guest_host_mask = vmcs12->cr0_guest_host_mask; 1872 * evmcs->cr4_guest_host_mask = vmcs12->cr4_guest_host_mask; 1873 * evmcs->cr0_read_shadow = vmcs12->cr0_read_shadow; 1874 * evmcs->cr4_read_shadow = vmcs12->cr4_read_shadow; 1875 * evmcs->vm_exit_msr_store_count = vmcs12->vm_exit_msr_store_count; 1876 * evmcs->vm_exit_msr_load_count = vmcs12->vm_exit_msr_load_count; 1877 * evmcs->vm_entry_msr_load_count = vmcs12->vm_entry_msr_load_count; 1878 * evmcs->guest_ia32_perf_global_ctrl = vmcs12->guest_ia32_perf_global_ctrl; 1879 * evmcs->host_ia32_perf_global_ctrl = vmcs12->host_ia32_perf_global_ctrl; 1880 * evmcs->encls_exiting_bitmap = vmcs12->encls_exiting_bitmap; 1881 * evmcs->tsc_multiplier = vmcs12->tsc_multiplier; 1882 * 1883 * Not present in struct vmcs12: 1884 * evmcs->exit_io_instruction_ecx = vmcs12->exit_io_instruction_ecx; 1885 * evmcs->exit_io_instruction_esi = vmcs12->exit_io_instruction_esi; 1886 * evmcs->exit_io_instruction_edi = vmcs12->exit_io_instruction_edi; 1887 * evmcs->exit_io_instruction_eip = vmcs12->exit_io_instruction_eip; 1888 * evmcs->host_ia32_s_cet = vmcs12->host_ia32_s_cet; 1889 * evmcs->host_ssp = vmcs12->host_ssp; 1890 * evmcs->host_ia32_int_ssp_table_addr = vmcs12->host_ia32_int_ssp_table_addr; 1891 * evmcs->guest_ia32_s_cet = vmcs12->guest_ia32_s_cet; 1892 * evmcs->guest_ia32_lbr_ctl = vmcs12->guest_ia32_lbr_ctl; 1893 * evmcs->guest_ia32_int_ssp_table_addr = vmcs12->guest_ia32_int_ssp_table_addr; 1894 * evmcs->guest_ssp = vmcs12->guest_ssp; 1895 */ 1896 1897 evmcs->guest_es_selector = vmcs12->guest_es_selector; 1898 evmcs->guest_cs_selector = vmcs12->guest_cs_selector; 1899 evmcs->guest_ss_selector = vmcs12->guest_ss_selector; 1900 evmcs->guest_ds_selector = vmcs12->guest_ds_selector; 1901 evmcs->guest_fs_selector = vmcs12->guest_fs_selector; 1902 evmcs->guest_gs_selector = vmcs12->guest_gs_selector; 1903 evmcs->guest_ldtr_selector = vmcs12->guest_ldtr_selector; 1904 evmcs->guest_tr_selector = vmcs12->guest_tr_selector; 1905 1906 evmcs->guest_es_limit = vmcs12->guest_es_limit; 1907 evmcs->guest_cs_limit = vmcs12->guest_cs_limit; 1908 evmcs->guest_ss_limit = vmcs12->guest_ss_limit; 1909 evmcs->guest_ds_limit = vmcs12->guest_ds_limit; 1910 evmcs->guest_fs_limit = vmcs12->guest_fs_limit; 1911 evmcs->guest_gs_limit = vmcs12->guest_gs_limit; 1912 evmcs->guest_ldtr_limit = vmcs12->guest_ldtr_limit; 1913 evmcs->guest_tr_limit = vmcs12->guest_tr_limit; 1914 evmcs->guest_gdtr_limit = vmcs12->guest_gdtr_limit; 1915 evmcs->guest_idtr_limit = vmcs12->guest_idtr_limit; 1916 1917 evmcs->guest_es_ar_bytes = vmcs12->guest_es_ar_bytes; 1918 evmcs->guest_cs_ar_bytes = vmcs12->guest_cs_ar_bytes; 1919 evmcs->guest_ss_ar_bytes = vmcs12->guest_ss_ar_bytes; 1920 evmcs->guest_ds_ar_bytes = vmcs12->guest_ds_ar_bytes; 1921 evmcs->guest_fs_ar_bytes = vmcs12->guest_fs_ar_bytes; 1922 evmcs->guest_gs_ar_bytes = vmcs12->guest_gs_ar_bytes; 1923 evmcs->guest_ldtr_ar_bytes = vmcs12->guest_ldtr_ar_bytes; 1924 evmcs->guest_tr_ar_bytes = vmcs12->guest_tr_ar_bytes; 1925 1926 evmcs->guest_es_base = vmcs12->guest_es_base; 1927 evmcs->guest_cs_base = vmcs12->guest_cs_base; 1928 evmcs->guest_ss_base = vmcs12->guest_ss_base; 1929 evmcs->guest_ds_base = vmcs12->guest_ds_base; 1930 evmcs->guest_fs_base = vmcs12->guest_fs_base; 1931 evmcs->guest_gs_base = vmcs12->guest_gs_base; 1932 evmcs->guest_ldtr_base = vmcs12->guest_ldtr_base; 1933 evmcs->guest_tr_base = vmcs12->guest_tr_base; 1934 evmcs->guest_gdtr_base = vmcs12->guest_gdtr_base; 1935 evmcs->guest_idtr_base = vmcs12->guest_idtr_base; 1936 1937 evmcs->guest_ia32_pat = vmcs12->guest_ia32_pat; 1938 evmcs->guest_ia32_efer = vmcs12->guest_ia32_efer; 1939 1940 evmcs->guest_pdptr0 = vmcs12->guest_pdptr0; 1941 evmcs->guest_pdptr1 = vmcs12->guest_pdptr1; 1942 evmcs->guest_pdptr2 = vmcs12->guest_pdptr2; 1943 evmcs->guest_pdptr3 = vmcs12->guest_pdptr3; 1944 1945 evmcs->guest_pending_dbg_exceptions = 1946 vmcs12->guest_pending_dbg_exceptions; 1947 evmcs->guest_sysenter_esp = vmcs12->guest_sysenter_esp; 1948 evmcs->guest_sysenter_eip = vmcs12->guest_sysenter_eip; 1949 1950 evmcs->guest_activity_state = vmcs12->guest_activity_state; 1951 evmcs->guest_sysenter_cs = vmcs12->guest_sysenter_cs; 1952 1953 evmcs->guest_cr0 = vmcs12->guest_cr0; 1954 evmcs->guest_cr3 = vmcs12->guest_cr3; 1955 evmcs->guest_cr4 = vmcs12->guest_cr4; 1956 evmcs->guest_dr7 = vmcs12->guest_dr7; 1957 1958 evmcs->guest_physical_address = vmcs12->guest_physical_address; 1959 1960 evmcs->vm_instruction_error = vmcs12->vm_instruction_error; 1961 evmcs->vm_exit_reason = vmcs12->vm_exit_reason; 1962 evmcs->vm_exit_intr_info = vmcs12->vm_exit_intr_info; 1963 evmcs->vm_exit_intr_error_code = vmcs12->vm_exit_intr_error_code; 1964 evmcs->idt_vectoring_info_field = vmcs12->idt_vectoring_info_field; 1965 evmcs->idt_vectoring_error_code = vmcs12->idt_vectoring_error_code; 1966 evmcs->vm_exit_instruction_len = vmcs12->vm_exit_instruction_len; 1967 evmcs->vmx_instruction_info = vmcs12->vmx_instruction_info; 1968 1969 evmcs->exit_qualification = vmcs12->exit_qualification; 1970 1971 evmcs->guest_linear_address = vmcs12->guest_linear_address; 1972 evmcs->guest_rsp = vmcs12->guest_rsp; 1973 evmcs->guest_rflags = vmcs12->guest_rflags; 1974 1975 evmcs->guest_interruptibility_info = 1976 vmcs12->guest_interruptibility_info; 1977 evmcs->cpu_based_vm_exec_control = vmcs12->cpu_based_vm_exec_control; 1978 evmcs->vm_entry_controls = vmcs12->vm_entry_controls; 1979 evmcs->vm_entry_intr_info_field = vmcs12->vm_entry_intr_info_field; 1980 evmcs->vm_entry_exception_error_code = 1981 vmcs12->vm_entry_exception_error_code; 1982 evmcs->vm_entry_instruction_len = vmcs12->vm_entry_instruction_len; 1983 1984 evmcs->guest_rip = vmcs12->guest_rip; 1985 1986 evmcs->guest_bndcfgs = vmcs12->guest_bndcfgs; 1987 1988 return; 1989 } 1990 1991 /* 1992 * This is an equivalent of the nested hypervisor executing the vmptrld 1993 * instruction. 1994 */ 1995 static enum nested_evmptrld_status nested_vmx_handle_enlightened_vmptrld( 1996 struct kvm_vcpu *vcpu, bool from_launch) 1997 { 1998 struct vcpu_vmx *vmx = to_vmx(vcpu); 1999 bool evmcs_gpa_changed = false; 2000 u64 evmcs_gpa; 2001 2002 if (likely(!guest_cpuid_has_evmcs(vcpu))) 2003 return EVMPTRLD_DISABLED; 2004 2005 evmcs_gpa = nested_get_evmptr(vcpu); 2006 if (!evmptr_is_valid(evmcs_gpa)) { 2007 nested_release_evmcs(vcpu); 2008 return EVMPTRLD_DISABLED; 2009 } 2010 2011 if (unlikely(evmcs_gpa != vmx->nested.hv_evmcs_vmptr)) { 2012 vmx->nested.current_vmptr = INVALID_GPA; 2013 2014 nested_release_evmcs(vcpu); 2015 2016 if (kvm_vcpu_map(vcpu, gpa_to_gfn(evmcs_gpa), 2017 &vmx->nested.hv_evmcs_map)) 2018 return EVMPTRLD_ERROR; 2019 2020 vmx->nested.hv_evmcs = vmx->nested.hv_evmcs_map.hva; 2021 2022 /* 2023 * Currently, KVM only supports eVMCS version 1 2024 * (== KVM_EVMCS_VERSION) and thus we expect guest to set this 2025 * value to first u32 field of eVMCS which should specify eVMCS 2026 * VersionNumber. 2027 * 2028 * Guest should be aware of supported eVMCS versions by host by 2029 * examining CPUID.0x4000000A.EAX[0:15]. Host userspace VMM is 2030 * expected to set this CPUID leaf according to the value 2031 * returned in vmcs_version from nested_enable_evmcs(). 2032 * 2033 * However, it turns out that Microsoft Hyper-V fails to comply 2034 * to their own invented interface: When Hyper-V use eVMCS, it 2035 * just sets first u32 field of eVMCS to revision_id specified 2036 * in MSR_IA32_VMX_BASIC. Instead of used eVMCS version number 2037 * which is one of the supported versions specified in 2038 * CPUID.0x4000000A.EAX[0:15]. 2039 * 2040 * To overcome Hyper-V bug, we accept here either a supported 2041 * eVMCS version or VMCS12 revision_id as valid values for first 2042 * u32 field of eVMCS. 2043 */ 2044 if ((vmx->nested.hv_evmcs->revision_id != KVM_EVMCS_VERSION) && 2045 (vmx->nested.hv_evmcs->revision_id != VMCS12_REVISION)) { 2046 nested_release_evmcs(vcpu); 2047 return EVMPTRLD_VMFAIL; 2048 } 2049 2050 vmx->nested.hv_evmcs_vmptr = evmcs_gpa; 2051 2052 evmcs_gpa_changed = true; 2053 /* 2054 * Unlike normal vmcs12, enlightened vmcs12 is not fully 2055 * reloaded from guest's memory (read only fields, fields not 2056 * present in struct hv_enlightened_vmcs, ...). Make sure there 2057 * are no leftovers. 2058 */ 2059 if (from_launch) { 2060 struct vmcs12 *vmcs12 = get_vmcs12(vcpu); 2061 memset(vmcs12, 0, sizeof(*vmcs12)); 2062 vmcs12->hdr.revision_id = VMCS12_REVISION; 2063 } 2064 2065 } 2066 2067 /* 2068 * Clean fields data can't be used on VMLAUNCH and when we switch 2069 * between different L2 guests as KVM keeps a single VMCS12 per L1. 2070 */ 2071 if (from_launch || evmcs_gpa_changed) { 2072 vmx->nested.hv_evmcs->hv_clean_fields &= 2073 ~HV_VMX_ENLIGHTENED_CLEAN_FIELD_ALL; 2074 2075 vmx->nested.force_msr_bitmap_recalc = true; 2076 } 2077 2078 return EVMPTRLD_SUCCEEDED; 2079 } 2080 2081 void nested_sync_vmcs12_to_shadow(struct kvm_vcpu *vcpu) 2082 { 2083 struct vcpu_vmx *vmx = to_vmx(vcpu); 2084 2085 if (evmptr_is_valid(vmx->nested.hv_evmcs_vmptr)) 2086 copy_vmcs12_to_enlightened(vmx); 2087 else 2088 copy_vmcs12_to_shadow(vmx); 2089 2090 vmx->nested.need_vmcs12_to_shadow_sync = false; 2091 } 2092 2093 static enum hrtimer_restart vmx_preemption_timer_fn(struct hrtimer *timer) 2094 { 2095 struct vcpu_vmx *vmx = 2096 container_of(timer, struct vcpu_vmx, nested.preemption_timer); 2097 2098 vmx->nested.preemption_timer_expired = true; 2099 kvm_make_request(KVM_REQ_EVENT, &vmx->vcpu); 2100 kvm_vcpu_kick(&vmx->vcpu); 2101 2102 return HRTIMER_NORESTART; 2103 } 2104 2105 static u64 vmx_calc_preemption_timer_value(struct kvm_vcpu *vcpu) 2106 { 2107 struct vcpu_vmx *vmx = to_vmx(vcpu); 2108 struct vmcs12 *vmcs12 = get_vmcs12(vcpu); 2109 2110 u64 l1_scaled_tsc = kvm_read_l1_tsc(vcpu, rdtsc()) >> 2111 VMX_MISC_EMULATED_PREEMPTION_TIMER_RATE; 2112 2113 if (!vmx->nested.has_preemption_timer_deadline) { 2114 vmx->nested.preemption_timer_deadline = 2115 vmcs12->vmx_preemption_timer_value + l1_scaled_tsc; 2116 vmx->nested.has_preemption_timer_deadline = true; 2117 } 2118 return vmx->nested.preemption_timer_deadline - l1_scaled_tsc; 2119 } 2120 2121 static void vmx_start_preemption_timer(struct kvm_vcpu *vcpu, 2122 u64 preemption_timeout) 2123 { 2124 struct vcpu_vmx *vmx = to_vmx(vcpu); 2125 2126 /* 2127 * A timer value of zero is architecturally guaranteed to cause 2128 * a VMExit prior to executing any instructions in the guest. 2129 */ 2130 if (preemption_timeout == 0) { 2131 vmx_preemption_timer_fn(&vmx->nested.preemption_timer); 2132 return; 2133 } 2134 2135 if (vcpu->arch.virtual_tsc_khz == 0) 2136 return; 2137 2138 preemption_timeout <<= VMX_MISC_EMULATED_PREEMPTION_TIMER_RATE; 2139 preemption_timeout *= 1000000; 2140 do_div(preemption_timeout, vcpu->arch.virtual_tsc_khz); 2141 hrtimer_start(&vmx->nested.preemption_timer, 2142 ktime_add_ns(ktime_get(), preemption_timeout), 2143 HRTIMER_MODE_ABS_PINNED); 2144 } 2145 2146 static u64 nested_vmx_calc_efer(struct vcpu_vmx *vmx, struct vmcs12 *vmcs12) 2147 { 2148 if (vmx->nested.nested_run_pending && 2149 (vmcs12->vm_entry_controls & VM_ENTRY_LOAD_IA32_EFER)) 2150 return vmcs12->guest_ia32_efer; 2151 else if (vmcs12->vm_entry_controls & VM_ENTRY_IA32E_MODE) 2152 return vmx->vcpu.arch.efer | (EFER_LMA | EFER_LME); 2153 else 2154 return vmx->vcpu.arch.efer & ~(EFER_LMA | EFER_LME); 2155 } 2156 2157 static void prepare_vmcs02_constant_state(struct vcpu_vmx *vmx) 2158 { 2159 struct kvm *kvm = vmx->vcpu.kvm; 2160 2161 /* 2162 * If vmcs02 hasn't been initialized, set the constant vmcs02 state 2163 * according to L0's settings (vmcs12 is irrelevant here). Host 2164 * fields that come from L0 and are not constant, e.g. HOST_CR3, 2165 * will be set as needed prior to VMLAUNCH/VMRESUME. 2166 */ 2167 if (vmx->nested.vmcs02_initialized) 2168 return; 2169 vmx->nested.vmcs02_initialized = true; 2170 2171 /* 2172 * We don't care what the EPTP value is we just need to guarantee 2173 * it's valid so we don't get a false positive when doing early 2174 * consistency checks. 2175 */ 2176 if (enable_ept && nested_early_check) 2177 vmcs_write64(EPT_POINTER, 2178 construct_eptp(&vmx->vcpu, 0, PT64_ROOT_4LEVEL)); 2179 2180 /* All VMFUNCs are currently emulated through L0 vmexits. */ 2181 if (cpu_has_vmx_vmfunc()) 2182 vmcs_write64(VM_FUNCTION_CONTROL, 0); 2183 2184 if (cpu_has_vmx_posted_intr()) 2185 vmcs_write16(POSTED_INTR_NV, POSTED_INTR_NESTED_VECTOR); 2186 2187 if (cpu_has_vmx_msr_bitmap()) 2188 vmcs_write64(MSR_BITMAP, __pa(vmx->nested.vmcs02.msr_bitmap)); 2189 2190 /* 2191 * PML is emulated for L2, but never enabled in hardware as the MMU 2192 * handles A/D emulation. Disabling PML for L2 also avoids having to 2193 * deal with filtering out L2 GPAs from the buffer. 2194 */ 2195 if (enable_pml) { 2196 vmcs_write64(PML_ADDRESS, 0); 2197 vmcs_write16(GUEST_PML_INDEX, -1); 2198 } 2199 2200 if (cpu_has_vmx_encls_vmexit()) 2201 vmcs_write64(ENCLS_EXITING_BITMAP, INVALID_GPA); 2202 2203 if (kvm_notify_vmexit_enabled(kvm)) 2204 vmcs_write32(NOTIFY_WINDOW, kvm->arch.notify_window); 2205 2206 /* 2207 * Set the MSR load/store lists to match L0's settings. Only the 2208 * addresses are constant (for vmcs02), the counts can change based 2209 * on L2's behavior, e.g. switching to/from long mode. 2210 */ 2211 vmcs_write64(VM_EXIT_MSR_STORE_ADDR, __pa(vmx->msr_autostore.guest.val)); 2212 vmcs_write64(VM_EXIT_MSR_LOAD_ADDR, __pa(vmx->msr_autoload.host.val)); 2213 vmcs_write64(VM_ENTRY_MSR_LOAD_ADDR, __pa(vmx->msr_autoload.guest.val)); 2214 2215 vmx_set_constant_host_state(vmx); 2216 } 2217 2218 static void prepare_vmcs02_early_rare(struct vcpu_vmx *vmx, 2219 struct vmcs12 *vmcs12) 2220 { 2221 prepare_vmcs02_constant_state(vmx); 2222 2223 vmcs_write64(VMCS_LINK_POINTER, INVALID_GPA); 2224 2225 if (enable_vpid) { 2226 if (nested_cpu_has_vpid(vmcs12) && vmx->nested.vpid02) 2227 vmcs_write16(VIRTUAL_PROCESSOR_ID, vmx->nested.vpid02); 2228 else 2229 vmcs_write16(VIRTUAL_PROCESSOR_ID, vmx->vpid); 2230 } 2231 } 2232 2233 static void prepare_vmcs02_early(struct vcpu_vmx *vmx, struct loaded_vmcs *vmcs01, 2234 struct vmcs12 *vmcs12) 2235 { 2236 u32 exec_control; 2237 u64 guest_efer = nested_vmx_calc_efer(vmx, vmcs12); 2238 2239 if (vmx->nested.dirty_vmcs12 || evmptr_is_valid(vmx->nested.hv_evmcs_vmptr)) 2240 prepare_vmcs02_early_rare(vmx, vmcs12); 2241 2242 /* 2243 * PIN CONTROLS 2244 */ 2245 exec_control = __pin_controls_get(vmcs01); 2246 exec_control |= (vmcs12->pin_based_vm_exec_control & 2247 ~PIN_BASED_VMX_PREEMPTION_TIMER); 2248 2249 /* Posted interrupts setting is only taken from vmcs12. */ 2250 vmx->nested.pi_pending = false; 2251 if (nested_cpu_has_posted_intr(vmcs12)) 2252 vmx->nested.posted_intr_nv = vmcs12->posted_intr_nv; 2253 else 2254 exec_control &= ~PIN_BASED_POSTED_INTR; 2255 pin_controls_set(vmx, exec_control); 2256 2257 /* 2258 * EXEC CONTROLS 2259 */ 2260 exec_control = __exec_controls_get(vmcs01); /* L0's desires */ 2261 exec_control &= ~CPU_BASED_INTR_WINDOW_EXITING; 2262 exec_control &= ~CPU_BASED_NMI_WINDOW_EXITING; 2263 exec_control &= ~CPU_BASED_TPR_SHADOW; 2264 exec_control |= vmcs12->cpu_based_vm_exec_control; 2265 2266 vmx->nested.l1_tpr_threshold = -1; 2267 if (exec_control & CPU_BASED_TPR_SHADOW) 2268 vmcs_write32(TPR_THRESHOLD, vmcs12->tpr_threshold); 2269 #ifdef CONFIG_X86_64 2270 else 2271 exec_control |= CPU_BASED_CR8_LOAD_EXITING | 2272 CPU_BASED_CR8_STORE_EXITING; 2273 #endif 2274 2275 /* 2276 * A vmexit (to either L1 hypervisor or L0 userspace) is always needed 2277 * for I/O port accesses. 2278 */ 2279 exec_control |= CPU_BASED_UNCOND_IO_EXITING; 2280 exec_control &= ~CPU_BASED_USE_IO_BITMAPS; 2281 2282 /* 2283 * This bit will be computed in nested_get_vmcs12_pages, because 2284 * we do not have access to L1's MSR bitmap yet. For now, keep 2285 * the same bit as before, hoping to avoid multiple VMWRITEs that 2286 * only set/clear this bit. 2287 */ 2288 exec_control &= ~CPU_BASED_USE_MSR_BITMAPS; 2289 exec_control |= exec_controls_get(vmx) & CPU_BASED_USE_MSR_BITMAPS; 2290 2291 exec_controls_set(vmx, exec_control); 2292 2293 /* 2294 * SECONDARY EXEC CONTROLS 2295 */ 2296 if (cpu_has_secondary_exec_ctrls()) { 2297 exec_control = __secondary_exec_controls_get(vmcs01); 2298 2299 /* Take the following fields only from vmcs12 */ 2300 exec_control &= ~(SECONDARY_EXEC_VIRTUALIZE_APIC_ACCESSES | 2301 SECONDARY_EXEC_VIRTUALIZE_X2APIC_MODE | 2302 SECONDARY_EXEC_ENABLE_INVPCID | 2303 SECONDARY_EXEC_ENABLE_RDTSCP | 2304 SECONDARY_EXEC_XSAVES | 2305 SECONDARY_EXEC_ENABLE_USR_WAIT_PAUSE | 2306 SECONDARY_EXEC_VIRTUAL_INTR_DELIVERY | 2307 SECONDARY_EXEC_APIC_REGISTER_VIRT | 2308 SECONDARY_EXEC_ENABLE_VMFUNC | 2309 SECONDARY_EXEC_DESC); 2310 2311 if (nested_cpu_has(vmcs12, 2312 CPU_BASED_ACTIVATE_SECONDARY_CONTROLS)) 2313 exec_control |= vmcs12->secondary_vm_exec_control; 2314 2315 /* PML is emulated and never enabled in hardware for L2. */ 2316 exec_control &= ~SECONDARY_EXEC_ENABLE_PML; 2317 2318 /* VMCS shadowing for L2 is emulated for now */ 2319 exec_control &= ~SECONDARY_EXEC_SHADOW_VMCS; 2320 2321 /* 2322 * Preset *DT exiting when emulating UMIP, so that vmx_set_cr4() 2323 * will not have to rewrite the controls just for this bit. 2324 */ 2325 if (!boot_cpu_has(X86_FEATURE_UMIP) && vmx_umip_emulated() && 2326 (vmcs12->guest_cr4 & X86_CR4_UMIP)) 2327 exec_control |= SECONDARY_EXEC_DESC; 2328 2329 if (exec_control & SECONDARY_EXEC_VIRTUAL_INTR_DELIVERY) 2330 vmcs_write16(GUEST_INTR_STATUS, 2331 vmcs12->guest_intr_status); 2332 2333 if (!nested_cpu_has2(vmcs12, SECONDARY_EXEC_UNRESTRICTED_GUEST)) 2334 exec_control &= ~SECONDARY_EXEC_UNRESTRICTED_GUEST; 2335 2336 if (exec_control & SECONDARY_EXEC_ENCLS_EXITING) 2337 vmx_write_encls_bitmap(&vmx->vcpu, vmcs12); 2338 2339 secondary_exec_controls_set(vmx, exec_control); 2340 } 2341 2342 /* 2343 * ENTRY CONTROLS 2344 * 2345 * vmcs12's VM_{ENTRY,EXIT}_LOAD_IA32_EFER and VM_ENTRY_IA32E_MODE 2346 * are emulated by vmx_set_efer() in prepare_vmcs02(), but speculate 2347 * on the related bits (if supported by the CPU) in the hope that 2348 * we can avoid VMWrites during vmx_set_efer(). 2349 * 2350 * Similarly, take vmcs01's PERF_GLOBAL_CTRL in the hope that if KVM is 2351 * loading PERF_GLOBAL_CTRL via the VMCS for L1, then KVM will want to 2352 * do the same for L2. 2353 */ 2354 exec_control = __vm_entry_controls_get(vmcs01); 2355 exec_control |= (vmcs12->vm_entry_controls & 2356 ~VM_ENTRY_LOAD_IA32_PERF_GLOBAL_CTRL); 2357 exec_control &= ~(VM_ENTRY_IA32E_MODE | VM_ENTRY_LOAD_IA32_EFER); 2358 if (cpu_has_load_ia32_efer()) { 2359 if (guest_efer & EFER_LMA) 2360 exec_control |= VM_ENTRY_IA32E_MODE; 2361 if (guest_efer != host_efer) 2362 exec_control |= VM_ENTRY_LOAD_IA32_EFER; 2363 } 2364 vm_entry_controls_set(vmx, exec_control); 2365 2366 /* 2367 * EXIT CONTROLS 2368 * 2369 * L2->L1 exit controls are emulated - the hardware exit is to L0 so 2370 * we should use its exit controls. Note that VM_EXIT_LOAD_IA32_EFER 2371 * bits may be modified by vmx_set_efer() in prepare_vmcs02(). 2372 */ 2373 exec_control = __vm_exit_controls_get(vmcs01); 2374 if (cpu_has_load_ia32_efer() && guest_efer != host_efer) 2375 exec_control |= VM_EXIT_LOAD_IA32_EFER; 2376 else 2377 exec_control &= ~VM_EXIT_LOAD_IA32_EFER; 2378 vm_exit_controls_set(vmx, exec_control); 2379 2380 /* 2381 * Interrupt/Exception Fields 2382 */ 2383 if (vmx->nested.nested_run_pending) { 2384 vmcs_write32(VM_ENTRY_INTR_INFO_FIELD, 2385 vmcs12->vm_entry_intr_info_field); 2386 vmcs_write32(VM_ENTRY_EXCEPTION_ERROR_CODE, 2387 vmcs12->vm_entry_exception_error_code); 2388 vmcs_write32(VM_ENTRY_INSTRUCTION_LEN, 2389 vmcs12->vm_entry_instruction_len); 2390 vmcs_write32(GUEST_INTERRUPTIBILITY_INFO, 2391 vmcs12->guest_interruptibility_info); 2392 vmx->loaded_vmcs->nmi_known_unmasked = 2393 !(vmcs12->guest_interruptibility_info & GUEST_INTR_STATE_NMI); 2394 } else { 2395 vmcs_write32(VM_ENTRY_INTR_INFO_FIELD, 0); 2396 } 2397 } 2398 2399 static void prepare_vmcs02_rare(struct vcpu_vmx *vmx, struct vmcs12 *vmcs12) 2400 { 2401 struct hv_enlightened_vmcs *hv_evmcs = vmx->nested.hv_evmcs; 2402 2403 if (!hv_evmcs || !(hv_evmcs->hv_clean_fields & 2404 HV_VMX_ENLIGHTENED_CLEAN_FIELD_GUEST_GRP2)) { 2405 vmcs_write16(GUEST_ES_SELECTOR, vmcs12->guest_es_selector); 2406 vmcs_write16(GUEST_CS_SELECTOR, vmcs12->guest_cs_selector); 2407 vmcs_write16(GUEST_SS_SELECTOR, vmcs12->guest_ss_selector); 2408 vmcs_write16(GUEST_DS_SELECTOR, vmcs12->guest_ds_selector); 2409 vmcs_write16(GUEST_FS_SELECTOR, vmcs12->guest_fs_selector); 2410 vmcs_write16(GUEST_GS_SELECTOR, vmcs12->guest_gs_selector); 2411 vmcs_write16(GUEST_LDTR_SELECTOR, vmcs12->guest_ldtr_selector); 2412 vmcs_write16(GUEST_TR_SELECTOR, vmcs12->guest_tr_selector); 2413 vmcs_write32(GUEST_ES_LIMIT, vmcs12->guest_es_limit); 2414 vmcs_write32(GUEST_CS_LIMIT, vmcs12->guest_cs_limit); 2415 vmcs_write32(GUEST_SS_LIMIT, vmcs12->guest_ss_limit); 2416 vmcs_write32(GUEST_DS_LIMIT, vmcs12->guest_ds_limit); 2417 vmcs_write32(GUEST_FS_LIMIT, vmcs12->guest_fs_limit); 2418 vmcs_write32(GUEST_GS_LIMIT, vmcs12->guest_gs_limit); 2419 vmcs_write32(GUEST_LDTR_LIMIT, vmcs12->guest_ldtr_limit); 2420 vmcs_write32(GUEST_TR_LIMIT, vmcs12->guest_tr_limit); 2421 vmcs_write32(GUEST_GDTR_LIMIT, vmcs12->guest_gdtr_limit); 2422 vmcs_write32(GUEST_IDTR_LIMIT, vmcs12->guest_idtr_limit); 2423 vmcs_write32(GUEST_CS_AR_BYTES, vmcs12->guest_cs_ar_bytes); 2424 vmcs_write32(GUEST_SS_AR_BYTES, vmcs12->guest_ss_ar_bytes); 2425 vmcs_write32(GUEST_ES_AR_BYTES, vmcs12->guest_es_ar_bytes); 2426 vmcs_write32(GUEST_DS_AR_BYTES, vmcs12->guest_ds_ar_bytes); 2427 vmcs_write32(GUEST_FS_AR_BYTES, vmcs12->guest_fs_ar_bytes); 2428 vmcs_write32(GUEST_GS_AR_BYTES, vmcs12->guest_gs_ar_bytes); 2429 vmcs_write32(GUEST_LDTR_AR_BYTES, vmcs12->guest_ldtr_ar_bytes); 2430 vmcs_write32(GUEST_TR_AR_BYTES, vmcs12->guest_tr_ar_bytes); 2431 vmcs_writel(GUEST_ES_BASE, vmcs12->guest_es_base); 2432 vmcs_writel(GUEST_CS_BASE, vmcs12->guest_cs_base); 2433 vmcs_writel(GUEST_SS_BASE, vmcs12->guest_ss_base); 2434 vmcs_writel(GUEST_DS_BASE, vmcs12->guest_ds_base); 2435 vmcs_writel(GUEST_FS_BASE, vmcs12->guest_fs_base); 2436 vmcs_writel(GUEST_GS_BASE, vmcs12->guest_gs_base); 2437 vmcs_writel(GUEST_LDTR_BASE, vmcs12->guest_ldtr_base); 2438 vmcs_writel(GUEST_TR_BASE, vmcs12->guest_tr_base); 2439 vmcs_writel(GUEST_GDTR_BASE, vmcs12->guest_gdtr_base); 2440 vmcs_writel(GUEST_IDTR_BASE, vmcs12->guest_idtr_base); 2441 2442 vmx->segment_cache.bitmask = 0; 2443 } 2444 2445 if (!hv_evmcs || !(hv_evmcs->hv_clean_fields & 2446 HV_VMX_ENLIGHTENED_CLEAN_FIELD_GUEST_GRP1)) { 2447 vmcs_write32(GUEST_SYSENTER_CS, vmcs12->guest_sysenter_cs); 2448 vmcs_writel(GUEST_PENDING_DBG_EXCEPTIONS, 2449 vmcs12->guest_pending_dbg_exceptions); 2450 vmcs_writel(GUEST_SYSENTER_ESP, vmcs12->guest_sysenter_esp); 2451 vmcs_writel(GUEST_SYSENTER_EIP, vmcs12->guest_sysenter_eip); 2452 2453 /* 2454 * L1 may access the L2's PDPTR, so save them to construct 2455 * vmcs12 2456 */ 2457 if (enable_ept) { 2458 vmcs_write64(GUEST_PDPTR0, vmcs12->guest_pdptr0); 2459 vmcs_write64(GUEST_PDPTR1, vmcs12->guest_pdptr1); 2460 vmcs_write64(GUEST_PDPTR2, vmcs12->guest_pdptr2); 2461 vmcs_write64(GUEST_PDPTR3, vmcs12->guest_pdptr3); 2462 } 2463 2464 if (kvm_mpx_supported() && vmx->nested.nested_run_pending && 2465 (vmcs12->vm_entry_controls & VM_ENTRY_LOAD_BNDCFGS)) 2466 vmcs_write64(GUEST_BNDCFGS, vmcs12->guest_bndcfgs); 2467 } 2468 2469 if (nested_cpu_has_xsaves(vmcs12)) 2470 vmcs_write64(XSS_EXIT_BITMAP, vmcs12->xss_exit_bitmap); 2471 2472 /* 2473 * Whether page-faults are trapped is determined by a combination of 2474 * 3 settings: PFEC_MASK, PFEC_MATCH and EXCEPTION_BITMAP.PF. If L0 2475 * doesn't care about page faults then we should set all of these to 2476 * L1's desires. However, if L0 does care about (some) page faults, it 2477 * is not easy (if at all possible?) to merge L0 and L1's desires, we 2478 * simply ask to exit on each and every L2 page fault. This is done by 2479 * setting MASK=MATCH=0 and (see below) EB.PF=1. 2480 * Note that below we don't need special code to set EB.PF beyond the 2481 * "or"ing of the EB of vmcs01 and vmcs12, because when enable_ept, 2482 * vmcs01's EB.PF is 0 so the "or" will take vmcs12's value, and when 2483 * !enable_ept, EB.PF is 1, so the "or" will always be 1. 2484 */ 2485 if (vmx_need_pf_intercept(&vmx->vcpu)) { 2486 /* 2487 * TODO: if both L0 and L1 need the same MASK and MATCH, 2488 * go ahead and use it? 2489 */ 2490 vmcs_write32(PAGE_FAULT_ERROR_CODE_MASK, 0); 2491 vmcs_write32(PAGE_FAULT_ERROR_CODE_MATCH, 0); 2492 } else { 2493 vmcs_write32(PAGE_FAULT_ERROR_CODE_MASK, vmcs12->page_fault_error_code_mask); 2494 vmcs_write32(PAGE_FAULT_ERROR_CODE_MATCH, vmcs12->page_fault_error_code_match); 2495 } 2496 2497 if (cpu_has_vmx_apicv()) { 2498 vmcs_write64(EOI_EXIT_BITMAP0, vmcs12->eoi_exit_bitmap0); 2499 vmcs_write64(EOI_EXIT_BITMAP1, vmcs12->eoi_exit_bitmap1); 2500 vmcs_write64(EOI_EXIT_BITMAP2, vmcs12->eoi_exit_bitmap2); 2501 vmcs_write64(EOI_EXIT_BITMAP3, vmcs12->eoi_exit_bitmap3); 2502 } 2503 2504 /* 2505 * Make sure the msr_autostore list is up to date before we set the 2506 * count in the vmcs02. 2507 */ 2508 prepare_vmx_msr_autostore_list(&vmx->vcpu, MSR_IA32_TSC); 2509 2510 vmcs_write32(VM_EXIT_MSR_STORE_COUNT, vmx->msr_autostore.guest.nr); 2511 vmcs_write32(VM_EXIT_MSR_LOAD_COUNT, vmx->msr_autoload.host.nr); 2512 vmcs_write32(VM_ENTRY_MSR_LOAD_COUNT, vmx->msr_autoload.guest.nr); 2513 2514 set_cr4_guest_host_mask(vmx); 2515 } 2516 2517 /* 2518 * prepare_vmcs02 is called when the L1 guest hypervisor runs its nested 2519 * L2 guest. L1 has a vmcs for L2 (vmcs12), and this function "merges" it 2520 * with L0's requirements for its guest (a.k.a. vmcs01), so we can run the L2 2521 * guest in a way that will both be appropriate to L1's requests, and our 2522 * needs. In addition to modifying the active vmcs (which is vmcs02), this 2523 * function also has additional necessary side-effects, like setting various 2524 * vcpu->arch fields. 2525 * Returns 0 on success, 1 on failure. Invalid state exit qualification code 2526 * is assigned to entry_failure_code on failure. 2527 */ 2528 static int prepare_vmcs02(struct kvm_vcpu *vcpu, struct vmcs12 *vmcs12, 2529 bool from_vmentry, 2530 enum vm_entry_failure_code *entry_failure_code) 2531 { 2532 struct vcpu_vmx *vmx = to_vmx(vcpu); 2533 bool load_guest_pdptrs_vmcs12 = false; 2534 2535 if (vmx->nested.dirty_vmcs12 || evmptr_is_valid(vmx->nested.hv_evmcs_vmptr)) { 2536 prepare_vmcs02_rare(vmx, vmcs12); 2537 vmx->nested.dirty_vmcs12 = false; 2538 2539 load_guest_pdptrs_vmcs12 = !evmptr_is_valid(vmx->nested.hv_evmcs_vmptr) || 2540 !(vmx->nested.hv_evmcs->hv_clean_fields & 2541 HV_VMX_ENLIGHTENED_CLEAN_FIELD_GUEST_GRP1); 2542 } 2543 2544 if (vmx->nested.nested_run_pending && 2545 (vmcs12->vm_entry_controls & VM_ENTRY_LOAD_DEBUG_CONTROLS)) { 2546 kvm_set_dr(vcpu, 7, vmcs12->guest_dr7); 2547 vmcs_write64(GUEST_IA32_DEBUGCTL, vmcs12->guest_ia32_debugctl); 2548 } else { 2549 kvm_set_dr(vcpu, 7, vcpu->arch.dr7); 2550 vmcs_write64(GUEST_IA32_DEBUGCTL, vmx->nested.pre_vmenter_debugctl); 2551 } 2552 if (kvm_mpx_supported() && (!vmx->nested.nested_run_pending || 2553 !(vmcs12->vm_entry_controls & VM_ENTRY_LOAD_BNDCFGS))) 2554 vmcs_write64(GUEST_BNDCFGS, vmx->nested.pre_vmenter_bndcfgs); 2555 vmx_set_rflags(vcpu, vmcs12->guest_rflags); 2556 2557 /* EXCEPTION_BITMAP and CR0_GUEST_HOST_MASK should basically be the 2558 * bitwise-or of what L1 wants to trap for L2, and what we want to 2559 * trap. Note that CR0.TS also needs updating - we do this later. 2560 */ 2561 vmx_update_exception_bitmap(vcpu); 2562 vcpu->arch.cr0_guest_owned_bits &= ~vmcs12->cr0_guest_host_mask; 2563 vmcs_writel(CR0_GUEST_HOST_MASK, ~vcpu->arch.cr0_guest_owned_bits); 2564 2565 if (vmx->nested.nested_run_pending && 2566 (vmcs12->vm_entry_controls & VM_ENTRY_LOAD_IA32_PAT)) { 2567 vmcs_write64(GUEST_IA32_PAT, vmcs12->guest_ia32_pat); 2568 vcpu->arch.pat = vmcs12->guest_ia32_pat; 2569 } else if (vmcs_config.vmentry_ctrl & VM_ENTRY_LOAD_IA32_PAT) { 2570 vmcs_write64(GUEST_IA32_PAT, vmx->vcpu.arch.pat); 2571 } 2572 2573 vcpu->arch.tsc_offset = kvm_calc_nested_tsc_offset( 2574 vcpu->arch.l1_tsc_offset, 2575 vmx_get_l2_tsc_offset(vcpu), 2576 vmx_get_l2_tsc_multiplier(vcpu)); 2577 2578 vcpu->arch.tsc_scaling_ratio = kvm_calc_nested_tsc_multiplier( 2579 vcpu->arch.l1_tsc_scaling_ratio, 2580 vmx_get_l2_tsc_multiplier(vcpu)); 2581 2582 vmcs_write64(TSC_OFFSET, vcpu->arch.tsc_offset); 2583 if (kvm_caps.has_tsc_control) 2584 vmcs_write64(TSC_MULTIPLIER, vcpu->arch.tsc_scaling_ratio); 2585 2586 nested_vmx_transition_tlb_flush(vcpu, vmcs12, true); 2587 2588 if (nested_cpu_has_ept(vmcs12)) 2589 nested_ept_init_mmu_context(vcpu); 2590 2591 /* 2592 * Override the CR0/CR4 read shadows after setting the effective guest 2593 * CR0/CR4. The common helpers also set the shadows, but they don't 2594 * account for vmcs12's cr0/4_guest_host_mask. 2595 */ 2596 vmx_set_cr0(vcpu, vmcs12->guest_cr0); 2597 vmcs_writel(CR0_READ_SHADOW, nested_read_cr0(vmcs12)); 2598 2599 vmx_set_cr4(vcpu, vmcs12->guest_cr4); 2600 vmcs_writel(CR4_READ_SHADOW, nested_read_cr4(vmcs12)); 2601 2602 vcpu->arch.efer = nested_vmx_calc_efer(vmx, vmcs12); 2603 /* Note: may modify VM_ENTRY/EXIT_CONTROLS and GUEST/HOST_IA32_EFER */ 2604 vmx_set_efer(vcpu, vcpu->arch.efer); 2605 2606 /* 2607 * Guest state is invalid and unrestricted guest is disabled, 2608 * which means L1 attempted VMEntry to L2 with invalid state. 2609 * Fail the VMEntry. 2610 * 2611 * However when force loading the guest state (SMM exit or 2612 * loading nested state after migration, it is possible to 2613 * have invalid guest state now, which will be later fixed by 2614 * restoring L2 register state 2615 */ 2616 if (CC(from_vmentry && !vmx_guest_state_valid(vcpu))) { 2617 *entry_failure_code = ENTRY_FAIL_DEFAULT; 2618 return -EINVAL; 2619 } 2620 2621 /* Shadow page tables on either EPT or shadow page tables. */ 2622 if (nested_vmx_load_cr3(vcpu, vmcs12->guest_cr3, nested_cpu_has_ept(vmcs12), 2623 from_vmentry, entry_failure_code)) 2624 return -EINVAL; 2625 2626 /* 2627 * Immediately write vmcs02.GUEST_CR3. It will be propagated to vmcs12 2628 * on nested VM-Exit, which can occur without actually running L2 and 2629 * thus without hitting vmx_load_mmu_pgd(), e.g. if L1 is entering L2 with 2630 * vmcs12.GUEST_ACTIVITYSTATE=HLT, in which case KVM will intercept the 2631 * transition to HLT instead of running L2. 2632 */ 2633 if (enable_ept) 2634 vmcs_writel(GUEST_CR3, vmcs12->guest_cr3); 2635 2636 /* Late preparation of GUEST_PDPTRs now that EFER and CRs are set. */ 2637 if (load_guest_pdptrs_vmcs12 && nested_cpu_has_ept(vmcs12) && 2638 is_pae_paging(vcpu)) { 2639 vmcs_write64(GUEST_PDPTR0, vmcs12->guest_pdptr0); 2640 vmcs_write64(GUEST_PDPTR1, vmcs12->guest_pdptr1); 2641 vmcs_write64(GUEST_PDPTR2, vmcs12->guest_pdptr2); 2642 vmcs_write64(GUEST_PDPTR3, vmcs12->guest_pdptr3); 2643 } 2644 2645 if ((vmcs12->vm_entry_controls & VM_ENTRY_LOAD_IA32_PERF_GLOBAL_CTRL) && 2646 intel_pmu_has_perf_global_ctrl(vcpu_to_pmu(vcpu)) && 2647 WARN_ON_ONCE(kvm_set_msr(vcpu, MSR_CORE_PERF_GLOBAL_CTRL, 2648 vmcs12->guest_ia32_perf_global_ctrl))) { 2649 *entry_failure_code = ENTRY_FAIL_DEFAULT; 2650 return -EINVAL; 2651 } 2652 2653 kvm_rsp_write(vcpu, vmcs12->guest_rsp); 2654 kvm_rip_write(vcpu, vmcs12->guest_rip); 2655 2656 /* 2657 * It was observed that genuine Hyper-V running in L1 doesn't reset 2658 * 'hv_clean_fields' by itself, it only sets the corresponding dirty 2659 * bits when it changes a field in eVMCS. Mark all fields as clean 2660 * here. 2661 */ 2662 if (evmptr_is_valid(vmx->nested.hv_evmcs_vmptr)) 2663 vmx->nested.hv_evmcs->hv_clean_fields |= 2664 HV_VMX_ENLIGHTENED_CLEAN_FIELD_ALL; 2665 2666 return 0; 2667 } 2668 2669 static int nested_vmx_check_nmi_controls(struct vmcs12 *vmcs12) 2670 { 2671 if (CC(!nested_cpu_has_nmi_exiting(vmcs12) && 2672 nested_cpu_has_virtual_nmis(vmcs12))) 2673 return -EINVAL; 2674 2675 if (CC(!nested_cpu_has_virtual_nmis(vmcs12) && 2676 nested_cpu_has(vmcs12, CPU_BASED_NMI_WINDOW_EXITING))) 2677 return -EINVAL; 2678 2679 return 0; 2680 } 2681 2682 static bool nested_vmx_check_eptp(struct kvm_vcpu *vcpu, u64 new_eptp) 2683 { 2684 struct vcpu_vmx *vmx = to_vmx(vcpu); 2685 2686 /* Check for memory type validity */ 2687 switch (new_eptp & VMX_EPTP_MT_MASK) { 2688 case VMX_EPTP_MT_UC: 2689 if (CC(!(vmx->nested.msrs.ept_caps & VMX_EPTP_UC_BIT))) 2690 return false; 2691 break; 2692 case VMX_EPTP_MT_WB: 2693 if (CC(!(vmx->nested.msrs.ept_caps & VMX_EPTP_WB_BIT))) 2694 return false; 2695 break; 2696 default: 2697 return false; 2698 } 2699 2700 /* Page-walk levels validity. */ 2701 switch (new_eptp & VMX_EPTP_PWL_MASK) { 2702 case VMX_EPTP_PWL_5: 2703 if (CC(!(vmx->nested.msrs.ept_caps & VMX_EPT_PAGE_WALK_5_BIT))) 2704 return false; 2705 break; 2706 case VMX_EPTP_PWL_4: 2707 if (CC(!(vmx->nested.msrs.ept_caps & VMX_EPT_PAGE_WALK_4_BIT))) 2708 return false; 2709 break; 2710 default: 2711 return false; 2712 } 2713 2714 /* Reserved bits should not be set */ 2715 if (CC(kvm_vcpu_is_illegal_gpa(vcpu, new_eptp) || ((new_eptp >> 7) & 0x1f))) 2716 return false; 2717 2718 /* AD, if set, should be supported */ 2719 if (new_eptp & VMX_EPTP_AD_ENABLE_BIT) { 2720 if (CC(!(vmx->nested.msrs.ept_caps & VMX_EPT_AD_BIT))) 2721 return false; 2722 } 2723 2724 return true; 2725 } 2726 2727 /* 2728 * Checks related to VM-Execution Control Fields 2729 */ 2730 static int nested_check_vm_execution_controls(struct kvm_vcpu *vcpu, 2731 struct vmcs12 *vmcs12) 2732 { 2733 struct vcpu_vmx *vmx = to_vmx(vcpu); 2734 2735 if (CC(!vmx_control_verify(vmcs12->pin_based_vm_exec_control, 2736 vmx->nested.msrs.pinbased_ctls_low, 2737 vmx->nested.msrs.pinbased_ctls_high)) || 2738 CC(!vmx_control_verify(vmcs12->cpu_based_vm_exec_control, 2739 vmx->nested.msrs.procbased_ctls_low, 2740 vmx->nested.msrs.procbased_ctls_high))) 2741 return -EINVAL; 2742 2743 if (nested_cpu_has(vmcs12, CPU_BASED_ACTIVATE_SECONDARY_CONTROLS) && 2744 CC(!vmx_control_verify(vmcs12->secondary_vm_exec_control, 2745 vmx->nested.msrs.secondary_ctls_low, 2746 vmx->nested.msrs.secondary_ctls_high))) 2747 return -EINVAL; 2748 2749 if (CC(vmcs12->cr3_target_count > nested_cpu_vmx_misc_cr3_count(vcpu)) || 2750 nested_vmx_check_io_bitmap_controls(vcpu, vmcs12) || 2751 nested_vmx_check_msr_bitmap_controls(vcpu, vmcs12) || 2752 nested_vmx_check_tpr_shadow_controls(vcpu, vmcs12) || 2753 nested_vmx_check_apic_access_controls(vcpu, vmcs12) || 2754 nested_vmx_check_apicv_controls(vcpu, vmcs12) || 2755 nested_vmx_check_nmi_controls(vmcs12) || 2756 nested_vmx_check_pml_controls(vcpu, vmcs12) || 2757 nested_vmx_check_unrestricted_guest_controls(vcpu, vmcs12) || 2758 nested_vmx_check_mode_based_ept_exec_controls(vcpu, vmcs12) || 2759 nested_vmx_check_shadow_vmcs_controls(vcpu, vmcs12) || 2760 CC(nested_cpu_has_vpid(vmcs12) && !vmcs12->virtual_processor_id)) 2761 return -EINVAL; 2762 2763 if (!nested_cpu_has_preemption_timer(vmcs12) && 2764 nested_cpu_has_save_preemption_timer(vmcs12)) 2765 return -EINVAL; 2766 2767 if (nested_cpu_has_ept(vmcs12) && 2768 CC(!nested_vmx_check_eptp(vcpu, vmcs12->ept_pointer))) 2769 return -EINVAL; 2770 2771 if (nested_cpu_has_vmfunc(vmcs12)) { 2772 if (CC(vmcs12->vm_function_control & 2773 ~vmx->nested.msrs.vmfunc_controls)) 2774 return -EINVAL; 2775 2776 if (nested_cpu_has_eptp_switching(vmcs12)) { 2777 if (CC(!nested_cpu_has_ept(vmcs12)) || 2778 CC(!page_address_valid(vcpu, vmcs12->eptp_list_address))) 2779 return -EINVAL; 2780 } 2781 } 2782 2783 return 0; 2784 } 2785 2786 /* 2787 * Checks related to VM-Exit Control Fields 2788 */ 2789 static int nested_check_vm_exit_controls(struct kvm_vcpu *vcpu, 2790 struct vmcs12 *vmcs12) 2791 { 2792 struct vcpu_vmx *vmx = to_vmx(vcpu); 2793 2794 if (CC(!vmx_control_verify(vmcs12->vm_exit_controls, 2795 vmx->nested.msrs.exit_ctls_low, 2796 vmx->nested.msrs.exit_ctls_high)) || 2797 CC(nested_vmx_check_exit_msr_switch_controls(vcpu, vmcs12))) 2798 return -EINVAL; 2799 2800 return 0; 2801 } 2802 2803 /* 2804 * Checks related to VM-Entry Control Fields 2805 */ 2806 static int nested_check_vm_entry_controls(struct kvm_vcpu *vcpu, 2807 struct vmcs12 *vmcs12) 2808 { 2809 struct vcpu_vmx *vmx = to_vmx(vcpu); 2810 2811 if (CC(!vmx_control_verify(vmcs12->vm_entry_controls, 2812 vmx->nested.msrs.entry_ctls_low, 2813 vmx->nested.msrs.entry_ctls_high))) 2814 return -EINVAL; 2815 2816 /* 2817 * From the Intel SDM, volume 3: 2818 * Fields relevant to VM-entry event injection must be set properly. 2819 * These fields are the VM-entry interruption-information field, the 2820 * VM-entry exception error code, and the VM-entry instruction length. 2821 */ 2822 if (vmcs12->vm_entry_intr_info_field & INTR_INFO_VALID_MASK) { 2823 u32 intr_info = vmcs12->vm_entry_intr_info_field; 2824 u8 vector = intr_info & INTR_INFO_VECTOR_MASK; 2825 u32 intr_type = intr_info & INTR_INFO_INTR_TYPE_MASK; 2826 bool has_error_code = intr_info & INTR_INFO_DELIVER_CODE_MASK; 2827 bool should_have_error_code; 2828 bool urg = nested_cpu_has2(vmcs12, 2829 SECONDARY_EXEC_UNRESTRICTED_GUEST); 2830 bool prot_mode = !urg || vmcs12->guest_cr0 & X86_CR0_PE; 2831 2832 /* VM-entry interruption-info field: interruption type */ 2833 if (CC(intr_type == INTR_TYPE_RESERVED) || 2834 CC(intr_type == INTR_TYPE_OTHER_EVENT && 2835 !nested_cpu_supports_monitor_trap_flag(vcpu))) 2836 return -EINVAL; 2837 2838 /* VM-entry interruption-info field: vector */ 2839 if (CC(intr_type == INTR_TYPE_NMI_INTR && vector != NMI_VECTOR) || 2840 CC(intr_type == INTR_TYPE_HARD_EXCEPTION && vector > 31) || 2841 CC(intr_type == INTR_TYPE_OTHER_EVENT && vector != 0)) 2842 return -EINVAL; 2843 2844 /* VM-entry interruption-info field: deliver error code */ 2845 should_have_error_code = 2846 intr_type == INTR_TYPE_HARD_EXCEPTION && prot_mode && 2847 x86_exception_has_error_code(vector); 2848 if (CC(has_error_code != should_have_error_code)) 2849 return -EINVAL; 2850 2851 /* VM-entry exception error code */ 2852 if (CC(has_error_code && 2853 vmcs12->vm_entry_exception_error_code & GENMASK(31, 16))) 2854 return -EINVAL; 2855 2856 /* VM-entry interruption-info field: reserved bits */ 2857 if (CC(intr_info & INTR_INFO_RESVD_BITS_MASK)) 2858 return -EINVAL; 2859 2860 /* VM-entry instruction length */ 2861 switch (intr_type) { 2862 case INTR_TYPE_SOFT_EXCEPTION: 2863 case INTR_TYPE_SOFT_INTR: 2864 case INTR_TYPE_PRIV_SW_EXCEPTION: 2865 if (CC(vmcs12->vm_entry_instruction_len > 15) || 2866 CC(vmcs12->vm_entry_instruction_len == 0 && 2867 CC(!nested_cpu_has_zero_length_injection(vcpu)))) 2868 return -EINVAL; 2869 } 2870 } 2871 2872 if (nested_vmx_check_entry_msr_switch_controls(vcpu, vmcs12)) 2873 return -EINVAL; 2874 2875 return 0; 2876 } 2877 2878 static int nested_vmx_check_controls(struct kvm_vcpu *vcpu, 2879 struct vmcs12 *vmcs12) 2880 { 2881 if (nested_check_vm_execution_controls(vcpu, vmcs12) || 2882 nested_check_vm_exit_controls(vcpu, vmcs12) || 2883 nested_check_vm_entry_controls(vcpu, vmcs12)) 2884 return -EINVAL; 2885 2886 if (guest_cpuid_has_evmcs(vcpu)) 2887 return nested_evmcs_check_controls(vmcs12); 2888 2889 return 0; 2890 } 2891 2892 static int nested_vmx_check_address_space_size(struct kvm_vcpu *vcpu, 2893 struct vmcs12 *vmcs12) 2894 { 2895 #ifdef CONFIG_X86_64 2896 if (CC(!!(vmcs12->vm_exit_controls & VM_EXIT_HOST_ADDR_SPACE_SIZE) != 2897 !!(vcpu->arch.efer & EFER_LMA))) 2898 return -EINVAL; 2899 #endif 2900 return 0; 2901 } 2902 2903 static int nested_vmx_check_host_state(struct kvm_vcpu *vcpu, 2904 struct vmcs12 *vmcs12) 2905 { 2906 bool ia32e = !!(vmcs12->vm_exit_controls & VM_EXIT_HOST_ADDR_SPACE_SIZE); 2907 2908 if (CC(!nested_host_cr0_valid(vcpu, vmcs12->host_cr0)) || 2909 CC(!nested_host_cr4_valid(vcpu, vmcs12->host_cr4)) || 2910 CC(kvm_vcpu_is_illegal_gpa(vcpu, vmcs12->host_cr3))) 2911 return -EINVAL; 2912 2913 if (CC(is_noncanonical_address(vmcs12->host_ia32_sysenter_esp, vcpu)) || 2914 CC(is_noncanonical_address(vmcs12->host_ia32_sysenter_eip, vcpu))) 2915 return -EINVAL; 2916 2917 if ((vmcs12->vm_exit_controls & VM_EXIT_LOAD_IA32_PAT) && 2918 CC(!kvm_pat_valid(vmcs12->host_ia32_pat))) 2919 return -EINVAL; 2920 2921 if ((vmcs12->vm_exit_controls & VM_EXIT_LOAD_IA32_PERF_GLOBAL_CTRL) && 2922 CC(!kvm_valid_perf_global_ctrl(vcpu_to_pmu(vcpu), 2923 vmcs12->host_ia32_perf_global_ctrl))) 2924 return -EINVAL; 2925 2926 if (ia32e) { 2927 if (CC(!(vmcs12->host_cr4 & X86_CR4_PAE))) 2928 return -EINVAL; 2929 } else { 2930 if (CC(vmcs12->vm_entry_controls & VM_ENTRY_IA32E_MODE) || 2931 CC(vmcs12->host_cr4 & X86_CR4_PCIDE) || 2932 CC((vmcs12->host_rip) >> 32)) 2933 return -EINVAL; 2934 } 2935 2936 if (CC(vmcs12->host_cs_selector & (SEGMENT_RPL_MASK | SEGMENT_TI_MASK)) || 2937 CC(vmcs12->host_ss_selector & (SEGMENT_RPL_MASK | SEGMENT_TI_MASK)) || 2938 CC(vmcs12->host_ds_selector & (SEGMENT_RPL_MASK | SEGMENT_TI_MASK)) || 2939 CC(vmcs12->host_es_selector & (SEGMENT_RPL_MASK | SEGMENT_TI_MASK)) || 2940 CC(vmcs12->host_fs_selector & (SEGMENT_RPL_MASK | SEGMENT_TI_MASK)) || 2941 CC(vmcs12->host_gs_selector & (SEGMENT_RPL_MASK | SEGMENT_TI_MASK)) || 2942 CC(vmcs12->host_tr_selector & (SEGMENT_RPL_MASK | SEGMENT_TI_MASK)) || 2943 CC(vmcs12->host_cs_selector == 0) || 2944 CC(vmcs12->host_tr_selector == 0) || 2945 CC(vmcs12->host_ss_selector == 0 && !ia32e)) 2946 return -EINVAL; 2947 2948 if (CC(is_noncanonical_address(vmcs12->host_fs_base, vcpu)) || 2949 CC(is_noncanonical_address(vmcs12->host_gs_base, vcpu)) || 2950 CC(is_noncanonical_address(vmcs12->host_gdtr_base, vcpu)) || 2951 CC(is_noncanonical_address(vmcs12->host_idtr_base, vcpu)) || 2952 CC(is_noncanonical_address(vmcs12->host_tr_base, vcpu)) || 2953 CC(is_noncanonical_address(vmcs12->host_rip, vcpu))) 2954 return -EINVAL; 2955 2956 /* 2957 * If the load IA32_EFER VM-exit control is 1, bits reserved in the 2958 * IA32_EFER MSR must be 0 in the field for that register. In addition, 2959 * the values of the LMA and LME bits in the field must each be that of 2960 * the host address-space size VM-exit control. 2961 */ 2962 if (vmcs12->vm_exit_controls & VM_EXIT_LOAD_IA32_EFER) { 2963 if (CC(!kvm_valid_efer(vcpu, vmcs12->host_ia32_efer)) || 2964 CC(ia32e != !!(vmcs12->host_ia32_efer & EFER_LMA)) || 2965 CC(ia32e != !!(vmcs12->host_ia32_efer & EFER_LME))) 2966 return -EINVAL; 2967 } 2968 2969 return 0; 2970 } 2971 2972 static int nested_vmx_check_vmcs_link_ptr(struct kvm_vcpu *vcpu, 2973 struct vmcs12 *vmcs12) 2974 { 2975 struct vcpu_vmx *vmx = to_vmx(vcpu); 2976 struct gfn_to_hva_cache *ghc = &vmx->nested.shadow_vmcs12_cache; 2977 struct vmcs_hdr hdr; 2978 2979 if (vmcs12->vmcs_link_pointer == INVALID_GPA) 2980 return 0; 2981 2982 if (CC(!page_address_valid(vcpu, vmcs12->vmcs_link_pointer))) 2983 return -EINVAL; 2984 2985 if (ghc->gpa != vmcs12->vmcs_link_pointer && 2986 CC(kvm_gfn_to_hva_cache_init(vcpu->kvm, ghc, 2987 vmcs12->vmcs_link_pointer, VMCS12_SIZE))) 2988 return -EINVAL; 2989 2990 if (CC(kvm_read_guest_offset_cached(vcpu->kvm, ghc, &hdr, 2991 offsetof(struct vmcs12, hdr), 2992 sizeof(hdr)))) 2993 return -EINVAL; 2994 2995 if (CC(hdr.revision_id != VMCS12_REVISION) || 2996 CC(hdr.shadow_vmcs != nested_cpu_has_shadow_vmcs(vmcs12))) 2997 return -EINVAL; 2998 2999 return 0; 3000 } 3001 3002 /* 3003 * Checks related to Guest Non-register State 3004 */ 3005 static int nested_check_guest_non_reg_state(struct vmcs12 *vmcs12) 3006 { 3007 if (CC(vmcs12->guest_activity_state != GUEST_ACTIVITY_ACTIVE && 3008 vmcs12->guest_activity_state != GUEST_ACTIVITY_HLT && 3009 vmcs12->guest_activity_state != GUEST_ACTIVITY_WAIT_SIPI)) 3010 return -EINVAL; 3011 3012 return 0; 3013 } 3014 3015 static int nested_vmx_check_guest_state(struct kvm_vcpu *vcpu, 3016 struct vmcs12 *vmcs12, 3017 enum vm_entry_failure_code *entry_failure_code) 3018 { 3019 bool ia32e = !!(vmcs12->vm_entry_controls & VM_ENTRY_IA32E_MODE); 3020 3021 *entry_failure_code = ENTRY_FAIL_DEFAULT; 3022 3023 if (CC(!nested_guest_cr0_valid(vcpu, vmcs12->guest_cr0)) || 3024 CC(!nested_guest_cr4_valid(vcpu, vmcs12->guest_cr4))) 3025 return -EINVAL; 3026 3027 if ((vmcs12->vm_entry_controls & VM_ENTRY_LOAD_DEBUG_CONTROLS) && 3028 CC(!kvm_dr7_valid(vmcs12->guest_dr7))) 3029 return -EINVAL; 3030 3031 if ((vmcs12->vm_entry_controls & VM_ENTRY_LOAD_IA32_PAT) && 3032 CC(!kvm_pat_valid(vmcs12->guest_ia32_pat))) 3033 return -EINVAL; 3034 3035 if (nested_vmx_check_vmcs_link_ptr(vcpu, vmcs12)) { 3036 *entry_failure_code = ENTRY_FAIL_VMCS_LINK_PTR; 3037 return -EINVAL; 3038 } 3039 3040 if ((vmcs12->vm_entry_controls & VM_ENTRY_LOAD_IA32_PERF_GLOBAL_CTRL) && 3041 CC(!kvm_valid_perf_global_ctrl(vcpu_to_pmu(vcpu), 3042 vmcs12->guest_ia32_perf_global_ctrl))) 3043 return -EINVAL; 3044 3045 if (CC((vmcs12->guest_cr0 & (X86_CR0_PG | X86_CR0_PE)) == X86_CR0_PG)) 3046 return -EINVAL; 3047 3048 if (CC(ia32e && !(vmcs12->guest_cr4 & X86_CR4_PAE)) || 3049 CC(ia32e && !(vmcs12->guest_cr0 & X86_CR0_PG))) 3050 return -EINVAL; 3051 3052 /* 3053 * If the load IA32_EFER VM-entry control is 1, the following checks 3054 * are performed on the field for the IA32_EFER MSR: 3055 * - Bits reserved in the IA32_EFER MSR must be 0. 3056 * - Bit 10 (corresponding to IA32_EFER.LMA) must equal the value of 3057 * the IA-32e mode guest VM-exit control. It must also be identical 3058 * to bit 8 (LME) if bit 31 in the CR0 field (corresponding to 3059 * CR0.PG) is 1. 3060 */ 3061 if (to_vmx(vcpu)->nested.nested_run_pending && 3062 (vmcs12->vm_entry_controls & VM_ENTRY_LOAD_IA32_EFER)) { 3063 if (CC(!kvm_valid_efer(vcpu, vmcs12->guest_ia32_efer)) || 3064 CC(ia32e != !!(vmcs12->guest_ia32_efer & EFER_LMA)) || 3065 CC(((vmcs12->guest_cr0 & X86_CR0_PG) && 3066 ia32e != !!(vmcs12->guest_ia32_efer & EFER_LME)))) 3067 return -EINVAL; 3068 } 3069 3070 if ((vmcs12->vm_entry_controls & VM_ENTRY_LOAD_BNDCFGS) && 3071 (CC(is_noncanonical_address(vmcs12->guest_bndcfgs & PAGE_MASK, vcpu)) || 3072 CC((vmcs12->guest_bndcfgs & MSR_IA32_BNDCFGS_RSVD)))) 3073 return -EINVAL; 3074 3075 if (nested_check_guest_non_reg_state(vmcs12)) 3076 return -EINVAL; 3077 3078 return 0; 3079 } 3080 3081 static int nested_vmx_check_vmentry_hw(struct kvm_vcpu *vcpu) 3082 { 3083 struct vcpu_vmx *vmx = to_vmx(vcpu); 3084 unsigned long cr3, cr4; 3085 bool vm_fail; 3086 3087 if (!nested_early_check) 3088 return 0; 3089 3090 if (vmx->msr_autoload.host.nr) 3091 vmcs_write32(VM_EXIT_MSR_LOAD_COUNT, 0); 3092 if (vmx->msr_autoload.guest.nr) 3093 vmcs_write32(VM_ENTRY_MSR_LOAD_COUNT, 0); 3094 3095 preempt_disable(); 3096 3097 vmx_prepare_switch_to_guest(vcpu); 3098 3099 /* 3100 * Induce a consistency check VMExit by clearing bit 1 in GUEST_RFLAGS, 3101 * which is reserved to '1' by hardware. GUEST_RFLAGS is guaranteed to 3102 * be written (by prepare_vmcs02()) before the "real" VMEnter, i.e. 3103 * there is no need to preserve other bits or save/restore the field. 3104 */ 3105 vmcs_writel(GUEST_RFLAGS, 0); 3106 3107 cr3 = __get_current_cr3_fast(); 3108 if (unlikely(cr3 != vmx->loaded_vmcs->host_state.cr3)) { 3109 vmcs_writel(HOST_CR3, cr3); 3110 vmx->loaded_vmcs->host_state.cr3 = cr3; 3111 } 3112 3113 cr4 = cr4_read_shadow(); 3114 if (unlikely(cr4 != vmx->loaded_vmcs->host_state.cr4)) { 3115 vmcs_writel(HOST_CR4, cr4); 3116 vmx->loaded_vmcs->host_state.cr4 = cr4; 3117 } 3118 3119 vm_fail = __vmx_vcpu_run(vmx, (unsigned long *)&vcpu->arch.regs, 3120 __vmx_vcpu_run_flags(vmx)); 3121 3122 if (vmx->msr_autoload.host.nr) 3123 vmcs_write32(VM_EXIT_MSR_LOAD_COUNT, vmx->msr_autoload.host.nr); 3124 if (vmx->msr_autoload.guest.nr) 3125 vmcs_write32(VM_ENTRY_MSR_LOAD_COUNT, vmx->msr_autoload.guest.nr); 3126 3127 if (vm_fail) { 3128 u32 error = vmcs_read32(VM_INSTRUCTION_ERROR); 3129 3130 preempt_enable(); 3131 3132 trace_kvm_nested_vmenter_failed( 3133 "early hardware check VM-instruction error: ", error); 3134 WARN_ON_ONCE(error != VMXERR_ENTRY_INVALID_CONTROL_FIELD); 3135 return 1; 3136 } 3137 3138 /* 3139 * VMExit clears RFLAGS.IF and DR7, even on a consistency check. 3140 */ 3141 if (hw_breakpoint_active()) 3142 set_debugreg(__this_cpu_read(cpu_dr7), 7); 3143 local_irq_enable(); 3144 preempt_enable(); 3145 3146 /* 3147 * A non-failing VMEntry means we somehow entered guest mode with 3148 * an illegal RIP, and that's just the tip of the iceberg. There 3149 * is no telling what memory has been modified or what state has 3150 * been exposed to unknown code. Hitting this all but guarantees 3151 * a (very critical) hardware issue. 3152 */ 3153 WARN_ON(!(vmcs_read32(VM_EXIT_REASON) & 3154 VMX_EXIT_REASONS_FAILED_VMENTRY)); 3155 3156 return 0; 3157 } 3158 3159 static bool nested_get_evmcs_page(struct kvm_vcpu *vcpu) 3160 { 3161 struct vcpu_vmx *vmx = to_vmx(vcpu); 3162 3163 /* 3164 * hv_evmcs may end up being not mapped after migration (when 3165 * L2 was running), map it here to make sure vmcs12 changes are 3166 * properly reflected. 3167 */ 3168 if (guest_cpuid_has_evmcs(vcpu) && 3169 vmx->nested.hv_evmcs_vmptr == EVMPTR_MAP_PENDING) { 3170 enum nested_evmptrld_status evmptrld_status = 3171 nested_vmx_handle_enlightened_vmptrld(vcpu, false); 3172 3173 if (evmptrld_status == EVMPTRLD_VMFAIL || 3174 evmptrld_status == EVMPTRLD_ERROR) 3175 return false; 3176 3177 /* 3178 * Post migration VMCS12 always provides the most actual 3179 * information, copy it to eVMCS upon entry. 3180 */ 3181 vmx->nested.need_vmcs12_to_shadow_sync = true; 3182 } 3183 3184 return true; 3185 } 3186 3187 static bool nested_get_vmcs12_pages(struct kvm_vcpu *vcpu) 3188 { 3189 struct vmcs12 *vmcs12 = get_vmcs12(vcpu); 3190 struct vcpu_vmx *vmx = to_vmx(vcpu); 3191 struct kvm_host_map *map; 3192 3193 if (!vcpu->arch.pdptrs_from_userspace && 3194 !nested_cpu_has_ept(vmcs12) && is_pae_paging(vcpu)) { 3195 /* 3196 * Reload the guest's PDPTRs since after a migration 3197 * the guest CR3 might be restored prior to setting the nested 3198 * state which can lead to a load of wrong PDPTRs. 3199 */ 3200 if (CC(!load_pdptrs(vcpu, vcpu->arch.cr3))) 3201 return false; 3202 } 3203 3204 3205 if (nested_cpu_has2(vmcs12, SECONDARY_EXEC_VIRTUALIZE_APIC_ACCESSES)) { 3206 map = &vmx->nested.apic_access_page_map; 3207 3208 if (!kvm_vcpu_map(vcpu, gpa_to_gfn(vmcs12->apic_access_addr), map)) { 3209 vmcs_write64(APIC_ACCESS_ADDR, pfn_to_hpa(map->pfn)); 3210 } else { 3211 pr_debug_ratelimited("%s: no backing for APIC-access address in vmcs12\n", 3212 __func__); 3213 vcpu->run->exit_reason = KVM_EXIT_INTERNAL_ERROR; 3214 vcpu->run->internal.suberror = 3215 KVM_INTERNAL_ERROR_EMULATION; 3216 vcpu->run->internal.ndata = 0; 3217 return false; 3218 } 3219 } 3220 3221 if (nested_cpu_has(vmcs12, CPU_BASED_TPR_SHADOW)) { 3222 map = &vmx->nested.virtual_apic_map; 3223 3224 if (!kvm_vcpu_map(vcpu, gpa_to_gfn(vmcs12->virtual_apic_page_addr), map)) { 3225 vmcs_write64(VIRTUAL_APIC_PAGE_ADDR, pfn_to_hpa(map->pfn)); 3226 } else if (nested_cpu_has(vmcs12, CPU_BASED_CR8_LOAD_EXITING) && 3227 nested_cpu_has(vmcs12, CPU_BASED_CR8_STORE_EXITING) && 3228 !nested_cpu_has2(vmcs12, SECONDARY_EXEC_VIRTUALIZE_APIC_ACCESSES)) { 3229 /* 3230 * The processor will never use the TPR shadow, simply 3231 * clear the bit from the execution control. Such a 3232 * configuration is useless, but it happens in tests. 3233 * For any other configuration, failing the vm entry is 3234 * _not_ what the processor does but it's basically the 3235 * only possibility we have. 3236 */ 3237 exec_controls_clearbit(vmx, CPU_BASED_TPR_SHADOW); 3238 } else { 3239 /* 3240 * Write an illegal value to VIRTUAL_APIC_PAGE_ADDR to 3241 * force VM-Entry to fail. 3242 */ 3243 vmcs_write64(VIRTUAL_APIC_PAGE_ADDR, INVALID_GPA); 3244 } 3245 } 3246 3247 if (nested_cpu_has_posted_intr(vmcs12)) { 3248 map = &vmx->nested.pi_desc_map; 3249 3250 if (!kvm_vcpu_map(vcpu, gpa_to_gfn(vmcs12->posted_intr_desc_addr), map)) { 3251 vmx->nested.pi_desc = 3252 (struct pi_desc *)(((void *)map->hva) + 3253 offset_in_page(vmcs12->posted_intr_desc_addr)); 3254 vmcs_write64(POSTED_INTR_DESC_ADDR, 3255 pfn_to_hpa(map->pfn) + offset_in_page(vmcs12->posted_intr_desc_addr)); 3256 } else { 3257 /* 3258 * Defer the KVM_INTERNAL_EXIT until KVM tries to 3259 * access the contents of the VMCS12 posted interrupt 3260 * descriptor. (Note that KVM may do this when it 3261 * should not, per the architectural specification.) 3262 */ 3263 vmx->nested.pi_desc = NULL; 3264 pin_controls_clearbit(vmx, PIN_BASED_POSTED_INTR); 3265 } 3266 } 3267 if (nested_vmx_prepare_msr_bitmap(vcpu, vmcs12)) 3268 exec_controls_setbit(vmx, CPU_BASED_USE_MSR_BITMAPS); 3269 else 3270 exec_controls_clearbit(vmx, CPU_BASED_USE_MSR_BITMAPS); 3271 3272 return true; 3273 } 3274 3275 static bool vmx_get_nested_state_pages(struct kvm_vcpu *vcpu) 3276 { 3277 /* 3278 * Note: nested_get_evmcs_page() also updates 'vp_assist_page' copy 3279 * in 'struct kvm_vcpu_hv' in case eVMCS is in use, this is mandatory 3280 * to make nested_evmcs_l2_tlb_flush_enabled() work correctly post 3281 * migration. 3282 */ 3283 if (!nested_get_evmcs_page(vcpu)) { 3284 pr_debug_ratelimited("%s: enlightened vmptrld failed\n", 3285 __func__); 3286 vcpu->run->exit_reason = KVM_EXIT_INTERNAL_ERROR; 3287 vcpu->run->internal.suberror = 3288 KVM_INTERNAL_ERROR_EMULATION; 3289 vcpu->run->internal.ndata = 0; 3290 3291 return false; 3292 } 3293 3294 if (is_guest_mode(vcpu) && !nested_get_vmcs12_pages(vcpu)) 3295 return false; 3296 3297 return true; 3298 } 3299 3300 static int nested_vmx_write_pml_buffer(struct kvm_vcpu *vcpu, gpa_t gpa) 3301 { 3302 struct vmcs12 *vmcs12; 3303 struct vcpu_vmx *vmx = to_vmx(vcpu); 3304 gpa_t dst; 3305 3306 if (WARN_ON_ONCE(!is_guest_mode(vcpu))) 3307 return 0; 3308 3309 if (WARN_ON_ONCE(vmx->nested.pml_full)) 3310 return 1; 3311 3312 /* 3313 * Check if PML is enabled for the nested guest. Whether eptp bit 6 is 3314 * set is already checked as part of A/D emulation. 3315 */ 3316 vmcs12 = get_vmcs12(vcpu); 3317 if (!nested_cpu_has_pml(vmcs12)) 3318 return 0; 3319 3320 if (vmcs12->guest_pml_index >= PML_ENTITY_NUM) { 3321 vmx->nested.pml_full = true; 3322 return 1; 3323 } 3324 3325 gpa &= ~0xFFFull; 3326 dst = vmcs12->pml_address + sizeof(u64) * vmcs12->guest_pml_index; 3327 3328 if (kvm_write_guest_page(vcpu->kvm, gpa_to_gfn(dst), &gpa, 3329 offset_in_page(dst), sizeof(gpa))) 3330 return 0; 3331 3332 vmcs12->guest_pml_index--; 3333 3334 return 0; 3335 } 3336 3337 /* 3338 * Intel's VMX Instruction Reference specifies a common set of prerequisites 3339 * for running VMX instructions (except VMXON, whose prerequisites are 3340 * slightly different). It also specifies what exception to inject otherwise. 3341 * Note that many of these exceptions have priority over VM exits, so they 3342 * don't have to be checked again here. 3343 */ 3344 static int nested_vmx_check_permission(struct kvm_vcpu *vcpu) 3345 { 3346 if (!to_vmx(vcpu)->nested.vmxon) { 3347 kvm_queue_exception(vcpu, UD_VECTOR); 3348 return 0; 3349 } 3350 3351 if (vmx_get_cpl(vcpu)) { 3352 kvm_inject_gp(vcpu, 0); 3353 return 0; 3354 } 3355 3356 return 1; 3357 } 3358 3359 static u8 vmx_has_apicv_interrupt(struct kvm_vcpu *vcpu) 3360 { 3361 u8 rvi = vmx_get_rvi(); 3362 u8 vppr = kvm_lapic_get_reg(vcpu->arch.apic, APIC_PROCPRI); 3363 3364 return ((rvi & 0xf0) > (vppr & 0xf0)); 3365 } 3366 3367 static void load_vmcs12_host_state(struct kvm_vcpu *vcpu, 3368 struct vmcs12 *vmcs12); 3369 3370 /* 3371 * If from_vmentry is false, this is being called from state restore (either RSM 3372 * or KVM_SET_NESTED_STATE). Otherwise it's called from vmlaunch/vmresume. 3373 * 3374 * Returns: 3375 * NVMX_VMENTRY_SUCCESS: Entered VMX non-root mode 3376 * NVMX_VMENTRY_VMFAIL: Consistency check VMFail 3377 * NVMX_VMENTRY_VMEXIT: Consistency check VMExit 3378 * NVMX_VMENTRY_KVM_INTERNAL_ERROR: KVM internal error 3379 */ 3380 enum nvmx_vmentry_status nested_vmx_enter_non_root_mode(struct kvm_vcpu *vcpu, 3381 bool from_vmentry) 3382 { 3383 struct vcpu_vmx *vmx = to_vmx(vcpu); 3384 struct vmcs12 *vmcs12 = get_vmcs12(vcpu); 3385 enum vm_entry_failure_code entry_failure_code; 3386 bool evaluate_pending_interrupts; 3387 union vmx_exit_reason exit_reason = { 3388 .basic = EXIT_REASON_INVALID_STATE, 3389 .failed_vmentry = 1, 3390 }; 3391 u32 failed_index; 3392 3393 trace_kvm_nested_vmenter(kvm_rip_read(vcpu), 3394 vmx->nested.current_vmptr, 3395 vmcs12->guest_rip, 3396 vmcs12->guest_intr_status, 3397 vmcs12->vm_entry_intr_info_field, 3398 vmcs12->secondary_vm_exec_control & SECONDARY_EXEC_ENABLE_EPT, 3399 vmcs12->ept_pointer, 3400 vmcs12->guest_cr3, 3401 KVM_ISA_VMX); 3402 3403 kvm_service_local_tlb_flush_requests(vcpu); 3404 3405 evaluate_pending_interrupts = exec_controls_get(vmx) & 3406 (CPU_BASED_INTR_WINDOW_EXITING | CPU_BASED_NMI_WINDOW_EXITING); 3407 if (likely(!evaluate_pending_interrupts) && kvm_vcpu_apicv_active(vcpu)) 3408 evaluate_pending_interrupts |= vmx_has_apicv_interrupt(vcpu); 3409 if (!evaluate_pending_interrupts) 3410 evaluate_pending_interrupts |= kvm_apic_has_pending_init_or_sipi(vcpu); 3411 3412 if (!vmx->nested.nested_run_pending || 3413 !(vmcs12->vm_entry_controls & VM_ENTRY_LOAD_DEBUG_CONTROLS)) 3414 vmx->nested.pre_vmenter_debugctl = vmcs_read64(GUEST_IA32_DEBUGCTL); 3415 if (kvm_mpx_supported() && 3416 (!vmx->nested.nested_run_pending || 3417 !(vmcs12->vm_entry_controls & VM_ENTRY_LOAD_BNDCFGS))) 3418 vmx->nested.pre_vmenter_bndcfgs = vmcs_read64(GUEST_BNDCFGS); 3419 3420 /* 3421 * Overwrite vmcs01.GUEST_CR3 with L1's CR3 if EPT is disabled *and* 3422 * nested early checks are disabled. In the event of a "late" VM-Fail, 3423 * i.e. a VM-Fail detected by hardware but not KVM, KVM must unwind its 3424 * software model to the pre-VMEntry host state. When EPT is disabled, 3425 * GUEST_CR3 holds KVM's shadow CR3, not L1's "real" CR3, which causes 3426 * nested_vmx_restore_host_state() to corrupt vcpu->arch.cr3. Stuffing 3427 * vmcs01.GUEST_CR3 results in the unwind naturally setting arch.cr3 to 3428 * the correct value. Smashing vmcs01.GUEST_CR3 is safe because nested 3429 * VM-Exits, and the unwind, reset KVM's MMU, i.e. vmcs01.GUEST_CR3 is 3430 * guaranteed to be overwritten with a shadow CR3 prior to re-entering 3431 * L1. Don't stuff vmcs01.GUEST_CR3 when using nested early checks as 3432 * KVM modifies vcpu->arch.cr3 if and only if the early hardware checks 3433 * pass, and early VM-Fails do not reset KVM's MMU, i.e. the VM-Fail 3434 * path would need to manually save/restore vmcs01.GUEST_CR3. 3435 */ 3436 if (!enable_ept && !nested_early_check) 3437 vmcs_writel(GUEST_CR3, vcpu->arch.cr3); 3438 3439 vmx_switch_vmcs(vcpu, &vmx->nested.vmcs02); 3440 3441 prepare_vmcs02_early(vmx, &vmx->vmcs01, vmcs12); 3442 3443 if (from_vmentry) { 3444 if (unlikely(!nested_get_vmcs12_pages(vcpu))) { 3445 vmx_switch_vmcs(vcpu, &vmx->vmcs01); 3446 return NVMX_VMENTRY_KVM_INTERNAL_ERROR; 3447 } 3448 3449 if (nested_vmx_check_vmentry_hw(vcpu)) { 3450 vmx_switch_vmcs(vcpu, &vmx->vmcs01); 3451 return NVMX_VMENTRY_VMFAIL; 3452 } 3453 3454 if (nested_vmx_check_guest_state(vcpu, vmcs12, 3455 &entry_failure_code)) { 3456 exit_reason.basic = EXIT_REASON_INVALID_STATE; 3457 vmcs12->exit_qualification = entry_failure_code; 3458 goto vmentry_fail_vmexit; 3459 } 3460 } 3461 3462 enter_guest_mode(vcpu); 3463 3464 if (prepare_vmcs02(vcpu, vmcs12, from_vmentry, &entry_failure_code)) { 3465 exit_reason.basic = EXIT_REASON_INVALID_STATE; 3466 vmcs12->exit_qualification = entry_failure_code; 3467 goto vmentry_fail_vmexit_guest_mode; 3468 } 3469 3470 if (from_vmentry) { 3471 failed_index = nested_vmx_load_msr(vcpu, 3472 vmcs12->vm_entry_msr_load_addr, 3473 vmcs12->vm_entry_msr_load_count); 3474 if (failed_index) { 3475 exit_reason.basic = EXIT_REASON_MSR_LOAD_FAIL; 3476 vmcs12->exit_qualification = failed_index; 3477 goto vmentry_fail_vmexit_guest_mode; 3478 } 3479 } else { 3480 /* 3481 * The MMU is not initialized to point at the right entities yet and 3482 * "get pages" would need to read data from the guest (i.e. we will 3483 * need to perform gpa to hpa translation). Request a call 3484 * to nested_get_vmcs12_pages before the next VM-entry. The MSRs 3485 * have already been set at vmentry time and should not be reset. 3486 */ 3487 kvm_make_request(KVM_REQ_GET_NESTED_STATE_PAGES, vcpu); 3488 } 3489 3490 /* 3491 * Re-evaluate pending events if L1 had a pending IRQ/NMI/INIT/SIPI 3492 * when it executed VMLAUNCH/VMRESUME, as entering non-root mode can 3493 * effectively unblock various events, e.g. INIT/SIPI cause VM-Exit 3494 * unconditionally. 3495 */ 3496 if (unlikely(evaluate_pending_interrupts)) 3497 kvm_make_request(KVM_REQ_EVENT, vcpu); 3498 3499 /* 3500 * Do not start the preemption timer hrtimer until after we know 3501 * we are successful, so that only nested_vmx_vmexit needs to cancel 3502 * the timer. 3503 */ 3504 vmx->nested.preemption_timer_expired = false; 3505 if (nested_cpu_has_preemption_timer(vmcs12)) { 3506 u64 timer_value = vmx_calc_preemption_timer_value(vcpu); 3507 vmx_start_preemption_timer(vcpu, timer_value); 3508 } 3509 3510 /* 3511 * Note no nested_vmx_succeed or nested_vmx_fail here. At this point 3512 * we are no longer running L1, and VMLAUNCH/VMRESUME has not yet 3513 * returned as far as L1 is concerned. It will only return (and set 3514 * the success flag) when L2 exits (see nested_vmx_vmexit()). 3515 */ 3516 return NVMX_VMENTRY_SUCCESS; 3517 3518 /* 3519 * A failed consistency check that leads to a VMExit during L1's 3520 * VMEnter to L2 is a variation of a normal VMexit, as explained in 3521 * 26.7 "VM-entry failures during or after loading guest state". 3522 */ 3523 vmentry_fail_vmexit_guest_mode: 3524 if (vmcs12->cpu_based_vm_exec_control & CPU_BASED_USE_TSC_OFFSETTING) 3525 vcpu->arch.tsc_offset -= vmcs12->tsc_offset; 3526 leave_guest_mode(vcpu); 3527 3528 vmentry_fail_vmexit: 3529 vmx_switch_vmcs(vcpu, &vmx->vmcs01); 3530 3531 if (!from_vmentry) 3532 return NVMX_VMENTRY_VMEXIT; 3533 3534 load_vmcs12_host_state(vcpu, vmcs12); 3535 vmcs12->vm_exit_reason = exit_reason.full; 3536 if (enable_shadow_vmcs || evmptr_is_valid(vmx->nested.hv_evmcs_vmptr)) 3537 vmx->nested.need_vmcs12_to_shadow_sync = true; 3538 return NVMX_VMENTRY_VMEXIT; 3539 } 3540 3541 /* 3542 * nested_vmx_run() handles a nested entry, i.e., a VMLAUNCH or VMRESUME on L1 3543 * for running an L2 nested guest. 3544 */ 3545 static int nested_vmx_run(struct kvm_vcpu *vcpu, bool launch) 3546 { 3547 struct vmcs12 *vmcs12; 3548 enum nvmx_vmentry_status status; 3549 struct vcpu_vmx *vmx = to_vmx(vcpu); 3550 u32 interrupt_shadow = vmx_get_interrupt_shadow(vcpu); 3551 enum nested_evmptrld_status evmptrld_status; 3552 3553 if (!nested_vmx_check_permission(vcpu)) 3554 return 1; 3555 3556 evmptrld_status = nested_vmx_handle_enlightened_vmptrld(vcpu, launch); 3557 if (evmptrld_status == EVMPTRLD_ERROR) { 3558 kvm_queue_exception(vcpu, UD_VECTOR); 3559 return 1; 3560 } 3561 3562 kvm_pmu_trigger_event(vcpu, PERF_COUNT_HW_BRANCH_INSTRUCTIONS); 3563 3564 if (CC(evmptrld_status == EVMPTRLD_VMFAIL)) 3565 return nested_vmx_failInvalid(vcpu); 3566 3567 if (CC(!evmptr_is_valid(vmx->nested.hv_evmcs_vmptr) && 3568 vmx->nested.current_vmptr == INVALID_GPA)) 3569 return nested_vmx_failInvalid(vcpu); 3570 3571 vmcs12 = get_vmcs12(vcpu); 3572 3573 /* 3574 * Can't VMLAUNCH or VMRESUME a shadow VMCS. Despite the fact 3575 * that there *is* a valid VMCS pointer, RFLAGS.CF is set 3576 * rather than RFLAGS.ZF, and no error number is stored to the 3577 * VM-instruction error field. 3578 */ 3579 if (CC(vmcs12->hdr.shadow_vmcs)) 3580 return nested_vmx_failInvalid(vcpu); 3581 3582 if (evmptr_is_valid(vmx->nested.hv_evmcs_vmptr)) { 3583 copy_enlightened_to_vmcs12(vmx, vmx->nested.hv_evmcs->hv_clean_fields); 3584 /* Enlightened VMCS doesn't have launch state */ 3585 vmcs12->launch_state = !launch; 3586 } else if (enable_shadow_vmcs) { 3587 copy_shadow_to_vmcs12(vmx); 3588 } 3589 3590 /* 3591 * The nested entry process starts with enforcing various prerequisites 3592 * on vmcs12 as required by the Intel SDM, and act appropriately when 3593 * they fail: As the SDM explains, some conditions should cause the 3594 * instruction to fail, while others will cause the instruction to seem 3595 * to succeed, but return an EXIT_REASON_INVALID_STATE. 3596 * To speed up the normal (success) code path, we should avoid checking 3597 * for misconfigurations which will anyway be caught by the processor 3598 * when using the merged vmcs02. 3599 */ 3600 if (CC(interrupt_shadow & KVM_X86_SHADOW_INT_MOV_SS)) 3601 return nested_vmx_fail(vcpu, VMXERR_ENTRY_EVENTS_BLOCKED_BY_MOV_SS); 3602 3603 if (CC(vmcs12->launch_state == launch)) 3604 return nested_vmx_fail(vcpu, 3605 launch ? VMXERR_VMLAUNCH_NONCLEAR_VMCS 3606 : VMXERR_VMRESUME_NONLAUNCHED_VMCS); 3607 3608 if (nested_vmx_check_controls(vcpu, vmcs12)) 3609 return nested_vmx_fail(vcpu, VMXERR_ENTRY_INVALID_CONTROL_FIELD); 3610 3611 if (nested_vmx_check_address_space_size(vcpu, vmcs12)) 3612 return nested_vmx_fail(vcpu, VMXERR_ENTRY_INVALID_HOST_STATE_FIELD); 3613 3614 if (nested_vmx_check_host_state(vcpu, vmcs12)) 3615 return nested_vmx_fail(vcpu, VMXERR_ENTRY_INVALID_HOST_STATE_FIELD); 3616 3617 /* 3618 * We're finally done with prerequisite checking, and can start with 3619 * the nested entry. 3620 */ 3621 vmx->nested.nested_run_pending = 1; 3622 vmx->nested.has_preemption_timer_deadline = false; 3623 status = nested_vmx_enter_non_root_mode(vcpu, true); 3624 if (unlikely(status != NVMX_VMENTRY_SUCCESS)) 3625 goto vmentry_failed; 3626 3627 /* Emulate processing of posted interrupts on VM-Enter. */ 3628 if (nested_cpu_has_posted_intr(vmcs12) && 3629 kvm_apic_has_interrupt(vcpu) == vmx->nested.posted_intr_nv) { 3630 vmx->nested.pi_pending = true; 3631 kvm_make_request(KVM_REQ_EVENT, vcpu); 3632 kvm_apic_clear_irr(vcpu, vmx->nested.posted_intr_nv); 3633 } 3634 3635 /* Hide L1D cache contents from the nested guest. */ 3636 vmx->vcpu.arch.l1tf_flush_l1d = true; 3637 3638 /* 3639 * Must happen outside of nested_vmx_enter_non_root_mode() as it will 3640 * also be used as part of restoring nVMX state for 3641 * snapshot restore (migration). 3642 * 3643 * In this flow, it is assumed that vmcs12 cache was 3644 * transferred as part of captured nVMX state and should 3645 * therefore not be read from guest memory (which may not 3646 * exist on destination host yet). 3647 */ 3648 nested_cache_shadow_vmcs12(vcpu, vmcs12); 3649 3650 switch (vmcs12->guest_activity_state) { 3651 case GUEST_ACTIVITY_HLT: 3652 /* 3653 * If we're entering a halted L2 vcpu and the L2 vcpu won't be 3654 * awakened by event injection or by an NMI-window VM-exit or 3655 * by an interrupt-window VM-exit, halt the vcpu. 3656 */ 3657 if (!(vmcs12->vm_entry_intr_info_field & INTR_INFO_VALID_MASK) && 3658 !nested_cpu_has(vmcs12, CPU_BASED_NMI_WINDOW_EXITING) && 3659 !(nested_cpu_has(vmcs12, CPU_BASED_INTR_WINDOW_EXITING) && 3660 (vmcs12->guest_rflags & X86_EFLAGS_IF))) { 3661 vmx->nested.nested_run_pending = 0; 3662 return kvm_emulate_halt_noskip(vcpu); 3663 } 3664 break; 3665 case GUEST_ACTIVITY_WAIT_SIPI: 3666 vmx->nested.nested_run_pending = 0; 3667 vcpu->arch.mp_state = KVM_MP_STATE_INIT_RECEIVED; 3668 break; 3669 default: 3670 break; 3671 } 3672 3673 return 1; 3674 3675 vmentry_failed: 3676 vmx->nested.nested_run_pending = 0; 3677 if (status == NVMX_VMENTRY_KVM_INTERNAL_ERROR) 3678 return 0; 3679 if (status == NVMX_VMENTRY_VMEXIT) 3680 return 1; 3681 WARN_ON_ONCE(status != NVMX_VMENTRY_VMFAIL); 3682 return nested_vmx_fail(vcpu, VMXERR_ENTRY_INVALID_CONTROL_FIELD); 3683 } 3684 3685 /* 3686 * On a nested exit from L2 to L1, vmcs12.guest_cr0 might not be up-to-date 3687 * because L2 may have changed some cr0 bits directly (CR0_GUEST_HOST_MASK). 3688 * This function returns the new value we should put in vmcs12.guest_cr0. 3689 * It's not enough to just return the vmcs02 GUEST_CR0. Rather, 3690 * 1. Bits that neither L0 nor L1 trapped, were set directly by L2 and are now 3691 * available in vmcs02 GUEST_CR0. (Note: It's enough to check that L0 3692 * didn't trap the bit, because if L1 did, so would L0). 3693 * 2. Bits that L1 asked to trap (and therefore L0 also did) could not have 3694 * been modified by L2, and L1 knows it. So just leave the old value of 3695 * the bit from vmcs12.guest_cr0. Note that the bit from vmcs02 GUEST_CR0 3696 * isn't relevant, because if L0 traps this bit it can set it to anything. 3697 * 3. Bits that L1 didn't trap, but L0 did. L1 believes the guest could have 3698 * changed these bits, and therefore they need to be updated, but L0 3699 * didn't necessarily allow them to be changed in GUEST_CR0 - and rather 3700 * put them in vmcs02 CR0_READ_SHADOW. So take these bits from there. 3701 */ 3702 static inline unsigned long 3703 vmcs12_guest_cr0(struct kvm_vcpu *vcpu, struct vmcs12 *vmcs12) 3704 { 3705 return 3706 /*1*/ (vmcs_readl(GUEST_CR0) & vcpu->arch.cr0_guest_owned_bits) | 3707 /*2*/ (vmcs12->guest_cr0 & vmcs12->cr0_guest_host_mask) | 3708 /*3*/ (vmcs_readl(CR0_READ_SHADOW) & ~(vmcs12->cr0_guest_host_mask | 3709 vcpu->arch.cr0_guest_owned_bits)); 3710 } 3711 3712 static inline unsigned long 3713 vmcs12_guest_cr4(struct kvm_vcpu *vcpu, struct vmcs12 *vmcs12) 3714 { 3715 return 3716 /*1*/ (vmcs_readl(GUEST_CR4) & vcpu->arch.cr4_guest_owned_bits) | 3717 /*2*/ (vmcs12->guest_cr4 & vmcs12->cr4_guest_host_mask) | 3718 /*3*/ (vmcs_readl(CR4_READ_SHADOW) & ~(vmcs12->cr4_guest_host_mask | 3719 vcpu->arch.cr4_guest_owned_bits)); 3720 } 3721 3722 static void vmcs12_save_pending_event(struct kvm_vcpu *vcpu, 3723 struct vmcs12 *vmcs12, 3724 u32 vm_exit_reason, u32 exit_intr_info) 3725 { 3726 u32 idt_vectoring; 3727 unsigned int nr; 3728 3729 /* 3730 * Per the SDM, VM-Exits due to double and triple faults are never 3731 * considered to occur during event delivery, even if the double/triple 3732 * fault is the result of an escalating vectoring issue. 3733 * 3734 * Note, the SDM qualifies the double fault behavior with "The original 3735 * event results in a double-fault exception". It's unclear why the 3736 * qualification exists since exits due to double fault can occur only 3737 * while vectoring a different exception (injected events are never 3738 * subject to interception), i.e. there's _always_ an original event. 3739 * 3740 * The SDM also uses NMI as a confusing example for the "original event 3741 * causes the VM exit directly" clause. NMI isn't special in any way, 3742 * the same rule applies to all events that cause an exit directly. 3743 * NMI is an odd choice for the example because NMIs can only occur on 3744 * instruction boundaries, i.e. they _can't_ occur during vectoring. 3745 */ 3746 if ((u16)vm_exit_reason == EXIT_REASON_TRIPLE_FAULT || 3747 ((u16)vm_exit_reason == EXIT_REASON_EXCEPTION_NMI && 3748 is_double_fault(exit_intr_info))) { 3749 vmcs12->idt_vectoring_info_field = 0; 3750 } else if (vcpu->arch.exception.injected) { 3751 nr = vcpu->arch.exception.vector; 3752 idt_vectoring = nr | VECTORING_INFO_VALID_MASK; 3753 3754 if (kvm_exception_is_soft(nr)) { 3755 vmcs12->vm_exit_instruction_len = 3756 vcpu->arch.event_exit_inst_len; 3757 idt_vectoring |= INTR_TYPE_SOFT_EXCEPTION; 3758 } else 3759 idt_vectoring |= INTR_TYPE_HARD_EXCEPTION; 3760 3761 if (vcpu->arch.exception.has_error_code) { 3762 idt_vectoring |= VECTORING_INFO_DELIVER_CODE_MASK; 3763 vmcs12->idt_vectoring_error_code = 3764 vcpu->arch.exception.error_code; 3765 } 3766 3767 vmcs12->idt_vectoring_info_field = idt_vectoring; 3768 } else if (vcpu->arch.nmi_injected) { 3769 vmcs12->idt_vectoring_info_field = 3770 INTR_TYPE_NMI_INTR | INTR_INFO_VALID_MASK | NMI_VECTOR; 3771 } else if (vcpu->arch.interrupt.injected) { 3772 nr = vcpu->arch.interrupt.nr; 3773 idt_vectoring = nr | VECTORING_INFO_VALID_MASK; 3774 3775 if (vcpu->arch.interrupt.soft) { 3776 idt_vectoring |= INTR_TYPE_SOFT_INTR; 3777 vmcs12->vm_entry_instruction_len = 3778 vcpu->arch.event_exit_inst_len; 3779 } else 3780 idt_vectoring |= INTR_TYPE_EXT_INTR; 3781 3782 vmcs12->idt_vectoring_info_field = idt_vectoring; 3783 } else { 3784 vmcs12->idt_vectoring_info_field = 0; 3785 } 3786 } 3787 3788 3789 void nested_mark_vmcs12_pages_dirty(struct kvm_vcpu *vcpu) 3790 { 3791 struct vmcs12 *vmcs12 = get_vmcs12(vcpu); 3792 gfn_t gfn; 3793 3794 /* 3795 * Don't need to mark the APIC access page dirty; it is never 3796 * written to by the CPU during APIC virtualization. 3797 */ 3798 3799 if (nested_cpu_has(vmcs12, CPU_BASED_TPR_SHADOW)) { 3800 gfn = vmcs12->virtual_apic_page_addr >> PAGE_SHIFT; 3801 kvm_vcpu_mark_page_dirty(vcpu, gfn); 3802 } 3803 3804 if (nested_cpu_has_posted_intr(vmcs12)) { 3805 gfn = vmcs12->posted_intr_desc_addr >> PAGE_SHIFT; 3806 kvm_vcpu_mark_page_dirty(vcpu, gfn); 3807 } 3808 } 3809 3810 static int vmx_complete_nested_posted_interrupt(struct kvm_vcpu *vcpu) 3811 { 3812 struct vcpu_vmx *vmx = to_vmx(vcpu); 3813 int max_irr; 3814 void *vapic_page; 3815 u16 status; 3816 3817 if (!vmx->nested.pi_pending) 3818 return 0; 3819 3820 if (!vmx->nested.pi_desc) 3821 goto mmio_needed; 3822 3823 vmx->nested.pi_pending = false; 3824 3825 if (!pi_test_and_clear_on(vmx->nested.pi_desc)) 3826 return 0; 3827 3828 max_irr = find_last_bit((unsigned long *)vmx->nested.pi_desc->pir, 256); 3829 if (max_irr != 256) { 3830 vapic_page = vmx->nested.virtual_apic_map.hva; 3831 if (!vapic_page) 3832 goto mmio_needed; 3833 3834 __kvm_apic_update_irr(vmx->nested.pi_desc->pir, 3835 vapic_page, &max_irr); 3836 status = vmcs_read16(GUEST_INTR_STATUS); 3837 if ((u8)max_irr > ((u8)status & 0xff)) { 3838 status &= ~0xff; 3839 status |= (u8)max_irr; 3840 vmcs_write16(GUEST_INTR_STATUS, status); 3841 } 3842 } 3843 3844 nested_mark_vmcs12_pages_dirty(vcpu); 3845 return 0; 3846 3847 mmio_needed: 3848 kvm_handle_memory_failure(vcpu, X86EMUL_IO_NEEDED, NULL); 3849 return -ENXIO; 3850 } 3851 3852 static void nested_vmx_inject_exception_vmexit(struct kvm_vcpu *vcpu) 3853 { 3854 struct kvm_queued_exception *ex = &vcpu->arch.exception_vmexit; 3855 u32 intr_info = ex->vector | INTR_INFO_VALID_MASK; 3856 struct vmcs12 *vmcs12 = get_vmcs12(vcpu); 3857 unsigned long exit_qual; 3858 3859 if (ex->has_payload) { 3860 exit_qual = ex->payload; 3861 } else if (ex->vector == PF_VECTOR) { 3862 exit_qual = vcpu->arch.cr2; 3863 } else if (ex->vector == DB_VECTOR) { 3864 exit_qual = vcpu->arch.dr6; 3865 exit_qual &= ~DR6_BT; 3866 exit_qual ^= DR6_ACTIVE_LOW; 3867 } else { 3868 exit_qual = 0; 3869 } 3870 3871 if (ex->has_error_code) { 3872 /* 3873 * Intel CPUs do not generate error codes with bits 31:16 set, 3874 * and more importantly VMX disallows setting bits 31:16 in the 3875 * injected error code for VM-Entry. Drop the bits to mimic 3876 * hardware and avoid inducing failure on nested VM-Entry if L1 3877 * chooses to inject the exception back to L2. AMD CPUs _do_ 3878 * generate "full" 32-bit error codes, so KVM allows userspace 3879 * to inject exception error codes with bits 31:16 set. 3880 */ 3881 vmcs12->vm_exit_intr_error_code = (u16)ex->error_code; 3882 intr_info |= INTR_INFO_DELIVER_CODE_MASK; 3883 } 3884 3885 if (kvm_exception_is_soft(ex->vector)) 3886 intr_info |= INTR_TYPE_SOFT_EXCEPTION; 3887 else 3888 intr_info |= INTR_TYPE_HARD_EXCEPTION; 3889 3890 if (!(vmcs12->idt_vectoring_info_field & VECTORING_INFO_VALID_MASK) && 3891 vmx_get_nmi_mask(vcpu)) 3892 intr_info |= INTR_INFO_UNBLOCK_NMI; 3893 3894 nested_vmx_vmexit(vcpu, EXIT_REASON_EXCEPTION_NMI, intr_info, exit_qual); 3895 } 3896 3897 /* 3898 * Returns true if a debug trap is (likely) pending delivery. Infer the class 3899 * of a #DB (trap-like vs. fault-like) from the exception payload (to-be-DR6). 3900 * Using the payload is flawed because code breakpoints (fault-like) and data 3901 * breakpoints (trap-like) set the same bits in DR6 (breakpoint detected), i.e. 3902 * this will return false positives if a to-be-injected code breakpoint #DB is 3903 * pending (from KVM's perspective, but not "pending" across an instruction 3904 * boundary). ICEBP, a.k.a. INT1, is also not reflected here even though it 3905 * too is trap-like. 3906 * 3907 * KVM "works" despite these flaws as ICEBP isn't currently supported by the 3908 * emulator, Monitor Trap Flag is not marked pending on intercepted #DBs (the 3909 * #DB has already happened), and MTF isn't marked pending on code breakpoints 3910 * from the emulator (because such #DBs are fault-like and thus don't trigger 3911 * actions that fire on instruction retire). 3912 */ 3913 static unsigned long vmx_get_pending_dbg_trap(struct kvm_queued_exception *ex) 3914 { 3915 if (!ex->pending || ex->vector != DB_VECTOR) 3916 return 0; 3917 3918 /* General Detect #DBs are always fault-like. */ 3919 return ex->payload & ~DR6_BD; 3920 } 3921 3922 /* 3923 * Returns true if there's a pending #DB exception that is lower priority than 3924 * a pending Monitor Trap Flag VM-Exit. TSS T-flag #DBs are not emulated by 3925 * KVM, but could theoretically be injected by userspace. Note, this code is 3926 * imperfect, see above. 3927 */ 3928 static bool vmx_is_low_priority_db_trap(struct kvm_queued_exception *ex) 3929 { 3930 return vmx_get_pending_dbg_trap(ex) & ~DR6_BT; 3931 } 3932 3933 /* 3934 * Certain VM-exits set the 'pending debug exceptions' field to indicate a 3935 * recognized #DB (data or single-step) that has yet to be delivered. Since KVM 3936 * represents these debug traps with a payload that is said to be compatible 3937 * with the 'pending debug exceptions' field, write the payload to the VMCS 3938 * field if a VM-exit is delivered before the debug trap. 3939 */ 3940 static void nested_vmx_update_pending_dbg(struct kvm_vcpu *vcpu) 3941 { 3942 unsigned long pending_dbg; 3943 3944 pending_dbg = vmx_get_pending_dbg_trap(&vcpu->arch.exception); 3945 if (pending_dbg) 3946 vmcs_writel(GUEST_PENDING_DBG_EXCEPTIONS, pending_dbg); 3947 } 3948 3949 static bool nested_vmx_preemption_timer_pending(struct kvm_vcpu *vcpu) 3950 { 3951 return nested_cpu_has_preemption_timer(get_vmcs12(vcpu)) && 3952 to_vmx(vcpu)->nested.preemption_timer_expired; 3953 } 3954 3955 static bool vmx_has_nested_events(struct kvm_vcpu *vcpu) 3956 { 3957 return nested_vmx_preemption_timer_pending(vcpu) || 3958 to_vmx(vcpu)->nested.mtf_pending; 3959 } 3960 3961 /* 3962 * Per the Intel SDM's table "Priority Among Concurrent Events", with minor 3963 * edits to fill in missing examples, e.g. #DB due to split-lock accesses, 3964 * and less minor edits to splice in the priority of VMX Non-Root specific 3965 * events, e.g. MTF and NMI/INTR-window exiting. 3966 * 3967 * 1 Hardware Reset and Machine Checks 3968 * - RESET 3969 * - Machine Check 3970 * 3971 * 2 Trap on Task Switch 3972 * - T flag in TSS is set (on task switch) 3973 * 3974 * 3 External Hardware Interventions 3975 * - FLUSH 3976 * - STOPCLK 3977 * - SMI 3978 * - INIT 3979 * 3980 * 3.5 Monitor Trap Flag (MTF) VM-exit[1] 3981 * 3982 * 4 Traps on Previous Instruction 3983 * - Breakpoints 3984 * - Trap-class Debug Exceptions (#DB due to TF flag set, data/I-O 3985 * breakpoint, or #DB due to a split-lock access) 3986 * 3987 * 4.3 VMX-preemption timer expired VM-exit 3988 * 3989 * 4.6 NMI-window exiting VM-exit[2] 3990 * 3991 * 5 Nonmaskable Interrupts (NMI) 3992 * 3993 * 5.5 Interrupt-window exiting VM-exit and Virtual-interrupt delivery 3994 * 3995 * 6 Maskable Hardware Interrupts 3996 * 3997 * 7 Code Breakpoint Fault 3998 * 3999 * 8 Faults from Fetching Next Instruction 4000 * - Code-Segment Limit Violation 4001 * - Code Page Fault 4002 * - Control protection exception (missing ENDBRANCH at target of indirect 4003 * call or jump) 4004 * 4005 * 9 Faults from Decoding Next Instruction 4006 * - Instruction length > 15 bytes 4007 * - Invalid Opcode 4008 * - Coprocessor Not Available 4009 * 4010 *10 Faults on Executing Instruction 4011 * - Overflow 4012 * - Bound error 4013 * - Invalid TSS 4014 * - Segment Not Present 4015 * - Stack fault 4016 * - General Protection 4017 * - Data Page Fault 4018 * - Alignment Check 4019 * - x86 FPU Floating-point exception 4020 * - SIMD floating-point exception 4021 * - Virtualization exception 4022 * - Control protection exception 4023 * 4024 * [1] Per the "Monitor Trap Flag" section: System-management interrupts (SMIs), 4025 * INIT signals, and higher priority events take priority over MTF VM exits. 4026 * MTF VM exits take priority over debug-trap exceptions and lower priority 4027 * events. 4028 * 4029 * [2] Debug-trap exceptions and higher priority events take priority over VM exits 4030 * caused by the VMX-preemption timer. VM exits caused by the VMX-preemption 4031 * timer take priority over VM exits caused by the "NMI-window exiting" 4032 * VM-execution control and lower priority events. 4033 * 4034 * [3] Debug-trap exceptions and higher priority events take priority over VM exits 4035 * caused by "NMI-window exiting". VM exits caused by this control take 4036 * priority over non-maskable interrupts (NMIs) and lower priority events. 4037 * 4038 * [4] Virtual-interrupt delivery has the same priority as that of VM exits due to 4039 * the 1-setting of the "interrupt-window exiting" VM-execution control. Thus, 4040 * non-maskable interrupts (NMIs) and higher priority events take priority over 4041 * delivery of a virtual interrupt; delivery of a virtual interrupt takes 4042 * priority over external interrupts and lower priority events. 4043 */ 4044 static int vmx_check_nested_events(struct kvm_vcpu *vcpu) 4045 { 4046 struct kvm_lapic *apic = vcpu->arch.apic; 4047 struct vcpu_vmx *vmx = to_vmx(vcpu); 4048 /* 4049 * Only a pending nested run blocks a pending exception. If there is a 4050 * previously injected event, the pending exception occurred while said 4051 * event was being delivered and thus needs to be handled. 4052 */ 4053 bool block_nested_exceptions = vmx->nested.nested_run_pending; 4054 /* 4055 * New events (not exceptions) are only recognized at instruction 4056 * boundaries. If an event needs reinjection, then KVM is handling a 4057 * VM-Exit that occurred _during_ instruction execution; new events are 4058 * blocked until the instruction completes. 4059 */ 4060 bool block_nested_events = block_nested_exceptions || 4061 kvm_event_needs_reinjection(vcpu); 4062 4063 if (lapic_in_kernel(vcpu) && 4064 test_bit(KVM_APIC_INIT, &apic->pending_events)) { 4065 if (block_nested_events) 4066 return -EBUSY; 4067 nested_vmx_update_pending_dbg(vcpu); 4068 clear_bit(KVM_APIC_INIT, &apic->pending_events); 4069 if (vcpu->arch.mp_state != KVM_MP_STATE_INIT_RECEIVED) 4070 nested_vmx_vmexit(vcpu, EXIT_REASON_INIT_SIGNAL, 0, 0); 4071 4072 /* MTF is discarded if the vCPU is in WFS. */ 4073 vmx->nested.mtf_pending = false; 4074 return 0; 4075 } 4076 4077 if (lapic_in_kernel(vcpu) && 4078 test_bit(KVM_APIC_SIPI, &apic->pending_events)) { 4079 if (block_nested_events) 4080 return -EBUSY; 4081 4082 clear_bit(KVM_APIC_SIPI, &apic->pending_events); 4083 if (vcpu->arch.mp_state == KVM_MP_STATE_INIT_RECEIVED) { 4084 nested_vmx_vmexit(vcpu, EXIT_REASON_SIPI_SIGNAL, 0, 4085 apic->sipi_vector & 0xFFUL); 4086 return 0; 4087 } 4088 /* Fallthrough, the SIPI is completely ignored. */ 4089 } 4090 4091 /* 4092 * Process exceptions that are higher priority than Monitor Trap Flag: 4093 * fault-like exceptions, TSS T flag #DB (not emulated by KVM, but 4094 * could theoretically come in from userspace), and ICEBP (INT1). 4095 * 4096 * TODO: SMIs have higher priority than MTF and trap-like #DBs (except 4097 * for TSS T flag #DBs). KVM also doesn't save/restore pending MTF 4098 * across SMI/RSM as it should; that needs to be addressed in order to 4099 * prioritize SMI over MTF and trap-like #DBs. 4100 */ 4101 if (vcpu->arch.exception_vmexit.pending && 4102 !vmx_is_low_priority_db_trap(&vcpu->arch.exception_vmexit)) { 4103 if (block_nested_exceptions) 4104 return -EBUSY; 4105 4106 nested_vmx_inject_exception_vmexit(vcpu); 4107 return 0; 4108 } 4109 4110 if (vcpu->arch.exception.pending && 4111 !vmx_is_low_priority_db_trap(&vcpu->arch.exception)) { 4112 if (block_nested_exceptions) 4113 return -EBUSY; 4114 goto no_vmexit; 4115 } 4116 4117 if (vmx->nested.mtf_pending) { 4118 if (block_nested_events) 4119 return -EBUSY; 4120 nested_vmx_update_pending_dbg(vcpu); 4121 nested_vmx_vmexit(vcpu, EXIT_REASON_MONITOR_TRAP_FLAG, 0, 0); 4122 return 0; 4123 } 4124 4125 if (vcpu->arch.exception_vmexit.pending) { 4126 if (block_nested_exceptions) 4127 return -EBUSY; 4128 4129 nested_vmx_inject_exception_vmexit(vcpu); 4130 return 0; 4131 } 4132 4133 if (vcpu->arch.exception.pending) { 4134 if (block_nested_exceptions) 4135 return -EBUSY; 4136 goto no_vmexit; 4137 } 4138 4139 if (nested_vmx_preemption_timer_pending(vcpu)) { 4140 if (block_nested_events) 4141 return -EBUSY; 4142 nested_vmx_vmexit(vcpu, EXIT_REASON_PREEMPTION_TIMER, 0, 0); 4143 return 0; 4144 } 4145 4146 if (vcpu->arch.smi_pending && !is_smm(vcpu)) { 4147 if (block_nested_events) 4148 return -EBUSY; 4149 goto no_vmexit; 4150 } 4151 4152 if (vcpu->arch.nmi_pending && !vmx_nmi_blocked(vcpu)) { 4153 if (block_nested_events) 4154 return -EBUSY; 4155 if (!nested_exit_on_nmi(vcpu)) 4156 goto no_vmexit; 4157 4158 nested_vmx_vmexit(vcpu, EXIT_REASON_EXCEPTION_NMI, 4159 NMI_VECTOR | INTR_TYPE_NMI_INTR | 4160 INTR_INFO_VALID_MASK, 0); 4161 /* 4162 * The NMI-triggered VM exit counts as injection: 4163 * clear this one and block further NMIs. 4164 */ 4165 vcpu->arch.nmi_pending = 0; 4166 vmx_set_nmi_mask(vcpu, true); 4167 return 0; 4168 } 4169 4170 if (kvm_cpu_has_interrupt(vcpu) && !vmx_interrupt_blocked(vcpu)) { 4171 if (block_nested_events) 4172 return -EBUSY; 4173 if (!nested_exit_on_intr(vcpu)) 4174 goto no_vmexit; 4175 nested_vmx_vmexit(vcpu, EXIT_REASON_EXTERNAL_INTERRUPT, 0, 0); 4176 return 0; 4177 } 4178 4179 no_vmexit: 4180 return vmx_complete_nested_posted_interrupt(vcpu); 4181 } 4182 4183 static u32 vmx_get_preemption_timer_value(struct kvm_vcpu *vcpu) 4184 { 4185 ktime_t remaining = 4186 hrtimer_get_remaining(&to_vmx(vcpu)->nested.preemption_timer); 4187 u64 value; 4188 4189 if (ktime_to_ns(remaining) <= 0) 4190 return 0; 4191 4192 value = ktime_to_ns(remaining) * vcpu->arch.virtual_tsc_khz; 4193 do_div(value, 1000000); 4194 return value >> VMX_MISC_EMULATED_PREEMPTION_TIMER_RATE; 4195 } 4196 4197 static bool is_vmcs12_ext_field(unsigned long field) 4198 { 4199 switch (field) { 4200 case GUEST_ES_SELECTOR: 4201 case GUEST_CS_SELECTOR: 4202 case GUEST_SS_SELECTOR: 4203 case GUEST_DS_SELECTOR: 4204 case GUEST_FS_SELECTOR: 4205 case GUEST_GS_SELECTOR: 4206 case GUEST_LDTR_SELECTOR: 4207 case GUEST_TR_SELECTOR: 4208 case GUEST_ES_LIMIT: 4209 case GUEST_CS_LIMIT: 4210 case GUEST_SS_LIMIT: 4211 case GUEST_DS_LIMIT: 4212 case GUEST_FS_LIMIT: 4213 case GUEST_GS_LIMIT: 4214 case GUEST_LDTR_LIMIT: 4215 case GUEST_TR_LIMIT: 4216 case GUEST_GDTR_LIMIT: 4217 case GUEST_IDTR_LIMIT: 4218 case GUEST_ES_AR_BYTES: 4219 case GUEST_DS_AR_BYTES: 4220 case GUEST_FS_AR_BYTES: 4221 case GUEST_GS_AR_BYTES: 4222 case GUEST_LDTR_AR_BYTES: 4223 case GUEST_TR_AR_BYTES: 4224 case GUEST_ES_BASE: 4225 case GUEST_CS_BASE: 4226 case GUEST_SS_BASE: 4227 case GUEST_DS_BASE: 4228 case GUEST_FS_BASE: 4229 case GUEST_GS_BASE: 4230 case GUEST_LDTR_BASE: 4231 case GUEST_TR_BASE: 4232 case GUEST_GDTR_BASE: 4233 case GUEST_IDTR_BASE: 4234 case GUEST_PENDING_DBG_EXCEPTIONS: 4235 case GUEST_BNDCFGS: 4236 return true; 4237 default: 4238 break; 4239 } 4240 4241 return false; 4242 } 4243 4244 static void sync_vmcs02_to_vmcs12_rare(struct kvm_vcpu *vcpu, 4245 struct vmcs12 *vmcs12) 4246 { 4247 struct vcpu_vmx *vmx = to_vmx(vcpu); 4248 4249 vmcs12->guest_es_selector = vmcs_read16(GUEST_ES_SELECTOR); 4250 vmcs12->guest_cs_selector = vmcs_read16(GUEST_CS_SELECTOR); 4251 vmcs12->guest_ss_selector = vmcs_read16(GUEST_SS_SELECTOR); 4252 vmcs12->guest_ds_selector = vmcs_read16(GUEST_DS_SELECTOR); 4253 vmcs12->guest_fs_selector = vmcs_read16(GUEST_FS_SELECTOR); 4254 vmcs12->guest_gs_selector = vmcs_read16(GUEST_GS_SELECTOR); 4255 vmcs12->guest_ldtr_selector = vmcs_read16(GUEST_LDTR_SELECTOR); 4256 vmcs12->guest_tr_selector = vmcs_read16(GUEST_TR_SELECTOR); 4257 vmcs12->guest_es_limit = vmcs_read32(GUEST_ES_LIMIT); 4258 vmcs12->guest_cs_limit = vmcs_read32(GUEST_CS_LIMIT); 4259 vmcs12->guest_ss_limit = vmcs_read32(GUEST_SS_LIMIT); 4260 vmcs12->guest_ds_limit = vmcs_read32(GUEST_DS_LIMIT); 4261 vmcs12->guest_fs_limit = vmcs_read32(GUEST_FS_LIMIT); 4262 vmcs12->guest_gs_limit = vmcs_read32(GUEST_GS_LIMIT); 4263 vmcs12->guest_ldtr_limit = vmcs_read32(GUEST_LDTR_LIMIT); 4264 vmcs12->guest_tr_limit = vmcs_read32(GUEST_TR_LIMIT); 4265 vmcs12->guest_gdtr_limit = vmcs_read32(GUEST_GDTR_LIMIT); 4266 vmcs12->guest_idtr_limit = vmcs_read32(GUEST_IDTR_LIMIT); 4267 vmcs12->guest_es_ar_bytes = vmcs_read32(GUEST_ES_AR_BYTES); 4268 vmcs12->guest_ds_ar_bytes = vmcs_read32(GUEST_DS_AR_BYTES); 4269 vmcs12->guest_fs_ar_bytes = vmcs_read32(GUEST_FS_AR_BYTES); 4270 vmcs12->guest_gs_ar_bytes = vmcs_read32(GUEST_GS_AR_BYTES); 4271 vmcs12->guest_ldtr_ar_bytes = vmcs_read32(GUEST_LDTR_AR_BYTES); 4272 vmcs12->guest_tr_ar_bytes = vmcs_read32(GUEST_TR_AR_BYTES); 4273 vmcs12->guest_es_base = vmcs_readl(GUEST_ES_BASE); 4274 vmcs12->guest_cs_base = vmcs_readl(GUEST_CS_BASE); 4275 vmcs12->guest_ss_base = vmcs_readl(GUEST_SS_BASE); 4276 vmcs12->guest_ds_base = vmcs_readl(GUEST_DS_BASE); 4277 vmcs12->guest_fs_base = vmcs_readl(GUEST_FS_BASE); 4278 vmcs12->guest_gs_base = vmcs_readl(GUEST_GS_BASE); 4279 vmcs12->guest_ldtr_base = vmcs_readl(GUEST_LDTR_BASE); 4280 vmcs12->guest_tr_base = vmcs_readl(GUEST_TR_BASE); 4281 vmcs12->guest_gdtr_base = vmcs_readl(GUEST_GDTR_BASE); 4282 vmcs12->guest_idtr_base = vmcs_readl(GUEST_IDTR_BASE); 4283 vmcs12->guest_pending_dbg_exceptions = 4284 vmcs_readl(GUEST_PENDING_DBG_EXCEPTIONS); 4285 4286 vmx->nested.need_sync_vmcs02_to_vmcs12_rare = false; 4287 } 4288 4289 static void copy_vmcs02_to_vmcs12_rare(struct kvm_vcpu *vcpu, 4290 struct vmcs12 *vmcs12) 4291 { 4292 struct vcpu_vmx *vmx = to_vmx(vcpu); 4293 int cpu; 4294 4295 if (!vmx->nested.need_sync_vmcs02_to_vmcs12_rare) 4296 return; 4297 4298 4299 WARN_ON_ONCE(vmx->loaded_vmcs != &vmx->vmcs01); 4300 4301 cpu = get_cpu(); 4302 vmx->loaded_vmcs = &vmx->nested.vmcs02; 4303 vmx_vcpu_load_vmcs(vcpu, cpu, &vmx->vmcs01); 4304 4305 sync_vmcs02_to_vmcs12_rare(vcpu, vmcs12); 4306 4307 vmx->loaded_vmcs = &vmx->vmcs01; 4308 vmx_vcpu_load_vmcs(vcpu, cpu, &vmx->nested.vmcs02); 4309 put_cpu(); 4310 } 4311 4312 /* 4313 * Update the guest state fields of vmcs12 to reflect changes that 4314 * occurred while L2 was running. (The "IA-32e mode guest" bit of the 4315 * VM-entry controls is also updated, since this is really a guest 4316 * state bit.) 4317 */ 4318 static void sync_vmcs02_to_vmcs12(struct kvm_vcpu *vcpu, struct vmcs12 *vmcs12) 4319 { 4320 struct vcpu_vmx *vmx = to_vmx(vcpu); 4321 4322 if (evmptr_is_valid(vmx->nested.hv_evmcs_vmptr)) 4323 sync_vmcs02_to_vmcs12_rare(vcpu, vmcs12); 4324 4325 vmx->nested.need_sync_vmcs02_to_vmcs12_rare = 4326 !evmptr_is_valid(vmx->nested.hv_evmcs_vmptr); 4327 4328 vmcs12->guest_cr0 = vmcs12_guest_cr0(vcpu, vmcs12); 4329 vmcs12->guest_cr4 = vmcs12_guest_cr4(vcpu, vmcs12); 4330 4331 vmcs12->guest_rsp = kvm_rsp_read(vcpu); 4332 vmcs12->guest_rip = kvm_rip_read(vcpu); 4333 vmcs12->guest_rflags = vmcs_readl(GUEST_RFLAGS); 4334 4335 vmcs12->guest_cs_ar_bytes = vmcs_read32(GUEST_CS_AR_BYTES); 4336 vmcs12->guest_ss_ar_bytes = vmcs_read32(GUEST_SS_AR_BYTES); 4337 4338 vmcs12->guest_interruptibility_info = 4339 vmcs_read32(GUEST_INTERRUPTIBILITY_INFO); 4340 4341 if (vcpu->arch.mp_state == KVM_MP_STATE_HALTED) 4342 vmcs12->guest_activity_state = GUEST_ACTIVITY_HLT; 4343 else if (vcpu->arch.mp_state == KVM_MP_STATE_INIT_RECEIVED) 4344 vmcs12->guest_activity_state = GUEST_ACTIVITY_WAIT_SIPI; 4345 else 4346 vmcs12->guest_activity_state = GUEST_ACTIVITY_ACTIVE; 4347 4348 if (nested_cpu_has_preemption_timer(vmcs12) && 4349 vmcs12->vm_exit_controls & VM_EXIT_SAVE_VMX_PREEMPTION_TIMER && 4350 !vmx->nested.nested_run_pending) 4351 vmcs12->vmx_preemption_timer_value = 4352 vmx_get_preemption_timer_value(vcpu); 4353 4354 /* 4355 * In some cases (usually, nested EPT), L2 is allowed to change its 4356 * own CR3 without exiting. If it has changed it, we must keep it. 4357 * Of course, if L0 is using shadow page tables, GUEST_CR3 was defined 4358 * by L0, not L1 or L2, so we mustn't unconditionally copy it to vmcs12. 4359 * 4360 * Additionally, restore L2's PDPTR to vmcs12. 4361 */ 4362 if (enable_ept) { 4363 vmcs12->guest_cr3 = vmcs_readl(GUEST_CR3); 4364 if (nested_cpu_has_ept(vmcs12) && is_pae_paging(vcpu)) { 4365 vmcs12->guest_pdptr0 = vmcs_read64(GUEST_PDPTR0); 4366 vmcs12->guest_pdptr1 = vmcs_read64(GUEST_PDPTR1); 4367 vmcs12->guest_pdptr2 = vmcs_read64(GUEST_PDPTR2); 4368 vmcs12->guest_pdptr3 = vmcs_read64(GUEST_PDPTR3); 4369 } 4370 } 4371 4372 vmcs12->guest_linear_address = vmcs_readl(GUEST_LINEAR_ADDRESS); 4373 4374 if (nested_cpu_has_vid(vmcs12)) 4375 vmcs12->guest_intr_status = vmcs_read16(GUEST_INTR_STATUS); 4376 4377 vmcs12->vm_entry_controls = 4378 (vmcs12->vm_entry_controls & ~VM_ENTRY_IA32E_MODE) | 4379 (vm_entry_controls_get(to_vmx(vcpu)) & VM_ENTRY_IA32E_MODE); 4380 4381 if (vmcs12->vm_exit_controls & VM_EXIT_SAVE_DEBUG_CONTROLS) 4382 kvm_get_dr(vcpu, 7, (unsigned long *)&vmcs12->guest_dr7); 4383 4384 if (vmcs12->vm_exit_controls & VM_EXIT_SAVE_IA32_EFER) 4385 vmcs12->guest_ia32_efer = vcpu->arch.efer; 4386 } 4387 4388 /* 4389 * prepare_vmcs12 is part of what we need to do when the nested L2 guest exits 4390 * and we want to prepare to run its L1 parent. L1 keeps a vmcs for L2 (vmcs12), 4391 * and this function updates it to reflect the changes to the guest state while 4392 * L2 was running (and perhaps made some exits which were handled directly by L0 4393 * without going back to L1), and to reflect the exit reason. 4394 * Note that we do not have to copy here all VMCS fields, just those that 4395 * could have changed by the L2 guest or the exit - i.e., the guest-state and 4396 * exit-information fields only. Other fields are modified by L1 with VMWRITE, 4397 * which already writes to vmcs12 directly. 4398 */ 4399 static void prepare_vmcs12(struct kvm_vcpu *vcpu, struct vmcs12 *vmcs12, 4400 u32 vm_exit_reason, u32 exit_intr_info, 4401 unsigned long exit_qualification) 4402 { 4403 /* update exit information fields: */ 4404 vmcs12->vm_exit_reason = vm_exit_reason; 4405 if (to_vmx(vcpu)->exit_reason.enclave_mode) 4406 vmcs12->vm_exit_reason |= VMX_EXIT_REASONS_SGX_ENCLAVE_MODE; 4407 vmcs12->exit_qualification = exit_qualification; 4408 4409 /* 4410 * On VM-Exit due to a failed VM-Entry, the VMCS isn't marked launched 4411 * and only EXIT_REASON and EXIT_QUALIFICATION are updated, all other 4412 * exit info fields are unmodified. 4413 */ 4414 if (!(vmcs12->vm_exit_reason & VMX_EXIT_REASONS_FAILED_VMENTRY)) { 4415 vmcs12->launch_state = 1; 4416 4417 /* vm_entry_intr_info_field is cleared on exit. Emulate this 4418 * instead of reading the real value. */ 4419 vmcs12->vm_entry_intr_info_field &= ~INTR_INFO_VALID_MASK; 4420 4421 /* 4422 * Transfer the event that L0 or L1 may wanted to inject into 4423 * L2 to IDT_VECTORING_INFO_FIELD. 4424 */ 4425 vmcs12_save_pending_event(vcpu, vmcs12, 4426 vm_exit_reason, exit_intr_info); 4427 4428 vmcs12->vm_exit_intr_info = exit_intr_info; 4429 vmcs12->vm_exit_instruction_len = vmcs_read32(VM_EXIT_INSTRUCTION_LEN); 4430 vmcs12->vmx_instruction_info = vmcs_read32(VMX_INSTRUCTION_INFO); 4431 4432 /* 4433 * According to spec, there's no need to store the guest's 4434 * MSRs if the exit is due to a VM-entry failure that occurs 4435 * during or after loading the guest state. Since this exit 4436 * does not fall in that category, we need to save the MSRs. 4437 */ 4438 if (nested_vmx_store_msr(vcpu, 4439 vmcs12->vm_exit_msr_store_addr, 4440 vmcs12->vm_exit_msr_store_count)) 4441 nested_vmx_abort(vcpu, 4442 VMX_ABORT_SAVE_GUEST_MSR_FAIL); 4443 } 4444 } 4445 4446 /* 4447 * A part of what we need to when the nested L2 guest exits and we want to 4448 * run its L1 parent, is to reset L1's guest state to the host state specified 4449 * in vmcs12. 4450 * This function is to be called not only on normal nested exit, but also on 4451 * a nested entry failure, as explained in Intel's spec, 3B.23.7 ("VM-Entry 4452 * Failures During or After Loading Guest State"). 4453 * This function should be called when the active VMCS is L1's (vmcs01). 4454 */ 4455 static void load_vmcs12_host_state(struct kvm_vcpu *vcpu, 4456 struct vmcs12 *vmcs12) 4457 { 4458 enum vm_entry_failure_code ignored; 4459 struct kvm_segment seg; 4460 4461 if (vmcs12->vm_exit_controls & VM_EXIT_LOAD_IA32_EFER) 4462 vcpu->arch.efer = vmcs12->host_ia32_efer; 4463 else if (vmcs12->vm_exit_controls & VM_EXIT_HOST_ADDR_SPACE_SIZE) 4464 vcpu->arch.efer |= (EFER_LMA | EFER_LME); 4465 else 4466 vcpu->arch.efer &= ~(EFER_LMA | EFER_LME); 4467 vmx_set_efer(vcpu, vcpu->arch.efer); 4468 4469 kvm_rsp_write(vcpu, vmcs12->host_rsp); 4470 kvm_rip_write(vcpu, vmcs12->host_rip); 4471 vmx_set_rflags(vcpu, X86_EFLAGS_FIXED); 4472 vmx_set_interrupt_shadow(vcpu, 0); 4473 4474 /* 4475 * Note that calling vmx_set_cr0 is important, even if cr0 hasn't 4476 * actually changed, because vmx_set_cr0 refers to efer set above. 4477 * 4478 * CR0_GUEST_HOST_MASK is already set in the original vmcs01 4479 * (KVM doesn't change it); 4480 */ 4481 vcpu->arch.cr0_guest_owned_bits = KVM_POSSIBLE_CR0_GUEST_BITS; 4482 vmx_set_cr0(vcpu, vmcs12->host_cr0); 4483 4484 /* Same as above - no reason to call set_cr4_guest_host_mask(). */ 4485 vcpu->arch.cr4_guest_owned_bits = ~vmcs_readl(CR4_GUEST_HOST_MASK); 4486 vmx_set_cr4(vcpu, vmcs12->host_cr4); 4487 4488 nested_ept_uninit_mmu_context(vcpu); 4489 4490 /* 4491 * Only PDPTE load can fail as the value of cr3 was checked on entry and 4492 * couldn't have changed. 4493 */ 4494 if (nested_vmx_load_cr3(vcpu, vmcs12->host_cr3, false, true, &ignored)) 4495 nested_vmx_abort(vcpu, VMX_ABORT_LOAD_HOST_PDPTE_FAIL); 4496 4497 nested_vmx_transition_tlb_flush(vcpu, vmcs12, false); 4498 4499 vmcs_write32(GUEST_SYSENTER_CS, vmcs12->host_ia32_sysenter_cs); 4500 vmcs_writel(GUEST_SYSENTER_ESP, vmcs12->host_ia32_sysenter_esp); 4501 vmcs_writel(GUEST_SYSENTER_EIP, vmcs12->host_ia32_sysenter_eip); 4502 vmcs_writel(GUEST_IDTR_BASE, vmcs12->host_idtr_base); 4503 vmcs_writel(GUEST_GDTR_BASE, vmcs12->host_gdtr_base); 4504 vmcs_write32(GUEST_IDTR_LIMIT, 0xFFFF); 4505 vmcs_write32(GUEST_GDTR_LIMIT, 0xFFFF); 4506 4507 /* If not VM_EXIT_CLEAR_BNDCFGS, the L2 value propagates to L1. */ 4508 if (vmcs12->vm_exit_controls & VM_EXIT_CLEAR_BNDCFGS) 4509 vmcs_write64(GUEST_BNDCFGS, 0); 4510 4511 if (vmcs12->vm_exit_controls & VM_EXIT_LOAD_IA32_PAT) { 4512 vmcs_write64(GUEST_IA32_PAT, vmcs12->host_ia32_pat); 4513 vcpu->arch.pat = vmcs12->host_ia32_pat; 4514 } 4515 if ((vmcs12->vm_exit_controls & VM_EXIT_LOAD_IA32_PERF_GLOBAL_CTRL) && 4516 intel_pmu_has_perf_global_ctrl(vcpu_to_pmu(vcpu))) 4517 WARN_ON_ONCE(kvm_set_msr(vcpu, MSR_CORE_PERF_GLOBAL_CTRL, 4518 vmcs12->host_ia32_perf_global_ctrl)); 4519 4520 /* Set L1 segment info according to Intel SDM 4521 27.5.2 Loading Host Segment and Descriptor-Table Registers */ 4522 seg = (struct kvm_segment) { 4523 .base = 0, 4524 .limit = 0xFFFFFFFF, 4525 .selector = vmcs12->host_cs_selector, 4526 .type = 11, 4527 .present = 1, 4528 .s = 1, 4529 .g = 1 4530 }; 4531 if (vmcs12->vm_exit_controls & VM_EXIT_HOST_ADDR_SPACE_SIZE) 4532 seg.l = 1; 4533 else 4534 seg.db = 1; 4535 __vmx_set_segment(vcpu, &seg, VCPU_SREG_CS); 4536 seg = (struct kvm_segment) { 4537 .base = 0, 4538 .limit = 0xFFFFFFFF, 4539 .type = 3, 4540 .present = 1, 4541 .s = 1, 4542 .db = 1, 4543 .g = 1 4544 }; 4545 seg.selector = vmcs12->host_ds_selector; 4546 __vmx_set_segment(vcpu, &seg, VCPU_SREG_DS); 4547 seg.selector = vmcs12->host_es_selector; 4548 __vmx_set_segment(vcpu, &seg, VCPU_SREG_ES); 4549 seg.selector = vmcs12->host_ss_selector; 4550 __vmx_set_segment(vcpu, &seg, VCPU_SREG_SS); 4551 seg.selector = vmcs12->host_fs_selector; 4552 seg.base = vmcs12->host_fs_base; 4553 __vmx_set_segment(vcpu, &seg, VCPU_SREG_FS); 4554 seg.selector = vmcs12->host_gs_selector; 4555 seg.base = vmcs12->host_gs_base; 4556 __vmx_set_segment(vcpu, &seg, VCPU_SREG_GS); 4557 seg = (struct kvm_segment) { 4558 .base = vmcs12->host_tr_base, 4559 .limit = 0x67, 4560 .selector = vmcs12->host_tr_selector, 4561 .type = 11, 4562 .present = 1 4563 }; 4564 __vmx_set_segment(vcpu, &seg, VCPU_SREG_TR); 4565 4566 memset(&seg, 0, sizeof(seg)); 4567 seg.unusable = 1; 4568 __vmx_set_segment(vcpu, &seg, VCPU_SREG_LDTR); 4569 4570 kvm_set_dr(vcpu, 7, 0x400); 4571 vmcs_write64(GUEST_IA32_DEBUGCTL, 0); 4572 4573 if (nested_vmx_load_msr(vcpu, vmcs12->vm_exit_msr_load_addr, 4574 vmcs12->vm_exit_msr_load_count)) 4575 nested_vmx_abort(vcpu, VMX_ABORT_LOAD_HOST_MSR_FAIL); 4576 4577 to_vmx(vcpu)->emulation_required = vmx_emulation_required(vcpu); 4578 } 4579 4580 static inline u64 nested_vmx_get_vmcs01_guest_efer(struct vcpu_vmx *vmx) 4581 { 4582 struct vmx_uret_msr *efer_msr; 4583 unsigned int i; 4584 4585 if (vm_entry_controls_get(vmx) & VM_ENTRY_LOAD_IA32_EFER) 4586 return vmcs_read64(GUEST_IA32_EFER); 4587 4588 if (cpu_has_load_ia32_efer()) 4589 return host_efer; 4590 4591 for (i = 0; i < vmx->msr_autoload.guest.nr; ++i) { 4592 if (vmx->msr_autoload.guest.val[i].index == MSR_EFER) 4593 return vmx->msr_autoload.guest.val[i].value; 4594 } 4595 4596 efer_msr = vmx_find_uret_msr(vmx, MSR_EFER); 4597 if (efer_msr) 4598 return efer_msr->data; 4599 4600 return host_efer; 4601 } 4602 4603 static void nested_vmx_restore_host_state(struct kvm_vcpu *vcpu) 4604 { 4605 struct vmcs12 *vmcs12 = get_vmcs12(vcpu); 4606 struct vcpu_vmx *vmx = to_vmx(vcpu); 4607 struct vmx_msr_entry g, h; 4608 gpa_t gpa; 4609 u32 i, j; 4610 4611 vcpu->arch.pat = vmcs_read64(GUEST_IA32_PAT); 4612 4613 if (vmcs12->vm_entry_controls & VM_ENTRY_LOAD_DEBUG_CONTROLS) { 4614 /* 4615 * L1's host DR7 is lost if KVM_GUESTDBG_USE_HW_BP is set 4616 * as vmcs01.GUEST_DR7 contains a userspace defined value 4617 * and vcpu->arch.dr7 is not squirreled away before the 4618 * nested VMENTER (not worth adding a variable in nested_vmx). 4619 */ 4620 if (vcpu->guest_debug & KVM_GUESTDBG_USE_HW_BP) 4621 kvm_set_dr(vcpu, 7, DR7_FIXED_1); 4622 else 4623 WARN_ON(kvm_set_dr(vcpu, 7, vmcs_readl(GUEST_DR7))); 4624 } 4625 4626 /* 4627 * Note that calling vmx_set_{efer,cr0,cr4} is important as they 4628 * handle a variety of side effects to KVM's software model. 4629 */ 4630 vmx_set_efer(vcpu, nested_vmx_get_vmcs01_guest_efer(vmx)); 4631 4632 vcpu->arch.cr0_guest_owned_bits = KVM_POSSIBLE_CR0_GUEST_BITS; 4633 vmx_set_cr0(vcpu, vmcs_readl(CR0_READ_SHADOW)); 4634 4635 vcpu->arch.cr4_guest_owned_bits = ~vmcs_readl(CR4_GUEST_HOST_MASK); 4636 vmx_set_cr4(vcpu, vmcs_readl(CR4_READ_SHADOW)); 4637 4638 nested_ept_uninit_mmu_context(vcpu); 4639 vcpu->arch.cr3 = vmcs_readl(GUEST_CR3); 4640 kvm_register_mark_available(vcpu, VCPU_EXREG_CR3); 4641 4642 /* 4643 * Use ept_save_pdptrs(vcpu) to load the MMU's cached PDPTRs 4644 * from vmcs01 (if necessary). The PDPTRs are not loaded on 4645 * VMFail, like everything else we just need to ensure our 4646 * software model is up-to-date. 4647 */ 4648 if (enable_ept && is_pae_paging(vcpu)) 4649 ept_save_pdptrs(vcpu); 4650 4651 kvm_mmu_reset_context(vcpu); 4652 4653 /* 4654 * This nasty bit of open coding is a compromise between blindly 4655 * loading L1's MSRs using the exit load lists (incorrect emulation 4656 * of VMFail), leaving the nested VM's MSRs in the software model 4657 * (incorrect behavior) and snapshotting the modified MSRs (too 4658 * expensive since the lists are unbound by hardware). For each 4659 * MSR that was (prematurely) loaded from the nested VMEntry load 4660 * list, reload it from the exit load list if it exists and differs 4661 * from the guest value. The intent is to stuff host state as 4662 * silently as possible, not to fully process the exit load list. 4663 */ 4664 for (i = 0; i < vmcs12->vm_entry_msr_load_count; i++) { 4665 gpa = vmcs12->vm_entry_msr_load_addr + (i * sizeof(g)); 4666 if (kvm_vcpu_read_guest(vcpu, gpa, &g, sizeof(g))) { 4667 pr_debug_ratelimited( 4668 "%s read MSR index failed (%u, 0x%08llx)\n", 4669 __func__, i, gpa); 4670 goto vmabort; 4671 } 4672 4673 for (j = 0; j < vmcs12->vm_exit_msr_load_count; j++) { 4674 gpa = vmcs12->vm_exit_msr_load_addr + (j * sizeof(h)); 4675 if (kvm_vcpu_read_guest(vcpu, gpa, &h, sizeof(h))) { 4676 pr_debug_ratelimited( 4677 "%s read MSR failed (%u, 0x%08llx)\n", 4678 __func__, j, gpa); 4679 goto vmabort; 4680 } 4681 if (h.index != g.index) 4682 continue; 4683 if (h.value == g.value) 4684 break; 4685 4686 if (nested_vmx_load_msr_check(vcpu, &h)) { 4687 pr_debug_ratelimited( 4688 "%s check failed (%u, 0x%x, 0x%x)\n", 4689 __func__, j, h.index, h.reserved); 4690 goto vmabort; 4691 } 4692 4693 if (kvm_set_msr(vcpu, h.index, h.value)) { 4694 pr_debug_ratelimited( 4695 "%s WRMSR failed (%u, 0x%x, 0x%llx)\n", 4696 __func__, j, h.index, h.value); 4697 goto vmabort; 4698 } 4699 } 4700 } 4701 4702 return; 4703 4704 vmabort: 4705 nested_vmx_abort(vcpu, VMX_ABORT_LOAD_HOST_MSR_FAIL); 4706 } 4707 4708 /* 4709 * Emulate an exit from nested guest (L2) to L1, i.e., prepare to run L1 4710 * and modify vmcs12 to make it see what it would expect to see there if 4711 * L2 was its real guest. Must only be called when in L2 (is_guest_mode()) 4712 */ 4713 void nested_vmx_vmexit(struct kvm_vcpu *vcpu, u32 vm_exit_reason, 4714 u32 exit_intr_info, unsigned long exit_qualification) 4715 { 4716 struct vcpu_vmx *vmx = to_vmx(vcpu); 4717 struct vmcs12 *vmcs12 = get_vmcs12(vcpu); 4718 4719 /* Pending MTF traps are discarded on VM-Exit. */ 4720 vmx->nested.mtf_pending = false; 4721 4722 /* trying to cancel vmlaunch/vmresume is a bug */ 4723 WARN_ON_ONCE(vmx->nested.nested_run_pending); 4724 4725 if (kvm_check_request(KVM_REQ_GET_NESTED_STATE_PAGES, vcpu)) { 4726 /* 4727 * KVM_REQ_GET_NESTED_STATE_PAGES is also used to map 4728 * Enlightened VMCS after migration and we still need to 4729 * do that when something is forcing L2->L1 exit prior to 4730 * the first L2 run. 4731 */ 4732 (void)nested_get_evmcs_page(vcpu); 4733 } 4734 4735 /* Service pending TLB flush requests for L2 before switching to L1. */ 4736 kvm_service_local_tlb_flush_requests(vcpu); 4737 4738 /* 4739 * VCPU_EXREG_PDPTR will be clobbered in arch/x86/kvm/vmx/vmx.h between 4740 * now and the new vmentry. Ensure that the VMCS02 PDPTR fields are 4741 * up-to-date before switching to L1. 4742 */ 4743 if (enable_ept && is_pae_paging(vcpu)) 4744 vmx_ept_load_pdptrs(vcpu); 4745 4746 leave_guest_mode(vcpu); 4747 4748 if (nested_cpu_has_preemption_timer(vmcs12)) 4749 hrtimer_cancel(&to_vmx(vcpu)->nested.preemption_timer); 4750 4751 if (nested_cpu_has(vmcs12, CPU_BASED_USE_TSC_OFFSETTING)) { 4752 vcpu->arch.tsc_offset = vcpu->arch.l1_tsc_offset; 4753 if (nested_cpu_has2(vmcs12, SECONDARY_EXEC_TSC_SCALING)) 4754 vcpu->arch.tsc_scaling_ratio = vcpu->arch.l1_tsc_scaling_ratio; 4755 } 4756 4757 if (likely(!vmx->fail)) { 4758 sync_vmcs02_to_vmcs12(vcpu, vmcs12); 4759 4760 if (vm_exit_reason != -1) 4761 prepare_vmcs12(vcpu, vmcs12, vm_exit_reason, 4762 exit_intr_info, exit_qualification); 4763 4764 /* 4765 * Must happen outside of sync_vmcs02_to_vmcs12() as it will 4766 * also be used to capture vmcs12 cache as part of 4767 * capturing nVMX state for snapshot (migration). 4768 * 4769 * Otherwise, this flush will dirty guest memory at a 4770 * point it is already assumed by user-space to be 4771 * immutable. 4772 */ 4773 nested_flush_cached_shadow_vmcs12(vcpu, vmcs12); 4774 } else { 4775 /* 4776 * The only expected VM-instruction error is "VM entry with 4777 * invalid control field(s)." Anything else indicates a 4778 * problem with L0. And we should never get here with a 4779 * VMFail of any type if early consistency checks are enabled. 4780 */ 4781 WARN_ON_ONCE(vmcs_read32(VM_INSTRUCTION_ERROR) != 4782 VMXERR_ENTRY_INVALID_CONTROL_FIELD); 4783 WARN_ON_ONCE(nested_early_check); 4784 } 4785 4786 /* 4787 * Drop events/exceptions that were queued for re-injection to L2 4788 * (picked up via vmx_complete_interrupts()), as well as exceptions 4789 * that were pending for L2. Note, this must NOT be hoisted above 4790 * prepare_vmcs12(), events/exceptions queued for re-injection need to 4791 * be captured in vmcs12 (see vmcs12_save_pending_event()). 4792 */ 4793 vcpu->arch.nmi_injected = false; 4794 kvm_clear_exception_queue(vcpu); 4795 kvm_clear_interrupt_queue(vcpu); 4796 4797 vmx_switch_vmcs(vcpu, &vmx->vmcs01); 4798 4799 /* 4800 * If IBRS is advertised to the vCPU, KVM must flush the indirect 4801 * branch predictors when transitioning from L2 to L1, as L1 expects 4802 * hardware (KVM in this case) to provide separate predictor modes. 4803 * Bare metal isolates VMX root (host) from VMX non-root (guest), but 4804 * doesn't isolate different VMCSs, i.e. in this case, doesn't provide 4805 * separate modes for L2 vs L1. 4806 */ 4807 if (guest_cpuid_has(vcpu, X86_FEATURE_SPEC_CTRL)) 4808 indirect_branch_prediction_barrier(); 4809 4810 /* Update any VMCS fields that might have changed while L2 ran */ 4811 vmcs_write32(VM_EXIT_MSR_LOAD_COUNT, vmx->msr_autoload.host.nr); 4812 vmcs_write32(VM_ENTRY_MSR_LOAD_COUNT, vmx->msr_autoload.guest.nr); 4813 vmcs_write64(TSC_OFFSET, vcpu->arch.tsc_offset); 4814 if (kvm_caps.has_tsc_control) 4815 vmcs_write64(TSC_MULTIPLIER, vcpu->arch.tsc_scaling_ratio); 4816 4817 if (vmx->nested.l1_tpr_threshold != -1) 4818 vmcs_write32(TPR_THRESHOLD, vmx->nested.l1_tpr_threshold); 4819 4820 if (vmx->nested.change_vmcs01_virtual_apic_mode) { 4821 vmx->nested.change_vmcs01_virtual_apic_mode = false; 4822 vmx_set_virtual_apic_mode(vcpu); 4823 } 4824 4825 if (vmx->nested.update_vmcs01_cpu_dirty_logging) { 4826 vmx->nested.update_vmcs01_cpu_dirty_logging = false; 4827 vmx_update_cpu_dirty_logging(vcpu); 4828 } 4829 4830 /* Unpin physical memory we referred to in vmcs02 */ 4831 kvm_vcpu_unmap(vcpu, &vmx->nested.apic_access_page_map, false); 4832 kvm_vcpu_unmap(vcpu, &vmx->nested.virtual_apic_map, true); 4833 kvm_vcpu_unmap(vcpu, &vmx->nested.pi_desc_map, true); 4834 vmx->nested.pi_desc = NULL; 4835 4836 if (vmx->nested.reload_vmcs01_apic_access_page) { 4837 vmx->nested.reload_vmcs01_apic_access_page = false; 4838 kvm_make_request(KVM_REQ_APIC_PAGE_RELOAD, vcpu); 4839 } 4840 4841 if (vmx->nested.update_vmcs01_apicv_status) { 4842 vmx->nested.update_vmcs01_apicv_status = false; 4843 kvm_make_request(KVM_REQ_APICV_UPDATE, vcpu); 4844 } 4845 4846 if ((vm_exit_reason != -1) && 4847 (enable_shadow_vmcs || evmptr_is_valid(vmx->nested.hv_evmcs_vmptr))) 4848 vmx->nested.need_vmcs12_to_shadow_sync = true; 4849 4850 /* in case we halted in L2 */ 4851 vcpu->arch.mp_state = KVM_MP_STATE_RUNNABLE; 4852 4853 if (likely(!vmx->fail)) { 4854 if ((u16)vm_exit_reason == EXIT_REASON_EXTERNAL_INTERRUPT && 4855 nested_exit_intr_ack_set(vcpu)) { 4856 int irq = kvm_cpu_get_interrupt(vcpu); 4857 WARN_ON(irq < 0); 4858 vmcs12->vm_exit_intr_info = irq | 4859 INTR_INFO_VALID_MASK | INTR_TYPE_EXT_INTR; 4860 } 4861 4862 if (vm_exit_reason != -1) 4863 trace_kvm_nested_vmexit_inject(vmcs12->vm_exit_reason, 4864 vmcs12->exit_qualification, 4865 vmcs12->idt_vectoring_info_field, 4866 vmcs12->vm_exit_intr_info, 4867 vmcs12->vm_exit_intr_error_code, 4868 KVM_ISA_VMX); 4869 4870 load_vmcs12_host_state(vcpu, vmcs12); 4871 4872 return; 4873 } 4874 4875 /* 4876 * After an early L2 VM-entry failure, we're now back 4877 * in L1 which thinks it just finished a VMLAUNCH or 4878 * VMRESUME instruction, so we need to set the failure 4879 * flag and the VM-instruction error field of the VMCS 4880 * accordingly, and skip the emulated instruction. 4881 */ 4882 (void)nested_vmx_fail(vcpu, VMXERR_ENTRY_INVALID_CONTROL_FIELD); 4883 4884 /* 4885 * Restore L1's host state to KVM's software model. We're here 4886 * because a consistency check was caught by hardware, which 4887 * means some amount of guest state has been propagated to KVM's 4888 * model and needs to be unwound to the host's state. 4889 */ 4890 nested_vmx_restore_host_state(vcpu); 4891 4892 vmx->fail = 0; 4893 } 4894 4895 static void nested_vmx_triple_fault(struct kvm_vcpu *vcpu) 4896 { 4897 kvm_clear_request(KVM_REQ_TRIPLE_FAULT, vcpu); 4898 nested_vmx_vmexit(vcpu, EXIT_REASON_TRIPLE_FAULT, 0, 0); 4899 } 4900 4901 /* 4902 * Decode the memory-address operand of a vmx instruction, as recorded on an 4903 * exit caused by such an instruction (run by a guest hypervisor). 4904 * On success, returns 0. When the operand is invalid, returns 1 and throws 4905 * #UD, #GP, or #SS. 4906 */ 4907 int get_vmx_mem_address(struct kvm_vcpu *vcpu, unsigned long exit_qualification, 4908 u32 vmx_instruction_info, bool wr, int len, gva_t *ret) 4909 { 4910 gva_t off; 4911 bool exn; 4912 struct kvm_segment s; 4913 4914 /* 4915 * According to Vol. 3B, "Information for VM Exits Due to Instruction 4916 * Execution", on an exit, vmx_instruction_info holds most of the 4917 * addressing components of the operand. Only the displacement part 4918 * is put in exit_qualification (see 3B, "Basic VM-Exit Information"). 4919 * For how an actual address is calculated from all these components, 4920 * refer to Vol. 1, "Operand Addressing". 4921 */ 4922 int scaling = vmx_instruction_info & 3; 4923 int addr_size = (vmx_instruction_info >> 7) & 7; 4924 bool is_reg = vmx_instruction_info & (1u << 10); 4925 int seg_reg = (vmx_instruction_info >> 15) & 7; 4926 int index_reg = (vmx_instruction_info >> 18) & 0xf; 4927 bool index_is_valid = !(vmx_instruction_info & (1u << 22)); 4928 int base_reg = (vmx_instruction_info >> 23) & 0xf; 4929 bool base_is_valid = !(vmx_instruction_info & (1u << 27)); 4930 4931 if (is_reg) { 4932 kvm_queue_exception(vcpu, UD_VECTOR); 4933 return 1; 4934 } 4935 4936 /* Addr = segment_base + offset */ 4937 /* offset = base + [index * scale] + displacement */ 4938 off = exit_qualification; /* holds the displacement */ 4939 if (addr_size == 1) 4940 off = (gva_t)sign_extend64(off, 31); 4941 else if (addr_size == 0) 4942 off = (gva_t)sign_extend64(off, 15); 4943 if (base_is_valid) 4944 off += kvm_register_read(vcpu, base_reg); 4945 if (index_is_valid) 4946 off += kvm_register_read(vcpu, index_reg) << scaling; 4947 vmx_get_segment(vcpu, &s, seg_reg); 4948 4949 /* 4950 * The effective address, i.e. @off, of a memory operand is truncated 4951 * based on the address size of the instruction. Note that this is 4952 * the *effective address*, i.e. the address prior to accounting for 4953 * the segment's base. 4954 */ 4955 if (addr_size == 1) /* 32 bit */ 4956 off &= 0xffffffff; 4957 else if (addr_size == 0) /* 16 bit */ 4958 off &= 0xffff; 4959 4960 /* Checks for #GP/#SS exceptions. */ 4961 exn = false; 4962 if (is_long_mode(vcpu)) { 4963 /* 4964 * The virtual/linear address is never truncated in 64-bit 4965 * mode, e.g. a 32-bit address size can yield a 64-bit virtual 4966 * address when using FS/GS with a non-zero base. 4967 */ 4968 if (seg_reg == VCPU_SREG_FS || seg_reg == VCPU_SREG_GS) 4969 *ret = s.base + off; 4970 else 4971 *ret = off; 4972 4973 /* Long mode: #GP(0)/#SS(0) if the memory address is in a 4974 * non-canonical form. This is the only check on the memory 4975 * destination for long mode! 4976 */ 4977 exn = is_noncanonical_address(*ret, vcpu); 4978 } else { 4979 /* 4980 * When not in long mode, the virtual/linear address is 4981 * unconditionally truncated to 32 bits regardless of the 4982 * address size. 4983 */ 4984 *ret = (s.base + off) & 0xffffffff; 4985 4986 /* Protected mode: apply checks for segment validity in the 4987 * following order: 4988 * - segment type check (#GP(0) may be thrown) 4989 * - usability check (#GP(0)/#SS(0)) 4990 * - limit check (#GP(0)/#SS(0)) 4991 */ 4992 if (wr) 4993 /* #GP(0) if the destination operand is located in a 4994 * read-only data segment or any code segment. 4995 */ 4996 exn = ((s.type & 0xa) == 0 || (s.type & 8)); 4997 else 4998 /* #GP(0) if the source operand is located in an 4999 * execute-only code segment 5000 */ 5001 exn = ((s.type & 0xa) == 8); 5002 if (exn) { 5003 kvm_queue_exception_e(vcpu, GP_VECTOR, 0); 5004 return 1; 5005 } 5006 /* Protected mode: #GP(0)/#SS(0) if the segment is unusable. 5007 */ 5008 exn = (s.unusable != 0); 5009 5010 /* 5011 * Protected mode: #GP(0)/#SS(0) if the memory operand is 5012 * outside the segment limit. All CPUs that support VMX ignore 5013 * limit checks for flat segments, i.e. segments with base==0, 5014 * limit==0xffffffff and of type expand-up data or code. 5015 */ 5016 if (!(s.base == 0 && s.limit == 0xffffffff && 5017 ((s.type & 8) || !(s.type & 4)))) 5018 exn = exn || ((u64)off + len - 1 > s.limit); 5019 } 5020 if (exn) { 5021 kvm_queue_exception_e(vcpu, 5022 seg_reg == VCPU_SREG_SS ? 5023 SS_VECTOR : GP_VECTOR, 5024 0); 5025 return 1; 5026 } 5027 5028 return 0; 5029 } 5030 5031 static int nested_vmx_get_vmptr(struct kvm_vcpu *vcpu, gpa_t *vmpointer, 5032 int *ret) 5033 { 5034 gva_t gva; 5035 struct x86_exception e; 5036 int r; 5037 5038 if (get_vmx_mem_address(vcpu, vmx_get_exit_qual(vcpu), 5039 vmcs_read32(VMX_INSTRUCTION_INFO), false, 5040 sizeof(*vmpointer), &gva)) { 5041 *ret = 1; 5042 return -EINVAL; 5043 } 5044 5045 r = kvm_read_guest_virt(vcpu, gva, vmpointer, sizeof(*vmpointer), &e); 5046 if (r != X86EMUL_CONTINUE) { 5047 *ret = kvm_handle_memory_failure(vcpu, r, &e); 5048 return -EINVAL; 5049 } 5050 5051 return 0; 5052 } 5053 5054 /* 5055 * Allocate a shadow VMCS and associate it with the currently loaded 5056 * VMCS, unless such a shadow VMCS already exists. The newly allocated 5057 * VMCS is also VMCLEARed, so that it is ready for use. 5058 */ 5059 static struct vmcs *alloc_shadow_vmcs(struct kvm_vcpu *vcpu) 5060 { 5061 struct vcpu_vmx *vmx = to_vmx(vcpu); 5062 struct loaded_vmcs *loaded_vmcs = vmx->loaded_vmcs; 5063 5064 /* 5065 * KVM allocates a shadow VMCS only when L1 executes VMXON and frees it 5066 * when L1 executes VMXOFF or the vCPU is forced out of nested 5067 * operation. VMXON faults if the CPU is already post-VMXON, so it 5068 * should be impossible to already have an allocated shadow VMCS. KVM 5069 * doesn't support virtualization of VMCS shadowing, so vmcs01 should 5070 * always be the loaded VMCS. 5071 */ 5072 if (WARN_ON(loaded_vmcs != &vmx->vmcs01 || loaded_vmcs->shadow_vmcs)) 5073 return loaded_vmcs->shadow_vmcs; 5074 5075 loaded_vmcs->shadow_vmcs = alloc_vmcs(true); 5076 if (loaded_vmcs->shadow_vmcs) 5077 vmcs_clear(loaded_vmcs->shadow_vmcs); 5078 5079 return loaded_vmcs->shadow_vmcs; 5080 } 5081 5082 static int enter_vmx_operation(struct kvm_vcpu *vcpu) 5083 { 5084 struct vcpu_vmx *vmx = to_vmx(vcpu); 5085 int r; 5086 5087 r = alloc_loaded_vmcs(&vmx->nested.vmcs02); 5088 if (r < 0) 5089 goto out_vmcs02; 5090 5091 vmx->nested.cached_vmcs12 = kzalloc(VMCS12_SIZE, GFP_KERNEL_ACCOUNT); 5092 if (!vmx->nested.cached_vmcs12) 5093 goto out_cached_vmcs12; 5094 5095 vmx->nested.shadow_vmcs12_cache.gpa = INVALID_GPA; 5096 vmx->nested.cached_shadow_vmcs12 = kzalloc(VMCS12_SIZE, GFP_KERNEL_ACCOUNT); 5097 if (!vmx->nested.cached_shadow_vmcs12) 5098 goto out_cached_shadow_vmcs12; 5099 5100 if (enable_shadow_vmcs && !alloc_shadow_vmcs(vcpu)) 5101 goto out_shadow_vmcs; 5102 5103 hrtimer_init(&vmx->nested.preemption_timer, CLOCK_MONOTONIC, 5104 HRTIMER_MODE_ABS_PINNED); 5105 vmx->nested.preemption_timer.function = vmx_preemption_timer_fn; 5106 5107 vmx->nested.vpid02 = allocate_vpid(); 5108 5109 vmx->nested.vmcs02_initialized = false; 5110 vmx->nested.vmxon = true; 5111 5112 if (vmx_pt_mode_is_host_guest()) { 5113 vmx->pt_desc.guest.ctl = 0; 5114 pt_update_intercept_for_msr(vcpu); 5115 } 5116 5117 return 0; 5118 5119 out_shadow_vmcs: 5120 kfree(vmx->nested.cached_shadow_vmcs12); 5121 5122 out_cached_shadow_vmcs12: 5123 kfree(vmx->nested.cached_vmcs12); 5124 5125 out_cached_vmcs12: 5126 free_loaded_vmcs(&vmx->nested.vmcs02); 5127 5128 out_vmcs02: 5129 return -ENOMEM; 5130 } 5131 5132 /* Emulate the VMXON instruction. */ 5133 static int handle_vmxon(struct kvm_vcpu *vcpu) 5134 { 5135 int ret; 5136 gpa_t vmptr; 5137 uint32_t revision; 5138 struct vcpu_vmx *vmx = to_vmx(vcpu); 5139 const u64 VMXON_NEEDED_FEATURES = FEAT_CTL_LOCKED 5140 | FEAT_CTL_VMX_ENABLED_OUTSIDE_SMX; 5141 5142 /* 5143 * Manually check CR4.VMXE checks, KVM must force CR4.VMXE=1 to enter 5144 * the guest and so cannot rely on hardware to perform the check, 5145 * which has higher priority than VM-Exit (see Intel SDM's pseudocode 5146 * for VMXON). 5147 * 5148 * Rely on hardware for the other pre-VM-Exit checks, CR0.PE=1, !VM86 5149 * and !COMPATIBILITY modes. For an unrestricted guest, KVM doesn't 5150 * force any of the relevant guest state. For a restricted guest, KVM 5151 * does force CR0.PE=1, but only to also force VM86 in order to emulate 5152 * Real Mode, and so there's no need to check CR0.PE manually. 5153 */ 5154 if (!kvm_read_cr4_bits(vcpu, X86_CR4_VMXE)) { 5155 kvm_queue_exception(vcpu, UD_VECTOR); 5156 return 1; 5157 } 5158 5159 /* 5160 * The CPL is checked for "not in VMX operation" and for "in VMX root", 5161 * and has higher priority than the VM-Fail due to being post-VMXON, 5162 * i.e. VMXON #GPs outside of VMX non-root if CPL!=0. In VMX non-root, 5163 * VMXON causes VM-Exit and KVM unconditionally forwards VMXON VM-Exits 5164 * from L2 to L1, i.e. there's no need to check for the vCPU being in 5165 * VMX non-root. 5166 * 5167 * Forwarding the VM-Exit unconditionally, i.e. without performing the 5168 * #UD checks (see above), is functionally ok because KVM doesn't allow 5169 * L1 to run L2 without CR4.VMXE=0, and because KVM never modifies L2's 5170 * CR0 or CR4, i.e. it's L2's responsibility to emulate #UDs that are 5171 * missed by hardware due to shadowing CR0 and/or CR4. 5172 */ 5173 if (vmx_get_cpl(vcpu)) { 5174 kvm_inject_gp(vcpu, 0); 5175 return 1; 5176 } 5177 5178 if (vmx->nested.vmxon) 5179 return nested_vmx_fail(vcpu, VMXERR_VMXON_IN_VMX_ROOT_OPERATION); 5180 5181 /* 5182 * Invalid CR0/CR4 generates #GP. These checks are performed if and 5183 * only if the vCPU isn't already in VMX operation, i.e. effectively 5184 * have lower priority than the VM-Fail above. 5185 */ 5186 if (!nested_host_cr0_valid(vcpu, kvm_read_cr0(vcpu)) || 5187 !nested_host_cr4_valid(vcpu, kvm_read_cr4(vcpu))) { 5188 kvm_inject_gp(vcpu, 0); 5189 return 1; 5190 } 5191 5192 if ((vmx->msr_ia32_feature_control & VMXON_NEEDED_FEATURES) 5193 != VMXON_NEEDED_FEATURES) { 5194 kvm_inject_gp(vcpu, 0); 5195 return 1; 5196 } 5197 5198 if (nested_vmx_get_vmptr(vcpu, &vmptr, &ret)) 5199 return ret; 5200 5201 /* 5202 * SDM 3: 24.11.5 5203 * The first 4 bytes of VMXON region contain the supported 5204 * VMCS revision identifier 5205 * 5206 * Note - IA32_VMX_BASIC[48] will never be 1 for the nested case; 5207 * which replaces physical address width with 32 5208 */ 5209 if (!page_address_valid(vcpu, vmptr)) 5210 return nested_vmx_failInvalid(vcpu); 5211 5212 if (kvm_read_guest(vcpu->kvm, vmptr, &revision, sizeof(revision)) || 5213 revision != VMCS12_REVISION) 5214 return nested_vmx_failInvalid(vcpu); 5215 5216 vmx->nested.vmxon_ptr = vmptr; 5217 ret = enter_vmx_operation(vcpu); 5218 if (ret) 5219 return ret; 5220 5221 return nested_vmx_succeed(vcpu); 5222 } 5223 5224 static inline void nested_release_vmcs12(struct kvm_vcpu *vcpu) 5225 { 5226 struct vcpu_vmx *vmx = to_vmx(vcpu); 5227 5228 if (vmx->nested.current_vmptr == INVALID_GPA) 5229 return; 5230 5231 copy_vmcs02_to_vmcs12_rare(vcpu, get_vmcs12(vcpu)); 5232 5233 if (enable_shadow_vmcs) { 5234 /* copy to memory all shadowed fields in case 5235 they were modified */ 5236 copy_shadow_to_vmcs12(vmx); 5237 vmx_disable_shadow_vmcs(vmx); 5238 } 5239 vmx->nested.posted_intr_nv = -1; 5240 5241 /* Flush VMCS12 to guest memory */ 5242 kvm_vcpu_write_guest_page(vcpu, 5243 vmx->nested.current_vmptr >> PAGE_SHIFT, 5244 vmx->nested.cached_vmcs12, 0, VMCS12_SIZE); 5245 5246 kvm_mmu_free_roots(vcpu->kvm, &vcpu->arch.guest_mmu, KVM_MMU_ROOTS_ALL); 5247 5248 vmx->nested.current_vmptr = INVALID_GPA; 5249 } 5250 5251 /* Emulate the VMXOFF instruction */ 5252 static int handle_vmxoff(struct kvm_vcpu *vcpu) 5253 { 5254 if (!nested_vmx_check_permission(vcpu)) 5255 return 1; 5256 5257 free_nested(vcpu); 5258 5259 if (kvm_apic_has_pending_init_or_sipi(vcpu)) 5260 kvm_make_request(KVM_REQ_EVENT, vcpu); 5261 5262 return nested_vmx_succeed(vcpu); 5263 } 5264 5265 /* Emulate the VMCLEAR instruction */ 5266 static int handle_vmclear(struct kvm_vcpu *vcpu) 5267 { 5268 struct vcpu_vmx *vmx = to_vmx(vcpu); 5269 u32 zero = 0; 5270 gpa_t vmptr; 5271 int r; 5272 5273 if (!nested_vmx_check_permission(vcpu)) 5274 return 1; 5275 5276 if (nested_vmx_get_vmptr(vcpu, &vmptr, &r)) 5277 return r; 5278 5279 if (!page_address_valid(vcpu, vmptr)) 5280 return nested_vmx_fail(vcpu, VMXERR_VMCLEAR_INVALID_ADDRESS); 5281 5282 if (vmptr == vmx->nested.vmxon_ptr) 5283 return nested_vmx_fail(vcpu, VMXERR_VMCLEAR_VMXON_POINTER); 5284 5285 /* 5286 * When Enlightened VMEntry is enabled on the calling CPU we treat 5287 * memory area pointer by vmptr as Enlightened VMCS (as there's no good 5288 * way to distinguish it from VMCS12) and we must not corrupt it by 5289 * writing to the non-existent 'launch_state' field. The area doesn't 5290 * have to be the currently active EVMCS on the calling CPU and there's 5291 * nothing KVM has to do to transition it from 'active' to 'non-active' 5292 * state. It is possible that the area will stay mapped as 5293 * vmx->nested.hv_evmcs but this shouldn't be a problem. 5294 */ 5295 if (likely(!guest_cpuid_has_evmcs(vcpu) || 5296 !evmptr_is_valid(nested_get_evmptr(vcpu)))) { 5297 if (vmptr == vmx->nested.current_vmptr) 5298 nested_release_vmcs12(vcpu); 5299 5300 /* 5301 * Silently ignore memory errors on VMCLEAR, Intel's pseudocode 5302 * for VMCLEAR includes a "ensure that data for VMCS referenced 5303 * by the operand is in memory" clause that guards writes to 5304 * memory, i.e. doing nothing for I/O is architecturally valid. 5305 * 5306 * FIXME: Suppress failures if and only if no memslot is found, 5307 * i.e. exit to userspace if __copy_to_user() fails. 5308 */ 5309 (void)kvm_vcpu_write_guest(vcpu, 5310 vmptr + offsetof(struct vmcs12, 5311 launch_state), 5312 &zero, sizeof(zero)); 5313 } else if (vmx->nested.hv_evmcs && vmptr == vmx->nested.hv_evmcs_vmptr) { 5314 nested_release_evmcs(vcpu); 5315 } 5316 5317 return nested_vmx_succeed(vcpu); 5318 } 5319 5320 /* Emulate the VMLAUNCH instruction */ 5321 static int handle_vmlaunch(struct kvm_vcpu *vcpu) 5322 { 5323 return nested_vmx_run(vcpu, true); 5324 } 5325 5326 /* Emulate the VMRESUME instruction */ 5327 static int handle_vmresume(struct kvm_vcpu *vcpu) 5328 { 5329 5330 return nested_vmx_run(vcpu, false); 5331 } 5332 5333 static int handle_vmread(struct kvm_vcpu *vcpu) 5334 { 5335 struct vmcs12 *vmcs12 = is_guest_mode(vcpu) ? get_shadow_vmcs12(vcpu) 5336 : get_vmcs12(vcpu); 5337 unsigned long exit_qualification = vmx_get_exit_qual(vcpu); 5338 u32 instr_info = vmcs_read32(VMX_INSTRUCTION_INFO); 5339 struct vcpu_vmx *vmx = to_vmx(vcpu); 5340 struct x86_exception e; 5341 unsigned long field; 5342 u64 value; 5343 gva_t gva = 0; 5344 short offset; 5345 int len, r; 5346 5347 if (!nested_vmx_check_permission(vcpu)) 5348 return 1; 5349 5350 /* Decode instruction info and find the field to read */ 5351 field = kvm_register_read(vcpu, (((instr_info) >> 28) & 0xf)); 5352 5353 if (!evmptr_is_valid(vmx->nested.hv_evmcs_vmptr)) { 5354 /* 5355 * In VMX non-root operation, when the VMCS-link pointer is INVALID_GPA, 5356 * any VMREAD sets the ALU flags for VMfailInvalid. 5357 */ 5358 if (vmx->nested.current_vmptr == INVALID_GPA || 5359 (is_guest_mode(vcpu) && 5360 get_vmcs12(vcpu)->vmcs_link_pointer == INVALID_GPA)) 5361 return nested_vmx_failInvalid(vcpu); 5362 5363 offset = get_vmcs12_field_offset(field); 5364 if (offset < 0) 5365 return nested_vmx_fail(vcpu, VMXERR_UNSUPPORTED_VMCS_COMPONENT); 5366 5367 if (!is_guest_mode(vcpu) && is_vmcs12_ext_field(field)) 5368 copy_vmcs02_to_vmcs12_rare(vcpu, vmcs12); 5369 5370 /* Read the field, zero-extended to a u64 value */ 5371 value = vmcs12_read_any(vmcs12, field, offset); 5372 } else { 5373 /* 5374 * Hyper-V TLFS (as of 6.0b) explicitly states, that while an 5375 * enlightened VMCS is active VMREAD/VMWRITE instructions are 5376 * unsupported. Unfortunately, certain versions of Windows 11 5377 * don't comply with this requirement which is not enforced in 5378 * genuine Hyper-V. Allow VMREAD from an enlightened VMCS as a 5379 * workaround, as misbehaving guests will panic on VM-Fail. 5380 * Note, enlightened VMCS is incompatible with shadow VMCS so 5381 * all VMREADs from L2 should go to L1. 5382 */ 5383 if (WARN_ON_ONCE(is_guest_mode(vcpu))) 5384 return nested_vmx_failInvalid(vcpu); 5385 5386 offset = evmcs_field_offset(field, NULL); 5387 if (offset < 0) 5388 return nested_vmx_fail(vcpu, VMXERR_UNSUPPORTED_VMCS_COMPONENT); 5389 5390 /* Read the field, zero-extended to a u64 value */ 5391 value = evmcs_read_any(vmx->nested.hv_evmcs, field, offset); 5392 } 5393 5394 /* 5395 * Now copy part of this value to register or memory, as requested. 5396 * Note that the number of bits actually copied is 32 or 64 depending 5397 * on the guest's mode (32 or 64 bit), not on the given field's length. 5398 */ 5399 if (instr_info & BIT(10)) { 5400 kvm_register_write(vcpu, (((instr_info) >> 3) & 0xf), value); 5401 } else { 5402 len = is_64_bit_mode(vcpu) ? 8 : 4; 5403 if (get_vmx_mem_address(vcpu, exit_qualification, 5404 instr_info, true, len, &gva)) 5405 return 1; 5406 /* _system ok, nested_vmx_check_permission has verified cpl=0 */ 5407 r = kvm_write_guest_virt_system(vcpu, gva, &value, len, &e); 5408 if (r != X86EMUL_CONTINUE) 5409 return kvm_handle_memory_failure(vcpu, r, &e); 5410 } 5411 5412 return nested_vmx_succeed(vcpu); 5413 } 5414 5415 static bool is_shadow_field_rw(unsigned long field) 5416 { 5417 switch (field) { 5418 #define SHADOW_FIELD_RW(x, y) case x: 5419 #include "vmcs_shadow_fields.h" 5420 return true; 5421 default: 5422 break; 5423 } 5424 return false; 5425 } 5426 5427 static bool is_shadow_field_ro(unsigned long field) 5428 { 5429 switch (field) { 5430 #define SHADOW_FIELD_RO(x, y) case x: 5431 #include "vmcs_shadow_fields.h" 5432 return true; 5433 default: 5434 break; 5435 } 5436 return false; 5437 } 5438 5439 static int handle_vmwrite(struct kvm_vcpu *vcpu) 5440 { 5441 struct vmcs12 *vmcs12 = is_guest_mode(vcpu) ? get_shadow_vmcs12(vcpu) 5442 : get_vmcs12(vcpu); 5443 unsigned long exit_qualification = vmx_get_exit_qual(vcpu); 5444 u32 instr_info = vmcs_read32(VMX_INSTRUCTION_INFO); 5445 struct vcpu_vmx *vmx = to_vmx(vcpu); 5446 struct x86_exception e; 5447 unsigned long field; 5448 short offset; 5449 gva_t gva; 5450 int len, r; 5451 5452 /* 5453 * The value to write might be 32 or 64 bits, depending on L1's long 5454 * mode, and eventually we need to write that into a field of several 5455 * possible lengths. The code below first zero-extends the value to 64 5456 * bit (value), and then copies only the appropriate number of 5457 * bits into the vmcs12 field. 5458 */ 5459 u64 value = 0; 5460 5461 if (!nested_vmx_check_permission(vcpu)) 5462 return 1; 5463 5464 /* 5465 * In VMX non-root operation, when the VMCS-link pointer is INVALID_GPA, 5466 * any VMWRITE sets the ALU flags for VMfailInvalid. 5467 */ 5468 if (vmx->nested.current_vmptr == INVALID_GPA || 5469 (is_guest_mode(vcpu) && 5470 get_vmcs12(vcpu)->vmcs_link_pointer == INVALID_GPA)) 5471 return nested_vmx_failInvalid(vcpu); 5472 5473 if (instr_info & BIT(10)) 5474 value = kvm_register_read(vcpu, (((instr_info) >> 3) & 0xf)); 5475 else { 5476 len = is_64_bit_mode(vcpu) ? 8 : 4; 5477 if (get_vmx_mem_address(vcpu, exit_qualification, 5478 instr_info, false, len, &gva)) 5479 return 1; 5480 r = kvm_read_guest_virt(vcpu, gva, &value, len, &e); 5481 if (r != X86EMUL_CONTINUE) 5482 return kvm_handle_memory_failure(vcpu, r, &e); 5483 } 5484 5485 field = kvm_register_read(vcpu, (((instr_info) >> 28) & 0xf)); 5486 5487 offset = get_vmcs12_field_offset(field); 5488 if (offset < 0) 5489 return nested_vmx_fail(vcpu, VMXERR_UNSUPPORTED_VMCS_COMPONENT); 5490 5491 /* 5492 * If the vCPU supports "VMWRITE to any supported field in the 5493 * VMCS," then the "read-only" fields are actually read/write. 5494 */ 5495 if (vmcs_field_readonly(field) && 5496 !nested_cpu_has_vmwrite_any_field(vcpu)) 5497 return nested_vmx_fail(vcpu, VMXERR_VMWRITE_READ_ONLY_VMCS_COMPONENT); 5498 5499 /* 5500 * Ensure vmcs12 is up-to-date before any VMWRITE that dirties 5501 * vmcs12, else we may crush a field or consume a stale value. 5502 */ 5503 if (!is_guest_mode(vcpu) && !is_shadow_field_rw(field)) 5504 copy_vmcs02_to_vmcs12_rare(vcpu, vmcs12); 5505 5506 /* 5507 * Some Intel CPUs intentionally drop the reserved bits of the AR byte 5508 * fields on VMWRITE. Emulate this behavior to ensure consistent KVM 5509 * behavior regardless of the underlying hardware, e.g. if an AR_BYTE 5510 * field is intercepted for VMWRITE but not VMREAD (in L1), then VMREAD 5511 * from L1 will return a different value than VMREAD from L2 (L1 sees 5512 * the stripped down value, L2 sees the full value as stored by KVM). 5513 */ 5514 if (field >= GUEST_ES_AR_BYTES && field <= GUEST_TR_AR_BYTES) 5515 value &= 0x1f0ff; 5516 5517 vmcs12_write_any(vmcs12, field, offset, value); 5518 5519 /* 5520 * Do not track vmcs12 dirty-state if in guest-mode as we actually 5521 * dirty shadow vmcs12 instead of vmcs12. Fields that can be updated 5522 * by L1 without a vmexit are always updated in the vmcs02, i.e. don't 5523 * "dirty" vmcs12, all others go down the prepare_vmcs02() slow path. 5524 */ 5525 if (!is_guest_mode(vcpu) && !is_shadow_field_rw(field)) { 5526 /* 5527 * L1 can read these fields without exiting, ensure the 5528 * shadow VMCS is up-to-date. 5529 */ 5530 if (enable_shadow_vmcs && is_shadow_field_ro(field)) { 5531 preempt_disable(); 5532 vmcs_load(vmx->vmcs01.shadow_vmcs); 5533 5534 __vmcs_writel(field, value); 5535 5536 vmcs_clear(vmx->vmcs01.shadow_vmcs); 5537 vmcs_load(vmx->loaded_vmcs->vmcs); 5538 preempt_enable(); 5539 } 5540 vmx->nested.dirty_vmcs12 = true; 5541 } 5542 5543 return nested_vmx_succeed(vcpu); 5544 } 5545 5546 static void set_current_vmptr(struct vcpu_vmx *vmx, gpa_t vmptr) 5547 { 5548 vmx->nested.current_vmptr = vmptr; 5549 if (enable_shadow_vmcs) { 5550 secondary_exec_controls_setbit(vmx, SECONDARY_EXEC_SHADOW_VMCS); 5551 vmcs_write64(VMCS_LINK_POINTER, 5552 __pa(vmx->vmcs01.shadow_vmcs)); 5553 vmx->nested.need_vmcs12_to_shadow_sync = true; 5554 } 5555 vmx->nested.dirty_vmcs12 = true; 5556 vmx->nested.force_msr_bitmap_recalc = true; 5557 } 5558 5559 /* Emulate the VMPTRLD instruction */ 5560 static int handle_vmptrld(struct kvm_vcpu *vcpu) 5561 { 5562 struct vcpu_vmx *vmx = to_vmx(vcpu); 5563 gpa_t vmptr; 5564 int r; 5565 5566 if (!nested_vmx_check_permission(vcpu)) 5567 return 1; 5568 5569 if (nested_vmx_get_vmptr(vcpu, &vmptr, &r)) 5570 return r; 5571 5572 if (!page_address_valid(vcpu, vmptr)) 5573 return nested_vmx_fail(vcpu, VMXERR_VMPTRLD_INVALID_ADDRESS); 5574 5575 if (vmptr == vmx->nested.vmxon_ptr) 5576 return nested_vmx_fail(vcpu, VMXERR_VMPTRLD_VMXON_POINTER); 5577 5578 /* Forbid normal VMPTRLD if Enlightened version was used */ 5579 if (evmptr_is_valid(vmx->nested.hv_evmcs_vmptr)) 5580 return 1; 5581 5582 if (vmx->nested.current_vmptr != vmptr) { 5583 struct gfn_to_hva_cache *ghc = &vmx->nested.vmcs12_cache; 5584 struct vmcs_hdr hdr; 5585 5586 if (kvm_gfn_to_hva_cache_init(vcpu->kvm, ghc, vmptr, VMCS12_SIZE)) { 5587 /* 5588 * Reads from an unbacked page return all 1s, 5589 * which means that the 32 bits located at the 5590 * given physical address won't match the required 5591 * VMCS12_REVISION identifier. 5592 */ 5593 return nested_vmx_fail(vcpu, 5594 VMXERR_VMPTRLD_INCORRECT_VMCS_REVISION_ID); 5595 } 5596 5597 if (kvm_read_guest_offset_cached(vcpu->kvm, ghc, &hdr, 5598 offsetof(struct vmcs12, hdr), 5599 sizeof(hdr))) { 5600 return nested_vmx_fail(vcpu, 5601 VMXERR_VMPTRLD_INCORRECT_VMCS_REVISION_ID); 5602 } 5603 5604 if (hdr.revision_id != VMCS12_REVISION || 5605 (hdr.shadow_vmcs && 5606 !nested_cpu_has_vmx_shadow_vmcs(vcpu))) { 5607 return nested_vmx_fail(vcpu, 5608 VMXERR_VMPTRLD_INCORRECT_VMCS_REVISION_ID); 5609 } 5610 5611 nested_release_vmcs12(vcpu); 5612 5613 /* 5614 * Load VMCS12 from guest memory since it is not already 5615 * cached. 5616 */ 5617 if (kvm_read_guest_cached(vcpu->kvm, ghc, vmx->nested.cached_vmcs12, 5618 VMCS12_SIZE)) { 5619 return nested_vmx_fail(vcpu, 5620 VMXERR_VMPTRLD_INCORRECT_VMCS_REVISION_ID); 5621 } 5622 5623 set_current_vmptr(vmx, vmptr); 5624 } 5625 5626 return nested_vmx_succeed(vcpu); 5627 } 5628 5629 /* Emulate the VMPTRST instruction */ 5630 static int handle_vmptrst(struct kvm_vcpu *vcpu) 5631 { 5632 unsigned long exit_qual = vmx_get_exit_qual(vcpu); 5633 u32 instr_info = vmcs_read32(VMX_INSTRUCTION_INFO); 5634 gpa_t current_vmptr = to_vmx(vcpu)->nested.current_vmptr; 5635 struct x86_exception e; 5636 gva_t gva; 5637 int r; 5638 5639 if (!nested_vmx_check_permission(vcpu)) 5640 return 1; 5641 5642 if (unlikely(evmptr_is_valid(to_vmx(vcpu)->nested.hv_evmcs_vmptr))) 5643 return 1; 5644 5645 if (get_vmx_mem_address(vcpu, exit_qual, instr_info, 5646 true, sizeof(gpa_t), &gva)) 5647 return 1; 5648 /* *_system ok, nested_vmx_check_permission has verified cpl=0 */ 5649 r = kvm_write_guest_virt_system(vcpu, gva, (void *)¤t_vmptr, 5650 sizeof(gpa_t), &e); 5651 if (r != X86EMUL_CONTINUE) 5652 return kvm_handle_memory_failure(vcpu, r, &e); 5653 5654 return nested_vmx_succeed(vcpu); 5655 } 5656 5657 /* Emulate the INVEPT instruction */ 5658 static int handle_invept(struct kvm_vcpu *vcpu) 5659 { 5660 struct vcpu_vmx *vmx = to_vmx(vcpu); 5661 u32 vmx_instruction_info, types; 5662 unsigned long type, roots_to_free; 5663 struct kvm_mmu *mmu; 5664 gva_t gva; 5665 struct x86_exception e; 5666 struct { 5667 u64 eptp, gpa; 5668 } operand; 5669 int i, r, gpr_index; 5670 5671 if (!(vmx->nested.msrs.secondary_ctls_high & 5672 SECONDARY_EXEC_ENABLE_EPT) || 5673 !(vmx->nested.msrs.ept_caps & VMX_EPT_INVEPT_BIT)) { 5674 kvm_queue_exception(vcpu, UD_VECTOR); 5675 return 1; 5676 } 5677 5678 if (!nested_vmx_check_permission(vcpu)) 5679 return 1; 5680 5681 vmx_instruction_info = vmcs_read32(VMX_INSTRUCTION_INFO); 5682 gpr_index = vmx_get_instr_info_reg2(vmx_instruction_info); 5683 type = kvm_register_read(vcpu, gpr_index); 5684 5685 types = (vmx->nested.msrs.ept_caps >> VMX_EPT_EXTENT_SHIFT) & 6; 5686 5687 if (type >= 32 || !(types & (1 << type))) 5688 return nested_vmx_fail(vcpu, VMXERR_INVALID_OPERAND_TO_INVEPT_INVVPID); 5689 5690 /* According to the Intel VMX instruction reference, the memory 5691 * operand is read even if it isn't needed (e.g., for type==global) 5692 */ 5693 if (get_vmx_mem_address(vcpu, vmx_get_exit_qual(vcpu), 5694 vmx_instruction_info, false, sizeof(operand), &gva)) 5695 return 1; 5696 r = kvm_read_guest_virt(vcpu, gva, &operand, sizeof(operand), &e); 5697 if (r != X86EMUL_CONTINUE) 5698 return kvm_handle_memory_failure(vcpu, r, &e); 5699 5700 /* 5701 * Nested EPT roots are always held through guest_mmu, 5702 * not root_mmu. 5703 */ 5704 mmu = &vcpu->arch.guest_mmu; 5705 5706 switch (type) { 5707 case VMX_EPT_EXTENT_CONTEXT: 5708 if (!nested_vmx_check_eptp(vcpu, operand.eptp)) 5709 return nested_vmx_fail(vcpu, 5710 VMXERR_INVALID_OPERAND_TO_INVEPT_INVVPID); 5711 5712 roots_to_free = 0; 5713 if (nested_ept_root_matches(mmu->root.hpa, mmu->root.pgd, 5714 operand.eptp)) 5715 roots_to_free |= KVM_MMU_ROOT_CURRENT; 5716 5717 for (i = 0; i < KVM_MMU_NUM_PREV_ROOTS; i++) { 5718 if (nested_ept_root_matches(mmu->prev_roots[i].hpa, 5719 mmu->prev_roots[i].pgd, 5720 operand.eptp)) 5721 roots_to_free |= KVM_MMU_ROOT_PREVIOUS(i); 5722 } 5723 break; 5724 case VMX_EPT_EXTENT_GLOBAL: 5725 roots_to_free = KVM_MMU_ROOTS_ALL; 5726 break; 5727 default: 5728 BUG(); 5729 break; 5730 } 5731 5732 if (roots_to_free) 5733 kvm_mmu_free_roots(vcpu->kvm, mmu, roots_to_free); 5734 5735 return nested_vmx_succeed(vcpu); 5736 } 5737 5738 static int handle_invvpid(struct kvm_vcpu *vcpu) 5739 { 5740 struct vcpu_vmx *vmx = to_vmx(vcpu); 5741 u32 vmx_instruction_info; 5742 unsigned long type, types; 5743 gva_t gva; 5744 struct x86_exception e; 5745 struct { 5746 u64 vpid; 5747 u64 gla; 5748 } operand; 5749 u16 vpid02; 5750 int r, gpr_index; 5751 5752 if (!(vmx->nested.msrs.secondary_ctls_high & 5753 SECONDARY_EXEC_ENABLE_VPID) || 5754 !(vmx->nested.msrs.vpid_caps & VMX_VPID_INVVPID_BIT)) { 5755 kvm_queue_exception(vcpu, UD_VECTOR); 5756 return 1; 5757 } 5758 5759 if (!nested_vmx_check_permission(vcpu)) 5760 return 1; 5761 5762 vmx_instruction_info = vmcs_read32(VMX_INSTRUCTION_INFO); 5763 gpr_index = vmx_get_instr_info_reg2(vmx_instruction_info); 5764 type = kvm_register_read(vcpu, gpr_index); 5765 5766 types = (vmx->nested.msrs.vpid_caps & 5767 VMX_VPID_EXTENT_SUPPORTED_MASK) >> 8; 5768 5769 if (type >= 32 || !(types & (1 << type))) 5770 return nested_vmx_fail(vcpu, 5771 VMXERR_INVALID_OPERAND_TO_INVEPT_INVVPID); 5772 5773 /* according to the intel vmx instruction reference, the memory 5774 * operand is read even if it isn't needed (e.g., for type==global) 5775 */ 5776 if (get_vmx_mem_address(vcpu, vmx_get_exit_qual(vcpu), 5777 vmx_instruction_info, false, sizeof(operand), &gva)) 5778 return 1; 5779 r = kvm_read_guest_virt(vcpu, gva, &operand, sizeof(operand), &e); 5780 if (r != X86EMUL_CONTINUE) 5781 return kvm_handle_memory_failure(vcpu, r, &e); 5782 5783 if (operand.vpid >> 16) 5784 return nested_vmx_fail(vcpu, 5785 VMXERR_INVALID_OPERAND_TO_INVEPT_INVVPID); 5786 5787 vpid02 = nested_get_vpid02(vcpu); 5788 switch (type) { 5789 case VMX_VPID_EXTENT_INDIVIDUAL_ADDR: 5790 if (!operand.vpid || 5791 is_noncanonical_address(operand.gla, vcpu)) 5792 return nested_vmx_fail(vcpu, 5793 VMXERR_INVALID_OPERAND_TO_INVEPT_INVVPID); 5794 vpid_sync_vcpu_addr(vpid02, operand.gla); 5795 break; 5796 case VMX_VPID_EXTENT_SINGLE_CONTEXT: 5797 case VMX_VPID_EXTENT_SINGLE_NON_GLOBAL: 5798 if (!operand.vpid) 5799 return nested_vmx_fail(vcpu, 5800 VMXERR_INVALID_OPERAND_TO_INVEPT_INVVPID); 5801 vpid_sync_context(vpid02); 5802 break; 5803 case VMX_VPID_EXTENT_ALL_CONTEXT: 5804 vpid_sync_context(vpid02); 5805 break; 5806 default: 5807 WARN_ON_ONCE(1); 5808 return kvm_skip_emulated_instruction(vcpu); 5809 } 5810 5811 /* 5812 * Sync the shadow page tables if EPT is disabled, L1 is invalidating 5813 * linear mappings for L2 (tagged with L2's VPID). Free all guest 5814 * roots as VPIDs are not tracked in the MMU role. 5815 * 5816 * Note, this operates on root_mmu, not guest_mmu, as L1 and L2 share 5817 * an MMU when EPT is disabled. 5818 * 5819 * TODO: sync only the affected SPTEs for INVDIVIDUAL_ADDR. 5820 */ 5821 if (!enable_ept) 5822 kvm_mmu_free_guest_mode_roots(vcpu->kvm, &vcpu->arch.root_mmu); 5823 5824 return nested_vmx_succeed(vcpu); 5825 } 5826 5827 static int nested_vmx_eptp_switching(struct kvm_vcpu *vcpu, 5828 struct vmcs12 *vmcs12) 5829 { 5830 u32 index = kvm_rcx_read(vcpu); 5831 u64 new_eptp; 5832 5833 if (WARN_ON_ONCE(!nested_cpu_has_ept(vmcs12))) 5834 return 1; 5835 if (index >= VMFUNC_EPTP_ENTRIES) 5836 return 1; 5837 5838 if (kvm_vcpu_read_guest_page(vcpu, vmcs12->eptp_list_address >> PAGE_SHIFT, 5839 &new_eptp, index * 8, 8)) 5840 return 1; 5841 5842 /* 5843 * If the (L2) guest does a vmfunc to the currently 5844 * active ept pointer, we don't have to do anything else 5845 */ 5846 if (vmcs12->ept_pointer != new_eptp) { 5847 if (!nested_vmx_check_eptp(vcpu, new_eptp)) 5848 return 1; 5849 5850 vmcs12->ept_pointer = new_eptp; 5851 nested_ept_new_eptp(vcpu); 5852 5853 if (!nested_cpu_has_vpid(vmcs12)) 5854 kvm_make_request(KVM_REQ_TLB_FLUSH_GUEST, vcpu); 5855 } 5856 5857 return 0; 5858 } 5859 5860 static int handle_vmfunc(struct kvm_vcpu *vcpu) 5861 { 5862 struct vcpu_vmx *vmx = to_vmx(vcpu); 5863 struct vmcs12 *vmcs12; 5864 u32 function = kvm_rax_read(vcpu); 5865 5866 /* 5867 * VMFUNC should never execute cleanly while L1 is active; KVM supports 5868 * VMFUNC for nested VMs, but not for L1. 5869 */ 5870 if (WARN_ON_ONCE(!is_guest_mode(vcpu))) { 5871 kvm_queue_exception(vcpu, UD_VECTOR); 5872 return 1; 5873 } 5874 5875 vmcs12 = get_vmcs12(vcpu); 5876 5877 /* 5878 * #UD on out-of-bounds function has priority over VM-Exit, and VMFUNC 5879 * is enabled in vmcs02 if and only if it's enabled in vmcs12. 5880 */ 5881 if (WARN_ON_ONCE((function > 63) || !nested_cpu_has_vmfunc(vmcs12))) { 5882 kvm_queue_exception(vcpu, UD_VECTOR); 5883 return 1; 5884 } 5885 5886 if (!(vmcs12->vm_function_control & BIT_ULL(function))) 5887 goto fail; 5888 5889 switch (function) { 5890 case 0: 5891 if (nested_vmx_eptp_switching(vcpu, vmcs12)) 5892 goto fail; 5893 break; 5894 default: 5895 goto fail; 5896 } 5897 return kvm_skip_emulated_instruction(vcpu); 5898 5899 fail: 5900 /* 5901 * This is effectively a reflected VM-Exit, as opposed to a synthesized 5902 * nested VM-Exit. Pass the original exit reason, i.e. don't hardcode 5903 * EXIT_REASON_VMFUNC as the exit reason. 5904 */ 5905 nested_vmx_vmexit(vcpu, vmx->exit_reason.full, 5906 vmx_get_intr_info(vcpu), 5907 vmx_get_exit_qual(vcpu)); 5908 return 1; 5909 } 5910 5911 /* 5912 * Return true if an IO instruction with the specified port and size should cause 5913 * a VM-exit into L1. 5914 */ 5915 bool nested_vmx_check_io_bitmaps(struct kvm_vcpu *vcpu, unsigned int port, 5916 int size) 5917 { 5918 struct vmcs12 *vmcs12 = get_vmcs12(vcpu); 5919 gpa_t bitmap, last_bitmap; 5920 u8 b; 5921 5922 last_bitmap = INVALID_GPA; 5923 b = -1; 5924 5925 while (size > 0) { 5926 if (port < 0x8000) 5927 bitmap = vmcs12->io_bitmap_a; 5928 else if (port < 0x10000) 5929 bitmap = vmcs12->io_bitmap_b; 5930 else 5931 return true; 5932 bitmap += (port & 0x7fff) / 8; 5933 5934 if (last_bitmap != bitmap) 5935 if (kvm_vcpu_read_guest(vcpu, bitmap, &b, 1)) 5936 return true; 5937 if (b & (1 << (port & 7))) 5938 return true; 5939 5940 port++; 5941 size--; 5942 last_bitmap = bitmap; 5943 } 5944 5945 return false; 5946 } 5947 5948 static bool nested_vmx_exit_handled_io(struct kvm_vcpu *vcpu, 5949 struct vmcs12 *vmcs12) 5950 { 5951 unsigned long exit_qualification; 5952 unsigned short port; 5953 int size; 5954 5955 if (!nested_cpu_has(vmcs12, CPU_BASED_USE_IO_BITMAPS)) 5956 return nested_cpu_has(vmcs12, CPU_BASED_UNCOND_IO_EXITING); 5957 5958 exit_qualification = vmx_get_exit_qual(vcpu); 5959 5960 port = exit_qualification >> 16; 5961 size = (exit_qualification & 7) + 1; 5962 5963 return nested_vmx_check_io_bitmaps(vcpu, port, size); 5964 } 5965 5966 /* 5967 * Return 1 if we should exit from L2 to L1 to handle an MSR access, 5968 * rather than handle it ourselves in L0. I.e., check whether L1 expressed 5969 * disinterest in the current event (read or write a specific MSR) by using an 5970 * MSR bitmap. This may be the case even when L0 doesn't use MSR bitmaps. 5971 */ 5972 static bool nested_vmx_exit_handled_msr(struct kvm_vcpu *vcpu, 5973 struct vmcs12 *vmcs12, 5974 union vmx_exit_reason exit_reason) 5975 { 5976 u32 msr_index = kvm_rcx_read(vcpu); 5977 gpa_t bitmap; 5978 5979 if (!nested_cpu_has(vmcs12, CPU_BASED_USE_MSR_BITMAPS)) 5980 return true; 5981 5982 /* 5983 * The MSR_BITMAP page is divided into four 1024-byte bitmaps, 5984 * for the four combinations of read/write and low/high MSR numbers. 5985 * First we need to figure out which of the four to use: 5986 */ 5987 bitmap = vmcs12->msr_bitmap; 5988 if (exit_reason.basic == EXIT_REASON_MSR_WRITE) 5989 bitmap += 2048; 5990 if (msr_index >= 0xc0000000) { 5991 msr_index -= 0xc0000000; 5992 bitmap += 1024; 5993 } 5994 5995 /* Then read the msr_index'th bit from this bitmap: */ 5996 if (msr_index < 1024*8) { 5997 unsigned char b; 5998 if (kvm_vcpu_read_guest(vcpu, bitmap + msr_index/8, &b, 1)) 5999 return true; 6000 return 1 & (b >> (msr_index & 7)); 6001 } else 6002 return true; /* let L1 handle the wrong parameter */ 6003 } 6004 6005 /* 6006 * Return 1 if we should exit from L2 to L1 to handle a CR access exit, 6007 * rather than handle it ourselves in L0. I.e., check if L1 wanted to 6008 * intercept (via guest_host_mask etc.) the current event. 6009 */ 6010 static bool nested_vmx_exit_handled_cr(struct kvm_vcpu *vcpu, 6011 struct vmcs12 *vmcs12) 6012 { 6013 unsigned long exit_qualification = vmx_get_exit_qual(vcpu); 6014 int cr = exit_qualification & 15; 6015 int reg; 6016 unsigned long val; 6017 6018 switch ((exit_qualification >> 4) & 3) { 6019 case 0: /* mov to cr */ 6020 reg = (exit_qualification >> 8) & 15; 6021 val = kvm_register_read(vcpu, reg); 6022 switch (cr) { 6023 case 0: 6024 if (vmcs12->cr0_guest_host_mask & 6025 (val ^ vmcs12->cr0_read_shadow)) 6026 return true; 6027 break; 6028 case 3: 6029 if (nested_cpu_has(vmcs12, CPU_BASED_CR3_LOAD_EXITING)) 6030 return true; 6031 break; 6032 case 4: 6033 if (vmcs12->cr4_guest_host_mask & 6034 (vmcs12->cr4_read_shadow ^ val)) 6035 return true; 6036 break; 6037 case 8: 6038 if (nested_cpu_has(vmcs12, CPU_BASED_CR8_LOAD_EXITING)) 6039 return true; 6040 break; 6041 } 6042 break; 6043 case 2: /* clts */ 6044 if ((vmcs12->cr0_guest_host_mask & X86_CR0_TS) && 6045 (vmcs12->cr0_read_shadow & X86_CR0_TS)) 6046 return true; 6047 break; 6048 case 1: /* mov from cr */ 6049 switch (cr) { 6050 case 3: 6051 if (vmcs12->cpu_based_vm_exec_control & 6052 CPU_BASED_CR3_STORE_EXITING) 6053 return true; 6054 break; 6055 case 8: 6056 if (vmcs12->cpu_based_vm_exec_control & 6057 CPU_BASED_CR8_STORE_EXITING) 6058 return true; 6059 break; 6060 } 6061 break; 6062 case 3: /* lmsw */ 6063 /* 6064 * lmsw can change bits 1..3 of cr0, and only set bit 0 of 6065 * cr0. Other attempted changes are ignored, with no exit. 6066 */ 6067 val = (exit_qualification >> LMSW_SOURCE_DATA_SHIFT) & 0x0f; 6068 if (vmcs12->cr0_guest_host_mask & 0xe & 6069 (val ^ vmcs12->cr0_read_shadow)) 6070 return true; 6071 if ((vmcs12->cr0_guest_host_mask & 0x1) && 6072 !(vmcs12->cr0_read_shadow & 0x1) && 6073 (val & 0x1)) 6074 return true; 6075 break; 6076 } 6077 return false; 6078 } 6079 6080 static bool nested_vmx_exit_handled_encls(struct kvm_vcpu *vcpu, 6081 struct vmcs12 *vmcs12) 6082 { 6083 u32 encls_leaf; 6084 6085 if (!guest_cpuid_has(vcpu, X86_FEATURE_SGX) || 6086 !nested_cpu_has2(vmcs12, SECONDARY_EXEC_ENCLS_EXITING)) 6087 return false; 6088 6089 encls_leaf = kvm_rax_read(vcpu); 6090 if (encls_leaf > 62) 6091 encls_leaf = 63; 6092 return vmcs12->encls_exiting_bitmap & BIT_ULL(encls_leaf); 6093 } 6094 6095 static bool nested_vmx_exit_handled_vmcs_access(struct kvm_vcpu *vcpu, 6096 struct vmcs12 *vmcs12, gpa_t bitmap) 6097 { 6098 u32 vmx_instruction_info; 6099 unsigned long field; 6100 u8 b; 6101 6102 if (!nested_cpu_has_shadow_vmcs(vmcs12)) 6103 return true; 6104 6105 /* Decode instruction info and find the field to access */ 6106 vmx_instruction_info = vmcs_read32(VMX_INSTRUCTION_INFO); 6107 field = kvm_register_read(vcpu, (((vmx_instruction_info) >> 28) & 0xf)); 6108 6109 /* Out-of-range fields always cause a VM exit from L2 to L1 */ 6110 if (field >> 15) 6111 return true; 6112 6113 if (kvm_vcpu_read_guest(vcpu, bitmap + field/8, &b, 1)) 6114 return true; 6115 6116 return 1 & (b >> (field & 7)); 6117 } 6118 6119 static bool nested_vmx_exit_handled_mtf(struct vmcs12 *vmcs12) 6120 { 6121 u32 entry_intr_info = vmcs12->vm_entry_intr_info_field; 6122 6123 if (nested_cpu_has_mtf(vmcs12)) 6124 return true; 6125 6126 /* 6127 * An MTF VM-exit may be injected into the guest by setting the 6128 * interruption-type to 7 (other event) and the vector field to 0. Such 6129 * is the case regardless of the 'monitor trap flag' VM-execution 6130 * control. 6131 */ 6132 return entry_intr_info == (INTR_INFO_VALID_MASK 6133 | INTR_TYPE_OTHER_EVENT); 6134 } 6135 6136 /* 6137 * Return true if L0 wants to handle an exit from L2 regardless of whether or not 6138 * L1 wants the exit. Only call this when in is_guest_mode (L2). 6139 */ 6140 static bool nested_vmx_l0_wants_exit(struct kvm_vcpu *vcpu, 6141 union vmx_exit_reason exit_reason) 6142 { 6143 u32 intr_info; 6144 6145 switch ((u16)exit_reason.basic) { 6146 case EXIT_REASON_EXCEPTION_NMI: 6147 intr_info = vmx_get_intr_info(vcpu); 6148 if (is_nmi(intr_info)) 6149 return true; 6150 else if (is_page_fault(intr_info)) 6151 return vcpu->arch.apf.host_apf_flags || 6152 vmx_need_pf_intercept(vcpu); 6153 else if (is_debug(intr_info) && 6154 vcpu->guest_debug & 6155 (KVM_GUESTDBG_SINGLESTEP | KVM_GUESTDBG_USE_HW_BP)) 6156 return true; 6157 else if (is_breakpoint(intr_info) && 6158 vcpu->guest_debug & KVM_GUESTDBG_USE_SW_BP) 6159 return true; 6160 else if (is_alignment_check(intr_info) && 6161 !vmx_guest_inject_ac(vcpu)) 6162 return true; 6163 return false; 6164 case EXIT_REASON_EXTERNAL_INTERRUPT: 6165 return true; 6166 case EXIT_REASON_MCE_DURING_VMENTRY: 6167 return true; 6168 case EXIT_REASON_EPT_VIOLATION: 6169 /* 6170 * L0 always deals with the EPT violation. If nested EPT is 6171 * used, and the nested mmu code discovers that the address is 6172 * missing in the guest EPT table (EPT12), the EPT violation 6173 * will be injected with nested_ept_inject_page_fault() 6174 */ 6175 return true; 6176 case EXIT_REASON_EPT_MISCONFIG: 6177 /* 6178 * L2 never uses directly L1's EPT, but rather L0's own EPT 6179 * table (shadow on EPT) or a merged EPT table that L0 built 6180 * (EPT on EPT). So any problems with the structure of the 6181 * table is L0's fault. 6182 */ 6183 return true; 6184 case EXIT_REASON_PREEMPTION_TIMER: 6185 return true; 6186 case EXIT_REASON_PML_FULL: 6187 /* 6188 * PML is emulated for an L1 VMM and should never be enabled in 6189 * vmcs02, always "handle" PML_FULL by exiting to userspace. 6190 */ 6191 return true; 6192 case EXIT_REASON_VMFUNC: 6193 /* VM functions are emulated through L2->L0 vmexits. */ 6194 return true; 6195 case EXIT_REASON_BUS_LOCK: 6196 /* 6197 * At present, bus lock VM exit is never exposed to L1. 6198 * Handle L2's bus locks in L0 directly. 6199 */ 6200 return true; 6201 case EXIT_REASON_VMCALL: 6202 /* Hyper-V L2 TLB flush hypercall is handled by L0 */ 6203 return guest_hv_cpuid_has_l2_tlb_flush(vcpu) && 6204 nested_evmcs_l2_tlb_flush_enabled(vcpu) && 6205 kvm_hv_is_tlb_flush_hcall(vcpu); 6206 default: 6207 break; 6208 } 6209 return false; 6210 } 6211 6212 /* 6213 * Return 1 if L1 wants to intercept an exit from L2. Only call this when in 6214 * is_guest_mode (L2). 6215 */ 6216 static bool nested_vmx_l1_wants_exit(struct kvm_vcpu *vcpu, 6217 union vmx_exit_reason exit_reason) 6218 { 6219 struct vmcs12 *vmcs12 = get_vmcs12(vcpu); 6220 u32 intr_info; 6221 6222 switch ((u16)exit_reason.basic) { 6223 case EXIT_REASON_EXCEPTION_NMI: 6224 intr_info = vmx_get_intr_info(vcpu); 6225 if (is_nmi(intr_info)) 6226 return true; 6227 else if (is_page_fault(intr_info)) 6228 return true; 6229 return vmcs12->exception_bitmap & 6230 (1u << (intr_info & INTR_INFO_VECTOR_MASK)); 6231 case EXIT_REASON_EXTERNAL_INTERRUPT: 6232 return nested_exit_on_intr(vcpu); 6233 case EXIT_REASON_TRIPLE_FAULT: 6234 return true; 6235 case EXIT_REASON_INTERRUPT_WINDOW: 6236 return nested_cpu_has(vmcs12, CPU_BASED_INTR_WINDOW_EXITING); 6237 case EXIT_REASON_NMI_WINDOW: 6238 return nested_cpu_has(vmcs12, CPU_BASED_NMI_WINDOW_EXITING); 6239 case EXIT_REASON_TASK_SWITCH: 6240 return true; 6241 case EXIT_REASON_CPUID: 6242 return true; 6243 case EXIT_REASON_HLT: 6244 return nested_cpu_has(vmcs12, CPU_BASED_HLT_EXITING); 6245 case EXIT_REASON_INVD: 6246 return true; 6247 case EXIT_REASON_INVLPG: 6248 return nested_cpu_has(vmcs12, CPU_BASED_INVLPG_EXITING); 6249 case EXIT_REASON_RDPMC: 6250 return nested_cpu_has(vmcs12, CPU_BASED_RDPMC_EXITING); 6251 case EXIT_REASON_RDRAND: 6252 return nested_cpu_has2(vmcs12, SECONDARY_EXEC_RDRAND_EXITING); 6253 case EXIT_REASON_RDSEED: 6254 return nested_cpu_has2(vmcs12, SECONDARY_EXEC_RDSEED_EXITING); 6255 case EXIT_REASON_RDTSC: case EXIT_REASON_RDTSCP: 6256 return nested_cpu_has(vmcs12, CPU_BASED_RDTSC_EXITING); 6257 case EXIT_REASON_VMREAD: 6258 return nested_vmx_exit_handled_vmcs_access(vcpu, vmcs12, 6259 vmcs12->vmread_bitmap); 6260 case EXIT_REASON_VMWRITE: 6261 return nested_vmx_exit_handled_vmcs_access(vcpu, vmcs12, 6262 vmcs12->vmwrite_bitmap); 6263 case EXIT_REASON_VMCALL: case EXIT_REASON_VMCLEAR: 6264 case EXIT_REASON_VMLAUNCH: case EXIT_REASON_VMPTRLD: 6265 case EXIT_REASON_VMPTRST: case EXIT_REASON_VMRESUME: 6266 case EXIT_REASON_VMOFF: case EXIT_REASON_VMON: 6267 case EXIT_REASON_INVEPT: case EXIT_REASON_INVVPID: 6268 /* 6269 * VMX instructions trap unconditionally. This allows L1 to 6270 * emulate them for its L2 guest, i.e., allows 3-level nesting! 6271 */ 6272 return true; 6273 case EXIT_REASON_CR_ACCESS: 6274 return nested_vmx_exit_handled_cr(vcpu, vmcs12); 6275 case EXIT_REASON_DR_ACCESS: 6276 return nested_cpu_has(vmcs12, CPU_BASED_MOV_DR_EXITING); 6277 case EXIT_REASON_IO_INSTRUCTION: 6278 return nested_vmx_exit_handled_io(vcpu, vmcs12); 6279 case EXIT_REASON_GDTR_IDTR: case EXIT_REASON_LDTR_TR: 6280 return nested_cpu_has2(vmcs12, SECONDARY_EXEC_DESC); 6281 case EXIT_REASON_MSR_READ: 6282 case EXIT_REASON_MSR_WRITE: 6283 return nested_vmx_exit_handled_msr(vcpu, vmcs12, exit_reason); 6284 case EXIT_REASON_INVALID_STATE: 6285 return true; 6286 case EXIT_REASON_MWAIT_INSTRUCTION: 6287 return nested_cpu_has(vmcs12, CPU_BASED_MWAIT_EXITING); 6288 case EXIT_REASON_MONITOR_TRAP_FLAG: 6289 return nested_vmx_exit_handled_mtf(vmcs12); 6290 case EXIT_REASON_MONITOR_INSTRUCTION: 6291 return nested_cpu_has(vmcs12, CPU_BASED_MONITOR_EXITING); 6292 case EXIT_REASON_PAUSE_INSTRUCTION: 6293 return nested_cpu_has(vmcs12, CPU_BASED_PAUSE_EXITING) || 6294 nested_cpu_has2(vmcs12, 6295 SECONDARY_EXEC_PAUSE_LOOP_EXITING); 6296 case EXIT_REASON_MCE_DURING_VMENTRY: 6297 return true; 6298 case EXIT_REASON_TPR_BELOW_THRESHOLD: 6299 return nested_cpu_has(vmcs12, CPU_BASED_TPR_SHADOW); 6300 case EXIT_REASON_APIC_ACCESS: 6301 case EXIT_REASON_APIC_WRITE: 6302 case EXIT_REASON_EOI_INDUCED: 6303 /* 6304 * The controls for "virtualize APIC accesses," "APIC- 6305 * register virtualization," and "virtual-interrupt 6306 * delivery" only come from vmcs12. 6307 */ 6308 return true; 6309 case EXIT_REASON_INVPCID: 6310 return 6311 nested_cpu_has2(vmcs12, SECONDARY_EXEC_ENABLE_INVPCID) && 6312 nested_cpu_has(vmcs12, CPU_BASED_INVLPG_EXITING); 6313 case EXIT_REASON_WBINVD: 6314 return nested_cpu_has2(vmcs12, SECONDARY_EXEC_WBINVD_EXITING); 6315 case EXIT_REASON_XSETBV: 6316 return true; 6317 case EXIT_REASON_XSAVES: case EXIT_REASON_XRSTORS: 6318 /* 6319 * This should never happen, since it is not possible to 6320 * set XSS to a non-zero value---neither in L1 nor in L2. 6321 * If if it were, XSS would have to be checked against 6322 * the XSS exit bitmap in vmcs12. 6323 */ 6324 return nested_cpu_has2(vmcs12, SECONDARY_EXEC_XSAVES); 6325 case EXIT_REASON_UMWAIT: 6326 case EXIT_REASON_TPAUSE: 6327 return nested_cpu_has2(vmcs12, 6328 SECONDARY_EXEC_ENABLE_USR_WAIT_PAUSE); 6329 case EXIT_REASON_ENCLS: 6330 return nested_vmx_exit_handled_encls(vcpu, vmcs12); 6331 case EXIT_REASON_NOTIFY: 6332 /* Notify VM exit is not exposed to L1 */ 6333 return false; 6334 default: 6335 return true; 6336 } 6337 } 6338 6339 /* 6340 * Conditionally reflect a VM-Exit into L1. Returns %true if the VM-Exit was 6341 * reflected into L1. 6342 */ 6343 bool nested_vmx_reflect_vmexit(struct kvm_vcpu *vcpu) 6344 { 6345 struct vcpu_vmx *vmx = to_vmx(vcpu); 6346 union vmx_exit_reason exit_reason = vmx->exit_reason; 6347 unsigned long exit_qual; 6348 u32 exit_intr_info; 6349 6350 WARN_ON_ONCE(vmx->nested.nested_run_pending); 6351 6352 /* 6353 * Late nested VM-Fail shares the same flow as nested VM-Exit since KVM 6354 * has already loaded L2's state. 6355 */ 6356 if (unlikely(vmx->fail)) { 6357 trace_kvm_nested_vmenter_failed( 6358 "hardware VM-instruction error: ", 6359 vmcs_read32(VM_INSTRUCTION_ERROR)); 6360 exit_intr_info = 0; 6361 exit_qual = 0; 6362 goto reflect_vmexit; 6363 } 6364 6365 trace_kvm_nested_vmexit(vcpu, KVM_ISA_VMX); 6366 6367 /* If L0 (KVM) wants the exit, it trumps L1's desires. */ 6368 if (nested_vmx_l0_wants_exit(vcpu, exit_reason)) 6369 return false; 6370 6371 /* If L1 doesn't want the exit, handle it in L0. */ 6372 if (!nested_vmx_l1_wants_exit(vcpu, exit_reason)) 6373 return false; 6374 6375 /* 6376 * vmcs.VM_EXIT_INTR_INFO is only valid for EXCEPTION_NMI exits. For 6377 * EXTERNAL_INTERRUPT, the value for vmcs12->vm_exit_intr_info would 6378 * need to be synthesized by querying the in-kernel LAPIC, but external 6379 * interrupts are never reflected to L1 so it's a non-issue. 6380 */ 6381 exit_intr_info = vmx_get_intr_info(vcpu); 6382 if (is_exception_with_error_code(exit_intr_info)) { 6383 struct vmcs12 *vmcs12 = get_vmcs12(vcpu); 6384 6385 vmcs12->vm_exit_intr_error_code = 6386 vmcs_read32(VM_EXIT_INTR_ERROR_CODE); 6387 } 6388 exit_qual = vmx_get_exit_qual(vcpu); 6389 6390 reflect_vmexit: 6391 nested_vmx_vmexit(vcpu, exit_reason.full, exit_intr_info, exit_qual); 6392 return true; 6393 } 6394 6395 static int vmx_get_nested_state(struct kvm_vcpu *vcpu, 6396 struct kvm_nested_state __user *user_kvm_nested_state, 6397 u32 user_data_size) 6398 { 6399 struct vcpu_vmx *vmx; 6400 struct vmcs12 *vmcs12; 6401 struct kvm_nested_state kvm_state = { 6402 .flags = 0, 6403 .format = KVM_STATE_NESTED_FORMAT_VMX, 6404 .size = sizeof(kvm_state), 6405 .hdr.vmx.flags = 0, 6406 .hdr.vmx.vmxon_pa = INVALID_GPA, 6407 .hdr.vmx.vmcs12_pa = INVALID_GPA, 6408 .hdr.vmx.preemption_timer_deadline = 0, 6409 }; 6410 struct kvm_vmx_nested_state_data __user *user_vmx_nested_state = 6411 &user_kvm_nested_state->data.vmx[0]; 6412 6413 if (!vcpu) 6414 return kvm_state.size + sizeof(*user_vmx_nested_state); 6415 6416 vmx = to_vmx(vcpu); 6417 vmcs12 = get_vmcs12(vcpu); 6418 6419 if (nested_vmx_allowed(vcpu) && 6420 (vmx->nested.vmxon || vmx->nested.smm.vmxon)) { 6421 kvm_state.hdr.vmx.vmxon_pa = vmx->nested.vmxon_ptr; 6422 kvm_state.hdr.vmx.vmcs12_pa = vmx->nested.current_vmptr; 6423 6424 if (vmx_has_valid_vmcs12(vcpu)) { 6425 kvm_state.size += sizeof(user_vmx_nested_state->vmcs12); 6426 6427 /* 'hv_evmcs_vmptr' can also be EVMPTR_MAP_PENDING here */ 6428 if (vmx->nested.hv_evmcs_vmptr != EVMPTR_INVALID) 6429 kvm_state.flags |= KVM_STATE_NESTED_EVMCS; 6430 6431 if (is_guest_mode(vcpu) && 6432 nested_cpu_has_shadow_vmcs(vmcs12) && 6433 vmcs12->vmcs_link_pointer != INVALID_GPA) 6434 kvm_state.size += sizeof(user_vmx_nested_state->shadow_vmcs12); 6435 } 6436 6437 if (vmx->nested.smm.vmxon) 6438 kvm_state.hdr.vmx.smm.flags |= KVM_STATE_NESTED_SMM_VMXON; 6439 6440 if (vmx->nested.smm.guest_mode) 6441 kvm_state.hdr.vmx.smm.flags |= KVM_STATE_NESTED_SMM_GUEST_MODE; 6442 6443 if (is_guest_mode(vcpu)) { 6444 kvm_state.flags |= KVM_STATE_NESTED_GUEST_MODE; 6445 6446 if (vmx->nested.nested_run_pending) 6447 kvm_state.flags |= KVM_STATE_NESTED_RUN_PENDING; 6448 6449 if (vmx->nested.mtf_pending) 6450 kvm_state.flags |= KVM_STATE_NESTED_MTF_PENDING; 6451 6452 if (nested_cpu_has_preemption_timer(vmcs12) && 6453 vmx->nested.has_preemption_timer_deadline) { 6454 kvm_state.hdr.vmx.flags |= 6455 KVM_STATE_VMX_PREEMPTION_TIMER_DEADLINE; 6456 kvm_state.hdr.vmx.preemption_timer_deadline = 6457 vmx->nested.preemption_timer_deadline; 6458 } 6459 } 6460 } 6461 6462 if (user_data_size < kvm_state.size) 6463 goto out; 6464 6465 if (copy_to_user(user_kvm_nested_state, &kvm_state, sizeof(kvm_state))) 6466 return -EFAULT; 6467 6468 if (!vmx_has_valid_vmcs12(vcpu)) 6469 goto out; 6470 6471 /* 6472 * When running L2, the authoritative vmcs12 state is in the 6473 * vmcs02. When running L1, the authoritative vmcs12 state is 6474 * in the shadow or enlightened vmcs linked to vmcs01, unless 6475 * need_vmcs12_to_shadow_sync is set, in which case, the authoritative 6476 * vmcs12 state is in the vmcs12 already. 6477 */ 6478 if (is_guest_mode(vcpu)) { 6479 sync_vmcs02_to_vmcs12(vcpu, vmcs12); 6480 sync_vmcs02_to_vmcs12_rare(vcpu, vmcs12); 6481 } else { 6482 copy_vmcs02_to_vmcs12_rare(vcpu, get_vmcs12(vcpu)); 6483 if (!vmx->nested.need_vmcs12_to_shadow_sync) { 6484 if (evmptr_is_valid(vmx->nested.hv_evmcs_vmptr)) 6485 /* 6486 * L1 hypervisor is not obliged to keep eVMCS 6487 * clean fields data always up-to-date while 6488 * not in guest mode, 'hv_clean_fields' is only 6489 * supposed to be actual upon vmentry so we need 6490 * to ignore it here and do full copy. 6491 */ 6492 copy_enlightened_to_vmcs12(vmx, 0); 6493 else if (enable_shadow_vmcs) 6494 copy_shadow_to_vmcs12(vmx); 6495 } 6496 } 6497 6498 BUILD_BUG_ON(sizeof(user_vmx_nested_state->vmcs12) < VMCS12_SIZE); 6499 BUILD_BUG_ON(sizeof(user_vmx_nested_state->shadow_vmcs12) < VMCS12_SIZE); 6500 6501 /* 6502 * Copy over the full allocated size of vmcs12 rather than just the size 6503 * of the struct. 6504 */ 6505 if (copy_to_user(user_vmx_nested_state->vmcs12, vmcs12, VMCS12_SIZE)) 6506 return -EFAULT; 6507 6508 if (nested_cpu_has_shadow_vmcs(vmcs12) && 6509 vmcs12->vmcs_link_pointer != INVALID_GPA) { 6510 if (copy_to_user(user_vmx_nested_state->shadow_vmcs12, 6511 get_shadow_vmcs12(vcpu), VMCS12_SIZE)) 6512 return -EFAULT; 6513 } 6514 out: 6515 return kvm_state.size; 6516 } 6517 6518 void vmx_leave_nested(struct kvm_vcpu *vcpu) 6519 { 6520 if (is_guest_mode(vcpu)) { 6521 to_vmx(vcpu)->nested.nested_run_pending = 0; 6522 nested_vmx_vmexit(vcpu, -1, 0, 0); 6523 } 6524 free_nested(vcpu); 6525 } 6526 6527 static int vmx_set_nested_state(struct kvm_vcpu *vcpu, 6528 struct kvm_nested_state __user *user_kvm_nested_state, 6529 struct kvm_nested_state *kvm_state) 6530 { 6531 struct vcpu_vmx *vmx = to_vmx(vcpu); 6532 struct vmcs12 *vmcs12; 6533 enum vm_entry_failure_code ignored; 6534 struct kvm_vmx_nested_state_data __user *user_vmx_nested_state = 6535 &user_kvm_nested_state->data.vmx[0]; 6536 int ret; 6537 6538 if (kvm_state->format != KVM_STATE_NESTED_FORMAT_VMX) 6539 return -EINVAL; 6540 6541 if (kvm_state->hdr.vmx.vmxon_pa == INVALID_GPA) { 6542 if (kvm_state->hdr.vmx.smm.flags) 6543 return -EINVAL; 6544 6545 if (kvm_state->hdr.vmx.vmcs12_pa != INVALID_GPA) 6546 return -EINVAL; 6547 6548 /* 6549 * KVM_STATE_NESTED_EVMCS used to signal that KVM should 6550 * enable eVMCS capability on vCPU. However, since then 6551 * code was changed such that flag signals vmcs12 should 6552 * be copied into eVMCS in guest memory. 6553 * 6554 * To preserve backwards compatability, allow user 6555 * to set this flag even when there is no VMXON region. 6556 */ 6557 if (kvm_state->flags & ~KVM_STATE_NESTED_EVMCS) 6558 return -EINVAL; 6559 } else { 6560 if (!nested_vmx_allowed(vcpu)) 6561 return -EINVAL; 6562 6563 if (!page_address_valid(vcpu, kvm_state->hdr.vmx.vmxon_pa)) 6564 return -EINVAL; 6565 } 6566 6567 if ((kvm_state->hdr.vmx.smm.flags & KVM_STATE_NESTED_SMM_GUEST_MODE) && 6568 (kvm_state->flags & KVM_STATE_NESTED_GUEST_MODE)) 6569 return -EINVAL; 6570 6571 if (kvm_state->hdr.vmx.smm.flags & 6572 ~(KVM_STATE_NESTED_SMM_GUEST_MODE | KVM_STATE_NESTED_SMM_VMXON)) 6573 return -EINVAL; 6574 6575 if (kvm_state->hdr.vmx.flags & ~KVM_STATE_VMX_PREEMPTION_TIMER_DEADLINE) 6576 return -EINVAL; 6577 6578 /* 6579 * SMM temporarily disables VMX, so we cannot be in guest mode, 6580 * nor can VMLAUNCH/VMRESUME be pending. Outside SMM, SMM flags 6581 * must be zero. 6582 */ 6583 if (is_smm(vcpu) ? 6584 (kvm_state->flags & 6585 (KVM_STATE_NESTED_GUEST_MODE | KVM_STATE_NESTED_RUN_PENDING)) 6586 : kvm_state->hdr.vmx.smm.flags) 6587 return -EINVAL; 6588 6589 if ((kvm_state->hdr.vmx.smm.flags & KVM_STATE_NESTED_SMM_GUEST_MODE) && 6590 !(kvm_state->hdr.vmx.smm.flags & KVM_STATE_NESTED_SMM_VMXON)) 6591 return -EINVAL; 6592 6593 if ((kvm_state->flags & KVM_STATE_NESTED_EVMCS) && 6594 (!nested_vmx_allowed(vcpu) || !vmx->nested.enlightened_vmcs_enabled)) 6595 return -EINVAL; 6596 6597 vmx_leave_nested(vcpu); 6598 6599 if (kvm_state->hdr.vmx.vmxon_pa == INVALID_GPA) 6600 return 0; 6601 6602 vmx->nested.vmxon_ptr = kvm_state->hdr.vmx.vmxon_pa; 6603 ret = enter_vmx_operation(vcpu); 6604 if (ret) 6605 return ret; 6606 6607 /* Empty 'VMXON' state is permitted if no VMCS loaded */ 6608 if (kvm_state->size < sizeof(*kvm_state) + sizeof(*vmcs12)) { 6609 /* See vmx_has_valid_vmcs12. */ 6610 if ((kvm_state->flags & KVM_STATE_NESTED_GUEST_MODE) || 6611 (kvm_state->flags & KVM_STATE_NESTED_EVMCS) || 6612 (kvm_state->hdr.vmx.vmcs12_pa != INVALID_GPA)) 6613 return -EINVAL; 6614 else 6615 return 0; 6616 } 6617 6618 if (kvm_state->hdr.vmx.vmcs12_pa != INVALID_GPA) { 6619 if (kvm_state->hdr.vmx.vmcs12_pa == kvm_state->hdr.vmx.vmxon_pa || 6620 !page_address_valid(vcpu, kvm_state->hdr.vmx.vmcs12_pa)) 6621 return -EINVAL; 6622 6623 set_current_vmptr(vmx, kvm_state->hdr.vmx.vmcs12_pa); 6624 } else if (kvm_state->flags & KVM_STATE_NESTED_EVMCS) { 6625 /* 6626 * nested_vmx_handle_enlightened_vmptrld() cannot be called 6627 * directly from here as HV_X64_MSR_VP_ASSIST_PAGE may not be 6628 * restored yet. EVMCS will be mapped from 6629 * nested_get_vmcs12_pages(). 6630 */ 6631 vmx->nested.hv_evmcs_vmptr = EVMPTR_MAP_PENDING; 6632 kvm_make_request(KVM_REQ_GET_NESTED_STATE_PAGES, vcpu); 6633 } else { 6634 return -EINVAL; 6635 } 6636 6637 if (kvm_state->hdr.vmx.smm.flags & KVM_STATE_NESTED_SMM_VMXON) { 6638 vmx->nested.smm.vmxon = true; 6639 vmx->nested.vmxon = false; 6640 6641 if (kvm_state->hdr.vmx.smm.flags & KVM_STATE_NESTED_SMM_GUEST_MODE) 6642 vmx->nested.smm.guest_mode = true; 6643 } 6644 6645 vmcs12 = get_vmcs12(vcpu); 6646 if (copy_from_user(vmcs12, user_vmx_nested_state->vmcs12, sizeof(*vmcs12))) 6647 return -EFAULT; 6648 6649 if (vmcs12->hdr.revision_id != VMCS12_REVISION) 6650 return -EINVAL; 6651 6652 if (!(kvm_state->flags & KVM_STATE_NESTED_GUEST_MODE)) 6653 return 0; 6654 6655 vmx->nested.nested_run_pending = 6656 !!(kvm_state->flags & KVM_STATE_NESTED_RUN_PENDING); 6657 6658 vmx->nested.mtf_pending = 6659 !!(kvm_state->flags & KVM_STATE_NESTED_MTF_PENDING); 6660 6661 ret = -EINVAL; 6662 if (nested_cpu_has_shadow_vmcs(vmcs12) && 6663 vmcs12->vmcs_link_pointer != INVALID_GPA) { 6664 struct vmcs12 *shadow_vmcs12 = get_shadow_vmcs12(vcpu); 6665 6666 if (kvm_state->size < 6667 sizeof(*kvm_state) + 6668 sizeof(user_vmx_nested_state->vmcs12) + sizeof(*shadow_vmcs12)) 6669 goto error_guest_mode; 6670 6671 if (copy_from_user(shadow_vmcs12, 6672 user_vmx_nested_state->shadow_vmcs12, 6673 sizeof(*shadow_vmcs12))) { 6674 ret = -EFAULT; 6675 goto error_guest_mode; 6676 } 6677 6678 if (shadow_vmcs12->hdr.revision_id != VMCS12_REVISION || 6679 !shadow_vmcs12->hdr.shadow_vmcs) 6680 goto error_guest_mode; 6681 } 6682 6683 vmx->nested.has_preemption_timer_deadline = false; 6684 if (kvm_state->hdr.vmx.flags & KVM_STATE_VMX_PREEMPTION_TIMER_DEADLINE) { 6685 vmx->nested.has_preemption_timer_deadline = true; 6686 vmx->nested.preemption_timer_deadline = 6687 kvm_state->hdr.vmx.preemption_timer_deadline; 6688 } 6689 6690 if (nested_vmx_check_controls(vcpu, vmcs12) || 6691 nested_vmx_check_host_state(vcpu, vmcs12) || 6692 nested_vmx_check_guest_state(vcpu, vmcs12, &ignored)) 6693 goto error_guest_mode; 6694 6695 vmx->nested.dirty_vmcs12 = true; 6696 vmx->nested.force_msr_bitmap_recalc = true; 6697 ret = nested_vmx_enter_non_root_mode(vcpu, false); 6698 if (ret) 6699 goto error_guest_mode; 6700 6701 if (vmx->nested.mtf_pending) 6702 kvm_make_request(KVM_REQ_EVENT, vcpu); 6703 6704 return 0; 6705 6706 error_guest_mode: 6707 vmx->nested.nested_run_pending = 0; 6708 return ret; 6709 } 6710 6711 void nested_vmx_set_vmcs_shadowing_bitmap(void) 6712 { 6713 if (enable_shadow_vmcs) { 6714 vmcs_write64(VMREAD_BITMAP, __pa(vmx_vmread_bitmap)); 6715 vmcs_write64(VMWRITE_BITMAP, __pa(vmx_vmwrite_bitmap)); 6716 } 6717 } 6718 6719 /* 6720 * Indexing into the vmcs12 uses the VMCS encoding rotated left by 6. Undo 6721 * that madness to get the encoding for comparison. 6722 */ 6723 #define VMCS12_IDX_TO_ENC(idx) ((u16)(((u16)(idx) >> 6) | ((u16)(idx) << 10))) 6724 6725 static u64 nested_vmx_calc_vmcs_enum_msr(void) 6726 { 6727 /* 6728 * Note these are the so called "index" of the VMCS field encoding, not 6729 * the index into vmcs12. 6730 */ 6731 unsigned int max_idx, idx; 6732 int i; 6733 6734 /* 6735 * For better or worse, KVM allows VMREAD/VMWRITE to all fields in 6736 * vmcs12, regardless of whether or not the associated feature is 6737 * exposed to L1. Simply find the field with the highest index. 6738 */ 6739 max_idx = 0; 6740 for (i = 0; i < nr_vmcs12_fields; i++) { 6741 /* The vmcs12 table is very, very sparsely populated. */ 6742 if (!vmcs12_field_offsets[i]) 6743 continue; 6744 6745 idx = vmcs_field_index(VMCS12_IDX_TO_ENC(i)); 6746 if (idx > max_idx) 6747 max_idx = idx; 6748 } 6749 6750 return (u64)max_idx << VMCS_FIELD_INDEX_SHIFT; 6751 } 6752 6753 /* 6754 * nested_vmx_setup_ctls_msrs() sets up variables containing the values to be 6755 * returned for the various VMX controls MSRs when nested VMX is enabled. 6756 * The same values should also be used to verify that vmcs12 control fields are 6757 * valid during nested entry from L1 to L2. 6758 * Each of these control msrs has a low and high 32-bit half: A low bit is on 6759 * if the corresponding bit in the (32-bit) control field *must* be on, and a 6760 * bit in the high half is on if the corresponding bit in the control field 6761 * may be on. See also vmx_control_verify(). 6762 */ 6763 void nested_vmx_setup_ctls_msrs(struct vmcs_config *vmcs_conf, u32 ept_caps) 6764 { 6765 struct nested_vmx_msrs *msrs = &vmcs_conf->nested; 6766 6767 /* 6768 * Note that as a general rule, the high half of the MSRs (bits in 6769 * the control fields which may be 1) should be initialized by the 6770 * intersection of the underlying hardware's MSR (i.e., features which 6771 * can be supported) and the list of features we want to expose - 6772 * because they are known to be properly supported in our code. 6773 * Also, usually, the low half of the MSRs (bits which must be 1) can 6774 * be set to 0, meaning that L1 may turn off any of these bits. The 6775 * reason is that if one of these bits is necessary, it will appear 6776 * in vmcs01 and prepare_vmcs02, when it bitwise-or's the control 6777 * fields of vmcs01 and vmcs02, will turn these bits off - and 6778 * nested_vmx_l1_wants_exit() will not pass related exits to L1. 6779 * These rules have exceptions below. 6780 */ 6781 6782 /* pin-based controls */ 6783 msrs->pinbased_ctls_low = 6784 PIN_BASED_ALWAYSON_WITHOUT_TRUE_MSR; 6785 6786 msrs->pinbased_ctls_high = vmcs_conf->pin_based_exec_ctrl; 6787 msrs->pinbased_ctls_high &= 6788 PIN_BASED_EXT_INTR_MASK | 6789 PIN_BASED_NMI_EXITING | 6790 PIN_BASED_VIRTUAL_NMIS | 6791 (enable_apicv ? PIN_BASED_POSTED_INTR : 0); 6792 msrs->pinbased_ctls_high |= 6793 PIN_BASED_ALWAYSON_WITHOUT_TRUE_MSR | 6794 PIN_BASED_VMX_PREEMPTION_TIMER; 6795 6796 /* exit controls */ 6797 msrs->exit_ctls_low = 6798 VM_EXIT_ALWAYSON_WITHOUT_TRUE_MSR; 6799 6800 msrs->exit_ctls_high = vmcs_conf->vmexit_ctrl; 6801 msrs->exit_ctls_high &= 6802 #ifdef CONFIG_X86_64 6803 VM_EXIT_HOST_ADDR_SPACE_SIZE | 6804 #endif 6805 VM_EXIT_LOAD_IA32_PAT | VM_EXIT_SAVE_IA32_PAT | 6806 VM_EXIT_CLEAR_BNDCFGS; 6807 msrs->exit_ctls_high |= 6808 VM_EXIT_ALWAYSON_WITHOUT_TRUE_MSR | 6809 VM_EXIT_LOAD_IA32_EFER | VM_EXIT_SAVE_IA32_EFER | 6810 VM_EXIT_SAVE_VMX_PREEMPTION_TIMER | VM_EXIT_ACK_INTR_ON_EXIT | 6811 VM_EXIT_LOAD_IA32_PERF_GLOBAL_CTRL; 6812 6813 /* We support free control of debug control saving. */ 6814 msrs->exit_ctls_low &= ~VM_EXIT_SAVE_DEBUG_CONTROLS; 6815 6816 /* entry controls */ 6817 msrs->entry_ctls_low = 6818 VM_ENTRY_ALWAYSON_WITHOUT_TRUE_MSR; 6819 6820 msrs->entry_ctls_high = vmcs_conf->vmentry_ctrl; 6821 msrs->entry_ctls_high &= 6822 #ifdef CONFIG_X86_64 6823 VM_ENTRY_IA32E_MODE | 6824 #endif 6825 VM_ENTRY_LOAD_IA32_PAT | VM_ENTRY_LOAD_BNDCFGS; 6826 msrs->entry_ctls_high |= 6827 (VM_ENTRY_ALWAYSON_WITHOUT_TRUE_MSR | VM_ENTRY_LOAD_IA32_EFER | 6828 VM_ENTRY_LOAD_IA32_PERF_GLOBAL_CTRL); 6829 6830 /* We support free control of debug control loading. */ 6831 msrs->entry_ctls_low &= ~VM_ENTRY_LOAD_DEBUG_CONTROLS; 6832 6833 /* cpu-based controls */ 6834 msrs->procbased_ctls_low = 6835 CPU_BASED_ALWAYSON_WITHOUT_TRUE_MSR; 6836 6837 msrs->procbased_ctls_high = vmcs_conf->cpu_based_exec_ctrl; 6838 msrs->procbased_ctls_high &= 6839 CPU_BASED_INTR_WINDOW_EXITING | 6840 CPU_BASED_NMI_WINDOW_EXITING | CPU_BASED_USE_TSC_OFFSETTING | 6841 CPU_BASED_HLT_EXITING | CPU_BASED_INVLPG_EXITING | 6842 CPU_BASED_MWAIT_EXITING | CPU_BASED_CR3_LOAD_EXITING | 6843 CPU_BASED_CR3_STORE_EXITING | 6844 #ifdef CONFIG_X86_64 6845 CPU_BASED_CR8_LOAD_EXITING | CPU_BASED_CR8_STORE_EXITING | 6846 #endif 6847 CPU_BASED_MOV_DR_EXITING | CPU_BASED_UNCOND_IO_EXITING | 6848 CPU_BASED_USE_IO_BITMAPS | CPU_BASED_MONITOR_TRAP_FLAG | 6849 CPU_BASED_MONITOR_EXITING | CPU_BASED_RDPMC_EXITING | 6850 CPU_BASED_RDTSC_EXITING | CPU_BASED_PAUSE_EXITING | 6851 CPU_BASED_TPR_SHADOW | CPU_BASED_ACTIVATE_SECONDARY_CONTROLS; 6852 /* 6853 * We can allow some features even when not supported by the 6854 * hardware. For example, L1 can specify an MSR bitmap - and we 6855 * can use it to avoid exits to L1 - even when L0 runs L2 6856 * without MSR bitmaps. 6857 */ 6858 msrs->procbased_ctls_high |= 6859 CPU_BASED_ALWAYSON_WITHOUT_TRUE_MSR | 6860 CPU_BASED_USE_MSR_BITMAPS; 6861 6862 /* We support free control of CR3 access interception. */ 6863 msrs->procbased_ctls_low &= 6864 ~(CPU_BASED_CR3_LOAD_EXITING | CPU_BASED_CR3_STORE_EXITING); 6865 6866 /* 6867 * secondary cpu-based controls. Do not include those that 6868 * depend on CPUID bits, they are added later by 6869 * vmx_vcpu_after_set_cpuid. 6870 */ 6871 msrs->secondary_ctls_low = 0; 6872 6873 msrs->secondary_ctls_high = vmcs_conf->cpu_based_2nd_exec_ctrl; 6874 msrs->secondary_ctls_high &= 6875 SECONDARY_EXEC_DESC | 6876 SECONDARY_EXEC_ENABLE_RDTSCP | 6877 SECONDARY_EXEC_VIRTUALIZE_X2APIC_MODE | 6878 SECONDARY_EXEC_WBINVD_EXITING | 6879 SECONDARY_EXEC_APIC_REGISTER_VIRT | 6880 SECONDARY_EXEC_VIRTUAL_INTR_DELIVERY | 6881 SECONDARY_EXEC_RDRAND_EXITING | 6882 SECONDARY_EXEC_ENABLE_INVPCID | 6883 SECONDARY_EXEC_ENABLE_VMFUNC | 6884 SECONDARY_EXEC_RDSEED_EXITING | 6885 SECONDARY_EXEC_XSAVES | 6886 SECONDARY_EXEC_TSC_SCALING | 6887 SECONDARY_EXEC_ENABLE_USR_WAIT_PAUSE; 6888 6889 /* 6890 * We can emulate "VMCS shadowing," even if the hardware 6891 * doesn't support it. 6892 */ 6893 msrs->secondary_ctls_high |= 6894 SECONDARY_EXEC_SHADOW_VMCS; 6895 6896 if (enable_ept) { 6897 /* nested EPT: emulate EPT also to L1 */ 6898 msrs->secondary_ctls_high |= 6899 SECONDARY_EXEC_ENABLE_EPT; 6900 msrs->ept_caps = 6901 VMX_EPT_PAGE_WALK_4_BIT | 6902 VMX_EPT_PAGE_WALK_5_BIT | 6903 VMX_EPTP_WB_BIT | 6904 VMX_EPT_INVEPT_BIT | 6905 VMX_EPT_EXECUTE_ONLY_BIT; 6906 6907 msrs->ept_caps &= ept_caps; 6908 msrs->ept_caps |= VMX_EPT_EXTENT_GLOBAL_BIT | 6909 VMX_EPT_EXTENT_CONTEXT_BIT | VMX_EPT_2MB_PAGE_BIT | 6910 VMX_EPT_1GB_PAGE_BIT; 6911 if (enable_ept_ad_bits) { 6912 msrs->secondary_ctls_high |= 6913 SECONDARY_EXEC_ENABLE_PML; 6914 msrs->ept_caps |= VMX_EPT_AD_BIT; 6915 } 6916 6917 /* 6918 * Advertise EPTP switching irrespective of hardware support, 6919 * KVM emulates it in software so long as VMFUNC is supported. 6920 */ 6921 if (cpu_has_vmx_vmfunc()) 6922 msrs->vmfunc_controls = VMX_VMFUNC_EPTP_SWITCHING; 6923 } 6924 6925 /* 6926 * Old versions of KVM use the single-context version without 6927 * checking for support, so declare that it is supported even 6928 * though it is treated as global context. The alternative is 6929 * not failing the single-context invvpid, and it is worse. 6930 */ 6931 if (enable_vpid) { 6932 msrs->secondary_ctls_high |= 6933 SECONDARY_EXEC_ENABLE_VPID; 6934 msrs->vpid_caps = VMX_VPID_INVVPID_BIT | 6935 VMX_VPID_EXTENT_SUPPORTED_MASK; 6936 } 6937 6938 if (enable_unrestricted_guest) 6939 msrs->secondary_ctls_high |= 6940 SECONDARY_EXEC_UNRESTRICTED_GUEST; 6941 6942 if (flexpriority_enabled) 6943 msrs->secondary_ctls_high |= 6944 SECONDARY_EXEC_VIRTUALIZE_APIC_ACCESSES; 6945 6946 if (enable_sgx) 6947 msrs->secondary_ctls_high |= SECONDARY_EXEC_ENCLS_EXITING; 6948 6949 /* miscellaneous data */ 6950 msrs->misc_low = (u32)vmcs_conf->misc & VMX_MISC_SAVE_EFER_LMA; 6951 msrs->misc_low |= 6952 MSR_IA32_VMX_MISC_VMWRITE_SHADOW_RO_FIELDS | 6953 VMX_MISC_EMULATED_PREEMPTION_TIMER_RATE | 6954 VMX_MISC_ACTIVITY_HLT | 6955 VMX_MISC_ACTIVITY_WAIT_SIPI; 6956 msrs->misc_high = 0; 6957 6958 /* 6959 * This MSR reports some information about VMX support. We 6960 * should return information about the VMX we emulate for the 6961 * guest, and the VMCS structure we give it - not about the 6962 * VMX support of the underlying hardware. 6963 */ 6964 msrs->basic = 6965 VMCS12_REVISION | 6966 VMX_BASIC_TRUE_CTLS | 6967 ((u64)VMCS12_SIZE << VMX_BASIC_VMCS_SIZE_SHIFT) | 6968 (VMX_BASIC_MEM_TYPE_WB << VMX_BASIC_MEM_TYPE_SHIFT); 6969 6970 if (cpu_has_vmx_basic_inout()) 6971 msrs->basic |= VMX_BASIC_INOUT; 6972 6973 /* 6974 * These MSRs specify bits which the guest must keep fixed on 6975 * while L1 is in VMXON mode (in L1's root mode, or running an L2). 6976 * We picked the standard core2 setting. 6977 */ 6978 #define VMXON_CR0_ALWAYSON (X86_CR0_PE | X86_CR0_PG | X86_CR0_NE) 6979 #define VMXON_CR4_ALWAYSON X86_CR4_VMXE 6980 msrs->cr0_fixed0 = VMXON_CR0_ALWAYSON; 6981 msrs->cr4_fixed0 = VMXON_CR4_ALWAYSON; 6982 6983 /* These MSRs specify bits which the guest must keep fixed off. */ 6984 rdmsrl(MSR_IA32_VMX_CR0_FIXED1, msrs->cr0_fixed1); 6985 rdmsrl(MSR_IA32_VMX_CR4_FIXED1, msrs->cr4_fixed1); 6986 6987 if (vmx_umip_emulated()) 6988 msrs->cr4_fixed1 |= X86_CR4_UMIP; 6989 6990 msrs->vmcs_enum = nested_vmx_calc_vmcs_enum_msr(); 6991 } 6992 6993 void nested_vmx_hardware_unsetup(void) 6994 { 6995 int i; 6996 6997 if (enable_shadow_vmcs) { 6998 for (i = 0; i < VMX_BITMAP_NR; i++) 6999 free_page((unsigned long)vmx_bitmap[i]); 7000 } 7001 } 7002 7003 __init int nested_vmx_hardware_setup(int (*exit_handlers[])(struct kvm_vcpu *)) 7004 { 7005 int i; 7006 7007 if (!cpu_has_vmx_shadow_vmcs()) 7008 enable_shadow_vmcs = 0; 7009 if (enable_shadow_vmcs) { 7010 for (i = 0; i < VMX_BITMAP_NR; i++) { 7011 /* 7012 * The vmx_bitmap is not tied to a VM and so should 7013 * not be charged to a memcg. 7014 */ 7015 vmx_bitmap[i] = (unsigned long *) 7016 __get_free_page(GFP_KERNEL); 7017 if (!vmx_bitmap[i]) { 7018 nested_vmx_hardware_unsetup(); 7019 return -ENOMEM; 7020 } 7021 } 7022 7023 init_vmcs_shadow_fields(); 7024 } 7025 7026 exit_handlers[EXIT_REASON_VMCLEAR] = handle_vmclear; 7027 exit_handlers[EXIT_REASON_VMLAUNCH] = handle_vmlaunch; 7028 exit_handlers[EXIT_REASON_VMPTRLD] = handle_vmptrld; 7029 exit_handlers[EXIT_REASON_VMPTRST] = handle_vmptrst; 7030 exit_handlers[EXIT_REASON_VMREAD] = handle_vmread; 7031 exit_handlers[EXIT_REASON_VMRESUME] = handle_vmresume; 7032 exit_handlers[EXIT_REASON_VMWRITE] = handle_vmwrite; 7033 exit_handlers[EXIT_REASON_VMOFF] = handle_vmxoff; 7034 exit_handlers[EXIT_REASON_VMON] = handle_vmxon; 7035 exit_handlers[EXIT_REASON_INVEPT] = handle_invept; 7036 exit_handlers[EXIT_REASON_INVVPID] = handle_invvpid; 7037 exit_handlers[EXIT_REASON_VMFUNC] = handle_vmfunc; 7038 7039 return 0; 7040 } 7041 7042 struct kvm_x86_nested_ops vmx_nested_ops = { 7043 .leave_nested = vmx_leave_nested, 7044 .is_exception_vmexit = nested_vmx_is_exception_vmexit, 7045 .check_events = vmx_check_nested_events, 7046 .has_events = vmx_has_nested_events, 7047 .triple_fault = nested_vmx_triple_fault, 7048 .get_state = vmx_get_nested_state, 7049 .set_state = vmx_set_nested_state, 7050 .get_nested_state_pages = vmx_get_nested_state_pages, 7051 .write_log_dirty = nested_vmx_write_pml_buffer, 7052 .enable_evmcs = nested_enable_evmcs, 7053 .get_evmcs_version = nested_get_evmcs_version, 7054 .hv_inject_synthetic_vmexit_post_tlb_flush = vmx_hv_inject_synthetic_vmexit_post_tlb_flush, 7055 }; 7056