1 // SPDX-License-Identifier: GPL-2.0 2 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt 3 4 #include <linux/objtool.h> 5 #include <linux/percpu.h> 6 7 #include <asm/debugreg.h> 8 #include <asm/mmu_context.h> 9 10 #include "cpuid.h" 11 #include "hyperv.h" 12 #include "mmu.h" 13 #include "nested.h" 14 #include "pmu.h" 15 #include "sgx.h" 16 #include "trace.h" 17 #include "vmx.h" 18 #include "x86.h" 19 #include "smm.h" 20 21 static bool __read_mostly enable_shadow_vmcs = 1; 22 module_param_named(enable_shadow_vmcs, enable_shadow_vmcs, bool, S_IRUGO); 23 24 static bool __read_mostly nested_early_check = 0; 25 module_param(nested_early_check, bool, S_IRUGO); 26 27 #define CC KVM_NESTED_VMENTER_CONSISTENCY_CHECK 28 29 /* 30 * Hyper-V requires all of these, so mark them as supported even though 31 * they are just treated the same as all-context. 32 */ 33 #define VMX_VPID_EXTENT_SUPPORTED_MASK \ 34 (VMX_VPID_EXTENT_INDIVIDUAL_ADDR_BIT | \ 35 VMX_VPID_EXTENT_SINGLE_CONTEXT_BIT | \ 36 VMX_VPID_EXTENT_GLOBAL_CONTEXT_BIT | \ 37 VMX_VPID_EXTENT_SINGLE_NON_GLOBAL_BIT) 38 39 #define VMX_MISC_EMULATED_PREEMPTION_TIMER_RATE 5 40 41 enum { 42 VMX_VMREAD_BITMAP, 43 VMX_VMWRITE_BITMAP, 44 VMX_BITMAP_NR 45 }; 46 static unsigned long *vmx_bitmap[VMX_BITMAP_NR]; 47 48 #define vmx_vmread_bitmap (vmx_bitmap[VMX_VMREAD_BITMAP]) 49 #define vmx_vmwrite_bitmap (vmx_bitmap[VMX_VMWRITE_BITMAP]) 50 51 struct shadow_vmcs_field { 52 u16 encoding; 53 u16 offset; 54 }; 55 static struct shadow_vmcs_field shadow_read_only_fields[] = { 56 #define SHADOW_FIELD_RO(x, y) { x, offsetof(struct vmcs12, y) }, 57 #include "vmcs_shadow_fields.h" 58 }; 59 static int max_shadow_read_only_fields = 60 ARRAY_SIZE(shadow_read_only_fields); 61 62 static struct shadow_vmcs_field shadow_read_write_fields[] = { 63 #define SHADOW_FIELD_RW(x, y) { x, offsetof(struct vmcs12, y) }, 64 #include "vmcs_shadow_fields.h" 65 }; 66 static int max_shadow_read_write_fields = 67 ARRAY_SIZE(shadow_read_write_fields); 68 69 static void init_vmcs_shadow_fields(void) 70 { 71 int i, j; 72 73 memset(vmx_vmread_bitmap, 0xff, PAGE_SIZE); 74 memset(vmx_vmwrite_bitmap, 0xff, PAGE_SIZE); 75 76 for (i = j = 0; i < max_shadow_read_only_fields; i++) { 77 struct shadow_vmcs_field entry = shadow_read_only_fields[i]; 78 u16 field = entry.encoding; 79 80 if (vmcs_field_width(field) == VMCS_FIELD_WIDTH_U64 && 81 (i + 1 == max_shadow_read_only_fields || 82 shadow_read_only_fields[i + 1].encoding != field + 1)) 83 pr_err("Missing field from shadow_read_only_field %x\n", 84 field + 1); 85 86 clear_bit(field, vmx_vmread_bitmap); 87 if (field & 1) 88 #ifdef CONFIG_X86_64 89 continue; 90 #else 91 entry.offset += sizeof(u32); 92 #endif 93 shadow_read_only_fields[j++] = entry; 94 } 95 max_shadow_read_only_fields = j; 96 97 for (i = j = 0; i < max_shadow_read_write_fields; i++) { 98 struct shadow_vmcs_field entry = shadow_read_write_fields[i]; 99 u16 field = entry.encoding; 100 101 if (vmcs_field_width(field) == VMCS_FIELD_WIDTH_U64 && 102 (i + 1 == max_shadow_read_write_fields || 103 shadow_read_write_fields[i + 1].encoding != field + 1)) 104 pr_err("Missing field from shadow_read_write_field %x\n", 105 field + 1); 106 107 WARN_ONCE(field >= GUEST_ES_AR_BYTES && 108 field <= GUEST_TR_AR_BYTES, 109 "Update vmcs12_write_any() to drop reserved bits from AR_BYTES"); 110 111 /* 112 * PML and the preemption timer can be emulated, but the 113 * processor cannot vmwrite to fields that don't exist 114 * on bare metal. 115 */ 116 switch (field) { 117 case GUEST_PML_INDEX: 118 if (!cpu_has_vmx_pml()) 119 continue; 120 break; 121 case VMX_PREEMPTION_TIMER_VALUE: 122 if (!cpu_has_vmx_preemption_timer()) 123 continue; 124 break; 125 case GUEST_INTR_STATUS: 126 if (!cpu_has_vmx_apicv()) 127 continue; 128 break; 129 default: 130 break; 131 } 132 133 clear_bit(field, vmx_vmwrite_bitmap); 134 clear_bit(field, vmx_vmread_bitmap); 135 if (field & 1) 136 #ifdef CONFIG_X86_64 137 continue; 138 #else 139 entry.offset += sizeof(u32); 140 #endif 141 shadow_read_write_fields[j++] = entry; 142 } 143 max_shadow_read_write_fields = j; 144 } 145 146 /* 147 * The following 3 functions, nested_vmx_succeed()/failValid()/failInvalid(), 148 * set the success or error code of an emulated VMX instruction (as specified 149 * by Vol 2B, VMX Instruction Reference, "Conventions"), and skip the emulated 150 * instruction. 151 */ 152 static int nested_vmx_succeed(struct kvm_vcpu *vcpu) 153 { 154 vmx_set_rflags(vcpu, vmx_get_rflags(vcpu) 155 & ~(X86_EFLAGS_CF | X86_EFLAGS_PF | X86_EFLAGS_AF | 156 X86_EFLAGS_ZF | X86_EFLAGS_SF | X86_EFLAGS_OF)); 157 return kvm_skip_emulated_instruction(vcpu); 158 } 159 160 static int nested_vmx_failInvalid(struct kvm_vcpu *vcpu) 161 { 162 vmx_set_rflags(vcpu, (vmx_get_rflags(vcpu) 163 & ~(X86_EFLAGS_PF | X86_EFLAGS_AF | X86_EFLAGS_ZF | 164 X86_EFLAGS_SF | X86_EFLAGS_OF)) 165 | X86_EFLAGS_CF); 166 return kvm_skip_emulated_instruction(vcpu); 167 } 168 169 static int nested_vmx_failValid(struct kvm_vcpu *vcpu, 170 u32 vm_instruction_error) 171 { 172 vmx_set_rflags(vcpu, (vmx_get_rflags(vcpu) 173 & ~(X86_EFLAGS_CF | X86_EFLAGS_PF | X86_EFLAGS_AF | 174 X86_EFLAGS_SF | X86_EFLAGS_OF)) 175 | X86_EFLAGS_ZF); 176 get_vmcs12(vcpu)->vm_instruction_error = vm_instruction_error; 177 /* 178 * We don't need to force sync to shadow VMCS because 179 * VM_INSTRUCTION_ERROR is not shadowed. Enlightened VMCS 'shadows' all 180 * fields and thus must be synced. 181 */ 182 if (to_vmx(vcpu)->nested.hv_evmcs_vmptr != EVMPTR_INVALID) 183 to_vmx(vcpu)->nested.need_vmcs12_to_shadow_sync = true; 184 185 return kvm_skip_emulated_instruction(vcpu); 186 } 187 188 static int nested_vmx_fail(struct kvm_vcpu *vcpu, u32 vm_instruction_error) 189 { 190 struct vcpu_vmx *vmx = to_vmx(vcpu); 191 192 /* 193 * failValid writes the error number to the current VMCS, which 194 * can't be done if there isn't a current VMCS. 195 */ 196 if (vmx->nested.current_vmptr == INVALID_GPA && 197 !evmptr_is_valid(vmx->nested.hv_evmcs_vmptr)) 198 return nested_vmx_failInvalid(vcpu); 199 200 return nested_vmx_failValid(vcpu, vm_instruction_error); 201 } 202 203 static void nested_vmx_abort(struct kvm_vcpu *vcpu, u32 indicator) 204 { 205 /* TODO: not to reset guest simply here. */ 206 kvm_make_request(KVM_REQ_TRIPLE_FAULT, vcpu); 207 pr_debug_ratelimited("nested vmx abort, indicator %d\n", indicator); 208 } 209 210 static inline bool vmx_control_verify(u32 control, u32 low, u32 high) 211 { 212 return fixed_bits_valid(control, low, high); 213 } 214 215 static inline u64 vmx_control_msr(u32 low, u32 high) 216 { 217 return low | ((u64)high << 32); 218 } 219 220 static void vmx_disable_shadow_vmcs(struct vcpu_vmx *vmx) 221 { 222 secondary_exec_controls_clearbit(vmx, SECONDARY_EXEC_SHADOW_VMCS); 223 vmcs_write64(VMCS_LINK_POINTER, INVALID_GPA); 224 vmx->nested.need_vmcs12_to_shadow_sync = false; 225 } 226 227 static inline void nested_release_evmcs(struct kvm_vcpu *vcpu) 228 { 229 struct kvm_vcpu_hv *hv_vcpu = to_hv_vcpu(vcpu); 230 struct vcpu_vmx *vmx = to_vmx(vcpu); 231 232 if (evmptr_is_valid(vmx->nested.hv_evmcs_vmptr)) { 233 kvm_vcpu_unmap(vcpu, &vmx->nested.hv_evmcs_map, true); 234 vmx->nested.hv_evmcs = NULL; 235 } 236 237 vmx->nested.hv_evmcs_vmptr = EVMPTR_INVALID; 238 239 if (hv_vcpu) { 240 hv_vcpu->nested.pa_page_gpa = INVALID_GPA; 241 hv_vcpu->nested.vm_id = 0; 242 hv_vcpu->nested.vp_id = 0; 243 } 244 } 245 246 static void vmx_sync_vmcs_host_state(struct vcpu_vmx *vmx, 247 struct loaded_vmcs *prev) 248 { 249 struct vmcs_host_state *dest, *src; 250 251 if (unlikely(!vmx->guest_state_loaded)) 252 return; 253 254 src = &prev->host_state; 255 dest = &vmx->loaded_vmcs->host_state; 256 257 vmx_set_host_fs_gs(dest, src->fs_sel, src->gs_sel, src->fs_base, src->gs_base); 258 dest->ldt_sel = src->ldt_sel; 259 #ifdef CONFIG_X86_64 260 dest->ds_sel = src->ds_sel; 261 dest->es_sel = src->es_sel; 262 #endif 263 } 264 265 static void vmx_switch_vmcs(struct kvm_vcpu *vcpu, struct loaded_vmcs *vmcs) 266 { 267 struct vcpu_vmx *vmx = to_vmx(vcpu); 268 struct loaded_vmcs *prev; 269 int cpu; 270 271 if (WARN_ON_ONCE(vmx->loaded_vmcs == vmcs)) 272 return; 273 274 cpu = get_cpu(); 275 prev = vmx->loaded_vmcs; 276 vmx->loaded_vmcs = vmcs; 277 vmx_vcpu_load_vmcs(vcpu, cpu, prev); 278 vmx_sync_vmcs_host_state(vmx, prev); 279 put_cpu(); 280 281 vcpu->arch.regs_avail = ~VMX_REGS_LAZY_LOAD_SET; 282 283 /* 284 * All lazily updated registers will be reloaded from VMCS12 on both 285 * vmentry and vmexit. 286 */ 287 vcpu->arch.regs_dirty = 0; 288 } 289 290 /* 291 * Free whatever needs to be freed from vmx->nested when L1 goes down, or 292 * just stops using VMX. 293 */ 294 static void free_nested(struct kvm_vcpu *vcpu) 295 { 296 struct vcpu_vmx *vmx = to_vmx(vcpu); 297 298 if (WARN_ON_ONCE(vmx->loaded_vmcs != &vmx->vmcs01)) 299 vmx_switch_vmcs(vcpu, &vmx->vmcs01); 300 301 if (!vmx->nested.vmxon && !vmx->nested.smm.vmxon) 302 return; 303 304 kvm_clear_request(KVM_REQ_GET_NESTED_STATE_PAGES, vcpu); 305 306 vmx->nested.vmxon = false; 307 vmx->nested.smm.vmxon = false; 308 vmx->nested.vmxon_ptr = INVALID_GPA; 309 free_vpid(vmx->nested.vpid02); 310 vmx->nested.posted_intr_nv = -1; 311 vmx->nested.current_vmptr = INVALID_GPA; 312 if (enable_shadow_vmcs) { 313 vmx_disable_shadow_vmcs(vmx); 314 vmcs_clear(vmx->vmcs01.shadow_vmcs); 315 free_vmcs(vmx->vmcs01.shadow_vmcs); 316 vmx->vmcs01.shadow_vmcs = NULL; 317 } 318 kfree(vmx->nested.cached_vmcs12); 319 vmx->nested.cached_vmcs12 = NULL; 320 kfree(vmx->nested.cached_shadow_vmcs12); 321 vmx->nested.cached_shadow_vmcs12 = NULL; 322 /* 323 * Unpin physical memory we referred to in the vmcs02. The APIC access 324 * page's backing page (yeah, confusing) shouldn't actually be accessed, 325 * and if it is written, the contents are irrelevant. 326 */ 327 kvm_vcpu_unmap(vcpu, &vmx->nested.apic_access_page_map, false); 328 kvm_vcpu_unmap(vcpu, &vmx->nested.virtual_apic_map, true); 329 kvm_vcpu_unmap(vcpu, &vmx->nested.pi_desc_map, true); 330 vmx->nested.pi_desc = NULL; 331 332 kvm_mmu_free_roots(vcpu->kvm, &vcpu->arch.guest_mmu, KVM_MMU_ROOTS_ALL); 333 334 nested_release_evmcs(vcpu); 335 336 free_loaded_vmcs(&vmx->nested.vmcs02); 337 } 338 339 /* 340 * Ensure that the current vmcs of the logical processor is the 341 * vmcs01 of the vcpu before calling free_nested(). 342 */ 343 void nested_vmx_free_vcpu(struct kvm_vcpu *vcpu) 344 { 345 vcpu_load(vcpu); 346 vmx_leave_nested(vcpu); 347 vcpu_put(vcpu); 348 } 349 350 #define EPTP_PA_MASK GENMASK_ULL(51, 12) 351 352 static bool nested_ept_root_matches(hpa_t root_hpa, u64 root_eptp, u64 eptp) 353 { 354 return VALID_PAGE(root_hpa) && 355 ((root_eptp & EPTP_PA_MASK) == (eptp & EPTP_PA_MASK)); 356 } 357 358 static void nested_ept_invalidate_addr(struct kvm_vcpu *vcpu, gpa_t eptp, 359 gpa_t addr) 360 { 361 uint i; 362 struct kvm_mmu_root_info *cached_root; 363 364 WARN_ON_ONCE(!mmu_is_nested(vcpu)); 365 366 for (i = 0; i < KVM_MMU_NUM_PREV_ROOTS; i++) { 367 cached_root = &vcpu->arch.mmu->prev_roots[i]; 368 369 if (nested_ept_root_matches(cached_root->hpa, cached_root->pgd, 370 eptp)) 371 vcpu->arch.mmu->invlpg(vcpu, addr, cached_root->hpa); 372 } 373 } 374 375 static void nested_ept_inject_page_fault(struct kvm_vcpu *vcpu, 376 struct x86_exception *fault) 377 { 378 struct vmcs12 *vmcs12 = get_vmcs12(vcpu); 379 struct vcpu_vmx *vmx = to_vmx(vcpu); 380 u32 vm_exit_reason; 381 unsigned long exit_qualification = vcpu->arch.exit_qualification; 382 383 if (vmx->nested.pml_full) { 384 vm_exit_reason = EXIT_REASON_PML_FULL; 385 vmx->nested.pml_full = false; 386 exit_qualification &= INTR_INFO_UNBLOCK_NMI; 387 } else { 388 if (fault->error_code & PFERR_RSVD_MASK) 389 vm_exit_reason = EXIT_REASON_EPT_MISCONFIG; 390 else 391 vm_exit_reason = EXIT_REASON_EPT_VIOLATION; 392 393 /* 394 * Although the caller (kvm_inject_emulated_page_fault) would 395 * have already synced the faulting address in the shadow EPT 396 * tables for the current EPTP12, we also need to sync it for 397 * any other cached EPTP02s based on the same EP4TA, since the 398 * TLB associates mappings to the EP4TA rather than the full EPTP. 399 */ 400 nested_ept_invalidate_addr(vcpu, vmcs12->ept_pointer, 401 fault->address); 402 } 403 404 nested_vmx_vmexit(vcpu, vm_exit_reason, 0, exit_qualification); 405 vmcs12->guest_physical_address = fault->address; 406 } 407 408 static void nested_ept_new_eptp(struct kvm_vcpu *vcpu) 409 { 410 struct vcpu_vmx *vmx = to_vmx(vcpu); 411 bool execonly = vmx->nested.msrs.ept_caps & VMX_EPT_EXECUTE_ONLY_BIT; 412 int ept_lpage_level = ept_caps_to_lpage_level(vmx->nested.msrs.ept_caps); 413 414 kvm_init_shadow_ept_mmu(vcpu, execonly, ept_lpage_level, 415 nested_ept_ad_enabled(vcpu), 416 nested_ept_get_eptp(vcpu)); 417 } 418 419 static void nested_ept_init_mmu_context(struct kvm_vcpu *vcpu) 420 { 421 WARN_ON(mmu_is_nested(vcpu)); 422 423 vcpu->arch.mmu = &vcpu->arch.guest_mmu; 424 nested_ept_new_eptp(vcpu); 425 vcpu->arch.mmu->get_guest_pgd = nested_ept_get_eptp; 426 vcpu->arch.mmu->inject_page_fault = nested_ept_inject_page_fault; 427 vcpu->arch.mmu->get_pdptr = kvm_pdptr_read; 428 429 vcpu->arch.walk_mmu = &vcpu->arch.nested_mmu; 430 } 431 432 static void nested_ept_uninit_mmu_context(struct kvm_vcpu *vcpu) 433 { 434 vcpu->arch.mmu = &vcpu->arch.root_mmu; 435 vcpu->arch.walk_mmu = &vcpu->arch.root_mmu; 436 } 437 438 static bool nested_vmx_is_page_fault_vmexit(struct vmcs12 *vmcs12, 439 u16 error_code) 440 { 441 bool inequality, bit; 442 443 bit = (vmcs12->exception_bitmap & (1u << PF_VECTOR)) != 0; 444 inequality = 445 (error_code & vmcs12->page_fault_error_code_mask) != 446 vmcs12->page_fault_error_code_match; 447 return inequality ^ bit; 448 } 449 450 static bool nested_vmx_is_exception_vmexit(struct kvm_vcpu *vcpu, u8 vector, 451 u32 error_code) 452 { 453 struct vmcs12 *vmcs12 = get_vmcs12(vcpu); 454 455 /* 456 * Drop bits 31:16 of the error code when performing the #PF mask+match 457 * check. All VMCS fields involved are 32 bits, but Intel CPUs never 458 * set bits 31:16 and VMX disallows setting bits 31:16 in the injected 459 * error code. Including the to-be-dropped bits in the check might 460 * result in an "impossible" or missed exit from L1's perspective. 461 */ 462 if (vector == PF_VECTOR) 463 return nested_vmx_is_page_fault_vmexit(vmcs12, (u16)error_code); 464 465 return (vmcs12->exception_bitmap & (1u << vector)); 466 } 467 468 static int nested_vmx_check_io_bitmap_controls(struct kvm_vcpu *vcpu, 469 struct vmcs12 *vmcs12) 470 { 471 if (!nested_cpu_has(vmcs12, CPU_BASED_USE_IO_BITMAPS)) 472 return 0; 473 474 if (CC(!page_address_valid(vcpu, vmcs12->io_bitmap_a)) || 475 CC(!page_address_valid(vcpu, vmcs12->io_bitmap_b))) 476 return -EINVAL; 477 478 return 0; 479 } 480 481 static int nested_vmx_check_msr_bitmap_controls(struct kvm_vcpu *vcpu, 482 struct vmcs12 *vmcs12) 483 { 484 if (!nested_cpu_has(vmcs12, CPU_BASED_USE_MSR_BITMAPS)) 485 return 0; 486 487 if (CC(!page_address_valid(vcpu, vmcs12->msr_bitmap))) 488 return -EINVAL; 489 490 return 0; 491 } 492 493 static int nested_vmx_check_tpr_shadow_controls(struct kvm_vcpu *vcpu, 494 struct vmcs12 *vmcs12) 495 { 496 if (!nested_cpu_has(vmcs12, CPU_BASED_TPR_SHADOW)) 497 return 0; 498 499 if (CC(!page_address_valid(vcpu, vmcs12->virtual_apic_page_addr))) 500 return -EINVAL; 501 502 return 0; 503 } 504 505 /* 506 * For x2APIC MSRs, ignore the vmcs01 bitmap. L1 can enable x2APIC without L1 507 * itself utilizing x2APIC. All MSRs were previously set to be intercepted, 508 * only the "disable intercept" case needs to be handled. 509 */ 510 static void nested_vmx_disable_intercept_for_x2apic_msr(unsigned long *msr_bitmap_l1, 511 unsigned long *msr_bitmap_l0, 512 u32 msr, int type) 513 { 514 if (type & MSR_TYPE_R && !vmx_test_msr_bitmap_read(msr_bitmap_l1, msr)) 515 vmx_clear_msr_bitmap_read(msr_bitmap_l0, msr); 516 517 if (type & MSR_TYPE_W && !vmx_test_msr_bitmap_write(msr_bitmap_l1, msr)) 518 vmx_clear_msr_bitmap_write(msr_bitmap_l0, msr); 519 } 520 521 static inline void enable_x2apic_msr_intercepts(unsigned long *msr_bitmap) 522 { 523 int msr; 524 525 for (msr = 0x800; msr <= 0x8ff; msr += BITS_PER_LONG) { 526 unsigned word = msr / BITS_PER_LONG; 527 528 msr_bitmap[word] = ~0; 529 msr_bitmap[word + (0x800 / sizeof(long))] = ~0; 530 } 531 } 532 533 #define BUILD_NVMX_MSR_INTERCEPT_HELPER(rw) \ 534 static inline \ 535 void nested_vmx_set_msr_##rw##_intercept(struct vcpu_vmx *vmx, \ 536 unsigned long *msr_bitmap_l1, \ 537 unsigned long *msr_bitmap_l0, u32 msr) \ 538 { \ 539 if (vmx_test_msr_bitmap_##rw(vmx->vmcs01.msr_bitmap, msr) || \ 540 vmx_test_msr_bitmap_##rw(msr_bitmap_l1, msr)) \ 541 vmx_set_msr_bitmap_##rw(msr_bitmap_l0, msr); \ 542 else \ 543 vmx_clear_msr_bitmap_##rw(msr_bitmap_l0, msr); \ 544 } 545 BUILD_NVMX_MSR_INTERCEPT_HELPER(read) 546 BUILD_NVMX_MSR_INTERCEPT_HELPER(write) 547 548 static inline void nested_vmx_set_intercept_for_msr(struct vcpu_vmx *vmx, 549 unsigned long *msr_bitmap_l1, 550 unsigned long *msr_bitmap_l0, 551 u32 msr, int types) 552 { 553 if (types & MSR_TYPE_R) 554 nested_vmx_set_msr_read_intercept(vmx, msr_bitmap_l1, 555 msr_bitmap_l0, msr); 556 if (types & MSR_TYPE_W) 557 nested_vmx_set_msr_write_intercept(vmx, msr_bitmap_l1, 558 msr_bitmap_l0, msr); 559 } 560 561 /* 562 * Merge L0's and L1's MSR bitmap, return false to indicate that 563 * we do not use the hardware. 564 */ 565 static inline bool nested_vmx_prepare_msr_bitmap(struct kvm_vcpu *vcpu, 566 struct vmcs12 *vmcs12) 567 { 568 struct vcpu_vmx *vmx = to_vmx(vcpu); 569 int msr; 570 unsigned long *msr_bitmap_l1; 571 unsigned long *msr_bitmap_l0 = vmx->nested.vmcs02.msr_bitmap; 572 struct hv_enlightened_vmcs *evmcs = vmx->nested.hv_evmcs; 573 struct kvm_host_map *map = &vmx->nested.msr_bitmap_map; 574 575 /* Nothing to do if the MSR bitmap is not in use. */ 576 if (!cpu_has_vmx_msr_bitmap() || 577 !nested_cpu_has(vmcs12, CPU_BASED_USE_MSR_BITMAPS)) 578 return false; 579 580 /* 581 * MSR bitmap update can be skipped when: 582 * - MSR bitmap for L1 hasn't changed. 583 * - Nested hypervisor (L1) is attempting to launch the same L2 as 584 * before. 585 * - Nested hypervisor (L1) has enabled 'Enlightened MSR Bitmap' feature 586 * and tells KVM (L0) there were no changes in MSR bitmap for L2. 587 */ 588 if (!vmx->nested.force_msr_bitmap_recalc && evmcs && 589 evmcs->hv_enlightenments_control.msr_bitmap && 590 evmcs->hv_clean_fields & HV_VMX_ENLIGHTENED_CLEAN_FIELD_MSR_BITMAP) 591 return true; 592 593 if (kvm_vcpu_map(vcpu, gpa_to_gfn(vmcs12->msr_bitmap), map)) 594 return false; 595 596 msr_bitmap_l1 = (unsigned long *)map->hva; 597 598 /* 599 * To keep the control flow simple, pay eight 8-byte writes (sixteen 600 * 4-byte writes on 32-bit systems) up front to enable intercepts for 601 * the x2APIC MSR range and selectively toggle those relevant to L2. 602 */ 603 enable_x2apic_msr_intercepts(msr_bitmap_l0); 604 605 if (nested_cpu_has_virt_x2apic_mode(vmcs12)) { 606 if (nested_cpu_has_apic_reg_virt(vmcs12)) { 607 /* 608 * L0 need not intercept reads for MSRs between 0x800 609 * and 0x8ff, it just lets the processor take the value 610 * from the virtual-APIC page; take those 256 bits 611 * directly from the L1 bitmap. 612 */ 613 for (msr = 0x800; msr <= 0x8ff; msr += BITS_PER_LONG) { 614 unsigned word = msr / BITS_PER_LONG; 615 616 msr_bitmap_l0[word] = msr_bitmap_l1[word]; 617 } 618 } 619 620 nested_vmx_disable_intercept_for_x2apic_msr( 621 msr_bitmap_l1, msr_bitmap_l0, 622 X2APIC_MSR(APIC_TASKPRI), 623 MSR_TYPE_R | MSR_TYPE_W); 624 625 if (nested_cpu_has_vid(vmcs12)) { 626 nested_vmx_disable_intercept_for_x2apic_msr( 627 msr_bitmap_l1, msr_bitmap_l0, 628 X2APIC_MSR(APIC_EOI), 629 MSR_TYPE_W); 630 nested_vmx_disable_intercept_for_x2apic_msr( 631 msr_bitmap_l1, msr_bitmap_l0, 632 X2APIC_MSR(APIC_SELF_IPI), 633 MSR_TYPE_W); 634 } 635 } 636 637 /* 638 * Always check vmcs01's bitmap to honor userspace MSR filters and any 639 * other runtime changes to vmcs01's bitmap, e.g. dynamic pass-through. 640 */ 641 #ifdef CONFIG_X86_64 642 nested_vmx_set_intercept_for_msr(vmx, msr_bitmap_l1, msr_bitmap_l0, 643 MSR_FS_BASE, MSR_TYPE_RW); 644 645 nested_vmx_set_intercept_for_msr(vmx, msr_bitmap_l1, msr_bitmap_l0, 646 MSR_GS_BASE, MSR_TYPE_RW); 647 648 nested_vmx_set_intercept_for_msr(vmx, msr_bitmap_l1, msr_bitmap_l0, 649 MSR_KERNEL_GS_BASE, MSR_TYPE_RW); 650 #endif 651 nested_vmx_set_intercept_for_msr(vmx, msr_bitmap_l1, msr_bitmap_l0, 652 MSR_IA32_SPEC_CTRL, MSR_TYPE_RW); 653 654 nested_vmx_set_intercept_for_msr(vmx, msr_bitmap_l1, msr_bitmap_l0, 655 MSR_IA32_PRED_CMD, MSR_TYPE_W); 656 657 kvm_vcpu_unmap(vcpu, &vmx->nested.msr_bitmap_map, false); 658 659 vmx->nested.force_msr_bitmap_recalc = false; 660 661 return true; 662 } 663 664 static void nested_cache_shadow_vmcs12(struct kvm_vcpu *vcpu, 665 struct vmcs12 *vmcs12) 666 { 667 struct vcpu_vmx *vmx = to_vmx(vcpu); 668 struct gfn_to_hva_cache *ghc = &vmx->nested.shadow_vmcs12_cache; 669 670 if (!nested_cpu_has_shadow_vmcs(vmcs12) || 671 vmcs12->vmcs_link_pointer == INVALID_GPA) 672 return; 673 674 if (ghc->gpa != vmcs12->vmcs_link_pointer && 675 kvm_gfn_to_hva_cache_init(vcpu->kvm, ghc, 676 vmcs12->vmcs_link_pointer, VMCS12_SIZE)) 677 return; 678 679 kvm_read_guest_cached(vmx->vcpu.kvm, ghc, get_shadow_vmcs12(vcpu), 680 VMCS12_SIZE); 681 } 682 683 static void nested_flush_cached_shadow_vmcs12(struct kvm_vcpu *vcpu, 684 struct vmcs12 *vmcs12) 685 { 686 struct vcpu_vmx *vmx = to_vmx(vcpu); 687 struct gfn_to_hva_cache *ghc = &vmx->nested.shadow_vmcs12_cache; 688 689 if (!nested_cpu_has_shadow_vmcs(vmcs12) || 690 vmcs12->vmcs_link_pointer == INVALID_GPA) 691 return; 692 693 if (ghc->gpa != vmcs12->vmcs_link_pointer && 694 kvm_gfn_to_hva_cache_init(vcpu->kvm, ghc, 695 vmcs12->vmcs_link_pointer, VMCS12_SIZE)) 696 return; 697 698 kvm_write_guest_cached(vmx->vcpu.kvm, ghc, get_shadow_vmcs12(vcpu), 699 VMCS12_SIZE); 700 } 701 702 /* 703 * In nested virtualization, check if L1 has set 704 * VM_EXIT_ACK_INTR_ON_EXIT 705 */ 706 static bool nested_exit_intr_ack_set(struct kvm_vcpu *vcpu) 707 { 708 return get_vmcs12(vcpu)->vm_exit_controls & 709 VM_EXIT_ACK_INTR_ON_EXIT; 710 } 711 712 static int nested_vmx_check_apic_access_controls(struct kvm_vcpu *vcpu, 713 struct vmcs12 *vmcs12) 714 { 715 if (nested_cpu_has2(vmcs12, SECONDARY_EXEC_VIRTUALIZE_APIC_ACCESSES) && 716 CC(!page_address_valid(vcpu, vmcs12->apic_access_addr))) 717 return -EINVAL; 718 else 719 return 0; 720 } 721 722 static int nested_vmx_check_apicv_controls(struct kvm_vcpu *vcpu, 723 struct vmcs12 *vmcs12) 724 { 725 if (!nested_cpu_has_virt_x2apic_mode(vmcs12) && 726 !nested_cpu_has_apic_reg_virt(vmcs12) && 727 !nested_cpu_has_vid(vmcs12) && 728 !nested_cpu_has_posted_intr(vmcs12)) 729 return 0; 730 731 /* 732 * If virtualize x2apic mode is enabled, 733 * virtualize apic access must be disabled. 734 */ 735 if (CC(nested_cpu_has_virt_x2apic_mode(vmcs12) && 736 nested_cpu_has2(vmcs12, SECONDARY_EXEC_VIRTUALIZE_APIC_ACCESSES))) 737 return -EINVAL; 738 739 /* 740 * If virtual interrupt delivery is enabled, 741 * we must exit on external interrupts. 742 */ 743 if (CC(nested_cpu_has_vid(vmcs12) && !nested_exit_on_intr(vcpu))) 744 return -EINVAL; 745 746 /* 747 * bits 15:8 should be zero in posted_intr_nv, 748 * the descriptor address has been already checked 749 * in nested_get_vmcs12_pages. 750 * 751 * bits 5:0 of posted_intr_desc_addr should be zero. 752 */ 753 if (nested_cpu_has_posted_intr(vmcs12) && 754 (CC(!nested_cpu_has_vid(vmcs12)) || 755 CC(!nested_exit_intr_ack_set(vcpu)) || 756 CC((vmcs12->posted_intr_nv & 0xff00)) || 757 CC(!kvm_vcpu_is_legal_aligned_gpa(vcpu, vmcs12->posted_intr_desc_addr, 64)))) 758 return -EINVAL; 759 760 /* tpr shadow is needed by all apicv features. */ 761 if (CC(!nested_cpu_has(vmcs12, CPU_BASED_TPR_SHADOW))) 762 return -EINVAL; 763 764 return 0; 765 } 766 767 static int nested_vmx_check_msr_switch(struct kvm_vcpu *vcpu, 768 u32 count, u64 addr) 769 { 770 if (count == 0) 771 return 0; 772 773 if (!kvm_vcpu_is_legal_aligned_gpa(vcpu, addr, 16) || 774 !kvm_vcpu_is_legal_gpa(vcpu, (addr + count * sizeof(struct vmx_msr_entry) - 1))) 775 return -EINVAL; 776 777 return 0; 778 } 779 780 static int nested_vmx_check_exit_msr_switch_controls(struct kvm_vcpu *vcpu, 781 struct vmcs12 *vmcs12) 782 { 783 if (CC(nested_vmx_check_msr_switch(vcpu, 784 vmcs12->vm_exit_msr_load_count, 785 vmcs12->vm_exit_msr_load_addr)) || 786 CC(nested_vmx_check_msr_switch(vcpu, 787 vmcs12->vm_exit_msr_store_count, 788 vmcs12->vm_exit_msr_store_addr))) 789 return -EINVAL; 790 791 return 0; 792 } 793 794 static int nested_vmx_check_entry_msr_switch_controls(struct kvm_vcpu *vcpu, 795 struct vmcs12 *vmcs12) 796 { 797 if (CC(nested_vmx_check_msr_switch(vcpu, 798 vmcs12->vm_entry_msr_load_count, 799 vmcs12->vm_entry_msr_load_addr))) 800 return -EINVAL; 801 802 return 0; 803 } 804 805 static int nested_vmx_check_pml_controls(struct kvm_vcpu *vcpu, 806 struct vmcs12 *vmcs12) 807 { 808 if (!nested_cpu_has_pml(vmcs12)) 809 return 0; 810 811 if (CC(!nested_cpu_has_ept(vmcs12)) || 812 CC(!page_address_valid(vcpu, vmcs12->pml_address))) 813 return -EINVAL; 814 815 return 0; 816 } 817 818 static int nested_vmx_check_unrestricted_guest_controls(struct kvm_vcpu *vcpu, 819 struct vmcs12 *vmcs12) 820 { 821 if (CC(nested_cpu_has2(vmcs12, SECONDARY_EXEC_UNRESTRICTED_GUEST) && 822 !nested_cpu_has_ept(vmcs12))) 823 return -EINVAL; 824 return 0; 825 } 826 827 static int nested_vmx_check_mode_based_ept_exec_controls(struct kvm_vcpu *vcpu, 828 struct vmcs12 *vmcs12) 829 { 830 if (CC(nested_cpu_has2(vmcs12, SECONDARY_EXEC_MODE_BASED_EPT_EXEC) && 831 !nested_cpu_has_ept(vmcs12))) 832 return -EINVAL; 833 return 0; 834 } 835 836 static int nested_vmx_check_shadow_vmcs_controls(struct kvm_vcpu *vcpu, 837 struct vmcs12 *vmcs12) 838 { 839 if (!nested_cpu_has_shadow_vmcs(vmcs12)) 840 return 0; 841 842 if (CC(!page_address_valid(vcpu, vmcs12->vmread_bitmap)) || 843 CC(!page_address_valid(vcpu, vmcs12->vmwrite_bitmap))) 844 return -EINVAL; 845 846 return 0; 847 } 848 849 static int nested_vmx_msr_check_common(struct kvm_vcpu *vcpu, 850 struct vmx_msr_entry *e) 851 { 852 /* x2APIC MSR accesses are not allowed */ 853 if (CC(vcpu->arch.apic_base & X2APIC_ENABLE && e->index >> 8 == 0x8)) 854 return -EINVAL; 855 if (CC(e->index == MSR_IA32_UCODE_WRITE) || /* SDM Table 35-2 */ 856 CC(e->index == MSR_IA32_UCODE_REV)) 857 return -EINVAL; 858 if (CC(e->reserved != 0)) 859 return -EINVAL; 860 return 0; 861 } 862 863 static int nested_vmx_load_msr_check(struct kvm_vcpu *vcpu, 864 struct vmx_msr_entry *e) 865 { 866 if (CC(e->index == MSR_FS_BASE) || 867 CC(e->index == MSR_GS_BASE) || 868 CC(e->index == MSR_IA32_SMM_MONITOR_CTL) || /* SMM is not supported */ 869 nested_vmx_msr_check_common(vcpu, e)) 870 return -EINVAL; 871 return 0; 872 } 873 874 static int nested_vmx_store_msr_check(struct kvm_vcpu *vcpu, 875 struct vmx_msr_entry *e) 876 { 877 if (CC(e->index == MSR_IA32_SMBASE) || /* SMM is not supported */ 878 nested_vmx_msr_check_common(vcpu, e)) 879 return -EINVAL; 880 return 0; 881 } 882 883 static u32 nested_vmx_max_atomic_switch_msrs(struct kvm_vcpu *vcpu) 884 { 885 struct vcpu_vmx *vmx = to_vmx(vcpu); 886 u64 vmx_misc = vmx_control_msr(vmx->nested.msrs.misc_low, 887 vmx->nested.msrs.misc_high); 888 889 return (vmx_misc_max_msr(vmx_misc) + 1) * VMX_MISC_MSR_LIST_MULTIPLIER; 890 } 891 892 /* 893 * Load guest's/host's msr at nested entry/exit. 894 * return 0 for success, entry index for failure. 895 * 896 * One of the failure modes for MSR load/store is when a list exceeds the 897 * virtual hardware's capacity. To maintain compatibility with hardware inasmuch 898 * as possible, process all valid entries before failing rather than precheck 899 * for a capacity violation. 900 */ 901 static u32 nested_vmx_load_msr(struct kvm_vcpu *vcpu, u64 gpa, u32 count) 902 { 903 u32 i; 904 struct vmx_msr_entry e; 905 u32 max_msr_list_size = nested_vmx_max_atomic_switch_msrs(vcpu); 906 907 for (i = 0; i < count; i++) { 908 if (unlikely(i >= max_msr_list_size)) 909 goto fail; 910 911 if (kvm_vcpu_read_guest(vcpu, gpa + i * sizeof(e), 912 &e, sizeof(e))) { 913 pr_debug_ratelimited( 914 "%s cannot read MSR entry (%u, 0x%08llx)\n", 915 __func__, i, gpa + i * sizeof(e)); 916 goto fail; 917 } 918 if (nested_vmx_load_msr_check(vcpu, &e)) { 919 pr_debug_ratelimited( 920 "%s check failed (%u, 0x%x, 0x%x)\n", 921 __func__, i, e.index, e.reserved); 922 goto fail; 923 } 924 if (kvm_set_msr(vcpu, e.index, e.value)) { 925 pr_debug_ratelimited( 926 "%s cannot write MSR (%u, 0x%x, 0x%llx)\n", 927 __func__, i, e.index, e.value); 928 goto fail; 929 } 930 } 931 return 0; 932 fail: 933 /* Note, max_msr_list_size is at most 4096, i.e. this can't wrap. */ 934 return i + 1; 935 } 936 937 static bool nested_vmx_get_vmexit_msr_value(struct kvm_vcpu *vcpu, 938 u32 msr_index, 939 u64 *data) 940 { 941 struct vcpu_vmx *vmx = to_vmx(vcpu); 942 943 /* 944 * If the L0 hypervisor stored a more accurate value for the TSC that 945 * does not include the time taken for emulation of the L2->L1 946 * VM-exit in L0, use the more accurate value. 947 */ 948 if (msr_index == MSR_IA32_TSC) { 949 int i = vmx_find_loadstore_msr_slot(&vmx->msr_autostore.guest, 950 MSR_IA32_TSC); 951 952 if (i >= 0) { 953 u64 val = vmx->msr_autostore.guest.val[i].value; 954 955 *data = kvm_read_l1_tsc(vcpu, val); 956 return true; 957 } 958 } 959 960 if (kvm_get_msr(vcpu, msr_index, data)) { 961 pr_debug_ratelimited("%s cannot read MSR (0x%x)\n", __func__, 962 msr_index); 963 return false; 964 } 965 return true; 966 } 967 968 static bool read_and_check_msr_entry(struct kvm_vcpu *vcpu, u64 gpa, int i, 969 struct vmx_msr_entry *e) 970 { 971 if (kvm_vcpu_read_guest(vcpu, 972 gpa + i * sizeof(*e), 973 e, 2 * sizeof(u32))) { 974 pr_debug_ratelimited( 975 "%s cannot read MSR entry (%u, 0x%08llx)\n", 976 __func__, i, gpa + i * sizeof(*e)); 977 return false; 978 } 979 if (nested_vmx_store_msr_check(vcpu, e)) { 980 pr_debug_ratelimited( 981 "%s check failed (%u, 0x%x, 0x%x)\n", 982 __func__, i, e->index, e->reserved); 983 return false; 984 } 985 return true; 986 } 987 988 static int nested_vmx_store_msr(struct kvm_vcpu *vcpu, u64 gpa, u32 count) 989 { 990 u64 data; 991 u32 i; 992 struct vmx_msr_entry e; 993 u32 max_msr_list_size = nested_vmx_max_atomic_switch_msrs(vcpu); 994 995 for (i = 0; i < count; i++) { 996 if (unlikely(i >= max_msr_list_size)) 997 return -EINVAL; 998 999 if (!read_and_check_msr_entry(vcpu, gpa, i, &e)) 1000 return -EINVAL; 1001 1002 if (!nested_vmx_get_vmexit_msr_value(vcpu, e.index, &data)) 1003 return -EINVAL; 1004 1005 if (kvm_vcpu_write_guest(vcpu, 1006 gpa + i * sizeof(e) + 1007 offsetof(struct vmx_msr_entry, value), 1008 &data, sizeof(data))) { 1009 pr_debug_ratelimited( 1010 "%s cannot write MSR (%u, 0x%x, 0x%llx)\n", 1011 __func__, i, e.index, data); 1012 return -EINVAL; 1013 } 1014 } 1015 return 0; 1016 } 1017 1018 static bool nested_msr_store_list_has_msr(struct kvm_vcpu *vcpu, u32 msr_index) 1019 { 1020 struct vmcs12 *vmcs12 = get_vmcs12(vcpu); 1021 u32 count = vmcs12->vm_exit_msr_store_count; 1022 u64 gpa = vmcs12->vm_exit_msr_store_addr; 1023 struct vmx_msr_entry e; 1024 u32 i; 1025 1026 for (i = 0; i < count; i++) { 1027 if (!read_and_check_msr_entry(vcpu, gpa, i, &e)) 1028 return false; 1029 1030 if (e.index == msr_index) 1031 return true; 1032 } 1033 return false; 1034 } 1035 1036 static void prepare_vmx_msr_autostore_list(struct kvm_vcpu *vcpu, 1037 u32 msr_index) 1038 { 1039 struct vcpu_vmx *vmx = to_vmx(vcpu); 1040 struct vmx_msrs *autostore = &vmx->msr_autostore.guest; 1041 bool in_vmcs12_store_list; 1042 int msr_autostore_slot; 1043 bool in_autostore_list; 1044 int last; 1045 1046 msr_autostore_slot = vmx_find_loadstore_msr_slot(autostore, msr_index); 1047 in_autostore_list = msr_autostore_slot >= 0; 1048 in_vmcs12_store_list = nested_msr_store_list_has_msr(vcpu, msr_index); 1049 1050 if (in_vmcs12_store_list && !in_autostore_list) { 1051 if (autostore->nr == MAX_NR_LOADSTORE_MSRS) { 1052 /* 1053 * Emulated VMEntry does not fail here. Instead a less 1054 * accurate value will be returned by 1055 * nested_vmx_get_vmexit_msr_value() using kvm_get_msr() 1056 * instead of reading the value from the vmcs02 VMExit 1057 * MSR-store area. 1058 */ 1059 pr_warn_ratelimited( 1060 "Not enough msr entries in msr_autostore. Can't add msr %x\n", 1061 msr_index); 1062 return; 1063 } 1064 last = autostore->nr++; 1065 autostore->val[last].index = msr_index; 1066 } else if (!in_vmcs12_store_list && in_autostore_list) { 1067 last = --autostore->nr; 1068 autostore->val[msr_autostore_slot] = autostore->val[last]; 1069 } 1070 } 1071 1072 /* 1073 * Load guest's/host's cr3 at nested entry/exit. @nested_ept is true if we are 1074 * emulating VM-Entry into a guest with EPT enabled. On failure, the expected 1075 * Exit Qualification (for a VM-Entry consistency check VM-Exit) is assigned to 1076 * @entry_failure_code. 1077 */ 1078 static int nested_vmx_load_cr3(struct kvm_vcpu *vcpu, unsigned long cr3, 1079 bool nested_ept, bool reload_pdptrs, 1080 enum vm_entry_failure_code *entry_failure_code) 1081 { 1082 if (CC(kvm_vcpu_is_illegal_gpa(vcpu, cr3))) { 1083 *entry_failure_code = ENTRY_FAIL_DEFAULT; 1084 return -EINVAL; 1085 } 1086 1087 /* 1088 * If PAE paging and EPT are both on, CR3 is not used by the CPU and 1089 * must not be dereferenced. 1090 */ 1091 if (reload_pdptrs && !nested_ept && is_pae_paging(vcpu) && 1092 CC(!load_pdptrs(vcpu, cr3))) { 1093 *entry_failure_code = ENTRY_FAIL_PDPTE; 1094 return -EINVAL; 1095 } 1096 1097 vcpu->arch.cr3 = cr3; 1098 kvm_register_mark_dirty(vcpu, VCPU_EXREG_CR3); 1099 1100 /* Re-initialize the MMU, e.g. to pick up CR4 MMU role changes. */ 1101 kvm_init_mmu(vcpu); 1102 1103 if (!nested_ept) 1104 kvm_mmu_new_pgd(vcpu, cr3); 1105 1106 return 0; 1107 } 1108 1109 /* 1110 * Returns if KVM is able to config CPU to tag TLB entries 1111 * populated by L2 differently than TLB entries populated 1112 * by L1. 1113 * 1114 * If L0 uses EPT, L1 and L2 run with different EPTP because 1115 * guest_mode is part of kvm_mmu_page_role. Thus, TLB entries 1116 * are tagged with different EPTP. 1117 * 1118 * If L1 uses VPID and we allocated a vpid02, TLB entries are tagged 1119 * with different VPID (L1 entries are tagged with vmx->vpid 1120 * while L2 entries are tagged with vmx->nested.vpid02). 1121 */ 1122 static bool nested_has_guest_tlb_tag(struct kvm_vcpu *vcpu) 1123 { 1124 struct vmcs12 *vmcs12 = get_vmcs12(vcpu); 1125 1126 return enable_ept || 1127 (nested_cpu_has_vpid(vmcs12) && to_vmx(vcpu)->nested.vpid02); 1128 } 1129 1130 static void nested_vmx_transition_tlb_flush(struct kvm_vcpu *vcpu, 1131 struct vmcs12 *vmcs12, 1132 bool is_vmenter) 1133 { 1134 struct vcpu_vmx *vmx = to_vmx(vcpu); 1135 1136 /* 1137 * KVM_REQ_HV_TLB_FLUSH flushes entries from either L1's VP_ID or 1138 * L2's VP_ID upon request from the guest. Make sure we check for 1139 * pending entries in the right FIFO upon L1/L2 transition as these 1140 * requests are put by other vCPUs asynchronously. 1141 */ 1142 if (to_hv_vcpu(vcpu) && enable_ept) 1143 kvm_make_request(KVM_REQ_HV_TLB_FLUSH, vcpu); 1144 1145 /* 1146 * If vmcs12 doesn't use VPID, L1 expects linear and combined mappings 1147 * for *all* contexts to be flushed on VM-Enter/VM-Exit, i.e. it's a 1148 * full TLB flush from the guest's perspective. This is required even 1149 * if VPID is disabled in the host as KVM may need to synchronize the 1150 * MMU in response to the guest TLB flush. 1151 * 1152 * Note, using TLB_FLUSH_GUEST is correct even if nested EPT is in use. 1153 * EPT is a special snowflake, as guest-physical mappings aren't 1154 * flushed on VPID invalidations, including VM-Enter or VM-Exit with 1155 * VPID disabled. As a result, KVM _never_ needs to sync nEPT 1156 * entries on VM-Enter because L1 can't rely on VM-Enter to flush 1157 * those mappings. 1158 */ 1159 if (!nested_cpu_has_vpid(vmcs12)) { 1160 kvm_make_request(KVM_REQ_TLB_FLUSH_GUEST, vcpu); 1161 return; 1162 } 1163 1164 /* L2 should never have a VPID if VPID is disabled. */ 1165 WARN_ON(!enable_vpid); 1166 1167 /* 1168 * VPID is enabled and in use by vmcs12. If vpid12 is changing, then 1169 * emulate a guest TLB flush as KVM does not track vpid12 history nor 1170 * is the VPID incorporated into the MMU context. I.e. KVM must assume 1171 * that the new vpid12 has never been used and thus represents a new 1172 * guest ASID that cannot have entries in the TLB. 1173 */ 1174 if (is_vmenter && vmcs12->virtual_processor_id != vmx->nested.last_vpid) { 1175 vmx->nested.last_vpid = vmcs12->virtual_processor_id; 1176 kvm_make_request(KVM_REQ_TLB_FLUSH_GUEST, vcpu); 1177 return; 1178 } 1179 1180 /* 1181 * If VPID is enabled, used by vmc12, and vpid12 is not changing but 1182 * does not have a unique TLB tag (ASID), i.e. EPT is disabled and 1183 * KVM was unable to allocate a VPID for L2, flush the current context 1184 * as the effective ASID is common to both L1 and L2. 1185 */ 1186 if (!nested_has_guest_tlb_tag(vcpu)) 1187 kvm_make_request(KVM_REQ_TLB_FLUSH_CURRENT, vcpu); 1188 } 1189 1190 static bool is_bitwise_subset(u64 superset, u64 subset, u64 mask) 1191 { 1192 superset &= mask; 1193 subset &= mask; 1194 1195 return (superset | subset) == superset; 1196 } 1197 1198 static int vmx_restore_vmx_basic(struct vcpu_vmx *vmx, u64 data) 1199 { 1200 const u64 feature_and_reserved = 1201 /* feature (except bit 48; see below) */ 1202 BIT_ULL(49) | BIT_ULL(54) | BIT_ULL(55) | 1203 /* reserved */ 1204 BIT_ULL(31) | GENMASK_ULL(47, 45) | GENMASK_ULL(63, 56); 1205 u64 vmx_basic = vmcs_config.nested.basic; 1206 1207 if (!is_bitwise_subset(vmx_basic, data, feature_and_reserved)) 1208 return -EINVAL; 1209 1210 /* 1211 * KVM does not emulate a version of VMX that constrains physical 1212 * addresses of VMX structures (e.g. VMCS) to 32-bits. 1213 */ 1214 if (data & BIT_ULL(48)) 1215 return -EINVAL; 1216 1217 if (vmx_basic_vmcs_revision_id(vmx_basic) != 1218 vmx_basic_vmcs_revision_id(data)) 1219 return -EINVAL; 1220 1221 if (vmx_basic_vmcs_size(vmx_basic) > vmx_basic_vmcs_size(data)) 1222 return -EINVAL; 1223 1224 vmx->nested.msrs.basic = data; 1225 return 0; 1226 } 1227 1228 static void vmx_get_control_msr(struct nested_vmx_msrs *msrs, u32 msr_index, 1229 u32 **low, u32 **high) 1230 { 1231 switch (msr_index) { 1232 case MSR_IA32_VMX_TRUE_PINBASED_CTLS: 1233 *low = &msrs->pinbased_ctls_low; 1234 *high = &msrs->pinbased_ctls_high; 1235 break; 1236 case MSR_IA32_VMX_TRUE_PROCBASED_CTLS: 1237 *low = &msrs->procbased_ctls_low; 1238 *high = &msrs->procbased_ctls_high; 1239 break; 1240 case MSR_IA32_VMX_TRUE_EXIT_CTLS: 1241 *low = &msrs->exit_ctls_low; 1242 *high = &msrs->exit_ctls_high; 1243 break; 1244 case MSR_IA32_VMX_TRUE_ENTRY_CTLS: 1245 *low = &msrs->entry_ctls_low; 1246 *high = &msrs->entry_ctls_high; 1247 break; 1248 case MSR_IA32_VMX_PROCBASED_CTLS2: 1249 *low = &msrs->secondary_ctls_low; 1250 *high = &msrs->secondary_ctls_high; 1251 break; 1252 default: 1253 BUG(); 1254 } 1255 } 1256 1257 static int 1258 vmx_restore_control_msr(struct vcpu_vmx *vmx, u32 msr_index, u64 data) 1259 { 1260 u32 *lowp, *highp; 1261 u64 supported; 1262 1263 vmx_get_control_msr(&vmcs_config.nested, msr_index, &lowp, &highp); 1264 1265 supported = vmx_control_msr(*lowp, *highp); 1266 1267 /* Check must-be-1 bits are still 1. */ 1268 if (!is_bitwise_subset(data, supported, GENMASK_ULL(31, 0))) 1269 return -EINVAL; 1270 1271 /* Check must-be-0 bits are still 0. */ 1272 if (!is_bitwise_subset(supported, data, GENMASK_ULL(63, 32))) 1273 return -EINVAL; 1274 1275 vmx_get_control_msr(&vmx->nested.msrs, msr_index, &lowp, &highp); 1276 *lowp = data; 1277 *highp = data >> 32; 1278 return 0; 1279 } 1280 1281 static int vmx_restore_vmx_misc(struct vcpu_vmx *vmx, u64 data) 1282 { 1283 const u64 feature_and_reserved_bits = 1284 /* feature */ 1285 BIT_ULL(5) | GENMASK_ULL(8, 6) | BIT_ULL(14) | BIT_ULL(15) | 1286 BIT_ULL(28) | BIT_ULL(29) | BIT_ULL(30) | 1287 /* reserved */ 1288 GENMASK_ULL(13, 9) | BIT_ULL(31); 1289 u64 vmx_misc = vmx_control_msr(vmcs_config.nested.misc_low, 1290 vmcs_config.nested.misc_high); 1291 1292 if (!is_bitwise_subset(vmx_misc, data, feature_and_reserved_bits)) 1293 return -EINVAL; 1294 1295 if ((vmx->nested.msrs.pinbased_ctls_high & 1296 PIN_BASED_VMX_PREEMPTION_TIMER) && 1297 vmx_misc_preemption_timer_rate(data) != 1298 vmx_misc_preemption_timer_rate(vmx_misc)) 1299 return -EINVAL; 1300 1301 if (vmx_misc_cr3_count(data) > vmx_misc_cr3_count(vmx_misc)) 1302 return -EINVAL; 1303 1304 if (vmx_misc_max_msr(data) > vmx_misc_max_msr(vmx_misc)) 1305 return -EINVAL; 1306 1307 if (vmx_misc_mseg_revid(data) != vmx_misc_mseg_revid(vmx_misc)) 1308 return -EINVAL; 1309 1310 vmx->nested.msrs.misc_low = data; 1311 vmx->nested.msrs.misc_high = data >> 32; 1312 1313 return 0; 1314 } 1315 1316 static int vmx_restore_vmx_ept_vpid_cap(struct vcpu_vmx *vmx, u64 data) 1317 { 1318 u64 vmx_ept_vpid_cap = vmx_control_msr(vmcs_config.nested.ept_caps, 1319 vmcs_config.nested.vpid_caps); 1320 1321 /* Every bit is either reserved or a feature bit. */ 1322 if (!is_bitwise_subset(vmx_ept_vpid_cap, data, -1ULL)) 1323 return -EINVAL; 1324 1325 vmx->nested.msrs.ept_caps = data; 1326 vmx->nested.msrs.vpid_caps = data >> 32; 1327 return 0; 1328 } 1329 1330 static u64 *vmx_get_fixed0_msr(struct nested_vmx_msrs *msrs, u32 msr_index) 1331 { 1332 switch (msr_index) { 1333 case MSR_IA32_VMX_CR0_FIXED0: 1334 return &msrs->cr0_fixed0; 1335 case MSR_IA32_VMX_CR4_FIXED0: 1336 return &msrs->cr4_fixed0; 1337 default: 1338 BUG(); 1339 } 1340 } 1341 1342 static int vmx_restore_fixed0_msr(struct vcpu_vmx *vmx, u32 msr_index, u64 data) 1343 { 1344 const u64 *msr = vmx_get_fixed0_msr(&vmcs_config.nested, msr_index); 1345 1346 /* 1347 * 1 bits (which indicates bits which "must-be-1" during VMX operation) 1348 * must be 1 in the restored value. 1349 */ 1350 if (!is_bitwise_subset(data, *msr, -1ULL)) 1351 return -EINVAL; 1352 1353 *vmx_get_fixed0_msr(&vmx->nested.msrs, msr_index) = data; 1354 return 0; 1355 } 1356 1357 /* 1358 * Called when userspace is restoring VMX MSRs. 1359 * 1360 * Returns 0 on success, non-0 otherwise. 1361 */ 1362 int vmx_set_vmx_msr(struct kvm_vcpu *vcpu, u32 msr_index, u64 data) 1363 { 1364 struct vcpu_vmx *vmx = to_vmx(vcpu); 1365 1366 /* 1367 * Don't allow changes to the VMX capability MSRs while the vCPU 1368 * is in VMX operation. 1369 */ 1370 if (vmx->nested.vmxon) 1371 return -EBUSY; 1372 1373 switch (msr_index) { 1374 case MSR_IA32_VMX_BASIC: 1375 return vmx_restore_vmx_basic(vmx, data); 1376 case MSR_IA32_VMX_PINBASED_CTLS: 1377 case MSR_IA32_VMX_PROCBASED_CTLS: 1378 case MSR_IA32_VMX_EXIT_CTLS: 1379 case MSR_IA32_VMX_ENTRY_CTLS: 1380 /* 1381 * The "non-true" VMX capability MSRs are generated from the 1382 * "true" MSRs, so we do not support restoring them directly. 1383 * 1384 * If userspace wants to emulate VMX_BASIC[55]=0, userspace 1385 * should restore the "true" MSRs with the must-be-1 bits 1386 * set according to the SDM Vol 3. A.2 "RESERVED CONTROLS AND 1387 * DEFAULT SETTINGS". 1388 */ 1389 return -EINVAL; 1390 case MSR_IA32_VMX_TRUE_PINBASED_CTLS: 1391 case MSR_IA32_VMX_TRUE_PROCBASED_CTLS: 1392 case MSR_IA32_VMX_TRUE_EXIT_CTLS: 1393 case MSR_IA32_VMX_TRUE_ENTRY_CTLS: 1394 case MSR_IA32_VMX_PROCBASED_CTLS2: 1395 return vmx_restore_control_msr(vmx, msr_index, data); 1396 case MSR_IA32_VMX_MISC: 1397 return vmx_restore_vmx_misc(vmx, data); 1398 case MSR_IA32_VMX_CR0_FIXED0: 1399 case MSR_IA32_VMX_CR4_FIXED0: 1400 return vmx_restore_fixed0_msr(vmx, msr_index, data); 1401 case MSR_IA32_VMX_CR0_FIXED1: 1402 case MSR_IA32_VMX_CR4_FIXED1: 1403 /* 1404 * These MSRs are generated based on the vCPU's CPUID, so we 1405 * do not support restoring them directly. 1406 */ 1407 return -EINVAL; 1408 case MSR_IA32_VMX_EPT_VPID_CAP: 1409 return vmx_restore_vmx_ept_vpid_cap(vmx, data); 1410 case MSR_IA32_VMX_VMCS_ENUM: 1411 vmx->nested.msrs.vmcs_enum = data; 1412 return 0; 1413 case MSR_IA32_VMX_VMFUNC: 1414 if (data & ~vmcs_config.nested.vmfunc_controls) 1415 return -EINVAL; 1416 vmx->nested.msrs.vmfunc_controls = data; 1417 return 0; 1418 default: 1419 /* 1420 * The rest of the VMX capability MSRs do not support restore. 1421 */ 1422 return -EINVAL; 1423 } 1424 } 1425 1426 /* Returns 0 on success, non-0 otherwise. */ 1427 int vmx_get_vmx_msr(struct nested_vmx_msrs *msrs, u32 msr_index, u64 *pdata) 1428 { 1429 switch (msr_index) { 1430 case MSR_IA32_VMX_BASIC: 1431 *pdata = msrs->basic; 1432 break; 1433 case MSR_IA32_VMX_TRUE_PINBASED_CTLS: 1434 case MSR_IA32_VMX_PINBASED_CTLS: 1435 *pdata = vmx_control_msr( 1436 msrs->pinbased_ctls_low, 1437 msrs->pinbased_ctls_high); 1438 if (msr_index == MSR_IA32_VMX_PINBASED_CTLS) 1439 *pdata |= PIN_BASED_ALWAYSON_WITHOUT_TRUE_MSR; 1440 break; 1441 case MSR_IA32_VMX_TRUE_PROCBASED_CTLS: 1442 case MSR_IA32_VMX_PROCBASED_CTLS: 1443 *pdata = vmx_control_msr( 1444 msrs->procbased_ctls_low, 1445 msrs->procbased_ctls_high); 1446 if (msr_index == MSR_IA32_VMX_PROCBASED_CTLS) 1447 *pdata |= CPU_BASED_ALWAYSON_WITHOUT_TRUE_MSR; 1448 break; 1449 case MSR_IA32_VMX_TRUE_EXIT_CTLS: 1450 case MSR_IA32_VMX_EXIT_CTLS: 1451 *pdata = vmx_control_msr( 1452 msrs->exit_ctls_low, 1453 msrs->exit_ctls_high); 1454 if (msr_index == MSR_IA32_VMX_EXIT_CTLS) 1455 *pdata |= VM_EXIT_ALWAYSON_WITHOUT_TRUE_MSR; 1456 break; 1457 case MSR_IA32_VMX_TRUE_ENTRY_CTLS: 1458 case MSR_IA32_VMX_ENTRY_CTLS: 1459 *pdata = vmx_control_msr( 1460 msrs->entry_ctls_low, 1461 msrs->entry_ctls_high); 1462 if (msr_index == MSR_IA32_VMX_ENTRY_CTLS) 1463 *pdata |= VM_ENTRY_ALWAYSON_WITHOUT_TRUE_MSR; 1464 break; 1465 case MSR_IA32_VMX_MISC: 1466 *pdata = vmx_control_msr( 1467 msrs->misc_low, 1468 msrs->misc_high); 1469 break; 1470 case MSR_IA32_VMX_CR0_FIXED0: 1471 *pdata = msrs->cr0_fixed0; 1472 break; 1473 case MSR_IA32_VMX_CR0_FIXED1: 1474 *pdata = msrs->cr0_fixed1; 1475 break; 1476 case MSR_IA32_VMX_CR4_FIXED0: 1477 *pdata = msrs->cr4_fixed0; 1478 break; 1479 case MSR_IA32_VMX_CR4_FIXED1: 1480 *pdata = msrs->cr4_fixed1; 1481 break; 1482 case MSR_IA32_VMX_VMCS_ENUM: 1483 *pdata = msrs->vmcs_enum; 1484 break; 1485 case MSR_IA32_VMX_PROCBASED_CTLS2: 1486 *pdata = vmx_control_msr( 1487 msrs->secondary_ctls_low, 1488 msrs->secondary_ctls_high); 1489 break; 1490 case MSR_IA32_VMX_EPT_VPID_CAP: 1491 *pdata = msrs->ept_caps | 1492 ((u64)msrs->vpid_caps << 32); 1493 break; 1494 case MSR_IA32_VMX_VMFUNC: 1495 *pdata = msrs->vmfunc_controls; 1496 break; 1497 default: 1498 return 1; 1499 } 1500 1501 return 0; 1502 } 1503 1504 /* 1505 * Copy the writable VMCS shadow fields back to the VMCS12, in case they have 1506 * been modified by the L1 guest. Note, "writable" in this context means 1507 * "writable by the guest", i.e. tagged SHADOW_FIELD_RW; the set of 1508 * fields tagged SHADOW_FIELD_RO may or may not align with the "read-only" 1509 * VM-exit information fields (which are actually writable if the vCPU is 1510 * configured to support "VMWRITE to any supported field in the VMCS"). 1511 */ 1512 static void copy_shadow_to_vmcs12(struct vcpu_vmx *vmx) 1513 { 1514 struct vmcs *shadow_vmcs = vmx->vmcs01.shadow_vmcs; 1515 struct vmcs12 *vmcs12 = get_vmcs12(&vmx->vcpu); 1516 struct shadow_vmcs_field field; 1517 unsigned long val; 1518 int i; 1519 1520 if (WARN_ON(!shadow_vmcs)) 1521 return; 1522 1523 preempt_disable(); 1524 1525 vmcs_load(shadow_vmcs); 1526 1527 for (i = 0; i < max_shadow_read_write_fields; i++) { 1528 field = shadow_read_write_fields[i]; 1529 val = __vmcs_readl(field.encoding); 1530 vmcs12_write_any(vmcs12, field.encoding, field.offset, val); 1531 } 1532 1533 vmcs_clear(shadow_vmcs); 1534 vmcs_load(vmx->loaded_vmcs->vmcs); 1535 1536 preempt_enable(); 1537 } 1538 1539 static void copy_vmcs12_to_shadow(struct vcpu_vmx *vmx) 1540 { 1541 const struct shadow_vmcs_field *fields[] = { 1542 shadow_read_write_fields, 1543 shadow_read_only_fields 1544 }; 1545 const int max_fields[] = { 1546 max_shadow_read_write_fields, 1547 max_shadow_read_only_fields 1548 }; 1549 struct vmcs *shadow_vmcs = vmx->vmcs01.shadow_vmcs; 1550 struct vmcs12 *vmcs12 = get_vmcs12(&vmx->vcpu); 1551 struct shadow_vmcs_field field; 1552 unsigned long val; 1553 int i, q; 1554 1555 if (WARN_ON(!shadow_vmcs)) 1556 return; 1557 1558 vmcs_load(shadow_vmcs); 1559 1560 for (q = 0; q < ARRAY_SIZE(fields); q++) { 1561 for (i = 0; i < max_fields[q]; i++) { 1562 field = fields[q][i]; 1563 val = vmcs12_read_any(vmcs12, field.encoding, 1564 field.offset); 1565 __vmcs_writel(field.encoding, val); 1566 } 1567 } 1568 1569 vmcs_clear(shadow_vmcs); 1570 vmcs_load(vmx->loaded_vmcs->vmcs); 1571 } 1572 1573 static void copy_enlightened_to_vmcs12(struct vcpu_vmx *vmx, u32 hv_clean_fields) 1574 { 1575 struct vmcs12 *vmcs12 = vmx->nested.cached_vmcs12; 1576 struct hv_enlightened_vmcs *evmcs = vmx->nested.hv_evmcs; 1577 struct kvm_vcpu_hv *hv_vcpu = to_hv_vcpu(&vmx->vcpu); 1578 1579 /* HV_VMX_ENLIGHTENED_CLEAN_FIELD_NONE */ 1580 vmcs12->tpr_threshold = evmcs->tpr_threshold; 1581 vmcs12->guest_rip = evmcs->guest_rip; 1582 1583 if (unlikely(!(hv_clean_fields & 1584 HV_VMX_ENLIGHTENED_CLEAN_FIELD_ENLIGHTENMENTSCONTROL))) { 1585 hv_vcpu->nested.pa_page_gpa = evmcs->partition_assist_page; 1586 hv_vcpu->nested.vm_id = evmcs->hv_vm_id; 1587 hv_vcpu->nested.vp_id = evmcs->hv_vp_id; 1588 } 1589 1590 if (unlikely(!(hv_clean_fields & 1591 HV_VMX_ENLIGHTENED_CLEAN_FIELD_GUEST_BASIC))) { 1592 vmcs12->guest_rsp = evmcs->guest_rsp; 1593 vmcs12->guest_rflags = evmcs->guest_rflags; 1594 vmcs12->guest_interruptibility_info = 1595 evmcs->guest_interruptibility_info; 1596 /* 1597 * Not present in struct vmcs12: 1598 * vmcs12->guest_ssp = evmcs->guest_ssp; 1599 */ 1600 } 1601 1602 if (unlikely(!(hv_clean_fields & 1603 HV_VMX_ENLIGHTENED_CLEAN_FIELD_CONTROL_PROC))) { 1604 vmcs12->cpu_based_vm_exec_control = 1605 evmcs->cpu_based_vm_exec_control; 1606 } 1607 1608 if (unlikely(!(hv_clean_fields & 1609 HV_VMX_ENLIGHTENED_CLEAN_FIELD_CONTROL_EXCPN))) { 1610 vmcs12->exception_bitmap = evmcs->exception_bitmap; 1611 } 1612 1613 if (unlikely(!(hv_clean_fields & 1614 HV_VMX_ENLIGHTENED_CLEAN_FIELD_CONTROL_ENTRY))) { 1615 vmcs12->vm_entry_controls = evmcs->vm_entry_controls; 1616 } 1617 1618 if (unlikely(!(hv_clean_fields & 1619 HV_VMX_ENLIGHTENED_CLEAN_FIELD_CONTROL_EVENT))) { 1620 vmcs12->vm_entry_intr_info_field = 1621 evmcs->vm_entry_intr_info_field; 1622 vmcs12->vm_entry_exception_error_code = 1623 evmcs->vm_entry_exception_error_code; 1624 vmcs12->vm_entry_instruction_len = 1625 evmcs->vm_entry_instruction_len; 1626 } 1627 1628 if (unlikely(!(hv_clean_fields & 1629 HV_VMX_ENLIGHTENED_CLEAN_FIELD_HOST_GRP1))) { 1630 vmcs12->host_ia32_pat = evmcs->host_ia32_pat; 1631 vmcs12->host_ia32_efer = evmcs->host_ia32_efer; 1632 vmcs12->host_cr0 = evmcs->host_cr0; 1633 vmcs12->host_cr3 = evmcs->host_cr3; 1634 vmcs12->host_cr4 = evmcs->host_cr4; 1635 vmcs12->host_ia32_sysenter_esp = evmcs->host_ia32_sysenter_esp; 1636 vmcs12->host_ia32_sysenter_eip = evmcs->host_ia32_sysenter_eip; 1637 vmcs12->host_rip = evmcs->host_rip; 1638 vmcs12->host_ia32_sysenter_cs = evmcs->host_ia32_sysenter_cs; 1639 vmcs12->host_es_selector = evmcs->host_es_selector; 1640 vmcs12->host_cs_selector = evmcs->host_cs_selector; 1641 vmcs12->host_ss_selector = evmcs->host_ss_selector; 1642 vmcs12->host_ds_selector = evmcs->host_ds_selector; 1643 vmcs12->host_fs_selector = evmcs->host_fs_selector; 1644 vmcs12->host_gs_selector = evmcs->host_gs_selector; 1645 vmcs12->host_tr_selector = evmcs->host_tr_selector; 1646 vmcs12->host_ia32_perf_global_ctrl = evmcs->host_ia32_perf_global_ctrl; 1647 /* 1648 * Not present in struct vmcs12: 1649 * vmcs12->host_ia32_s_cet = evmcs->host_ia32_s_cet; 1650 * vmcs12->host_ssp = evmcs->host_ssp; 1651 * vmcs12->host_ia32_int_ssp_table_addr = evmcs->host_ia32_int_ssp_table_addr; 1652 */ 1653 } 1654 1655 if (unlikely(!(hv_clean_fields & 1656 HV_VMX_ENLIGHTENED_CLEAN_FIELD_CONTROL_GRP1))) { 1657 vmcs12->pin_based_vm_exec_control = 1658 evmcs->pin_based_vm_exec_control; 1659 vmcs12->vm_exit_controls = evmcs->vm_exit_controls; 1660 vmcs12->secondary_vm_exec_control = 1661 evmcs->secondary_vm_exec_control; 1662 } 1663 1664 if (unlikely(!(hv_clean_fields & 1665 HV_VMX_ENLIGHTENED_CLEAN_FIELD_IO_BITMAP))) { 1666 vmcs12->io_bitmap_a = evmcs->io_bitmap_a; 1667 vmcs12->io_bitmap_b = evmcs->io_bitmap_b; 1668 } 1669 1670 if (unlikely(!(hv_clean_fields & 1671 HV_VMX_ENLIGHTENED_CLEAN_FIELD_MSR_BITMAP))) { 1672 vmcs12->msr_bitmap = evmcs->msr_bitmap; 1673 } 1674 1675 if (unlikely(!(hv_clean_fields & 1676 HV_VMX_ENLIGHTENED_CLEAN_FIELD_GUEST_GRP2))) { 1677 vmcs12->guest_es_base = evmcs->guest_es_base; 1678 vmcs12->guest_cs_base = evmcs->guest_cs_base; 1679 vmcs12->guest_ss_base = evmcs->guest_ss_base; 1680 vmcs12->guest_ds_base = evmcs->guest_ds_base; 1681 vmcs12->guest_fs_base = evmcs->guest_fs_base; 1682 vmcs12->guest_gs_base = evmcs->guest_gs_base; 1683 vmcs12->guest_ldtr_base = evmcs->guest_ldtr_base; 1684 vmcs12->guest_tr_base = evmcs->guest_tr_base; 1685 vmcs12->guest_gdtr_base = evmcs->guest_gdtr_base; 1686 vmcs12->guest_idtr_base = evmcs->guest_idtr_base; 1687 vmcs12->guest_es_limit = evmcs->guest_es_limit; 1688 vmcs12->guest_cs_limit = evmcs->guest_cs_limit; 1689 vmcs12->guest_ss_limit = evmcs->guest_ss_limit; 1690 vmcs12->guest_ds_limit = evmcs->guest_ds_limit; 1691 vmcs12->guest_fs_limit = evmcs->guest_fs_limit; 1692 vmcs12->guest_gs_limit = evmcs->guest_gs_limit; 1693 vmcs12->guest_ldtr_limit = evmcs->guest_ldtr_limit; 1694 vmcs12->guest_tr_limit = evmcs->guest_tr_limit; 1695 vmcs12->guest_gdtr_limit = evmcs->guest_gdtr_limit; 1696 vmcs12->guest_idtr_limit = evmcs->guest_idtr_limit; 1697 vmcs12->guest_es_ar_bytes = evmcs->guest_es_ar_bytes; 1698 vmcs12->guest_cs_ar_bytes = evmcs->guest_cs_ar_bytes; 1699 vmcs12->guest_ss_ar_bytes = evmcs->guest_ss_ar_bytes; 1700 vmcs12->guest_ds_ar_bytes = evmcs->guest_ds_ar_bytes; 1701 vmcs12->guest_fs_ar_bytes = evmcs->guest_fs_ar_bytes; 1702 vmcs12->guest_gs_ar_bytes = evmcs->guest_gs_ar_bytes; 1703 vmcs12->guest_ldtr_ar_bytes = evmcs->guest_ldtr_ar_bytes; 1704 vmcs12->guest_tr_ar_bytes = evmcs->guest_tr_ar_bytes; 1705 vmcs12->guest_es_selector = evmcs->guest_es_selector; 1706 vmcs12->guest_cs_selector = evmcs->guest_cs_selector; 1707 vmcs12->guest_ss_selector = evmcs->guest_ss_selector; 1708 vmcs12->guest_ds_selector = evmcs->guest_ds_selector; 1709 vmcs12->guest_fs_selector = evmcs->guest_fs_selector; 1710 vmcs12->guest_gs_selector = evmcs->guest_gs_selector; 1711 vmcs12->guest_ldtr_selector = evmcs->guest_ldtr_selector; 1712 vmcs12->guest_tr_selector = evmcs->guest_tr_selector; 1713 } 1714 1715 if (unlikely(!(hv_clean_fields & 1716 HV_VMX_ENLIGHTENED_CLEAN_FIELD_CONTROL_GRP2))) { 1717 vmcs12->tsc_offset = evmcs->tsc_offset; 1718 vmcs12->virtual_apic_page_addr = evmcs->virtual_apic_page_addr; 1719 vmcs12->xss_exit_bitmap = evmcs->xss_exit_bitmap; 1720 vmcs12->encls_exiting_bitmap = evmcs->encls_exiting_bitmap; 1721 vmcs12->tsc_multiplier = evmcs->tsc_multiplier; 1722 } 1723 1724 if (unlikely(!(hv_clean_fields & 1725 HV_VMX_ENLIGHTENED_CLEAN_FIELD_CRDR))) { 1726 vmcs12->cr0_guest_host_mask = evmcs->cr0_guest_host_mask; 1727 vmcs12->cr4_guest_host_mask = evmcs->cr4_guest_host_mask; 1728 vmcs12->cr0_read_shadow = evmcs->cr0_read_shadow; 1729 vmcs12->cr4_read_shadow = evmcs->cr4_read_shadow; 1730 vmcs12->guest_cr0 = evmcs->guest_cr0; 1731 vmcs12->guest_cr3 = evmcs->guest_cr3; 1732 vmcs12->guest_cr4 = evmcs->guest_cr4; 1733 vmcs12->guest_dr7 = evmcs->guest_dr7; 1734 } 1735 1736 if (unlikely(!(hv_clean_fields & 1737 HV_VMX_ENLIGHTENED_CLEAN_FIELD_HOST_POINTER))) { 1738 vmcs12->host_fs_base = evmcs->host_fs_base; 1739 vmcs12->host_gs_base = evmcs->host_gs_base; 1740 vmcs12->host_tr_base = evmcs->host_tr_base; 1741 vmcs12->host_gdtr_base = evmcs->host_gdtr_base; 1742 vmcs12->host_idtr_base = evmcs->host_idtr_base; 1743 vmcs12->host_rsp = evmcs->host_rsp; 1744 } 1745 1746 if (unlikely(!(hv_clean_fields & 1747 HV_VMX_ENLIGHTENED_CLEAN_FIELD_CONTROL_XLAT))) { 1748 vmcs12->ept_pointer = evmcs->ept_pointer; 1749 vmcs12->virtual_processor_id = evmcs->virtual_processor_id; 1750 } 1751 1752 if (unlikely(!(hv_clean_fields & 1753 HV_VMX_ENLIGHTENED_CLEAN_FIELD_GUEST_GRP1))) { 1754 vmcs12->vmcs_link_pointer = evmcs->vmcs_link_pointer; 1755 vmcs12->guest_ia32_debugctl = evmcs->guest_ia32_debugctl; 1756 vmcs12->guest_ia32_pat = evmcs->guest_ia32_pat; 1757 vmcs12->guest_ia32_efer = evmcs->guest_ia32_efer; 1758 vmcs12->guest_pdptr0 = evmcs->guest_pdptr0; 1759 vmcs12->guest_pdptr1 = evmcs->guest_pdptr1; 1760 vmcs12->guest_pdptr2 = evmcs->guest_pdptr2; 1761 vmcs12->guest_pdptr3 = evmcs->guest_pdptr3; 1762 vmcs12->guest_pending_dbg_exceptions = 1763 evmcs->guest_pending_dbg_exceptions; 1764 vmcs12->guest_sysenter_esp = evmcs->guest_sysenter_esp; 1765 vmcs12->guest_sysenter_eip = evmcs->guest_sysenter_eip; 1766 vmcs12->guest_bndcfgs = evmcs->guest_bndcfgs; 1767 vmcs12->guest_activity_state = evmcs->guest_activity_state; 1768 vmcs12->guest_sysenter_cs = evmcs->guest_sysenter_cs; 1769 vmcs12->guest_ia32_perf_global_ctrl = evmcs->guest_ia32_perf_global_ctrl; 1770 /* 1771 * Not present in struct vmcs12: 1772 * vmcs12->guest_ia32_s_cet = evmcs->guest_ia32_s_cet; 1773 * vmcs12->guest_ia32_lbr_ctl = evmcs->guest_ia32_lbr_ctl; 1774 * vmcs12->guest_ia32_int_ssp_table_addr = evmcs->guest_ia32_int_ssp_table_addr; 1775 */ 1776 } 1777 1778 /* 1779 * Not used? 1780 * vmcs12->vm_exit_msr_store_addr = evmcs->vm_exit_msr_store_addr; 1781 * vmcs12->vm_exit_msr_load_addr = evmcs->vm_exit_msr_load_addr; 1782 * vmcs12->vm_entry_msr_load_addr = evmcs->vm_entry_msr_load_addr; 1783 * vmcs12->page_fault_error_code_mask = 1784 * evmcs->page_fault_error_code_mask; 1785 * vmcs12->page_fault_error_code_match = 1786 * evmcs->page_fault_error_code_match; 1787 * vmcs12->cr3_target_count = evmcs->cr3_target_count; 1788 * vmcs12->vm_exit_msr_store_count = evmcs->vm_exit_msr_store_count; 1789 * vmcs12->vm_exit_msr_load_count = evmcs->vm_exit_msr_load_count; 1790 * vmcs12->vm_entry_msr_load_count = evmcs->vm_entry_msr_load_count; 1791 */ 1792 1793 /* 1794 * Read only fields: 1795 * vmcs12->guest_physical_address = evmcs->guest_physical_address; 1796 * vmcs12->vm_instruction_error = evmcs->vm_instruction_error; 1797 * vmcs12->vm_exit_reason = evmcs->vm_exit_reason; 1798 * vmcs12->vm_exit_intr_info = evmcs->vm_exit_intr_info; 1799 * vmcs12->vm_exit_intr_error_code = evmcs->vm_exit_intr_error_code; 1800 * vmcs12->idt_vectoring_info_field = evmcs->idt_vectoring_info_field; 1801 * vmcs12->idt_vectoring_error_code = evmcs->idt_vectoring_error_code; 1802 * vmcs12->vm_exit_instruction_len = evmcs->vm_exit_instruction_len; 1803 * vmcs12->vmx_instruction_info = evmcs->vmx_instruction_info; 1804 * vmcs12->exit_qualification = evmcs->exit_qualification; 1805 * vmcs12->guest_linear_address = evmcs->guest_linear_address; 1806 * 1807 * Not present in struct vmcs12: 1808 * vmcs12->exit_io_instruction_ecx = evmcs->exit_io_instruction_ecx; 1809 * vmcs12->exit_io_instruction_esi = evmcs->exit_io_instruction_esi; 1810 * vmcs12->exit_io_instruction_edi = evmcs->exit_io_instruction_edi; 1811 * vmcs12->exit_io_instruction_eip = evmcs->exit_io_instruction_eip; 1812 */ 1813 1814 return; 1815 } 1816 1817 static void copy_vmcs12_to_enlightened(struct vcpu_vmx *vmx) 1818 { 1819 struct vmcs12 *vmcs12 = vmx->nested.cached_vmcs12; 1820 struct hv_enlightened_vmcs *evmcs = vmx->nested.hv_evmcs; 1821 1822 /* 1823 * Should not be changed by KVM: 1824 * 1825 * evmcs->host_es_selector = vmcs12->host_es_selector; 1826 * evmcs->host_cs_selector = vmcs12->host_cs_selector; 1827 * evmcs->host_ss_selector = vmcs12->host_ss_selector; 1828 * evmcs->host_ds_selector = vmcs12->host_ds_selector; 1829 * evmcs->host_fs_selector = vmcs12->host_fs_selector; 1830 * evmcs->host_gs_selector = vmcs12->host_gs_selector; 1831 * evmcs->host_tr_selector = vmcs12->host_tr_selector; 1832 * evmcs->host_ia32_pat = vmcs12->host_ia32_pat; 1833 * evmcs->host_ia32_efer = vmcs12->host_ia32_efer; 1834 * evmcs->host_cr0 = vmcs12->host_cr0; 1835 * evmcs->host_cr3 = vmcs12->host_cr3; 1836 * evmcs->host_cr4 = vmcs12->host_cr4; 1837 * evmcs->host_ia32_sysenter_esp = vmcs12->host_ia32_sysenter_esp; 1838 * evmcs->host_ia32_sysenter_eip = vmcs12->host_ia32_sysenter_eip; 1839 * evmcs->host_rip = vmcs12->host_rip; 1840 * evmcs->host_ia32_sysenter_cs = vmcs12->host_ia32_sysenter_cs; 1841 * evmcs->host_fs_base = vmcs12->host_fs_base; 1842 * evmcs->host_gs_base = vmcs12->host_gs_base; 1843 * evmcs->host_tr_base = vmcs12->host_tr_base; 1844 * evmcs->host_gdtr_base = vmcs12->host_gdtr_base; 1845 * evmcs->host_idtr_base = vmcs12->host_idtr_base; 1846 * evmcs->host_rsp = vmcs12->host_rsp; 1847 * sync_vmcs02_to_vmcs12() doesn't read these: 1848 * evmcs->io_bitmap_a = vmcs12->io_bitmap_a; 1849 * evmcs->io_bitmap_b = vmcs12->io_bitmap_b; 1850 * evmcs->msr_bitmap = vmcs12->msr_bitmap; 1851 * evmcs->ept_pointer = vmcs12->ept_pointer; 1852 * evmcs->xss_exit_bitmap = vmcs12->xss_exit_bitmap; 1853 * evmcs->vm_exit_msr_store_addr = vmcs12->vm_exit_msr_store_addr; 1854 * evmcs->vm_exit_msr_load_addr = vmcs12->vm_exit_msr_load_addr; 1855 * evmcs->vm_entry_msr_load_addr = vmcs12->vm_entry_msr_load_addr; 1856 * evmcs->tpr_threshold = vmcs12->tpr_threshold; 1857 * evmcs->virtual_processor_id = vmcs12->virtual_processor_id; 1858 * evmcs->exception_bitmap = vmcs12->exception_bitmap; 1859 * evmcs->vmcs_link_pointer = vmcs12->vmcs_link_pointer; 1860 * evmcs->pin_based_vm_exec_control = vmcs12->pin_based_vm_exec_control; 1861 * evmcs->vm_exit_controls = vmcs12->vm_exit_controls; 1862 * evmcs->secondary_vm_exec_control = vmcs12->secondary_vm_exec_control; 1863 * evmcs->page_fault_error_code_mask = 1864 * vmcs12->page_fault_error_code_mask; 1865 * evmcs->page_fault_error_code_match = 1866 * vmcs12->page_fault_error_code_match; 1867 * evmcs->cr3_target_count = vmcs12->cr3_target_count; 1868 * evmcs->virtual_apic_page_addr = vmcs12->virtual_apic_page_addr; 1869 * evmcs->tsc_offset = vmcs12->tsc_offset; 1870 * evmcs->guest_ia32_debugctl = vmcs12->guest_ia32_debugctl; 1871 * evmcs->cr0_guest_host_mask = vmcs12->cr0_guest_host_mask; 1872 * evmcs->cr4_guest_host_mask = vmcs12->cr4_guest_host_mask; 1873 * evmcs->cr0_read_shadow = vmcs12->cr0_read_shadow; 1874 * evmcs->cr4_read_shadow = vmcs12->cr4_read_shadow; 1875 * evmcs->vm_exit_msr_store_count = vmcs12->vm_exit_msr_store_count; 1876 * evmcs->vm_exit_msr_load_count = vmcs12->vm_exit_msr_load_count; 1877 * evmcs->vm_entry_msr_load_count = vmcs12->vm_entry_msr_load_count; 1878 * evmcs->guest_ia32_perf_global_ctrl = vmcs12->guest_ia32_perf_global_ctrl; 1879 * evmcs->host_ia32_perf_global_ctrl = vmcs12->host_ia32_perf_global_ctrl; 1880 * evmcs->encls_exiting_bitmap = vmcs12->encls_exiting_bitmap; 1881 * evmcs->tsc_multiplier = vmcs12->tsc_multiplier; 1882 * 1883 * Not present in struct vmcs12: 1884 * evmcs->exit_io_instruction_ecx = vmcs12->exit_io_instruction_ecx; 1885 * evmcs->exit_io_instruction_esi = vmcs12->exit_io_instruction_esi; 1886 * evmcs->exit_io_instruction_edi = vmcs12->exit_io_instruction_edi; 1887 * evmcs->exit_io_instruction_eip = vmcs12->exit_io_instruction_eip; 1888 * evmcs->host_ia32_s_cet = vmcs12->host_ia32_s_cet; 1889 * evmcs->host_ssp = vmcs12->host_ssp; 1890 * evmcs->host_ia32_int_ssp_table_addr = vmcs12->host_ia32_int_ssp_table_addr; 1891 * evmcs->guest_ia32_s_cet = vmcs12->guest_ia32_s_cet; 1892 * evmcs->guest_ia32_lbr_ctl = vmcs12->guest_ia32_lbr_ctl; 1893 * evmcs->guest_ia32_int_ssp_table_addr = vmcs12->guest_ia32_int_ssp_table_addr; 1894 * evmcs->guest_ssp = vmcs12->guest_ssp; 1895 */ 1896 1897 evmcs->guest_es_selector = vmcs12->guest_es_selector; 1898 evmcs->guest_cs_selector = vmcs12->guest_cs_selector; 1899 evmcs->guest_ss_selector = vmcs12->guest_ss_selector; 1900 evmcs->guest_ds_selector = vmcs12->guest_ds_selector; 1901 evmcs->guest_fs_selector = vmcs12->guest_fs_selector; 1902 evmcs->guest_gs_selector = vmcs12->guest_gs_selector; 1903 evmcs->guest_ldtr_selector = vmcs12->guest_ldtr_selector; 1904 evmcs->guest_tr_selector = vmcs12->guest_tr_selector; 1905 1906 evmcs->guest_es_limit = vmcs12->guest_es_limit; 1907 evmcs->guest_cs_limit = vmcs12->guest_cs_limit; 1908 evmcs->guest_ss_limit = vmcs12->guest_ss_limit; 1909 evmcs->guest_ds_limit = vmcs12->guest_ds_limit; 1910 evmcs->guest_fs_limit = vmcs12->guest_fs_limit; 1911 evmcs->guest_gs_limit = vmcs12->guest_gs_limit; 1912 evmcs->guest_ldtr_limit = vmcs12->guest_ldtr_limit; 1913 evmcs->guest_tr_limit = vmcs12->guest_tr_limit; 1914 evmcs->guest_gdtr_limit = vmcs12->guest_gdtr_limit; 1915 evmcs->guest_idtr_limit = vmcs12->guest_idtr_limit; 1916 1917 evmcs->guest_es_ar_bytes = vmcs12->guest_es_ar_bytes; 1918 evmcs->guest_cs_ar_bytes = vmcs12->guest_cs_ar_bytes; 1919 evmcs->guest_ss_ar_bytes = vmcs12->guest_ss_ar_bytes; 1920 evmcs->guest_ds_ar_bytes = vmcs12->guest_ds_ar_bytes; 1921 evmcs->guest_fs_ar_bytes = vmcs12->guest_fs_ar_bytes; 1922 evmcs->guest_gs_ar_bytes = vmcs12->guest_gs_ar_bytes; 1923 evmcs->guest_ldtr_ar_bytes = vmcs12->guest_ldtr_ar_bytes; 1924 evmcs->guest_tr_ar_bytes = vmcs12->guest_tr_ar_bytes; 1925 1926 evmcs->guest_es_base = vmcs12->guest_es_base; 1927 evmcs->guest_cs_base = vmcs12->guest_cs_base; 1928 evmcs->guest_ss_base = vmcs12->guest_ss_base; 1929 evmcs->guest_ds_base = vmcs12->guest_ds_base; 1930 evmcs->guest_fs_base = vmcs12->guest_fs_base; 1931 evmcs->guest_gs_base = vmcs12->guest_gs_base; 1932 evmcs->guest_ldtr_base = vmcs12->guest_ldtr_base; 1933 evmcs->guest_tr_base = vmcs12->guest_tr_base; 1934 evmcs->guest_gdtr_base = vmcs12->guest_gdtr_base; 1935 evmcs->guest_idtr_base = vmcs12->guest_idtr_base; 1936 1937 evmcs->guest_ia32_pat = vmcs12->guest_ia32_pat; 1938 evmcs->guest_ia32_efer = vmcs12->guest_ia32_efer; 1939 1940 evmcs->guest_pdptr0 = vmcs12->guest_pdptr0; 1941 evmcs->guest_pdptr1 = vmcs12->guest_pdptr1; 1942 evmcs->guest_pdptr2 = vmcs12->guest_pdptr2; 1943 evmcs->guest_pdptr3 = vmcs12->guest_pdptr3; 1944 1945 evmcs->guest_pending_dbg_exceptions = 1946 vmcs12->guest_pending_dbg_exceptions; 1947 evmcs->guest_sysenter_esp = vmcs12->guest_sysenter_esp; 1948 evmcs->guest_sysenter_eip = vmcs12->guest_sysenter_eip; 1949 1950 evmcs->guest_activity_state = vmcs12->guest_activity_state; 1951 evmcs->guest_sysenter_cs = vmcs12->guest_sysenter_cs; 1952 1953 evmcs->guest_cr0 = vmcs12->guest_cr0; 1954 evmcs->guest_cr3 = vmcs12->guest_cr3; 1955 evmcs->guest_cr4 = vmcs12->guest_cr4; 1956 evmcs->guest_dr7 = vmcs12->guest_dr7; 1957 1958 evmcs->guest_physical_address = vmcs12->guest_physical_address; 1959 1960 evmcs->vm_instruction_error = vmcs12->vm_instruction_error; 1961 evmcs->vm_exit_reason = vmcs12->vm_exit_reason; 1962 evmcs->vm_exit_intr_info = vmcs12->vm_exit_intr_info; 1963 evmcs->vm_exit_intr_error_code = vmcs12->vm_exit_intr_error_code; 1964 evmcs->idt_vectoring_info_field = vmcs12->idt_vectoring_info_field; 1965 evmcs->idt_vectoring_error_code = vmcs12->idt_vectoring_error_code; 1966 evmcs->vm_exit_instruction_len = vmcs12->vm_exit_instruction_len; 1967 evmcs->vmx_instruction_info = vmcs12->vmx_instruction_info; 1968 1969 evmcs->exit_qualification = vmcs12->exit_qualification; 1970 1971 evmcs->guest_linear_address = vmcs12->guest_linear_address; 1972 evmcs->guest_rsp = vmcs12->guest_rsp; 1973 evmcs->guest_rflags = vmcs12->guest_rflags; 1974 1975 evmcs->guest_interruptibility_info = 1976 vmcs12->guest_interruptibility_info; 1977 evmcs->cpu_based_vm_exec_control = vmcs12->cpu_based_vm_exec_control; 1978 evmcs->vm_entry_controls = vmcs12->vm_entry_controls; 1979 evmcs->vm_entry_intr_info_field = vmcs12->vm_entry_intr_info_field; 1980 evmcs->vm_entry_exception_error_code = 1981 vmcs12->vm_entry_exception_error_code; 1982 evmcs->vm_entry_instruction_len = vmcs12->vm_entry_instruction_len; 1983 1984 evmcs->guest_rip = vmcs12->guest_rip; 1985 1986 evmcs->guest_bndcfgs = vmcs12->guest_bndcfgs; 1987 1988 return; 1989 } 1990 1991 /* 1992 * This is an equivalent of the nested hypervisor executing the vmptrld 1993 * instruction. 1994 */ 1995 static enum nested_evmptrld_status nested_vmx_handle_enlightened_vmptrld( 1996 struct kvm_vcpu *vcpu, bool from_launch) 1997 { 1998 struct vcpu_vmx *vmx = to_vmx(vcpu); 1999 bool evmcs_gpa_changed = false; 2000 u64 evmcs_gpa; 2001 2002 if (likely(!guest_cpuid_has_evmcs(vcpu))) 2003 return EVMPTRLD_DISABLED; 2004 2005 evmcs_gpa = nested_get_evmptr(vcpu); 2006 if (!evmptr_is_valid(evmcs_gpa)) { 2007 nested_release_evmcs(vcpu); 2008 return EVMPTRLD_DISABLED; 2009 } 2010 2011 if (unlikely(evmcs_gpa != vmx->nested.hv_evmcs_vmptr)) { 2012 vmx->nested.current_vmptr = INVALID_GPA; 2013 2014 nested_release_evmcs(vcpu); 2015 2016 if (kvm_vcpu_map(vcpu, gpa_to_gfn(evmcs_gpa), 2017 &vmx->nested.hv_evmcs_map)) 2018 return EVMPTRLD_ERROR; 2019 2020 vmx->nested.hv_evmcs = vmx->nested.hv_evmcs_map.hva; 2021 2022 /* 2023 * Currently, KVM only supports eVMCS version 1 2024 * (== KVM_EVMCS_VERSION) and thus we expect guest to set this 2025 * value to first u32 field of eVMCS which should specify eVMCS 2026 * VersionNumber. 2027 * 2028 * Guest should be aware of supported eVMCS versions by host by 2029 * examining CPUID.0x4000000A.EAX[0:15]. Host userspace VMM is 2030 * expected to set this CPUID leaf according to the value 2031 * returned in vmcs_version from nested_enable_evmcs(). 2032 * 2033 * However, it turns out that Microsoft Hyper-V fails to comply 2034 * to their own invented interface: When Hyper-V use eVMCS, it 2035 * just sets first u32 field of eVMCS to revision_id specified 2036 * in MSR_IA32_VMX_BASIC. Instead of used eVMCS version number 2037 * which is one of the supported versions specified in 2038 * CPUID.0x4000000A.EAX[0:15]. 2039 * 2040 * To overcome Hyper-V bug, we accept here either a supported 2041 * eVMCS version or VMCS12 revision_id as valid values for first 2042 * u32 field of eVMCS. 2043 */ 2044 if ((vmx->nested.hv_evmcs->revision_id != KVM_EVMCS_VERSION) && 2045 (vmx->nested.hv_evmcs->revision_id != VMCS12_REVISION)) { 2046 nested_release_evmcs(vcpu); 2047 return EVMPTRLD_VMFAIL; 2048 } 2049 2050 vmx->nested.hv_evmcs_vmptr = evmcs_gpa; 2051 2052 evmcs_gpa_changed = true; 2053 /* 2054 * Unlike normal vmcs12, enlightened vmcs12 is not fully 2055 * reloaded from guest's memory (read only fields, fields not 2056 * present in struct hv_enlightened_vmcs, ...). Make sure there 2057 * are no leftovers. 2058 */ 2059 if (from_launch) { 2060 struct vmcs12 *vmcs12 = get_vmcs12(vcpu); 2061 memset(vmcs12, 0, sizeof(*vmcs12)); 2062 vmcs12->hdr.revision_id = VMCS12_REVISION; 2063 } 2064 2065 } 2066 2067 /* 2068 * Clean fields data can't be used on VMLAUNCH and when we switch 2069 * between different L2 guests as KVM keeps a single VMCS12 per L1. 2070 */ 2071 if (from_launch || evmcs_gpa_changed) { 2072 vmx->nested.hv_evmcs->hv_clean_fields &= 2073 ~HV_VMX_ENLIGHTENED_CLEAN_FIELD_ALL; 2074 2075 vmx->nested.force_msr_bitmap_recalc = true; 2076 } 2077 2078 return EVMPTRLD_SUCCEEDED; 2079 } 2080 2081 void nested_sync_vmcs12_to_shadow(struct kvm_vcpu *vcpu) 2082 { 2083 struct vcpu_vmx *vmx = to_vmx(vcpu); 2084 2085 if (evmptr_is_valid(vmx->nested.hv_evmcs_vmptr)) 2086 copy_vmcs12_to_enlightened(vmx); 2087 else 2088 copy_vmcs12_to_shadow(vmx); 2089 2090 vmx->nested.need_vmcs12_to_shadow_sync = false; 2091 } 2092 2093 static enum hrtimer_restart vmx_preemption_timer_fn(struct hrtimer *timer) 2094 { 2095 struct vcpu_vmx *vmx = 2096 container_of(timer, struct vcpu_vmx, nested.preemption_timer); 2097 2098 vmx->nested.preemption_timer_expired = true; 2099 kvm_make_request(KVM_REQ_EVENT, &vmx->vcpu); 2100 kvm_vcpu_kick(&vmx->vcpu); 2101 2102 return HRTIMER_NORESTART; 2103 } 2104 2105 static u64 vmx_calc_preemption_timer_value(struct kvm_vcpu *vcpu) 2106 { 2107 struct vcpu_vmx *vmx = to_vmx(vcpu); 2108 struct vmcs12 *vmcs12 = get_vmcs12(vcpu); 2109 2110 u64 l1_scaled_tsc = kvm_read_l1_tsc(vcpu, rdtsc()) >> 2111 VMX_MISC_EMULATED_PREEMPTION_TIMER_RATE; 2112 2113 if (!vmx->nested.has_preemption_timer_deadline) { 2114 vmx->nested.preemption_timer_deadline = 2115 vmcs12->vmx_preemption_timer_value + l1_scaled_tsc; 2116 vmx->nested.has_preemption_timer_deadline = true; 2117 } 2118 return vmx->nested.preemption_timer_deadline - l1_scaled_tsc; 2119 } 2120 2121 static void vmx_start_preemption_timer(struct kvm_vcpu *vcpu, 2122 u64 preemption_timeout) 2123 { 2124 struct vcpu_vmx *vmx = to_vmx(vcpu); 2125 2126 /* 2127 * A timer value of zero is architecturally guaranteed to cause 2128 * a VMExit prior to executing any instructions in the guest. 2129 */ 2130 if (preemption_timeout == 0) { 2131 vmx_preemption_timer_fn(&vmx->nested.preemption_timer); 2132 return; 2133 } 2134 2135 if (vcpu->arch.virtual_tsc_khz == 0) 2136 return; 2137 2138 preemption_timeout <<= VMX_MISC_EMULATED_PREEMPTION_TIMER_RATE; 2139 preemption_timeout *= 1000000; 2140 do_div(preemption_timeout, vcpu->arch.virtual_tsc_khz); 2141 hrtimer_start(&vmx->nested.preemption_timer, 2142 ktime_add_ns(ktime_get(), preemption_timeout), 2143 HRTIMER_MODE_ABS_PINNED); 2144 } 2145 2146 static u64 nested_vmx_calc_efer(struct vcpu_vmx *vmx, struct vmcs12 *vmcs12) 2147 { 2148 if (vmx->nested.nested_run_pending && 2149 (vmcs12->vm_entry_controls & VM_ENTRY_LOAD_IA32_EFER)) 2150 return vmcs12->guest_ia32_efer; 2151 else if (vmcs12->vm_entry_controls & VM_ENTRY_IA32E_MODE) 2152 return vmx->vcpu.arch.efer | (EFER_LMA | EFER_LME); 2153 else 2154 return vmx->vcpu.arch.efer & ~(EFER_LMA | EFER_LME); 2155 } 2156 2157 static void prepare_vmcs02_constant_state(struct vcpu_vmx *vmx) 2158 { 2159 struct kvm *kvm = vmx->vcpu.kvm; 2160 2161 /* 2162 * If vmcs02 hasn't been initialized, set the constant vmcs02 state 2163 * according to L0's settings (vmcs12 is irrelevant here). Host 2164 * fields that come from L0 and are not constant, e.g. HOST_CR3, 2165 * will be set as needed prior to VMLAUNCH/VMRESUME. 2166 */ 2167 if (vmx->nested.vmcs02_initialized) 2168 return; 2169 vmx->nested.vmcs02_initialized = true; 2170 2171 /* 2172 * We don't care what the EPTP value is we just need to guarantee 2173 * it's valid so we don't get a false positive when doing early 2174 * consistency checks. 2175 */ 2176 if (enable_ept && nested_early_check) 2177 vmcs_write64(EPT_POINTER, 2178 construct_eptp(&vmx->vcpu, 0, PT64_ROOT_4LEVEL)); 2179 2180 /* All VMFUNCs are currently emulated through L0 vmexits. */ 2181 if (cpu_has_vmx_vmfunc()) 2182 vmcs_write64(VM_FUNCTION_CONTROL, 0); 2183 2184 if (cpu_has_vmx_posted_intr()) 2185 vmcs_write16(POSTED_INTR_NV, POSTED_INTR_NESTED_VECTOR); 2186 2187 if (cpu_has_vmx_msr_bitmap()) 2188 vmcs_write64(MSR_BITMAP, __pa(vmx->nested.vmcs02.msr_bitmap)); 2189 2190 /* 2191 * PML is emulated for L2, but never enabled in hardware as the MMU 2192 * handles A/D emulation. Disabling PML for L2 also avoids having to 2193 * deal with filtering out L2 GPAs from the buffer. 2194 */ 2195 if (enable_pml) { 2196 vmcs_write64(PML_ADDRESS, 0); 2197 vmcs_write16(GUEST_PML_INDEX, -1); 2198 } 2199 2200 if (cpu_has_vmx_encls_vmexit()) 2201 vmcs_write64(ENCLS_EXITING_BITMAP, INVALID_GPA); 2202 2203 if (kvm_notify_vmexit_enabled(kvm)) 2204 vmcs_write32(NOTIFY_WINDOW, kvm->arch.notify_window); 2205 2206 /* 2207 * Set the MSR load/store lists to match L0's settings. Only the 2208 * addresses are constant (for vmcs02), the counts can change based 2209 * on L2's behavior, e.g. switching to/from long mode. 2210 */ 2211 vmcs_write64(VM_EXIT_MSR_STORE_ADDR, __pa(vmx->msr_autostore.guest.val)); 2212 vmcs_write64(VM_EXIT_MSR_LOAD_ADDR, __pa(vmx->msr_autoload.host.val)); 2213 vmcs_write64(VM_ENTRY_MSR_LOAD_ADDR, __pa(vmx->msr_autoload.guest.val)); 2214 2215 vmx_set_constant_host_state(vmx); 2216 } 2217 2218 static void prepare_vmcs02_early_rare(struct vcpu_vmx *vmx, 2219 struct vmcs12 *vmcs12) 2220 { 2221 prepare_vmcs02_constant_state(vmx); 2222 2223 vmcs_write64(VMCS_LINK_POINTER, INVALID_GPA); 2224 2225 if (enable_vpid) { 2226 if (nested_cpu_has_vpid(vmcs12) && vmx->nested.vpid02) 2227 vmcs_write16(VIRTUAL_PROCESSOR_ID, vmx->nested.vpid02); 2228 else 2229 vmcs_write16(VIRTUAL_PROCESSOR_ID, vmx->vpid); 2230 } 2231 } 2232 2233 static void prepare_vmcs02_early(struct vcpu_vmx *vmx, struct loaded_vmcs *vmcs01, 2234 struct vmcs12 *vmcs12) 2235 { 2236 u32 exec_control; 2237 u64 guest_efer = nested_vmx_calc_efer(vmx, vmcs12); 2238 2239 if (vmx->nested.dirty_vmcs12 || evmptr_is_valid(vmx->nested.hv_evmcs_vmptr)) 2240 prepare_vmcs02_early_rare(vmx, vmcs12); 2241 2242 /* 2243 * PIN CONTROLS 2244 */ 2245 exec_control = __pin_controls_get(vmcs01); 2246 exec_control |= (vmcs12->pin_based_vm_exec_control & 2247 ~PIN_BASED_VMX_PREEMPTION_TIMER); 2248 2249 /* Posted interrupts setting is only taken from vmcs12. */ 2250 vmx->nested.pi_pending = false; 2251 if (nested_cpu_has_posted_intr(vmcs12)) 2252 vmx->nested.posted_intr_nv = vmcs12->posted_intr_nv; 2253 else 2254 exec_control &= ~PIN_BASED_POSTED_INTR; 2255 pin_controls_set(vmx, exec_control); 2256 2257 /* 2258 * EXEC CONTROLS 2259 */ 2260 exec_control = __exec_controls_get(vmcs01); /* L0's desires */ 2261 exec_control &= ~CPU_BASED_INTR_WINDOW_EXITING; 2262 exec_control &= ~CPU_BASED_NMI_WINDOW_EXITING; 2263 exec_control &= ~CPU_BASED_TPR_SHADOW; 2264 exec_control |= vmcs12->cpu_based_vm_exec_control; 2265 2266 vmx->nested.l1_tpr_threshold = -1; 2267 if (exec_control & CPU_BASED_TPR_SHADOW) 2268 vmcs_write32(TPR_THRESHOLD, vmcs12->tpr_threshold); 2269 #ifdef CONFIG_X86_64 2270 else 2271 exec_control |= CPU_BASED_CR8_LOAD_EXITING | 2272 CPU_BASED_CR8_STORE_EXITING; 2273 #endif 2274 2275 /* 2276 * A vmexit (to either L1 hypervisor or L0 userspace) is always needed 2277 * for I/O port accesses. 2278 */ 2279 exec_control |= CPU_BASED_UNCOND_IO_EXITING; 2280 exec_control &= ~CPU_BASED_USE_IO_BITMAPS; 2281 2282 /* 2283 * This bit will be computed in nested_get_vmcs12_pages, because 2284 * we do not have access to L1's MSR bitmap yet. For now, keep 2285 * the same bit as before, hoping to avoid multiple VMWRITEs that 2286 * only set/clear this bit. 2287 */ 2288 exec_control &= ~CPU_BASED_USE_MSR_BITMAPS; 2289 exec_control |= exec_controls_get(vmx) & CPU_BASED_USE_MSR_BITMAPS; 2290 2291 exec_controls_set(vmx, exec_control); 2292 2293 /* 2294 * SECONDARY EXEC CONTROLS 2295 */ 2296 if (cpu_has_secondary_exec_ctrls()) { 2297 exec_control = __secondary_exec_controls_get(vmcs01); 2298 2299 /* Take the following fields only from vmcs12 */ 2300 exec_control &= ~(SECONDARY_EXEC_VIRTUALIZE_APIC_ACCESSES | 2301 SECONDARY_EXEC_VIRTUALIZE_X2APIC_MODE | 2302 SECONDARY_EXEC_ENABLE_INVPCID | 2303 SECONDARY_EXEC_ENABLE_RDTSCP | 2304 SECONDARY_EXEC_XSAVES | 2305 SECONDARY_EXEC_ENABLE_USR_WAIT_PAUSE | 2306 SECONDARY_EXEC_VIRTUAL_INTR_DELIVERY | 2307 SECONDARY_EXEC_APIC_REGISTER_VIRT | 2308 SECONDARY_EXEC_ENABLE_VMFUNC | 2309 SECONDARY_EXEC_DESC); 2310 2311 if (nested_cpu_has(vmcs12, 2312 CPU_BASED_ACTIVATE_SECONDARY_CONTROLS)) 2313 exec_control |= vmcs12->secondary_vm_exec_control; 2314 2315 /* PML is emulated and never enabled in hardware for L2. */ 2316 exec_control &= ~SECONDARY_EXEC_ENABLE_PML; 2317 2318 /* VMCS shadowing for L2 is emulated for now */ 2319 exec_control &= ~SECONDARY_EXEC_SHADOW_VMCS; 2320 2321 /* 2322 * Preset *DT exiting when emulating UMIP, so that vmx_set_cr4() 2323 * will not have to rewrite the controls just for this bit. 2324 */ 2325 if (!boot_cpu_has(X86_FEATURE_UMIP) && vmx_umip_emulated() && 2326 (vmcs12->guest_cr4 & X86_CR4_UMIP)) 2327 exec_control |= SECONDARY_EXEC_DESC; 2328 2329 if (exec_control & SECONDARY_EXEC_VIRTUAL_INTR_DELIVERY) 2330 vmcs_write16(GUEST_INTR_STATUS, 2331 vmcs12->guest_intr_status); 2332 2333 if (!nested_cpu_has2(vmcs12, SECONDARY_EXEC_UNRESTRICTED_GUEST)) 2334 exec_control &= ~SECONDARY_EXEC_UNRESTRICTED_GUEST; 2335 2336 if (exec_control & SECONDARY_EXEC_ENCLS_EXITING) 2337 vmx_write_encls_bitmap(&vmx->vcpu, vmcs12); 2338 2339 secondary_exec_controls_set(vmx, exec_control); 2340 } 2341 2342 /* 2343 * ENTRY CONTROLS 2344 * 2345 * vmcs12's VM_{ENTRY,EXIT}_LOAD_IA32_EFER and VM_ENTRY_IA32E_MODE 2346 * are emulated by vmx_set_efer() in prepare_vmcs02(), but speculate 2347 * on the related bits (if supported by the CPU) in the hope that 2348 * we can avoid VMWrites during vmx_set_efer(). 2349 * 2350 * Similarly, take vmcs01's PERF_GLOBAL_CTRL in the hope that if KVM is 2351 * loading PERF_GLOBAL_CTRL via the VMCS for L1, then KVM will want to 2352 * do the same for L2. 2353 */ 2354 exec_control = __vm_entry_controls_get(vmcs01); 2355 exec_control |= (vmcs12->vm_entry_controls & 2356 ~VM_ENTRY_LOAD_IA32_PERF_GLOBAL_CTRL); 2357 exec_control &= ~(VM_ENTRY_IA32E_MODE | VM_ENTRY_LOAD_IA32_EFER); 2358 if (cpu_has_load_ia32_efer()) { 2359 if (guest_efer & EFER_LMA) 2360 exec_control |= VM_ENTRY_IA32E_MODE; 2361 if (guest_efer != host_efer) 2362 exec_control |= VM_ENTRY_LOAD_IA32_EFER; 2363 } 2364 vm_entry_controls_set(vmx, exec_control); 2365 2366 /* 2367 * EXIT CONTROLS 2368 * 2369 * L2->L1 exit controls are emulated - the hardware exit is to L0 so 2370 * we should use its exit controls. Note that VM_EXIT_LOAD_IA32_EFER 2371 * bits may be modified by vmx_set_efer() in prepare_vmcs02(). 2372 */ 2373 exec_control = __vm_exit_controls_get(vmcs01); 2374 if (cpu_has_load_ia32_efer() && guest_efer != host_efer) 2375 exec_control |= VM_EXIT_LOAD_IA32_EFER; 2376 else 2377 exec_control &= ~VM_EXIT_LOAD_IA32_EFER; 2378 vm_exit_controls_set(vmx, exec_control); 2379 2380 /* 2381 * Interrupt/Exception Fields 2382 */ 2383 if (vmx->nested.nested_run_pending) { 2384 vmcs_write32(VM_ENTRY_INTR_INFO_FIELD, 2385 vmcs12->vm_entry_intr_info_field); 2386 vmcs_write32(VM_ENTRY_EXCEPTION_ERROR_CODE, 2387 vmcs12->vm_entry_exception_error_code); 2388 vmcs_write32(VM_ENTRY_INSTRUCTION_LEN, 2389 vmcs12->vm_entry_instruction_len); 2390 vmcs_write32(GUEST_INTERRUPTIBILITY_INFO, 2391 vmcs12->guest_interruptibility_info); 2392 vmx->loaded_vmcs->nmi_known_unmasked = 2393 !(vmcs12->guest_interruptibility_info & GUEST_INTR_STATE_NMI); 2394 } else { 2395 vmcs_write32(VM_ENTRY_INTR_INFO_FIELD, 0); 2396 } 2397 } 2398 2399 static void prepare_vmcs02_rare(struct vcpu_vmx *vmx, struct vmcs12 *vmcs12) 2400 { 2401 struct hv_enlightened_vmcs *hv_evmcs = vmx->nested.hv_evmcs; 2402 2403 if (!hv_evmcs || !(hv_evmcs->hv_clean_fields & 2404 HV_VMX_ENLIGHTENED_CLEAN_FIELD_GUEST_GRP2)) { 2405 vmcs_write16(GUEST_ES_SELECTOR, vmcs12->guest_es_selector); 2406 vmcs_write16(GUEST_CS_SELECTOR, vmcs12->guest_cs_selector); 2407 vmcs_write16(GUEST_SS_SELECTOR, vmcs12->guest_ss_selector); 2408 vmcs_write16(GUEST_DS_SELECTOR, vmcs12->guest_ds_selector); 2409 vmcs_write16(GUEST_FS_SELECTOR, vmcs12->guest_fs_selector); 2410 vmcs_write16(GUEST_GS_SELECTOR, vmcs12->guest_gs_selector); 2411 vmcs_write16(GUEST_LDTR_SELECTOR, vmcs12->guest_ldtr_selector); 2412 vmcs_write16(GUEST_TR_SELECTOR, vmcs12->guest_tr_selector); 2413 vmcs_write32(GUEST_ES_LIMIT, vmcs12->guest_es_limit); 2414 vmcs_write32(GUEST_CS_LIMIT, vmcs12->guest_cs_limit); 2415 vmcs_write32(GUEST_SS_LIMIT, vmcs12->guest_ss_limit); 2416 vmcs_write32(GUEST_DS_LIMIT, vmcs12->guest_ds_limit); 2417 vmcs_write32(GUEST_FS_LIMIT, vmcs12->guest_fs_limit); 2418 vmcs_write32(GUEST_GS_LIMIT, vmcs12->guest_gs_limit); 2419 vmcs_write32(GUEST_LDTR_LIMIT, vmcs12->guest_ldtr_limit); 2420 vmcs_write32(GUEST_TR_LIMIT, vmcs12->guest_tr_limit); 2421 vmcs_write32(GUEST_GDTR_LIMIT, vmcs12->guest_gdtr_limit); 2422 vmcs_write32(GUEST_IDTR_LIMIT, vmcs12->guest_idtr_limit); 2423 vmcs_write32(GUEST_CS_AR_BYTES, vmcs12->guest_cs_ar_bytes); 2424 vmcs_write32(GUEST_SS_AR_BYTES, vmcs12->guest_ss_ar_bytes); 2425 vmcs_write32(GUEST_ES_AR_BYTES, vmcs12->guest_es_ar_bytes); 2426 vmcs_write32(GUEST_DS_AR_BYTES, vmcs12->guest_ds_ar_bytes); 2427 vmcs_write32(GUEST_FS_AR_BYTES, vmcs12->guest_fs_ar_bytes); 2428 vmcs_write32(GUEST_GS_AR_BYTES, vmcs12->guest_gs_ar_bytes); 2429 vmcs_write32(GUEST_LDTR_AR_BYTES, vmcs12->guest_ldtr_ar_bytes); 2430 vmcs_write32(GUEST_TR_AR_BYTES, vmcs12->guest_tr_ar_bytes); 2431 vmcs_writel(GUEST_ES_BASE, vmcs12->guest_es_base); 2432 vmcs_writel(GUEST_CS_BASE, vmcs12->guest_cs_base); 2433 vmcs_writel(GUEST_SS_BASE, vmcs12->guest_ss_base); 2434 vmcs_writel(GUEST_DS_BASE, vmcs12->guest_ds_base); 2435 vmcs_writel(GUEST_FS_BASE, vmcs12->guest_fs_base); 2436 vmcs_writel(GUEST_GS_BASE, vmcs12->guest_gs_base); 2437 vmcs_writel(GUEST_LDTR_BASE, vmcs12->guest_ldtr_base); 2438 vmcs_writel(GUEST_TR_BASE, vmcs12->guest_tr_base); 2439 vmcs_writel(GUEST_GDTR_BASE, vmcs12->guest_gdtr_base); 2440 vmcs_writel(GUEST_IDTR_BASE, vmcs12->guest_idtr_base); 2441 2442 vmx->segment_cache.bitmask = 0; 2443 } 2444 2445 if (!hv_evmcs || !(hv_evmcs->hv_clean_fields & 2446 HV_VMX_ENLIGHTENED_CLEAN_FIELD_GUEST_GRP1)) { 2447 vmcs_write32(GUEST_SYSENTER_CS, vmcs12->guest_sysenter_cs); 2448 vmcs_writel(GUEST_PENDING_DBG_EXCEPTIONS, 2449 vmcs12->guest_pending_dbg_exceptions); 2450 vmcs_writel(GUEST_SYSENTER_ESP, vmcs12->guest_sysenter_esp); 2451 vmcs_writel(GUEST_SYSENTER_EIP, vmcs12->guest_sysenter_eip); 2452 2453 /* 2454 * L1 may access the L2's PDPTR, so save them to construct 2455 * vmcs12 2456 */ 2457 if (enable_ept) { 2458 vmcs_write64(GUEST_PDPTR0, vmcs12->guest_pdptr0); 2459 vmcs_write64(GUEST_PDPTR1, vmcs12->guest_pdptr1); 2460 vmcs_write64(GUEST_PDPTR2, vmcs12->guest_pdptr2); 2461 vmcs_write64(GUEST_PDPTR3, vmcs12->guest_pdptr3); 2462 } 2463 2464 if (kvm_mpx_supported() && vmx->nested.nested_run_pending && 2465 (vmcs12->vm_entry_controls & VM_ENTRY_LOAD_BNDCFGS)) 2466 vmcs_write64(GUEST_BNDCFGS, vmcs12->guest_bndcfgs); 2467 } 2468 2469 if (nested_cpu_has_xsaves(vmcs12)) 2470 vmcs_write64(XSS_EXIT_BITMAP, vmcs12->xss_exit_bitmap); 2471 2472 /* 2473 * Whether page-faults are trapped is determined by a combination of 2474 * 3 settings: PFEC_MASK, PFEC_MATCH and EXCEPTION_BITMAP.PF. If L0 2475 * doesn't care about page faults then we should set all of these to 2476 * L1's desires. However, if L0 does care about (some) page faults, it 2477 * is not easy (if at all possible?) to merge L0 and L1's desires, we 2478 * simply ask to exit on each and every L2 page fault. This is done by 2479 * setting MASK=MATCH=0 and (see below) EB.PF=1. 2480 * Note that below we don't need special code to set EB.PF beyond the 2481 * "or"ing of the EB of vmcs01 and vmcs12, because when enable_ept, 2482 * vmcs01's EB.PF is 0 so the "or" will take vmcs12's value, and when 2483 * !enable_ept, EB.PF is 1, so the "or" will always be 1. 2484 */ 2485 if (vmx_need_pf_intercept(&vmx->vcpu)) { 2486 /* 2487 * TODO: if both L0 and L1 need the same MASK and MATCH, 2488 * go ahead and use it? 2489 */ 2490 vmcs_write32(PAGE_FAULT_ERROR_CODE_MASK, 0); 2491 vmcs_write32(PAGE_FAULT_ERROR_CODE_MATCH, 0); 2492 } else { 2493 vmcs_write32(PAGE_FAULT_ERROR_CODE_MASK, vmcs12->page_fault_error_code_mask); 2494 vmcs_write32(PAGE_FAULT_ERROR_CODE_MATCH, vmcs12->page_fault_error_code_match); 2495 } 2496 2497 if (cpu_has_vmx_apicv()) { 2498 vmcs_write64(EOI_EXIT_BITMAP0, vmcs12->eoi_exit_bitmap0); 2499 vmcs_write64(EOI_EXIT_BITMAP1, vmcs12->eoi_exit_bitmap1); 2500 vmcs_write64(EOI_EXIT_BITMAP2, vmcs12->eoi_exit_bitmap2); 2501 vmcs_write64(EOI_EXIT_BITMAP3, vmcs12->eoi_exit_bitmap3); 2502 } 2503 2504 /* 2505 * Make sure the msr_autostore list is up to date before we set the 2506 * count in the vmcs02. 2507 */ 2508 prepare_vmx_msr_autostore_list(&vmx->vcpu, MSR_IA32_TSC); 2509 2510 vmcs_write32(VM_EXIT_MSR_STORE_COUNT, vmx->msr_autostore.guest.nr); 2511 vmcs_write32(VM_EXIT_MSR_LOAD_COUNT, vmx->msr_autoload.host.nr); 2512 vmcs_write32(VM_ENTRY_MSR_LOAD_COUNT, vmx->msr_autoload.guest.nr); 2513 2514 set_cr4_guest_host_mask(vmx); 2515 } 2516 2517 /* 2518 * prepare_vmcs02 is called when the L1 guest hypervisor runs its nested 2519 * L2 guest. L1 has a vmcs for L2 (vmcs12), and this function "merges" it 2520 * with L0's requirements for its guest (a.k.a. vmcs01), so we can run the L2 2521 * guest in a way that will both be appropriate to L1's requests, and our 2522 * needs. In addition to modifying the active vmcs (which is vmcs02), this 2523 * function also has additional necessary side-effects, like setting various 2524 * vcpu->arch fields. 2525 * Returns 0 on success, 1 on failure. Invalid state exit qualification code 2526 * is assigned to entry_failure_code on failure. 2527 */ 2528 static int prepare_vmcs02(struct kvm_vcpu *vcpu, struct vmcs12 *vmcs12, 2529 bool from_vmentry, 2530 enum vm_entry_failure_code *entry_failure_code) 2531 { 2532 struct vcpu_vmx *vmx = to_vmx(vcpu); 2533 bool load_guest_pdptrs_vmcs12 = false; 2534 2535 if (vmx->nested.dirty_vmcs12 || evmptr_is_valid(vmx->nested.hv_evmcs_vmptr)) { 2536 prepare_vmcs02_rare(vmx, vmcs12); 2537 vmx->nested.dirty_vmcs12 = false; 2538 2539 load_guest_pdptrs_vmcs12 = !evmptr_is_valid(vmx->nested.hv_evmcs_vmptr) || 2540 !(vmx->nested.hv_evmcs->hv_clean_fields & 2541 HV_VMX_ENLIGHTENED_CLEAN_FIELD_GUEST_GRP1); 2542 } 2543 2544 if (vmx->nested.nested_run_pending && 2545 (vmcs12->vm_entry_controls & VM_ENTRY_LOAD_DEBUG_CONTROLS)) { 2546 kvm_set_dr(vcpu, 7, vmcs12->guest_dr7); 2547 vmcs_write64(GUEST_IA32_DEBUGCTL, vmcs12->guest_ia32_debugctl); 2548 } else { 2549 kvm_set_dr(vcpu, 7, vcpu->arch.dr7); 2550 vmcs_write64(GUEST_IA32_DEBUGCTL, vmx->nested.pre_vmenter_debugctl); 2551 } 2552 if (kvm_mpx_supported() && (!vmx->nested.nested_run_pending || 2553 !(vmcs12->vm_entry_controls & VM_ENTRY_LOAD_BNDCFGS))) 2554 vmcs_write64(GUEST_BNDCFGS, vmx->nested.pre_vmenter_bndcfgs); 2555 vmx_set_rflags(vcpu, vmcs12->guest_rflags); 2556 2557 /* EXCEPTION_BITMAP and CR0_GUEST_HOST_MASK should basically be the 2558 * bitwise-or of what L1 wants to trap for L2, and what we want to 2559 * trap. Note that CR0.TS also needs updating - we do this later. 2560 */ 2561 vmx_update_exception_bitmap(vcpu); 2562 vcpu->arch.cr0_guest_owned_bits &= ~vmcs12->cr0_guest_host_mask; 2563 vmcs_writel(CR0_GUEST_HOST_MASK, ~vcpu->arch.cr0_guest_owned_bits); 2564 2565 if (vmx->nested.nested_run_pending && 2566 (vmcs12->vm_entry_controls & VM_ENTRY_LOAD_IA32_PAT)) { 2567 vmcs_write64(GUEST_IA32_PAT, vmcs12->guest_ia32_pat); 2568 vcpu->arch.pat = vmcs12->guest_ia32_pat; 2569 } else if (vmcs_config.vmentry_ctrl & VM_ENTRY_LOAD_IA32_PAT) { 2570 vmcs_write64(GUEST_IA32_PAT, vmx->vcpu.arch.pat); 2571 } 2572 2573 vcpu->arch.tsc_offset = kvm_calc_nested_tsc_offset( 2574 vcpu->arch.l1_tsc_offset, 2575 vmx_get_l2_tsc_offset(vcpu), 2576 vmx_get_l2_tsc_multiplier(vcpu)); 2577 2578 vcpu->arch.tsc_scaling_ratio = kvm_calc_nested_tsc_multiplier( 2579 vcpu->arch.l1_tsc_scaling_ratio, 2580 vmx_get_l2_tsc_multiplier(vcpu)); 2581 2582 vmcs_write64(TSC_OFFSET, vcpu->arch.tsc_offset); 2583 if (kvm_caps.has_tsc_control) 2584 vmcs_write64(TSC_MULTIPLIER, vcpu->arch.tsc_scaling_ratio); 2585 2586 nested_vmx_transition_tlb_flush(vcpu, vmcs12, true); 2587 2588 if (nested_cpu_has_ept(vmcs12)) 2589 nested_ept_init_mmu_context(vcpu); 2590 2591 /* 2592 * Override the CR0/CR4 read shadows after setting the effective guest 2593 * CR0/CR4. The common helpers also set the shadows, but they don't 2594 * account for vmcs12's cr0/4_guest_host_mask. 2595 */ 2596 vmx_set_cr0(vcpu, vmcs12->guest_cr0); 2597 vmcs_writel(CR0_READ_SHADOW, nested_read_cr0(vmcs12)); 2598 2599 vmx_set_cr4(vcpu, vmcs12->guest_cr4); 2600 vmcs_writel(CR4_READ_SHADOW, nested_read_cr4(vmcs12)); 2601 2602 vcpu->arch.efer = nested_vmx_calc_efer(vmx, vmcs12); 2603 /* Note: may modify VM_ENTRY/EXIT_CONTROLS and GUEST/HOST_IA32_EFER */ 2604 vmx_set_efer(vcpu, vcpu->arch.efer); 2605 2606 /* 2607 * Guest state is invalid and unrestricted guest is disabled, 2608 * which means L1 attempted VMEntry to L2 with invalid state. 2609 * Fail the VMEntry. 2610 * 2611 * However when force loading the guest state (SMM exit or 2612 * loading nested state after migration, it is possible to 2613 * have invalid guest state now, which will be later fixed by 2614 * restoring L2 register state 2615 */ 2616 if (CC(from_vmentry && !vmx_guest_state_valid(vcpu))) { 2617 *entry_failure_code = ENTRY_FAIL_DEFAULT; 2618 return -EINVAL; 2619 } 2620 2621 /* Shadow page tables on either EPT or shadow page tables. */ 2622 if (nested_vmx_load_cr3(vcpu, vmcs12->guest_cr3, nested_cpu_has_ept(vmcs12), 2623 from_vmentry, entry_failure_code)) 2624 return -EINVAL; 2625 2626 /* 2627 * Immediately write vmcs02.GUEST_CR3. It will be propagated to vmcs12 2628 * on nested VM-Exit, which can occur without actually running L2 and 2629 * thus without hitting vmx_load_mmu_pgd(), e.g. if L1 is entering L2 with 2630 * vmcs12.GUEST_ACTIVITYSTATE=HLT, in which case KVM will intercept the 2631 * transition to HLT instead of running L2. 2632 */ 2633 if (enable_ept) 2634 vmcs_writel(GUEST_CR3, vmcs12->guest_cr3); 2635 2636 /* Late preparation of GUEST_PDPTRs now that EFER and CRs are set. */ 2637 if (load_guest_pdptrs_vmcs12 && nested_cpu_has_ept(vmcs12) && 2638 is_pae_paging(vcpu)) { 2639 vmcs_write64(GUEST_PDPTR0, vmcs12->guest_pdptr0); 2640 vmcs_write64(GUEST_PDPTR1, vmcs12->guest_pdptr1); 2641 vmcs_write64(GUEST_PDPTR2, vmcs12->guest_pdptr2); 2642 vmcs_write64(GUEST_PDPTR3, vmcs12->guest_pdptr3); 2643 } 2644 2645 if ((vmcs12->vm_entry_controls & VM_ENTRY_LOAD_IA32_PERF_GLOBAL_CTRL) && 2646 intel_pmu_has_perf_global_ctrl(vcpu_to_pmu(vcpu)) && 2647 WARN_ON_ONCE(kvm_set_msr(vcpu, MSR_CORE_PERF_GLOBAL_CTRL, 2648 vmcs12->guest_ia32_perf_global_ctrl))) { 2649 *entry_failure_code = ENTRY_FAIL_DEFAULT; 2650 return -EINVAL; 2651 } 2652 2653 kvm_rsp_write(vcpu, vmcs12->guest_rsp); 2654 kvm_rip_write(vcpu, vmcs12->guest_rip); 2655 2656 /* 2657 * It was observed that genuine Hyper-V running in L1 doesn't reset 2658 * 'hv_clean_fields' by itself, it only sets the corresponding dirty 2659 * bits when it changes a field in eVMCS. Mark all fields as clean 2660 * here. 2661 */ 2662 if (evmptr_is_valid(vmx->nested.hv_evmcs_vmptr)) 2663 vmx->nested.hv_evmcs->hv_clean_fields |= 2664 HV_VMX_ENLIGHTENED_CLEAN_FIELD_ALL; 2665 2666 return 0; 2667 } 2668 2669 static int nested_vmx_check_nmi_controls(struct vmcs12 *vmcs12) 2670 { 2671 if (CC(!nested_cpu_has_nmi_exiting(vmcs12) && 2672 nested_cpu_has_virtual_nmis(vmcs12))) 2673 return -EINVAL; 2674 2675 if (CC(!nested_cpu_has_virtual_nmis(vmcs12) && 2676 nested_cpu_has(vmcs12, CPU_BASED_NMI_WINDOW_EXITING))) 2677 return -EINVAL; 2678 2679 return 0; 2680 } 2681 2682 static bool nested_vmx_check_eptp(struct kvm_vcpu *vcpu, u64 new_eptp) 2683 { 2684 struct vcpu_vmx *vmx = to_vmx(vcpu); 2685 2686 /* Check for memory type validity */ 2687 switch (new_eptp & VMX_EPTP_MT_MASK) { 2688 case VMX_EPTP_MT_UC: 2689 if (CC(!(vmx->nested.msrs.ept_caps & VMX_EPTP_UC_BIT))) 2690 return false; 2691 break; 2692 case VMX_EPTP_MT_WB: 2693 if (CC(!(vmx->nested.msrs.ept_caps & VMX_EPTP_WB_BIT))) 2694 return false; 2695 break; 2696 default: 2697 return false; 2698 } 2699 2700 /* Page-walk levels validity. */ 2701 switch (new_eptp & VMX_EPTP_PWL_MASK) { 2702 case VMX_EPTP_PWL_5: 2703 if (CC(!(vmx->nested.msrs.ept_caps & VMX_EPT_PAGE_WALK_5_BIT))) 2704 return false; 2705 break; 2706 case VMX_EPTP_PWL_4: 2707 if (CC(!(vmx->nested.msrs.ept_caps & VMX_EPT_PAGE_WALK_4_BIT))) 2708 return false; 2709 break; 2710 default: 2711 return false; 2712 } 2713 2714 /* Reserved bits should not be set */ 2715 if (CC(kvm_vcpu_is_illegal_gpa(vcpu, new_eptp) || ((new_eptp >> 7) & 0x1f))) 2716 return false; 2717 2718 /* AD, if set, should be supported */ 2719 if (new_eptp & VMX_EPTP_AD_ENABLE_BIT) { 2720 if (CC(!(vmx->nested.msrs.ept_caps & VMX_EPT_AD_BIT))) 2721 return false; 2722 } 2723 2724 return true; 2725 } 2726 2727 /* 2728 * Checks related to VM-Execution Control Fields 2729 */ 2730 static int nested_check_vm_execution_controls(struct kvm_vcpu *vcpu, 2731 struct vmcs12 *vmcs12) 2732 { 2733 struct vcpu_vmx *vmx = to_vmx(vcpu); 2734 2735 if (CC(!vmx_control_verify(vmcs12->pin_based_vm_exec_control, 2736 vmx->nested.msrs.pinbased_ctls_low, 2737 vmx->nested.msrs.pinbased_ctls_high)) || 2738 CC(!vmx_control_verify(vmcs12->cpu_based_vm_exec_control, 2739 vmx->nested.msrs.procbased_ctls_low, 2740 vmx->nested.msrs.procbased_ctls_high))) 2741 return -EINVAL; 2742 2743 if (nested_cpu_has(vmcs12, CPU_BASED_ACTIVATE_SECONDARY_CONTROLS) && 2744 CC(!vmx_control_verify(vmcs12->secondary_vm_exec_control, 2745 vmx->nested.msrs.secondary_ctls_low, 2746 vmx->nested.msrs.secondary_ctls_high))) 2747 return -EINVAL; 2748 2749 if (CC(vmcs12->cr3_target_count > nested_cpu_vmx_misc_cr3_count(vcpu)) || 2750 nested_vmx_check_io_bitmap_controls(vcpu, vmcs12) || 2751 nested_vmx_check_msr_bitmap_controls(vcpu, vmcs12) || 2752 nested_vmx_check_tpr_shadow_controls(vcpu, vmcs12) || 2753 nested_vmx_check_apic_access_controls(vcpu, vmcs12) || 2754 nested_vmx_check_apicv_controls(vcpu, vmcs12) || 2755 nested_vmx_check_nmi_controls(vmcs12) || 2756 nested_vmx_check_pml_controls(vcpu, vmcs12) || 2757 nested_vmx_check_unrestricted_guest_controls(vcpu, vmcs12) || 2758 nested_vmx_check_mode_based_ept_exec_controls(vcpu, vmcs12) || 2759 nested_vmx_check_shadow_vmcs_controls(vcpu, vmcs12) || 2760 CC(nested_cpu_has_vpid(vmcs12) && !vmcs12->virtual_processor_id)) 2761 return -EINVAL; 2762 2763 if (!nested_cpu_has_preemption_timer(vmcs12) && 2764 nested_cpu_has_save_preemption_timer(vmcs12)) 2765 return -EINVAL; 2766 2767 if (nested_cpu_has_ept(vmcs12) && 2768 CC(!nested_vmx_check_eptp(vcpu, vmcs12->ept_pointer))) 2769 return -EINVAL; 2770 2771 if (nested_cpu_has_vmfunc(vmcs12)) { 2772 if (CC(vmcs12->vm_function_control & 2773 ~vmx->nested.msrs.vmfunc_controls)) 2774 return -EINVAL; 2775 2776 if (nested_cpu_has_eptp_switching(vmcs12)) { 2777 if (CC(!nested_cpu_has_ept(vmcs12)) || 2778 CC(!page_address_valid(vcpu, vmcs12->eptp_list_address))) 2779 return -EINVAL; 2780 } 2781 } 2782 2783 return 0; 2784 } 2785 2786 /* 2787 * Checks related to VM-Exit Control Fields 2788 */ 2789 static int nested_check_vm_exit_controls(struct kvm_vcpu *vcpu, 2790 struct vmcs12 *vmcs12) 2791 { 2792 struct vcpu_vmx *vmx = to_vmx(vcpu); 2793 2794 if (CC(!vmx_control_verify(vmcs12->vm_exit_controls, 2795 vmx->nested.msrs.exit_ctls_low, 2796 vmx->nested.msrs.exit_ctls_high)) || 2797 CC(nested_vmx_check_exit_msr_switch_controls(vcpu, vmcs12))) 2798 return -EINVAL; 2799 2800 return 0; 2801 } 2802 2803 /* 2804 * Checks related to VM-Entry Control Fields 2805 */ 2806 static int nested_check_vm_entry_controls(struct kvm_vcpu *vcpu, 2807 struct vmcs12 *vmcs12) 2808 { 2809 struct vcpu_vmx *vmx = to_vmx(vcpu); 2810 2811 if (CC(!vmx_control_verify(vmcs12->vm_entry_controls, 2812 vmx->nested.msrs.entry_ctls_low, 2813 vmx->nested.msrs.entry_ctls_high))) 2814 return -EINVAL; 2815 2816 /* 2817 * From the Intel SDM, volume 3: 2818 * Fields relevant to VM-entry event injection must be set properly. 2819 * These fields are the VM-entry interruption-information field, the 2820 * VM-entry exception error code, and the VM-entry instruction length. 2821 */ 2822 if (vmcs12->vm_entry_intr_info_field & INTR_INFO_VALID_MASK) { 2823 u32 intr_info = vmcs12->vm_entry_intr_info_field; 2824 u8 vector = intr_info & INTR_INFO_VECTOR_MASK; 2825 u32 intr_type = intr_info & INTR_INFO_INTR_TYPE_MASK; 2826 bool has_error_code = intr_info & INTR_INFO_DELIVER_CODE_MASK; 2827 bool should_have_error_code; 2828 bool urg = nested_cpu_has2(vmcs12, 2829 SECONDARY_EXEC_UNRESTRICTED_GUEST); 2830 bool prot_mode = !urg || vmcs12->guest_cr0 & X86_CR0_PE; 2831 2832 /* VM-entry interruption-info field: interruption type */ 2833 if (CC(intr_type == INTR_TYPE_RESERVED) || 2834 CC(intr_type == INTR_TYPE_OTHER_EVENT && 2835 !nested_cpu_supports_monitor_trap_flag(vcpu))) 2836 return -EINVAL; 2837 2838 /* VM-entry interruption-info field: vector */ 2839 if (CC(intr_type == INTR_TYPE_NMI_INTR && vector != NMI_VECTOR) || 2840 CC(intr_type == INTR_TYPE_HARD_EXCEPTION && vector > 31) || 2841 CC(intr_type == INTR_TYPE_OTHER_EVENT && vector != 0)) 2842 return -EINVAL; 2843 2844 /* VM-entry interruption-info field: deliver error code */ 2845 should_have_error_code = 2846 intr_type == INTR_TYPE_HARD_EXCEPTION && prot_mode && 2847 x86_exception_has_error_code(vector); 2848 if (CC(has_error_code != should_have_error_code)) 2849 return -EINVAL; 2850 2851 /* VM-entry exception error code */ 2852 if (CC(has_error_code && 2853 vmcs12->vm_entry_exception_error_code & GENMASK(31, 16))) 2854 return -EINVAL; 2855 2856 /* VM-entry interruption-info field: reserved bits */ 2857 if (CC(intr_info & INTR_INFO_RESVD_BITS_MASK)) 2858 return -EINVAL; 2859 2860 /* VM-entry instruction length */ 2861 switch (intr_type) { 2862 case INTR_TYPE_SOFT_EXCEPTION: 2863 case INTR_TYPE_SOFT_INTR: 2864 case INTR_TYPE_PRIV_SW_EXCEPTION: 2865 if (CC(vmcs12->vm_entry_instruction_len > 15) || 2866 CC(vmcs12->vm_entry_instruction_len == 0 && 2867 CC(!nested_cpu_has_zero_length_injection(vcpu)))) 2868 return -EINVAL; 2869 } 2870 } 2871 2872 if (nested_vmx_check_entry_msr_switch_controls(vcpu, vmcs12)) 2873 return -EINVAL; 2874 2875 return 0; 2876 } 2877 2878 static int nested_vmx_check_controls(struct kvm_vcpu *vcpu, 2879 struct vmcs12 *vmcs12) 2880 { 2881 if (nested_check_vm_execution_controls(vcpu, vmcs12) || 2882 nested_check_vm_exit_controls(vcpu, vmcs12) || 2883 nested_check_vm_entry_controls(vcpu, vmcs12)) 2884 return -EINVAL; 2885 2886 if (guest_cpuid_has_evmcs(vcpu)) 2887 return nested_evmcs_check_controls(vmcs12); 2888 2889 return 0; 2890 } 2891 2892 static int nested_vmx_check_address_space_size(struct kvm_vcpu *vcpu, 2893 struct vmcs12 *vmcs12) 2894 { 2895 #ifdef CONFIG_X86_64 2896 if (CC(!!(vmcs12->vm_exit_controls & VM_EXIT_HOST_ADDR_SPACE_SIZE) != 2897 !!(vcpu->arch.efer & EFER_LMA))) 2898 return -EINVAL; 2899 #endif 2900 return 0; 2901 } 2902 2903 static int nested_vmx_check_host_state(struct kvm_vcpu *vcpu, 2904 struct vmcs12 *vmcs12) 2905 { 2906 bool ia32e = !!(vmcs12->vm_exit_controls & VM_EXIT_HOST_ADDR_SPACE_SIZE); 2907 2908 if (CC(!nested_host_cr0_valid(vcpu, vmcs12->host_cr0)) || 2909 CC(!nested_host_cr4_valid(vcpu, vmcs12->host_cr4)) || 2910 CC(kvm_vcpu_is_illegal_gpa(vcpu, vmcs12->host_cr3))) 2911 return -EINVAL; 2912 2913 if (CC(is_noncanonical_address(vmcs12->host_ia32_sysenter_esp, vcpu)) || 2914 CC(is_noncanonical_address(vmcs12->host_ia32_sysenter_eip, vcpu))) 2915 return -EINVAL; 2916 2917 if ((vmcs12->vm_exit_controls & VM_EXIT_LOAD_IA32_PAT) && 2918 CC(!kvm_pat_valid(vmcs12->host_ia32_pat))) 2919 return -EINVAL; 2920 2921 if ((vmcs12->vm_exit_controls & VM_EXIT_LOAD_IA32_PERF_GLOBAL_CTRL) && 2922 CC(!kvm_valid_perf_global_ctrl(vcpu_to_pmu(vcpu), 2923 vmcs12->host_ia32_perf_global_ctrl))) 2924 return -EINVAL; 2925 2926 if (ia32e) { 2927 if (CC(!(vmcs12->host_cr4 & X86_CR4_PAE))) 2928 return -EINVAL; 2929 } else { 2930 if (CC(vmcs12->vm_entry_controls & VM_ENTRY_IA32E_MODE) || 2931 CC(vmcs12->host_cr4 & X86_CR4_PCIDE) || 2932 CC((vmcs12->host_rip) >> 32)) 2933 return -EINVAL; 2934 } 2935 2936 if (CC(vmcs12->host_cs_selector & (SEGMENT_RPL_MASK | SEGMENT_TI_MASK)) || 2937 CC(vmcs12->host_ss_selector & (SEGMENT_RPL_MASK | SEGMENT_TI_MASK)) || 2938 CC(vmcs12->host_ds_selector & (SEGMENT_RPL_MASK | SEGMENT_TI_MASK)) || 2939 CC(vmcs12->host_es_selector & (SEGMENT_RPL_MASK | SEGMENT_TI_MASK)) || 2940 CC(vmcs12->host_fs_selector & (SEGMENT_RPL_MASK | SEGMENT_TI_MASK)) || 2941 CC(vmcs12->host_gs_selector & (SEGMENT_RPL_MASK | SEGMENT_TI_MASK)) || 2942 CC(vmcs12->host_tr_selector & (SEGMENT_RPL_MASK | SEGMENT_TI_MASK)) || 2943 CC(vmcs12->host_cs_selector == 0) || 2944 CC(vmcs12->host_tr_selector == 0) || 2945 CC(vmcs12->host_ss_selector == 0 && !ia32e)) 2946 return -EINVAL; 2947 2948 if (CC(is_noncanonical_address(vmcs12->host_fs_base, vcpu)) || 2949 CC(is_noncanonical_address(vmcs12->host_gs_base, vcpu)) || 2950 CC(is_noncanonical_address(vmcs12->host_gdtr_base, vcpu)) || 2951 CC(is_noncanonical_address(vmcs12->host_idtr_base, vcpu)) || 2952 CC(is_noncanonical_address(vmcs12->host_tr_base, vcpu)) || 2953 CC(is_noncanonical_address(vmcs12->host_rip, vcpu))) 2954 return -EINVAL; 2955 2956 /* 2957 * If the load IA32_EFER VM-exit control is 1, bits reserved in the 2958 * IA32_EFER MSR must be 0 in the field for that register. In addition, 2959 * the values of the LMA and LME bits in the field must each be that of 2960 * the host address-space size VM-exit control. 2961 */ 2962 if (vmcs12->vm_exit_controls & VM_EXIT_LOAD_IA32_EFER) { 2963 if (CC(!kvm_valid_efer(vcpu, vmcs12->host_ia32_efer)) || 2964 CC(ia32e != !!(vmcs12->host_ia32_efer & EFER_LMA)) || 2965 CC(ia32e != !!(vmcs12->host_ia32_efer & EFER_LME))) 2966 return -EINVAL; 2967 } 2968 2969 return 0; 2970 } 2971 2972 static int nested_vmx_check_vmcs_link_ptr(struct kvm_vcpu *vcpu, 2973 struct vmcs12 *vmcs12) 2974 { 2975 struct vcpu_vmx *vmx = to_vmx(vcpu); 2976 struct gfn_to_hva_cache *ghc = &vmx->nested.shadow_vmcs12_cache; 2977 struct vmcs_hdr hdr; 2978 2979 if (vmcs12->vmcs_link_pointer == INVALID_GPA) 2980 return 0; 2981 2982 if (CC(!page_address_valid(vcpu, vmcs12->vmcs_link_pointer))) 2983 return -EINVAL; 2984 2985 if (ghc->gpa != vmcs12->vmcs_link_pointer && 2986 CC(kvm_gfn_to_hva_cache_init(vcpu->kvm, ghc, 2987 vmcs12->vmcs_link_pointer, VMCS12_SIZE))) 2988 return -EINVAL; 2989 2990 if (CC(kvm_read_guest_offset_cached(vcpu->kvm, ghc, &hdr, 2991 offsetof(struct vmcs12, hdr), 2992 sizeof(hdr)))) 2993 return -EINVAL; 2994 2995 if (CC(hdr.revision_id != VMCS12_REVISION) || 2996 CC(hdr.shadow_vmcs != nested_cpu_has_shadow_vmcs(vmcs12))) 2997 return -EINVAL; 2998 2999 return 0; 3000 } 3001 3002 /* 3003 * Checks related to Guest Non-register State 3004 */ 3005 static int nested_check_guest_non_reg_state(struct vmcs12 *vmcs12) 3006 { 3007 if (CC(vmcs12->guest_activity_state != GUEST_ACTIVITY_ACTIVE && 3008 vmcs12->guest_activity_state != GUEST_ACTIVITY_HLT && 3009 vmcs12->guest_activity_state != GUEST_ACTIVITY_WAIT_SIPI)) 3010 return -EINVAL; 3011 3012 return 0; 3013 } 3014 3015 static int nested_vmx_check_guest_state(struct kvm_vcpu *vcpu, 3016 struct vmcs12 *vmcs12, 3017 enum vm_entry_failure_code *entry_failure_code) 3018 { 3019 bool ia32e = !!(vmcs12->vm_entry_controls & VM_ENTRY_IA32E_MODE); 3020 3021 *entry_failure_code = ENTRY_FAIL_DEFAULT; 3022 3023 if (CC(!nested_guest_cr0_valid(vcpu, vmcs12->guest_cr0)) || 3024 CC(!nested_guest_cr4_valid(vcpu, vmcs12->guest_cr4))) 3025 return -EINVAL; 3026 3027 if ((vmcs12->vm_entry_controls & VM_ENTRY_LOAD_DEBUG_CONTROLS) && 3028 CC(!kvm_dr7_valid(vmcs12->guest_dr7))) 3029 return -EINVAL; 3030 3031 if ((vmcs12->vm_entry_controls & VM_ENTRY_LOAD_IA32_PAT) && 3032 CC(!kvm_pat_valid(vmcs12->guest_ia32_pat))) 3033 return -EINVAL; 3034 3035 if (nested_vmx_check_vmcs_link_ptr(vcpu, vmcs12)) { 3036 *entry_failure_code = ENTRY_FAIL_VMCS_LINK_PTR; 3037 return -EINVAL; 3038 } 3039 3040 if ((vmcs12->vm_entry_controls & VM_ENTRY_LOAD_IA32_PERF_GLOBAL_CTRL) && 3041 CC(!kvm_valid_perf_global_ctrl(vcpu_to_pmu(vcpu), 3042 vmcs12->guest_ia32_perf_global_ctrl))) 3043 return -EINVAL; 3044 3045 if (CC((vmcs12->guest_cr0 & (X86_CR0_PG | X86_CR0_PE)) == X86_CR0_PG)) 3046 return -EINVAL; 3047 3048 if (CC(ia32e && !(vmcs12->guest_cr4 & X86_CR4_PAE)) || 3049 CC(ia32e && !(vmcs12->guest_cr0 & X86_CR0_PG))) 3050 return -EINVAL; 3051 3052 /* 3053 * If the load IA32_EFER VM-entry control is 1, the following checks 3054 * are performed on the field for the IA32_EFER MSR: 3055 * - Bits reserved in the IA32_EFER MSR must be 0. 3056 * - Bit 10 (corresponding to IA32_EFER.LMA) must equal the value of 3057 * the IA-32e mode guest VM-exit control. It must also be identical 3058 * to bit 8 (LME) if bit 31 in the CR0 field (corresponding to 3059 * CR0.PG) is 1. 3060 */ 3061 if (to_vmx(vcpu)->nested.nested_run_pending && 3062 (vmcs12->vm_entry_controls & VM_ENTRY_LOAD_IA32_EFER)) { 3063 if (CC(!kvm_valid_efer(vcpu, vmcs12->guest_ia32_efer)) || 3064 CC(ia32e != !!(vmcs12->guest_ia32_efer & EFER_LMA)) || 3065 CC(((vmcs12->guest_cr0 & X86_CR0_PG) && 3066 ia32e != !!(vmcs12->guest_ia32_efer & EFER_LME)))) 3067 return -EINVAL; 3068 } 3069 3070 if ((vmcs12->vm_entry_controls & VM_ENTRY_LOAD_BNDCFGS) && 3071 (CC(is_noncanonical_address(vmcs12->guest_bndcfgs & PAGE_MASK, vcpu)) || 3072 CC((vmcs12->guest_bndcfgs & MSR_IA32_BNDCFGS_RSVD)))) 3073 return -EINVAL; 3074 3075 if (nested_check_guest_non_reg_state(vmcs12)) 3076 return -EINVAL; 3077 3078 return 0; 3079 } 3080 3081 static int nested_vmx_check_vmentry_hw(struct kvm_vcpu *vcpu) 3082 { 3083 struct vcpu_vmx *vmx = to_vmx(vcpu); 3084 unsigned long cr3, cr4; 3085 bool vm_fail; 3086 3087 if (!nested_early_check) 3088 return 0; 3089 3090 if (vmx->msr_autoload.host.nr) 3091 vmcs_write32(VM_EXIT_MSR_LOAD_COUNT, 0); 3092 if (vmx->msr_autoload.guest.nr) 3093 vmcs_write32(VM_ENTRY_MSR_LOAD_COUNT, 0); 3094 3095 preempt_disable(); 3096 3097 vmx_prepare_switch_to_guest(vcpu); 3098 3099 /* 3100 * Induce a consistency check VMExit by clearing bit 1 in GUEST_RFLAGS, 3101 * which is reserved to '1' by hardware. GUEST_RFLAGS is guaranteed to 3102 * be written (by prepare_vmcs02()) before the "real" VMEnter, i.e. 3103 * there is no need to preserve other bits or save/restore the field. 3104 */ 3105 vmcs_writel(GUEST_RFLAGS, 0); 3106 3107 cr3 = __get_current_cr3_fast(); 3108 if (unlikely(cr3 != vmx->loaded_vmcs->host_state.cr3)) { 3109 vmcs_writel(HOST_CR3, cr3); 3110 vmx->loaded_vmcs->host_state.cr3 = cr3; 3111 } 3112 3113 cr4 = cr4_read_shadow(); 3114 if (unlikely(cr4 != vmx->loaded_vmcs->host_state.cr4)) { 3115 vmcs_writel(HOST_CR4, cr4); 3116 vmx->loaded_vmcs->host_state.cr4 = cr4; 3117 } 3118 3119 vm_fail = __vmx_vcpu_run(vmx, (unsigned long *)&vcpu->arch.regs, 3120 __vmx_vcpu_run_flags(vmx)); 3121 3122 if (vmx->msr_autoload.host.nr) 3123 vmcs_write32(VM_EXIT_MSR_LOAD_COUNT, vmx->msr_autoload.host.nr); 3124 if (vmx->msr_autoload.guest.nr) 3125 vmcs_write32(VM_ENTRY_MSR_LOAD_COUNT, vmx->msr_autoload.guest.nr); 3126 3127 if (vm_fail) { 3128 u32 error = vmcs_read32(VM_INSTRUCTION_ERROR); 3129 3130 preempt_enable(); 3131 3132 trace_kvm_nested_vmenter_failed( 3133 "early hardware check VM-instruction error: ", error); 3134 WARN_ON_ONCE(error != VMXERR_ENTRY_INVALID_CONTROL_FIELD); 3135 return 1; 3136 } 3137 3138 /* 3139 * VMExit clears RFLAGS.IF and DR7, even on a consistency check. 3140 */ 3141 if (hw_breakpoint_active()) 3142 set_debugreg(__this_cpu_read(cpu_dr7), 7); 3143 local_irq_enable(); 3144 preempt_enable(); 3145 3146 /* 3147 * A non-failing VMEntry means we somehow entered guest mode with 3148 * an illegal RIP, and that's just the tip of the iceberg. There 3149 * is no telling what memory has been modified or what state has 3150 * been exposed to unknown code. Hitting this all but guarantees 3151 * a (very critical) hardware issue. 3152 */ 3153 WARN_ON(!(vmcs_read32(VM_EXIT_REASON) & 3154 VMX_EXIT_REASONS_FAILED_VMENTRY)); 3155 3156 return 0; 3157 } 3158 3159 static bool nested_get_evmcs_page(struct kvm_vcpu *vcpu) 3160 { 3161 struct vcpu_vmx *vmx = to_vmx(vcpu); 3162 3163 /* 3164 * hv_evmcs may end up being not mapped after migration (when 3165 * L2 was running), map it here to make sure vmcs12 changes are 3166 * properly reflected. 3167 */ 3168 if (guest_cpuid_has_evmcs(vcpu) && 3169 vmx->nested.hv_evmcs_vmptr == EVMPTR_MAP_PENDING) { 3170 enum nested_evmptrld_status evmptrld_status = 3171 nested_vmx_handle_enlightened_vmptrld(vcpu, false); 3172 3173 if (evmptrld_status == EVMPTRLD_VMFAIL || 3174 evmptrld_status == EVMPTRLD_ERROR) 3175 return false; 3176 3177 /* 3178 * Post migration VMCS12 always provides the most actual 3179 * information, copy it to eVMCS upon entry. 3180 */ 3181 vmx->nested.need_vmcs12_to_shadow_sync = true; 3182 } 3183 3184 return true; 3185 } 3186 3187 static bool nested_get_vmcs12_pages(struct kvm_vcpu *vcpu) 3188 { 3189 struct vmcs12 *vmcs12 = get_vmcs12(vcpu); 3190 struct vcpu_vmx *vmx = to_vmx(vcpu); 3191 struct kvm_host_map *map; 3192 3193 if (!vcpu->arch.pdptrs_from_userspace && 3194 !nested_cpu_has_ept(vmcs12) && is_pae_paging(vcpu)) { 3195 /* 3196 * Reload the guest's PDPTRs since after a migration 3197 * the guest CR3 might be restored prior to setting the nested 3198 * state which can lead to a load of wrong PDPTRs. 3199 */ 3200 if (CC(!load_pdptrs(vcpu, vcpu->arch.cr3))) 3201 return false; 3202 } 3203 3204 3205 if (nested_cpu_has2(vmcs12, SECONDARY_EXEC_VIRTUALIZE_APIC_ACCESSES)) { 3206 map = &vmx->nested.apic_access_page_map; 3207 3208 if (!kvm_vcpu_map(vcpu, gpa_to_gfn(vmcs12->apic_access_addr), map)) { 3209 vmcs_write64(APIC_ACCESS_ADDR, pfn_to_hpa(map->pfn)); 3210 } else { 3211 pr_debug_ratelimited("%s: no backing for APIC-access address in vmcs12\n", 3212 __func__); 3213 vcpu->run->exit_reason = KVM_EXIT_INTERNAL_ERROR; 3214 vcpu->run->internal.suberror = 3215 KVM_INTERNAL_ERROR_EMULATION; 3216 vcpu->run->internal.ndata = 0; 3217 return false; 3218 } 3219 } 3220 3221 if (nested_cpu_has(vmcs12, CPU_BASED_TPR_SHADOW)) { 3222 map = &vmx->nested.virtual_apic_map; 3223 3224 if (!kvm_vcpu_map(vcpu, gpa_to_gfn(vmcs12->virtual_apic_page_addr), map)) { 3225 vmcs_write64(VIRTUAL_APIC_PAGE_ADDR, pfn_to_hpa(map->pfn)); 3226 } else if (nested_cpu_has(vmcs12, CPU_BASED_CR8_LOAD_EXITING) && 3227 nested_cpu_has(vmcs12, CPU_BASED_CR8_STORE_EXITING) && 3228 !nested_cpu_has2(vmcs12, SECONDARY_EXEC_VIRTUALIZE_APIC_ACCESSES)) { 3229 /* 3230 * The processor will never use the TPR shadow, simply 3231 * clear the bit from the execution control. Such a 3232 * configuration is useless, but it happens in tests. 3233 * For any other configuration, failing the vm entry is 3234 * _not_ what the processor does but it's basically the 3235 * only possibility we have. 3236 */ 3237 exec_controls_clearbit(vmx, CPU_BASED_TPR_SHADOW); 3238 } else { 3239 /* 3240 * Write an illegal value to VIRTUAL_APIC_PAGE_ADDR to 3241 * force VM-Entry to fail. 3242 */ 3243 vmcs_write64(VIRTUAL_APIC_PAGE_ADDR, INVALID_GPA); 3244 } 3245 } 3246 3247 if (nested_cpu_has_posted_intr(vmcs12)) { 3248 map = &vmx->nested.pi_desc_map; 3249 3250 if (!kvm_vcpu_map(vcpu, gpa_to_gfn(vmcs12->posted_intr_desc_addr), map)) { 3251 vmx->nested.pi_desc = 3252 (struct pi_desc *)(((void *)map->hva) + 3253 offset_in_page(vmcs12->posted_intr_desc_addr)); 3254 vmcs_write64(POSTED_INTR_DESC_ADDR, 3255 pfn_to_hpa(map->pfn) + offset_in_page(vmcs12->posted_intr_desc_addr)); 3256 } else { 3257 /* 3258 * Defer the KVM_INTERNAL_EXIT until KVM tries to 3259 * access the contents of the VMCS12 posted interrupt 3260 * descriptor. (Note that KVM may do this when it 3261 * should not, per the architectural specification.) 3262 */ 3263 vmx->nested.pi_desc = NULL; 3264 pin_controls_clearbit(vmx, PIN_BASED_POSTED_INTR); 3265 } 3266 } 3267 if (nested_vmx_prepare_msr_bitmap(vcpu, vmcs12)) 3268 exec_controls_setbit(vmx, CPU_BASED_USE_MSR_BITMAPS); 3269 else 3270 exec_controls_clearbit(vmx, CPU_BASED_USE_MSR_BITMAPS); 3271 3272 return true; 3273 } 3274 3275 static bool vmx_get_nested_state_pages(struct kvm_vcpu *vcpu) 3276 { 3277 /* 3278 * Note: nested_get_evmcs_page() also updates 'vp_assist_page' copy 3279 * in 'struct kvm_vcpu_hv' in case eVMCS is in use, this is mandatory 3280 * to make nested_evmcs_l2_tlb_flush_enabled() work correctly post 3281 * migration. 3282 */ 3283 if (!nested_get_evmcs_page(vcpu)) { 3284 pr_debug_ratelimited("%s: enlightened vmptrld failed\n", 3285 __func__); 3286 vcpu->run->exit_reason = KVM_EXIT_INTERNAL_ERROR; 3287 vcpu->run->internal.suberror = 3288 KVM_INTERNAL_ERROR_EMULATION; 3289 vcpu->run->internal.ndata = 0; 3290 3291 return false; 3292 } 3293 3294 if (is_guest_mode(vcpu) && !nested_get_vmcs12_pages(vcpu)) 3295 return false; 3296 3297 return true; 3298 } 3299 3300 static int nested_vmx_write_pml_buffer(struct kvm_vcpu *vcpu, gpa_t gpa) 3301 { 3302 struct vmcs12 *vmcs12; 3303 struct vcpu_vmx *vmx = to_vmx(vcpu); 3304 gpa_t dst; 3305 3306 if (WARN_ON_ONCE(!is_guest_mode(vcpu))) 3307 return 0; 3308 3309 if (WARN_ON_ONCE(vmx->nested.pml_full)) 3310 return 1; 3311 3312 /* 3313 * Check if PML is enabled for the nested guest. Whether eptp bit 6 is 3314 * set is already checked as part of A/D emulation. 3315 */ 3316 vmcs12 = get_vmcs12(vcpu); 3317 if (!nested_cpu_has_pml(vmcs12)) 3318 return 0; 3319 3320 if (vmcs12->guest_pml_index >= PML_ENTITY_NUM) { 3321 vmx->nested.pml_full = true; 3322 return 1; 3323 } 3324 3325 gpa &= ~0xFFFull; 3326 dst = vmcs12->pml_address + sizeof(u64) * vmcs12->guest_pml_index; 3327 3328 if (kvm_write_guest_page(vcpu->kvm, gpa_to_gfn(dst), &gpa, 3329 offset_in_page(dst), sizeof(gpa))) 3330 return 0; 3331 3332 vmcs12->guest_pml_index--; 3333 3334 return 0; 3335 } 3336 3337 /* 3338 * Intel's VMX Instruction Reference specifies a common set of prerequisites 3339 * for running VMX instructions (except VMXON, whose prerequisites are 3340 * slightly different). It also specifies what exception to inject otherwise. 3341 * Note that many of these exceptions have priority over VM exits, so they 3342 * don't have to be checked again here. 3343 */ 3344 static int nested_vmx_check_permission(struct kvm_vcpu *vcpu) 3345 { 3346 if (!to_vmx(vcpu)->nested.vmxon) { 3347 kvm_queue_exception(vcpu, UD_VECTOR); 3348 return 0; 3349 } 3350 3351 if (vmx_get_cpl(vcpu)) { 3352 kvm_inject_gp(vcpu, 0); 3353 return 0; 3354 } 3355 3356 return 1; 3357 } 3358 3359 static u8 vmx_has_apicv_interrupt(struct kvm_vcpu *vcpu) 3360 { 3361 u8 rvi = vmx_get_rvi(); 3362 u8 vppr = kvm_lapic_get_reg(vcpu->arch.apic, APIC_PROCPRI); 3363 3364 return ((rvi & 0xf0) > (vppr & 0xf0)); 3365 } 3366 3367 static void load_vmcs12_host_state(struct kvm_vcpu *vcpu, 3368 struct vmcs12 *vmcs12); 3369 3370 /* 3371 * If from_vmentry is false, this is being called from state restore (either RSM 3372 * or KVM_SET_NESTED_STATE). Otherwise it's called from vmlaunch/vmresume. 3373 * 3374 * Returns: 3375 * NVMX_VMENTRY_SUCCESS: Entered VMX non-root mode 3376 * NVMX_VMENTRY_VMFAIL: Consistency check VMFail 3377 * NVMX_VMENTRY_VMEXIT: Consistency check VMExit 3378 * NVMX_VMENTRY_KVM_INTERNAL_ERROR: KVM internal error 3379 */ 3380 enum nvmx_vmentry_status nested_vmx_enter_non_root_mode(struct kvm_vcpu *vcpu, 3381 bool from_vmentry) 3382 { 3383 struct vcpu_vmx *vmx = to_vmx(vcpu); 3384 struct vmcs12 *vmcs12 = get_vmcs12(vcpu); 3385 enum vm_entry_failure_code entry_failure_code; 3386 bool evaluate_pending_interrupts; 3387 union vmx_exit_reason exit_reason = { 3388 .basic = EXIT_REASON_INVALID_STATE, 3389 .failed_vmentry = 1, 3390 }; 3391 u32 failed_index; 3392 3393 trace_kvm_nested_vmenter(kvm_rip_read(vcpu), 3394 vmx->nested.current_vmptr, 3395 vmcs12->guest_rip, 3396 vmcs12->guest_intr_status, 3397 vmcs12->vm_entry_intr_info_field, 3398 vmcs12->secondary_vm_exec_control & SECONDARY_EXEC_ENABLE_EPT, 3399 vmcs12->ept_pointer, 3400 vmcs12->guest_cr3, 3401 KVM_ISA_VMX); 3402 3403 kvm_service_local_tlb_flush_requests(vcpu); 3404 3405 evaluate_pending_interrupts = exec_controls_get(vmx) & 3406 (CPU_BASED_INTR_WINDOW_EXITING | CPU_BASED_NMI_WINDOW_EXITING); 3407 if (likely(!evaluate_pending_interrupts) && kvm_vcpu_apicv_active(vcpu)) 3408 evaluate_pending_interrupts |= vmx_has_apicv_interrupt(vcpu); 3409 if (!evaluate_pending_interrupts) 3410 evaluate_pending_interrupts |= kvm_apic_has_pending_init_or_sipi(vcpu); 3411 3412 if (!vmx->nested.nested_run_pending || 3413 !(vmcs12->vm_entry_controls & VM_ENTRY_LOAD_DEBUG_CONTROLS)) 3414 vmx->nested.pre_vmenter_debugctl = vmcs_read64(GUEST_IA32_DEBUGCTL); 3415 if (kvm_mpx_supported() && 3416 (!vmx->nested.nested_run_pending || 3417 !(vmcs12->vm_entry_controls & VM_ENTRY_LOAD_BNDCFGS))) 3418 vmx->nested.pre_vmenter_bndcfgs = vmcs_read64(GUEST_BNDCFGS); 3419 3420 /* 3421 * Overwrite vmcs01.GUEST_CR3 with L1's CR3 if EPT is disabled *and* 3422 * nested early checks are disabled. In the event of a "late" VM-Fail, 3423 * i.e. a VM-Fail detected by hardware but not KVM, KVM must unwind its 3424 * software model to the pre-VMEntry host state. When EPT is disabled, 3425 * GUEST_CR3 holds KVM's shadow CR3, not L1's "real" CR3, which causes 3426 * nested_vmx_restore_host_state() to corrupt vcpu->arch.cr3. Stuffing 3427 * vmcs01.GUEST_CR3 results in the unwind naturally setting arch.cr3 to 3428 * the correct value. Smashing vmcs01.GUEST_CR3 is safe because nested 3429 * VM-Exits, and the unwind, reset KVM's MMU, i.e. vmcs01.GUEST_CR3 is 3430 * guaranteed to be overwritten with a shadow CR3 prior to re-entering 3431 * L1. Don't stuff vmcs01.GUEST_CR3 when using nested early checks as 3432 * KVM modifies vcpu->arch.cr3 if and only if the early hardware checks 3433 * pass, and early VM-Fails do not reset KVM's MMU, i.e. the VM-Fail 3434 * path would need to manually save/restore vmcs01.GUEST_CR3. 3435 */ 3436 if (!enable_ept && !nested_early_check) 3437 vmcs_writel(GUEST_CR3, vcpu->arch.cr3); 3438 3439 vmx_switch_vmcs(vcpu, &vmx->nested.vmcs02); 3440 3441 prepare_vmcs02_early(vmx, &vmx->vmcs01, vmcs12); 3442 3443 if (from_vmentry) { 3444 if (unlikely(!nested_get_vmcs12_pages(vcpu))) { 3445 vmx_switch_vmcs(vcpu, &vmx->vmcs01); 3446 return NVMX_VMENTRY_KVM_INTERNAL_ERROR; 3447 } 3448 3449 if (nested_vmx_check_vmentry_hw(vcpu)) { 3450 vmx_switch_vmcs(vcpu, &vmx->vmcs01); 3451 return NVMX_VMENTRY_VMFAIL; 3452 } 3453 3454 if (nested_vmx_check_guest_state(vcpu, vmcs12, 3455 &entry_failure_code)) { 3456 exit_reason.basic = EXIT_REASON_INVALID_STATE; 3457 vmcs12->exit_qualification = entry_failure_code; 3458 goto vmentry_fail_vmexit; 3459 } 3460 } 3461 3462 enter_guest_mode(vcpu); 3463 3464 if (prepare_vmcs02(vcpu, vmcs12, from_vmentry, &entry_failure_code)) { 3465 exit_reason.basic = EXIT_REASON_INVALID_STATE; 3466 vmcs12->exit_qualification = entry_failure_code; 3467 goto vmentry_fail_vmexit_guest_mode; 3468 } 3469 3470 if (from_vmentry) { 3471 failed_index = nested_vmx_load_msr(vcpu, 3472 vmcs12->vm_entry_msr_load_addr, 3473 vmcs12->vm_entry_msr_load_count); 3474 if (failed_index) { 3475 exit_reason.basic = EXIT_REASON_MSR_LOAD_FAIL; 3476 vmcs12->exit_qualification = failed_index; 3477 goto vmentry_fail_vmexit_guest_mode; 3478 } 3479 } else { 3480 /* 3481 * The MMU is not initialized to point at the right entities yet and 3482 * "get pages" would need to read data from the guest (i.e. we will 3483 * need to perform gpa to hpa translation). Request a call 3484 * to nested_get_vmcs12_pages before the next VM-entry. The MSRs 3485 * have already been set at vmentry time and should not be reset. 3486 */ 3487 kvm_make_request(KVM_REQ_GET_NESTED_STATE_PAGES, vcpu); 3488 } 3489 3490 /* 3491 * Re-evaluate pending events if L1 had a pending IRQ/NMI/INIT/SIPI 3492 * when it executed VMLAUNCH/VMRESUME, as entering non-root mode can 3493 * effectively unblock various events, e.g. INIT/SIPI cause VM-Exit 3494 * unconditionally. 3495 */ 3496 if (unlikely(evaluate_pending_interrupts)) 3497 kvm_make_request(KVM_REQ_EVENT, vcpu); 3498 3499 /* 3500 * Do not start the preemption timer hrtimer until after we know 3501 * we are successful, so that only nested_vmx_vmexit needs to cancel 3502 * the timer. 3503 */ 3504 vmx->nested.preemption_timer_expired = false; 3505 if (nested_cpu_has_preemption_timer(vmcs12)) { 3506 u64 timer_value = vmx_calc_preemption_timer_value(vcpu); 3507 vmx_start_preemption_timer(vcpu, timer_value); 3508 } 3509 3510 /* 3511 * Note no nested_vmx_succeed or nested_vmx_fail here. At this point 3512 * we are no longer running L1, and VMLAUNCH/VMRESUME has not yet 3513 * returned as far as L1 is concerned. It will only return (and set 3514 * the success flag) when L2 exits (see nested_vmx_vmexit()). 3515 */ 3516 return NVMX_VMENTRY_SUCCESS; 3517 3518 /* 3519 * A failed consistency check that leads to a VMExit during L1's 3520 * VMEnter to L2 is a variation of a normal VMexit, as explained in 3521 * 26.7 "VM-entry failures during or after loading guest state". 3522 */ 3523 vmentry_fail_vmexit_guest_mode: 3524 if (vmcs12->cpu_based_vm_exec_control & CPU_BASED_USE_TSC_OFFSETTING) 3525 vcpu->arch.tsc_offset -= vmcs12->tsc_offset; 3526 leave_guest_mode(vcpu); 3527 3528 vmentry_fail_vmexit: 3529 vmx_switch_vmcs(vcpu, &vmx->vmcs01); 3530 3531 if (!from_vmentry) 3532 return NVMX_VMENTRY_VMEXIT; 3533 3534 load_vmcs12_host_state(vcpu, vmcs12); 3535 vmcs12->vm_exit_reason = exit_reason.full; 3536 if (enable_shadow_vmcs || evmptr_is_valid(vmx->nested.hv_evmcs_vmptr)) 3537 vmx->nested.need_vmcs12_to_shadow_sync = true; 3538 return NVMX_VMENTRY_VMEXIT; 3539 } 3540 3541 /* 3542 * nested_vmx_run() handles a nested entry, i.e., a VMLAUNCH or VMRESUME on L1 3543 * for running an L2 nested guest. 3544 */ 3545 static int nested_vmx_run(struct kvm_vcpu *vcpu, bool launch) 3546 { 3547 struct vmcs12 *vmcs12; 3548 enum nvmx_vmentry_status status; 3549 struct vcpu_vmx *vmx = to_vmx(vcpu); 3550 u32 interrupt_shadow = vmx_get_interrupt_shadow(vcpu); 3551 enum nested_evmptrld_status evmptrld_status; 3552 3553 if (!nested_vmx_check_permission(vcpu)) 3554 return 1; 3555 3556 evmptrld_status = nested_vmx_handle_enlightened_vmptrld(vcpu, launch); 3557 if (evmptrld_status == EVMPTRLD_ERROR) { 3558 kvm_queue_exception(vcpu, UD_VECTOR); 3559 return 1; 3560 } 3561 3562 kvm_pmu_trigger_event(vcpu, PERF_COUNT_HW_BRANCH_INSTRUCTIONS); 3563 3564 if (CC(evmptrld_status == EVMPTRLD_VMFAIL)) 3565 return nested_vmx_failInvalid(vcpu); 3566 3567 if (CC(!evmptr_is_valid(vmx->nested.hv_evmcs_vmptr) && 3568 vmx->nested.current_vmptr == INVALID_GPA)) 3569 return nested_vmx_failInvalid(vcpu); 3570 3571 vmcs12 = get_vmcs12(vcpu); 3572 3573 /* 3574 * Can't VMLAUNCH or VMRESUME a shadow VMCS. Despite the fact 3575 * that there *is* a valid VMCS pointer, RFLAGS.CF is set 3576 * rather than RFLAGS.ZF, and no error number is stored to the 3577 * VM-instruction error field. 3578 */ 3579 if (CC(vmcs12->hdr.shadow_vmcs)) 3580 return nested_vmx_failInvalid(vcpu); 3581 3582 if (evmptr_is_valid(vmx->nested.hv_evmcs_vmptr)) { 3583 copy_enlightened_to_vmcs12(vmx, vmx->nested.hv_evmcs->hv_clean_fields); 3584 /* Enlightened VMCS doesn't have launch state */ 3585 vmcs12->launch_state = !launch; 3586 } else if (enable_shadow_vmcs) { 3587 copy_shadow_to_vmcs12(vmx); 3588 } 3589 3590 /* 3591 * The nested entry process starts with enforcing various prerequisites 3592 * on vmcs12 as required by the Intel SDM, and act appropriately when 3593 * they fail: As the SDM explains, some conditions should cause the 3594 * instruction to fail, while others will cause the instruction to seem 3595 * to succeed, but return an EXIT_REASON_INVALID_STATE. 3596 * To speed up the normal (success) code path, we should avoid checking 3597 * for misconfigurations which will anyway be caught by the processor 3598 * when using the merged vmcs02. 3599 */ 3600 if (CC(interrupt_shadow & KVM_X86_SHADOW_INT_MOV_SS)) 3601 return nested_vmx_fail(vcpu, VMXERR_ENTRY_EVENTS_BLOCKED_BY_MOV_SS); 3602 3603 if (CC(vmcs12->launch_state == launch)) 3604 return nested_vmx_fail(vcpu, 3605 launch ? VMXERR_VMLAUNCH_NONCLEAR_VMCS 3606 : VMXERR_VMRESUME_NONLAUNCHED_VMCS); 3607 3608 if (nested_vmx_check_controls(vcpu, vmcs12)) 3609 return nested_vmx_fail(vcpu, VMXERR_ENTRY_INVALID_CONTROL_FIELD); 3610 3611 if (nested_vmx_check_address_space_size(vcpu, vmcs12)) 3612 return nested_vmx_fail(vcpu, VMXERR_ENTRY_INVALID_HOST_STATE_FIELD); 3613 3614 if (nested_vmx_check_host_state(vcpu, vmcs12)) 3615 return nested_vmx_fail(vcpu, VMXERR_ENTRY_INVALID_HOST_STATE_FIELD); 3616 3617 /* 3618 * We're finally done with prerequisite checking, and can start with 3619 * the nested entry. 3620 */ 3621 vmx->nested.nested_run_pending = 1; 3622 vmx->nested.has_preemption_timer_deadline = false; 3623 status = nested_vmx_enter_non_root_mode(vcpu, true); 3624 if (unlikely(status != NVMX_VMENTRY_SUCCESS)) 3625 goto vmentry_failed; 3626 3627 /* Emulate processing of posted interrupts on VM-Enter. */ 3628 if (nested_cpu_has_posted_intr(vmcs12) && 3629 kvm_apic_has_interrupt(vcpu) == vmx->nested.posted_intr_nv) { 3630 vmx->nested.pi_pending = true; 3631 kvm_make_request(KVM_REQ_EVENT, vcpu); 3632 kvm_apic_clear_irr(vcpu, vmx->nested.posted_intr_nv); 3633 } 3634 3635 /* Hide L1D cache contents from the nested guest. */ 3636 vmx->vcpu.arch.l1tf_flush_l1d = true; 3637 3638 /* 3639 * Must happen outside of nested_vmx_enter_non_root_mode() as it will 3640 * also be used as part of restoring nVMX state for 3641 * snapshot restore (migration). 3642 * 3643 * In this flow, it is assumed that vmcs12 cache was 3644 * transferred as part of captured nVMX state and should 3645 * therefore not be read from guest memory (which may not 3646 * exist on destination host yet). 3647 */ 3648 nested_cache_shadow_vmcs12(vcpu, vmcs12); 3649 3650 switch (vmcs12->guest_activity_state) { 3651 case GUEST_ACTIVITY_HLT: 3652 /* 3653 * If we're entering a halted L2 vcpu and the L2 vcpu won't be 3654 * awakened by event injection or by an NMI-window VM-exit or 3655 * by an interrupt-window VM-exit, halt the vcpu. 3656 */ 3657 if (!(vmcs12->vm_entry_intr_info_field & INTR_INFO_VALID_MASK) && 3658 !nested_cpu_has(vmcs12, CPU_BASED_NMI_WINDOW_EXITING) && 3659 !(nested_cpu_has(vmcs12, CPU_BASED_INTR_WINDOW_EXITING) && 3660 (vmcs12->guest_rflags & X86_EFLAGS_IF))) { 3661 vmx->nested.nested_run_pending = 0; 3662 return kvm_emulate_halt_noskip(vcpu); 3663 } 3664 break; 3665 case GUEST_ACTIVITY_WAIT_SIPI: 3666 vmx->nested.nested_run_pending = 0; 3667 vcpu->arch.mp_state = KVM_MP_STATE_INIT_RECEIVED; 3668 break; 3669 default: 3670 break; 3671 } 3672 3673 return 1; 3674 3675 vmentry_failed: 3676 vmx->nested.nested_run_pending = 0; 3677 if (status == NVMX_VMENTRY_KVM_INTERNAL_ERROR) 3678 return 0; 3679 if (status == NVMX_VMENTRY_VMEXIT) 3680 return 1; 3681 WARN_ON_ONCE(status != NVMX_VMENTRY_VMFAIL); 3682 return nested_vmx_fail(vcpu, VMXERR_ENTRY_INVALID_CONTROL_FIELD); 3683 } 3684 3685 /* 3686 * On a nested exit from L2 to L1, vmcs12.guest_cr0 might not be up-to-date 3687 * because L2 may have changed some cr0 bits directly (CR0_GUEST_HOST_MASK). 3688 * This function returns the new value we should put in vmcs12.guest_cr0. 3689 * It's not enough to just return the vmcs02 GUEST_CR0. Rather, 3690 * 1. Bits that neither L0 nor L1 trapped, were set directly by L2 and are now 3691 * available in vmcs02 GUEST_CR0. (Note: It's enough to check that L0 3692 * didn't trap the bit, because if L1 did, so would L0). 3693 * 2. Bits that L1 asked to trap (and therefore L0 also did) could not have 3694 * been modified by L2, and L1 knows it. So just leave the old value of 3695 * the bit from vmcs12.guest_cr0. Note that the bit from vmcs02 GUEST_CR0 3696 * isn't relevant, because if L0 traps this bit it can set it to anything. 3697 * 3. Bits that L1 didn't trap, but L0 did. L1 believes the guest could have 3698 * changed these bits, and therefore they need to be updated, but L0 3699 * didn't necessarily allow them to be changed in GUEST_CR0 - and rather 3700 * put them in vmcs02 CR0_READ_SHADOW. So take these bits from there. 3701 */ 3702 static inline unsigned long 3703 vmcs12_guest_cr0(struct kvm_vcpu *vcpu, struct vmcs12 *vmcs12) 3704 { 3705 return 3706 /*1*/ (vmcs_readl(GUEST_CR0) & vcpu->arch.cr0_guest_owned_bits) | 3707 /*2*/ (vmcs12->guest_cr0 & vmcs12->cr0_guest_host_mask) | 3708 /*3*/ (vmcs_readl(CR0_READ_SHADOW) & ~(vmcs12->cr0_guest_host_mask | 3709 vcpu->arch.cr0_guest_owned_bits)); 3710 } 3711 3712 static inline unsigned long 3713 vmcs12_guest_cr4(struct kvm_vcpu *vcpu, struct vmcs12 *vmcs12) 3714 { 3715 return 3716 /*1*/ (vmcs_readl(GUEST_CR4) & vcpu->arch.cr4_guest_owned_bits) | 3717 /*2*/ (vmcs12->guest_cr4 & vmcs12->cr4_guest_host_mask) | 3718 /*3*/ (vmcs_readl(CR4_READ_SHADOW) & ~(vmcs12->cr4_guest_host_mask | 3719 vcpu->arch.cr4_guest_owned_bits)); 3720 } 3721 3722 static void vmcs12_save_pending_event(struct kvm_vcpu *vcpu, 3723 struct vmcs12 *vmcs12, 3724 u32 vm_exit_reason, u32 exit_intr_info) 3725 { 3726 u32 idt_vectoring; 3727 unsigned int nr; 3728 3729 /* 3730 * Per the SDM, VM-Exits due to double and triple faults are never 3731 * considered to occur during event delivery, even if the double/triple 3732 * fault is the result of an escalating vectoring issue. 3733 * 3734 * Note, the SDM qualifies the double fault behavior with "The original 3735 * event results in a double-fault exception". It's unclear why the 3736 * qualification exists since exits due to double fault can occur only 3737 * while vectoring a different exception (injected events are never 3738 * subject to interception), i.e. there's _always_ an original event. 3739 * 3740 * The SDM also uses NMI as a confusing example for the "original event 3741 * causes the VM exit directly" clause. NMI isn't special in any way, 3742 * the same rule applies to all events that cause an exit directly. 3743 * NMI is an odd choice for the example because NMIs can only occur on 3744 * instruction boundaries, i.e. they _can't_ occur during vectoring. 3745 */ 3746 if ((u16)vm_exit_reason == EXIT_REASON_TRIPLE_FAULT || 3747 ((u16)vm_exit_reason == EXIT_REASON_EXCEPTION_NMI && 3748 is_double_fault(exit_intr_info))) { 3749 vmcs12->idt_vectoring_info_field = 0; 3750 } else if (vcpu->arch.exception.injected) { 3751 nr = vcpu->arch.exception.vector; 3752 idt_vectoring = nr | VECTORING_INFO_VALID_MASK; 3753 3754 if (kvm_exception_is_soft(nr)) { 3755 vmcs12->vm_exit_instruction_len = 3756 vcpu->arch.event_exit_inst_len; 3757 idt_vectoring |= INTR_TYPE_SOFT_EXCEPTION; 3758 } else 3759 idt_vectoring |= INTR_TYPE_HARD_EXCEPTION; 3760 3761 if (vcpu->arch.exception.has_error_code) { 3762 idt_vectoring |= VECTORING_INFO_DELIVER_CODE_MASK; 3763 vmcs12->idt_vectoring_error_code = 3764 vcpu->arch.exception.error_code; 3765 } 3766 3767 vmcs12->idt_vectoring_info_field = idt_vectoring; 3768 } else if (vcpu->arch.nmi_injected) { 3769 vmcs12->idt_vectoring_info_field = 3770 INTR_TYPE_NMI_INTR | INTR_INFO_VALID_MASK | NMI_VECTOR; 3771 } else if (vcpu->arch.interrupt.injected) { 3772 nr = vcpu->arch.interrupt.nr; 3773 idt_vectoring = nr | VECTORING_INFO_VALID_MASK; 3774 3775 if (vcpu->arch.interrupt.soft) { 3776 idt_vectoring |= INTR_TYPE_SOFT_INTR; 3777 vmcs12->vm_entry_instruction_len = 3778 vcpu->arch.event_exit_inst_len; 3779 } else 3780 idt_vectoring |= INTR_TYPE_EXT_INTR; 3781 3782 vmcs12->idt_vectoring_info_field = idt_vectoring; 3783 } else { 3784 vmcs12->idt_vectoring_info_field = 0; 3785 } 3786 } 3787 3788 3789 void nested_mark_vmcs12_pages_dirty(struct kvm_vcpu *vcpu) 3790 { 3791 struct vmcs12 *vmcs12 = get_vmcs12(vcpu); 3792 gfn_t gfn; 3793 3794 /* 3795 * Don't need to mark the APIC access page dirty; it is never 3796 * written to by the CPU during APIC virtualization. 3797 */ 3798 3799 if (nested_cpu_has(vmcs12, CPU_BASED_TPR_SHADOW)) { 3800 gfn = vmcs12->virtual_apic_page_addr >> PAGE_SHIFT; 3801 kvm_vcpu_mark_page_dirty(vcpu, gfn); 3802 } 3803 3804 if (nested_cpu_has_posted_intr(vmcs12)) { 3805 gfn = vmcs12->posted_intr_desc_addr >> PAGE_SHIFT; 3806 kvm_vcpu_mark_page_dirty(vcpu, gfn); 3807 } 3808 } 3809 3810 static int vmx_complete_nested_posted_interrupt(struct kvm_vcpu *vcpu) 3811 { 3812 struct vcpu_vmx *vmx = to_vmx(vcpu); 3813 int max_irr; 3814 void *vapic_page; 3815 u16 status; 3816 3817 if (!vmx->nested.pi_pending) 3818 return 0; 3819 3820 if (!vmx->nested.pi_desc) 3821 goto mmio_needed; 3822 3823 vmx->nested.pi_pending = false; 3824 3825 if (!pi_test_and_clear_on(vmx->nested.pi_desc)) 3826 return 0; 3827 3828 max_irr = find_last_bit((unsigned long *)vmx->nested.pi_desc->pir, 256); 3829 if (max_irr != 256) { 3830 vapic_page = vmx->nested.virtual_apic_map.hva; 3831 if (!vapic_page) 3832 goto mmio_needed; 3833 3834 __kvm_apic_update_irr(vmx->nested.pi_desc->pir, 3835 vapic_page, &max_irr); 3836 status = vmcs_read16(GUEST_INTR_STATUS); 3837 if ((u8)max_irr > ((u8)status & 0xff)) { 3838 status &= ~0xff; 3839 status |= (u8)max_irr; 3840 vmcs_write16(GUEST_INTR_STATUS, status); 3841 } 3842 } 3843 3844 nested_mark_vmcs12_pages_dirty(vcpu); 3845 return 0; 3846 3847 mmio_needed: 3848 kvm_handle_memory_failure(vcpu, X86EMUL_IO_NEEDED, NULL); 3849 return -ENXIO; 3850 } 3851 3852 static void nested_vmx_inject_exception_vmexit(struct kvm_vcpu *vcpu) 3853 { 3854 struct kvm_queued_exception *ex = &vcpu->arch.exception_vmexit; 3855 u32 intr_info = ex->vector | INTR_INFO_VALID_MASK; 3856 struct vmcs12 *vmcs12 = get_vmcs12(vcpu); 3857 unsigned long exit_qual; 3858 3859 if (ex->has_payload) { 3860 exit_qual = ex->payload; 3861 } else if (ex->vector == PF_VECTOR) { 3862 exit_qual = vcpu->arch.cr2; 3863 } else if (ex->vector == DB_VECTOR) { 3864 exit_qual = vcpu->arch.dr6; 3865 exit_qual &= ~DR6_BT; 3866 exit_qual ^= DR6_ACTIVE_LOW; 3867 } else { 3868 exit_qual = 0; 3869 } 3870 3871 /* 3872 * Unlike AMD's Paged Real Mode, which reports an error code on #PF 3873 * VM-Exits even if the CPU is in Real Mode, Intel VMX never sets the 3874 * "has error code" flags on VM-Exit if the CPU is in Real Mode. 3875 */ 3876 if (ex->has_error_code && is_protmode(vcpu)) { 3877 /* 3878 * Intel CPUs do not generate error codes with bits 31:16 set, 3879 * and more importantly VMX disallows setting bits 31:16 in the 3880 * injected error code for VM-Entry. Drop the bits to mimic 3881 * hardware and avoid inducing failure on nested VM-Entry if L1 3882 * chooses to inject the exception back to L2. AMD CPUs _do_ 3883 * generate "full" 32-bit error codes, so KVM allows userspace 3884 * to inject exception error codes with bits 31:16 set. 3885 */ 3886 vmcs12->vm_exit_intr_error_code = (u16)ex->error_code; 3887 intr_info |= INTR_INFO_DELIVER_CODE_MASK; 3888 } 3889 3890 if (kvm_exception_is_soft(ex->vector)) 3891 intr_info |= INTR_TYPE_SOFT_EXCEPTION; 3892 else 3893 intr_info |= INTR_TYPE_HARD_EXCEPTION; 3894 3895 if (!(vmcs12->idt_vectoring_info_field & VECTORING_INFO_VALID_MASK) && 3896 vmx_get_nmi_mask(vcpu)) 3897 intr_info |= INTR_INFO_UNBLOCK_NMI; 3898 3899 nested_vmx_vmexit(vcpu, EXIT_REASON_EXCEPTION_NMI, intr_info, exit_qual); 3900 } 3901 3902 /* 3903 * Returns true if a debug trap is (likely) pending delivery. Infer the class 3904 * of a #DB (trap-like vs. fault-like) from the exception payload (to-be-DR6). 3905 * Using the payload is flawed because code breakpoints (fault-like) and data 3906 * breakpoints (trap-like) set the same bits in DR6 (breakpoint detected), i.e. 3907 * this will return false positives if a to-be-injected code breakpoint #DB is 3908 * pending (from KVM's perspective, but not "pending" across an instruction 3909 * boundary). ICEBP, a.k.a. INT1, is also not reflected here even though it 3910 * too is trap-like. 3911 * 3912 * KVM "works" despite these flaws as ICEBP isn't currently supported by the 3913 * emulator, Monitor Trap Flag is not marked pending on intercepted #DBs (the 3914 * #DB has already happened), and MTF isn't marked pending on code breakpoints 3915 * from the emulator (because such #DBs are fault-like and thus don't trigger 3916 * actions that fire on instruction retire). 3917 */ 3918 static unsigned long vmx_get_pending_dbg_trap(struct kvm_queued_exception *ex) 3919 { 3920 if (!ex->pending || ex->vector != DB_VECTOR) 3921 return 0; 3922 3923 /* General Detect #DBs are always fault-like. */ 3924 return ex->payload & ~DR6_BD; 3925 } 3926 3927 /* 3928 * Returns true if there's a pending #DB exception that is lower priority than 3929 * a pending Monitor Trap Flag VM-Exit. TSS T-flag #DBs are not emulated by 3930 * KVM, but could theoretically be injected by userspace. Note, this code is 3931 * imperfect, see above. 3932 */ 3933 static bool vmx_is_low_priority_db_trap(struct kvm_queued_exception *ex) 3934 { 3935 return vmx_get_pending_dbg_trap(ex) & ~DR6_BT; 3936 } 3937 3938 /* 3939 * Certain VM-exits set the 'pending debug exceptions' field to indicate a 3940 * recognized #DB (data or single-step) that has yet to be delivered. Since KVM 3941 * represents these debug traps with a payload that is said to be compatible 3942 * with the 'pending debug exceptions' field, write the payload to the VMCS 3943 * field if a VM-exit is delivered before the debug trap. 3944 */ 3945 static void nested_vmx_update_pending_dbg(struct kvm_vcpu *vcpu) 3946 { 3947 unsigned long pending_dbg; 3948 3949 pending_dbg = vmx_get_pending_dbg_trap(&vcpu->arch.exception); 3950 if (pending_dbg) 3951 vmcs_writel(GUEST_PENDING_DBG_EXCEPTIONS, pending_dbg); 3952 } 3953 3954 static bool nested_vmx_preemption_timer_pending(struct kvm_vcpu *vcpu) 3955 { 3956 return nested_cpu_has_preemption_timer(get_vmcs12(vcpu)) && 3957 to_vmx(vcpu)->nested.preemption_timer_expired; 3958 } 3959 3960 static bool vmx_has_nested_events(struct kvm_vcpu *vcpu) 3961 { 3962 return nested_vmx_preemption_timer_pending(vcpu) || 3963 to_vmx(vcpu)->nested.mtf_pending; 3964 } 3965 3966 /* 3967 * Per the Intel SDM's table "Priority Among Concurrent Events", with minor 3968 * edits to fill in missing examples, e.g. #DB due to split-lock accesses, 3969 * and less minor edits to splice in the priority of VMX Non-Root specific 3970 * events, e.g. MTF and NMI/INTR-window exiting. 3971 * 3972 * 1 Hardware Reset and Machine Checks 3973 * - RESET 3974 * - Machine Check 3975 * 3976 * 2 Trap on Task Switch 3977 * - T flag in TSS is set (on task switch) 3978 * 3979 * 3 External Hardware Interventions 3980 * - FLUSH 3981 * - STOPCLK 3982 * - SMI 3983 * - INIT 3984 * 3985 * 3.5 Monitor Trap Flag (MTF) VM-exit[1] 3986 * 3987 * 4 Traps on Previous Instruction 3988 * - Breakpoints 3989 * - Trap-class Debug Exceptions (#DB due to TF flag set, data/I-O 3990 * breakpoint, or #DB due to a split-lock access) 3991 * 3992 * 4.3 VMX-preemption timer expired VM-exit 3993 * 3994 * 4.6 NMI-window exiting VM-exit[2] 3995 * 3996 * 5 Nonmaskable Interrupts (NMI) 3997 * 3998 * 5.5 Interrupt-window exiting VM-exit and Virtual-interrupt delivery 3999 * 4000 * 6 Maskable Hardware Interrupts 4001 * 4002 * 7 Code Breakpoint Fault 4003 * 4004 * 8 Faults from Fetching Next Instruction 4005 * - Code-Segment Limit Violation 4006 * - Code Page Fault 4007 * - Control protection exception (missing ENDBRANCH at target of indirect 4008 * call or jump) 4009 * 4010 * 9 Faults from Decoding Next Instruction 4011 * - Instruction length > 15 bytes 4012 * - Invalid Opcode 4013 * - Coprocessor Not Available 4014 * 4015 *10 Faults on Executing Instruction 4016 * - Overflow 4017 * - Bound error 4018 * - Invalid TSS 4019 * - Segment Not Present 4020 * - Stack fault 4021 * - General Protection 4022 * - Data Page Fault 4023 * - Alignment Check 4024 * - x86 FPU Floating-point exception 4025 * - SIMD floating-point exception 4026 * - Virtualization exception 4027 * - Control protection exception 4028 * 4029 * [1] Per the "Monitor Trap Flag" section: System-management interrupts (SMIs), 4030 * INIT signals, and higher priority events take priority over MTF VM exits. 4031 * MTF VM exits take priority over debug-trap exceptions and lower priority 4032 * events. 4033 * 4034 * [2] Debug-trap exceptions and higher priority events take priority over VM exits 4035 * caused by the VMX-preemption timer. VM exits caused by the VMX-preemption 4036 * timer take priority over VM exits caused by the "NMI-window exiting" 4037 * VM-execution control and lower priority events. 4038 * 4039 * [3] Debug-trap exceptions and higher priority events take priority over VM exits 4040 * caused by "NMI-window exiting". VM exits caused by this control take 4041 * priority over non-maskable interrupts (NMIs) and lower priority events. 4042 * 4043 * [4] Virtual-interrupt delivery has the same priority as that of VM exits due to 4044 * the 1-setting of the "interrupt-window exiting" VM-execution control. Thus, 4045 * non-maskable interrupts (NMIs) and higher priority events take priority over 4046 * delivery of a virtual interrupt; delivery of a virtual interrupt takes 4047 * priority over external interrupts and lower priority events. 4048 */ 4049 static int vmx_check_nested_events(struct kvm_vcpu *vcpu) 4050 { 4051 struct kvm_lapic *apic = vcpu->arch.apic; 4052 struct vcpu_vmx *vmx = to_vmx(vcpu); 4053 /* 4054 * Only a pending nested run blocks a pending exception. If there is a 4055 * previously injected event, the pending exception occurred while said 4056 * event was being delivered and thus needs to be handled. 4057 */ 4058 bool block_nested_exceptions = vmx->nested.nested_run_pending; 4059 /* 4060 * New events (not exceptions) are only recognized at instruction 4061 * boundaries. If an event needs reinjection, then KVM is handling a 4062 * VM-Exit that occurred _during_ instruction execution; new events are 4063 * blocked until the instruction completes. 4064 */ 4065 bool block_nested_events = block_nested_exceptions || 4066 kvm_event_needs_reinjection(vcpu); 4067 4068 if (lapic_in_kernel(vcpu) && 4069 test_bit(KVM_APIC_INIT, &apic->pending_events)) { 4070 if (block_nested_events) 4071 return -EBUSY; 4072 nested_vmx_update_pending_dbg(vcpu); 4073 clear_bit(KVM_APIC_INIT, &apic->pending_events); 4074 if (vcpu->arch.mp_state != KVM_MP_STATE_INIT_RECEIVED) 4075 nested_vmx_vmexit(vcpu, EXIT_REASON_INIT_SIGNAL, 0, 0); 4076 4077 /* MTF is discarded if the vCPU is in WFS. */ 4078 vmx->nested.mtf_pending = false; 4079 return 0; 4080 } 4081 4082 if (lapic_in_kernel(vcpu) && 4083 test_bit(KVM_APIC_SIPI, &apic->pending_events)) { 4084 if (block_nested_events) 4085 return -EBUSY; 4086 4087 clear_bit(KVM_APIC_SIPI, &apic->pending_events); 4088 if (vcpu->arch.mp_state == KVM_MP_STATE_INIT_RECEIVED) { 4089 nested_vmx_vmexit(vcpu, EXIT_REASON_SIPI_SIGNAL, 0, 4090 apic->sipi_vector & 0xFFUL); 4091 return 0; 4092 } 4093 /* Fallthrough, the SIPI is completely ignored. */ 4094 } 4095 4096 /* 4097 * Process exceptions that are higher priority than Monitor Trap Flag: 4098 * fault-like exceptions, TSS T flag #DB (not emulated by KVM, but 4099 * could theoretically come in from userspace), and ICEBP (INT1). 4100 * 4101 * TODO: SMIs have higher priority than MTF and trap-like #DBs (except 4102 * for TSS T flag #DBs). KVM also doesn't save/restore pending MTF 4103 * across SMI/RSM as it should; that needs to be addressed in order to 4104 * prioritize SMI over MTF and trap-like #DBs. 4105 */ 4106 if (vcpu->arch.exception_vmexit.pending && 4107 !vmx_is_low_priority_db_trap(&vcpu->arch.exception_vmexit)) { 4108 if (block_nested_exceptions) 4109 return -EBUSY; 4110 4111 nested_vmx_inject_exception_vmexit(vcpu); 4112 return 0; 4113 } 4114 4115 if (vcpu->arch.exception.pending && 4116 !vmx_is_low_priority_db_trap(&vcpu->arch.exception)) { 4117 if (block_nested_exceptions) 4118 return -EBUSY; 4119 goto no_vmexit; 4120 } 4121 4122 if (vmx->nested.mtf_pending) { 4123 if (block_nested_events) 4124 return -EBUSY; 4125 nested_vmx_update_pending_dbg(vcpu); 4126 nested_vmx_vmexit(vcpu, EXIT_REASON_MONITOR_TRAP_FLAG, 0, 0); 4127 return 0; 4128 } 4129 4130 if (vcpu->arch.exception_vmexit.pending) { 4131 if (block_nested_exceptions) 4132 return -EBUSY; 4133 4134 nested_vmx_inject_exception_vmexit(vcpu); 4135 return 0; 4136 } 4137 4138 if (vcpu->arch.exception.pending) { 4139 if (block_nested_exceptions) 4140 return -EBUSY; 4141 goto no_vmexit; 4142 } 4143 4144 if (nested_vmx_preemption_timer_pending(vcpu)) { 4145 if (block_nested_events) 4146 return -EBUSY; 4147 nested_vmx_vmexit(vcpu, EXIT_REASON_PREEMPTION_TIMER, 0, 0); 4148 return 0; 4149 } 4150 4151 if (vcpu->arch.smi_pending && !is_smm(vcpu)) { 4152 if (block_nested_events) 4153 return -EBUSY; 4154 goto no_vmexit; 4155 } 4156 4157 if (vcpu->arch.nmi_pending && !vmx_nmi_blocked(vcpu)) { 4158 if (block_nested_events) 4159 return -EBUSY; 4160 if (!nested_exit_on_nmi(vcpu)) 4161 goto no_vmexit; 4162 4163 nested_vmx_vmexit(vcpu, EXIT_REASON_EXCEPTION_NMI, 4164 NMI_VECTOR | INTR_TYPE_NMI_INTR | 4165 INTR_INFO_VALID_MASK, 0); 4166 /* 4167 * The NMI-triggered VM exit counts as injection: 4168 * clear this one and block further NMIs. 4169 */ 4170 vcpu->arch.nmi_pending = 0; 4171 vmx_set_nmi_mask(vcpu, true); 4172 return 0; 4173 } 4174 4175 if (kvm_cpu_has_interrupt(vcpu) && !vmx_interrupt_blocked(vcpu)) { 4176 if (block_nested_events) 4177 return -EBUSY; 4178 if (!nested_exit_on_intr(vcpu)) 4179 goto no_vmexit; 4180 nested_vmx_vmexit(vcpu, EXIT_REASON_EXTERNAL_INTERRUPT, 0, 0); 4181 return 0; 4182 } 4183 4184 no_vmexit: 4185 return vmx_complete_nested_posted_interrupt(vcpu); 4186 } 4187 4188 static u32 vmx_get_preemption_timer_value(struct kvm_vcpu *vcpu) 4189 { 4190 ktime_t remaining = 4191 hrtimer_get_remaining(&to_vmx(vcpu)->nested.preemption_timer); 4192 u64 value; 4193 4194 if (ktime_to_ns(remaining) <= 0) 4195 return 0; 4196 4197 value = ktime_to_ns(remaining) * vcpu->arch.virtual_tsc_khz; 4198 do_div(value, 1000000); 4199 return value >> VMX_MISC_EMULATED_PREEMPTION_TIMER_RATE; 4200 } 4201 4202 static bool is_vmcs12_ext_field(unsigned long field) 4203 { 4204 switch (field) { 4205 case GUEST_ES_SELECTOR: 4206 case GUEST_CS_SELECTOR: 4207 case GUEST_SS_SELECTOR: 4208 case GUEST_DS_SELECTOR: 4209 case GUEST_FS_SELECTOR: 4210 case GUEST_GS_SELECTOR: 4211 case GUEST_LDTR_SELECTOR: 4212 case GUEST_TR_SELECTOR: 4213 case GUEST_ES_LIMIT: 4214 case GUEST_CS_LIMIT: 4215 case GUEST_SS_LIMIT: 4216 case GUEST_DS_LIMIT: 4217 case GUEST_FS_LIMIT: 4218 case GUEST_GS_LIMIT: 4219 case GUEST_LDTR_LIMIT: 4220 case GUEST_TR_LIMIT: 4221 case GUEST_GDTR_LIMIT: 4222 case GUEST_IDTR_LIMIT: 4223 case GUEST_ES_AR_BYTES: 4224 case GUEST_DS_AR_BYTES: 4225 case GUEST_FS_AR_BYTES: 4226 case GUEST_GS_AR_BYTES: 4227 case GUEST_LDTR_AR_BYTES: 4228 case GUEST_TR_AR_BYTES: 4229 case GUEST_ES_BASE: 4230 case GUEST_CS_BASE: 4231 case GUEST_SS_BASE: 4232 case GUEST_DS_BASE: 4233 case GUEST_FS_BASE: 4234 case GUEST_GS_BASE: 4235 case GUEST_LDTR_BASE: 4236 case GUEST_TR_BASE: 4237 case GUEST_GDTR_BASE: 4238 case GUEST_IDTR_BASE: 4239 case GUEST_PENDING_DBG_EXCEPTIONS: 4240 case GUEST_BNDCFGS: 4241 return true; 4242 default: 4243 break; 4244 } 4245 4246 return false; 4247 } 4248 4249 static void sync_vmcs02_to_vmcs12_rare(struct kvm_vcpu *vcpu, 4250 struct vmcs12 *vmcs12) 4251 { 4252 struct vcpu_vmx *vmx = to_vmx(vcpu); 4253 4254 vmcs12->guest_es_selector = vmcs_read16(GUEST_ES_SELECTOR); 4255 vmcs12->guest_cs_selector = vmcs_read16(GUEST_CS_SELECTOR); 4256 vmcs12->guest_ss_selector = vmcs_read16(GUEST_SS_SELECTOR); 4257 vmcs12->guest_ds_selector = vmcs_read16(GUEST_DS_SELECTOR); 4258 vmcs12->guest_fs_selector = vmcs_read16(GUEST_FS_SELECTOR); 4259 vmcs12->guest_gs_selector = vmcs_read16(GUEST_GS_SELECTOR); 4260 vmcs12->guest_ldtr_selector = vmcs_read16(GUEST_LDTR_SELECTOR); 4261 vmcs12->guest_tr_selector = vmcs_read16(GUEST_TR_SELECTOR); 4262 vmcs12->guest_es_limit = vmcs_read32(GUEST_ES_LIMIT); 4263 vmcs12->guest_cs_limit = vmcs_read32(GUEST_CS_LIMIT); 4264 vmcs12->guest_ss_limit = vmcs_read32(GUEST_SS_LIMIT); 4265 vmcs12->guest_ds_limit = vmcs_read32(GUEST_DS_LIMIT); 4266 vmcs12->guest_fs_limit = vmcs_read32(GUEST_FS_LIMIT); 4267 vmcs12->guest_gs_limit = vmcs_read32(GUEST_GS_LIMIT); 4268 vmcs12->guest_ldtr_limit = vmcs_read32(GUEST_LDTR_LIMIT); 4269 vmcs12->guest_tr_limit = vmcs_read32(GUEST_TR_LIMIT); 4270 vmcs12->guest_gdtr_limit = vmcs_read32(GUEST_GDTR_LIMIT); 4271 vmcs12->guest_idtr_limit = vmcs_read32(GUEST_IDTR_LIMIT); 4272 vmcs12->guest_es_ar_bytes = vmcs_read32(GUEST_ES_AR_BYTES); 4273 vmcs12->guest_ds_ar_bytes = vmcs_read32(GUEST_DS_AR_BYTES); 4274 vmcs12->guest_fs_ar_bytes = vmcs_read32(GUEST_FS_AR_BYTES); 4275 vmcs12->guest_gs_ar_bytes = vmcs_read32(GUEST_GS_AR_BYTES); 4276 vmcs12->guest_ldtr_ar_bytes = vmcs_read32(GUEST_LDTR_AR_BYTES); 4277 vmcs12->guest_tr_ar_bytes = vmcs_read32(GUEST_TR_AR_BYTES); 4278 vmcs12->guest_es_base = vmcs_readl(GUEST_ES_BASE); 4279 vmcs12->guest_cs_base = vmcs_readl(GUEST_CS_BASE); 4280 vmcs12->guest_ss_base = vmcs_readl(GUEST_SS_BASE); 4281 vmcs12->guest_ds_base = vmcs_readl(GUEST_DS_BASE); 4282 vmcs12->guest_fs_base = vmcs_readl(GUEST_FS_BASE); 4283 vmcs12->guest_gs_base = vmcs_readl(GUEST_GS_BASE); 4284 vmcs12->guest_ldtr_base = vmcs_readl(GUEST_LDTR_BASE); 4285 vmcs12->guest_tr_base = vmcs_readl(GUEST_TR_BASE); 4286 vmcs12->guest_gdtr_base = vmcs_readl(GUEST_GDTR_BASE); 4287 vmcs12->guest_idtr_base = vmcs_readl(GUEST_IDTR_BASE); 4288 vmcs12->guest_pending_dbg_exceptions = 4289 vmcs_readl(GUEST_PENDING_DBG_EXCEPTIONS); 4290 4291 vmx->nested.need_sync_vmcs02_to_vmcs12_rare = false; 4292 } 4293 4294 static void copy_vmcs02_to_vmcs12_rare(struct kvm_vcpu *vcpu, 4295 struct vmcs12 *vmcs12) 4296 { 4297 struct vcpu_vmx *vmx = to_vmx(vcpu); 4298 int cpu; 4299 4300 if (!vmx->nested.need_sync_vmcs02_to_vmcs12_rare) 4301 return; 4302 4303 4304 WARN_ON_ONCE(vmx->loaded_vmcs != &vmx->vmcs01); 4305 4306 cpu = get_cpu(); 4307 vmx->loaded_vmcs = &vmx->nested.vmcs02; 4308 vmx_vcpu_load_vmcs(vcpu, cpu, &vmx->vmcs01); 4309 4310 sync_vmcs02_to_vmcs12_rare(vcpu, vmcs12); 4311 4312 vmx->loaded_vmcs = &vmx->vmcs01; 4313 vmx_vcpu_load_vmcs(vcpu, cpu, &vmx->nested.vmcs02); 4314 put_cpu(); 4315 } 4316 4317 /* 4318 * Update the guest state fields of vmcs12 to reflect changes that 4319 * occurred while L2 was running. (The "IA-32e mode guest" bit of the 4320 * VM-entry controls is also updated, since this is really a guest 4321 * state bit.) 4322 */ 4323 static void sync_vmcs02_to_vmcs12(struct kvm_vcpu *vcpu, struct vmcs12 *vmcs12) 4324 { 4325 struct vcpu_vmx *vmx = to_vmx(vcpu); 4326 4327 if (evmptr_is_valid(vmx->nested.hv_evmcs_vmptr)) 4328 sync_vmcs02_to_vmcs12_rare(vcpu, vmcs12); 4329 4330 vmx->nested.need_sync_vmcs02_to_vmcs12_rare = 4331 !evmptr_is_valid(vmx->nested.hv_evmcs_vmptr); 4332 4333 vmcs12->guest_cr0 = vmcs12_guest_cr0(vcpu, vmcs12); 4334 vmcs12->guest_cr4 = vmcs12_guest_cr4(vcpu, vmcs12); 4335 4336 vmcs12->guest_rsp = kvm_rsp_read(vcpu); 4337 vmcs12->guest_rip = kvm_rip_read(vcpu); 4338 vmcs12->guest_rflags = vmcs_readl(GUEST_RFLAGS); 4339 4340 vmcs12->guest_cs_ar_bytes = vmcs_read32(GUEST_CS_AR_BYTES); 4341 vmcs12->guest_ss_ar_bytes = vmcs_read32(GUEST_SS_AR_BYTES); 4342 4343 vmcs12->guest_interruptibility_info = 4344 vmcs_read32(GUEST_INTERRUPTIBILITY_INFO); 4345 4346 if (vcpu->arch.mp_state == KVM_MP_STATE_HALTED) 4347 vmcs12->guest_activity_state = GUEST_ACTIVITY_HLT; 4348 else if (vcpu->arch.mp_state == KVM_MP_STATE_INIT_RECEIVED) 4349 vmcs12->guest_activity_state = GUEST_ACTIVITY_WAIT_SIPI; 4350 else 4351 vmcs12->guest_activity_state = GUEST_ACTIVITY_ACTIVE; 4352 4353 if (nested_cpu_has_preemption_timer(vmcs12) && 4354 vmcs12->vm_exit_controls & VM_EXIT_SAVE_VMX_PREEMPTION_TIMER && 4355 !vmx->nested.nested_run_pending) 4356 vmcs12->vmx_preemption_timer_value = 4357 vmx_get_preemption_timer_value(vcpu); 4358 4359 /* 4360 * In some cases (usually, nested EPT), L2 is allowed to change its 4361 * own CR3 without exiting. If it has changed it, we must keep it. 4362 * Of course, if L0 is using shadow page tables, GUEST_CR3 was defined 4363 * by L0, not L1 or L2, so we mustn't unconditionally copy it to vmcs12. 4364 * 4365 * Additionally, restore L2's PDPTR to vmcs12. 4366 */ 4367 if (enable_ept) { 4368 vmcs12->guest_cr3 = vmcs_readl(GUEST_CR3); 4369 if (nested_cpu_has_ept(vmcs12) && is_pae_paging(vcpu)) { 4370 vmcs12->guest_pdptr0 = vmcs_read64(GUEST_PDPTR0); 4371 vmcs12->guest_pdptr1 = vmcs_read64(GUEST_PDPTR1); 4372 vmcs12->guest_pdptr2 = vmcs_read64(GUEST_PDPTR2); 4373 vmcs12->guest_pdptr3 = vmcs_read64(GUEST_PDPTR3); 4374 } 4375 } 4376 4377 vmcs12->guest_linear_address = vmcs_readl(GUEST_LINEAR_ADDRESS); 4378 4379 if (nested_cpu_has_vid(vmcs12)) 4380 vmcs12->guest_intr_status = vmcs_read16(GUEST_INTR_STATUS); 4381 4382 vmcs12->vm_entry_controls = 4383 (vmcs12->vm_entry_controls & ~VM_ENTRY_IA32E_MODE) | 4384 (vm_entry_controls_get(to_vmx(vcpu)) & VM_ENTRY_IA32E_MODE); 4385 4386 if (vmcs12->vm_exit_controls & VM_EXIT_SAVE_DEBUG_CONTROLS) 4387 kvm_get_dr(vcpu, 7, (unsigned long *)&vmcs12->guest_dr7); 4388 4389 if (vmcs12->vm_exit_controls & VM_EXIT_SAVE_IA32_EFER) 4390 vmcs12->guest_ia32_efer = vcpu->arch.efer; 4391 } 4392 4393 /* 4394 * prepare_vmcs12 is part of what we need to do when the nested L2 guest exits 4395 * and we want to prepare to run its L1 parent. L1 keeps a vmcs for L2 (vmcs12), 4396 * and this function updates it to reflect the changes to the guest state while 4397 * L2 was running (and perhaps made some exits which were handled directly by L0 4398 * without going back to L1), and to reflect the exit reason. 4399 * Note that we do not have to copy here all VMCS fields, just those that 4400 * could have changed by the L2 guest or the exit - i.e., the guest-state and 4401 * exit-information fields only. Other fields are modified by L1 with VMWRITE, 4402 * which already writes to vmcs12 directly. 4403 */ 4404 static void prepare_vmcs12(struct kvm_vcpu *vcpu, struct vmcs12 *vmcs12, 4405 u32 vm_exit_reason, u32 exit_intr_info, 4406 unsigned long exit_qualification) 4407 { 4408 /* update exit information fields: */ 4409 vmcs12->vm_exit_reason = vm_exit_reason; 4410 if (to_vmx(vcpu)->exit_reason.enclave_mode) 4411 vmcs12->vm_exit_reason |= VMX_EXIT_REASONS_SGX_ENCLAVE_MODE; 4412 vmcs12->exit_qualification = exit_qualification; 4413 4414 /* 4415 * On VM-Exit due to a failed VM-Entry, the VMCS isn't marked launched 4416 * and only EXIT_REASON and EXIT_QUALIFICATION are updated, all other 4417 * exit info fields are unmodified. 4418 */ 4419 if (!(vmcs12->vm_exit_reason & VMX_EXIT_REASONS_FAILED_VMENTRY)) { 4420 vmcs12->launch_state = 1; 4421 4422 /* vm_entry_intr_info_field is cleared on exit. Emulate this 4423 * instead of reading the real value. */ 4424 vmcs12->vm_entry_intr_info_field &= ~INTR_INFO_VALID_MASK; 4425 4426 /* 4427 * Transfer the event that L0 or L1 may wanted to inject into 4428 * L2 to IDT_VECTORING_INFO_FIELD. 4429 */ 4430 vmcs12_save_pending_event(vcpu, vmcs12, 4431 vm_exit_reason, exit_intr_info); 4432 4433 vmcs12->vm_exit_intr_info = exit_intr_info; 4434 vmcs12->vm_exit_instruction_len = vmcs_read32(VM_EXIT_INSTRUCTION_LEN); 4435 vmcs12->vmx_instruction_info = vmcs_read32(VMX_INSTRUCTION_INFO); 4436 4437 /* 4438 * According to spec, there's no need to store the guest's 4439 * MSRs if the exit is due to a VM-entry failure that occurs 4440 * during or after loading the guest state. Since this exit 4441 * does not fall in that category, we need to save the MSRs. 4442 */ 4443 if (nested_vmx_store_msr(vcpu, 4444 vmcs12->vm_exit_msr_store_addr, 4445 vmcs12->vm_exit_msr_store_count)) 4446 nested_vmx_abort(vcpu, 4447 VMX_ABORT_SAVE_GUEST_MSR_FAIL); 4448 } 4449 } 4450 4451 /* 4452 * A part of what we need to when the nested L2 guest exits and we want to 4453 * run its L1 parent, is to reset L1's guest state to the host state specified 4454 * in vmcs12. 4455 * This function is to be called not only on normal nested exit, but also on 4456 * a nested entry failure, as explained in Intel's spec, 3B.23.7 ("VM-Entry 4457 * Failures During or After Loading Guest State"). 4458 * This function should be called when the active VMCS is L1's (vmcs01). 4459 */ 4460 static void load_vmcs12_host_state(struct kvm_vcpu *vcpu, 4461 struct vmcs12 *vmcs12) 4462 { 4463 enum vm_entry_failure_code ignored; 4464 struct kvm_segment seg; 4465 4466 if (vmcs12->vm_exit_controls & VM_EXIT_LOAD_IA32_EFER) 4467 vcpu->arch.efer = vmcs12->host_ia32_efer; 4468 else if (vmcs12->vm_exit_controls & VM_EXIT_HOST_ADDR_SPACE_SIZE) 4469 vcpu->arch.efer |= (EFER_LMA | EFER_LME); 4470 else 4471 vcpu->arch.efer &= ~(EFER_LMA | EFER_LME); 4472 vmx_set_efer(vcpu, vcpu->arch.efer); 4473 4474 kvm_rsp_write(vcpu, vmcs12->host_rsp); 4475 kvm_rip_write(vcpu, vmcs12->host_rip); 4476 vmx_set_rflags(vcpu, X86_EFLAGS_FIXED); 4477 vmx_set_interrupt_shadow(vcpu, 0); 4478 4479 /* 4480 * Note that calling vmx_set_cr0 is important, even if cr0 hasn't 4481 * actually changed, because vmx_set_cr0 refers to efer set above. 4482 * 4483 * CR0_GUEST_HOST_MASK is already set in the original vmcs01 4484 * (KVM doesn't change it); 4485 */ 4486 vcpu->arch.cr0_guest_owned_bits = KVM_POSSIBLE_CR0_GUEST_BITS; 4487 vmx_set_cr0(vcpu, vmcs12->host_cr0); 4488 4489 /* Same as above - no reason to call set_cr4_guest_host_mask(). */ 4490 vcpu->arch.cr4_guest_owned_bits = ~vmcs_readl(CR4_GUEST_HOST_MASK); 4491 vmx_set_cr4(vcpu, vmcs12->host_cr4); 4492 4493 nested_ept_uninit_mmu_context(vcpu); 4494 4495 /* 4496 * Only PDPTE load can fail as the value of cr3 was checked on entry and 4497 * couldn't have changed. 4498 */ 4499 if (nested_vmx_load_cr3(vcpu, vmcs12->host_cr3, false, true, &ignored)) 4500 nested_vmx_abort(vcpu, VMX_ABORT_LOAD_HOST_PDPTE_FAIL); 4501 4502 nested_vmx_transition_tlb_flush(vcpu, vmcs12, false); 4503 4504 vmcs_write32(GUEST_SYSENTER_CS, vmcs12->host_ia32_sysenter_cs); 4505 vmcs_writel(GUEST_SYSENTER_ESP, vmcs12->host_ia32_sysenter_esp); 4506 vmcs_writel(GUEST_SYSENTER_EIP, vmcs12->host_ia32_sysenter_eip); 4507 vmcs_writel(GUEST_IDTR_BASE, vmcs12->host_idtr_base); 4508 vmcs_writel(GUEST_GDTR_BASE, vmcs12->host_gdtr_base); 4509 vmcs_write32(GUEST_IDTR_LIMIT, 0xFFFF); 4510 vmcs_write32(GUEST_GDTR_LIMIT, 0xFFFF); 4511 4512 /* If not VM_EXIT_CLEAR_BNDCFGS, the L2 value propagates to L1. */ 4513 if (vmcs12->vm_exit_controls & VM_EXIT_CLEAR_BNDCFGS) 4514 vmcs_write64(GUEST_BNDCFGS, 0); 4515 4516 if (vmcs12->vm_exit_controls & VM_EXIT_LOAD_IA32_PAT) { 4517 vmcs_write64(GUEST_IA32_PAT, vmcs12->host_ia32_pat); 4518 vcpu->arch.pat = vmcs12->host_ia32_pat; 4519 } 4520 if ((vmcs12->vm_exit_controls & VM_EXIT_LOAD_IA32_PERF_GLOBAL_CTRL) && 4521 intel_pmu_has_perf_global_ctrl(vcpu_to_pmu(vcpu))) 4522 WARN_ON_ONCE(kvm_set_msr(vcpu, MSR_CORE_PERF_GLOBAL_CTRL, 4523 vmcs12->host_ia32_perf_global_ctrl)); 4524 4525 /* Set L1 segment info according to Intel SDM 4526 27.5.2 Loading Host Segment and Descriptor-Table Registers */ 4527 seg = (struct kvm_segment) { 4528 .base = 0, 4529 .limit = 0xFFFFFFFF, 4530 .selector = vmcs12->host_cs_selector, 4531 .type = 11, 4532 .present = 1, 4533 .s = 1, 4534 .g = 1 4535 }; 4536 if (vmcs12->vm_exit_controls & VM_EXIT_HOST_ADDR_SPACE_SIZE) 4537 seg.l = 1; 4538 else 4539 seg.db = 1; 4540 __vmx_set_segment(vcpu, &seg, VCPU_SREG_CS); 4541 seg = (struct kvm_segment) { 4542 .base = 0, 4543 .limit = 0xFFFFFFFF, 4544 .type = 3, 4545 .present = 1, 4546 .s = 1, 4547 .db = 1, 4548 .g = 1 4549 }; 4550 seg.selector = vmcs12->host_ds_selector; 4551 __vmx_set_segment(vcpu, &seg, VCPU_SREG_DS); 4552 seg.selector = vmcs12->host_es_selector; 4553 __vmx_set_segment(vcpu, &seg, VCPU_SREG_ES); 4554 seg.selector = vmcs12->host_ss_selector; 4555 __vmx_set_segment(vcpu, &seg, VCPU_SREG_SS); 4556 seg.selector = vmcs12->host_fs_selector; 4557 seg.base = vmcs12->host_fs_base; 4558 __vmx_set_segment(vcpu, &seg, VCPU_SREG_FS); 4559 seg.selector = vmcs12->host_gs_selector; 4560 seg.base = vmcs12->host_gs_base; 4561 __vmx_set_segment(vcpu, &seg, VCPU_SREG_GS); 4562 seg = (struct kvm_segment) { 4563 .base = vmcs12->host_tr_base, 4564 .limit = 0x67, 4565 .selector = vmcs12->host_tr_selector, 4566 .type = 11, 4567 .present = 1 4568 }; 4569 __vmx_set_segment(vcpu, &seg, VCPU_SREG_TR); 4570 4571 memset(&seg, 0, sizeof(seg)); 4572 seg.unusable = 1; 4573 __vmx_set_segment(vcpu, &seg, VCPU_SREG_LDTR); 4574 4575 kvm_set_dr(vcpu, 7, 0x400); 4576 vmcs_write64(GUEST_IA32_DEBUGCTL, 0); 4577 4578 if (nested_vmx_load_msr(vcpu, vmcs12->vm_exit_msr_load_addr, 4579 vmcs12->vm_exit_msr_load_count)) 4580 nested_vmx_abort(vcpu, VMX_ABORT_LOAD_HOST_MSR_FAIL); 4581 4582 to_vmx(vcpu)->emulation_required = vmx_emulation_required(vcpu); 4583 } 4584 4585 static inline u64 nested_vmx_get_vmcs01_guest_efer(struct vcpu_vmx *vmx) 4586 { 4587 struct vmx_uret_msr *efer_msr; 4588 unsigned int i; 4589 4590 if (vm_entry_controls_get(vmx) & VM_ENTRY_LOAD_IA32_EFER) 4591 return vmcs_read64(GUEST_IA32_EFER); 4592 4593 if (cpu_has_load_ia32_efer()) 4594 return host_efer; 4595 4596 for (i = 0; i < vmx->msr_autoload.guest.nr; ++i) { 4597 if (vmx->msr_autoload.guest.val[i].index == MSR_EFER) 4598 return vmx->msr_autoload.guest.val[i].value; 4599 } 4600 4601 efer_msr = vmx_find_uret_msr(vmx, MSR_EFER); 4602 if (efer_msr) 4603 return efer_msr->data; 4604 4605 return host_efer; 4606 } 4607 4608 static void nested_vmx_restore_host_state(struct kvm_vcpu *vcpu) 4609 { 4610 struct vmcs12 *vmcs12 = get_vmcs12(vcpu); 4611 struct vcpu_vmx *vmx = to_vmx(vcpu); 4612 struct vmx_msr_entry g, h; 4613 gpa_t gpa; 4614 u32 i, j; 4615 4616 vcpu->arch.pat = vmcs_read64(GUEST_IA32_PAT); 4617 4618 if (vmcs12->vm_entry_controls & VM_ENTRY_LOAD_DEBUG_CONTROLS) { 4619 /* 4620 * L1's host DR7 is lost if KVM_GUESTDBG_USE_HW_BP is set 4621 * as vmcs01.GUEST_DR7 contains a userspace defined value 4622 * and vcpu->arch.dr7 is not squirreled away before the 4623 * nested VMENTER (not worth adding a variable in nested_vmx). 4624 */ 4625 if (vcpu->guest_debug & KVM_GUESTDBG_USE_HW_BP) 4626 kvm_set_dr(vcpu, 7, DR7_FIXED_1); 4627 else 4628 WARN_ON(kvm_set_dr(vcpu, 7, vmcs_readl(GUEST_DR7))); 4629 } 4630 4631 /* 4632 * Note that calling vmx_set_{efer,cr0,cr4} is important as they 4633 * handle a variety of side effects to KVM's software model. 4634 */ 4635 vmx_set_efer(vcpu, nested_vmx_get_vmcs01_guest_efer(vmx)); 4636 4637 vcpu->arch.cr0_guest_owned_bits = KVM_POSSIBLE_CR0_GUEST_BITS; 4638 vmx_set_cr0(vcpu, vmcs_readl(CR0_READ_SHADOW)); 4639 4640 vcpu->arch.cr4_guest_owned_bits = ~vmcs_readl(CR4_GUEST_HOST_MASK); 4641 vmx_set_cr4(vcpu, vmcs_readl(CR4_READ_SHADOW)); 4642 4643 nested_ept_uninit_mmu_context(vcpu); 4644 vcpu->arch.cr3 = vmcs_readl(GUEST_CR3); 4645 kvm_register_mark_available(vcpu, VCPU_EXREG_CR3); 4646 4647 /* 4648 * Use ept_save_pdptrs(vcpu) to load the MMU's cached PDPTRs 4649 * from vmcs01 (if necessary). The PDPTRs are not loaded on 4650 * VMFail, like everything else we just need to ensure our 4651 * software model is up-to-date. 4652 */ 4653 if (enable_ept && is_pae_paging(vcpu)) 4654 ept_save_pdptrs(vcpu); 4655 4656 kvm_mmu_reset_context(vcpu); 4657 4658 /* 4659 * This nasty bit of open coding is a compromise between blindly 4660 * loading L1's MSRs using the exit load lists (incorrect emulation 4661 * of VMFail), leaving the nested VM's MSRs in the software model 4662 * (incorrect behavior) and snapshotting the modified MSRs (too 4663 * expensive since the lists are unbound by hardware). For each 4664 * MSR that was (prematurely) loaded from the nested VMEntry load 4665 * list, reload it from the exit load list if it exists and differs 4666 * from the guest value. The intent is to stuff host state as 4667 * silently as possible, not to fully process the exit load list. 4668 */ 4669 for (i = 0; i < vmcs12->vm_entry_msr_load_count; i++) { 4670 gpa = vmcs12->vm_entry_msr_load_addr + (i * sizeof(g)); 4671 if (kvm_vcpu_read_guest(vcpu, gpa, &g, sizeof(g))) { 4672 pr_debug_ratelimited( 4673 "%s read MSR index failed (%u, 0x%08llx)\n", 4674 __func__, i, gpa); 4675 goto vmabort; 4676 } 4677 4678 for (j = 0; j < vmcs12->vm_exit_msr_load_count; j++) { 4679 gpa = vmcs12->vm_exit_msr_load_addr + (j * sizeof(h)); 4680 if (kvm_vcpu_read_guest(vcpu, gpa, &h, sizeof(h))) { 4681 pr_debug_ratelimited( 4682 "%s read MSR failed (%u, 0x%08llx)\n", 4683 __func__, j, gpa); 4684 goto vmabort; 4685 } 4686 if (h.index != g.index) 4687 continue; 4688 if (h.value == g.value) 4689 break; 4690 4691 if (nested_vmx_load_msr_check(vcpu, &h)) { 4692 pr_debug_ratelimited( 4693 "%s check failed (%u, 0x%x, 0x%x)\n", 4694 __func__, j, h.index, h.reserved); 4695 goto vmabort; 4696 } 4697 4698 if (kvm_set_msr(vcpu, h.index, h.value)) { 4699 pr_debug_ratelimited( 4700 "%s WRMSR failed (%u, 0x%x, 0x%llx)\n", 4701 __func__, j, h.index, h.value); 4702 goto vmabort; 4703 } 4704 } 4705 } 4706 4707 return; 4708 4709 vmabort: 4710 nested_vmx_abort(vcpu, VMX_ABORT_LOAD_HOST_MSR_FAIL); 4711 } 4712 4713 /* 4714 * Emulate an exit from nested guest (L2) to L1, i.e., prepare to run L1 4715 * and modify vmcs12 to make it see what it would expect to see there if 4716 * L2 was its real guest. Must only be called when in L2 (is_guest_mode()) 4717 */ 4718 void nested_vmx_vmexit(struct kvm_vcpu *vcpu, u32 vm_exit_reason, 4719 u32 exit_intr_info, unsigned long exit_qualification) 4720 { 4721 struct vcpu_vmx *vmx = to_vmx(vcpu); 4722 struct vmcs12 *vmcs12 = get_vmcs12(vcpu); 4723 4724 /* Pending MTF traps are discarded on VM-Exit. */ 4725 vmx->nested.mtf_pending = false; 4726 4727 /* trying to cancel vmlaunch/vmresume is a bug */ 4728 WARN_ON_ONCE(vmx->nested.nested_run_pending); 4729 4730 if (kvm_check_request(KVM_REQ_GET_NESTED_STATE_PAGES, vcpu)) { 4731 /* 4732 * KVM_REQ_GET_NESTED_STATE_PAGES is also used to map 4733 * Enlightened VMCS after migration and we still need to 4734 * do that when something is forcing L2->L1 exit prior to 4735 * the first L2 run. 4736 */ 4737 (void)nested_get_evmcs_page(vcpu); 4738 } 4739 4740 /* Service pending TLB flush requests for L2 before switching to L1. */ 4741 kvm_service_local_tlb_flush_requests(vcpu); 4742 4743 /* 4744 * VCPU_EXREG_PDPTR will be clobbered in arch/x86/kvm/vmx/vmx.h between 4745 * now and the new vmentry. Ensure that the VMCS02 PDPTR fields are 4746 * up-to-date before switching to L1. 4747 */ 4748 if (enable_ept && is_pae_paging(vcpu)) 4749 vmx_ept_load_pdptrs(vcpu); 4750 4751 leave_guest_mode(vcpu); 4752 4753 if (nested_cpu_has_preemption_timer(vmcs12)) 4754 hrtimer_cancel(&to_vmx(vcpu)->nested.preemption_timer); 4755 4756 if (nested_cpu_has(vmcs12, CPU_BASED_USE_TSC_OFFSETTING)) { 4757 vcpu->arch.tsc_offset = vcpu->arch.l1_tsc_offset; 4758 if (nested_cpu_has2(vmcs12, SECONDARY_EXEC_TSC_SCALING)) 4759 vcpu->arch.tsc_scaling_ratio = vcpu->arch.l1_tsc_scaling_ratio; 4760 } 4761 4762 if (likely(!vmx->fail)) { 4763 sync_vmcs02_to_vmcs12(vcpu, vmcs12); 4764 4765 if (vm_exit_reason != -1) 4766 prepare_vmcs12(vcpu, vmcs12, vm_exit_reason, 4767 exit_intr_info, exit_qualification); 4768 4769 /* 4770 * Must happen outside of sync_vmcs02_to_vmcs12() as it will 4771 * also be used to capture vmcs12 cache as part of 4772 * capturing nVMX state for snapshot (migration). 4773 * 4774 * Otherwise, this flush will dirty guest memory at a 4775 * point it is already assumed by user-space to be 4776 * immutable. 4777 */ 4778 nested_flush_cached_shadow_vmcs12(vcpu, vmcs12); 4779 } else { 4780 /* 4781 * The only expected VM-instruction error is "VM entry with 4782 * invalid control field(s)." Anything else indicates a 4783 * problem with L0. And we should never get here with a 4784 * VMFail of any type if early consistency checks are enabled. 4785 */ 4786 WARN_ON_ONCE(vmcs_read32(VM_INSTRUCTION_ERROR) != 4787 VMXERR_ENTRY_INVALID_CONTROL_FIELD); 4788 WARN_ON_ONCE(nested_early_check); 4789 } 4790 4791 /* 4792 * Drop events/exceptions that were queued for re-injection to L2 4793 * (picked up via vmx_complete_interrupts()), as well as exceptions 4794 * that were pending for L2. Note, this must NOT be hoisted above 4795 * prepare_vmcs12(), events/exceptions queued for re-injection need to 4796 * be captured in vmcs12 (see vmcs12_save_pending_event()). 4797 */ 4798 vcpu->arch.nmi_injected = false; 4799 kvm_clear_exception_queue(vcpu); 4800 kvm_clear_interrupt_queue(vcpu); 4801 4802 vmx_switch_vmcs(vcpu, &vmx->vmcs01); 4803 4804 /* 4805 * If IBRS is advertised to the vCPU, KVM must flush the indirect 4806 * branch predictors when transitioning from L2 to L1, as L1 expects 4807 * hardware (KVM in this case) to provide separate predictor modes. 4808 * Bare metal isolates VMX root (host) from VMX non-root (guest), but 4809 * doesn't isolate different VMCSs, i.e. in this case, doesn't provide 4810 * separate modes for L2 vs L1. 4811 */ 4812 if (guest_cpuid_has(vcpu, X86_FEATURE_SPEC_CTRL)) 4813 indirect_branch_prediction_barrier(); 4814 4815 /* Update any VMCS fields that might have changed while L2 ran */ 4816 vmcs_write32(VM_EXIT_MSR_LOAD_COUNT, vmx->msr_autoload.host.nr); 4817 vmcs_write32(VM_ENTRY_MSR_LOAD_COUNT, vmx->msr_autoload.guest.nr); 4818 vmcs_write64(TSC_OFFSET, vcpu->arch.tsc_offset); 4819 if (kvm_caps.has_tsc_control) 4820 vmcs_write64(TSC_MULTIPLIER, vcpu->arch.tsc_scaling_ratio); 4821 4822 if (vmx->nested.l1_tpr_threshold != -1) 4823 vmcs_write32(TPR_THRESHOLD, vmx->nested.l1_tpr_threshold); 4824 4825 if (vmx->nested.change_vmcs01_virtual_apic_mode) { 4826 vmx->nested.change_vmcs01_virtual_apic_mode = false; 4827 vmx_set_virtual_apic_mode(vcpu); 4828 } 4829 4830 if (vmx->nested.update_vmcs01_cpu_dirty_logging) { 4831 vmx->nested.update_vmcs01_cpu_dirty_logging = false; 4832 vmx_update_cpu_dirty_logging(vcpu); 4833 } 4834 4835 /* Unpin physical memory we referred to in vmcs02 */ 4836 kvm_vcpu_unmap(vcpu, &vmx->nested.apic_access_page_map, false); 4837 kvm_vcpu_unmap(vcpu, &vmx->nested.virtual_apic_map, true); 4838 kvm_vcpu_unmap(vcpu, &vmx->nested.pi_desc_map, true); 4839 vmx->nested.pi_desc = NULL; 4840 4841 if (vmx->nested.reload_vmcs01_apic_access_page) { 4842 vmx->nested.reload_vmcs01_apic_access_page = false; 4843 kvm_make_request(KVM_REQ_APIC_PAGE_RELOAD, vcpu); 4844 } 4845 4846 if (vmx->nested.update_vmcs01_apicv_status) { 4847 vmx->nested.update_vmcs01_apicv_status = false; 4848 kvm_make_request(KVM_REQ_APICV_UPDATE, vcpu); 4849 } 4850 4851 if ((vm_exit_reason != -1) && 4852 (enable_shadow_vmcs || evmptr_is_valid(vmx->nested.hv_evmcs_vmptr))) 4853 vmx->nested.need_vmcs12_to_shadow_sync = true; 4854 4855 /* in case we halted in L2 */ 4856 vcpu->arch.mp_state = KVM_MP_STATE_RUNNABLE; 4857 4858 if (likely(!vmx->fail)) { 4859 if ((u16)vm_exit_reason == EXIT_REASON_EXTERNAL_INTERRUPT && 4860 nested_exit_intr_ack_set(vcpu)) { 4861 int irq = kvm_cpu_get_interrupt(vcpu); 4862 WARN_ON(irq < 0); 4863 vmcs12->vm_exit_intr_info = irq | 4864 INTR_INFO_VALID_MASK | INTR_TYPE_EXT_INTR; 4865 } 4866 4867 if (vm_exit_reason != -1) 4868 trace_kvm_nested_vmexit_inject(vmcs12->vm_exit_reason, 4869 vmcs12->exit_qualification, 4870 vmcs12->idt_vectoring_info_field, 4871 vmcs12->vm_exit_intr_info, 4872 vmcs12->vm_exit_intr_error_code, 4873 KVM_ISA_VMX); 4874 4875 load_vmcs12_host_state(vcpu, vmcs12); 4876 4877 return; 4878 } 4879 4880 /* 4881 * After an early L2 VM-entry failure, we're now back 4882 * in L1 which thinks it just finished a VMLAUNCH or 4883 * VMRESUME instruction, so we need to set the failure 4884 * flag and the VM-instruction error field of the VMCS 4885 * accordingly, and skip the emulated instruction. 4886 */ 4887 (void)nested_vmx_fail(vcpu, VMXERR_ENTRY_INVALID_CONTROL_FIELD); 4888 4889 /* 4890 * Restore L1's host state to KVM's software model. We're here 4891 * because a consistency check was caught by hardware, which 4892 * means some amount of guest state has been propagated to KVM's 4893 * model and needs to be unwound to the host's state. 4894 */ 4895 nested_vmx_restore_host_state(vcpu); 4896 4897 vmx->fail = 0; 4898 } 4899 4900 static void nested_vmx_triple_fault(struct kvm_vcpu *vcpu) 4901 { 4902 kvm_clear_request(KVM_REQ_TRIPLE_FAULT, vcpu); 4903 nested_vmx_vmexit(vcpu, EXIT_REASON_TRIPLE_FAULT, 0, 0); 4904 } 4905 4906 /* 4907 * Decode the memory-address operand of a vmx instruction, as recorded on an 4908 * exit caused by such an instruction (run by a guest hypervisor). 4909 * On success, returns 0. When the operand is invalid, returns 1 and throws 4910 * #UD, #GP, or #SS. 4911 */ 4912 int get_vmx_mem_address(struct kvm_vcpu *vcpu, unsigned long exit_qualification, 4913 u32 vmx_instruction_info, bool wr, int len, gva_t *ret) 4914 { 4915 gva_t off; 4916 bool exn; 4917 struct kvm_segment s; 4918 4919 /* 4920 * According to Vol. 3B, "Information for VM Exits Due to Instruction 4921 * Execution", on an exit, vmx_instruction_info holds most of the 4922 * addressing components of the operand. Only the displacement part 4923 * is put in exit_qualification (see 3B, "Basic VM-Exit Information"). 4924 * For how an actual address is calculated from all these components, 4925 * refer to Vol. 1, "Operand Addressing". 4926 */ 4927 int scaling = vmx_instruction_info & 3; 4928 int addr_size = (vmx_instruction_info >> 7) & 7; 4929 bool is_reg = vmx_instruction_info & (1u << 10); 4930 int seg_reg = (vmx_instruction_info >> 15) & 7; 4931 int index_reg = (vmx_instruction_info >> 18) & 0xf; 4932 bool index_is_valid = !(vmx_instruction_info & (1u << 22)); 4933 int base_reg = (vmx_instruction_info >> 23) & 0xf; 4934 bool base_is_valid = !(vmx_instruction_info & (1u << 27)); 4935 4936 if (is_reg) { 4937 kvm_queue_exception(vcpu, UD_VECTOR); 4938 return 1; 4939 } 4940 4941 /* Addr = segment_base + offset */ 4942 /* offset = base + [index * scale] + displacement */ 4943 off = exit_qualification; /* holds the displacement */ 4944 if (addr_size == 1) 4945 off = (gva_t)sign_extend64(off, 31); 4946 else if (addr_size == 0) 4947 off = (gva_t)sign_extend64(off, 15); 4948 if (base_is_valid) 4949 off += kvm_register_read(vcpu, base_reg); 4950 if (index_is_valid) 4951 off += kvm_register_read(vcpu, index_reg) << scaling; 4952 vmx_get_segment(vcpu, &s, seg_reg); 4953 4954 /* 4955 * The effective address, i.e. @off, of a memory operand is truncated 4956 * based on the address size of the instruction. Note that this is 4957 * the *effective address*, i.e. the address prior to accounting for 4958 * the segment's base. 4959 */ 4960 if (addr_size == 1) /* 32 bit */ 4961 off &= 0xffffffff; 4962 else if (addr_size == 0) /* 16 bit */ 4963 off &= 0xffff; 4964 4965 /* Checks for #GP/#SS exceptions. */ 4966 exn = false; 4967 if (is_long_mode(vcpu)) { 4968 /* 4969 * The virtual/linear address is never truncated in 64-bit 4970 * mode, e.g. a 32-bit address size can yield a 64-bit virtual 4971 * address when using FS/GS with a non-zero base. 4972 */ 4973 if (seg_reg == VCPU_SREG_FS || seg_reg == VCPU_SREG_GS) 4974 *ret = s.base + off; 4975 else 4976 *ret = off; 4977 4978 /* Long mode: #GP(0)/#SS(0) if the memory address is in a 4979 * non-canonical form. This is the only check on the memory 4980 * destination for long mode! 4981 */ 4982 exn = is_noncanonical_address(*ret, vcpu); 4983 } else { 4984 /* 4985 * When not in long mode, the virtual/linear address is 4986 * unconditionally truncated to 32 bits regardless of the 4987 * address size. 4988 */ 4989 *ret = (s.base + off) & 0xffffffff; 4990 4991 /* Protected mode: apply checks for segment validity in the 4992 * following order: 4993 * - segment type check (#GP(0) may be thrown) 4994 * - usability check (#GP(0)/#SS(0)) 4995 * - limit check (#GP(0)/#SS(0)) 4996 */ 4997 if (wr) 4998 /* #GP(0) if the destination operand is located in a 4999 * read-only data segment or any code segment. 5000 */ 5001 exn = ((s.type & 0xa) == 0 || (s.type & 8)); 5002 else 5003 /* #GP(0) if the source operand is located in an 5004 * execute-only code segment 5005 */ 5006 exn = ((s.type & 0xa) == 8); 5007 if (exn) { 5008 kvm_queue_exception_e(vcpu, GP_VECTOR, 0); 5009 return 1; 5010 } 5011 /* Protected mode: #GP(0)/#SS(0) if the segment is unusable. 5012 */ 5013 exn = (s.unusable != 0); 5014 5015 /* 5016 * Protected mode: #GP(0)/#SS(0) if the memory operand is 5017 * outside the segment limit. All CPUs that support VMX ignore 5018 * limit checks for flat segments, i.e. segments with base==0, 5019 * limit==0xffffffff and of type expand-up data or code. 5020 */ 5021 if (!(s.base == 0 && s.limit == 0xffffffff && 5022 ((s.type & 8) || !(s.type & 4)))) 5023 exn = exn || ((u64)off + len - 1 > s.limit); 5024 } 5025 if (exn) { 5026 kvm_queue_exception_e(vcpu, 5027 seg_reg == VCPU_SREG_SS ? 5028 SS_VECTOR : GP_VECTOR, 5029 0); 5030 return 1; 5031 } 5032 5033 return 0; 5034 } 5035 5036 static int nested_vmx_get_vmptr(struct kvm_vcpu *vcpu, gpa_t *vmpointer, 5037 int *ret) 5038 { 5039 gva_t gva; 5040 struct x86_exception e; 5041 int r; 5042 5043 if (get_vmx_mem_address(vcpu, vmx_get_exit_qual(vcpu), 5044 vmcs_read32(VMX_INSTRUCTION_INFO), false, 5045 sizeof(*vmpointer), &gva)) { 5046 *ret = 1; 5047 return -EINVAL; 5048 } 5049 5050 r = kvm_read_guest_virt(vcpu, gva, vmpointer, sizeof(*vmpointer), &e); 5051 if (r != X86EMUL_CONTINUE) { 5052 *ret = kvm_handle_memory_failure(vcpu, r, &e); 5053 return -EINVAL; 5054 } 5055 5056 return 0; 5057 } 5058 5059 /* 5060 * Allocate a shadow VMCS and associate it with the currently loaded 5061 * VMCS, unless such a shadow VMCS already exists. The newly allocated 5062 * VMCS is also VMCLEARed, so that it is ready for use. 5063 */ 5064 static struct vmcs *alloc_shadow_vmcs(struct kvm_vcpu *vcpu) 5065 { 5066 struct vcpu_vmx *vmx = to_vmx(vcpu); 5067 struct loaded_vmcs *loaded_vmcs = vmx->loaded_vmcs; 5068 5069 /* 5070 * KVM allocates a shadow VMCS only when L1 executes VMXON and frees it 5071 * when L1 executes VMXOFF or the vCPU is forced out of nested 5072 * operation. VMXON faults if the CPU is already post-VMXON, so it 5073 * should be impossible to already have an allocated shadow VMCS. KVM 5074 * doesn't support virtualization of VMCS shadowing, so vmcs01 should 5075 * always be the loaded VMCS. 5076 */ 5077 if (WARN_ON(loaded_vmcs != &vmx->vmcs01 || loaded_vmcs->shadow_vmcs)) 5078 return loaded_vmcs->shadow_vmcs; 5079 5080 loaded_vmcs->shadow_vmcs = alloc_vmcs(true); 5081 if (loaded_vmcs->shadow_vmcs) 5082 vmcs_clear(loaded_vmcs->shadow_vmcs); 5083 5084 return loaded_vmcs->shadow_vmcs; 5085 } 5086 5087 static int enter_vmx_operation(struct kvm_vcpu *vcpu) 5088 { 5089 struct vcpu_vmx *vmx = to_vmx(vcpu); 5090 int r; 5091 5092 r = alloc_loaded_vmcs(&vmx->nested.vmcs02); 5093 if (r < 0) 5094 goto out_vmcs02; 5095 5096 vmx->nested.cached_vmcs12 = kzalloc(VMCS12_SIZE, GFP_KERNEL_ACCOUNT); 5097 if (!vmx->nested.cached_vmcs12) 5098 goto out_cached_vmcs12; 5099 5100 vmx->nested.shadow_vmcs12_cache.gpa = INVALID_GPA; 5101 vmx->nested.cached_shadow_vmcs12 = kzalloc(VMCS12_SIZE, GFP_KERNEL_ACCOUNT); 5102 if (!vmx->nested.cached_shadow_vmcs12) 5103 goto out_cached_shadow_vmcs12; 5104 5105 if (enable_shadow_vmcs && !alloc_shadow_vmcs(vcpu)) 5106 goto out_shadow_vmcs; 5107 5108 hrtimer_init(&vmx->nested.preemption_timer, CLOCK_MONOTONIC, 5109 HRTIMER_MODE_ABS_PINNED); 5110 vmx->nested.preemption_timer.function = vmx_preemption_timer_fn; 5111 5112 vmx->nested.vpid02 = allocate_vpid(); 5113 5114 vmx->nested.vmcs02_initialized = false; 5115 vmx->nested.vmxon = true; 5116 5117 if (vmx_pt_mode_is_host_guest()) { 5118 vmx->pt_desc.guest.ctl = 0; 5119 pt_update_intercept_for_msr(vcpu); 5120 } 5121 5122 return 0; 5123 5124 out_shadow_vmcs: 5125 kfree(vmx->nested.cached_shadow_vmcs12); 5126 5127 out_cached_shadow_vmcs12: 5128 kfree(vmx->nested.cached_vmcs12); 5129 5130 out_cached_vmcs12: 5131 free_loaded_vmcs(&vmx->nested.vmcs02); 5132 5133 out_vmcs02: 5134 return -ENOMEM; 5135 } 5136 5137 /* Emulate the VMXON instruction. */ 5138 static int handle_vmxon(struct kvm_vcpu *vcpu) 5139 { 5140 int ret; 5141 gpa_t vmptr; 5142 uint32_t revision; 5143 struct vcpu_vmx *vmx = to_vmx(vcpu); 5144 const u64 VMXON_NEEDED_FEATURES = FEAT_CTL_LOCKED 5145 | FEAT_CTL_VMX_ENABLED_OUTSIDE_SMX; 5146 5147 /* 5148 * Manually check CR4.VMXE checks, KVM must force CR4.VMXE=1 to enter 5149 * the guest and so cannot rely on hardware to perform the check, 5150 * which has higher priority than VM-Exit (see Intel SDM's pseudocode 5151 * for VMXON). 5152 * 5153 * Rely on hardware for the other pre-VM-Exit checks, CR0.PE=1, !VM86 5154 * and !COMPATIBILITY modes. For an unrestricted guest, KVM doesn't 5155 * force any of the relevant guest state. For a restricted guest, KVM 5156 * does force CR0.PE=1, but only to also force VM86 in order to emulate 5157 * Real Mode, and so there's no need to check CR0.PE manually. 5158 */ 5159 if (!kvm_read_cr4_bits(vcpu, X86_CR4_VMXE)) { 5160 kvm_queue_exception(vcpu, UD_VECTOR); 5161 return 1; 5162 } 5163 5164 /* 5165 * The CPL is checked for "not in VMX operation" and for "in VMX root", 5166 * and has higher priority than the VM-Fail due to being post-VMXON, 5167 * i.e. VMXON #GPs outside of VMX non-root if CPL!=0. In VMX non-root, 5168 * VMXON causes VM-Exit and KVM unconditionally forwards VMXON VM-Exits 5169 * from L2 to L1, i.e. there's no need to check for the vCPU being in 5170 * VMX non-root. 5171 * 5172 * Forwarding the VM-Exit unconditionally, i.e. without performing the 5173 * #UD checks (see above), is functionally ok because KVM doesn't allow 5174 * L1 to run L2 without CR4.VMXE=0, and because KVM never modifies L2's 5175 * CR0 or CR4, i.e. it's L2's responsibility to emulate #UDs that are 5176 * missed by hardware due to shadowing CR0 and/or CR4. 5177 */ 5178 if (vmx_get_cpl(vcpu)) { 5179 kvm_inject_gp(vcpu, 0); 5180 return 1; 5181 } 5182 5183 if (vmx->nested.vmxon) 5184 return nested_vmx_fail(vcpu, VMXERR_VMXON_IN_VMX_ROOT_OPERATION); 5185 5186 /* 5187 * Invalid CR0/CR4 generates #GP. These checks are performed if and 5188 * only if the vCPU isn't already in VMX operation, i.e. effectively 5189 * have lower priority than the VM-Fail above. 5190 */ 5191 if (!nested_host_cr0_valid(vcpu, kvm_read_cr0(vcpu)) || 5192 !nested_host_cr4_valid(vcpu, kvm_read_cr4(vcpu))) { 5193 kvm_inject_gp(vcpu, 0); 5194 return 1; 5195 } 5196 5197 if ((vmx->msr_ia32_feature_control & VMXON_NEEDED_FEATURES) 5198 != VMXON_NEEDED_FEATURES) { 5199 kvm_inject_gp(vcpu, 0); 5200 return 1; 5201 } 5202 5203 if (nested_vmx_get_vmptr(vcpu, &vmptr, &ret)) 5204 return ret; 5205 5206 /* 5207 * SDM 3: 24.11.5 5208 * The first 4 bytes of VMXON region contain the supported 5209 * VMCS revision identifier 5210 * 5211 * Note - IA32_VMX_BASIC[48] will never be 1 for the nested case; 5212 * which replaces physical address width with 32 5213 */ 5214 if (!page_address_valid(vcpu, vmptr)) 5215 return nested_vmx_failInvalid(vcpu); 5216 5217 if (kvm_read_guest(vcpu->kvm, vmptr, &revision, sizeof(revision)) || 5218 revision != VMCS12_REVISION) 5219 return nested_vmx_failInvalid(vcpu); 5220 5221 vmx->nested.vmxon_ptr = vmptr; 5222 ret = enter_vmx_operation(vcpu); 5223 if (ret) 5224 return ret; 5225 5226 return nested_vmx_succeed(vcpu); 5227 } 5228 5229 static inline void nested_release_vmcs12(struct kvm_vcpu *vcpu) 5230 { 5231 struct vcpu_vmx *vmx = to_vmx(vcpu); 5232 5233 if (vmx->nested.current_vmptr == INVALID_GPA) 5234 return; 5235 5236 copy_vmcs02_to_vmcs12_rare(vcpu, get_vmcs12(vcpu)); 5237 5238 if (enable_shadow_vmcs) { 5239 /* copy to memory all shadowed fields in case 5240 they were modified */ 5241 copy_shadow_to_vmcs12(vmx); 5242 vmx_disable_shadow_vmcs(vmx); 5243 } 5244 vmx->nested.posted_intr_nv = -1; 5245 5246 /* Flush VMCS12 to guest memory */ 5247 kvm_vcpu_write_guest_page(vcpu, 5248 vmx->nested.current_vmptr >> PAGE_SHIFT, 5249 vmx->nested.cached_vmcs12, 0, VMCS12_SIZE); 5250 5251 kvm_mmu_free_roots(vcpu->kvm, &vcpu->arch.guest_mmu, KVM_MMU_ROOTS_ALL); 5252 5253 vmx->nested.current_vmptr = INVALID_GPA; 5254 } 5255 5256 /* Emulate the VMXOFF instruction */ 5257 static int handle_vmxoff(struct kvm_vcpu *vcpu) 5258 { 5259 if (!nested_vmx_check_permission(vcpu)) 5260 return 1; 5261 5262 free_nested(vcpu); 5263 5264 if (kvm_apic_has_pending_init_or_sipi(vcpu)) 5265 kvm_make_request(KVM_REQ_EVENT, vcpu); 5266 5267 return nested_vmx_succeed(vcpu); 5268 } 5269 5270 /* Emulate the VMCLEAR instruction */ 5271 static int handle_vmclear(struct kvm_vcpu *vcpu) 5272 { 5273 struct vcpu_vmx *vmx = to_vmx(vcpu); 5274 u32 zero = 0; 5275 gpa_t vmptr; 5276 int r; 5277 5278 if (!nested_vmx_check_permission(vcpu)) 5279 return 1; 5280 5281 if (nested_vmx_get_vmptr(vcpu, &vmptr, &r)) 5282 return r; 5283 5284 if (!page_address_valid(vcpu, vmptr)) 5285 return nested_vmx_fail(vcpu, VMXERR_VMCLEAR_INVALID_ADDRESS); 5286 5287 if (vmptr == vmx->nested.vmxon_ptr) 5288 return nested_vmx_fail(vcpu, VMXERR_VMCLEAR_VMXON_POINTER); 5289 5290 /* 5291 * When Enlightened VMEntry is enabled on the calling CPU we treat 5292 * memory area pointer by vmptr as Enlightened VMCS (as there's no good 5293 * way to distinguish it from VMCS12) and we must not corrupt it by 5294 * writing to the non-existent 'launch_state' field. The area doesn't 5295 * have to be the currently active EVMCS on the calling CPU and there's 5296 * nothing KVM has to do to transition it from 'active' to 'non-active' 5297 * state. It is possible that the area will stay mapped as 5298 * vmx->nested.hv_evmcs but this shouldn't be a problem. 5299 */ 5300 if (likely(!guest_cpuid_has_evmcs(vcpu) || 5301 !evmptr_is_valid(nested_get_evmptr(vcpu)))) { 5302 if (vmptr == vmx->nested.current_vmptr) 5303 nested_release_vmcs12(vcpu); 5304 5305 /* 5306 * Silently ignore memory errors on VMCLEAR, Intel's pseudocode 5307 * for VMCLEAR includes a "ensure that data for VMCS referenced 5308 * by the operand is in memory" clause that guards writes to 5309 * memory, i.e. doing nothing for I/O is architecturally valid. 5310 * 5311 * FIXME: Suppress failures if and only if no memslot is found, 5312 * i.e. exit to userspace if __copy_to_user() fails. 5313 */ 5314 (void)kvm_vcpu_write_guest(vcpu, 5315 vmptr + offsetof(struct vmcs12, 5316 launch_state), 5317 &zero, sizeof(zero)); 5318 } else if (vmx->nested.hv_evmcs && vmptr == vmx->nested.hv_evmcs_vmptr) { 5319 nested_release_evmcs(vcpu); 5320 } 5321 5322 return nested_vmx_succeed(vcpu); 5323 } 5324 5325 /* Emulate the VMLAUNCH instruction */ 5326 static int handle_vmlaunch(struct kvm_vcpu *vcpu) 5327 { 5328 return nested_vmx_run(vcpu, true); 5329 } 5330 5331 /* Emulate the VMRESUME instruction */ 5332 static int handle_vmresume(struct kvm_vcpu *vcpu) 5333 { 5334 5335 return nested_vmx_run(vcpu, false); 5336 } 5337 5338 static int handle_vmread(struct kvm_vcpu *vcpu) 5339 { 5340 struct vmcs12 *vmcs12 = is_guest_mode(vcpu) ? get_shadow_vmcs12(vcpu) 5341 : get_vmcs12(vcpu); 5342 unsigned long exit_qualification = vmx_get_exit_qual(vcpu); 5343 u32 instr_info = vmcs_read32(VMX_INSTRUCTION_INFO); 5344 struct vcpu_vmx *vmx = to_vmx(vcpu); 5345 struct x86_exception e; 5346 unsigned long field; 5347 u64 value; 5348 gva_t gva = 0; 5349 short offset; 5350 int len, r; 5351 5352 if (!nested_vmx_check_permission(vcpu)) 5353 return 1; 5354 5355 /* Decode instruction info and find the field to read */ 5356 field = kvm_register_read(vcpu, (((instr_info) >> 28) & 0xf)); 5357 5358 if (!evmptr_is_valid(vmx->nested.hv_evmcs_vmptr)) { 5359 /* 5360 * In VMX non-root operation, when the VMCS-link pointer is INVALID_GPA, 5361 * any VMREAD sets the ALU flags for VMfailInvalid. 5362 */ 5363 if (vmx->nested.current_vmptr == INVALID_GPA || 5364 (is_guest_mode(vcpu) && 5365 get_vmcs12(vcpu)->vmcs_link_pointer == INVALID_GPA)) 5366 return nested_vmx_failInvalid(vcpu); 5367 5368 offset = get_vmcs12_field_offset(field); 5369 if (offset < 0) 5370 return nested_vmx_fail(vcpu, VMXERR_UNSUPPORTED_VMCS_COMPONENT); 5371 5372 if (!is_guest_mode(vcpu) && is_vmcs12_ext_field(field)) 5373 copy_vmcs02_to_vmcs12_rare(vcpu, vmcs12); 5374 5375 /* Read the field, zero-extended to a u64 value */ 5376 value = vmcs12_read_any(vmcs12, field, offset); 5377 } else { 5378 /* 5379 * Hyper-V TLFS (as of 6.0b) explicitly states, that while an 5380 * enlightened VMCS is active VMREAD/VMWRITE instructions are 5381 * unsupported. Unfortunately, certain versions of Windows 11 5382 * don't comply with this requirement which is not enforced in 5383 * genuine Hyper-V. Allow VMREAD from an enlightened VMCS as a 5384 * workaround, as misbehaving guests will panic on VM-Fail. 5385 * Note, enlightened VMCS is incompatible with shadow VMCS so 5386 * all VMREADs from L2 should go to L1. 5387 */ 5388 if (WARN_ON_ONCE(is_guest_mode(vcpu))) 5389 return nested_vmx_failInvalid(vcpu); 5390 5391 offset = evmcs_field_offset(field, NULL); 5392 if (offset < 0) 5393 return nested_vmx_fail(vcpu, VMXERR_UNSUPPORTED_VMCS_COMPONENT); 5394 5395 /* Read the field, zero-extended to a u64 value */ 5396 value = evmcs_read_any(vmx->nested.hv_evmcs, field, offset); 5397 } 5398 5399 /* 5400 * Now copy part of this value to register or memory, as requested. 5401 * Note that the number of bits actually copied is 32 or 64 depending 5402 * on the guest's mode (32 or 64 bit), not on the given field's length. 5403 */ 5404 if (instr_info & BIT(10)) { 5405 kvm_register_write(vcpu, (((instr_info) >> 3) & 0xf), value); 5406 } else { 5407 len = is_64_bit_mode(vcpu) ? 8 : 4; 5408 if (get_vmx_mem_address(vcpu, exit_qualification, 5409 instr_info, true, len, &gva)) 5410 return 1; 5411 /* _system ok, nested_vmx_check_permission has verified cpl=0 */ 5412 r = kvm_write_guest_virt_system(vcpu, gva, &value, len, &e); 5413 if (r != X86EMUL_CONTINUE) 5414 return kvm_handle_memory_failure(vcpu, r, &e); 5415 } 5416 5417 return nested_vmx_succeed(vcpu); 5418 } 5419 5420 static bool is_shadow_field_rw(unsigned long field) 5421 { 5422 switch (field) { 5423 #define SHADOW_FIELD_RW(x, y) case x: 5424 #include "vmcs_shadow_fields.h" 5425 return true; 5426 default: 5427 break; 5428 } 5429 return false; 5430 } 5431 5432 static bool is_shadow_field_ro(unsigned long field) 5433 { 5434 switch (field) { 5435 #define SHADOW_FIELD_RO(x, y) case x: 5436 #include "vmcs_shadow_fields.h" 5437 return true; 5438 default: 5439 break; 5440 } 5441 return false; 5442 } 5443 5444 static int handle_vmwrite(struct kvm_vcpu *vcpu) 5445 { 5446 struct vmcs12 *vmcs12 = is_guest_mode(vcpu) ? get_shadow_vmcs12(vcpu) 5447 : get_vmcs12(vcpu); 5448 unsigned long exit_qualification = vmx_get_exit_qual(vcpu); 5449 u32 instr_info = vmcs_read32(VMX_INSTRUCTION_INFO); 5450 struct vcpu_vmx *vmx = to_vmx(vcpu); 5451 struct x86_exception e; 5452 unsigned long field; 5453 short offset; 5454 gva_t gva; 5455 int len, r; 5456 5457 /* 5458 * The value to write might be 32 or 64 bits, depending on L1's long 5459 * mode, and eventually we need to write that into a field of several 5460 * possible lengths. The code below first zero-extends the value to 64 5461 * bit (value), and then copies only the appropriate number of 5462 * bits into the vmcs12 field. 5463 */ 5464 u64 value = 0; 5465 5466 if (!nested_vmx_check_permission(vcpu)) 5467 return 1; 5468 5469 /* 5470 * In VMX non-root operation, when the VMCS-link pointer is INVALID_GPA, 5471 * any VMWRITE sets the ALU flags for VMfailInvalid. 5472 */ 5473 if (vmx->nested.current_vmptr == INVALID_GPA || 5474 (is_guest_mode(vcpu) && 5475 get_vmcs12(vcpu)->vmcs_link_pointer == INVALID_GPA)) 5476 return nested_vmx_failInvalid(vcpu); 5477 5478 if (instr_info & BIT(10)) 5479 value = kvm_register_read(vcpu, (((instr_info) >> 3) & 0xf)); 5480 else { 5481 len = is_64_bit_mode(vcpu) ? 8 : 4; 5482 if (get_vmx_mem_address(vcpu, exit_qualification, 5483 instr_info, false, len, &gva)) 5484 return 1; 5485 r = kvm_read_guest_virt(vcpu, gva, &value, len, &e); 5486 if (r != X86EMUL_CONTINUE) 5487 return kvm_handle_memory_failure(vcpu, r, &e); 5488 } 5489 5490 field = kvm_register_read(vcpu, (((instr_info) >> 28) & 0xf)); 5491 5492 offset = get_vmcs12_field_offset(field); 5493 if (offset < 0) 5494 return nested_vmx_fail(vcpu, VMXERR_UNSUPPORTED_VMCS_COMPONENT); 5495 5496 /* 5497 * If the vCPU supports "VMWRITE to any supported field in the 5498 * VMCS," then the "read-only" fields are actually read/write. 5499 */ 5500 if (vmcs_field_readonly(field) && 5501 !nested_cpu_has_vmwrite_any_field(vcpu)) 5502 return nested_vmx_fail(vcpu, VMXERR_VMWRITE_READ_ONLY_VMCS_COMPONENT); 5503 5504 /* 5505 * Ensure vmcs12 is up-to-date before any VMWRITE that dirties 5506 * vmcs12, else we may crush a field or consume a stale value. 5507 */ 5508 if (!is_guest_mode(vcpu) && !is_shadow_field_rw(field)) 5509 copy_vmcs02_to_vmcs12_rare(vcpu, vmcs12); 5510 5511 /* 5512 * Some Intel CPUs intentionally drop the reserved bits of the AR byte 5513 * fields on VMWRITE. Emulate this behavior to ensure consistent KVM 5514 * behavior regardless of the underlying hardware, e.g. if an AR_BYTE 5515 * field is intercepted for VMWRITE but not VMREAD (in L1), then VMREAD 5516 * from L1 will return a different value than VMREAD from L2 (L1 sees 5517 * the stripped down value, L2 sees the full value as stored by KVM). 5518 */ 5519 if (field >= GUEST_ES_AR_BYTES && field <= GUEST_TR_AR_BYTES) 5520 value &= 0x1f0ff; 5521 5522 vmcs12_write_any(vmcs12, field, offset, value); 5523 5524 /* 5525 * Do not track vmcs12 dirty-state if in guest-mode as we actually 5526 * dirty shadow vmcs12 instead of vmcs12. Fields that can be updated 5527 * by L1 without a vmexit are always updated in the vmcs02, i.e. don't 5528 * "dirty" vmcs12, all others go down the prepare_vmcs02() slow path. 5529 */ 5530 if (!is_guest_mode(vcpu) && !is_shadow_field_rw(field)) { 5531 /* 5532 * L1 can read these fields without exiting, ensure the 5533 * shadow VMCS is up-to-date. 5534 */ 5535 if (enable_shadow_vmcs && is_shadow_field_ro(field)) { 5536 preempt_disable(); 5537 vmcs_load(vmx->vmcs01.shadow_vmcs); 5538 5539 __vmcs_writel(field, value); 5540 5541 vmcs_clear(vmx->vmcs01.shadow_vmcs); 5542 vmcs_load(vmx->loaded_vmcs->vmcs); 5543 preempt_enable(); 5544 } 5545 vmx->nested.dirty_vmcs12 = true; 5546 } 5547 5548 return nested_vmx_succeed(vcpu); 5549 } 5550 5551 static void set_current_vmptr(struct vcpu_vmx *vmx, gpa_t vmptr) 5552 { 5553 vmx->nested.current_vmptr = vmptr; 5554 if (enable_shadow_vmcs) { 5555 secondary_exec_controls_setbit(vmx, SECONDARY_EXEC_SHADOW_VMCS); 5556 vmcs_write64(VMCS_LINK_POINTER, 5557 __pa(vmx->vmcs01.shadow_vmcs)); 5558 vmx->nested.need_vmcs12_to_shadow_sync = true; 5559 } 5560 vmx->nested.dirty_vmcs12 = true; 5561 vmx->nested.force_msr_bitmap_recalc = true; 5562 } 5563 5564 /* Emulate the VMPTRLD instruction */ 5565 static int handle_vmptrld(struct kvm_vcpu *vcpu) 5566 { 5567 struct vcpu_vmx *vmx = to_vmx(vcpu); 5568 gpa_t vmptr; 5569 int r; 5570 5571 if (!nested_vmx_check_permission(vcpu)) 5572 return 1; 5573 5574 if (nested_vmx_get_vmptr(vcpu, &vmptr, &r)) 5575 return r; 5576 5577 if (!page_address_valid(vcpu, vmptr)) 5578 return nested_vmx_fail(vcpu, VMXERR_VMPTRLD_INVALID_ADDRESS); 5579 5580 if (vmptr == vmx->nested.vmxon_ptr) 5581 return nested_vmx_fail(vcpu, VMXERR_VMPTRLD_VMXON_POINTER); 5582 5583 /* Forbid normal VMPTRLD if Enlightened version was used */ 5584 if (evmptr_is_valid(vmx->nested.hv_evmcs_vmptr)) 5585 return 1; 5586 5587 if (vmx->nested.current_vmptr != vmptr) { 5588 struct gfn_to_hva_cache *ghc = &vmx->nested.vmcs12_cache; 5589 struct vmcs_hdr hdr; 5590 5591 if (kvm_gfn_to_hva_cache_init(vcpu->kvm, ghc, vmptr, VMCS12_SIZE)) { 5592 /* 5593 * Reads from an unbacked page return all 1s, 5594 * which means that the 32 bits located at the 5595 * given physical address won't match the required 5596 * VMCS12_REVISION identifier. 5597 */ 5598 return nested_vmx_fail(vcpu, 5599 VMXERR_VMPTRLD_INCORRECT_VMCS_REVISION_ID); 5600 } 5601 5602 if (kvm_read_guest_offset_cached(vcpu->kvm, ghc, &hdr, 5603 offsetof(struct vmcs12, hdr), 5604 sizeof(hdr))) { 5605 return nested_vmx_fail(vcpu, 5606 VMXERR_VMPTRLD_INCORRECT_VMCS_REVISION_ID); 5607 } 5608 5609 if (hdr.revision_id != VMCS12_REVISION || 5610 (hdr.shadow_vmcs && 5611 !nested_cpu_has_vmx_shadow_vmcs(vcpu))) { 5612 return nested_vmx_fail(vcpu, 5613 VMXERR_VMPTRLD_INCORRECT_VMCS_REVISION_ID); 5614 } 5615 5616 nested_release_vmcs12(vcpu); 5617 5618 /* 5619 * Load VMCS12 from guest memory since it is not already 5620 * cached. 5621 */ 5622 if (kvm_read_guest_cached(vcpu->kvm, ghc, vmx->nested.cached_vmcs12, 5623 VMCS12_SIZE)) { 5624 return nested_vmx_fail(vcpu, 5625 VMXERR_VMPTRLD_INCORRECT_VMCS_REVISION_ID); 5626 } 5627 5628 set_current_vmptr(vmx, vmptr); 5629 } 5630 5631 return nested_vmx_succeed(vcpu); 5632 } 5633 5634 /* Emulate the VMPTRST instruction */ 5635 static int handle_vmptrst(struct kvm_vcpu *vcpu) 5636 { 5637 unsigned long exit_qual = vmx_get_exit_qual(vcpu); 5638 u32 instr_info = vmcs_read32(VMX_INSTRUCTION_INFO); 5639 gpa_t current_vmptr = to_vmx(vcpu)->nested.current_vmptr; 5640 struct x86_exception e; 5641 gva_t gva; 5642 int r; 5643 5644 if (!nested_vmx_check_permission(vcpu)) 5645 return 1; 5646 5647 if (unlikely(evmptr_is_valid(to_vmx(vcpu)->nested.hv_evmcs_vmptr))) 5648 return 1; 5649 5650 if (get_vmx_mem_address(vcpu, exit_qual, instr_info, 5651 true, sizeof(gpa_t), &gva)) 5652 return 1; 5653 /* *_system ok, nested_vmx_check_permission has verified cpl=0 */ 5654 r = kvm_write_guest_virt_system(vcpu, gva, (void *)¤t_vmptr, 5655 sizeof(gpa_t), &e); 5656 if (r != X86EMUL_CONTINUE) 5657 return kvm_handle_memory_failure(vcpu, r, &e); 5658 5659 return nested_vmx_succeed(vcpu); 5660 } 5661 5662 /* Emulate the INVEPT instruction */ 5663 static int handle_invept(struct kvm_vcpu *vcpu) 5664 { 5665 struct vcpu_vmx *vmx = to_vmx(vcpu); 5666 u32 vmx_instruction_info, types; 5667 unsigned long type, roots_to_free; 5668 struct kvm_mmu *mmu; 5669 gva_t gva; 5670 struct x86_exception e; 5671 struct { 5672 u64 eptp, gpa; 5673 } operand; 5674 int i, r, gpr_index; 5675 5676 if (!(vmx->nested.msrs.secondary_ctls_high & 5677 SECONDARY_EXEC_ENABLE_EPT) || 5678 !(vmx->nested.msrs.ept_caps & VMX_EPT_INVEPT_BIT)) { 5679 kvm_queue_exception(vcpu, UD_VECTOR); 5680 return 1; 5681 } 5682 5683 if (!nested_vmx_check_permission(vcpu)) 5684 return 1; 5685 5686 vmx_instruction_info = vmcs_read32(VMX_INSTRUCTION_INFO); 5687 gpr_index = vmx_get_instr_info_reg2(vmx_instruction_info); 5688 type = kvm_register_read(vcpu, gpr_index); 5689 5690 types = (vmx->nested.msrs.ept_caps >> VMX_EPT_EXTENT_SHIFT) & 6; 5691 5692 if (type >= 32 || !(types & (1 << type))) 5693 return nested_vmx_fail(vcpu, VMXERR_INVALID_OPERAND_TO_INVEPT_INVVPID); 5694 5695 /* According to the Intel VMX instruction reference, the memory 5696 * operand is read even if it isn't needed (e.g., for type==global) 5697 */ 5698 if (get_vmx_mem_address(vcpu, vmx_get_exit_qual(vcpu), 5699 vmx_instruction_info, false, sizeof(operand), &gva)) 5700 return 1; 5701 r = kvm_read_guest_virt(vcpu, gva, &operand, sizeof(operand), &e); 5702 if (r != X86EMUL_CONTINUE) 5703 return kvm_handle_memory_failure(vcpu, r, &e); 5704 5705 /* 5706 * Nested EPT roots are always held through guest_mmu, 5707 * not root_mmu. 5708 */ 5709 mmu = &vcpu->arch.guest_mmu; 5710 5711 switch (type) { 5712 case VMX_EPT_EXTENT_CONTEXT: 5713 if (!nested_vmx_check_eptp(vcpu, operand.eptp)) 5714 return nested_vmx_fail(vcpu, 5715 VMXERR_INVALID_OPERAND_TO_INVEPT_INVVPID); 5716 5717 roots_to_free = 0; 5718 if (nested_ept_root_matches(mmu->root.hpa, mmu->root.pgd, 5719 operand.eptp)) 5720 roots_to_free |= KVM_MMU_ROOT_CURRENT; 5721 5722 for (i = 0; i < KVM_MMU_NUM_PREV_ROOTS; i++) { 5723 if (nested_ept_root_matches(mmu->prev_roots[i].hpa, 5724 mmu->prev_roots[i].pgd, 5725 operand.eptp)) 5726 roots_to_free |= KVM_MMU_ROOT_PREVIOUS(i); 5727 } 5728 break; 5729 case VMX_EPT_EXTENT_GLOBAL: 5730 roots_to_free = KVM_MMU_ROOTS_ALL; 5731 break; 5732 default: 5733 BUG(); 5734 break; 5735 } 5736 5737 if (roots_to_free) 5738 kvm_mmu_free_roots(vcpu->kvm, mmu, roots_to_free); 5739 5740 return nested_vmx_succeed(vcpu); 5741 } 5742 5743 static int handle_invvpid(struct kvm_vcpu *vcpu) 5744 { 5745 struct vcpu_vmx *vmx = to_vmx(vcpu); 5746 u32 vmx_instruction_info; 5747 unsigned long type, types; 5748 gva_t gva; 5749 struct x86_exception e; 5750 struct { 5751 u64 vpid; 5752 u64 gla; 5753 } operand; 5754 u16 vpid02; 5755 int r, gpr_index; 5756 5757 if (!(vmx->nested.msrs.secondary_ctls_high & 5758 SECONDARY_EXEC_ENABLE_VPID) || 5759 !(vmx->nested.msrs.vpid_caps & VMX_VPID_INVVPID_BIT)) { 5760 kvm_queue_exception(vcpu, UD_VECTOR); 5761 return 1; 5762 } 5763 5764 if (!nested_vmx_check_permission(vcpu)) 5765 return 1; 5766 5767 vmx_instruction_info = vmcs_read32(VMX_INSTRUCTION_INFO); 5768 gpr_index = vmx_get_instr_info_reg2(vmx_instruction_info); 5769 type = kvm_register_read(vcpu, gpr_index); 5770 5771 types = (vmx->nested.msrs.vpid_caps & 5772 VMX_VPID_EXTENT_SUPPORTED_MASK) >> 8; 5773 5774 if (type >= 32 || !(types & (1 << type))) 5775 return nested_vmx_fail(vcpu, 5776 VMXERR_INVALID_OPERAND_TO_INVEPT_INVVPID); 5777 5778 /* according to the intel vmx instruction reference, the memory 5779 * operand is read even if it isn't needed (e.g., for type==global) 5780 */ 5781 if (get_vmx_mem_address(vcpu, vmx_get_exit_qual(vcpu), 5782 vmx_instruction_info, false, sizeof(operand), &gva)) 5783 return 1; 5784 r = kvm_read_guest_virt(vcpu, gva, &operand, sizeof(operand), &e); 5785 if (r != X86EMUL_CONTINUE) 5786 return kvm_handle_memory_failure(vcpu, r, &e); 5787 5788 if (operand.vpid >> 16) 5789 return nested_vmx_fail(vcpu, 5790 VMXERR_INVALID_OPERAND_TO_INVEPT_INVVPID); 5791 5792 vpid02 = nested_get_vpid02(vcpu); 5793 switch (type) { 5794 case VMX_VPID_EXTENT_INDIVIDUAL_ADDR: 5795 if (!operand.vpid || 5796 is_noncanonical_address(operand.gla, vcpu)) 5797 return nested_vmx_fail(vcpu, 5798 VMXERR_INVALID_OPERAND_TO_INVEPT_INVVPID); 5799 vpid_sync_vcpu_addr(vpid02, operand.gla); 5800 break; 5801 case VMX_VPID_EXTENT_SINGLE_CONTEXT: 5802 case VMX_VPID_EXTENT_SINGLE_NON_GLOBAL: 5803 if (!operand.vpid) 5804 return nested_vmx_fail(vcpu, 5805 VMXERR_INVALID_OPERAND_TO_INVEPT_INVVPID); 5806 vpid_sync_context(vpid02); 5807 break; 5808 case VMX_VPID_EXTENT_ALL_CONTEXT: 5809 vpid_sync_context(vpid02); 5810 break; 5811 default: 5812 WARN_ON_ONCE(1); 5813 return kvm_skip_emulated_instruction(vcpu); 5814 } 5815 5816 /* 5817 * Sync the shadow page tables if EPT is disabled, L1 is invalidating 5818 * linear mappings for L2 (tagged with L2's VPID). Free all guest 5819 * roots as VPIDs are not tracked in the MMU role. 5820 * 5821 * Note, this operates on root_mmu, not guest_mmu, as L1 and L2 share 5822 * an MMU when EPT is disabled. 5823 * 5824 * TODO: sync only the affected SPTEs for INVDIVIDUAL_ADDR. 5825 */ 5826 if (!enable_ept) 5827 kvm_mmu_free_guest_mode_roots(vcpu->kvm, &vcpu->arch.root_mmu); 5828 5829 return nested_vmx_succeed(vcpu); 5830 } 5831 5832 static int nested_vmx_eptp_switching(struct kvm_vcpu *vcpu, 5833 struct vmcs12 *vmcs12) 5834 { 5835 u32 index = kvm_rcx_read(vcpu); 5836 u64 new_eptp; 5837 5838 if (WARN_ON_ONCE(!nested_cpu_has_ept(vmcs12))) 5839 return 1; 5840 if (index >= VMFUNC_EPTP_ENTRIES) 5841 return 1; 5842 5843 if (kvm_vcpu_read_guest_page(vcpu, vmcs12->eptp_list_address >> PAGE_SHIFT, 5844 &new_eptp, index * 8, 8)) 5845 return 1; 5846 5847 /* 5848 * If the (L2) guest does a vmfunc to the currently 5849 * active ept pointer, we don't have to do anything else 5850 */ 5851 if (vmcs12->ept_pointer != new_eptp) { 5852 if (!nested_vmx_check_eptp(vcpu, new_eptp)) 5853 return 1; 5854 5855 vmcs12->ept_pointer = new_eptp; 5856 nested_ept_new_eptp(vcpu); 5857 5858 if (!nested_cpu_has_vpid(vmcs12)) 5859 kvm_make_request(KVM_REQ_TLB_FLUSH_GUEST, vcpu); 5860 } 5861 5862 return 0; 5863 } 5864 5865 static int handle_vmfunc(struct kvm_vcpu *vcpu) 5866 { 5867 struct vcpu_vmx *vmx = to_vmx(vcpu); 5868 struct vmcs12 *vmcs12; 5869 u32 function = kvm_rax_read(vcpu); 5870 5871 /* 5872 * VMFUNC should never execute cleanly while L1 is active; KVM supports 5873 * VMFUNC for nested VMs, but not for L1. 5874 */ 5875 if (WARN_ON_ONCE(!is_guest_mode(vcpu))) { 5876 kvm_queue_exception(vcpu, UD_VECTOR); 5877 return 1; 5878 } 5879 5880 vmcs12 = get_vmcs12(vcpu); 5881 5882 /* 5883 * #UD on out-of-bounds function has priority over VM-Exit, and VMFUNC 5884 * is enabled in vmcs02 if and only if it's enabled in vmcs12. 5885 */ 5886 if (WARN_ON_ONCE((function > 63) || !nested_cpu_has_vmfunc(vmcs12))) { 5887 kvm_queue_exception(vcpu, UD_VECTOR); 5888 return 1; 5889 } 5890 5891 if (!(vmcs12->vm_function_control & BIT_ULL(function))) 5892 goto fail; 5893 5894 switch (function) { 5895 case 0: 5896 if (nested_vmx_eptp_switching(vcpu, vmcs12)) 5897 goto fail; 5898 break; 5899 default: 5900 goto fail; 5901 } 5902 return kvm_skip_emulated_instruction(vcpu); 5903 5904 fail: 5905 /* 5906 * This is effectively a reflected VM-Exit, as opposed to a synthesized 5907 * nested VM-Exit. Pass the original exit reason, i.e. don't hardcode 5908 * EXIT_REASON_VMFUNC as the exit reason. 5909 */ 5910 nested_vmx_vmexit(vcpu, vmx->exit_reason.full, 5911 vmx_get_intr_info(vcpu), 5912 vmx_get_exit_qual(vcpu)); 5913 return 1; 5914 } 5915 5916 /* 5917 * Return true if an IO instruction with the specified port and size should cause 5918 * a VM-exit into L1. 5919 */ 5920 bool nested_vmx_check_io_bitmaps(struct kvm_vcpu *vcpu, unsigned int port, 5921 int size) 5922 { 5923 struct vmcs12 *vmcs12 = get_vmcs12(vcpu); 5924 gpa_t bitmap, last_bitmap; 5925 u8 b; 5926 5927 last_bitmap = INVALID_GPA; 5928 b = -1; 5929 5930 while (size > 0) { 5931 if (port < 0x8000) 5932 bitmap = vmcs12->io_bitmap_a; 5933 else if (port < 0x10000) 5934 bitmap = vmcs12->io_bitmap_b; 5935 else 5936 return true; 5937 bitmap += (port & 0x7fff) / 8; 5938 5939 if (last_bitmap != bitmap) 5940 if (kvm_vcpu_read_guest(vcpu, bitmap, &b, 1)) 5941 return true; 5942 if (b & (1 << (port & 7))) 5943 return true; 5944 5945 port++; 5946 size--; 5947 last_bitmap = bitmap; 5948 } 5949 5950 return false; 5951 } 5952 5953 static bool nested_vmx_exit_handled_io(struct kvm_vcpu *vcpu, 5954 struct vmcs12 *vmcs12) 5955 { 5956 unsigned long exit_qualification; 5957 unsigned short port; 5958 int size; 5959 5960 if (!nested_cpu_has(vmcs12, CPU_BASED_USE_IO_BITMAPS)) 5961 return nested_cpu_has(vmcs12, CPU_BASED_UNCOND_IO_EXITING); 5962 5963 exit_qualification = vmx_get_exit_qual(vcpu); 5964 5965 port = exit_qualification >> 16; 5966 size = (exit_qualification & 7) + 1; 5967 5968 return nested_vmx_check_io_bitmaps(vcpu, port, size); 5969 } 5970 5971 /* 5972 * Return 1 if we should exit from L2 to L1 to handle an MSR access, 5973 * rather than handle it ourselves in L0. I.e., check whether L1 expressed 5974 * disinterest in the current event (read or write a specific MSR) by using an 5975 * MSR bitmap. This may be the case even when L0 doesn't use MSR bitmaps. 5976 */ 5977 static bool nested_vmx_exit_handled_msr(struct kvm_vcpu *vcpu, 5978 struct vmcs12 *vmcs12, 5979 union vmx_exit_reason exit_reason) 5980 { 5981 u32 msr_index = kvm_rcx_read(vcpu); 5982 gpa_t bitmap; 5983 5984 if (!nested_cpu_has(vmcs12, CPU_BASED_USE_MSR_BITMAPS)) 5985 return true; 5986 5987 /* 5988 * The MSR_BITMAP page is divided into four 1024-byte bitmaps, 5989 * for the four combinations of read/write and low/high MSR numbers. 5990 * First we need to figure out which of the four to use: 5991 */ 5992 bitmap = vmcs12->msr_bitmap; 5993 if (exit_reason.basic == EXIT_REASON_MSR_WRITE) 5994 bitmap += 2048; 5995 if (msr_index >= 0xc0000000) { 5996 msr_index -= 0xc0000000; 5997 bitmap += 1024; 5998 } 5999 6000 /* Then read the msr_index'th bit from this bitmap: */ 6001 if (msr_index < 1024*8) { 6002 unsigned char b; 6003 if (kvm_vcpu_read_guest(vcpu, bitmap + msr_index/8, &b, 1)) 6004 return true; 6005 return 1 & (b >> (msr_index & 7)); 6006 } else 6007 return true; /* let L1 handle the wrong parameter */ 6008 } 6009 6010 /* 6011 * Return 1 if we should exit from L2 to L1 to handle a CR access exit, 6012 * rather than handle it ourselves in L0. I.e., check if L1 wanted to 6013 * intercept (via guest_host_mask etc.) the current event. 6014 */ 6015 static bool nested_vmx_exit_handled_cr(struct kvm_vcpu *vcpu, 6016 struct vmcs12 *vmcs12) 6017 { 6018 unsigned long exit_qualification = vmx_get_exit_qual(vcpu); 6019 int cr = exit_qualification & 15; 6020 int reg; 6021 unsigned long val; 6022 6023 switch ((exit_qualification >> 4) & 3) { 6024 case 0: /* mov to cr */ 6025 reg = (exit_qualification >> 8) & 15; 6026 val = kvm_register_read(vcpu, reg); 6027 switch (cr) { 6028 case 0: 6029 if (vmcs12->cr0_guest_host_mask & 6030 (val ^ vmcs12->cr0_read_shadow)) 6031 return true; 6032 break; 6033 case 3: 6034 if (nested_cpu_has(vmcs12, CPU_BASED_CR3_LOAD_EXITING)) 6035 return true; 6036 break; 6037 case 4: 6038 if (vmcs12->cr4_guest_host_mask & 6039 (vmcs12->cr4_read_shadow ^ val)) 6040 return true; 6041 break; 6042 case 8: 6043 if (nested_cpu_has(vmcs12, CPU_BASED_CR8_LOAD_EXITING)) 6044 return true; 6045 break; 6046 } 6047 break; 6048 case 2: /* clts */ 6049 if ((vmcs12->cr0_guest_host_mask & X86_CR0_TS) && 6050 (vmcs12->cr0_read_shadow & X86_CR0_TS)) 6051 return true; 6052 break; 6053 case 1: /* mov from cr */ 6054 switch (cr) { 6055 case 3: 6056 if (vmcs12->cpu_based_vm_exec_control & 6057 CPU_BASED_CR3_STORE_EXITING) 6058 return true; 6059 break; 6060 case 8: 6061 if (vmcs12->cpu_based_vm_exec_control & 6062 CPU_BASED_CR8_STORE_EXITING) 6063 return true; 6064 break; 6065 } 6066 break; 6067 case 3: /* lmsw */ 6068 /* 6069 * lmsw can change bits 1..3 of cr0, and only set bit 0 of 6070 * cr0. Other attempted changes are ignored, with no exit. 6071 */ 6072 val = (exit_qualification >> LMSW_SOURCE_DATA_SHIFT) & 0x0f; 6073 if (vmcs12->cr0_guest_host_mask & 0xe & 6074 (val ^ vmcs12->cr0_read_shadow)) 6075 return true; 6076 if ((vmcs12->cr0_guest_host_mask & 0x1) && 6077 !(vmcs12->cr0_read_shadow & 0x1) && 6078 (val & 0x1)) 6079 return true; 6080 break; 6081 } 6082 return false; 6083 } 6084 6085 static bool nested_vmx_exit_handled_encls(struct kvm_vcpu *vcpu, 6086 struct vmcs12 *vmcs12) 6087 { 6088 u32 encls_leaf; 6089 6090 if (!guest_cpuid_has(vcpu, X86_FEATURE_SGX) || 6091 !nested_cpu_has2(vmcs12, SECONDARY_EXEC_ENCLS_EXITING)) 6092 return false; 6093 6094 encls_leaf = kvm_rax_read(vcpu); 6095 if (encls_leaf > 62) 6096 encls_leaf = 63; 6097 return vmcs12->encls_exiting_bitmap & BIT_ULL(encls_leaf); 6098 } 6099 6100 static bool nested_vmx_exit_handled_vmcs_access(struct kvm_vcpu *vcpu, 6101 struct vmcs12 *vmcs12, gpa_t bitmap) 6102 { 6103 u32 vmx_instruction_info; 6104 unsigned long field; 6105 u8 b; 6106 6107 if (!nested_cpu_has_shadow_vmcs(vmcs12)) 6108 return true; 6109 6110 /* Decode instruction info and find the field to access */ 6111 vmx_instruction_info = vmcs_read32(VMX_INSTRUCTION_INFO); 6112 field = kvm_register_read(vcpu, (((vmx_instruction_info) >> 28) & 0xf)); 6113 6114 /* Out-of-range fields always cause a VM exit from L2 to L1 */ 6115 if (field >> 15) 6116 return true; 6117 6118 if (kvm_vcpu_read_guest(vcpu, bitmap + field/8, &b, 1)) 6119 return true; 6120 6121 return 1 & (b >> (field & 7)); 6122 } 6123 6124 static bool nested_vmx_exit_handled_mtf(struct vmcs12 *vmcs12) 6125 { 6126 u32 entry_intr_info = vmcs12->vm_entry_intr_info_field; 6127 6128 if (nested_cpu_has_mtf(vmcs12)) 6129 return true; 6130 6131 /* 6132 * An MTF VM-exit may be injected into the guest by setting the 6133 * interruption-type to 7 (other event) and the vector field to 0. Such 6134 * is the case regardless of the 'monitor trap flag' VM-execution 6135 * control. 6136 */ 6137 return entry_intr_info == (INTR_INFO_VALID_MASK 6138 | INTR_TYPE_OTHER_EVENT); 6139 } 6140 6141 /* 6142 * Return true if L0 wants to handle an exit from L2 regardless of whether or not 6143 * L1 wants the exit. Only call this when in is_guest_mode (L2). 6144 */ 6145 static bool nested_vmx_l0_wants_exit(struct kvm_vcpu *vcpu, 6146 union vmx_exit_reason exit_reason) 6147 { 6148 u32 intr_info; 6149 6150 switch ((u16)exit_reason.basic) { 6151 case EXIT_REASON_EXCEPTION_NMI: 6152 intr_info = vmx_get_intr_info(vcpu); 6153 if (is_nmi(intr_info)) 6154 return true; 6155 else if (is_page_fault(intr_info)) 6156 return vcpu->arch.apf.host_apf_flags || 6157 vmx_need_pf_intercept(vcpu); 6158 else if (is_debug(intr_info) && 6159 vcpu->guest_debug & 6160 (KVM_GUESTDBG_SINGLESTEP | KVM_GUESTDBG_USE_HW_BP)) 6161 return true; 6162 else if (is_breakpoint(intr_info) && 6163 vcpu->guest_debug & KVM_GUESTDBG_USE_SW_BP) 6164 return true; 6165 else if (is_alignment_check(intr_info) && 6166 !vmx_guest_inject_ac(vcpu)) 6167 return true; 6168 return false; 6169 case EXIT_REASON_EXTERNAL_INTERRUPT: 6170 return true; 6171 case EXIT_REASON_MCE_DURING_VMENTRY: 6172 return true; 6173 case EXIT_REASON_EPT_VIOLATION: 6174 /* 6175 * L0 always deals with the EPT violation. If nested EPT is 6176 * used, and the nested mmu code discovers that the address is 6177 * missing in the guest EPT table (EPT12), the EPT violation 6178 * will be injected with nested_ept_inject_page_fault() 6179 */ 6180 return true; 6181 case EXIT_REASON_EPT_MISCONFIG: 6182 /* 6183 * L2 never uses directly L1's EPT, but rather L0's own EPT 6184 * table (shadow on EPT) or a merged EPT table that L0 built 6185 * (EPT on EPT). So any problems with the structure of the 6186 * table is L0's fault. 6187 */ 6188 return true; 6189 case EXIT_REASON_PREEMPTION_TIMER: 6190 return true; 6191 case EXIT_REASON_PML_FULL: 6192 /* 6193 * PML is emulated for an L1 VMM and should never be enabled in 6194 * vmcs02, always "handle" PML_FULL by exiting to userspace. 6195 */ 6196 return true; 6197 case EXIT_REASON_VMFUNC: 6198 /* VM functions are emulated through L2->L0 vmexits. */ 6199 return true; 6200 case EXIT_REASON_BUS_LOCK: 6201 /* 6202 * At present, bus lock VM exit is never exposed to L1. 6203 * Handle L2's bus locks in L0 directly. 6204 */ 6205 return true; 6206 case EXIT_REASON_VMCALL: 6207 /* Hyper-V L2 TLB flush hypercall is handled by L0 */ 6208 return guest_hv_cpuid_has_l2_tlb_flush(vcpu) && 6209 nested_evmcs_l2_tlb_flush_enabled(vcpu) && 6210 kvm_hv_is_tlb_flush_hcall(vcpu); 6211 default: 6212 break; 6213 } 6214 return false; 6215 } 6216 6217 /* 6218 * Return 1 if L1 wants to intercept an exit from L2. Only call this when in 6219 * is_guest_mode (L2). 6220 */ 6221 static bool nested_vmx_l1_wants_exit(struct kvm_vcpu *vcpu, 6222 union vmx_exit_reason exit_reason) 6223 { 6224 struct vmcs12 *vmcs12 = get_vmcs12(vcpu); 6225 u32 intr_info; 6226 6227 switch ((u16)exit_reason.basic) { 6228 case EXIT_REASON_EXCEPTION_NMI: 6229 intr_info = vmx_get_intr_info(vcpu); 6230 if (is_nmi(intr_info)) 6231 return true; 6232 else if (is_page_fault(intr_info)) 6233 return true; 6234 return vmcs12->exception_bitmap & 6235 (1u << (intr_info & INTR_INFO_VECTOR_MASK)); 6236 case EXIT_REASON_EXTERNAL_INTERRUPT: 6237 return nested_exit_on_intr(vcpu); 6238 case EXIT_REASON_TRIPLE_FAULT: 6239 return true; 6240 case EXIT_REASON_INTERRUPT_WINDOW: 6241 return nested_cpu_has(vmcs12, CPU_BASED_INTR_WINDOW_EXITING); 6242 case EXIT_REASON_NMI_WINDOW: 6243 return nested_cpu_has(vmcs12, CPU_BASED_NMI_WINDOW_EXITING); 6244 case EXIT_REASON_TASK_SWITCH: 6245 return true; 6246 case EXIT_REASON_CPUID: 6247 return true; 6248 case EXIT_REASON_HLT: 6249 return nested_cpu_has(vmcs12, CPU_BASED_HLT_EXITING); 6250 case EXIT_REASON_INVD: 6251 return true; 6252 case EXIT_REASON_INVLPG: 6253 return nested_cpu_has(vmcs12, CPU_BASED_INVLPG_EXITING); 6254 case EXIT_REASON_RDPMC: 6255 return nested_cpu_has(vmcs12, CPU_BASED_RDPMC_EXITING); 6256 case EXIT_REASON_RDRAND: 6257 return nested_cpu_has2(vmcs12, SECONDARY_EXEC_RDRAND_EXITING); 6258 case EXIT_REASON_RDSEED: 6259 return nested_cpu_has2(vmcs12, SECONDARY_EXEC_RDSEED_EXITING); 6260 case EXIT_REASON_RDTSC: case EXIT_REASON_RDTSCP: 6261 return nested_cpu_has(vmcs12, CPU_BASED_RDTSC_EXITING); 6262 case EXIT_REASON_VMREAD: 6263 return nested_vmx_exit_handled_vmcs_access(vcpu, vmcs12, 6264 vmcs12->vmread_bitmap); 6265 case EXIT_REASON_VMWRITE: 6266 return nested_vmx_exit_handled_vmcs_access(vcpu, vmcs12, 6267 vmcs12->vmwrite_bitmap); 6268 case EXIT_REASON_VMCALL: case EXIT_REASON_VMCLEAR: 6269 case EXIT_REASON_VMLAUNCH: case EXIT_REASON_VMPTRLD: 6270 case EXIT_REASON_VMPTRST: case EXIT_REASON_VMRESUME: 6271 case EXIT_REASON_VMOFF: case EXIT_REASON_VMON: 6272 case EXIT_REASON_INVEPT: case EXIT_REASON_INVVPID: 6273 /* 6274 * VMX instructions trap unconditionally. This allows L1 to 6275 * emulate them for its L2 guest, i.e., allows 3-level nesting! 6276 */ 6277 return true; 6278 case EXIT_REASON_CR_ACCESS: 6279 return nested_vmx_exit_handled_cr(vcpu, vmcs12); 6280 case EXIT_REASON_DR_ACCESS: 6281 return nested_cpu_has(vmcs12, CPU_BASED_MOV_DR_EXITING); 6282 case EXIT_REASON_IO_INSTRUCTION: 6283 return nested_vmx_exit_handled_io(vcpu, vmcs12); 6284 case EXIT_REASON_GDTR_IDTR: case EXIT_REASON_LDTR_TR: 6285 return nested_cpu_has2(vmcs12, SECONDARY_EXEC_DESC); 6286 case EXIT_REASON_MSR_READ: 6287 case EXIT_REASON_MSR_WRITE: 6288 return nested_vmx_exit_handled_msr(vcpu, vmcs12, exit_reason); 6289 case EXIT_REASON_INVALID_STATE: 6290 return true; 6291 case EXIT_REASON_MWAIT_INSTRUCTION: 6292 return nested_cpu_has(vmcs12, CPU_BASED_MWAIT_EXITING); 6293 case EXIT_REASON_MONITOR_TRAP_FLAG: 6294 return nested_vmx_exit_handled_mtf(vmcs12); 6295 case EXIT_REASON_MONITOR_INSTRUCTION: 6296 return nested_cpu_has(vmcs12, CPU_BASED_MONITOR_EXITING); 6297 case EXIT_REASON_PAUSE_INSTRUCTION: 6298 return nested_cpu_has(vmcs12, CPU_BASED_PAUSE_EXITING) || 6299 nested_cpu_has2(vmcs12, 6300 SECONDARY_EXEC_PAUSE_LOOP_EXITING); 6301 case EXIT_REASON_MCE_DURING_VMENTRY: 6302 return true; 6303 case EXIT_REASON_TPR_BELOW_THRESHOLD: 6304 return nested_cpu_has(vmcs12, CPU_BASED_TPR_SHADOW); 6305 case EXIT_REASON_APIC_ACCESS: 6306 case EXIT_REASON_APIC_WRITE: 6307 case EXIT_REASON_EOI_INDUCED: 6308 /* 6309 * The controls for "virtualize APIC accesses," "APIC- 6310 * register virtualization," and "virtual-interrupt 6311 * delivery" only come from vmcs12. 6312 */ 6313 return true; 6314 case EXIT_REASON_INVPCID: 6315 return 6316 nested_cpu_has2(vmcs12, SECONDARY_EXEC_ENABLE_INVPCID) && 6317 nested_cpu_has(vmcs12, CPU_BASED_INVLPG_EXITING); 6318 case EXIT_REASON_WBINVD: 6319 return nested_cpu_has2(vmcs12, SECONDARY_EXEC_WBINVD_EXITING); 6320 case EXIT_REASON_XSETBV: 6321 return true; 6322 case EXIT_REASON_XSAVES: case EXIT_REASON_XRSTORS: 6323 /* 6324 * This should never happen, since it is not possible to 6325 * set XSS to a non-zero value---neither in L1 nor in L2. 6326 * If if it were, XSS would have to be checked against 6327 * the XSS exit bitmap in vmcs12. 6328 */ 6329 return nested_cpu_has2(vmcs12, SECONDARY_EXEC_XSAVES); 6330 case EXIT_REASON_UMWAIT: 6331 case EXIT_REASON_TPAUSE: 6332 return nested_cpu_has2(vmcs12, 6333 SECONDARY_EXEC_ENABLE_USR_WAIT_PAUSE); 6334 case EXIT_REASON_ENCLS: 6335 return nested_vmx_exit_handled_encls(vcpu, vmcs12); 6336 case EXIT_REASON_NOTIFY: 6337 /* Notify VM exit is not exposed to L1 */ 6338 return false; 6339 default: 6340 return true; 6341 } 6342 } 6343 6344 /* 6345 * Conditionally reflect a VM-Exit into L1. Returns %true if the VM-Exit was 6346 * reflected into L1. 6347 */ 6348 bool nested_vmx_reflect_vmexit(struct kvm_vcpu *vcpu) 6349 { 6350 struct vcpu_vmx *vmx = to_vmx(vcpu); 6351 union vmx_exit_reason exit_reason = vmx->exit_reason; 6352 unsigned long exit_qual; 6353 u32 exit_intr_info; 6354 6355 WARN_ON_ONCE(vmx->nested.nested_run_pending); 6356 6357 /* 6358 * Late nested VM-Fail shares the same flow as nested VM-Exit since KVM 6359 * has already loaded L2's state. 6360 */ 6361 if (unlikely(vmx->fail)) { 6362 trace_kvm_nested_vmenter_failed( 6363 "hardware VM-instruction error: ", 6364 vmcs_read32(VM_INSTRUCTION_ERROR)); 6365 exit_intr_info = 0; 6366 exit_qual = 0; 6367 goto reflect_vmexit; 6368 } 6369 6370 trace_kvm_nested_vmexit(vcpu, KVM_ISA_VMX); 6371 6372 /* If L0 (KVM) wants the exit, it trumps L1's desires. */ 6373 if (nested_vmx_l0_wants_exit(vcpu, exit_reason)) 6374 return false; 6375 6376 /* If L1 doesn't want the exit, handle it in L0. */ 6377 if (!nested_vmx_l1_wants_exit(vcpu, exit_reason)) 6378 return false; 6379 6380 /* 6381 * vmcs.VM_EXIT_INTR_INFO is only valid for EXCEPTION_NMI exits. For 6382 * EXTERNAL_INTERRUPT, the value for vmcs12->vm_exit_intr_info would 6383 * need to be synthesized by querying the in-kernel LAPIC, but external 6384 * interrupts are never reflected to L1 so it's a non-issue. 6385 */ 6386 exit_intr_info = vmx_get_intr_info(vcpu); 6387 if (is_exception_with_error_code(exit_intr_info)) { 6388 struct vmcs12 *vmcs12 = get_vmcs12(vcpu); 6389 6390 vmcs12->vm_exit_intr_error_code = 6391 vmcs_read32(VM_EXIT_INTR_ERROR_CODE); 6392 } 6393 exit_qual = vmx_get_exit_qual(vcpu); 6394 6395 reflect_vmexit: 6396 nested_vmx_vmexit(vcpu, exit_reason.full, exit_intr_info, exit_qual); 6397 return true; 6398 } 6399 6400 static int vmx_get_nested_state(struct kvm_vcpu *vcpu, 6401 struct kvm_nested_state __user *user_kvm_nested_state, 6402 u32 user_data_size) 6403 { 6404 struct vcpu_vmx *vmx; 6405 struct vmcs12 *vmcs12; 6406 struct kvm_nested_state kvm_state = { 6407 .flags = 0, 6408 .format = KVM_STATE_NESTED_FORMAT_VMX, 6409 .size = sizeof(kvm_state), 6410 .hdr.vmx.flags = 0, 6411 .hdr.vmx.vmxon_pa = INVALID_GPA, 6412 .hdr.vmx.vmcs12_pa = INVALID_GPA, 6413 .hdr.vmx.preemption_timer_deadline = 0, 6414 }; 6415 struct kvm_vmx_nested_state_data __user *user_vmx_nested_state = 6416 &user_kvm_nested_state->data.vmx[0]; 6417 6418 if (!vcpu) 6419 return kvm_state.size + sizeof(*user_vmx_nested_state); 6420 6421 vmx = to_vmx(vcpu); 6422 vmcs12 = get_vmcs12(vcpu); 6423 6424 if (nested_vmx_allowed(vcpu) && 6425 (vmx->nested.vmxon || vmx->nested.smm.vmxon)) { 6426 kvm_state.hdr.vmx.vmxon_pa = vmx->nested.vmxon_ptr; 6427 kvm_state.hdr.vmx.vmcs12_pa = vmx->nested.current_vmptr; 6428 6429 if (vmx_has_valid_vmcs12(vcpu)) { 6430 kvm_state.size += sizeof(user_vmx_nested_state->vmcs12); 6431 6432 /* 'hv_evmcs_vmptr' can also be EVMPTR_MAP_PENDING here */ 6433 if (vmx->nested.hv_evmcs_vmptr != EVMPTR_INVALID) 6434 kvm_state.flags |= KVM_STATE_NESTED_EVMCS; 6435 6436 if (is_guest_mode(vcpu) && 6437 nested_cpu_has_shadow_vmcs(vmcs12) && 6438 vmcs12->vmcs_link_pointer != INVALID_GPA) 6439 kvm_state.size += sizeof(user_vmx_nested_state->shadow_vmcs12); 6440 } 6441 6442 if (vmx->nested.smm.vmxon) 6443 kvm_state.hdr.vmx.smm.flags |= KVM_STATE_NESTED_SMM_VMXON; 6444 6445 if (vmx->nested.smm.guest_mode) 6446 kvm_state.hdr.vmx.smm.flags |= KVM_STATE_NESTED_SMM_GUEST_MODE; 6447 6448 if (is_guest_mode(vcpu)) { 6449 kvm_state.flags |= KVM_STATE_NESTED_GUEST_MODE; 6450 6451 if (vmx->nested.nested_run_pending) 6452 kvm_state.flags |= KVM_STATE_NESTED_RUN_PENDING; 6453 6454 if (vmx->nested.mtf_pending) 6455 kvm_state.flags |= KVM_STATE_NESTED_MTF_PENDING; 6456 6457 if (nested_cpu_has_preemption_timer(vmcs12) && 6458 vmx->nested.has_preemption_timer_deadline) { 6459 kvm_state.hdr.vmx.flags |= 6460 KVM_STATE_VMX_PREEMPTION_TIMER_DEADLINE; 6461 kvm_state.hdr.vmx.preemption_timer_deadline = 6462 vmx->nested.preemption_timer_deadline; 6463 } 6464 } 6465 } 6466 6467 if (user_data_size < kvm_state.size) 6468 goto out; 6469 6470 if (copy_to_user(user_kvm_nested_state, &kvm_state, sizeof(kvm_state))) 6471 return -EFAULT; 6472 6473 if (!vmx_has_valid_vmcs12(vcpu)) 6474 goto out; 6475 6476 /* 6477 * When running L2, the authoritative vmcs12 state is in the 6478 * vmcs02. When running L1, the authoritative vmcs12 state is 6479 * in the shadow or enlightened vmcs linked to vmcs01, unless 6480 * need_vmcs12_to_shadow_sync is set, in which case, the authoritative 6481 * vmcs12 state is in the vmcs12 already. 6482 */ 6483 if (is_guest_mode(vcpu)) { 6484 sync_vmcs02_to_vmcs12(vcpu, vmcs12); 6485 sync_vmcs02_to_vmcs12_rare(vcpu, vmcs12); 6486 } else { 6487 copy_vmcs02_to_vmcs12_rare(vcpu, get_vmcs12(vcpu)); 6488 if (!vmx->nested.need_vmcs12_to_shadow_sync) { 6489 if (evmptr_is_valid(vmx->nested.hv_evmcs_vmptr)) 6490 /* 6491 * L1 hypervisor is not obliged to keep eVMCS 6492 * clean fields data always up-to-date while 6493 * not in guest mode, 'hv_clean_fields' is only 6494 * supposed to be actual upon vmentry so we need 6495 * to ignore it here and do full copy. 6496 */ 6497 copy_enlightened_to_vmcs12(vmx, 0); 6498 else if (enable_shadow_vmcs) 6499 copy_shadow_to_vmcs12(vmx); 6500 } 6501 } 6502 6503 BUILD_BUG_ON(sizeof(user_vmx_nested_state->vmcs12) < VMCS12_SIZE); 6504 BUILD_BUG_ON(sizeof(user_vmx_nested_state->shadow_vmcs12) < VMCS12_SIZE); 6505 6506 /* 6507 * Copy over the full allocated size of vmcs12 rather than just the size 6508 * of the struct. 6509 */ 6510 if (copy_to_user(user_vmx_nested_state->vmcs12, vmcs12, VMCS12_SIZE)) 6511 return -EFAULT; 6512 6513 if (nested_cpu_has_shadow_vmcs(vmcs12) && 6514 vmcs12->vmcs_link_pointer != INVALID_GPA) { 6515 if (copy_to_user(user_vmx_nested_state->shadow_vmcs12, 6516 get_shadow_vmcs12(vcpu), VMCS12_SIZE)) 6517 return -EFAULT; 6518 } 6519 out: 6520 return kvm_state.size; 6521 } 6522 6523 void vmx_leave_nested(struct kvm_vcpu *vcpu) 6524 { 6525 if (is_guest_mode(vcpu)) { 6526 to_vmx(vcpu)->nested.nested_run_pending = 0; 6527 nested_vmx_vmexit(vcpu, -1, 0, 0); 6528 } 6529 free_nested(vcpu); 6530 } 6531 6532 static int vmx_set_nested_state(struct kvm_vcpu *vcpu, 6533 struct kvm_nested_state __user *user_kvm_nested_state, 6534 struct kvm_nested_state *kvm_state) 6535 { 6536 struct vcpu_vmx *vmx = to_vmx(vcpu); 6537 struct vmcs12 *vmcs12; 6538 enum vm_entry_failure_code ignored; 6539 struct kvm_vmx_nested_state_data __user *user_vmx_nested_state = 6540 &user_kvm_nested_state->data.vmx[0]; 6541 int ret; 6542 6543 if (kvm_state->format != KVM_STATE_NESTED_FORMAT_VMX) 6544 return -EINVAL; 6545 6546 if (kvm_state->hdr.vmx.vmxon_pa == INVALID_GPA) { 6547 if (kvm_state->hdr.vmx.smm.flags) 6548 return -EINVAL; 6549 6550 if (kvm_state->hdr.vmx.vmcs12_pa != INVALID_GPA) 6551 return -EINVAL; 6552 6553 /* 6554 * KVM_STATE_NESTED_EVMCS used to signal that KVM should 6555 * enable eVMCS capability on vCPU. However, since then 6556 * code was changed such that flag signals vmcs12 should 6557 * be copied into eVMCS in guest memory. 6558 * 6559 * To preserve backwards compatability, allow user 6560 * to set this flag even when there is no VMXON region. 6561 */ 6562 if (kvm_state->flags & ~KVM_STATE_NESTED_EVMCS) 6563 return -EINVAL; 6564 } else { 6565 if (!nested_vmx_allowed(vcpu)) 6566 return -EINVAL; 6567 6568 if (!page_address_valid(vcpu, kvm_state->hdr.vmx.vmxon_pa)) 6569 return -EINVAL; 6570 } 6571 6572 if ((kvm_state->hdr.vmx.smm.flags & KVM_STATE_NESTED_SMM_GUEST_MODE) && 6573 (kvm_state->flags & KVM_STATE_NESTED_GUEST_MODE)) 6574 return -EINVAL; 6575 6576 if (kvm_state->hdr.vmx.smm.flags & 6577 ~(KVM_STATE_NESTED_SMM_GUEST_MODE | KVM_STATE_NESTED_SMM_VMXON)) 6578 return -EINVAL; 6579 6580 if (kvm_state->hdr.vmx.flags & ~KVM_STATE_VMX_PREEMPTION_TIMER_DEADLINE) 6581 return -EINVAL; 6582 6583 /* 6584 * SMM temporarily disables VMX, so we cannot be in guest mode, 6585 * nor can VMLAUNCH/VMRESUME be pending. Outside SMM, SMM flags 6586 * must be zero. 6587 */ 6588 if (is_smm(vcpu) ? 6589 (kvm_state->flags & 6590 (KVM_STATE_NESTED_GUEST_MODE | KVM_STATE_NESTED_RUN_PENDING)) 6591 : kvm_state->hdr.vmx.smm.flags) 6592 return -EINVAL; 6593 6594 if ((kvm_state->hdr.vmx.smm.flags & KVM_STATE_NESTED_SMM_GUEST_MODE) && 6595 !(kvm_state->hdr.vmx.smm.flags & KVM_STATE_NESTED_SMM_VMXON)) 6596 return -EINVAL; 6597 6598 if ((kvm_state->flags & KVM_STATE_NESTED_EVMCS) && 6599 (!nested_vmx_allowed(vcpu) || !vmx->nested.enlightened_vmcs_enabled)) 6600 return -EINVAL; 6601 6602 vmx_leave_nested(vcpu); 6603 6604 if (kvm_state->hdr.vmx.vmxon_pa == INVALID_GPA) 6605 return 0; 6606 6607 vmx->nested.vmxon_ptr = kvm_state->hdr.vmx.vmxon_pa; 6608 ret = enter_vmx_operation(vcpu); 6609 if (ret) 6610 return ret; 6611 6612 /* Empty 'VMXON' state is permitted if no VMCS loaded */ 6613 if (kvm_state->size < sizeof(*kvm_state) + sizeof(*vmcs12)) { 6614 /* See vmx_has_valid_vmcs12. */ 6615 if ((kvm_state->flags & KVM_STATE_NESTED_GUEST_MODE) || 6616 (kvm_state->flags & KVM_STATE_NESTED_EVMCS) || 6617 (kvm_state->hdr.vmx.vmcs12_pa != INVALID_GPA)) 6618 return -EINVAL; 6619 else 6620 return 0; 6621 } 6622 6623 if (kvm_state->hdr.vmx.vmcs12_pa != INVALID_GPA) { 6624 if (kvm_state->hdr.vmx.vmcs12_pa == kvm_state->hdr.vmx.vmxon_pa || 6625 !page_address_valid(vcpu, kvm_state->hdr.vmx.vmcs12_pa)) 6626 return -EINVAL; 6627 6628 set_current_vmptr(vmx, kvm_state->hdr.vmx.vmcs12_pa); 6629 } else if (kvm_state->flags & KVM_STATE_NESTED_EVMCS) { 6630 /* 6631 * nested_vmx_handle_enlightened_vmptrld() cannot be called 6632 * directly from here as HV_X64_MSR_VP_ASSIST_PAGE may not be 6633 * restored yet. EVMCS will be mapped from 6634 * nested_get_vmcs12_pages(). 6635 */ 6636 vmx->nested.hv_evmcs_vmptr = EVMPTR_MAP_PENDING; 6637 kvm_make_request(KVM_REQ_GET_NESTED_STATE_PAGES, vcpu); 6638 } else { 6639 return -EINVAL; 6640 } 6641 6642 if (kvm_state->hdr.vmx.smm.flags & KVM_STATE_NESTED_SMM_VMXON) { 6643 vmx->nested.smm.vmxon = true; 6644 vmx->nested.vmxon = false; 6645 6646 if (kvm_state->hdr.vmx.smm.flags & KVM_STATE_NESTED_SMM_GUEST_MODE) 6647 vmx->nested.smm.guest_mode = true; 6648 } 6649 6650 vmcs12 = get_vmcs12(vcpu); 6651 if (copy_from_user(vmcs12, user_vmx_nested_state->vmcs12, sizeof(*vmcs12))) 6652 return -EFAULT; 6653 6654 if (vmcs12->hdr.revision_id != VMCS12_REVISION) 6655 return -EINVAL; 6656 6657 if (!(kvm_state->flags & KVM_STATE_NESTED_GUEST_MODE)) 6658 return 0; 6659 6660 vmx->nested.nested_run_pending = 6661 !!(kvm_state->flags & KVM_STATE_NESTED_RUN_PENDING); 6662 6663 vmx->nested.mtf_pending = 6664 !!(kvm_state->flags & KVM_STATE_NESTED_MTF_PENDING); 6665 6666 ret = -EINVAL; 6667 if (nested_cpu_has_shadow_vmcs(vmcs12) && 6668 vmcs12->vmcs_link_pointer != INVALID_GPA) { 6669 struct vmcs12 *shadow_vmcs12 = get_shadow_vmcs12(vcpu); 6670 6671 if (kvm_state->size < 6672 sizeof(*kvm_state) + 6673 sizeof(user_vmx_nested_state->vmcs12) + sizeof(*shadow_vmcs12)) 6674 goto error_guest_mode; 6675 6676 if (copy_from_user(shadow_vmcs12, 6677 user_vmx_nested_state->shadow_vmcs12, 6678 sizeof(*shadow_vmcs12))) { 6679 ret = -EFAULT; 6680 goto error_guest_mode; 6681 } 6682 6683 if (shadow_vmcs12->hdr.revision_id != VMCS12_REVISION || 6684 !shadow_vmcs12->hdr.shadow_vmcs) 6685 goto error_guest_mode; 6686 } 6687 6688 vmx->nested.has_preemption_timer_deadline = false; 6689 if (kvm_state->hdr.vmx.flags & KVM_STATE_VMX_PREEMPTION_TIMER_DEADLINE) { 6690 vmx->nested.has_preemption_timer_deadline = true; 6691 vmx->nested.preemption_timer_deadline = 6692 kvm_state->hdr.vmx.preemption_timer_deadline; 6693 } 6694 6695 if (nested_vmx_check_controls(vcpu, vmcs12) || 6696 nested_vmx_check_host_state(vcpu, vmcs12) || 6697 nested_vmx_check_guest_state(vcpu, vmcs12, &ignored)) 6698 goto error_guest_mode; 6699 6700 vmx->nested.dirty_vmcs12 = true; 6701 vmx->nested.force_msr_bitmap_recalc = true; 6702 ret = nested_vmx_enter_non_root_mode(vcpu, false); 6703 if (ret) 6704 goto error_guest_mode; 6705 6706 if (vmx->nested.mtf_pending) 6707 kvm_make_request(KVM_REQ_EVENT, vcpu); 6708 6709 return 0; 6710 6711 error_guest_mode: 6712 vmx->nested.nested_run_pending = 0; 6713 return ret; 6714 } 6715 6716 void nested_vmx_set_vmcs_shadowing_bitmap(void) 6717 { 6718 if (enable_shadow_vmcs) { 6719 vmcs_write64(VMREAD_BITMAP, __pa(vmx_vmread_bitmap)); 6720 vmcs_write64(VMWRITE_BITMAP, __pa(vmx_vmwrite_bitmap)); 6721 } 6722 } 6723 6724 /* 6725 * Indexing into the vmcs12 uses the VMCS encoding rotated left by 6. Undo 6726 * that madness to get the encoding for comparison. 6727 */ 6728 #define VMCS12_IDX_TO_ENC(idx) ((u16)(((u16)(idx) >> 6) | ((u16)(idx) << 10))) 6729 6730 static u64 nested_vmx_calc_vmcs_enum_msr(void) 6731 { 6732 /* 6733 * Note these are the so called "index" of the VMCS field encoding, not 6734 * the index into vmcs12. 6735 */ 6736 unsigned int max_idx, idx; 6737 int i; 6738 6739 /* 6740 * For better or worse, KVM allows VMREAD/VMWRITE to all fields in 6741 * vmcs12, regardless of whether or not the associated feature is 6742 * exposed to L1. Simply find the field with the highest index. 6743 */ 6744 max_idx = 0; 6745 for (i = 0; i < nr_vmcs12_fields; i++) { 6746 /* The vmcs12 table is very, very sparsely populated. */ 6747 if (!vmcs12_field_offsets[i]) 6748 continue; 6749 6750 idx = vmcs_field_index(VMCS12_IDX_TO_ENC(i)); 6751 if (idx > max_idx) 6752 max_idx = idx; 6753 } 6754 6755 return (u64)max_idx << VMCS_FIELD_INDEX_SHIFT; 6756 } 6757 6758 /* 6759 * nested_vmx_setup_ctls_msrs() sets up variables containing the values to be 6760 * returned for the various VMX controls MSRs when nested VMX is enabled. 6761 * The same values should also be used to verify that vmcs12 control fields are 6762 * valid during nested entry from L1 to L2. 6763 * Each of these control msrs has a low and high 32-bit half: A low bit is on 6764 * if the corresponding bit in the (32-bit) control field *must* be on, and a 6765 * bit in the high half is on if the corresponding bit in the control field 6766 * may be on. See also vmx_control_verify(). 6767 */ 6768 void nested_vmx_setup_ctls_msrs(struct vmcs_config *vmcs_conf, u32 ept_caps) 6769 { 6770 struct nested_vmx_msrs *msrs = &vmcs_conf->nested; 6771 6772 /* 6773 * Note that as a general rule, the high half of the MSRs (bits in 6774 * the control fields which may be 1) should be initialized by the 6775 * intersection of the underlying hardware's MSR (i.e., features which 6776 * can be supported) and the list of features we want to expose - 6777 * because they are known to be properly supported in our code. 6778 * Also, usually, the low half of the MSRs (bits which must be 1) can 6779 * be set to 0, meaning that L1 may turn off any of these bits. The 6780 * reason is that if one of these bits is necessary, it will appear 6781 * in vmcs01 and prepare_vmcs02, when it bitwise-or's the control 6782 * fields of vmcs01 and vmcs02, will turn these bits off - and 6783 * nested_vmx_l1_wants_exit() will not pass related exits to L1. 6784 * These rules have exceptions below. 6785 */ 6786 6787 /* pin-based controls */ 6788 msrs->pinbased_ctls_low = 6789 PIN_BASED_ALWAYSON_WITHOUT_TRUE_MSR; 6790 6791 msrs->pinbased_ctls_high = vmcs_conf->pin_based_exec_ctrl; 6792 msrs->pinbased_ctls_high &= 6793 PIN_BASED_EXT_INTR_MASK | 6794 PIN_BASED_NMI_EXITING | 6795 PIN_BASED_VIRTUAL_NMIS | 6796 (enable_apicv ? PIN_BASED_POSTED_INTR : 0); 6797 msrs->pinbased_ctls_high |= 6798 PIN_BASED_ALWAYSON_WITHOUT_TRUE_MSR | 6799 PIN_BASED_VMX_PREEMPTION_TIMER; 6800 6801 /* exit controls */ 6802 msrs->exit_ctls_low = 6803 VM_EXIT_ALWAYSON_WITHOUT_TRUE_MSR; 6804 6805 msrs->exit_ctls_high = vmcs_conf->vmexit_ctrl; 6806 msrs->exit_ctls_high &= 6807 #ifdef CONFIG_X86_64 6808 VM_EXIT_HOST_ADDR_SPACE_SIZE | 6809 #endif 6810 VM_EXIT_LOAD_IA32_PAT | VM_EXIT_SAVE_IA32_PAT | 6811 VM_EXIT_CLEAR_BNDCFGS; 6812 msrs->exit_ctls_high |= 6813 VM_EXIT_ALWAYSON_WITHOUT_TRUE_MSR | 6814 VM_EXIT_LOAD_IA32_EFER | VM_EXIT_SAVE_IA32_EFER | 6815 VM_EXIT_SAVE_VMX_PREEMPTION_TIMER | VM_EXIT_ACK_INTR_ON_EXIT | 6816 VM_EXIT_LOAD_IA32_PERF_GLOBAL_CTRL; 6817 6818 /* We support free control of debug control saving. */ 6819 msrs->exit_ctls_low &= ~VM_EXIT_SAVE_DEBUG_CONTROLS; 6820 6821 /* entry controls */ 6822 msrs->entry_ctls_low = 6823 VM_ENTRY_ALWAYSON_WITHOUT_TRUE_MSR; 6824 6825 msrs->entry_ctls_high = vmcs_conf->vmentry_ctrl; 6826 msrs->entry_ctls_high &= 6827 #ifdef CONFIG_X86_64 6828 VM_ENTRY_IA32E_MODE | 6829 #endif 6830 VM_ENTRY_LOAD_IA32_PAT | VM_ENTRY_LOAD_BNDCFGS; 6831 msrs->entry_ctls_high |= 6832 (VM_ENTRY_ALWAYSON_WITHOUT_TRUE_MSR | VM_ENTRY_LOAD_IA32_EFER | 6833 VM_ENTRY_LOAD_IA32_PERF_GLOBAL_CTRL); 6834 6835 /* We support free control of debug control loading. */ 6836 msrs->entry_ctls_low &= ~VM_ENTRY_LOAD_DEBUG_CONTROLS; 6837 6838 /* cpu-based controls */ 6839 msrs->procbased_ctls_low = 6840 CPU_BASED_ALWAYSON_WITHOUT_TRUE_MSR; 6841 6842 msrs->procbased_ctls_high = vmcs_conf->cpu_based_exec_ctrl; 6843 msrs->procbased_ctls_high &= 6844 CPU_BASED_INTR_WINDOW_EXITING | 6845 CPU_BASED_NMI_WINDOW_EXITING | CPU_BASED_USE_TSC_OFFSETTING | 6846 CPU_BASED_HLT_EXITING | CPU_BASED_INVLPG_EXITING | 6847 CPU_BASED_MWAIT_EXITING | CPU_BASED_CR3_LOAD_EXITING | 6848 CPU_BASED_CR3_STORE_EXITING | 6849 #ifdef CONFIG_X86_64 6850 CPU_BASED_CR8_LOAD_EXITING | CPU_BASED_CR8_STORE_EXITING | 6851 #endif 6852 CPU_BASED_MOV_DR_EXITING | CPU_BASED_UNCOND_IO_EXITING | 6853 CPU_BASED_USE_IO_BITMAPS | CPU_BASED_MONITOR_TRAP_FLAG | 6854 CPU_BASED_MONITOR_EXITING | CPU_BASED_RDPMC_EXITING | 6855 CPU_BASED_RDTSC_EXITING | CPU_BASED_PAUSE_EXITING | 6856 CPU_BASED_TPR_SHADOW | CPU_BASED_ACTIVATE_SECONDARY_CONTROLS; 6857 /* 6858 * We can allow some features even when not supported by the 6859 * hardware. For example, L1 can specify an MSR bitmap - and we 6860 * can use it to avoid exits to L1 - even when L0 runs L2 6861 * without MSR bitmaps. 6862 */ 6863 msrs->procbased_ctls_high |= 6864 CPU_BASED_ALWAYSON_WITHOUT_TRUE_MSR | 6865 CPU_BASED_USE_MSR_BITMAPS; 6866 6867 /* We support free control of CR3 access interception. */ 6868 msrs->procbased_ctls_low &= 6869 ~(CPU_BASED_CR3_LOAD_EXITING | CPU_BASED_CR3_STORE_EXITING); 6870 6871 /* 6872 * secondary cpu-based controls. Do not include those that 6873 * depend on CPUID bits, they are added later by 6874 * vmx_vcpu_after_set_cpuid. 6875 */ 6876 msrs->secondary_ctls_low = 0; 6877 6878 msrs->secondary_ctls_high = vmcs_conf->cpu_based_2nd_exec_ctrl; 6879 msrs->secondary_ctls_high &= 6880 SECONDARY_EXEC_DESC | 6881 SECONDARY_EXEC_ENABLE_RDTSCP | 6882 SECONDARY_EXEC_VIRTUALIZE_X2APIC_MODE | 6883 SECONDARY_EXEC_WBINVD_EXITING | 6884 SECONDARY_EXEC_APIC_REGISTER_VIRT | 6885 SECONDARY_EXEC_VIRTUAL_INTR_DELIVERY | 6886 SECONDARY_EXEC_RDRAND_EXITING | 6887 SECONDARY_EXEC_ENABLE_INVPCID | 6888 SECONDARY_EXEC_ENABLE_VMFUNC | 6889 SECONDARY_EXEC_RDSEED_EXITING | 6890 SECONDARY_EXEC_XSAVES | 6891 SECONDARY_EXEC_TSC_SCALING | 6892 SECONDARY_EXEC_ENABLE_USR_WAIT_PAUSE; 6893 6894 /* 6895 * We can emulate "VMCS shadowing," even if the hardware 6896 * doesn't support it. 6897 */ 6898 msrs->secondary_ctls_high |= 6899 SECONDARY_EXEC_SHADOW_VMCS; 6900 6901 if (enable_ept) { 6902 /* nested EPT: emulate EPT also to L1 */ 6903 msrs->secondary_ctls_high |= 6904 SECONDARY_EXEC_ENABLE_EPT; 6905 msrs->ept_caps = 6906 VMX_EPT_PAGE_WALK_4_BIT | 6907 VMX_EPT_PAGE_WALK_5_BIT | 6908 VMX_EPTP_WB_BIT | 6909 VMX_EPT_INVEPT_BIT | 6910 VMX_EPT_EXECUTE_ONLY_BIT; 6911 6912 msrs->ept_caps &= ept_caps; 6913 msrs->ept_caps |= VMX_EPT_EXTENT_GLOBAL_BIT | 6914 VMX_EPT_EXTENT_CONTEXT_BIT | VMX_EPT_2MB_PAGE_BIT | 6915 VMX_EPT_1GB_PAGE_BIT; 6916 if (enable_ept_ad_bits) { 6917 msrs->secondary_ctls_high |= 6918 SECONDARY_EXEC_ENABLE_PML; 6919 msrs->ept_caps |= VMX_EPT_AD_BIT; 6920 } 6921 6922 /* 6923 * Advertise EPTP switching irrespective of hardware support, 6924 * KVM emulates it in software so long as VMFUNC is supported. 6925 */ 6926 if (cpu_has_vmx_vmfunc()) 6927 msrs->vmfunc_controls = VMX_VMFUNC_EPTP_SWITCHING; 6928 } 6929 6930 /* 6931 * Old versions of KVM use the single-context version without 6932 * checking for support, so declare that it is supported even 6933 * though it is treated as global context. The alternative is 6934 * not failing the single-context invvpid, and it is worse. 6935 */ 6936 if (enable_vpid) { 6937 msrs->secondary_ctls_high |= 6938 SECONDARY_EXEC_ENABLE_VPID; 6939 msrs->vpid_caps = VMX_VPID_INVVPID_BIT | 6940 VMX_VPID_EXTENT_SUPPORTED_MASK; 6941 } 6942 6943 if (enable_unrestricted_guest) 6944 msrs->secondary_ctls_high |= 6945 SECONDARY_EXEC_UNRESTRICTED_GUEST; 6946 6947 if (flexpriority_enabled) 6948 msrs->secondary_ctls_high |= 6949 SECONDARY_EXEC_VIRTUALIZE_APIC_ACCESSES; 6950 6951 if (enable_sgx) 6952 msrs->secondary_ctls_high |= SECONDARY_EXEC_ENCLS_EXITING; 6953 6954 /* miscellaneous data */ 6955 msrs->misc_low = (u32)vmcs_conf->misc & VMX_MISC_SAVE_EFER_LMA; 6956 msrs->misc_low |= 6957 MSR_IA32_VMX_MISC_VMWRITE_SHADOW_RO_FIELDS | 6958 VMX_MISC_EMULATED_PREEMPTION_TIMER_RATE | 6959 VMX_MISC_ACTIVITY_HLT | 6960 VMX_MISC_ACTIVITY_WAIT_SIPI; 6961 msrs->misc_high = 0; 6962 6963 /* 6964 * This MSR reports some information about VMX support. We 6965 * should return information about the VMX we emulate for the 6966 * guest, and the VMCS structure we give it - not about the 6967 * VMX support of the underlying hardware. 6968 */ 6969 msrs->basic = 6970 VMCS12_REVISION | 6971 VMX_BASIC_TRUE_CTLS | 6972 ((u64)VMCS12_SIZE << VMX_BASIC_VMCS_SIZE_SHIFT) | 6973 (VMX_BASIC_MEM_TYPE_WB << VMX_BASIC_MEM_TYPE_SHIFT); 6974 6975 if (cpu_has_vmx_basic_inout()) 6976 msrs->basic |= VMX_BASIC_INOUT; 6977 6978 /* 6979 * These MSRs specify bits which the guest must keep fixed on 6980 * while L1 is in VMXON mode (in L1's root mode, or running an L2). 6981 * We picked the standard core2 setting. 6982 */ 6983 #define VMXON_CR0_ALWAYSON (X86_CR0_PE | X86_CR0_PG | X86_CR0_NE) 6984 #define VMXON_CR4_ALWAYSON X86_CR4_VMXE 6985 msrs->cr0_fixed0 = VMXON_CR0_ALWAYSON; 6986 msrs->cr4_fixed0 = VMXON_CR4_ALWAYSON; 6987 6988 /* These MSRs specify bits which the guest must keep fixed off. */ 6989 rdmsrl(MSR_IA32_VMX_CR0_FIXED1, msrs->cr0_fixed1); 6990 rdmsrl(MSR_IA32_VMX_CR4_FIXED1, msrs->cr4_fixed1); 6991 6992 if (vmx_umip_emulated()) 6993 msrs->cr4_fixed1 |= X86_CR4_UMIP; 6994 6995 msrs->vmcs_enum = nested_vmx_calc_vmcs_enum_msr(); 6996 } 6997 6998 void nested_vmx_hardware_unsetup(void) 6999 { 7000 int i; 7001 7002 if (enable_shadow_vmcs) { 7003 for (i = 0; i < VMX_BITMAP_NR; i++) 7004 free_page((unsigned long)vmx_bitmap[i]); 7005 } 7006 } 7007 7008 __init int nested_vmx_hardware_setup(int (*exit_handlers[])(struct kvm_vcpu *)) 7009 { 7010 int i; 7011 7012 if (!cpu_has_vmx_shadow_vmcs()) 7013 enable_shadow_vmcs = 0; 7014 if (enable_shadow_vmcs) { 7015 for (i = 0; i < VMX_BITMAP_NR; i++) { 7016 /* 7017 * The vmx_bitmap is not tied to a VM and so should 7018 * not be charged to a memcg. 7019 */ 7020 vmx_bitmap[i] = (unsigned long *) 7021 __get_free_page(GFP_KERNEL); 7022 if (!vmx_bitmap[i]) { 7023 nested_vmx_hardware_unsetup(); 7024 return -ENOMEM; 7025 } 7026 } 7027 7028 init_vmcs_shadow_fields(); 7029 } 7030 7031 exit_handlers[EXIT_REASON_VMCLEAR] = handle_vmclear; 7032 exit_handlers[EXIT_REASON_VMLAUNCH] = handle_vmlaunch; 7033 exit_handlers[EXIT_REASON_VMPTRLD] = handle_vmptrld; 7034 exit_handlers[EXIT_REASON_VMPTRST] = handle_vmptrst; 7035 exit_handlers[EXIT_REASON_VMREAD] = handle_vmread; 7036 exit_handlers[EXIT_REASON_VMRESUME] = handle_vmresume; 7037 exit_handlers[EXIT_REASON_VMWRITE] = handle_vmwrite; 7038 exit_handlers[EXIT_REASON_VMOFF] = handle_vmxoff; 7039 exit_handlers[EXIT_REASON_VMON] = handle_vmxon; 7040 exit_handlers[EXIT_REASON_INVEPT] = handle_invept; 7041 exit_handlers[EXIT_REASON_INVVPID] = handle_invvpid; 7042 exit_handlers[EXIT_REASON_VMFUNC] = handle_vmfunc; 7043 7044 return 0; 7045 } 7046 7047 struct kvm_x86_nested_ops vmx_nested_ops = { 7048 .leave_nested = vmx_leave_nested, 7049 .is_exception_vmexit = nested_vmx_is_exception_vmexit, 7050 .check_events = vmx_check_nested_events, 7051 .has_events = vmx_has_nested_events, 7052 .triple_fault = nested_vmx_triple_fault, 7053 .get_state = vmx_get_nested_state, 7054 .set_state = vmx_set_nested_state, 7055 .get_nested_state_pages = vmx_get_nested_state_pages, 7056 .write_log_dirty = nested_vmx_write_pml_buffer, 7057 .enable_evmcs = nested_enable_evmcs, 7058 .get_evmcs_version = nested_get_evmcs_version, 7059 .hv_inject_synthetic_vmexit_post_tlb_flush = vmx_hv_inject_synthetic_vmexit_post_tlb_flush, 7060 }; 7061