1 // SPDX-License-Identifier: GPL-2.0 2 3 #include <linux/objtool.h> 4 #include <linux/percpu.h> 5 6 #include <asm/debugreg.h> 7 #include <asm/mmu_context.h> 8 9 #include "cpuid.h" 10 #include "hyperv.h" 11 #include "mmu.h" 12 #include "nested.h" 13 #include "pmu.h" 14 #include "sgx.h" 15 #include "trace.h" 16 #include "vmx.h" 17 #include "x86.h" 18 19 static bool __read_mostly enable_shadow_vmcs = 1; 20 module_param_named(enable_shadow_vmcs, enable_shadow_vmcs, bool, S_IRUGO); 21 22 static bool __read_mostly nested_early_check = 0; 23 module_param(nested_early_check, bool, S_IRUGO); 24 25 #define CC KVM_NESTED_VMENTER_CONSISTENCY_CHECK 26 27 /* 28 * Hyper-V requires all of these, so mark them as supported even though 29 * they are just treated the same as all-context. 30 */ 31 #define VMX_VPID_EXTENT_SUPPORTED_MASK \ 32 (VMX_VPID_EXTENT_INDIVIDUAL_ADDR_BIT | \ 33 VMX_VPID_EXTENT_SINGLE_CONTEXT_BIT | \ 34 VMX_VPID_EXTENT_GLOBAL_CONTEXT_BIT | \ 35 VMX_VPID_EXTENT_SINGLE_NON_GLOBAL_BIT) 36 37 #define VMX_MISC_EMULATED_PREEMPTION_TIMER_RATE 5 38 39 enum { 40 VMX_VMREAD_BITMAP, 41 VMX_VMWRITE_BITMAP, 42 VMX_BITMAP_NR 43 }; 44 static unsigned long *vmx_bitmap[VMX_BITMAP_NR]; 45 46 #define vmx_vmread_bitmap (vmx_bitmap[VMX_VMREAD_BITMAP]) 47 #define vmx_vmwrite_bitmap (vmx_bitmap[VMX_VMWRITE_BITMAP]) 48 49 struct shadow_vmcs_field { 50 u16 encoding; 51 u16 offset; 52 }; 53 static struct shadow_vmcs_field shadow_read_only_fields[] = { 54 #define SHADOW_FIELD_RO(x, y) { x, offsetof(struct vmcs12, y) }, 55 #include "vmcs_shadow_fields.h" 56 }; 57 static int max_shadow_read_only_fields = 58 ARRAY_SIZE(shadow_read_only_fields); 59 60 static struct shadow_vmcs_field shadow_read_write_fields[] = { 61 #define SHADOW_FIELD_RW(x, y) { x, offsetof(struct vmcs12, y) }, 62 #include "vmcs_shadow_fields.h" 63 }; 64 static int max_shadow_read_write_fields = 65 ARRAY_SIZE(shadow_read_write_fields); 66 67 static void init_vmcs_shadow_fields(void) 68 { 69 int i, j; 70 71 memset(vmx_vmread_bitmap, 0xff, PAGE_SIZE); 72 memset(vmx_vmwrite_bitmap, 0xff, PAGE_SIZE); 73 74 for (i = j = 0; i < max_shadow_read_only_fields; i++) { 75 struct shadow_vmcs_field entry = shadow_read_only_fields[i]; 76 u16 field = entry.encoding; 77 78 if (vmcs_field_width(field) == VMCS_FIELD_WIDTH_U64 && 79 (i + 1 == max_shadow_read_only_fields || 80 shadow_read_only_fields[i + 1].encoding != field + 1)) 81 pr_err("Missing field from shadow_read_only_field %x\n", 82 field + 1); 83 84 clear_bit(field, vmx_vmread_bitmap); 85 if (field & 1) 86 #ifdef CONFIG_X86_64 87 continue; 88 #else 89 entry.offset += sizeof(u32); 90 #endif 91 shadow_read_only_fields[j++] = entry; 92 } 93 max_shadow_read_only_fields = j; 94 95 for (i = j = 0; i < max_shadow_read_write_fields; i++) { 96 struct shadow_vmcs_field entry = shadow_read_write_fields[i]; 97 u16 field = entry.encoding; 98 99 if (vmcs_field_width(field) == VMCS_FIELD_WIDTH_U64 && 100 (i + 1 == max_shadow_read_write_fields || 101 shadow_read_write_fields[i + 1].encoding != field + 1)) 102 pr_err("Missing field from shadow_read_write_field %x\n", 103 field + 1); 104 105 WARN_ONCE(field >= GUEST_ES_AR_BYTES && 106 field <= GUEST_TR_AR_BYTES, 107 "Update vmcs12_write_any() to drop reserved bits from AR_BYTES"); 108 109 /* 110 * PML and the preemption timer can be emulated, but the 111 * processor cannot vmwrite to fields that don't exist 112 * on bare metal. 113 */ 114 switch (field) { 115 case GUEST_PML_INDEX: 116 if (!cpu_has_vmx_pml()) 117 continue; 118 break; 119 case VMX_PREEMPTION_TIMER_VALUE: 120 if (!cpu_has_vmx_preemption_timer()) 121 continue; 122 break; 123 case GUEST_INTR_STATUS: 124 if (!cpu_has_vmx_apicv()) 125 continue; 126 break; 127 default: 128 break; 129 } 130 131 clear_bit(field, vmx_vmwrite_bitmap); 132 clear_bit(field, vmx_vmread_bitmap); 133 if (field & 1) 134 #ifdef CONFIG_X86_64 135 continue; 136 #else 137 entry.offset += sizeof(u32); 138 #endif 139 shadow_read_write_fields[j++] = entry; 140 } 141 max_shadow_read_write_fields = j; 142 } 143 144 /* 145 * The following 3 functions, nested_vmx_succeed()/failValid()/failInvalid(), 146 * set the success or error code of an emulated VMX instruction (as specified 147 * by Vol 2B, VMX Instruction Reference, "Conventions"), and skip the emulated 148 * instruction. 149 */ 150 static int nested_vmx_succeed(struct kvm_vcpu *vcpu) 151 { 152 vmx_set_rflags(vcpu, vmx_get_rflags(vcpu) 153 & ~(X86_EFLAGS_CF | X86_EFLAGS_PF | X86_EFLAGS_AF | 154 X86_EFLAGS_ZF | X86_EFLAGS_SF | X86_EFLAGS_OF)); 155 return kvm_skip_emulated_instruction(vcpu); 156 } 157 158 static int nested_vmx_failInvalid(struct kvm_vcpu *vcpu) 159 { 160 vmx_set_rflags(vcpu, (vmx_get_rflags(vcpu) 161 & ~(X86_EFLAGS_PF | X86_EFLAGS_AF | X86_EFLAGS_ZF | 162 X86_EFLAGS_SF | X86_EFLAGS_OF)) 163 | X86_EFLAGS_CF); 164 return kvm_skip_emulated_instruction(vcpu); 165 } 166 167 static int nested_vmx_failValid(struct kvm_vcpu *vcpu, 168 u32 vm_instruction_error) 169 { 170 vmx_set_rflags(vcpu, (vmx_get_rflags(vcpu) 171 & ~(X86_EFLAGS_CF | X86_EFLAGS_PF | X86_EFLAGS_AF | 172 X86_EFLAGS_SF | X86_EFLAGS_OF)) 173 | X86_EFLAGS_ZF); 174 get_vmcs12(vcpu)->vm_instruction_error = vm_instruction_error; 175 /* 176 * We don't need to force sync to shadow VMCS because 177 * VM_INSTRUCTION_ERROR is not shadowed. Enlightened VMCS 'shadows' all 178 * fields and thus must be synced. 179 */ 180 if (to_vmx(vcpu)->nested.hv_evmcs_vmptr != EVMPTR_INVALID) 181 to_vmx(vcpu)->nested.need_vmcs12_to_shadow_sync = true; 182 183 return kvm_skip_emulated_instruction(vcpu); 184 } 185 186 static int nested_vmx_fail(struct kvm_vcpu *vcpu, u32 vm_instruction_error) 187 { 188 struct vcpu_vmx *vmx = to_vmx(vcpu); 189 190 /* 191 * failValid writes the error number to the current VMCS, which 192 * can't be done if there isn't a current VMCS. 193 */ 194 if (vmx->nested.current_vmptr == INVALID_GPA && 195 !evmptr_is_valid(vmx->nested.hv_evmcs_vmptr)) 196 return nested_vmx_failInvalid(vcpu); 197 198 return nested_vmx_failValid(vcpu, vm_instruction_error); 199 } 200 201 static void nested_vmx_abort(struct kvm_vcpu *vcpu, u32 indicator) 202 { 203 /* TODO: not to reset guest simply here. */ 204 kvm_make_request(KVM_REQ_TRIPLE_FAULT, vcpu); 205 pr_debug_ratelimited("kvm: nested vmx abort, indicator %d\n", indicator); 206 } 207 208 static inline bool vmx_control_verify(u32 control, u32 low, u32 high) 209 { 210 return fixed_bits_valid(control, low, high); 211 } 212 213 static inline u64 vmx_control_msr(u32 low, u32 high) 214 { 215 return low | ((u64)high << 32); 216 } 217 218 static void vmx_disable_shadow_vmcs(struct vcpu_vmx *vmx) 219 { 220 secondary_exec_controls_clearbit(vmx, SECONDARY_EXEC_SHADOW_VMCS); 221 vmcs_write64(VMCS_LINK_POINTER, INVALID_GPA); 222 vmx->nested.need_vmcs12_to_shadow_sync = false; 223 } 224 225 static inline void nested_release_evmcs(struct kvm_vcpu *vcpu) 226 { 227 struct vcpu_vmx *vmx = to_vmx(vcpu); 228 229 if (evmptr_is_valid(vmx->nested.hv_evmcs_vmptr)) { 230 kvm_vcpu_unmap(vcpu, &vmx->nested.hv_evmcs_map, true); 231 vmx->nested.hv_evmcs = NULL; 232 } 233 234 vmx->nested.hv_evmcs_vmptr = EVMPTR_INVALID; 235 } 236 237 static void vmx_sync_vmcs_host_state(struct vcpu_vmx *vmx, 238 struct loaded_vmcs *prev) 239 { 240 struct vmcs_host_state *dest, *src; 241 242 if (unlikely(!vmx->guest_state_loaded)) 243 return; 244 245 src = &prev->host_state; 246 dest = &vmx->loaded_vmcs->host_state; 247 248 vmx_set_host_fs_gs(dest, src->fs_sel, src->gs_sel, src->fs_base, src->gs_base); 249 dest->ldt_sel = src->ldt_sel; 250 #ifdef CONFIG_X86_64 251 dest->ds_sel = src->ds_sel; 252 dest->es_sel = src->es_sel; 253 #endif 254 } 255 256 static void vmx_switch_vmcs(struct kvm_vcpu *vcpu, struct loaded_vmcs *vmcs) 257 { 258 struct vcpu_vmx *vmx = to_vmx(vcpu); 259 struct loaded_vmcs *prev; 260 int cpu; 261 262 if (WARN_ON_ONCE(vmx->loaded_vmcs == vmcs)) 263 return; 264 265 cpu = get_cpu(); 266 prev = vmx->loaded_vmcs; 267 vmx->loaded_vmcs = vmcs; 268 vmx_vcpu_load_vmcs(vcpu, cpu, prev); 269 vmx_sync_vmcs_host_state(vmx, prev); 270 put_cpu(); 271 272 vmx_register_cache_reset(vcpu); 273 } 274 275 /* 276 * Free whatever needs to be freed from vmx->nested when L1 goes down, or 277 * just stops using VMX. 278 */ 279 static void free_nested(struct kvm_vcpu *vcpu) 280 { 281 struct vcpu_vmx *vmx = to_vmx(vcpu); 282 283 if (WARN_ON_ONCE(vmx->loaded_vmcs != &vmx->vmcs01)) 284 vmx_switch_vmcs(vcpu, &vmx->vmcs01); 285 286 if (!vmx->nested.vmxon && !vmx->nested.smm.vmxon) 287 return; 288 289 kvm_clear_request(KVM_REQ_GET_NESTED_STATE_PAGES, vcpu); 290 291 vmx->nested.vmxon = false; 292 vmx->nested.smm.vmxon = false; 293 vmx->nested.vmxon_ptr = INVALID_GPA; 294 free_vpid(vmx->nested.vpid02); 295 vmx->nested.posted_intr_nv = -1; 296 vmx->nested.current_vmptr = INVALID_GPA; 297 if (enable_shadow_vmcs) { 298 vmx_disable_shadow_vmcs(vmx); 299 vmcs_clear(vmx->vmcs01.shadow_vmcs); 300 free_vmcs(vmx->vmcs01.shadow_vmcs); 301 vmx->vmcs01.shadow_vmcs = NULL; 302 } 303 kfree(vmx->nested.cached_vmcs12); 304 vmx->nested.cached_vmcs12 = NULL; 305 kfree(vmx->nested.cached_shadow_vmcs12); 306 vmx->nested.cached_shadow_vmcs12 = NULL; 307 /* Unpin physical memory we referred to in the vmcs02 */ 308 if (vmx->nested.apic_access_page) { 309 kvm_release_page_clean(vmx->nested.apic_access_page); 310 vmx->nested.apic_access_page = NULL; 311 } 312 kvm_vcpu_unmap(vcpu, &vmx->nested.virtual_apic_map, true); 313 kvm_vcpu_unmap(vcpu, &vmx->nested.pi_desc_map, true); 314 vmx->nested.pi_desc = NULL; 315 316 kvm_mmu_free_roots(vcpu, &vcpu->arch.guest_mmu, KVM_MMU_ROOTS_ALL); 317 318 nested_release_evmcs(vcpu); 319 320 free_loaded_vmcs(&vmx->nested.vmcs02); 321 } 322 323 /* 324 * Ensure that the current vmcs of the logical processor is the 325 * vmcs01 of the vcpu before calling free_nested(). 326 */ 327 void nested_vmx_free_vcpu(struct kvm_vcpu *vcpu) 328 { 329 vcpu_load(vcpu); 330 vmx_leave_nested(vcpu); 331 vcpu_put(vcpu); 332 } 333 334 #define EPTP_PA_MASK GENMASK_ULL(51, 12) 335 336 static bool nested_ept_root_matches(hpa_t root_hpa, u64 root_eptp, u64 eptp) 337 { 338 return VALID_PAGE(root_hpa) && 339 ((root_eptp & EPTP_PA_MASK) == (eptp & EPTP_PA_MASK)); 340 } 341 342 static void nested_ept_invalidate_addr(struct kvm_vcpu *vcpu, gpa_t eptp, 343 gpa_t addr) 344 { 345 uint i; 346 struct kvm_mmu_root_info *cached_root; 347 348 WARN_ON_ONCE(!mmu_is_nested(vcpu)); 349 350 for (i = 0; i < KVM_MMU_NUM_PREV_ROOTS; i++) { 351 cached_root = &vcpu->arch.mmu->prev_roots[i]; 352 353 if (nested_ept_root_matches(cached_root->hpa, cached_root->pgd, 354 eptp)) 355 vcpu->arch.mmu->invlpg(vcpu, addr, cached_root->hpa); 356 } 357 } 358 359 static void nested_ept_inject_page_fault(struct kvm_vcpu *vcpu, 360 struct x86_exception *fault) 361 { 362 struct vmcs12 *vmcs12 = get_vmcs12(vcpu); 363 struct vcpu_vmx *vmx = to_vmx(vcpu); 364 u32 vm_exit_reason; 365 unsigned long exit_qualification = vcpu->arch.exit_qualification; 366 367 if (vmx->nested.pml_full) { 368 vm_exit_reason = EXIT_REASON_PML_FULL; 369 vmx->nested.pml_full = false; 370 exit_qualification &= INTR_INFO_UNBLOCK_NMI; 371 } else { 372 if (fault->error_code & PFERR_RSVD_MASK) 373 vm_exit_reason = EXIT_REASON_EPT_MISCONFIG; 374 else 375 vm_exit_reason = EXIT_REASON_EPT_VIOLATION; 376 377 /* 378 * Although the caller (kvm_inject_emulated_page_fault) would 379 * have already synced the faulting address in the shadow EPT 380 * tables for the current EPTP12, we also need to sync it for 381 * any other cached EPTP02s based on the same EP4TA, since the 382 * TLB associates mappings to the EP4TA rather than the full EPTP. 383 */ 384 nested_ept_invalidate_addr(vcpu, vmcs12->ept_pointer, 385 fault->address); 386 } 387 388 nested_vmx_vmexit(vcpu, vm_exit_reason, 0, exit_qualification); 389 vmcs12->guest_physical_address = fault->address; 390 } 391 392 static void nested_ept_new_eptp(struct kvm_vcpu *vcpu) 393 { 394 kvm_init_shadow_ept_mmu(vcpu, 395 to_vmx(vcpu)->nested.msrs.ept_caps & 396 VMX_EPT_EXECUTE_ONLY_BIT, 397 nested_ept_ad_enabled(vcpu), 398 nested_ept_get_eptp(vcpu)); 399 } 400 401 static void nested_ept_init_mmu_context(struct kvm_vcpu *vcpu) 402 { 403 WARN_ON(mmu_is_nested(vcpu)); 404 405 vcpu->arch.mmu = &vcpu->arch.guest_mmu; 406 nested_ept_new_eptp(vcpu); 407 vcpu->arch.mmu->get_guest_pgd = nested_ept_get_eptp; 408 vcpu->arch.mmu->inject_page_fault = nested_ept_inject_page_fault; 409 vcpu->arch.mmu->get_pdptr = kvm_pdptr_read; 410 411 vcpu->arch.walk_mmu = &vcpu->arch.nested_mmu; 412 } 413 414 static void nested_ept_uninit_mmu_context(struct kvm_vcpu *vcpu) 415 { 416 vcpu->arch.mmu = &vcpu->arch.root_mmu; 417 vcpu->arch.walk_mmu = &vcpu->arch.root_mmu; 418 } 419 420 static bool nested_vmx_is_page_fault_vmexit(struct vmcs12 *vmcs12, 421 u16 error_code) 422 { 423 bool inequality, bit; 424 425 bit = (vmcs12->exception_bitmap & (1u << PF_VECTOR)) != 0; 426 inequality = 427 (error_code & vmcs12->page_fault_error_code_mask) != 428 vmcs12->page_fault_error_code_match; 429 return inequality ^ bit; 430 } 431 432 433 /* 434 * KVM wants to inject page-faults which it got to the guest. This function 435 * checks whether in a nested guest, we need to inject them to L1 or L2. 436 */ 437 static int nested_vmx_check_exception(struct kvm_vcpu *vcpu, unsigned long *exit_qual) 438 { 439 struct vmcs12 *vmcs12 = get_vmcs12(vcpu); 440 unsigned int nr = vcpu->arch.exception.nr; 441 bool has_payload = vcpu->arch.exception.has_payload; 442 unsigned long payload = vcpu->arch.exception.payload; 443 444 if (nr == PF_VECTOR) { 445 if (vcpu->arch.exception.nested_apf) { 446 *exit_qual = vcpu->arch.apf.nested_apf_token; 447 return 1; 448 } 449 if (nested_vmx_is_page_fault_vmexit(vmcs12, 450 vcpu->arch.exception.error_code)) { 451 *exit_qual = has_payload ? payload : vcpu->arch.cr2; 452 return 1; 453 } 454 } else if (vmcs12->exception_bitmap & (1u << nr)) { 455 if (nr == DB_VECTOR) { 456 if (!has_payload) { 457 payload = vcpu->arch.dr6; 458 payload &= ~DR6_BT; 459 payload ^= DR6_ACTIVE_LOW; 460 } 461 *exit_qual = payload; 462 } else 463 *exit_qual = 0; 464 return 1; 465 } 466 467 return 0; 468 } 469 470 471 static void vmx_inject_page_fault_nested(struct kvm_vcpu *vcpu, 472 struct x86_exception *fault) 473 { 474 struct vmcs12 *vmcs12 = get_vmcs12(vcpu); 475 476 WARN_ON(!is_guest_mode(vcpu)); 477 478 if (nested_vmx_is_page_fault_vmexit(vmcs12, fault->error_code) && 479 !to_vmx(vcpu)->nested.nested_run_pending) { 480 vmcs12->vm_exit_intr_error_code = fault->error_code; 481 nested_vmx_vmexit(vcpu, EXIT_REASON_EXCEPTION_NMI, 482 PF_VECTOR | INTR_TYPE_HARD_EXCEPTION | 483 INTR_INFO_DELIVER_CODE_MASK | INTR_INFO_VALID_MASK, 484 fault->address); 485 } else { 486 kvm_inject_page_fault(vcpu, fault); 487 } 488 } 489 490 static int nested_vmx_check_io_bitmap_controls(struct kvm_vcpu *vcpu, 491 struct vmcs12 *vmcs12) 492 { 493 if (!nested_cpu_has(vmcs12, CPU_BASED_USE_IO_BITMAPS)) 494 return 0; 495 496 if (CC(!page_address_valid(vcpu, vmcs12->io_bitmap_a)) || 497 CC(!page_address_valid(vcpu, vmcs12->io_bitmap_b))) 498 return -EINVAL; 499 500 return 0; 501 } 502 503 static int nested_vmx_check_msr_bitmap_controls(struct kvm_vcpu *vcpu, 504 struct vmcs12 *vmcs12) 505 { 506 if (!nested_cpu_has(vmcs12, CPU_BASED_USE_MSR_BITMAPS)) 507 return 0; 508 509 if (CC(!page_address_valid(vcpu, vmcs12->msr_bitmap))) 510 return -EINVAL; 511 512 return 0; 513 } 514 515 static int nested_vmx_check_tpr_shadow_controls(struct kvm_vcpu *vcpu, 516 struct vmcs12 *vmcs12) 517 { 518 if (!nested_cpu_has(vmcs12, CPU_BASED_TPR_SHADOW)) 519 return 0; 520 521 if (CC(!page_address_valid(vcpu, vmcs12->virtual_apic_page_addr))) 522 return -EINVAL; 523 524 return 0; 525 } 526 527 /* 528 * For x2APIC MSRs, ignore the vmcs01 bitmap. L1 can enable x2APIC without L1 529 * itself utilizing x2APIC. All MSRs were previously set to be intercepted, 530 * only the "disable intercept" case needs to be handled. 531 */ 532 static void nested_vmx_disable_intercept_for_x2apic_msr(unsigned long *msr_bitmap_l1, 533 unsigned long *msr_bitmap_l0, 534 u32 msr, int type) 535 { 536 if (type & MSR_TYPE_R && !vmx_test_msr_bitmap_read(msr_bitmap_l1, msr)) 537 vmx_clear_msr_bitmap_read(msr_bitmap_l0, msr); 538 539 if (type & MSR_TYPE_W && !vmx_test_msr_bitmap_write(msr_bitmap_l1, msr)) 540 vmx_clear_msr_bitmap_write(msr_bitmap_l0, msr); 541 } 542 543 static inline void enable_x2apic_msr_intercepts(unsigned long *msr_bitmap) 544 { 545 int msr; 546 547 for (msr = 0x800; msr <= 0x8ff; msr += BITS_PER_LONG) { 548 unsigned word = msr / BITS_PER_LONG; 549 550 msr_bitmap[word] = ~0; 551 msr_bitmap[word + (0x800 / sizeof(long))] = ~0; 552 } 553 } 554 555 #define BUILD_NVMX_MSR_INTERCEPT_HELPER(rw) \ 556 static inline \ 557 void nested_vmx_set_msr_##rw##_intercept(struct vcpu_vmx *vmx, \ 558 unsigned long *msr_bitmap_l1, \ 559 unsigned long *msr_bitmap_l0, u32 msr) \ 560 { \ 561 if (vmx_test_msr_bitmap_##rw(vmx->vmcs01.msr_bitmap, msr) || \ 562 vmx_test_msr_bitmap_##rw(msr_bitmap_l1, msr)) \ 563 vmx_set_msr_bitmap_##rw(msr_bitmap_l0, msr); \ 564 else \ 565 vmx_clear_msr_bitmap_##rw(msr_bitmap_l0, msr); \ 566 } 567 BUILD_NVMX_MSR_INTERCEPT_HELPER(read) 568 BUILD_NVMX_MSR_INTERCEPT_HELPER(write) 569 570 static inline void nested_vmx_set_intercept_for_msr(struct vcpu_vmx *vmx, 571 unsigned long *msr_bitmap_l1, 572 unsigned long *msr_bitmap_l0, 573 u32 msr, int types) 574 { 575 if (types & MSR_TYPE_R) 576 nested_vmx_set_msr_read_intercept(vmx, msr_bitmap_l1, 577 msr_bitmap_l0, msr); 578 if (types & MSR_TYPE_W) 579 nested_vmx_set_msr_write_intercept(vmx, msr_bitmap_l1, 580 msr_bitmap_l0, msr); 581 } 582 583 /* 584 * Merge L0's and L1's MSR bitmap, return false to indicate that 585 * we do not use the hardware. 586 */ 587 static inline bool nested_vmx_prepare_msr_bitmap(struct kvm_vcpu *vcpu, 588 struct vmcs12 *vmcs12) 589 { 590 struct vcpu_vmx *vmx = to_vmx(vcpu); 591 int msr; 592 unsigned long *msr_bitmap_l1; 593 unsigned long *msr_bitmap_l0 = vmx->nested.vmcs02.msr_bitmap; 594 struct kvm_host_map *map = &vmx->nested.msr_bitmap_map; 595 596 /* Nothing to do if the MSR bitmap is not in use. */ 597 if (!cpu_has_vmx_msr_bitmap() || 598 !nested_cpu_has(vmcs12, CPU_BASED_USE_MSR_BITMAPS)) 599 return false; 600 601 if (kvm_vcpu_map(vcpu, gpa_to_gfn(vmcs12->msr_bitmap), map)) 602 return false; 603 604 msr_bitmap_l1 = (unsigned long *)map->hva; 605 606 /* 607 * To keep the control flow simple, pay eight 8-byte writes (sixteen 608 * 4-byte writes on 32-bit systems) up front to enable intercepts for 609 * the x2APIC MSR range and selectively toggle those relevant to L2. 610 */ 611 enable_x2apic_msr_intercepts(msr_bitmap_l0); 612 613 if (nested_cpu_has_virt_x2apic_mode(vmcs12)) { 614 if (nested_cpu_has_apic_reg_virt(vmcs12)) { 615 /* 616 * L0 need not intercept reads for MSRs between 0x800 617 * and 0x8ff, it just lets the processor take the value 618 * from the virtual-APIC page; take those 256 bits 619 * directly from the L1 bitmap. 620 */ 621 for (msr = 0x800; msr <= 0x8ff; msr += BITS_PER_LONG) { 622 unsigned word = msr / BITS_PER_LONG; 623 624 msr_bitmap_l0[word] = msr_bitmap_l1[word]; 625 } 626 } 627 628 nested_vmx_disable_intercept_for_x2apic_msr( 629 msr_bitmap_l1, msr_bitmap_l0, 630 X2APIC_MSR(APIC_TASKPRI), 631 MSR_TYPE_R | MSR_TYPE_W); 632 633 if (nested_cpu_has_vid(vmcs12)) { 634 nested_vmx_disable_intercept_for_x2apic_msr( 635 msr_bitmap_l1, msr_bitmap_l0, 636 X2APIC_MSR(APIC_EOI), 637 MSR_TYPE_W); 638 nested_vmx_disable_intercept_for_x2apic_msr( 639 msr_bitmap_l1, msr_bitmap_l0, 640 X2APIC_MSR(APIC_SELF_IPI), 641 MSR_TYPE_W); 642 } 643 } 644 645 /* 646 * Always check vmcs01's bitmap to honor userspace MSR filters and any 647 * other runtime changes to vmcs01's bitmap, e.g. dynamic pass-through. 648 */ 649 #ifdef CONFIG_X86_64 650 nested_vmx_set_intercept_for_msr(vmx, msr_bitmap_l1, msr_bitmap_l0, 651 MSR_FS_BASE, MSR_TYPE_RW); 652 653 nested_vmx_set_intercept_for_msr(vmx, msr_bitmap_l1, msr_bitmap_l0, 654 MSR_GS_BASE, MSR_TYPE_RW); 655 656 nested_vmx_set_intercept_for_msr(vmx, msr_bitmap_l1, msr_bitmap_l0, 657 MSR_KERNEL_GS_BASE, MSR_TYPE_RW); 658 #endif 659 nested_vmx_set_intercept_for_msr(vmx, msr_bitmap_l1, msr_bitmap_l0, 660 MSR_IA32_SPEC_CTRL, MSR_TYPE_RW); 661 662 nested_vmx_set_intercept_for_msr(vmx, msr_bitmap_l1, msr_bitmap_l0, 663 MSR_IA32_PRED_CMD, MSR_TYPE_W); 664 665 kvm_vcpu_unmap(vcpu, &vmx->nested.msr_bitmap_map, false); 666 667 return true; 668 } 669 670 static void nested_cache_shadow_vmcs12(struct kvm_vcpu *vcpu, 671 struct vmcs12 *vmcs12) 672 { 673 struct vcpu_vmx *vmx = to_vmx(vcpu); 674 struct gfn_to_hva_cache *ghc = &vmx->nested.shadow_vmcs12_cache; 675 676 if (!nested_cpu_has_shadow_vmcs(vmcs12) || 677 vmcs12->vmcs_link_pointer == INVALID_GPA) 678 return; 679 680 if (ghc->gpa != vmcs12->vmcs_link_pointer && 681 kvm_gfn_to_hva_cache_init(vcpu->kvm, ghc, 682 vmcs12->vmcs_link_pointer, VMCS12_SIZE)) 683 return; 684 685 kvm_read_guest_cached(vmx->vcpu.kvm, ghc, get_shadow_vmcs12(vcpu), 686 VMCS12_SIZE); 687 } 688 689 static void nested_flush_cached_shadow_vmcs12(struct kvm_vcpu *vcpu, 690 struct vmcs12 *vmcs12) 691 { 692 struct vcpu_vmx *vmx = to_vmx(vcpu); 693 struct gfn_to_hva_cache *ghc = &vmx->nested.shadow_vmcs12_cache; 694 695 if (!nested_cpu_has_shadow_vmcs(vmcs12) || 696 vmcs12->vmcs_link_pointer == INVALID_GPA) 697 return; 698 699 if (ghc->gpa != vmcs12->vmcs_link_pointer && 700 kvm_gfn_to_hva_cache_init(vcpu->kvm, ghc, 701 vmcs12->vmcs_link_pointer, VMCS12_SIZE)) 702 return; 703 704 kvm_write_guest_cached(vmx->vcpu.kvm, ghc, get_shadow_vmcs12(vcpu), 705 VMCS12_SIZE); 706 } 707 708 /* 709 * In nested virtualization, check if L1 has set 710 * VM_EXIT_ACK_INTR_ON_EXIT 711 */ 712 static bool nested_exit_intr_ack_set(struct kvm_vcpu *vcpu) 713 { 714 return get_vmcs12(vcpu)->vm_exit_controls & 715 VM_EXIT_ACK_INTR_ON_EXIT; 716 } 717 718 static int nested_vmx_check_apic_access_controls(struct kvm_vcpu *vcpu, 719 struct vmcs12 *vmcs12) 720 { 721 if (nested_cpu_has2(vmcs12, SECONDARY_EXEC_VIRTUALIZE_APIC_ACCESSES) && 722 CC(!page_address_valid(vcpu, vmcs12->apic_access_addr))) 723 return -EINVAL; 724 else 725 return 0; 726 } 727 728 static int nested_vmx_check_apicv_controls(struct kvm_vcpu *vcpu, 729 struct vmcs12 *vmcs12) 730 { 731 if (!nested_cpu_has_virt_x2apic_mode(vmcs12) && 732 !nested_cpu_has_apic_reg_virt(vmcs12) && 733 !nested_cpu_has_vid(vmcs12) && 734 !nested_cpu_has_posted_intr(vmcs12)) 735 return 0; 736 737 /* 738 * If virtualize x2apic mode is enabled, 739 * virtualize apic access must be disabled. 740 */ 741 if (CC(nested_cpu_has_virt_x2apic_mode(vmcs12) && 742 nested_cpu_has2(vmcs12, SECONDARY_EXEC_VIRTUALIZE_APIC_ACCESSES))) 743 return -EINVAL; 744 745 /* 746 * If virtual interrupt delivery is enabled, 747 * we must exit on external interrupts. 748 */ 749 if (CC(nested_cpu_has_vid(vmcs12) && !nested_exit_on_intr(vcpu))) 750 return -EINVAL; 751 752 /* 753 * bits 15:8 should be zero in posted_intr_nv, 754 * the descriptor address has been already checked 755 * in nested_get_vmcs12_pages. 756 * 757 * bits 5:0 of posted_intr_desc_addr should be zero. 758 */ 759 if (nested_cpu_has_posted_intr(vmcs12) && 760 (CC(!nested_cpu_has_vid(vmcs12)) || 761 CC(!nested_exit_intr_ack_set(vcpu)) || 762 CC((vmcs12->posted_intr_nv & 0xff00)) || 763 CC(!kvm_vcpu_is_legal_aligned_gpa(vcpu, vmcs12->posted_intr_desc_addr, 64)))) 764 return -EINVAL; 765 766 /* tpr shadow is needed by all apicv features. */ 767 if (CC(!nested_cpu_has(vmcs12, CPU_BASED_TPR_SHADOW))) 768 return -EINVAL; 769 770 return 0; 771 } 772 773 static int nested_vmx_check_msr_switch(struct kvm_vcpu *vcpu, 774 u32 count, u64 addr) 775 { 776 if (count == 0) 777 return 0; 778 779 if (!kvm_vcpu_is_legal_aligned_gpa(vcpu, addr, 16) || 780 !kvm_vcpu_is_legal_gpa(vcpu, (addr + count * sizeof(struct vmx_msr_entry) - 1))) 781 return -EINVAL; 782 783 return 0; 784 } 785 786 static int nested_vmx_check_exit_msr_switch_controls(struct kvm_vcpu *vcpu, 787 struct vmcs12 *vmcs12) 788 { 789 if (CC(nested_vmx_check_msr_switch(vcpu, 790 vmcs12->vm_exit_msr_load_count, 791 vmcs12->vm_exit_msr_load_addr)) || 792 CC(nested_vmx_check_msr_switch(vcpu, 793 vmcs12->vm_exit_msr_store_count, 794 vmcs12->vm_exit_msr_store_addr))) 795 return -EINVAL; 796 797 return 0; 798 } 799 800 static int nested_vmx_check_entry_msr_switch_controls(struct kvm_vcpu *vcpu, 801 struct vmcs12 *vmcs12) 802 { 803 if (CC(nested_vmx_check_msr_switch(vcpu, 804 vmcs12->vm_entry_msr_load_count, 805 vmcs12->vm_entry_msr_load_addr))) 806 return -EINVAL; 807 808 return 0; 809 } 810 811 static int nested_vmx_check_pml_controls(struct kvm_vcpu *vcpu, 812 struct vmcs12 *vmcs12) 813 { 814 if (!nested_cpu_has_pml(vmcs12)) 815 return 0; 816 817 if (CC(!nested_cpu_has_ept(vmcs12)) || 818 CC(!page_address_valid(vcpu, vmcs12->pml_address))) 819 return -EINVAL; 820 821 return 0; 822 } 823 824 static int nested_vmx_check_unrestricted_guest_controls(struct kvm_vcpu *vcpu, 825 struct vmcs12 *vmcs12) 826 { 827 if (CC(nested_cpu_has2(vmcs12, SECONDARY_EXEC_UNRESTRICTED_GUEST) && 828 !nested_cpu_has_ept(vmcs12))) 829 return -EINVAL; 830 return 0; 831 } 832 833 static int nested_vmx_check_mode_based_ept_exec_controls(struct kvm_vcpu *vcpu, 834 struct vmcs12 *vmcs12) 835 { 836 if (CC(nested_cpu_has2(vmcs12, SECONDARY_EXEC_MODE_BASED_EPT_EXEC) && 837 !nested_cpu_has_ept(vmcs12))) 838 return -EINVAL; 839 return 0; 840 } 841 842 static int nested_vmx_check_shadow_vmcs_controls(struct kvm_vcpu *vcpu, 843 struct vmcs12 *vmcs12) 844 { 845 if (!nested_cpu_has_shadow_vmcs(vmcs12)) 846 return 0; 847 848 if (CC(!page_address_valid(vcpu, vmcs12->vmread_bitmap)) || 849 CC(!page_address_valid(vcpu, vmcs12->vmwrite_bitmap))) 850 return -EINVAL; 851 852 return 0; 853 } 854 855 static int nested_vmx_msr_check_common(struct kvm_vcpu *vcpu, 856 struct vmx_msr_entry *e) 857 { 858 /* x2APIC MSR accesses are not allowed */ 859 if (CC(vcpu->arch.apic_base & X2APIC_ENABLE && e->index >> 8 == 0x8)) 860 return -EINVAL; 861 if (CC(e->index == MSR_IA32_UCODE_WRITE) || /* SDM Table 35-2 */ 862 CC(e->index == MSR_IA32_UCODE_REV)) 863 return -EINVAL; 864 if (CC(e->reserved != 0)) 865 return -EINVAL; 866 return 0; 867 } 868 869 static int nested_vmx_load_msr_check(struct kvm_vcpu *vcpu, 870 struct vmx_msr_entry *e) 871 { 872 if (CC(e->index == MSR_FS_BASE) || 873 CC(e->index == MSR_GS_BASE) || 874 CC(e->index == MSR_IA32_SMM_MONITOR_CTL) || /* SMM is not supported */ 875 nested_vmx_msr_check_common(vcpu, e)) 876 return -EINVAL; 877 return 0; 878 } 879 880 static int nested_vmx_store_msr_check(struct kvm_vcpu *vcpu, 881 struct vmx_msr_entry *e) 882 { 883 if (CC(e->index == MSR_IA32_SMBASE) || /* SMM is not supported */ 884 nested_vmx_msr_check_common(vcpu, e)) 885 return -EINVAL; 886 return 0; 887 } 888 889 static u32 nested_vmx_max_atomic_switch_msrs(struct kvm_vcpu *vcpu) 890 { 891 struct vcpu_vmx *vmx = to_vmx(vcpu); 892 u64 vmx_misc = vmx_control_msr(vmx->nested.msrs.misc_low, 893 vmx->nested.msrs.misc_high); 894 895 return (vmx_misc_max_msr(vmx_misc) + 1) * VMX_MISC_MSR_LIST_MULTIPLIER; 896 } 897 898 /* 899 * Load guest's/host's msr at nested entry/exit. 900 * return 0 for success, entry index for failure. 901 * 902 * One of the failure modes for MSR load/store is when a list exceeds the 903 * virtual hardware's capacity. To maintain compatibility with hardware inasmuch 904 * as possible, process all valid entries before failing rather than precheck 905 * for a capacity violation. 906 */ 907 static u32 nested_vmx_load_msr(struct kvm_vcpu *vcpu, u64 gpa, u32 count) 908 { 909 u32 i; 910 struct vmx_msr_entry e; 911 u32 max_msr_list_size = nested_vmx_max_atomic_switch_msrs(vcpu); 912 913 for (i = 0; i < count; i++) { 914 if (unlikely(i >= max_msr_list_size)) 915 goto fail; 916 917 if (kvm_vcpu_read_guest(vcpu, gpa + i * sizeof(e), 918 &e, sizeof(e))) { 919 pr_debug_ratelimited( 920 "%s cannot read MSR entry (%u, 0x%08llx)\n", 921 __func__, i, gpa + i * sizeof(e)); 922 goto fail; 923 } 924 if (nested_vmx_load_msr_check(vcpu, &e)) { 925 pr_debug_ratelimited( 926 "%s check failed (%u, 0x%x, 0x%x)\n", 927 __func__, i, e.index, e.reserved); 928 goto fail; 929 } 930 if (kvm_set_msr(vcpu, e.index, e.value)) { 931 pr_debug_ratelimited( 932 "%s cannot write MSR (%u, 0x%x, 0x%llx)\n", 933 __func__, i, e.index, e.value); 934 goto fail; 935 } 936 } 937 return 0; 938 fail: 939 /* Note, max_msr_list_size is at most 4096, i.e. this can't wrap. */ 940 return i + 1; 941 } 942 943 static bool nested_vmx_get_vmexit_msr_value(struct kvm_vcpu *vcpu, 944 u32 msr_index, 945 u64 *data) 946 { 947 struct vcpu_vmx *vmx = to_vmx(vcpu); 948 949 /* 950 * If the L0 hypervisor stored a more accurate value for the TSC that 951 * does not include the time taken for emulation of the L2->L1 952 * VM-exit in L0, use the more accurate value. 953 */ 954 if (msr_index == MSR_IA32_TSC) { 955 int i = vmx_find_loadstore_msr_slot(&vmx->msr_autostore.guest, 956 MSR_IA32_TSC); 957 958 if (i >= 0) { 959 u64 val = vmx->msr_autostore.guest.val[i].value; 960 961 *data = kvm_read_l1_tsc(vcpu, val); 962 return true; 963 } 964 } 965 966 if (kvm_get_msr(vcpu, msr_index, data)) { 967 pr_debug_ratelimited("%s cannot read MSR (0x%x)\n", __func__, 968 msr_index); 969 return false; 970 } 971 return true; 972 } 973 974 static bool read_and_check_msr_entry(struct kvm_vcpu *vcpu, u64 gpa, int i, 975 struct vmx_msr_entry *e) 976 { 977 if (kvm_vcpu_read_guest(vcpu, 978 gpa + i * sizeof(*e), 979 e, 2 * sizeof(u32))) { 980 pr_debug_ratelimited( 981 "%s cannot read MSR entry (%u, 0x%08llx)\n", 982 __func__, i, gpa + i * sizeof(*e)); 983 return false; 984 } 985 if (nested_vmx_store_msr_check(vcpu, e)) { 986 pr_debug_ratelimited( 987 "%s check failed (%u, 0x%x, 0x%x)\n", 988 __func__, i, e->index, e->reserved); 989 return false; 990 } 991 return true; 992 } 993 994 static int nested_vmx_store_msr(struct kvm_vcpu *vcpu, u64 gpa, u32 count) 995 { 996 u64 data; 997 u32 i; 998 struct vmx_msr_entry e; 999 u32 max_msr_list_size = nested_vmx_max_atomic_switch_msrs(vcpu); 1000 1001 for (i = 0; i < count; i++) { 1002 if (unlikely(i >= max_msr_list_size)) 1003 return -EINVAL; 1004 1005 if (!read_and_check_msr_entry(vcpu, gpa, i, &e)) 1006 return -EINVAL; 1007 1008 if (!nested_vmx_get_vmexit_msr_value(vcpu, e.index, &data)) 1009 return -EINVAL; 1010 1011 if (kvm_vcpu_write_guest(vcpu, 1012 gpa + i * sizeof(e) + 1013 offsetof(struct vmx_msr_entry, value), 1014 &data, sizeof(data))) { 1015 pr_debug_ratelimited( 1016 "%s cannot write MSR (%u, 0x%x, 0x%llx)\n", 1017 __func__, i, e.index, data); 1018 return -EINVAL; 1019 } 1020 } 1021 return 0; 1022 } 1023 1024 static bool nested_msr_store_list_has_msr(struct kvm_vcpu *vcpu, u32 msr_index) 1025 { 1026 struct vmcs12 *vmcs12 = get_vmcs12(vcpu); 1027 u32 count = vmcs12->vm_exit_msr_store_count; 1028 u64 gpa = vmcs12->vm_exit_msr_store_addr; 1029 struct vmx_msr_entry e; 1030 u32 i; 1031 1032 for (i = 0; i < count; i++) { 1033 if (!read_and_check_msr_entry(vcpu, gpa, i, &e)) 1034 return false; 1035 1036 if (e.index == msr_index) 1037 return true; 1038 } 1039 return false; 1040 } 1041 1042 static void prepare_vmx_msr_autostore_list(struct kvm_vcpu *vcpu, 1043 u32 msr_index) 1044 { 1045 struct vcpu_vmx *vmx = to_vmx(vcpu); 1046 struct vmx_msrs *autostore = &vmx->msr_autostore.guest; 1047 bool in_vmcs12_store_list; 1048 int msr_autostore_slot; 1049 bool in_autostore_list; 1050 int last; 1051 1052 msr_autostore_slot = vmx_find_loadstore_msr_slot(autostore, msr_index); 1053 in_autostore_list = msr_autostore_slot >= 0; 1054 in_vmcs12_store_list = nested_msr_store_list_has_msr(vcpu, msr_index); 1055 1056 if (in_vmcs12_store_list && !in_autostore_list) { 1057 if (autostore->nr == MAX_NR_LOADSTORE_MSRS) { 1058 /* 1059 * Emulated VMEntry does not fail here. Instead a less 1060 * accurate value will be returned by 1061 * nested_vmx_get_vmexit_msr_value() using kvm_get_msr() 1062 * instead of reading the value from the vmcs02 VMExit 1063 * MSR-store area. 1064 */ 1065 pr_warn_ratelimited( 1066 "Not enough msr entries in msr_autostore. Can't add msr %x\n", 1067 msr_index); 1068 return; 1069 } 1070 last = autostore->nr++; 1071 autostore->val[last].index = msr_index; 1072 } else if (!in_vmcs12_store_list && in_autostore_list) { 1073 last = --autostore->nr; 1074 autostore->val[msr_autostore_slot] = autostore->val[last]; 1075 } 1076 } 1077 1078 /* 1079 * Load guest's/host's cr3 at nested entry/exit. @nested_ept is true if we are 1080 * emulating VM-Entry into a guest with EPT enabled. On failure, the expected 1081 * Exit Qualification (for a VM-Entry consistency check VM-Exit) is assigned to 1082 * @entry_failure_code. 1083 */ 1084 static int nested_vmx_load_cr3(struct kvm_vcpu *vcpu, unsigned long cr3, 1085 bool nested_ept, bool reload_pdptrs, 1086 enum vm_entry_failure_code *entry_failure_code) 1087 { 1088 if (CC(kvm_vcpu_is_illegal_gpa(vcpu, cr3))) { 1089 *entry_failure_code = ENTRY_FAIL_DEFAULT; 1090 return -EINVAL; 1091 } 1092 1093 /* 1094 * If PAE paging and EPT are both on, CR3 is not used by the CPU and 1095 * must not be dereferenced. 1096 */ 1097 if (reload_pdptrs && !nested_ept && is_pae_paging(vcpu) && 1098 CC(!load_pdptrs(vcpu, vcpu->arch.walk_mmu, cr3))) { 1099 *entry_failure_code = ENTRY_FAIL_PDPTE; 1100 return -EINVAL; 1101 } 1102 1103 if (!nested_ept) 1104 kvm_mmu_new_pgd(vcpu, cr3); 1105 1106 vcpu->arch.cr3 = cr3; 1107 kvm_register_mark_available(vcpu, VCPU_EXREG_CR3); 1108 1109 /* Re-initialize the MMU, e.g. to pick up CR4 MMU role changes. */ 1110 kvm_init_mmu(vcpu); 1111 1112 return 0; 1113 } 1114 1115 /* 1116 * Returns if KVM is able to config CPU to tag TLB entries 1117 * populated by L2 differently than TLB entries populated 1118 * by L1. 1119 * 1120 * If L0 uses EPT, L1 and L2 run with different EPTP because 1121 * guest_mode is part of kvm_mmu_page_role. Thus, TLB entries 1122 * are tagged with different EPTP. 1123 * 1124 * If L1 uses VPID and we allocated a vpid02, TLB entries are tagged 1125 * with different VPID (L1 entries are tagged with vmx->vpid 1126 * while L2 entries are tagged with vmx->nested.vpid02). 1127 */ 1128 static bool nested_has_guest_tlb_tag(struct kvm_vcpu *vcpu) 1129 { 1130 struct vmcs12 *vmcs12 = get_vmcs12(vcpu); 1131 1132 return enable_ept || 1133 (nested_cpu_has_vpid(vmcs12) && to_vmx(vcpu)->nested.vpid02); 1134 } 1135 1136 static void nested_vmx_transition_tlb_flush(struct kvm_vcpu *vcpu, 1137 struct vmcs12 *vmcs12, 1138 bool is_vmenter) 1139 { 1140 struct vcpu_vmx *vmx = to_vmx(vcpu); 1141 1142 /* 1143 * If vmcs12 doesn't use VPID, L1 expects linear and combined mappings 1144 * for *all* contexts to be flushed on VM-Enter/VM-Exit, i.e. it's a 1145 * full TLB flush from the guest's perspective. This is required even 1146 * if VPID is disabled in the host as KVM may need to synchronize the 1147 * MMU in response to the guest TLB flush. 1148 * 1149 * Note, using TLB_FLUSH_GUEST is correct even if nested EPT is in use. 1150 * EPT is a special snowflake, as guest-physical mappings aren't 1151 * flushed on VPID invalidations, including VM-Enter or VM-Exit with 1152 * VPID disabled. As a result, KVM _never_ needs to sync nEPT 1153 * entries on VM-Enter because L1 can't rely on VM-Enter to flush 1154 * those mappings. 1155 */ 1156 if (!nested_cpu_has_vpid(vmcs12)) { 1157 kvm_make_request(KVM_REQ_TLB_FLUSH_GUEST, vcpu); 1158 return; 1159 } 1160 1161 /* L2 should never have a VPID if VPID is disabled. */ 1162 WARN_ON(!enable_vpid); 1163 1164 /* 1165 * If VPID is enabled and used by vmc12, but L2 does not have a unique 1166 * TLB tag (ASID), i.e. EPT is disabled and KVM was unable to allocate 1167 * a VPID for L2, flush the current context as the effective ASID is 1168 * common to both L1 and L2. 1169 * 1170 * Defer the flush so that it runs after vmcs02.EPTP has been set by 1171 * KVM_REQ_LOAD_MMU_PGD (if nested EPT is enabled) and to avoid 1172 * redundant flushes further down the nested pipeline. 1173 * 1174 * If a TLB flush isn't required due to any of the above, and vpid12 is 1175 * changing then the new "virtual" VPID (vpid12) will reuse the same 1176 * "real" VPID (vpid02), and so needs to be flushed. There's no direct 1177 * mapping between vpid02 and vpid12, vpid02 is per-vCPU and reused for 1178 * all nested vCPUs. Remember, a flush on VM-Enter does not invalidate 1179 * guest-physical mappings, so there is no need to sync the nEPT MMU. 1180 */ 1181 if (!nested_has_guest_tlb_tag(vcpu)) { 1182 kvm_make_request(KVM_REQ_TLB_FLUSH_CURRENT, vcpu); 1183 } else if (is_vmenter && 1184 vmcs12->virtual_processor_id != vmx->nested.last_vpid) { 1185 vmx->nested.last_vpid = vmcs12->virtual_processor_id; 1186 vpid_sync_context(nested_get_vpid02(vcpu)); 1187 } 1188 } 1189 1190 static bool is_bitwise_subset(u64 superset, u64 subset, u64 mask) 1191 { 1192 superset &= mask; 1193 subset &= mask; 1194 1195 return (superset | subset) == superset; 1196 } 1197 1198 static int vmx_restore_vmx_basic(struct vcpu_vmx *vmx, u64 data) 1199 { 1200 const u64 feature_and_reserved = 1201 /* feature (except bit 48; see below) */ 1202 BIT_ULL(49) | BIT_ULL(54) | BIT_ULL(55) | 1203 /* reserved */ 1204 BIT_ULL(31) | GENMASK_ULL(47, 45) | GENMASK_ULL(63, 56); 1205 u64 vmx_basic = vmx->nested.msrs.basic; 1206 1207 if (!is_bitwise_subset(vmx_basic, data, feature_and_reserved)) 1208 return -EINVAL; 1209 1210 /* 1211 * KVM does not emulate a version of VMX that constrains physical 1212 * addresses of VMX structures (e.g. VMCS) to 32-bits. 1213 */ 1214 if (data & BIT_ULL(48)) 1215 return -EINVAL; 1216 1217 if (vmx_basic_vmcs_revision_id(vmx_basic) != 1218 vmx_basic_vmcs_revision_id(data)) 1219 return -EINVAL; 1220 1221 if (vmx_basic_vmcs_size(vmx_basic) > vmx_basic_vmcs_size(data)) 1222 return -EINVAL; 1223 1224 vmx->nested.msrs.basic = data; 1225 return 0; 1226 } 1227 1228 static int 1229 vmx_restore_control_msr(struct vcpu_vmx *vmx, u32 msr_index, u64 data) 1230 { 1231 u64 supported; 1232 u32 *lowp, *highp; 1233 1234 switch (msr_index) { 1235 case MSR_IA32_VMX_TRUE_PINBASED_CTLS: 1236 lowp = &vmx->nested.msrs.pinbased_ctls_low; 1237 highp = &vmx->nested.msrs.pinbased_ctls_high; 1238 break; 1239 case MSR_IA32_VMX_TRUE_PROCBASED_CTLS: 1240 lowp = &vmx->nested.msrs.procbased_ctls_low; 1241 highp = &vmx->nested.msrs.procbased_ctls_high; 1242 break; 1243 case MSR_IA32_VMX_TRUE_EXIT_CTLS: 1244 lowp = &vmx->nested.msrs.exit_ctls_low; 1245 highp = &vmx->nested.msrs.exit_ctls_high; 1246 break; 1247 case MSR_IA32_VMX_TRUE_ENTRY_CTLS: 1248 lowp = &vmx->nested.msrs.entry_ctls_low; 1249 highp = &vmx->nested.msrs.entry_ctls_high; 1250 break; 1251 case MSR_IA32_VMX_PROCBASED_CTLS2: 1252 lowp = &vmx->nested.msrs.secondary_ctls_low; 1253 highp = &vmx->nested.msrs.secondary_ctls_high; 1254 break; 1255 default: 1256 BUG(); 1257 } 1258 1259 supported = vmx_control_msr(*lowp, *highp); 1260 1261 /* Check must-be-1 bits are still 1. */ 1262 if (!is_bitwise_subset(data, supported, GENMASK_ULL(31, 0))) 1263 return -EINVAL; 1264 1265 /* Check must-be-0 bits are still 0. */ 1266 if (!is_bitwise_subset(supported, data, GENMASK_ULL(63, 32))) 1267 return -EINVAL; 1268 1269 *lowp = data; 1270 *highp = data >> 32; 1271 return 0; 1272 } 1273 1274 static int vmx_restore_vmx_misc(struct vcpu_vmx *vmx, u64 data) 1275 { 1276 const u64 feature_and_reserved_bits = 1277 /* feature */ 1278 BIT_ULL(5) | GENMASK_ULL(8, 6) | BIT_ULL(14) | BIT_ULL(15) | 1279 BIT_ULL(28) | BIT_ULL(29) | BIT_ULL(30) | 1280 /* reserved */ 1281 GENMASK_ULL(13, 9) | BIT_ULL(31); 1282 u64 vmx_misc; 1283 1284 vmx_misc = vmx_control_msr(vmx->nested.msrs.misc_low, 1285 vmx->nested.msrs.misc_high); 1286 1287 if (!is_bitwise_subset(vmx_misc, data, feature_and_reserved_bits)) 1288 return -EINVAL; 1289 1290 if ((vmx->nested.msrs.pinbased_ctls_high & 1291 PIN_BASED_VMX_PREEMPTION_TIMER) && 1292 vmx_misc_preemption_timer_rate(data) != 1293 vmx_misc_preemption_timer_rate(vmx_misc)) 1294 return -EINVAL; 1295 1296 if (vmx_misc_cr3_count(data) > vmx_misc_cr3_count(vmx_misc)) 1297 return -EINVAL; 1298 1299 if (vmx_misc_max_msr(data) > vmx_misc_max_msr(vmx_misc)) 1300 return -EINVAL; 1301 1302 if (vmx_misc_mseg_revid(data) != vmx_misc_mseg_revid(vmx_misc)) 1303 return -EINVAL; 1304 1305 vmx->nested.msrs.misc_low = data; 1306 vmx->nested.msrs.misc_high = data >> 32; 1307 1308 return 0; 1309 } 1310 1311 static int vmx_restore_vmx_ept_vpid_cap(struct vcpu_vmx *vmx, u64 data) 1312 { 1313 u64 vmx_ept_vpid_cap; 1314 1315 vmx_ept_vpid_cap = vmx_control_msr(vmx->nested.msrs.ept_caps, 1316 vmx->nested.msrs.vpid_caps); 1317 1318 /* Every bit is either reserved or a feature bit. */ 1319 if (!is_bitwise_subset(vmx_ept_vpid_cap, data, -1ULL)) 1320 return -EINVAL; 1321 1322 vmx->nested.msrs.ept_caps = data; 1323 vmx->nested.msrs.vpid_caps = data >> 32; 1324 return 0; 1325 } 1326 1327 static int vmx_restore_fixed0_msr(struct vcpu_vmx *vmx, u32 msr_index, u64 data) 1328 { 1329 u64 *msr; 1330 1331 switch (msr_index) { 1332 case MSR_IA32_VMX_CR0_FIXED0: 1333 msr = &vmx->nested.msrs.cr0_fixed0; 1334 break; 1335 case MSR_IA32_VMX_CR4_FIXED0: 1336 msr = &vmx->nested.msrs.cr4_fixed0; 1337 break; 1338 default: 1339 BUG(); 1340 } 1341 1342 /* 1343 * 1 bits (which indicates bits which "must-be-1" during VMX operation) 1344 * must be 1 in the restored value. 1345 */ 1346 if (!is_bitwise_subset(data, *msr, -1ULL)) 1347 return -EINVAL; 1348 1349 *msr = data; 1350 return 0; 1351 } 1352 1353 /* 1354 * Called when userspace is restoring VMX MSRs. 1355 * 1356 * Returns 0 on success, non-0 otherwise. 1357 */ 1358 int vmx_set_vmx_msr(struct kvm_vcpu *vcpu, u32 msr_index, u64 data) 1359 { 1360 struct vcpu_vmx *vmx = to_vmx(vcpu); 1361 1362 /* 1363 * Don't allow changes to the VMX capability MSRs while the vCPU 1364 * is in VMX operation. 1365 */ 1366 if (vmx->nested.vmxon) 1367 return -EBUSY; 1368 1369 switch (msr_index) { 1370 case MSR_IA32_VMX_BASIC: 1371 return vmx_restore_vmx_basic(vmx, data); 1372 case MSR_IA32_VMX_PINBASED_CTLS: 1373 case MSR_IA32_VMX_PROCBASED_CTLS: 1374 case MSR_IA32_VMX_EXIT_CTLS: 1375 case MSR_IA32_VMX_ENTRY_CTLS: 1376 /* 1377 * The "non-true" VMX capability MSRs are generated from the 1378 * "true" MSRs, so we do not support restoring them directly. 1379 * 1380 * If userspace wants to emulate VMX_BASIC[55]=0, userspace 1381 * should restore the "true" MSRs with the must-be-1 bits 1382 * set according to the SDM Vol 3. A.2 "RESERVED CONTROLS AND 1383 * DEFAULT SETTINGS". 1384 */ 1385 return -EINVAL; 1386 case MSR_IA32_VMX_TRUE_PINBASED_CTLS: 1387 case MSR_IA32_VMX_TRUE_PROCBASED_CTLS: 1388 case MSR_IA32_VMX_TRUE_EXIT_CTLS: 1389 case MSR_IA32_VMX_TRUE_ENTRY_CTLS: 1390 case MSR_IA32_VMX_PROCBASED_CTLS2: 1391 return vmx_restore_control_msr(vmx, msr_index, data); 1392 case MSR_IA32_VMX_MISC: 1393 return vmx_restore_vmx_misc(vmx, data); 1394 case MSR_IA32_VMX_CR0_FIXED0: 1395 case MSR_IA32_VMX_CR4_FIXED0: 1396 return vmx_restore_fixed0_msr(vmx, msr_index, data); 1397 case MSR_IA32_VMX_CR0_FIXED1: 1398 case MSR_IA32_VMX_CR4_FIXED1: 1399 /* 1400 * These MSRs are generated based on the vCPU's CPUID, so we 1401 * do not support restoring them directly. 1402 */ 1403 return -EINVAL; 1404 case MSR_IA32_VMX_EPT_VPID_CAP: 1405 return vmx_restore_vmx_ept_vpid_cap(vmx, data); 1406 case MSR_IA32_VMX_VMCS_ENUM: 1407 vmx->nested.msrs.vmcs_enum = data; 1408 return 0; 1409 case MSR_IA32_VMX_VMFUNC: 1410 if (data & ~vmx->nested.msrs.vmfunc_controls) 1411 return -EINVAL; 1412 vmx->nested.msrs.vmfunc_controls = data; 1413 return 0; 1414 default: 1415 /* 1416 * The rest of the VMX capability MSRs do not support restore. 1417 */ 1418 return -EINVAL; 1419 } 1420 } 1421 1422 /* Returns 0 on success, non-0 otherwise. */ 1423 int vmx_get_vmx_msr(struct nested_vmx_msrs *msrs, u32 msr_index, u64 *pdata) 1424 { 1425 switch (msr_index) { 1426 case MSR_IA32_VMX_BASIC: 1427 *pdata = msrs->basic; 1428 break; 1429 case MSR_IA32_VMX_TRUE_PINBASED_CTLS: 1430 case MSR_IA32_VMX_PINBASED_CTLS: 1431 *pdata = vmx_control_msr( 1432 msrs->pinbased_ctls_low, 1433 msrs->pinbased_ctls_high); 1434 if (msr_index == MSR_IA32_VMX_PINBASED_CTLS) 1435 *pdata |= PIN_BASED_ALWAYSON_WITHOUT_TRUE_MSR; 1436 break; 1437 case MSR_IA32_VMX_TRUE_PROCBASED_CTLS: 1438 case MSR_IA32_VMX_PROCBASED_CTLS: 1439 *pdata = vmx_control_msr( 1440 msrs->procbased_ctls_low, 1441 msrs->procbased_ctls_high); 1442 if (msr_index == MSR_IA32_VMX_PROCBASED_CTLS) 1443 *pdata |= CPU_BASED_ALWAYSON_WITHOUT_TRUE_MSR; 1444 break; 1445 case MSR_IA32_VMX_TRUE_EXIT_CTLS: 1446 case MSR_IA32_VMX_EXIT_CTLS: 1447 *pdata = vmx_control_msr( 1448 msrs->exit_ctls_low, 1449 msrs->exit_ctls_high); 1450 if (msr_index == MSR_IA32_VMX_EXIT_CTLS) 1451 *pdata |= VM_EXIT_ALWAYSON_WITHOUT_TRUE_MSR; 1452 break; 1453 case MSR_IA32_VMX_TRUE_ENTRY_CTLS: 1454 case MSR_IA32_VMX_ENTRY_CTLS: 1455 *pdata = vmx_control_msr( 1456 msrs->entry_ctls_low, 1457 msrs->entry_ctls_high); 1458 if (msr_index == MSR_IA32_VMX_ENTRY_CTLS) 1459 *pdata |= VM_ENTRY_ALWAYSON_WITHOUT_TRUE_MSR; 1460 break; 1461 case MSR_IA32_VMX_MISC: 1462 *pdata = vmx_control_msr( 1463 msrs->misc_low, 1464 msrs->misc_high); 1465 break; 1466 case MSR_IA32_VMX_CR0_FIXED0: 1467 *pdata = msrs->cr0_fixed0; 1468 break; 1469 case MSR_IA32_VMX_CR0_FIXED1: 1470 *pdata = msrs->cr0_fixed1; 1471 break; 1472 case MSR_IA32_VMX_CR4_FIXED0: 1473 *pdata = msrs->cr4_fixed0; 1474 break; 1475 case MSR_IA32_VMX_CR4_FIXED1: 1476 *pdata = msrs->cr4_fixed1; 1477 break; 1478 case MSR_IA32_VMX_VMCS_ENUM: 1479 *pdata = msrs->vmcs_enum; 1480 break; 1481 case MSR_IA32_VMX_PROCBASED_CTLS2: 1482 *pdata = vmx_control_msr( 1483 msrs->secondary_ctls_low, 1484 msrs->secondary_ctls_high); 1485 break; 1486 case MSR_IA32_VMX_EPT_VPID_CAP: 1487 *pdata = msrs->ept_caps | 1488 ((u64)msrs->vpid_caps << 32); 1489 break; 1490 case MSR_IA32_VMX_VMFUNC: 1491 *pdata = msrs->vmfunc_controls; 1492 break; 1493 default: 1494 return 1; 1495 } 1496 1497 return 0; 1498 } 1499 1500 /* 1501 * Copy the writable VMCS shadow fields back to the VMCS12, in case they have 1502 * been modified by the L1 guest. Note, "writable" in this context means 1503 * "writable by the guest", i.e. tagged SHADOW_FIELD_RW; the set of 1504 * fields tagged SHADOW_FIELD_RO may or may not align with the "read-only" 1505 * VM-exit information fields (which are actually writable if the vCPU is 1506 * configured to support "VMWRITE to any supported field in the VMCS"). 1507 */ 1508 static void copy_shadow_to_vmcs12(struct vcpu_vmx *vmx) 1509 { 1510 struct vmcs *shadow_vmcs = vmx->vmcs01.shadow_vmcs; 1511 struct vmcs12 *vmcs12 = get_vmcs12(&vmx->vcpu); 1512 struct shadow_vmcs_field field; 1513 unsigned long val; 1514 int i; 1515 1516 if (WARN_ON(!shadow_vmcs)) 1517 return; 1518 1519 preempt_disable(); 1520 1521 vmcs_load(shadow_vmcs); 1522 1523 for (i = 0; i < max_shadow_read_write_fields; i++) { 1524 field = shadow_read_write_fields[i]; 1525 val = __vmcs_readl(field.encoding); 1526 vmcs12_write_any(vmcs12, field.encoding, field.offset, val); 1527 } 1528 1529 vmcs_clear(shadow_vmcs); 1530 vmcs_load(vmx->loaded_vmcs->vmcs); 1531 1532 preempt_enable(); 1533 } 1534 1535 static void copy_vmcs12_to_shadow(struct vcpu_vmx *vmx) 1536 { 1537 const struct shadow_vmcs_field *fields[] = { 1538 shadow_read_write_fields, 1539 shadow_read_only_fields 1540 }; 1541 const int max_fields[] = { 1542 max_shadow_read_write_fields, 1543 max_shadow_read_only_fields 1544 }; 1545 struct vmcs *shadow_vmcs = vmx->vmcs01.shadow_vmcs; 1546 struct vmcs12 *vmcs12 = get_vmcs12(&vmx->vcpu); 1547 struct shadow_vmcs_field field; 1548 unsigned long val; 1549 int i, q; 1550 1551 if (WARN_ON(!shadow_vmcs)) 1552 return; 1553 1554 vmcs_load(shadow_vmcs); 1555 1556 for (q = 0; q < ARRAY_SIZE(fields); q++) { 1557 for (i = 0; i < max_fields[q]; i++) { 1558 field = fields[q][i]; 1559 val = vmcs12_read_any(vmcs12, field.encoding, 1560 field.offset); 1561 __vmcs_writel(field.encoding, val); 1562 } 1563 } 1564 1565 vmcs_clear(shadow_vmcs); 1566 vmcs_load(vmx->loaded_vmcs->vmcs); 1567 } 1568 1569 static void copy_enlightened_to_vmcs12(struct vcpu_vmx *vmx, u32 hv_clean_fields) 1570 { 1571 struct vmcs12 *vmcs12 = vmx->nested.cached_vmcs12; 1572 struct hv_enlightened_vmcs *evmcs = vmx->nested.hv_evmcs; 1573 1574 /* HV_VMX_ENLIGHTENED_CLEAN_FIELD_NONE */ 1575 vmcs12->tpr_threshold = evmcs->tpr_threshold; 1576 vmcs12->guest_rip = evmcs->guest_rip; 1577 1578 if (unlikely(!(hv_clean_fields & 1579 HV_VMX_ENLIGHTENED_CLEAN_FIELD_GUEST_BASIC))) { 1580 vmcs12->guest_rsp = evmcs->guest_rsp; 1581 vmcs12->guest_rflags = evmcs->guest_rflags; 1582 vmcs12->guest_interruptibility_info = 1583 evmcs->guest_interruptibility_info; 1584 } 1585 1586 if (unlikely(!(hv_clean_fields & 1587 HV_VMX_ENLIGHTENED_CLEAN_FIELD_CONTROL_PROC))) { 1588 vmcs12->cpu_based_vm_exec_control = 1589 evmcs->cpu_based_vm_exec_control; 1590 } 1591 1592 if (unlikely(!(hv_clean_fields & 1593 HV_VMX_ENLIGHTENED_CLEAN_FIELD_CONTROL_EXCPN))) { 1594 vmcs12->exception_bitmap = evmcs->exception_bitmap; 1595 } 1596 1597 if (unlikely(!(hv_clean_fields & 1598 HV_VMX_ENLIGHTENED_CLEAN_FIELD_CONTROL_ENTRY))) { 1599 vmcs12->vm_entry_controls = evmcs->vm_entry_controls; 1600 } 1601 1602 if (unlikely(!(hv_clean_fields & 1603 HV_VMX_ENLIGHTENED_CLEAN_FIELD_CONTROL_EVENT))) { 1604 vmcs12->vm_entry_intr_info_field = 1605 evmcs->vm_entry_intr_info_field; 1606 vmcs12->vm_entry_exception_error_code = 1607 evmcs->vm_entry_exception_error_code; 1608 vmcs12->vm_entry_instruction_len = 1609 evmcs->vm_entry_instruction_len; 1610 } 1611 1612 if (unlikely(!(hv_clean_fields & 1613 HV_VMX_ENLIGHTENED_CLEAN_FIELD_HOST_GRP1))) { 1614 vmcs12->host_ia32_pat = evmcs->host_ia32_pat; 1615 vmcs12->host_ia32_efer = evmcs->host_ia32_efer; 1616 vmcs12->host_cr0 = evmcs->host_cr0; 1617 vmcs12->host_cr3 = evmcs->host_cr3; 1618 vmcs12->host_cr4 = evmcs->host_cr4; 1619 vmcs12->host_ia32_sysenter_esp = evmcs->host_ia32_sysenter_esp; 1620 vmcs12->host_ia32_sysenter_eip = evmcs->host_ia32_sysenter_eip; 1621 vmcs12->host_rip = evmcs->host_rip; 1622 vmcs12->host_ia32_sysenter_cs = evmcs->host_ia32_sysenter_cs; 1623 vmcs12->host_es_selector = evmcs->host_es_selector; 1624 vmcs12->host_cs_selector = evmcs->host_cs_selector; 1625 vmcs12->host_ss_selector = evmcs->host_ss_selector; 1626 vmcs12->host_ds_selector = evmcs->host_ds_selector; 1627 vmcs12->host_fs_selector = evmcs->host_fs_selector; 1628 vmcs12->host_gs_selector = evmcs->host_gs_selector; 1629 vmcs12->host_tr_selector = evmcs->host_tr_selector; 1630 } 1631 1632 if (unlikely(!(hv_clean_fields & 1633 HV_VMX_ENLIGHTENED_CLEAN_FIELD_CONTROL_GRP1))) { 1634 vmcs12->pin_based_vm_exec_control = 1635 evmcs->pin_based_vm_exec_control; 1636 vmcs12->vm_exit_controls = evmcs->vm_exit_controls; 1637 vmcs12->secondary_vm_exec_control = 1638 evmcs->secondary_vm_exec_control; 1639 } 1640 1641 if (unlikely(!(hv_clean_fields & 1642 HV_VMX_ENLIGHTENED_CLEAN_FIELD_IO_BITMAP))) { 1643 vmcs12->io_bitmap_a = evmcs->io_bitmap_a; 1644 vmcs12->io_bitmap_b = evmcs->io_bitmap_b; 1645 } 1646 1647 if (unlikely(!(hv_clean_fields & 1648 HV_VMX_ENLIGHTENED_CLEAN_FIELD_MSR_BITMAP))) { 1649 vmcs12->msr_bitmap = evmcs->msr_bitmap; 1650 } 1651 1652 if (unlikely(!(hv_clean_fields & 1653 HV_VMX_ENLIGHTENED_CLEAN_FIELD_GUEST_GRP2))) { 1654 vmcs12->guest_es_base = evmcs->guest_es_base; 1655 vmcs12->guest_cs_base = evmcs->guest_cs_base; 1656 vmcs12->guest_ss_base = evmcs->guest_ss_base; 1657 vmcs12->guest_ds_base = evmcs->guest_ds_base; 1658 vmcs12->guest_fs_base = evmcs->guest_fs_base; 1659 vmcs12->guest_gs_base = evmcs->guest_gs_base; 1660 vmcs12->guest_ldtr_base = evmcs->guest_ldtr_base; 1661 vmcs12->guest_tr_base = evmcs->guest_tr_base; 1662 vmcs12->guest_gdtr_base = evmcs->guest_gdtr_base; 1663 vmcs12->guest_idtr_base = evmcs->guest_idtr_base; 1664 vmcs12->guest_es_limit = evmcs->guest_es_limit; 1665 vmcs12->guest_cs_limit = evmcs->guest_cs_limit; 1666 vmcs12->guest_ss_limit = evmcs->guest_ss_limit; 1667 vmcs12->guest_ds_limit = evmcs->guest_ds_limit; 1668 vmcs12->guest_fs_limit = evmcs->guest_fs_limit; 1669 vmcs12->guest_gs_limit = evmcs->guest_gs_limit; 1670 vmcs12->guest_ldtr_limit = evmcs->guest_ldtr_limit; 1671 vmcs12->guest_tr_limit = evmcs->guest_tr_limit; 1672 vmcs12->guest_gdtr_limit = evmcs->guest_gdtr_limit; 1673 vmcs12->guest_idtr_limit = evmcs->guest_idtr_limit; 1674 vmcs12->guest_es_ar_bytes = evmcs->guest_es_ar_bytes; 1675 vmcs12->guest_cs_ar_bytes = evmcs->guest_cs_ar_bytes; 1676 vmcs12->guest_ss_ar_bytes = evmcs->guest_ss_ar_bytes; 1677 vmcs12->guest_ds_ar_bytes = evmcs->guest_ds_ar_bytes; 1678 vmcs12->guest_fs_ar_bytes = evmcs->guest_fs_ar_bytes; 1679 vmcs12->guest_gs_ar_bytes = evmcs->guest_gs_ar_bytes; 1680 vmcs12->guest_ldtr_ar_bytes = evmcs->guest_ldtr_ar_bytes; 1681 vmcs12->guest_tr_ar_bytes = evmcs->guest_tr_ar_bytes; 1682 vmcs12->guest_es_selector = evmcs->guest_es_selector; 1683 vmcs12->guest_cs_selector = evmcs->guest_cs_selector; 1684 vmcs12->guest_ss_selector = evmcs->guest_ss_selector; 1685 vmcs12->guest_ds_selector = evmcs->guest_ds_selector; 1686 vmcs12->guest_fs_selector = evmcs->guest_fs_selector; 1687 vmcs12->guest_gs_selector = evmcs->guest_gs_selector; 1688 vmcs12->guest_ldtr_selector = evmcs->guest_ldtr_selector; 1689 vmcs12->guest_tr_selector = evmcs->guest_tr_selector; 1690 } 1691 1692 if (unlikely(!(hv_clean_fields & 1693 HV_VMX_ENLIGHTENED_CLEAN_FIELD_CONTROL_GRP2))) { 1694 vmcs12->tsc_offset = evmcs->tsc_offset; 1695 vmcs12->virtual_apic_page_addr = evmcs->virtual_apic_page_addr; 1696 vmcs12->xss_exit_bitmap = evmcs->xss_exit_bitmap; 1697 } 1698 1699 if (unlikely(!(hv_clean_fields & 1700 HV_VMX_ENLIGHTENED_CLEAN_FIELD_CRDR))) { 1701 vmcs12->cr0_guest_host_mask = evmcs->cr0_guest_host_mask; 1702 vmcs12->cr4_guest_host_mask = evmcs->cr4_guest_host_mask; 1703 vmcs12->cr0_read_shadow = evmcs->cr0_read_shadow; 1704 vmcs12->cr4_read_shadow = evmcs->cr4_read_shadow; 1705 vmcs12->guest_cr0 = evmcs->guest_cr0; 1706 vmcs12->guest_cr3 = evmcs->guest_cr3; 1707 vmcs12->guest_cr4 = evmcs->guest_cr4; 1708 vmcs12->guest_dr7 = evmcs->guest_dr7; 1709 } 1710 1711 if (unlikely(!(hv_clean_fields & 1712 HV_VMX_ENLIGHTENED_CLEAN_FIELD_HOST_POINTER))) { 1713 vmcs12->host_fs_base = evmcs->host_fs_base; 1714 vmcs12->host_gs_base = evmcs->host_gs_base; 1715 vmcs12->host_tr_base = evmcs->host_tr_base; 1716 vmcs12->host_gdtr_base = evmcs->host_gdtr_base; 1717 vmcs12->host_idtr_base = evmcs->host_idtr_base; 1718 vmcs12->host_rsp = evmcs->host_rsp; 1719 } 1720 1721 if (unlikely(!(hv_clean_fields & 1722 HV_VMX_ENLIGHTENED_CLEAN_FIELD_CONTROL_XLAT))) { 1723 vmcs12->ept_pointer = evmcs->ept_pointer; 1724 vmcs12->virtual_processor_id = evmcs->virtual_processor_id; 1725 } 1726 1727 if (unlikely(!(hv_clean_fields & 1728 HV_VMX_ENLIGHTENED_CLEAN_FIELD_GUEST_GRP1))) { 1729 vmcs12->vmcs_link_pointer = evmcs->vmcs_link_pointer; 1730 vmcs12->guest_ia32_debugctl = evmcs->guest_ia32_debugctl; 1731 vmcs12->guest_ia32_pat = evmcs->guest_ia32_pat; 1732 vmcs12->guest_ia32_efer = evmcs->guest_ia32_efer; 1733 vmcs12->guest_pdptr0 = evmcs->guest_pdptr0; 1734 vmcs12->guest_pdptr1 = evmcs->guest_pdptr1; 1735 vmcs12->guest_pdptr2 = evmcs->guest_pdptr2; 1736 vmcs12->guest_pdptr3 = evmcs->guest_pdptr3; 1737 vmcs12->guest_pending_dbg_exceptions = 1738 evmcs->guest_pending_dbg_exceptions; 1739 vmcs12->guest_sysenter_esp = evmcs->guest_sysenter_esp; 1740 vmcs12->guest_sysenter_eip = evmcs->guest_sysenter_eip; 1741 vmcs12->guest_bndcfgs = evmcs->guest_bndcfgs; 1742 vmcs12->guest_activity_state = evmcs->guest_activity_state; 1743 vmcs12->guest_sysenter_cs = evmcs->guest_sysenter_cs; 1744 } 1745 1746 /* 1747 * Not used? 1748 * vmcs12->vm_exit_msr_store_addr = evmcs->vm_exit_msr_store_addr; 1749 * vmcs12->vm_exit_msr_load_addr = evmcs->vm_exit_msr_load_addr; 1750 * vmcs12->vm_entry_msr_load_addr = evmcs->vm_entry_msr_load_addr; 1751 * vmcs12->page_fault_error_code_mask = 1752 * evmcs->page_fault_error_code_mask; 1753 * vmcs12->page_fault_error_code_match = 1754 * evmcs->page_fault_error_code_match; 1755 * vmcs12->cr3_target_count = evmcs->cr3_target_count; 1756 * vmcs12->vm_exit_msr_store_count = evmcs->vm_exit_msr_store_count; 1757 * vmcs12->vm_exit_msr_load_count = evmcs->vm_exit_msr_load_count; 1758 * vmcs12->vm_entry_msr_load_count = evmcs->vm_entry_msr_load_count; 1759 */ 1760 1761 /* 1762 * Read only fields: 1763 * vmcs12->guest_physical_address = evmcs->guest_physical_address; 1764 * vmcs12->vm_instruction_error = evmcs->vm_instruction_error; 1765 * vmcs12->vm_exit_reason = evmcs->vm_exit_reason; 1766 * vmcs12->vm_exit_intr_info = evmcs->vm_exit_intr_info; 1767 * vmcs12->vm_exit_intr_error_code = evmcs->vm_exit_intr_error_code; 1768 * vmcs12->idt_vectoring_info_field = evmcs->idt_vectoring_info_field; 1769 * vmcs12->idt_vectoring_error_code = evmcs->idt_vectoring_error_code; 1770 * vmcs12->vm_exit_instruction_len = evmcs->vm_exit_instruction_len; 1771 * vmcs12->vmx_instruction_info = evmcs->vmx_instruction_info; 1772 * vmcs12->exit_qualification = evmcs->exit_qualification; 1773 * vmcs12->guest_linear_address = evmcs->guest_linear_address; 1774 * 1775 * Not present in struct vmcs12: 1776 * vmcs12->exit_io_instruction_ecx = evmcs->exit_io_instruction_ecx; 1777 * vmcs12->exit_io_instruction_esi = evmcs->exit_io_instruction_esi; 1778 * vmcs12->exit_io_instruction_edi = evmcs->exit_io_instruction_edi; 1779 * vmcs12->exit_io_instruction_eip = evmcs->exit_io_instruction_eip; 1780 */ 1781 1782 return; 1783 } 1784 1785 static void copy_vmcs12_to_enlightened(struct vcpu_vmx *vmx) 1786 { 1787 struct vmcs12 *vmcs12 = vmx->nested.cached_vmcs12; 1788 struct hv_enlightened_vmcs *evmcs = vmx->nested.hv_evmcs; 1789 1790 /* 1791 * Should not be changed by KVM: 1792 * 1793 * evmcs->host_es_selector = vmcs12->host_es_selector; 1794 * evmcs->host_cs_selector = vmcs12->host_cs_selector; 1795 * evmcs->host_ss_selector = vmcs12->host_ss_selector; 1796 * evmcs->host_ds_selector = vmcs12->host_ds_selector; 1797 * evmcs->host_fs_selector = vmcs12->host_fs_selector; 1798 * evmcs->host_gs_selector = vmcs12->host_gs_selector; 1799 * evmcs->host_tr_selector = vmcs12->host_tr_selector; 1800 * evmcs->host_ia32_pat = vmcs12->host_ia32_pat; 1801 * evmcs->host_ia32_efer = vmcs12->host_ia32_efer; 1802 * evmcs->host_cr0 = vmcs12->host_cr0; 1803 * evmcs->host_cr3 = vmcs12->host_cr3; 1804 * evmcs->host_cr4 = vmcs12->host_cr4; 1805 * evmcs->host_ia32_sysenter_esp = vmcs12->host_ia32_sysenter_esp; 1806 * evmcs->host_ia32_sysenter_eip = vmcs12->host_ia32_sysenter_eip; 1807 * evmcs->host_rip = vmcs12->host_rip; 1808 * evmcs->host_ia32_sysenter_cs = vmcs12->host_ia32_sysenter_cs; 1809 * evmcs->host_fs_base = vmcs12->host_fs_base; 1810 * evmcs->host_gs_base = vmcs12->host_gs_base; 1811 * evmcs->host_tr_base = vmcs12->host_tr_base; 1812 * evmcs->host_gdtr_base = vmcs12->host_gdtr_base; 1813 * evmcs->host_idtr_base = vmcs12->host_idtr_base; 1814 * evmcs->host_rsp = vmcs12->host_rsp; 1815 * sync_vmcs02_to_vmcs12() doesn't read these: 1816 * evmcs->io_bitmap_a = vmcs12->io_bitmap_a; 1817 * evmcs->io_bitmap_b = vmcs12->io_bitmap_b; 1818 * evmcs->msr_bitmap = vmcs12->msr_bitmap; 1819 * evmcs->ept_pointer = vmcs12->ept_pointer; 1820 * evmcs->xss_exit_bitmap = vmcs12->xss_exit_bitmap; 1821 * evmcs->vm_exit_msr_store_addr = vmcs12->vm_exit_msr_store_addr; 1822 * evmcs->vm_exit_msr_load_addr = vmcs12->vm_exit_msr_load_addr; 1823 * evmcs->vm_entry_msr_load_addr = vmcs12->vm_entry_msr_load_addr; 1824 * evmcs->tpr_threshold = vmcs12->tpr_threshold; 1825 * evmcs->virtual_processor_id = vmcs12->virtual_processor_id; 1826 * evmcs->exception_bitmap = vmcs12->exception_bitmap; 1827 * evmcs->vmcs_link_pointer = vmcs12->vmcs_link_pointer; 1828 * evmcs->pin_based_vm_exec_control = vmcs12->pin_based_vm_exec_control; 1829 * evmcs->vm_exit_controls = vmcs12->vm_exit_controls; 1830 * evmcs->secondary_vm_exec_control = vmcs12->secondary_vm_exec_control; 1831 * evmcs->page_fault_error_code_mask = 1832 * vmcs12->page_fault_error_code_mask; 1833 * evmcs->page_fault_error_code_match = 1834 * vmcs12->page_fault_error_code_match; 1835 * evmcs->cr3_target_count = vmcs12->cr3_target_count; 1836 * evmcs->virtual_apic_page_addr = vmcs12->virtual_apic_page_addr; 1837 * evmcs->tsc_offset = vmcs12->tsc_offset; 1838 * evmcs->guest_ia32_debugctl = vmcs12->guest_ia32_debugctl; 1839 * evmcs->cr0_guest_host_mask = vmcs12->cr0_guest_host_mask; 1840 * evmcs->cr4_guest_host_mask = vmcs12->cr4_guest_host_mask; 1841 * evmcs->cr0_read_shadow = vmcs12->cr0_read_shadow; 1842 * evmcs->cr4_read_shadow = vmcs12->cr4_read_shadow; 1843 * evmcs->vm_exit_msr_store_count = vmcs12->vm_exit_msr_store_count; 1844 * evmcs->vm_exit_msr_load_count = vmcs12->vm_exit_msr_load_count; 1845 * evmcs->vm_entry_msr_load_count = vmcs12->vm_entry_msr_load_count; 1846 * 1847 * Not present in struct vmcs12: 1848 * evmcs->exit_io_instruction_ecx = vmcs12->exit_io_instruction_ecx; 1849 * evmcs->exit_io_instruction_esi = vmcs12->exit_io_instruction_esi; 1850 * evmcs->exit_io_instruction_edi = vmcs12->exit_io_instruction_edi; 1851 * evmcs->exit_io_instruction_eip = vmcs12->exit_io_instruction_eip; 1852 */ 1853 1854 evmcs->guest_es_selector = vmcs12->guest_es_selector; 1855 evmcs->guest_cs_selector = vmcs12->guest_cs_selector; 1856 evmcs->guest_ss_selector = vmcs12->guest_ss_selector; 1857 evmcs->guest_ds_selector = vmcs12->guest_ds_selector; 1858 evmcs->guest_fs_selector = vmcs12->guest_fs_selector; 1859 evmcs->guest_gs_selector = vmcs12->guest_gs_selector; 1860 evmcs->guest_ldtr_selector = vmcs12->guest_ldtr_selector; 1861 evmcs->guest_tr_selector = vmcs12->guest_tr_selector; 1862 1863 evmcs->guest_es_limit = vmcs12->guest_es_limit; 1864 evmcs->guest_cs_limit = vmcs12->guest_cs_limit; 1865 evmcs->guest_ss_limit = vmcs12->guest_ss_limit; 1866 evmcs->guest_ds_limit = vmcs12->guest_ds_limit; 1867 evmcs->guest_fs_limit = vmcs12->guest_fs_limit; 1868 evmcs->guest_gs_limit = vmcs12->guest_gs_limit; 1869 evmcs->guest_ldtr_limit = vmcs12->guest_ldtr_limit; 1870 evmcs->guest_tr_limit = vmcs12->guest_tr_limit; 1871 evmcs->guest_gdtr_limit = vmcs12->guest_gdtr_limit; 1872 evmcs->guest_idtr_limit = vmcs12->guest_idtr_limit; 1873 1874 evmcs->guest_es_ar_bytes = vmcs12->guest_es_ar_bytes; 1875 evmcs->guest_cs_ar_bytes = vmcs12->guest_cs_ar_bytes; 1876 evmcs->guest_ss_ar_bytes = vmcs12->guest_ss_ar_bytes; 1877 evmcs->guest_ds_ar_bytes = vmcs12->guest_ds_ar_bytes; 1878 evmcs->guest_fs_ar_bytes = vmcs12->guest_fs_ar_bytes; 1879 evmcs->guest_gs_ar_bytes = vmcs12->guest_gs_ar_bytes; 1880 evmcs->guest_ldtr_ar_bytes = vmcs12->guest_ldtr_ar_bytes; 1881 evmcs->guest_tr_ar_bytes = vmcs12->guest_tr_ar_bytes; 1882 1883 evmcs->guest_es_base = vmcs12->guest_es_base; 1884 evmcs->guest_cs_base = vmcs12->guest_cs_base; 1885 evmcs->guest_ss_base = vmcs12->guest_ss_base; 1886 evmcs->guest_ds_base = vmcs12->guest_ds_base; 1887 evmcs->guest_fs_base = vmcs12->guest_fs_base; 1888 evmcs->guest_gs_base = vmcs12->guest_gs_base; 1889 evmcs->guest_ldtr_base = vmcs12->guest_ldtr_base; 1890 evmcs->guest_tr_base = vmcs12->guest_tr_base; 1891 evmcs->guest_gdtr_base = vmcs12->guest_gdtr_base; 1892 evmcs->guest_idtr_base = vmcs12->guest_idtr_base; 1893 1894 evmcs->guest_ia32_pat = vmcs12->guest_ia32_pat; 1895 evmcs->guest_ia32_efer = vmcs12->guest_ia32_efer; 1896 1897 evmcs->guest_pdptr0 = vmcs12->guest_pdptr0; 1898 evmcs->guest_pdptr1 = vmcs12->guest_pdptr1; 1899 evmcs->guest_pdptr2 = vmcs12->guest_pdptr2; 1900 evmcs->guest_pdptr3 = vmcs12->guest_pdptr3; 1901 1902 evmcs->guest_pending_dbg_exceptions = 1903 vmcs12->guest_pending_dbg_exceptions; 1904 evmcs->guest_sysenter_esp = vmcs12->guest_sysenter_esp; 1905 evmcs->guest_sysenter_eip = vmcs12->guest_sysenter_eip; 1906 1907 evmcs->guest_activity_state = vmcs12->guest_activity_state; 1908 evmcs->guest_sysenter_cs = vmcs12->guest_sysenter_cs; 1909 1910 evmcs->guest_cr0 = vmcs12->guest_cr0; 1911 evmcs->guest_cr3 = vmcs12->guest_cr3; 1912 evmcs->guest_cr4 = vmcs12->guest_cr4; 1913 evmcs->guest_dr7 = vmcs12->guest_dr7; 1914 1915 evmcs->guest_physical_address = vmcs12->guest_physical_address; 1916 1917 evmcs->vm_instruction_error = vmcs12->vm_instruction_error; 1918 evmcs->vm_exit_reason = vmcs12->vm_exit_reason; 1919 evmcs->vm_exit_intr_info = vmcs12->vm_exit_intr_info; 1920 evmcs->vm_exit_intr_error_code = vmcs12->vm_exit_intr_error_code; 1921 evmcs->idt_vectoring_info_field = vmcs12->idt_vectoring_info_field; 1922 evmcs->idt_vectoring_error_code = vmcs12->idt_vectoring_error_code; 1923 evmcs->vm_exit_instruction_len = vmcs12->vm_exit_instruction_len; 1924 evmcs->vmx_instruction_info = vmcs12->vmx_instruction_info; 1925 1926 evmcs->exit_qualification = vmcs12->exit_qualification; 1927 1928 evmcs->guest_linear_address = vmcs12->guest_linear_address; 1929 evmcs->guest_rsp = vmcs12->guest_rsp; 1930 evmcs->guest_rflags = vmcs12->guest_rflags; 1931 1932 evmcs->guest_interruptibility_info = 1933 vmcs12->guest_interruptibility_info; 1934 evmcs->cpu_based_vm_exec_control = vmcs12->cpu_based_vm_exec_control; 1935 evmcs->vm_entry_controls = vmcs12->vm_entry_controls; 1936 evmcs->vm_entry_intr_info_field = vmcs12->vm_entry_intr_info_field; 1937 evmcs->vm_entry_exception_error_code = 1938 vmcs12->vm_entry_exception_error_code; 1939 evmcs->vm_entry_instruction_len = vmcs12->vm_entry_instruction_len; 1940 1941 evmcs->guest_rip = vmcs12->guest_rip; 1942 1943 evmcs->guest_bndcfgs = vmcs12->guest_bndcfgs; 1944 1945 return; 1946 } 1947 1948 /* 1949 * This is an equivalent of the nested hypervisor executing the vmptrld 1950 * instruction. 1951 */ 1952 static enum nested_evmptrld_status nested_vmx_handle_enlightened_vmptrld( 1953 struct kvm_vcpu *vcpu, bool from_launch) 1954 { 1955 struct vcpu_vmx *vmx = to_vmx(vcpu); 1956 bool evmcs_gpa_changed = false; 1957 u64 evmcs_gpa; 1958 1959 if (likely(!vmx->nested.enlightened_vmcs_enabled)) 1960 return EVMPTRLD_DISABLED; 1961 1962 if (!nested_enlightened_vmentry(vcpu, &evmcs_gpa)) { 1963 nested_release_evmcs(vcpu); 1964 return EVMPTRLD_DISABLED; 1965 } 1966 1967 if (unlikely(evmcs_gpa != vmx->nested.hv_evmcs_vmptr)) { 1968 vmx->nested.current_vmptr = INVALID_GPA; 1969 1970 nested_release_evmcs(vcpu); 1971 1972 if (kvm_vcpu_map(vcpu, gpa_to_gfn(evmcs_gpa), 1973 &vmx->nested.hv_evmcs_map)) 1974 return EVMPTRLD_ERROR; 1975 1976 vmx->nested.hv_evmcs = vmx->nested.hv_evmcs_map.hva; 1977 1978 /* 1979 * Currently, KVM only supports eVMCS version 1 1980 * (== KVM_EVMCS_VERSION) and thus we expect guest to set this 1981 * value to first u32 field of eVMCS which should specify eVMCS 1982 * VersionNumber. 1983 * 1984 * Guest should be aware of supported eVMCS versions by host by 1985 * examining CPUID.0x4000000A.EAX[0:15]. Host userspace VMM is 1986 * expected to set this CPUID leaf according to the value 1987 * returned in vmcs_version from nested_enable_evmcs(). 1988 * 1989 * However, it turns out that Microsoft Hyper-V fails to comply 1990 * to their own invented interface: When Hyper-V use eVMCS, it 1991 * just sets first u32 field of eVMCS to revision_id specified 1992 * in MSR_IA32_VMX_BASIC. Instead of used eVMCS version number 1993 * which is one of the supported versions specified in 1994 * CPUID.0x4000000A.EAX[0:15]. 1995 * 1996 * To overcome Hyper-V bug, we accept here either a supported 1997 * eVMCS version or VMCS12 revision_id as valid values for first 1998 * u32 field of eVMCS. 1999 */ 2000 if ((vmx->nested.hv_evmcs->revision_id != KVM_EVMCS_VERSION) && 2001 (vmx->nested.hv_evmcs->revision_id != VMCS12_REVISION)) { 2002 nested_release_evmcs(vcpu); 2003 return EVMPTRLD_VMFAIL; 2004 } 2005 2006 vmx->nested.hv_evmcs_vmptr = evmcs_gpa; 2007 2008 evmcs_gpa_changed = true; 2009 /* 2010 * Unlike normal vmcs12, enlightened vmcs12 is not fully 2011 * reloaded from guest's memory (read only fields, fields not 2012 * present in struct hv_enlightened_vmcs, ...). Make sure there 2013 * are no leftovers. 2014 */ 2015 if (from_launch) { 2016 struct vmcs12 *vmcs12 = get_vmcs12(vcpu); 2017 memset(vmcs12, 0, sizeof(*vmcs12)); 2018 vmcs12->hdr.revision_id = VMCS12_REVISION; 2019 } 2020 2021 } 2022 2023 /* 2024 * Clean fields data can't be used on VMLAUNCH and when we switch 2025 * between different L2 guests as KVM keeps a single VMCS12 per L1. 2026 */ 2027 if (from_launch || evmcs_gpa_changed) 2028 vmx->nested.hv_evmcs->hv_clean_fields &= 2029 ~HV_VMX_ENLIGHTENED_CLEAN_FIELD_ALL; 2030 2031 return EVMPTRLD_SUCCEEDED; 2032 } 2033 2034 void nested_sync_vmcs12_to_shadow(struct kvm_vcpu *vcpu) 2035 { 2036 struct vcpu_vmx *vmx = to_vmx(vcpu); 2037 2038 if (evmptr_is_valid(vmx->nested.hv_evmcs_vmptr)) 2039 copy_vmcs12_to_enlightened(vmx); 2040 else 2041 copy_vmcs12_to_shadow(vmx); 2042 2043 vmx->nested.need_vmcs12_to_shadow_sync = false; 2044 } 2045 2046 static enum hrtimer_restart vmx_preemption_timer_fn(struct hrtimer *timer) 2047 { 2048 struct vcpu_vmx *vmx = 2049 container_of(timer, struct vcpu_vmx, nested.preemption_timer); 2050 2051 vmx->nested.preemption_timer_expired = true; 2052 kvm_make_request(KVM_REQ_EVENT, &vmx->vcpu); 2053 kvm_vcpu_kick(&vmx->vcpu); 2054 2055 return HRTIMER_NORESTART; 2056 } 2057 2058 static u64 vmx_calc_preemption_timer_value(struct kvm_vcpu *vcpu) 2059 { 2060 struct vcpu_vmx *vmx = to_vmx(vcpu); 2061 struct vmcs12 *vmcs12 = get_vmcs12(vcpu); 2062 2063 u64 l1_scaled_tsc = kvm_read_l1_tsc(vcpu, rdtsc()) >> 2064 VMX_MISC_EMULATED_PREEMPTION_TIMER_RATE; 2065 2066 if (!vmx->nested.has_preemption_timer_deadline) { 2067 vmx->nested.preemption_timer_deadline = 2068 vmcs12->vmx_preemption_timer_value + l1_scaled_tsc; 2069 vmx->nested.has_preemption_timer_deadline = true; 2070 } 2071 return vmx->nested.preemption_timer_deadline - l1_scaled_tsc; 2072 } 2073 2074 static void vmx_start_preemption_timer(struct kvm_vcpu *vcpu, 2075 u64 preemption_timeout) 2076 { 2077 struct vcpu_vmx *vmx = to_vmx(vcpu); 2078 2079 /* 2080 * A timer value of zero is architecturally guaranteed to cause 2081 * a VMExit prior to executing any instructions in the guest. 2082 */ 2083 if (preemption_timeout == 0) { 2084 vmx_preemption_timer_fn(&vmx->nested.preemption_timer); 2085 return; 2086 } 2087 2088 if (vcpu->arch.virtual_tsc_khz == 0) 2089 return; 2090 2091 preemption_timeout <<= VMX_MISC_EMULATED_PREEMPTION_TIMER_RATE; 2092 preemption_timeout *= 1000000; 2093 do_div(preemption_timeout, vcpu->arch.virtual_tsc_khz); 2094 hrtimer_start(&vmx->nested.preemption_timer, 2095 ktime_add_ns(ktime_get(), preemption_timeout), 2096 HRTIMER_MODE_ABS_PINNED); 2097 } 2098 2099 static u64 nested_vmx_calc_efer(struct vcpu_vmx *vmx, struct vmcs12 *vmcs12) 2100 { 2101 if (vmx->nested.nested_run_pending && 2102 (vmcs12->vm_entry_controls & VM_ENTRY_LOAD_IA32_EFER)) 2103 return vmcs12->guest_ia32_efer; 2104 else if (vmcs12->vm_entry_controls & VM_ENTRY_IA32E_MODE) 2105 return vmx->vcpu.arch.efer | (EFER_LMA | EFER_LME); 2106 else 2107 return vmx->vcpu.arch.efer & ~(EFER_LMA | EFER_LME); 2108 } 2109 2110 static void prepare_vmcs02_constant_state(struct vcpu_vmx *vmx) 2111 { 2112 /* 2113 * If vmcs02 hasn't been initialized, set the constant vmcs02 state 2114 * according to L0's settings (vmcs12 is irrelevant here). Host 2115 * fields that come from L0 and are not constant, e.g. HOST_CR3, 2116 * will be set as needed prior to VMLAUNCH/VMRESUME. 2117 */ 2118 if (vmx->nested.vmcs02_initialized) 2119 return; 2120 vmx->nested.vmcs02_initialized = true; 2121 2122 /* 2123 * We don't care what the EPTP value is we just need to guarantee 2124 * it's valid so we don't get a false positive when doing early 2125 * consistency checks. 2126 */ 2127 if (enable_ept && nested_early_check) 2128 vmcs_write64(EPT_POINTER, 2129 construct_eptp(&vmx->vcpu, 0, PT64_ROOT_4LEVEL)); 2130 2131 /* All VMFUNCs are currently emulated through L0 vmexits. */ 2132 if (cpu_has_vmx_vmfunc()) 2133 vmcs_write64(VM_FUNCTION_CONTROL, 0); 2134 2135 if (cpu_has_vmx_posted_intr()) 2136 vmcs_write16(POSTED_INTR_NV, POSTED_INTR_NESTED_VECTOR); 2137 2138 if (cpu_has_vmx_msr_bitmap()) 2139 vmcs_write64(MSR_BITMAP, __pa(vmx->nested.vmcs02.msr_bitmap)); 2140 2141 /* 2142 * PML is emulated for L2, but never enabled in hardware as the MMU 2143 * handles A/D emulation. Disabling PML for L2 also avoids having to 2144 * deal with filtering out L2 GPAs from the buffer. 2145 */ 2146 if (enable_pml) { 2147 vmcs_write64(PML_ADDRESS, 0); 2148 vmcs_write16(GUEST_PML_INDEX, -1); 2149 } 2150 2151 if (cpu_has_vmx_encls_vmexit()) 2152 vmcs_write64(ENCLS_EXITING_BITMAP, INVALID_GPA); 2153 2154 /* 2155 * Set the MSR load/store lists to match L0's settings. Only the 2156 * addresses are constant (for vmcs02), the counts can change based 2157 * on L2's behavior, e.g. switching to/from long mode. 2158 */ 2159 vmcs_write64(VM_EXIT_MSR_STORE_ADDR, __pa(vmx->msr_autostore.guest.val)); 2160 vmcs_write64(VM_EXIT_MSR_LOAD_ADDR, __pa(vmx->msr_autoload.host.val)); 2161 vmcs_write64(VM_ENTRY_MSR_LOAD_ADDR, __pa(vmx->msr_autoload.guest.val)); 2162 2163 vmx_set_constant_host_state(vmx); 2164 } 2165 2166 static void prepare_vmcs02_early_rare(struct vcpu_vmx *vmx, 2167 struct vmcs12 *vmcs12) 2168 { 2169 prepare_vmcs02_constant_state(vmx); 2170 2171 vmcs_write64(VMCS_LINK_POINTER, INVALID_GPA); 2172 2173 if (enable_vpid) { 2174 if (nested_cpu_has_vpid(vmcs12) && vmx->nested.vpid02) 2175 vmcs_write16(VIRTUAL_PROCESSOR_ID, vmx->nested.vpid02); 2176 else 2177 vmcs_write16(VIRTUAL_PROCESSOR_ID, vmx->vpid); 2178 } 2179 } 2180 2181 static void prepare_vmcs02_early(struct vcpu_vmx *vmx, struct loaded_vmcs *vmcs01, 2182 struct vmcs12 *vmcs12) 2183 { 2184 u32 exec_control; 2185 u64 guest_efer = nested_vmx_calc_efer(vmx, vmcs12); 2186 2187 if (vmx->nested.dirty_vmcs12 || evmptr_is_valid(vmx->nested.hv_evmcs_vmptr)) 2188 prepare_vmcs02_early_rare(vmx, vmcs12); 2189 2190 /* 2191 * PIN CONTROLS 2192 */ 2193 exec_control = __pin_controls_get(vmcs01); 2194 exec_control |= (vmcs12->pin_based_vm_exec_control & 2195 ~PIN_BASED_VMX_PREEMPTION_TIMER); 2196 2197 /* Posted interrupts setting is only taken from vmcs12. */ 2198 vmx->nested.pi_pending = false; 2199 if (nested_cpu_has_posted_intr(vmcs12)) 2200 vmx->nested.posted_intr_nv = vmcs12->posted_intr_nv; 2201 else 2202 exec_control &= ~PIN_BASED_POSTED_INTR; 2203 pin_controls_set(vmx, exec_control); 2204 2205 /* 2206 * EXEC CONTROLS 2207 */ 2208 exec_control = __exec_controls_get(vmcs01); /* L0's desires */ 2209 exec_control &= ~CPU_BASED_INTR_WINDOW_EXITING; 2210 exec_control &= ~CPU_BASED_NMI_WINDOW_EXITING; 2211 exec_control &= ~CPU_BASED_TPR_SHADOW; 2212 exec_control |= vmcs12->cpu_based_vm_exec_control; 2213 2214 vmx->nested.l1_tpr_threshold = -1; 2215 if (exec_control & CPU_BASED_TPR_SHADOW) 2216 vmcs_write32(TPR_THRESHOLD, vmcs12->tpr_threshold); 2217 #ifdef CONFIG_X86_64 2218 else 2219 exec_control |= CPU_BASED_CR8_LOAD_EXITING | 2220 CPU_BASED_CR8_STORE_EXITING; 2221 #endif 2222 2223 /* 2224 * A vmexit (to either L1 hypervisor or L0 userspace) is always needed 2225 * for I/O port accesses. 2226 */ 2227 exec_control |= CPU_BASED_UNCOND_IO_EXITING; 2228 exec_control &= ~CPU_BASED_USE_IO_BITMAPS; 2229 2230 /* 2231 * This bit will be computed in nested_get_vmcs12_pages, because 2232 * we do not have access to L1's MSR bitmap yet. For now, keep 2233 * the same bit as before, hoping to avoid multiple VMWRITEs that 2234 * only set/clear this bit. 2235 */ 2236 exec_control &= ~CPU_BASED_USE_MSR_BITMAPS; 2237 exec_control |= exec_controls_get(vmx) & CPU_BASED_USE_MSR_BITMAPS; 2238 2239 exec_controls_set(vmx, exec_control); 2240 2241 /* 2242 * SECONDARY EXEC CONTROLS 2243 */ 2244 if (cpu_has_secondary_exec_ctrls()) { 2245 exec_control = __secondary_exec_controls_get(vmcs01); 2246 2247 /* Take the following fields only from vmcs12 */ 2248 exec_control &= ~(SECONDARY_EXEC_VIRTUALIZE_APIC_ACCESSES | 2249 SECONDARY_EXEC_VIRTUALIZE_X2APIC_MODE | 2250 SECONDARY_EXEC_ENABLE_INVPCID | 2251 SECONDARY_EXEC_ENABLE_RDTSCP | 2252 SECONDARY_EXEC_XSAVES | 2253 SECONDARY_EXEC_ENABLE_USR_WAIT_PAUSE | 2254 SECONDARY_EXEC_VIRTUAL_INTR_DELIVERY | 2255 SECONDARY_EXEC_APIC_REGISTER_VIRT | 2256 SECONDARY_EXEC_ENABLE_VMFUNC | 2257 SECONDARY_EXEC_TSC_SCALING | 2258 SECONDARY_EXEC_DESC); 2259 2260 if (nested_cpu_has(vmcs12, 2261 CPU_BASED_ACTIVATE_SECONDARY_CONTROLS)) 2262 exec_control |= vmcs12->secondary_vm_exec_control; 2263 2264 /* PML is emulated and never enabled in hardware for L2. */ 2265 exec_control &= ~SECONDARY_EXEC_ENABLE_PML; 2266 2267 /* VMCS shadowing for L2 is emulated for now */ 2268 exec_control &= ~SECONDARY_EXEC_SHADOW_VMCS; 2269 2270 /* 2271 * Preset *DT exiting when emulating UMIP, so that vmx_set_cr4() 2272 * will not have to rewrite the controls just for this bit. 2273 */ 2274 if (!boot_cpu_has(X86_FEATURE_UMIP) && vmx_umip_emulated() && 2275 (vmcs12->guest_cr4 & X86_CR4_UMIP)) 2276 exec_control |= SECONDARY_EXEC_DESC; 2277 2278 if (exec_control & SECONDARY_EXEC_VIRTUAL_INTR_DELIVERY) 2279 vmcs_write16(GUEST_INTR_STATUS, 2280 vmcs12->guest_intr_status); 2281 2282 if (!nested_cpu_has2(vmcs12, SECONDARY_EXEC_UNRESTRICTED_GUEST)) 2283 exec_control &= ~SECONDARY_EXEC_UNRESTRICTED_GUEST; 2284 2285 if (exec_control & SECONDARY_EXEC_ENCLS_EXITING) 2286 vmx_write_encls_bitmap(&vmx->vcpu, vmcs12); 2287 2288 secondary_exec_controls_set(vmx, exec_control); 2289 } 2290 2291 /* 2292 * ENTRY CONTROLS 2293 * 2294 * vmcs12's VM_{ENTRY,EXIT}_LOAD_IA32_EFER and VM_ENTRY_IA32E_MODE 2295 * are emulated by vmx_set_efer() in prepare_vmcs02(), but speculate 2296 * on the related bits (if supported by the CPU) in the hope that 2297 * we can avoid VMWrites during vmx_set_efer(). 2298 */ 2299 exec_control = __vm_entry_controls_get(vmcs01); 2300 exec_control |= vmcs12->vm_entry_controls; 2301 exec_control &= ~(VM_ENTRY_IA32E_MODE | VM_ENTRY_LOAD_IA32_EFER); 2302 if (cpu_has_load_ia32_efer()) { 2303 if (guest_efer & EFER_LMA) 2304 exec_control |= VM_ENTRY_IA32E_MODE; 2305 if (guest_efer != host_efer) 2306 exec_control |= VM_ENTRY_LOAD_IA32_EFER; 2307 } 2308 vm_entry_controls_set(vmx, exec_control); 2309 2310 /* 2311 * EXIT CONTROLS 2312 * 2313 * L2->L1 exit controls are emulated - the hardware exit is to L0 so 2314 * we should use its exit controls. Note that VM_EXIT_LOAD_IA32_EFER 2315 * bits may be modified by vmx_set_efer() in prepare_vmcs02(). 2316 */ 2317 exec_control = __vm_exit_controls_get(vmcs01); 2318 if (cpu_has_load_ia32_efer() && guest_efer != host_efer) 2319 exec_control |= VM_EXIT_LOAD_IA32_EFER; 2320 else 2321 exec_control &= ~VM_EXIT_LOAD_IA32_EFER; 2322 vm_exit_controls_set(vmx, exec_control); 2323 2324 /* 2325 * Interrupt/Exception Fields 2326 */ 2327 if (vmx->nested.nested_run_pending) { 2328 vmcs_write32(VM_ENTRY_INTR_INFO_FIELD, 2329 vmcs12->vm_entry_intr_info_field); 2330 vmcs_write32(VM_ENTRY_EXCEPTION_ERROR_CODE, 2331 vmcs12->vm_entry_exception_error_code); 2332 vmcs_write32(VM_ENTRY_INSTRUCTION_LEN, 2333 vmcs12->vm_entry_instruction_len); 2334 vmcs_write32(GUEST_INTERRUPTIBILITY_INFO, 2335 vmcs12->guest_interruptibility_info); 2336 vmx->loaded_vmcs->nmi_known_unmasked = 2337 !(vmcs12->guest_interruptibility_info & GUEST_INTR_STATE_NMI); 2338 } else { 2339 vmcs_write32(VM_ENTRY_INTR_INFO_FIELD, 0); 2340 } 2341 } 2342 2343 static void prepare_vmcs02_rare(struct vcpu_vmx *vmx, struct vmcs12 *vmcs12) 2344 { 2345 struct hv_enlightened_vmcs *hv_evmcs = vmx->nested.hv_evmcs; 2346 2347 if (!hv_evmcs || !(hv_evmcs->hv_clean_fields & 2348 HV_VMX_ENLIGHTENED_CLEAN_FIELD_GUEST_GRP2)) { 2349 vmcs_write16(GUEST_ES_SELECTOR, vmcs12->guest_es_selector); 2350 vmcs_write16(GUEST_CS_SELECTOR, vmcs12->guest_cs_selector); 2351 vmcs_write16(GUEST_SS_SELECTOR, vmcs12->guest_ss_selector); 2352 vmcs_write16(GUEST_DS_SELECTOR, vmcs12->guest_ds_selector); 2353 vmcs_write16(GUEST_FS_SELECTOR, vmcs12->guest_fs_selector); 2354 vmcs_write16(GUEST_GS_SELECTOR, vmcs12->guest_gs_selector); 2355 vmcs_write16(GUEST_LDTR_SELECTOR, vmcs12->guest_ldtr_selector); 2356 vmcs_write16(GUEST_TR_SELECTOR, vmcs12->guest_tr_selector); 2357 vmcs_write32(GUEST_ES_LIMIT, vmcs12->guest_es_limit); 2358 vmcs_write32(GUEST_CS_LIMIT, vmcs12->guest_cs_limit); 2359 vmcs_write32(GUEST_SS_LIMIT, vmcs12->guest_ss_limit); 2360 vmcs_write32(GUEST_DS_LIMIT, vmcs12->guest_ds_limit); 2361 vmcs_write32(GUEST_FS_LIMIT, vmcs12->guest_fs_limit); 2362 vmcs_write32(GUEST_GS_LIMIT, vmcs12->guest_gs_limit); 2363 vmcs_write32(GUEST_LDTR_LIMIT, vmcs12->guest_ldtr_limit); 2364 vmcs_write32(GUEST_TR_LIMIT, vmcs12->guest_tr_limit); 2365 vmcs_write32(GUEST_GDTR_LIMIT, vmcs12->guest_gdtr_limit); 2366 vmcs_write32(GUEST_IDTR_LIMIT, vmcs12->guest_idtr_limit); 2367 vmcs_write32(GUEST_CS_AR_BYTES, vmcs12->guest_cs_ar_bytes); 2368 vmcs_write32(GUEST_SS_AR_BYTES, vmcs12->guest_ss_ar_bytes); 2369 vmcs_write32(GUEST_ES_AR_BYTES, vmcs12->guest_es_ar_bytes); 2370 vmcs_write32(GUEST_DS_AR_BYTES, vmcs12->guest_ds_ar_bytes); 2371 vmcs_write32(GUEST_FS_AR_BYTES, vmcs12->guest_fs_ar_bytes); 2372 vmcs_write32(GUEST_GS_AR_BYTES, vmcs12->guest_gs_ar_bytes); 2373 vmcs_write32(GUEST_LDTR_AR_BYTES, vmcs12->guest_ldtr_ar_bytes); 2374 vmcs_write32(GUEST_TR_AR_BYTES, vmcs12->guest_tr_ar_bytes); 2375 vmcs_writel(GUEST_ES_BASE, vmcs12->guest_es_base); 2376 vmcs_writel(GUEST_CS_BASE, vmcs12->guest_cs_base); 2377 vmcs_writel(GUEST_SS_BASE, vmcs12->guest_ss_base); 2378 vmcs_writel(GUEST_DS_BASE, vmcs12->guest_ds_base); 2379 vmcs_writel(GUEST_FS_BASE, vmcs12->guest_fs_base); 2380 vmcs_writel(GUEST_GS_BASE, vmcs12->guest_gs_base); 2381 vmcs_writel(GUEST_LDTR_BASE, vmcs12->guest_ldtr_base); 2382 vmcs_writel(GUEST_TR_BASE, vmcs12->guest_tr_base); 2383 vmcs_writel(GUEST_GDTR_BASE, vmcs12->guest_gdtr_base); 2384 vmcs_writel(GUEST_IDTR_BASE, vmcs12->guest_idtr_base); 2385 2386 vmx->segment_cache.bitmask = 0; 2387 } 2388 2389 if (!hv_evmcs || !(hv_evmcs->hv_clean_fields & 2390 HV_VMX_ENLIGHTENED_CLEAN_FIELD_GUEST_GRP1)) { 2391 vmcs_write32(GUEST_SYSENTER_CS, vmcs12->guest_sysenter_cs); 2392 vmcs_writel(GUEST_PENDING_DBG_EXCEPTIONS, 2393 vmcs12->guest_pending_dbg_exceptions); 2394 vmcs_writel(GUEST_SYSENTER_ESP, vmcs12->guest_sysenter_esp); 2395 vmcs_writel(GUEST_SYSENTER_EIP, vmcs12->guest_sysenter_eip); 2396 2397 /* 2398 * L1 may access the L2's PDPTR, so save them to construct 2399 * vmcs12 2400 */ 2401 if (enable_ept) { 2402 vmcs_write64(GUEST_PDPTR0, vmcs12->guest_pdptr0); 2403 vmcs_write64(GUEST_PDPTR1, vmcs12->guest_pdptr1); 2404 vmcs_write64(GUEST_PDPTR2, vmcs12->guest_pdptr2); 2405 vmcs_write64(GUEST_PDPTR3, vmcs12->guest_pdptr3); 2406 } 2407 2408 if (kvm_mpx_supported() && vmx->nested.nested_run_pending && 2409 (vmcs12->vm_entry_controls & VM_ENTRY_LOAD_BNDCFGS)) 2410 vmcs_write64(GUEST_BNDCFGS, vmcs12->guest_bndcfgs); 2411 } 2412 2413 if (nested_cpu_has_xsaves(vmcs12)) 2414 vmcs_write64(XSS_EXIT_BITMAP, vmcs12->xss_exit_bitmap); 2415 2416 /* 2417 * Whether page-faults are trapped is determined by a combination of 2418 * 3 settings: PFEC_MASK, PFEC_MATCH and EXCEPTION_BITMAP.PF. If L0 2419 * doesn't care about page faults then we should set all of these to 2420 * L1's desires. However, if L0 does care about (some) page faults, it 2421 * is not easy (if at all possible?) to merge L0 and L1's desires, we 2422 * simply ask to exit on each and every L2 page fault. This is done by 2423 * setting MASK=MATCH=0 and (see below) EB.PF=1. 2424 * Note that below we don't need special code to set EB.PF beyond the 2425 * "or"ing of the EB of vmcs01 and vmcs12, because when enable_ept, 2426 * vmcs01's EB.PF is 0 so the "or" will take vmcs12's value, and when 2427 * !enable_ept, EB.PF is 1, so the "or" will always be 1. 2428 */ 2429 if (vmx_need_pf_intercept(&vmx->vcpu)) { 2430 /* 2431 * TODO: if both L0 and L1 need the same MASK and MATCH, 2432 * go ahead and use it? 2433 */ 2434 vmcs_write32(PAGE_FAULT_ERROR_CODE_MASK, 0); 2435 vmcs_write32(PAGE_FAULT_ERROR_CODE_MATCH, 0); 2436 } else { 2437 vmcs_write32(PAGE_FAULT_ERROR_CODE_MASK, vmcs12->page_fault_error_code_mask); 2438 vmcs_write32(PAGE_FAULT_ERROR_CODE_MATCH, vmcs12->page_fault_error_code_match); 2439 } 2440 2441 if (cpu_has_vmx_apicv()) { 2442 vmcs_write64(EOI_EXIT_BITMAP0, vmcs12->eoi_exit_bitmap0); 2443 vmcs_write64(EOI_EXIT_BITMAP1, vmcs12->eoi_exit_bitmap1); 2444 vmcs_write64(EOI_EXIT_BITMAP2, vmcs12->eoi_exit_bitmap2); 2445 vmcs_write64(EOI_EXIT_BITMAP3, vmcs12->eoi_exit_bitmap3); 2446 } 2447 2448 /* 2449 * Make sure the msr_autostore list is up to date before we set the 2450 * count in the vmcs02. 2451 */ 2452 prepare_vmx_msr_autostore_list(&vmx->vcpu, MSR_IA32_TSC); 2453 2454 vmcs_write32(VM_EXIT_MSR_STORE_COUNT, vmx->msr_autostore.guest.nr); 2455 vmcs_write32(VM_EXIT_MSR_LOAD_COUNT, vmx->msr_autoload.host.nr); 2456 vmcs_write32(VM_ENTRY_MSR_LOAD_COUNT, vmx->msr_autoload.guest.nr); 2457 2458 set_cr4_guest_host_mask(vmx); 2459 } 2460 2461 /* 2462 * prepare_vmcs02 is called when the L1 guest hypervisor runs its nested 2463 * L2 guest. L1 has a vmcs for L2 (vmcs12), and this function "merges" it 2464 * with L0's requirements for its guest (a.k.a. vmcs01), so we can run the L2 2465 * guest in a way that will both be appropriate to L1's requests, and our 2466 * needs. In addition to modifying the active vmcs (which is vmcs02), this 2467 * function also has additional necessary side-effects, like setting various 2468 * vcpu->arch fields. 2469 * Returns 0 on success, 1 on failure. Invalid state exit qualification code 2470 * is assigned to entry_failure_code on failure. 2471 */ 2472 static int prepare_vmcs02(struct kvm_vcpu *vcpu, struct vmcs12 *vmcs12, 2473 bool from_vmentry, 2474 enum vm_entry_failure_code *entry_failure_code) 2475 { 2476 struct vcpu_vmx *vmx = to_vmx(vcpu); 2477 bool load_guest_pdptrs_vmcs12 = false; 2478 2479 if (vmx->nested.dirty_vmcs12 || evmptr_is_valid(vmx->nested.hv_evmcs_vmptr)) { 2480 prepare_vmcs02_rare(vmx, vmcs12); 2481 vmx->nested.dirty_vmcs12 = false; 2482 2483 load_guest_pdptrs_vmcs12 = !evmptr_is_valid(vmx->nested.hv_evmcs_vmptr) || 2484 !(vmx->nested.hv_evmcs->hv_clean_fields & 2485 HV_VMX_ENLIGHTENED_CLEAN_FIELD_GUEST_GRP1); 2486 } 2487 2488 if (vmx->nested.nested_run_pending && 2489 (vmcs12->vm_entry_controls & VM_ENTRY_LOAD_DEBUG_CONTROLS)) { 2490 kvm_set_dr(vcpu, 7, vmcs12->guest_dr7); 2491 vmcs_write64(GUEST_IA32_DEBUGCTL, vmcs12->guest_ia32_debugctl); 2492 } else { 2493 kvm_set_dr(vcpu, 7, vcpu->arch.dr7); 2494 vmcs_write64(GUEST_IA32_DEBUGCTL, vmx->nested.vmcs01_debugctl); 2495 } 2496 if (kvm_mpx_supported() && (!vmx->nested.nested_run_pending || 2497 !(vmcs12->vm_entry_controls & VM_ENTRY_LOAD_BNDCFGS))) 2498 vmcs_write64(GUEST_BNDCFGS, vmx->nested.vmcs01_guest_bndcfgs); 2499 vmx_set_rflags(vcpu, vmcs12->guest_rflags); 2500 2501 /* EXCEPTION_BITMAP and CR0_GUEST_HOST_MASK should basically be the 2502 * bitwise-or of what L1 wants to trap for L2, and what we want to 2503 * trap. Note that CR0.TS also needs updating - we do this later. 2504 */ 2505 vmx_update_exception_bitmap(vcpu); 2506 vcpu->arch.cr0_guest_owned_bits &= ~vmcs12->cr0_guest_host_mask; 2507 vmcs_writel(CR0_GUEST_HOST_MASK, ~vcpu->arch.cr0_guest_owned_bits); 2508 2509 if (vmx->nested.nested_run_pending && 2510 (vmcs12->vm_entry_controls & VM_ENTRY_LOAD_IA32_PAT)) { 2511 vmcs_write64(GUEST_IA32_PAT, vmcs12->guest_ia32_pat); 2512 vcpu->arch.pat = vmcs12->guest_ia32_pat; 2513 } else if (vmcs_config.vmentry_ctrl & VM_ENTRY_LOAD_IA32_PAT) { 2514 vmcs_write64(GUEST_IA32_PAT, vmx->vcpu.arch.pat); 2515 } 2516 2517 vcpu->arch.tsc_offset = kvm_calc_nested_tsc_offset( 2518 vcpu->arch.l1_tsc_offset, 2519 vmx_get_l2_tsc_offset(vcpu), 2520 vmx_get_l2_tsc_multiplier(vcpu)); 2521 2522 vcpu->arch.tsc_scaling_ratio = kvm_calc_nested_tsc_multiplier( 2523 vcpu->arch.l1_tsc_scaling_ratio, 2524 vmx_get_l2_tsc_multiplier(vcpu)); 2525 2526 vmcs_write64(TSC_OFFSET, vcpu->arch.tsc_offset); 2527 if (kvm_has_tsc_control) 2528 vmcs_write64(TSC_MULTIPLIER, vcpu->arch.tsc_scaling_ratio); 2529 2530 nested_vmx_transition_tlb_flush(vcpu, vmcs12, true); 2531 2532 if (nested_cpu_has_ept(vmcs12)) 2533 nested_ept_init_mmu_context(vcpu); 2534 2535 /* 2536 * This sets GUEST_CR0 to vmcs12->guest_cr0, possibly modifying those 2537 * bits which we consider mandatory enabled. 2538 * The CR0_READ_SHADOW is what L2 should have expected to read given 2539 * the specifications by L1; It's not enough to take 2540 * vmcs12->cr0_read_shadow because on our cr0_guest_host_mask we we 2541 * have more bits than L1 expected. 2542 */ 2543 vmx_set_cr0(vcpu, vmcs12->guest_cr0); 2544 vmcs_writel(CR0_READ_SHADOW, nested_read_cr0(vmcs12)); 2545 2546 vmx_set_cr4(vcpu, vmcs12->guest_cr4); 2547 vmcs_writel(CR4_READ_SHADOW, nested_read_cr4(vmcs12)); 2548 2549 vcpu->arch.efer = nested_vmx_calc_efer(vmx, vmcs12); 2550 /* Note: may modify VM_ENTRY/EXIT_CONTROLS and GUEST/HOST_IA32_EFER */ 2551 vmx_set_efer(vcpu, vcpu->arch.efer); 2552 2553 /* 2554 * Guest state is invalid and unrestricted guest is disabled, 2555 * which means L1 attempted VMEntry to L2 with invalid state. 2556 * Fail the VMEntry. 2557 * 2558 * However when force loading the guest state (SMM exit or 2559 * loading nested state after migration, it is possible to 2560 * have invalid guest state now, which will be later fixed by 2561 * restoring L2 register state 2562 */ 2563 if (CC(from_vmentry && !vmx_guest_state_valid(vcpu))) { 2564 *entry_failure_code = ENTRY_FAIL_DEFAULT; 2565 return -EINVAL; 2566 } 2567 2568 /* Shadow page tables on either EPT or shadow page tables. */ 2569 if (nested_vmx_load_cr3(vcpu, vmcs12->guest_cr3, nested_cpu_has_ept(vmcs12), 2570 from_vmentry, entry_failure_code)) 2571 return -EINVAL; 2572 2573 /* 2574 * Immediately write vmcs02.GUEST_CR3. It will be propagated to vmcs12 2575 * on nested VM-Exit, which can occur without actually running L2 and 2576 * thus without hitting vmx_load_mmu_pgd(), e.g. if L1 is entering L2 with 2577 * vmcs12.GUEST_ACTIVITYSTATE=HLT, in which case KVM will intercept the 2578 * transition to HLT instead of running L2. 2579 */ 2580 if (enable_ept) 2581 vmcs_writel(GUEST_CR3, vmcs12->guest_cr3); 2582 2583 /* Late preparation of GUEST_PDPTRs now that EFER and CRs are set. */ 2584 if (load_guest_pdptrs_vmcs12 && nested_cpu_has_ept(vmcs12) && 2585 is_pae_paging(vcpu)) { 2586 vmcs_write64(GUEST_PDPTR0, vmcs12->guest_pdptr0); 2587 vmcs_write64(GUEST_PDPTR1, vmcs12->guest_pdptr1); 2588 vmcs_write64(GUEST_PDPTR2, vmcs12->guest_pdptr2); 2589 vmcs_write64(GUEST_PDPTR3, vmcs12->guest_pdptr3); 2590 } 2591 2592 if (!enable_ept) 2593 vcpu->arch.walk_mmu->inject_page_fault = vmx_inject_page_fault_nested; 2594 2595 if ((vmcs12->vm_entry_controls & VM_ENTRY_LOAD_IA32_PERF_GLOBAL_CTRL) && 2596 WARN_ON_ONCE(kvm_set_msr(vcpu, MSR_CORE_PERF_GLOBAL_CTRL, 2597 vmcs12->guest_ia32_perf_global_ctrl))) 2598 return -EINVAL; 2599 2600 kvm_rsp_write(vcpu, vmcs12->guest_rsp); 2601 kvm_rip_write(vcpu, vmcs12->guest_rip); 2602 2603 /* 2604 * It was observed that genuine Hyper-V running in L1 doesn't reset 2605 * 'hv_clean_fields' by itself, it only sets the corresponding dirty 2606 * bits when it changes a field in eVMCS. Mark all fields as clean 2607 * here. 2608 */ 2609 if (evmptr_is_valid(vmx->nested.hv_evmcs_vmptr)) 2610 vmx->nested.hv_evmcs->hv_clean_fields |= 2611 HV_VMX_ENLIGHTENED_CLEAN_FIELD_ALL; 2612 2613 return 0; 2614 } 2615 2616 static int nested_vmx_check_nmi_controls(struct vmcs12 *vmcs12) 2617 { 2618 if (CC(!nested_cpu_has_nmi_exiting(vmcs12) && 2619 nested_cpu_has_virtual_nmis(vmcs12))) 2620 return -EINVAL; 2621 2622 if (CC(!nested_cpu_has_virtual_nmis(vmcs12) && 2623 nested_cpu_has(vmcs12, CPU_BASED_NMI_WINDOW_EXITING))) 2624 return -EINVAL; 2625 2626 return 0; 2627 } 2628 2629 static bool nested_vmx_check_eptp(struct kvm_vcpu *vcpu, u64 new_eptp) 2630 { 2631 struct vcpu_vmx *vmx = to_vmx(vcpu); 2632 2633 /* Check for memory type validity */ 2634 switch (new_eptp & VMX_EPTP_MT_MASK) { 2635 case VMX_EPTP_MT_UC: 2636 if (CC(!(vmx->nested.msrs.ept_caps & VMX_EPTP_UC_BIT))) 2637 return false; 2638 break; 2639 case VMX_EPTP_MT_WB: 2640 if (CC(!(vmx->nested.msrs.ept_caps & VMX_EPTP_WB_BIT))) 2641 return false; 2642 break; 2643 default: 2644 return false; 2645 } 2646 2647 /* Page-walk levels validity. */ 2648 switch (new_eptp & VMX_EPTP_PWL_MASK) { 2649 case VMX_EPTP_PWL_5: 2650 if (CC(!(vmx->nested.msrs.ept_caps & VMX_EPT_PAGE_WALK_5_BIT))) 2651 return false; 2652 break; 2653 case VMX_EPTP_PWL_4: 2654 if (CC(!(vmx->nested.msrs.ept_caps & VMX_EPT_PAGE_WALK_4_BIT))) 2655 return false; 2656 break; 2657 default: 2658 return false; 2659 } 2660 2661 /* Reserved bits should not be set */ 2662 if (CC(kvm_vcpu_is_illegal_gpa(vcpu, new_eptp) || ((new_eptp >> 7) & 0x1f))) 2663 return false; 2664 2665 /* AD, if set, should be supported */ 2666 if (new_eptp & VMX_EPTP_AD_ENABLE_BIT) { 2667 if (CC(!(vmx->nested.msrs.ept_caps & VMX_EPT_AD_BIT))) 2668 return false; 2669 } 2670 2671 return true; 2672 } 2673 2674 /* 2675 * Checks related to VM-Execution Control Fields 2676 */ 2677 static int nested_check_vm_execution_controls(struct kvm_vcpu *vcpu, 2678 struct vmcs12 *vmcs12) 2679 { 2680 struct vcpu_vmx *vmx = to_vmx(vcpu); 2681 2682 if (CC(!vmx_control_verify(vmcs12->pin_based_vm_exec_control, 2683 vmx->nested.msrs.pinbased_ctls_low, 2684 vmx->nested.msrs.pinbased_ctls_high)) || 2685 CC(!vmx_control_verify(vmcs12->cpu_based_vm_exec_control, 2686 vmx->nested.msrs.procbased_ctls_low, 2687 vmx->nested.msrs.procbased_ctls_high))) 2688 return -EINVAL; 2689 2690 if (nested_cpu_has(vmcs12, CPU_BASED_ACTIVATE_SECONDARY_CONTROLS) && 2691 CC(!vmx_control_verify(vmcs12->secondary_vm_exec_control, 2692 vmx->nested.msrs.secondary_ctls_low, 2693 vmx->nested.msrs.secondary_ctls_high))) 2694 return -EINVAL; 2695 2696 if (CC(vmcs12->cr3_target_count > nested_cpu_vmx_misc_cr3_count(vcpu)) || 2697 nested_vmx_check_io_bitmap_controls(vcpu, vmcs12) || 2698 nested_vmx_check_msr_bitmap_controls(vcpu, vmcs12) || 2699 nested_vmx_check_tpr_shadow_controls(vcpu, vmcs12) || 2700 nested_vmx_check_apic_access_controls(vcpu, vmcs12) || 2701 nested_vmx_check_apicv_controls(vcpu, vmcs12) || 2702 nested_vmx_check_nmi_controls(vmcs12) || 2703 nested_vmx_check_pml_controls(vcpu, vmcs12) || 2704 nested_vmx_check_unrestricted_guest_controls(vcpu, vmcs12) || 2705 nested_vmx_check_mode_based_ept_exec_controls(vcpu, vmcs12) || 2706 nested_vmx_check_shadow_vmcs_controls(vcpu, vmcs12) || 2707 CC(nested_cpu_has_vpid(vmcs12) && !vmcs12->virtual_processor_id)) 2708 return -EINVAL; 2709 2710 if (!nested_cpu_has_preemption_timer(vmcs12) && 2711 nested_cpu_has_save_preemption_timer(vmcs12)) 2712 return -EINVAL; 2713 2714 if (nested_cpu_has_ept(vmcs12) && 2715 CC(!nested_vmx_check_eptp(vcpu, vmcs12->ept_pointer))) 2716 return -EINVAL; 2717 2718 if (nested_cpu_has_vmfunc(vmcs12)) { 2719 if (CC(vmcs12->vm_function_control & 2720 ~vmx->nested.msrs.vmfunc_controls)) 2721 return -EINVAL; 2722 2723 if (nested_cpu_has_eptp_switching(vmcs12)) { 2724 if (CC(!nested_cpu_has_ept(vmcs12)) || 2725 CC(!page_address_valid(vcpu, vmcs12->eptp_list_address))) 2726 return -EINVAL; 2727 } 2728 } 2729 2730 return 0; 2731 } 2732 2733 /* 2734 * Checks related to VM-Exit Control Fields 2735 */ 2736 static int nested_check_vm_exit_controls(struct kvm_vcpu *vcpu, 2737 struct vmcs12 *vmcs12) 2738 { 2739 struct vcpu_vmx *vmx = to_vmx(vcpu); 2740 2741 if (CC(!vmx_control_verify(vmcs12->vm_exit_controls, 2742 vmx->nested.msrs.exit_ctls_low, 2743 vmx->nested.msrs.exit_ctls_high)) || 2744 CC(nested_vmx_check_exit_msr_switch_controls(vcpu, vmcs12))) 2745 return -EINVAL; 2746 2747 return 0; 2748 } 2749 2750 /* 2751 * Checks related to VM-Entry Control Fields 2752 */ 2753 static int nested_check_vm_entry_controls(struct kvm_vcpu *vcpu, 2754 struct vmcs12 *vmcs12) 2755 { 2756 struct vcpu_vmx *vmx = to_vmx(vcpu); 2757 2758 if (CC(!vmx_control_verify(vmcs12->vm_entry_controls, 2759 vmx->nested.msrs.entry_ctls_low, 2760 vmx->nested.msrs.entry_ctls_high))) 2761 return -EINVAL; 2762 2763 /* 2764 * From the Intel SDM, volume 3: 2765 * Fields relevant to VM-entry event injection must be set properly. 2766 * These fields are the VM-entry interruption-information field, the 2767 * VM-entry exception error code, and the VM-entry instruction length. 2768 */ 2769 if (vmcs12->vm_entry_intr_info_field & INTR_INFO_VALID_MASK) { 2770 u32 intr_info = vmcs12->vm_entry_intr_info_field; 2771 u8 vector = intr_info & INTR_INFO_VECTOR_MASK; 2772 u32 intr_type = intr_info & INTR_INFO_INTR_TYPE_MASK; 2773 bool has_error_code = intr_info & INTR_INFO_DELIVER_CODE_MASK; 2774 bool should_have_error_code; 2775 bool urg = nested_cpu_has2(vmcs12, 2776 SECONDARY_EXEC_UNRESTRICTED_GUEST); 2777 bool prot_mode = !urg || vmcs12->guest_cr0 & X86_CR0_PE; 2778 2779 /* VM-entry interruption-info field: interruption type */ 2780 if (CC(intr_type == INTR_TYPE_RESERVED) || 2781 CC(intr_type == INTR_TYPE_OTHER_EVENT && 2782 !nested_cpu_supports_monitor_trap_flag(vcpu))) 2783 return -EINVAL; 2784 2785 /* VM-entry interruption-info field: vector */ 2786 if (CC(intr_type == INTR_TYPE_NMI_INTR && vector != NMI_VECTOR) || 2787 CC(intr_type == INTR_TYPE_HARD_EXCEPTION && vector > 31) || 2788 CC(intr_type == INTR_TYPE_OTHER_EVENT && vector != 0)) 2789 return -EINVAL; 2790 2791 /* VM-entry interruption-info field: deliver error code */ 2792 should_have_error_code = 2793 intr_type == INTR_TYPE_HARD_EXCEPTION && prot_mode && 2794 x86_exception_has_error_code(vector); 2795 if (CC(has_error_code != should_have_error_code)) 2796 return -EINVAL; 2797 2798 /* VM-entry exception error code */ 2799 if (CC(has_error_code && 2800 vmcs12->vm_entry_exception_error_code & GENMASK(31, 16))) 2801 return -EINVAL; 2802 2803 /* VM-entry interruption-info field: reserved bits */ 2804 if (CC(intr_info & INTR_INFO_RESVD_BITS_MASK)) 2805 return -EINVAL; 2806 2807 /* VM-entry instruction length */ 2808 switch (intr_type) { 2809 case INTR_TYPE_SOFT_EXCEPTION: 2810 case INTR_TYPE_SOFT_INTR: 2811 case INTR_TYPE_PRIV_SW_EXCEPTION: 2812 if (CC(vmcs12->vm_entry_instruction_len > 15) || 2813 CC(vmcs12->vm_entry_instruction_len == 0 && 2814 CC(!nested_cpu_has_zero_length_injection(vcpu)))) 2815 return -EINVAL; 2816 } 2817 } 2818 2819 if (nested_vmx_check_entry_msr_switch_controls(vcpu, vmcs12)) 2820 return -EINVAL; 2821 2822 return 0; 2823 } 2824 2825 static int nested_vmx_check_controls(struct kvm_vcpu *vcpu, 2826 struct vmcs12 *vmcs12) 2827 { 2828 if (nested_check_vm_execution_controls(vcpu, vmcs12) || 2829 nested_check_vm_exit_controls(vcpu, vmcs12) || 2830 nested_check_vm_entry_controls(vcpu, vmcs12)) 2831 return -EINVAL; 2832 2833 if (to_vmx(vcpu)->nested.enlightened_vmcs_enabled) 2834 return nested_evmcs_check_controls(vmcs12); 2835 2836 return 0; 2837 } 2838 2839 static int nested_vmx_check_address_space_size(struct kvm_vcpu *vcpu, 2840 struct vmcs12 *vmcs12) 2841 { 2842 #ifdef CONFIG_X86_64 2843 if (CC(!!(vmcs12->vm_exit_controls & VM_EXIT_HOST_ADDR_SPACE_SIZE) != 2844 !!(vcpu->arch.efer & EFER_LMA))) 2845 return -EINVAL; 2846 #endif 2847 return 0; 2848 } 2849 2850 static int nested_vmx_check_host_state(struct kvm_vcpu *vcpu, 2851 struct vmcs12 *vmcs12) 2852 { 2853 bool ia32e; 2854 2855 if (CC(!nested_host_cr0_valid(vcpu, vmcs12->host_cr0)) || 2856 CC(!nested_host_cr4_valid(vcpu, vmcs12->host_cr4)) || 2857 CC(kvm_vcpu_is_illegal_gpa(vcpu, vmcs12->host_cr3))) 2858 return -EINVAL; 2859 2860 if (CC(is_noncanonical_address(vmcs12->host_ia32_sysenter_esp, vcpu)) || 2861 CC(is_noncanonical_address(vmcs12->host_ia32_sysenter_eip, vcpu))) 2862 return -EINVAL; 2863 2864 if ((vmcs12->vm_exit_controls & VM_EXIT_LOAD_IA32_PAT) && 2865 CC(!kvm_pat_valid(vmcs12->host_ia32_pat))) 2866 return -EINVAL; 2867 2868 if ((vmcs12->vm_exit_controls & VM_EXIT_LOAD_IA32_PERF_GLOBAL_CTRL) && 2869 CC(!kvm_valid_perf_global_ctrl(vcpu_to_pmu(vcpu), 2870 vmcs12->host_ia32_perf_global_ctrl))) 2871 return -EINVAL; 2872 2873 #ifdef CONFIG_X86_64 2874 ia32e = !!(vmcs12->vm_exit_controls & VM_EXIT_HOST_ADDR_SPACE_SIZE); 2875 #else 2876 ia32e = false; 2877 #endif 2878 2879 if (ia32e) { 2880 if (CC(!(vmcs12->host_cr4 & X86_CR4_PAE))) 2881 return -EINVAL; 2882 } else { 2883 if (CC(vmcs12->vm_entry_controls & VM_ENTRY_IA32E_MODE) || 2884 CC(vmcs12->host_cr4 & X86_CR4_PCIDE) || 2885 CC((vmcs12->host_rip) >> 32)) 2886 return -EINVAL; 2887 } 2888 2889 if (CC(vmcs12->host_cs_selector & (SEGMENT_RPL_MASK | SEGMENT_TI_MASK)) || 2890 CC(vmcs12->host_ss_selector & (SEGMENT_RPL_MASK | SEGMENT_TI_MASK)) || 2891 CC(vmcs12->host_ds_selector & (SEGMENT_RPL_MASK | SEGMENT_TI_MASK)) || 2892 CC(vmcs12->host_es_selector & (SEGMENT_RPL_MASK | SEGMENT_TI_MASK)) || 2893 CC(vmcs12->host_fs_selector & (SEGMENT_RPL_MASK | SEGMENT_TI_MASK)) || 2894 CC(vmcs12->host_gs_selector & (SEGMENT_RPL_MASK | SEGMENT_TI_MASK)) || 2895 CC(vmcs12->host_tr_selector & (SEGMENT_RPL_MASK | SEGMENT_TI_MASK)) || 2896 CC(vmcs12->host_cs_selector == 0) || 2897 CC(vmcs12->host_tr_selector == 0) || 2898 CC(vmcs12->host_ss_selector == 0 && !ia32e)) 2899 return -EINVAL; 2900 2901 if (CC(is_noncanonical_address(vmcs12->host_fs_base, vcpu)) || 2902 CC(is_noncanonical_address(vmcs12->host_gs_base, vcpu)) || 2903 CC(is_noncanonical_address(vmcs12->host_gdtr_base, vcpu)) || 2904 CC(is_noncanonical_address(vmcs12->host_idtr_base, vcpu)) || 2905 CC(is_noncanonical_address(vmcs12->host_tr_base, vcpu)) || 2906 CC(is_noncanonical_address(vmcs12->host_rip, vcpu))) 2907 return -EINVAL; 2908 2909 /* 2910 * If the load IA32_EFER VM-exit control is 1, bits reserved in the 2911 * IA32_EFER MSR must be 0 in the field for that register. In addition, 2912 * the values of the LMA and LME bits in the field must each be that of 2913 * the host address-space size VM-exit control. 2914 */ 2915 if (vmcs12->vm_exit_controls & VM_EXIT_LOAD_IA32_EFER) { 2916 if (CC(!kvm_valid_efer(vcpu, vmcs12->host_ia32_efer)) || 2917 CC(ia32e != !!(vmcs12->host_ia32_efer & EFER_LMA)) || 2918 CC(ia32e != !!(vmcs12->host_ia32_efer & EFER_LME))) 2919 return -EINVAL; 2920 } 2921 2922 return 0; 2923 } 2924 2925 static int nested_vmx_check_vmcs_link_ptr(struct kvm_vcpu *vcpu, 2926 struct vmcs12 *vmcs12) 2927 { 2928 struct vcpu_vmx *vmx = to_vmx(vcpu); 2929 struct gfn_to_hva_cache *ghc = &vmx->nested.shadow_vmcs12_cache; 2930 struct vmcs_hdr hdr; 2931 2932 if (vmcs12->vmcs_link_pointer == INVALID_GPA) 2933 return 0; 2934 2935 if (CC(!page_address_valid(vcpu, vmcs12->vmcs_link_pointer))) 2936 return -EINVAL; 2937 2938 if (ghc->gpa != vmcs12->vmcs_link_pointer && 2939 CC(kvm_gfn_to_hva_cache_init(vcpu->kvm, ghc, 2940 vmcs12->vmcs_link_pointer, VMCS12_SIZE))) 2941 return -EINVAL; 2942 2943 if (CC(kvm_read_guest_offset_cached(vcpu->kvm, ghc, &hdr, 2944 offsetof(struct vmcs12, hdr), 2945 sizeof(hdr)))) 2946 return -EINVAL; 2947 2948 if (CC(hdr.revision_id != VMCS12_REVISION) || 2949 CC(hdr.shadow_vmcs != nested_cpu_has_shadow_vmcs(vmcs12))) 2950 return -EINVAL; 2951 2952 return 0; 2953 } 2954 2955 /* 2956 * Checks related to Guest Non-register State 2957 */ 2958 static int nested_check_guest_non_reg_state(struct vmcs12 *vmcs12) 2959 { 2960 if (CC(vmcs12->guest_activity_state != GUEST_ACTIVITY_ACTIVE && 2961 vmcs12->guest_activity_state != GUEST_ACTIVITY_HLT && 2962 vmcs12->guest_activity_state != GUEST_ACTIVITY_WAIT_SIPI)) 2963 return -EINVAL; 2964 2965 return 0; 2966 } 2967 2968 static int nested_vmx_check_guest_state(struct kvm_vcpu *vcpu, 2969 struct vmcs12 *vmcs12, 2970 enum vm_entry_failure_code *entry_failure_code) 2971 { 2972 bool ia32e; 2973 2974 *entry_failure_code = ENTRY_FAIL_DEFAULT; 2975 2976 if (CC(!nested_guest_cr0_valid(vcpu, vmcs12->guest_cr0)) || 2977 CC(!nested_guest_cr4_valid(vcpu, vmcs12->guest_cr4))) 2978 return -EINVAL; 2979 2980 if ((vmcs12->vm_entry_controls & VM_ENTRY_LOAD_DEBUG_CONTROLS) && 2981 CC(!kvm_dr7_valid(vmcs12->guest_dr7))) 2982 return -EINVAL; 2983 2984 if ((vmcs12->vm_entry_controls & VM_ENTRY_LOAD_IA32_PAT) && 2985 CC(!kvm_pat_valid(vmcs12->guest_ia32_pat))) 2986 return -EINVAL; 2987 2988 if (nested_vmx_check_vmcs_link_ptr(vcpu, vmcs12)) { 2989 *entry_failure_code = ENTRY_FAIL_VMCS_LINK_PTR; 2990 return -EINVAL; 2991 } 2992 2993 if ((vmcs12->vm_entry_controls & VM_ENTRY_LOAD_IA32_PERF_GLOBAL_CTRL) && 2994 CC(!kvm_valid_perf_global_ctrl(vcpu_to_pmu(vcpu), 2995 vmcs12->guest_ia32_perf_global_ctrl))) 2996 return -EINVAL; 2997 2998 /* 2999 * If the load IA32_EFER VM-entry control is 1, the following checks 3000 * are performed on the field for the IA32_EFER MSR: 3001 * - Bits reserved in the IA32_EFER MSR must be 0. 3002 * - Bit 10 (corresponding to IA32_EFER.LMA) must equal the value of 3003 * the IA-32e mode guest VM-exit control. It must also be identical 3004 * to bit 8 (LME) if bit 31 in the CR0 field (corresponding to 3005 * CR0.PG) is 1. 3006 */ 3007 if (to_vmx(vcpu)->nested.nested_run_pending && 3008 (vmcs12->vm_entry_controls & VM_ENTRY_LOAD_IA32_EFER)) { 3009 ia32e = (vmcs12->vm_entry_controls & VM_ENTRY_IA32E_MODE) != 0; 3010 if (CC(!kvm_valid_efer(vcpu, vmcs12->guest_ia32_efer)) || 3011 CC(ia32e != !!(vmcs12->guest_ia32_efer & EFER_LMA)) || 3012 CC(((vmcs12->guest_cr0 & X86_CR0_PG) && 3013 ia32e != !!(vmcs12->guest_ia32_efer & EFER_LME)))) 3014 return -EINVAL; 3015 } 3016 3017 if ((vmcs12->vm_entry_controls & VM_ENTRY_LOAD_BNDCFGS) && 3018 (CC(is_noncanonical_address(vmcs12->guest_bndcfgs & PAGE_MASK, vcpu)) || 3019 CC((vmcs12->guest_bndcfgs & MSR_IA32_BNDCFGS_RSVD)))) 3020 return -EINVAL; 3021 3022 if (nested_check_guest_non_reg_state(vmcs12)) 3023 return -EINVAL; 3024 3025 return 0; 3026 } 3027 3028 static int nested_vmx_check_vmentry_hw(struct kvm_vcpu *vcpu) 3029 { 3030 struct vcpu_vmx *vmx = to_vmx(vcpu); 3031 unsigned long cr3, cr4; 3032 bool vm_fail; 3033 3034 if (!nested_early_check) 3035 return 0; 3036 3037 if (vmx->msr_autoload.host.nr) 3038 vmcs_write32(VM_EXIT_MSR_LOAD_COUNT, 0); 3039 if (vmx->msr_autoload.guest.nr) 3040 vmcs_write32(VM_ENTRY_MSR_LOAD_COUNT, 0); 3041 3042 preempt_disable(); 3043 3044 vmx_prepare_switch_to_guest(vcpu); 3045 3046 /* 3047 * Induce a consistency check VMExit by clearing bit 1 in GUEST_RFLAGS, 3048 * which is reserved to '1' by hardware. GUEST_RFLAGS is guaranteed to 3049 * be written (by prepare_vmcs02()) before the "real" VMEnter, i.e. 3050 * there is no need to preserve other bits or save/restore the field. 3051 */ 3052 vmcs_writel(GUEST_RFLAGS, 0); 3053 3054 cr3 = __get_current_cr3_fast(); 3055 if (unlikely(cr3 != vmx->loaded_vmcs->host_state.cr3)) { 3056 vmcs_writel(HOST_CR3, cr3); 3057 vmx->loaded_vmcs->host_state.cr3 = cr3; 3058 } 3059 3060 cr4 = cr4_read_shadow(); 3061 if (unlikely(cr4 != vmx->loaded_vmcs->host_state.cr4)) { 3062 vmcs_writel(HOST_CR4, cr4); 3063 vmx->loaded_vmcs->host_state.cr4 = cr4; 3064 } 3065 3066 vm_fail = __vmx_vcpu_run(vmx, (unsigned long *)&vcpu->arch.regs, 3067 vmx->loaded_vmcs->launched); 3068 3069 if (vmx->msr_autoload.host.nr) 3070 vmcs_write32(VM_EXIT_MSR_LOAD_COUNT, vmx->msr_autoload.host.nr); 3071 if (vmx->msr_autoload.guest.nr) 3072 vmcs_write32(VM_ENTRY_MSR_LOAD_COUNT, vmx->msr_autoload.guest.nr); 3073 3074 if (vm_fail) { 3075 u32 error = vmcs_read32(VM_INSTRUCTION_ERROR); 3076 3077 preempt_enable(); 3078 3079 trace_kvm_nested_vmenter_failed( 3080 "early hardware check VM-instruction error: ", error); 3081 WARN_ON_ONCE(error != VMXERR_ENTRY_INVALID_CONTROL_FIELD); 3082 return 1; 3083 } 3084 3085 /* 3086 * VMExit clears RFLAGS.IF and DR7, even on a consistency check. 3087 */ 3088 if (hw_breakpoint_active()) 3089 set_debugreg(__this_cpu_read(cpu_dr7), 7); 3090 local_irq_enable(); 3091 preempt_enable(); 3092 3093 /* 3094 * A non-failing VMEntry means we somehow entered guest mode with 3095 * an illegal RIP, and that's just the tip of the iceberg. There 3096 * is no telling what memory has been modified or what state has 3097 * been exposed to unknown code. Hitting this all but guarantees 3098 * a (very critical) hardware issue. 3099 */ 3100 WARN_ON(!(vmcs_read32(VM_EXIT_REASON) & 3101 VMX_EXIT_REASONS_FAILED_VMENTRY)); 3102 3103 return 0; 3104 } 3105 3106 static bool nested_get_evmcs_page(struct kvm_vcpu *vcpu) 3107 { 3108 struct vcpu_vmx *vmx = to_vmx(vcpu); 3109 3110 /* 3111 * hv_evmcs may end up being not mapped after migration (when 3112 * L2 was running), map it here to make sure vmcs12 changes are 3113 * properly reflected. 3114 */ 3115 if (vmx->nested.enlightened_vmcs_enabled && 3116 vmx->nested.hv_evmcs_vmptr == EVMPTR_MAP_PENDING) { 3117 enum nested_evmptrld_status evmptrld_status = 3118 nested_vmx_handle_enlightened_vmptrld(vcpu, false); 3119 3120 if (evmptrld_status == EVMPTRLD_VMFAIL || 3121 evmptrld_status == EVMPTRLD_ERROR) 3122 return false; 3123 3124 /* 3125 * Post migration VMCS12 always provides the most actual 3126 * information, copy it to eVMCS upon entry. 3127 */ 3128 vmx->nested.need_vmcs12_to_shadow_sync = true; 3129 } 3130 3131 return true; 3132 } 3133 3134 static bool nested_get_vmcs12_pages(struct kvm_vcpu *vcpu) 3135 { 3136 struct vmcs12 *vmcs12 = get_vmcs12(vcpu); 3137 struct vcpu_vmx *vmx = to_vmx(vcpu); 3138 struct kvm_host_map *map; 3139 struct page *page; 3140 u64 hpa; 3141 3142 if (!vcpu->arch.pdptrs_from_userspace && 3143 !nested_cpu_has_ept(vmcs12) && is_pae_paging(vcpu)) { 3144 /* 3145 * Reload the guest's PDPTRs since after a migration 3146 * the guest CR3 might be restored prior to setting the nested 3147 * state which can lead to a load of wrong PDPTRs. 3148 */ 3149 if (CC(!load_pdptrs(vcpu, vcpu->arch.walk_mmu, vcpu->arch.cr3))) 3150 return false; 3151 } 3152 3153 3154 if (nested_cpu_has2(vmcs12, SECONDARY_EXEC_VIRTUALIZE_APIC_ACCESSES)) { 3155 /* 3156 * Translate L1 physical address to host physical 3157 * address for vmcs02. Keep the page pinned, so this 3158 * physical address remains valid. We keep a reference 3159 * to it so we can release it later. 3160 */ 3161 if (vmx->nested.apic_access_page) { /* shouldn't happen */ 3162 kvm_release_page_clean(vmx->nested.apic_access_page); 3163 vmx->nested.apic_access_page = NULL; 3164 } 3165 page = kvm_vcpu_gpa_to_page(vcpu, vmcs12->apic_access_addr); 3166 if (!is_error_page(page)) { 3167 vmx->nested.apic_access_page = page; 3168 hpa = page_to_phys(vmx->nested.apic_access_page); 3169 vmcs_write64(APIC_ACCESS_ADDR, hpa); 3170 } else { 3171 pr_debug_ratelimited("%s: no backing 'struct page' for APIC-access address in vmcs12\n", 3172 __func__); 3173 vcpu->run->exit_reason = KVM_EXIT_INTERNAL_ERROR; 3174 vcpu->run->internal.suberror = 3175 KVM_INTERNAL_ERROR_EMULATION; 3176 vcpu->run->internal.ndata = 0; 3177 return false; 3178 } 3179 } 3180 3181 if (nested_cpu_has(vmcs12, CPU_BASED_TPR_SHADOW)) { 3182 map = &vmx->nested.virtual_apic_map; 3183 3184 if (!kvm_vcpu_map(vcpu, gpa_to_gfn(vmcs12->virtual_apic_page_addr), map)) { 3185 vmcs_write64(VIRTUAL_APIC_PAGE_ADDR, pfn_to_hpa(map->pfn)); 3186 } else if (nested_cpu_has(vmcs12, CPU_BASED_CR8_LOAD_EXITING) && 3187 nested_cpu_has(vmcs12, CPU_BASED_CR8_STORE_EXITING) && 3188 !nested_cpu_has2(vmcs12, SECONDARY_EXEC_VIRTUALIZE_APIC_ACCESSES)) { 3189 /* 3190 * The processor will never use the TPR shadow, simply 3191 * clear the bit from the execution control. Such a 3192 * configuration is useless, but it happens in tests. 3193 * For any other configuration, failing the vm entry is 3194 * _not_ what the processor does but it's basically the 3195 * only possibility we have. 3196 */ 3197 exec_controls_clearbit(vmx, CPU_BASED_TPR_SHADOW); 3198 } else { 3199 /* 3200 * Write an illegal value to VIRTUAL_APIC_PAGE_ADDR to 3201 * force VM-Entry to fail. 3202 */ 3203 vmcs_write64(VIRTUAL_APIC_PAGE_ADDR, INVALID_GPA); 3204 } 3205 } 3206 3207 if (nested_cpu_has_posted_intr(vmcs12)) { 3208 map = &vmx->nested.pi_desc_map; 3209 3210 if (!kvm_vcpu_map(vcpu, gpa_to_gfn(vmcs12->posted_intr_desc_addr), map)) { 3211 vmx->nested.pi_desc = 3212 (struct pi_desc *)(((void *)map->hva) + 3213 offset_in_page(vmcs12->posted_intr_desc_addr)); 3214 vmcs_write64(POSTED_INTR_DESC_ADDR, 3215 pfn_to_hpa(map->pfn) + offset_in_page(vmcs12->posted_intr_desc_addr)); 3216 } else { 3217 /* 3218 * Defer the KVM_INTERNAL_EXIT until KVM tries to 3219 * access the contents of the VMCS12 posted interrupt 3220 * descriptor. (Note that KVM may do this when it 3221 * should not, per the architectural specification.) 3222 */ 3223 vmx->nested.pi_desc = NULL; 3224 pin_controls_clearbit(vmx, PIN_BASED_POSTED_INTR); 3225 } 3226 } 3227 if (nested_vmx_prepare_msr_bitmap(vcpu, vmcs12)) 3228 exec_controls_setbit(vmx, CPU_BASED_USE_MSR_BITMAPS); 3229 else 3230 exec_controls_clearbit(vmx, CPU_BASED_USE_MSR_BITMAPS); 3231 3232 return true; 3233 } 3234 3235 static bool vmx_get_nested_state_pages(struct kvm_vcpu *vcpu) 3236 { 3237 if (!nested_get_evmcs_page(vcpu)) { 3238 pr_debug_ratelimited("%s: enlightened vmptrld failed\n", 3239 __func__); 3240 vcpu->run->exit_reason = KVM_EXIT_INTERNAL_ERROR; 3241 vcpu->run->internal.suberror = 3242 KVM_INTERNAL_ERROR_EMULATION; 3243 vcpu->run->internal.ndata = 0; 3244 3245 return false; 3246 } 3247 3248 if (is_guest_mode(vcpu) && !nested_get_vmcs12_pages(vcpu)) 3249 return false; 3250 3251 return true; 3252 } 3253 3254 static int nested_vmx_write_pml_buffer(struct kvm_vcpu *vcpu, gpa_t gpa) 3255 { 3256 struct vmcs12 *vmcs12; 3257 struct vcpu_vmx *vmx = to_vmx(vcpu); 3258 gpa_t dst; 3259 3260 if (WARN_ON_ONCE(!is_guest_mode(vcpu))) 3261 return 0; 3262 3263 if (WARN_ON_ONCE(vmx->nested.pml_full)) 3264 return 1; 3265 3266 /* 3267 * Check if PML is enabled for the nested guest. Whether eptp bit 6 is 3268 * set is already checked as part of A/D emulation. 3269 */ 3270 vmcs12 = get_vmcs12(vcpu); 3271 if (!nested_cpu_has_pml(vmcs12)) 3272 return 0; 3273 3274 if (vmcs12->guest_pml_index >= PML_ENTITY_NUM) { 3275 vmx->nested.pml_full = true; 3276 return 1; 3277 } 3278 3279 gpa &= ~0xFFFull; 3280 dst = vmcs12->pml_address + sizeof(u64) * vmcs12->guest_pml_index; 3281 3282 if (kvm_write_guest_page(vcpu->kvm, gpa_to_gfn(dst), &gpa, 3283 offset_in_page(dst), sizeof(gpa))) 3284 return 0; 3285 3286 vmcs12->guest_pml_index--; 3287 3288 return 0; 3289 } 3290 3291 /* 3292 * Intel's VMX Instruction Reference specifies a common set of prerequisites 3293 * for running VMX instructions (except VMXON, whose prerequisites are 3294 * slightly different). It also specifies what exception to inject otherwise. 3295 * Note that many of these exceptions have priority over VM exits, so they 3296 * don't have to be checked again here. 3297 */ 3298 static int nested_vmx_check_permission(struct kvm_vcpu *vcpu) 3299 { 3300 if (!to_vmx(vcpu)->nested.vmxon) { 3301 kvm_queue_exception(vcpu, UD_VECTOR); 3302 return 0; 3303 } 3304 3305 if (vmx_get_cpl(vcpu)) { 3306 kvm_inject_gp(vcpu, 0); 3307 return 0; 3308 } 3309 3310 return 1; 3311 } 3312 3313 static u8 vmx_has_apicv_interrupt(struct kvm_vcpu *vcpu) 3314 { 3315 u8 rvi = vmx_get_rvi(); 3316 u8 vppr = kvm_lapic_get_reg(vcpu->arch.apic, APIC_PROCPRI); 3317 3318 return ((rvi & 0xf0) > (vppr & 0xf0)); 3319 } 3320 3321 static void load_vmcs12_host_state(struct kvm_vcpu *vcpu, 3322 struct vmcs12 *vmcs12); 3323 3324 /* 3325 * If from_vmentry is false, this is being called from state restore (either RSM 3326 * or KVM_SET_NESTED_STATE). Otherwise it's called from vmlaunch/vmresume. 3327 * 3328 * Returns: 3329 * NVMX_VMENTRY_SUCCESS: Entered VMX non-root mode 3330 * NVMX_VMENTRY_VMFAIL: Consistency check VMFail 3331 * NVMX_VMENTRY_VMEXIT: Consistency check VMExit 3332 * NVMX_VMENTRY_KVM_INTERNAL_ERROR: KVM internal error 3333 */ 3334 enum nvmx_vmentry_status nested_vmx_enter_non_root_mode(struct kvm_vcpu *vcpu, 3335 bool from_vmentry) 3336 { 3337 struct vcpu_vmx *vmx = to_vmx(vcpu); 3338 struct vmcs12 *vmcs12 = get_vmcs12(vcpu); 3339 enum vm_entry_failure_code entry_failure_code; 3340 bool evaluate_pending_interrupts; 3341 union vmx_exit_reason exit_reason = { 3342 .basic = EXIT_REASON_INVALID_STATE, 3343 .failed_vmentry = 1, 3344 }; 3345 u32 failed_index; 3346 3347 if (kvm_check_request(KVM_REQ_TLB_FLUSH_CURRENT, vcpu)) 3348 kvm_vcpu_flush_tlb_current(vcpu); 3349 3350 evaluate_pending_interrupts = exec_controls_get(vmx) & 3351 (CPU_BASED_INTR_WINDOW_EXITING | CPU_BASED_NMI_WINDOW_EXITING); 3352 if (likely(!evaluate_pending_interrupts) && kvm_vcpu_apicv_active(vcpu)) 3353 evaluate_pending_interrupts |= vmx_has_apicv_interrupt(vcpu); 3354 3355 if (!(vmcs12->vm_entry_controls & VM_ENTRY_LOAD_DEBUG_CONTROLS)) 3356 vmx->nested.vmcs01_debugctl = vmcs_read64(GUEST_IA32_DEBUGCTL); 3357 if (kvm_mpx_supported() && 3358 !(vmcs12->vm_entry_controls & VM_ENTRY_LOAD_BNDCFGS)) 3359 vmx->nested.vmcs01_guest_bndcfgs = vmcs_read64(GUEST_BNDCFGS); 3360 3361 /* 3362 * Overwrite vmcs01.GUEST_CR3 with L1's CR3 if EPT is disabled *and* 3363 * nested early checks are disabled. In the event of a "late" VM-Fail, 3364 * i.e. a VM-Fail detected by hardware but not KVM, KVM must unwind its 3365 * software model to the pre-VMEntry host state. When EPT is disabled, 3366 * GUEST_CR3 holds KVM's shadow CR3, not L1's "real" CR3, which causes 3367 * nested_vmx_restore_host_state() to corrupt vcpu->arch.cr3. Stuffing 3368 * vmcs01.GUEST_CR3 results in the unwind naturally setting arch.cr3 to 3369 * the correct value. Smashing vmcs01.GUEST_CR3 is safe because nested 3370 * VM-Exits, and the unwind, reset KVM's MMU, i.e. vmcs01.GUEST_CR3 is 3371 * guaranteed to be overwritten with a shadow CR3 prior to re-entering 3372 * L1. Don't stuff vmcs01.GUEST_CR3 when using nested early checks as 3373 * KVM modifies vcpu->arch.cr3 if and only if the early hardware checks 3374 * pass, and early VM-Fails do not reset KVM's MMU, i.e. the VM-Fail 3375 * path would need to manually save/restore vmcs01.GUEST_CR3. 3376 */ 3377 if (!enable_ept && !nested_early_check) 3378 vmcs_writel(GUEST_CR3, vcpu->arch.cr3); 3379 3380 vmx_switch_vmcs(vcpu, &vmx->nested.vmcs02); 3381 3382 prepare_vmcs02_early(vmx, &vmx->vmcs01, vmcs12); 3383 3384 if (from_vmentry) { 3385 if (unlikely(!nested_get_vmcs12_pages(vcpu))) { 3386 vmx_switch_vmcs(vcpu, &vmx->vmcs01); 3387 return NVMX_VMENTRY_KVM_INTERNAL_ERROR; 3388 } 3389 3390 if (nested_vmx_check_vmentry_hw(vcpu)) { 3391 vmx_switch_vmcs(vcpu, &vmx->vmcs01); 3392 return NVMX_VMENTRY_VMFAIL; 3393 } 3394 3395 if (nested_vmx_check_guest_state(vcpu, vmcs12, 3396 &entry_failure_code)) { 3397 exit_reason.basic = EXIT_REASON_INVALID_STATE; 3398 vmcs12->exit_qualification = entry_failure_code; 3399 goto vmentry_fail_vmexit; 3400 } 3401 } 3402 3403 enter_guest_mode(vcpu); 3404 3405 if (prepare_vmcs02(vcpu, vmcs12, from_vmentry, &entry_failure_code)) { 3406 exit_reason.basic = EXIT_REASON_INVALID_STATE; 3407 vmcs12->exit_qualification = entry_failure_code; 3408 goto vmentry_fail_vmexit_guest_mode; 3409 } 3410 3411 if (from_vmentry) { 3412 failed_index = nested_vmx_load_msr(vcpu, 3413 vmcs12->vm_entry_msr_load_addr, 3414 vmcs12->vm_entry_msr_load_count); 3415 if (failed_index) { 3416 exit_reason.basic = EXIT_REASON_MSR_LOAD_FAIL; 3417 vmcs12->exit_qualification = failed_index; 3418 goto vmentry_fail_vmexit_guest_mode; 3419 } 3420 } else { 3421 /* 3422 * The MMU is not initialized to point at the right entities yet and 3423 * "get pages" would need to read data from the guest (i.e. we will 3424 * need to perform gpa to hpa translation). Request a call 3425 * to nested_get_vmcs12_pages before the next VM-entry. The MSRs 3426 * have already been set at vmentry time and should not be reset. 3427 */ 3428 kvm_make_request(KVM_REQ_GET_NESTED_STATE_PAGES, vcpu); 3429 } 3430 3431 /* 3432 * If L1 had a pending IRQ/NMI until it executed 3433 * VMLAUNCH/VMRESUME which wasn't delivered because it was 3434 * disallowed (e.g. interrupts disabled), L0 needs to 3435 * evaluate if this pending event should cause an exit from L2 3436 * to L1 or delivered directly to L2 (e.g. In case L1 don't 3437 * intercept EXTERNAL_INTERRUPT). 3438 * 3439 * Usually this would be handled by the processor noticing an 3440 * IRQ/NMI window request, or checking RVI during evaluation of 3441 * pending virtual interrupts. However, this setting was done 3442 * on VMCS01 and now VMCS02 is active instead. Thus, we force L0 3443 * to perform pending event evaluation by requesting a KVM_REQ_EVENT. 3444 */ 3445 if (unlikely(evaluate_pending_interrupts)) 3446 kvm_make_request(KVM_REQ_EVENT, vcpu); 3447 3448 /* 3449 * Do not start the preemption timer hrtimer until after we know 3450 * we are successful, so that only nested_vmx_vmexit needs to cancel 3451 * the timer. 3452 */ 3453 vmx->nested.preemption_timer_expired = false; 3454 if (nested_cpu_has_preemption_timer(vmcs12)) { 3455 u64 timer_value = vmx_calc_preemption_timer_value(vcpu); 3456 vmx_start_preemption_timer(vcpu, timer_value); 3457 } 3458 3459 /* 3460 * Note no nested_vmx_succeed or nested_vmx_fail here. At this point 3461 * we are no longer running L1, and VMLAUNCH/VMRESUME has not yet 3462 * returned as far as L1 is concerned. It will only return (and set 3463 * the success flag) when L2 exits (see nested_vmx_vmexit()). 3464 */ 3465 return NVMX_VMENTRY_SUCCESS; 3466 3467 /* 3468 * A failed consistency check that leads to a VMExit during L1's 3469 * VMEnter to L2 is a variation of a normal VMexit, as explained in 3470 * 26.7 "VM-entry failures during or after loading guest state". 3471 */ 3472 vmentry_fail_vmexit_guest_mode: 3473 if (vmcs12->cpu_based_vm_exec_control & CPU_BASED_USE_TSC_OFFSETTING) 3474 vcpu->arch.tsc_offset -= vmcs12->tsc_offset; 3475 leave_guest_mode(vcpu); 3476 3477 vmentry_fail_vmexit: 3478 vmx_switch_vmcs(vcpu, &vmx->vmcs01); 3479 3480 if (!from_vmentry) 3481 return NVMX_VMENTRY_VMEXIT; 3482 3483 load_vmcs12_host_state(vcpu, vmcs12); 3484 vmcs12->vm_exit_reason = exit_reason.full; 3485 if (enable_shadow_vmcs || evmptr_is_valid(vmx->nested.hv_evmcs_vmptr)) 3486 vmx->nested.need_vmcs12_to_shadow_sync = true; 3487 return NVMX_VMENTRY_VMEXIT; 3488 } 3489 3490 /* 3491 * nested_vmx_run() handles a nested entry, i.e., a VMLAUNCH or VMRESUME on L1 3492 * for running an L2 nested guest. 3493 */ 3494 static int nested_vmx_run(struct kvm_vcpu *vcpu, bool launch) 3495 { 3496 struct vmcs12 *vmcs12; 3497 enum nvmx_vmentry_status status; 3498 struct vcpu_vmx *vmx = to_vmx(vcpu); 3499 u32 interrupt_shadow = vmx_get_interrupt_shadow(vcpu); 3500 enum nested_evmptrld_status evmptrld_status; 3501 3502 if (!nested_vmx_check_permission(vcpu)) 3503 return 1; 3504 3505 evmptrld_status = nested_vmx_handle_enlightened_vmptrld(vcpu, launch); 3506 if (evmptrld_status == EVMPTRLD_ERROR) { 3507 kvm_queue_exception(vcpu, UD_VECTOR); 3508 return 1; 3509 } else if (CC(evmptrld_status == EVMPTRLD_VMFAIL)) { 3510 return nested_vmx_failInvalid(vcpu); 3511 } 3512 3513 if (CC(!evmptr_is_valid(vmx->nested.hv_evmcs_vmptr) && 3514 vmx->nested.current_vmptr == INVALID_GPA)) 3515 return nested_vmx_failInvalid(vcpu); 3516 3517 vmcs12 = get_vmcs12(vcpu); 3518 3519 /* 3520 * Can't VMLAUNCH or VMRESUME a shadow VMCS. Despite the fact 3521 * that there *is* a valid VMCS pointer, RFLAGS.CF is set 3522 * rather than RFLAGS.ZF, and no error number is stored to the 3523 * VM-instruction error field. 3524 */ 3525 if (CC(vmcs12->hdr.shadow_vmcs)) 3526 return nested_vmx_failInvalid(vcpu); 3527 3528 if (evmptr_is_valid(vmx->nested.hv_evmcs_vmptr)) { 3529 copy_enlightened_to_vmcs12(vmx, vmx->nested.hv_evmcs->hv_clean_fields); 3530 /* Enlightened VMCS doesn't have launch state */ 3531 vmcs12->launch_state = !launch; 3532 } else if (enable_shadow_vmcs) { 3533 copy_shadow_to_vmcs12(vmx); 3534 } 3535 3536 /* 3537 * The nested entry process starts with enforcing various prerequisites 3538 * on vmcs12 as required by the Intel SDM, and act appropriately when 3539 * they fail: As the SDM explains, some conditions should cause the 3540 * instruction to fail, while others will cause the instruction to seem 3541 * to succeed, but return an EXIT_REASON_INVALID_STATE. 3542 * To speed up the normal (success) code path, we should avoid checking 3543 * for misconfigurations which will anyway be caught by the processor 3544 * when using the merged vmcs02. 3545 */ 3546 if (CC(interrupt_shadow & KVM_X86_SHADOW_INT_MOV_SS)) 3547 return nested_vmx_fail(vcpu, VMXERR_ENTRY_EVENTS_BLOCKED_BY_MOV_SS); 3548 3549 if (CC(vmcs12->launch_state == launch)) 3550 return nested_vmx_fail(vcpu, 3551 launch ? VMXERR_VMLAUNCH_NONCLEAR_VMCS 3552 : VMXERR_VMRESUME_NONLAUNCHED_VMCS); 3553 3554 if (nested_vmx_check_controls(vcpu, vmcs12)) 3555 return nested_vmx_fail(vcpu, VMXERR_ENTRY_INVALID_CONTROL_FIELD); 3556 3557 if (nested_vmx_check_address_space_size(vcpu, vmcs12)) 3558 return nested_vmx_fail(vcpu, VMXERR_ENTRY_INVALID_HOST_STATE_FIELD); 3559 3560 if (nested_vmx_check_host_state(vcpu, vmcs12)) 3561 return nested_vmx_fail(vcpu, VMXERR_ENTRY_INVALID_HOST_STATE_FIELD); 3562 3563 /* 3564 * We're finally done with prerequisite checking, and can start with 3565 * the nested entry. 3566 */ 3567 vmx->nested.nested_run_pending = 1; 3568 vmx->nested.has_preemption_timer_deadline = false; 3569 status = nested_vmx_enter_non_root_mode(vcpu, true); 3570 if (unlikely(status != NVMX_VMENTRY_SUCCESS)) 3571 goto vmentry_failed; 3572 3573 /* Emulate processing of posted interrupts on VM-Enter. */ 3574 if (nested_cpu_has_posted_intr(vmcs12) && 3575 kvm_apic_has_interrupt(vcpu) == vmx->nested.posted_intr_nv) { 3576 vmx->nested.pi_pending = true; 3577 kvm_make_request(KVM_REQ_EVENT, vcpu); 3578 kvm_apic_clear_irr(vcpu, vmx->nested.posted_intr_nv); 3579 } 3580 3581 /* Hide L1D cache contents from the nested guest. */ 3582 vmx->vcpu.arch.l1tf_flush_l1d = true; 3583 3584 /* 3585 * Must happen outside of nested_vmx_enter_non_root_mode() as it will 3586 * also be used as part of restoring nVMX state for 3587 * snapshot restore (migration). 3588 * 3589 * In this flow, it is assumed that vmcs12 cache was 3590 * transferred as part of captured nVMX state and should 3591 * therefore not be read from guest memory (which may not 3592 * exist on destination host yet). 3593 */ 3594 nested_cache_shadow_vmcs12(vcpu, vmcs12); 3595 3596 switch (vmcs12->guest_activity_state) { 3597 case GUEST_ACTIVITY_HLT: 3598 /* 3599 * If we're entering a halted L2 vcpu and the L2 vcpu won't be 3600 * awakened by event injection or by an NMI-window VM-exit or 3601 * by an interrupt-window VM-exit, halt the vcpu. 3602 */ 3603 if (!(vmcs12->vm_entry_intr_info_field & INTR_INFO_VALID_MASK) && 3604 !nested_cpu_has(vmcs12, CPU_BASED_NMI_WINDOW_EXITING) && 3605 !(nested_cpu_has(vmcs12, CPU_BASED_INTR_WINDOW_EXITING) && 3606 (vmcs12->guest_rflags & X86_EFLAGS_IF))) { 3607 vmx->nested.nested_run_pending = 0; 3608 return kvm_vcpu_halt(vcpu); 3609 } 3610 break; 3611 case GUEST_ACTIVITY_WAIT_SIPI: 3612 vmx->nested.nested_run_pending = 0; 3613 vcpu->arch.mp_state = KVM_MP_STATE_INIT_RECEIVED; 3614 break; 3615 default: 3616 break; 3617 } 3618 3619 return 1; 3620 3621 vmentry_failed: 3622 vmx->nested.nested_run_pending = 0; 3623 if (status == NVMX_VMENTRY_KVM_INTERNAL_ERROR) 3624 return 0; 3625 if (status == NVMX_VMENTRY_VMEXIT) 3626 return 1; 3627 WARN_ON_ONCE(status != NVMX_VMENTRY_VMFAIL); 3628 return nested_vmx_fail(vcpu, VMXERR_ENTRY_INVALID_CONTROL_FIELD); 3629 } 3630 3631 /* 3632 * On a nested exit from L2 to L1, vmcs12.guest_cr0 might not be up-to-date 3633 * because L2 may have changed some cr0 bits directly (CR0_GUEST_HOST_MASK). 3634 * This function returns the new value we should put in vmcs12.guest_cr0. 3635 * It's not enough to just return the vmcs02 GUEST_CR0. Rather, 3636 * 1. Bits that neither L0 nor L1 trapped, were set directly by L2 and are now 3637 * available in vmcs02 GUEST_CR0. (Note: It's enough to check that L0 3638 * didn't trap the bit, because if L1 did, so would L0). 3639 * 2. Bits that L1 asked to trap (and therefore L0 also did) could not have 3640 * been modified by L2, and L1 knows it. So just leave the old value of 3641 * the bit from vmcs12.guest_cr0. Note that the bit from vmcs02 GUEST_CR0 3642 * isn't relevant, because if L0 traps this bit it can set it to anything. 3643 * 3. Bits that L1 didn't trap, but L0 did. L1 believes the guest could have 3644 * changed these bits, and therefore they need to be updated, but L0 3645 * didn't necessarily allow them to be changed in GUEST_CR0 - and rather 3646 * put them in vmcs02 CR0_READ_SHADOW. So take these bits from there. 3647 */ 3648 static inline unsigned long 3649 vmcs12_guest_cr0(struct kvm_vcpu *vcpu, struct vmcs12 *vmcs12) 3650 { 3651 return 3652 /*1*/ (vmcs_readl(GUEST_CR0) & vcpu->arch.cr0_guest_owned_bits) | 3653 /*2*/ (vmcs12->guest_cr0 & vmcs12->cr0_guest_host_mask) | 3654 /*3*/ (vmcs_readl(CR0_READ_SHADOW) & ~(vmcs12->cr0_guest_host_mask | 3655 vcpu->arch.cr0_guest_owned_bits)); 3656 } 3657 3658 static inline unsigned long 3659 vmcs12_guest_cr4(struct kvm_vcpu *vcpu, struct vmcs12 *vmcs12) 3660 { 3661 return 3662 /*1*/ (vmcs_readl(GUEST_CR4) & vcpu->arch.cr4_guest_owned_bits) | 3663 /*2*/ (vmcs12->guest_cr4 & vmcs12->cr4_guest_host_mask) | 3664 /*3*/ (vmcs_readl(CR4_READ_SHADOW) & ~(vmcs12->cr4_guest_host_mask | 3665 vcpu->arch.cr4_guest_owned_bits)); 3666 } 3667 3668 static void vmcs12_save_pending_event(struct kvm_vcpu *vcpu, 3669 struct vmcs12 *vmcs12) 3670 { 3671 u32 idt_vectoring; 3672 unsigned int nr; 3673 3674 if (vcpu->arch.exception.injected) { 3675 nr = vcpu->arch.exception.nr; 3676 idt_vectoring = nr | VECTORING_INFO_VALID_MASK; 3677 3678 if (kvm_exception_is_soft(nr)) { 3679 vmcs12->vm_exit_instruction_len = 3680 vcpu->arch.event_exit_inst_len; 3681 idt_vectoring |= INTR_TYPE_SOFT_EXCEPTION; 3682 } else 3683 idt_vectoring |= INTR_TYPE_HARD_EXCEPTION; 3684 3685 if (vcpu->arch.exception.has_error_code) { 3686 idt_vectoring |= VECTORING_INFO_DELIVER_CODE_MASK; 3687 vmcs12->idt_vectoring_error_code = 3688 vcpu->arch.exception.error_code; 3689 } 3690 3691 vmcs12->idt_vectoring_info_field = idt_vectoring; 3692 } else if (vcpu->arch.nmi_injected) { 3693 vmcs12->idt_vectoring_info_field = 3694 INTR_TYPE_NMI_INTR | INTR_INFO_VALID_MASK | NMI_VECTOR; 3695 } else if (vcpu->arch.interrupt.injected) { 3696 nr = vcpu->arch.interrupt.nr; 3697 idt_vectoring = nr | VECTORING_INFO_VALID_MASK; 3698 3699 if (vcpu->arch.interrupt.soft) { 3700 idt_vectoring |= INTR_TYPE_SOFT_INTR; 3701 vmcs12->vm_entry_instruction_len = 3702 vcpu->arch.event_exit_inst_len; 3703 } else 3704 idt_vectoring |= INTR_TYPE_EXT_INTR; 3705 3706 vmcs12->idt_vectoring_info_field = idt_vectoring; 3707 } 3708 } 3709 3710 3711 void nested_mark_vmcs12_pages_dirty(struct kvm_vcpu *vcpu) 3712 { 3713 struct vmcs12 *vmcs12 = get_vmcs12(vcpu); 3714 gfn_t gfn; 3715 3716 /* 3717 * Don't need to mark the APIC access page dirty; it is never 3718 * written to by the CPU during APIC virtualization. 3719 */ 3720 3721 if (nested_cpu_has(vmcs12, CPU_BASED_TPR_SHADOW)) { 3722 gfn = vmcs12->virtual_apic_page_addr >> PAGE_SHIFT; 3723 kvm_vcpu_mark_page_dirty(vcpu, gfn); 3724 } 3725 3726 if (nested_cpu_has_posted_intr(vmcs12)) { 3727 gfn = vmcs12->posted_intr_desc_addr >> PAGE_SHIFT; 3728 kvm_vcpu_mark_page_dirty(vcpu, gfn); 3729 } 3730 } 3731 3732 static int vmx_complete_nested_posted_interrupt(struct kvm_vcpu *vcpu) 3733 { 3734 struct vcpu_vmx *vmx = to_vmx(vcpu); 3735 int max_irr; 3736 void *vapic_page; 3737 u16 status; 3738 3739 if (!vmx->nested.pi_pending) 3740 return 0; 3741 3742 if (!vmx->nested.pi_desc) 3743 goto mmio_needed; 3744 3745 vmx->nested.pi_pending = false; 3746 3747 if (!pi_test_and_clear_on(vmx->nested.pi_desc)) 3748 return 0; 3749 3750 max_irr = find_last_bit((unsigned long *)vmx->nested.pi_desc->pir, 256); 3751 if (max_irr != 256) { 3752 vapic_page = vmx->nested.virtual_apic_map.hva; 3753 if (!vapic_page) 3754 goto mmio_needed; 3755 3756 __kvm_apic_update_irr(vmx->nested.pi_desc->pir, 3757 vapic_page, &max_irr); 3758 status = vmcs_read16(GUEST_INTR_STATUS); 3759 if ((u8)max_irr > ((u8)status & 0xff)) { 3760 status &= ~0xff; 3761 status |= (u8)max_irr; 3762 vmcs_write16(GUEST_INTR_STATUS, status); 3763 } 3764 } 3765 3766 nested_mark_vmcs12_pages_dirty(vcpu); 3767 return 0; 3768 3769 mmio_needed: 3770 kvm_handle_memory_failure(vcpu, X86EMUL_IO_NEEDED, NULL); 3771 return -ENXIO; 3772 } 3773 3774 static void nested_vmx_inject_exception_vmexit(struct kvm_vcpu *vcpu, 3775 unsigned long exit_qual) 3776 { 3777 struct vmcs12 *vmcs12 = get_vmcs12(vcpu); 3778 unsigned int nr = vcpu->arch.exception.nr; 3779 u32 intr_info = nr | INTR_INFO_VALID_MASK; 3780 3781 if (vcpu->arch.exception.has_error_code) { 3782 vmcs12->vm_exit_intr_error_code = vcpu->arch.exception.error_code; 3783 intr_info |= INTR_INFO_DELIVER_CODE_MASK; 3784 } 3785 3786 if (kvm_exception_is_soft(nr)) 3787 intr_info |= INTR_TYPE_SOFT_EXCEPTION; 3788 else 3789 intr_info |= INTR_TYPE_HARD_EXCEPTION; 3790 3791 if (!(vmcs12->idt_vectoring_info_field & VECTORING_INFO_VALID_MASK) && 3792 vmx_get_nmi_mask(vcpu)) 3793 intr_info |= INTR_INFO_UNBLOCK_NMI; 3794 3795 nested_vmx_vmexit(vcpu, EXIT_REASON_EXCEPTION_NMI, intr_info, exit_qual); 3796 } 3797 3798 /* 3799 * Returns true if a debug trap is pending delivery. 3800 * 3801 * In KVM, debug traps bear an exception payload. As such, the class of a #DB 3802 * exception may be inferred from the presence of an exception payload. 3803 */ 3804 static inline bool vmx_pending_dbg_trap(struct kvm_vcpu *vcpu) 3805 { 3806 return vcpu->arch.exception.pending && 3807 vcpu->arch.exception.nr == DB_VECTOR && 3808 vcpu->arch.exception.payload; 3809 } 3810 3811 /* 3812 * Certain VM-exits set the 'pending debug exceptions' field to indicate a 3813 * recognized #DB (data or single-step) that has yet to be delivered. Since KVM 3814 * represents these debug traps with a payload that is said to be compatible 3815 * with the 'pending debug exceptions' field, write the payload to the VMCS 3816 * field if a VM-exit is delivered before the debug trap. 3817 */ 3818 static void nested_vmx_update_pending_dbg(struct kvm_vcpu *vcpu) 3819 { 3820 if (vmx_pending_dbg_trap(vcpu)) 3821 vmcs_writel(GUEST_PENDING_DBG_EXCEPTIONS, 3822 vcpu->arch.exception.payload); 3823 } 3824 3825 static bool nested_vmx_preemption_timer_pending(struct kvm_vcpu *vcpu) 3826 { 3827 return nested_cpu_has_preemption_timer(get_vmcs12(vcpu)) && 3828 to_vmx(vcpu)->nested.preemption_timer_expired; 3829 } 3830 3831 static int vmx_check_nested_events(struct kvm_vcpu *vcpu) 3832 { 3833 struct vcpu_vmx *vmx = to_vmx(vcpu); 3834 unsigned long exit_qual; 3835 bool block_nested_events = 3836 vmx->nested.nested_run_pending || kvm_event_needs_reinjection(vcpu); 3837 bool mtf_pending = vmx->nested.mtf_pending; 3838 struct kvm_lapic *apic = vcpu->arch.apic; 3839 3840 /* 3841 * Clear the MTF state. If a higher priority VM-exit is delivered first, 3842 * this state is discarded. 3843 */ 3844 if (!block_nested_events) 3845 vmx->nested.mtf_pending = false; 3846 3847 if (lapic_in_kernel(vcpu) && 3848 test_bit(KVM_APIC_INIT, &apic->pending_events)) { 3849 if (block_nested_events) 3850 return -EBUSY; 3851 nested_vmx_update_pending_dbg(vcpu); 3852 clear_bit(KVM_APIC_INIT, &apic->pending_events); 3853 if (vcpu->arch.mp_state != KVM_MP_STATE_INIT_RECEIVED) 3854 nested_vmx_vmexit(vcpu, EXIT_REASON_INIT_SIGNAL, 0, 0); 3855 return 0; 3856 } 3857 3858 if (lapic_in_kernel(vcpu) && 3859 test_bit(KVM_APIC_SIPI, &apic->pending_events)) { 3860 if (block_nested_events) 3861 return -EBUSY; 3862 3863 clear_bit(KVM_APIC_SIPI, &apic->pending_events); 3864 if (vcpu->arch.mp_state == KVM_MP_STATE_INIT_RECEIVED) 3865 nested_vmx_vmexit(vcpu, EXIT_REASON_SIPI_SIGNAL, 0, 3866 apic->sipi_vector & 0xFFUL); 3867 return 0; 3868 } 3869 3870 /* 3871 * Process any exceptions that are not debug traps before MTF. 3872 * 3873 * Note that only a pending nested run can block a pending exception. 3874 * Otherwise an injected NMI/interrupt should either be 3875 * lost or delivered to the nested hypervisor in the IDT_VECTORING_INFO, 3876 * while delivering the pending exception. 3877 */ 3878 3879 if (vcpu->arch.exception.pending && !vmx_pending_dbg_trap(vcpu)) { 3880 if (vmx->nested.nested_run_pending) 3881 return -EBUSY; 3882 if (!nested_vmx_check_exception(vcpu, &exit_qual)) 3883 goto no_vmexit; 3884 nested_vmx_inject_exception_vmexit(vcpu, exit_qual); 3885 return 0; 3886 } 3887 3888 if (mtf_pending) { 3889 if (block_nested_events) 3890 return -EBUSY; 3891 nested_vmx_update_pending_dbg(vcpu); 3892 nested_vmx_vmexit(vcpu, EXIT_REASON_MONITOR_TRAP_FLAG, 0, 0); 3893 return 0; 3894 } 3895 3896 if (vcpu->arch.exception.pending) { 3897 if (vmx->nested.nested_run_pending) 3898 return -EBUSY; 3899 if (!nested_vmx_check_exception(vcpu, &exit_qual)) 3900 goto no_vmexit; 3901 nested_vmx_inject_exception_vmexit(vcpu, exit_qual); 3902 return 0; 3903 } 3904 3905 if (nested_vmx_preemption_timer_pending(vcpu)) { 3906 if (block_nested_events) 3907 return -EBUSY; 3908 nested_vmx_vmexit(vcpu, EXIT_REASON_PREEMPTION_TIMER, 0, 0); 3909 return 0; 3910 } 3911 3912 if (vcpu->arch.smi_pending && !is_smm(vcpu)) { 3913 if (block_nested_events) 3914 return -EBUSY; 3915 goto no_vmexit; 3916 } 3917 3918 if (vcpu->arch.nmi_pending && !vmx_nmi_blocked(vcpu)) { 3919 if (block_nested_events) 3920 return -EBUSY; 3921 if (!nested_exit_on_nmi(vcpu)) 3922 goto no_vmexit; 3923 3924 nested_vmx_vmexit(vcpu, EXIT_REASON_EXCEPTION_NMI, 3925 NMI_VECTOR | INTR_TYPE_NMI_INTR | 3926 INTR_INFO_VALID_MASK, 0); 3927 /* 3928 * The NMI-triggered VM exit counts as injection: 3929 * clear this one and block further NMIs. 3930 */ 3931 vcpu->arch.nmi_pending = 0; 3932 vmx_set_nmi_mask(vcpu, true); 3933 return 0; 3934 } 3935 3936 if (kvm_cpu_has_interrupt(vcpu) && !vmx_interrupt_blocked(vcpu)) { 3937 if (block_nested_events) 3938 return -EBUSY; 3939 if (!nested_exit_on_intr(vcpu)) 3940 goto no_vmexit; 3941 nested_vmx_vmexit(vcpu, EXIT_REASON_EXTERNAL_INTERRUPT, 0, 0); 3942 return 0; 3943 } 3944 3945 no_vmexit: 3946 return vmx_complete_nested_posted_interrupt(vcpu); 3947 } 3948 3949 static u32 vmx_get_preemption_timer_value(struct kvm_vcpu *vcpu) 3950 { 3951 ktime_t remaining = 3952 hrtimer_get_remaining(&to_vmx(vcpu)->nested.preemption_timer); 3953 u64 value; 3954 3955 if (ktime_to_ns(remaining) <= 0) 3956 return 0; 3957 3958 value = ktime_to_ns(remaining) * vcpu->arch.virtual_tsc_khz; 3959 do_div(value, 1000000); 3960 return value >> VMX_MISC_EMULATED_PREEMPTION_TIMER_RATE; 3961 } 3962 3963 static bool is_vmcs12_ext_field(unsigned long field) 3964 { 3965 switch (field) { 3966 case GUEST_ES_SELECTOR: 3967 case GUEST_CS_SELECTOR: 3968 case GUEST_SS_SELECTOR: 3969 case GUEST_DS_SELECTOR: 3970 case GUEST_FS_SELECTOR: 3971 case GUEST_GS_SELECTOR: 3972 case GUEST_LDTR_SELECTOR: 3973 case GUEST_TR_SELECTOR: 3974 case GUEST_ES_LIMIT: 3975 case GUEST_CS_LIMIT: 3976 case GUEST_SS_LIMIT: 3977 case GUEST_DS_LIMIT: 3978 case GUEST_FS_LIMIT: 3979 case GUEST_GS_LIMIT: 3980 case GUEST_LDTR_LIMIT: 3981 case GUEST_TR_LIMIT: 3982 case GUEST_GDTR_LIMIT: 3983 case GUEST_IDTR_LIMIT: 3984 case GUEST_ES_AR_BYTES: 3985 case GUEST_DS_AR_BYTES: 3986 case GUEST_FS_AR_BYTES: 3987 case GUEST_GS_AR_BYTES: 3988 case GUEST_LDTR_AR_BYTES: 3989 case GUEST_TR_AR_BYTES: 3990 case GUEST_ES_BASE: 3991 case GUEST_CS_BASE: 3992 case GUEST_SS_BASE: 3993 case GUEST_DS_BASE: 3994 case GUEST_FS_BASE: 3995 case GUEST_GS_BASE: 3996 case GUEST_LDTR_BASE: 3997 case GUEST_TR_BASE: 3998 case GUEST_GDTR_BASE: 3999 case GUEST_IDTR_BASE: 4000 case GUEST_PENDING_DBG_EXCEPTIONS: 4001 case GUEST_BNDCFGS: 4002 return true; 4003 default: 4004 break; 4005 } 4006 4007 return false; 4008 } 4009 4010 static void sync_vmcs02_to_vmcs12_rare(struct kvm_vcpu *vcpu, 4011 struct vmcs12 *vmcs12) 4012 { 4013 struct vcpu_vmx *vmx = to_vmx(vcpu); 4014 4015 vmcs12->guest_es_selector = vmcs_read16(GUEST_ES_SELECTOR); 4016 vmcs12->guest_cs_selector = vmcs_read16(GUEST_CS_SELECTOR); 4017 vmcs12->guest_ss_selector = vmcs_read16(GUEST_SS_SELECTOR); 4018 vmcs12->guest_ds_selector = vmcs_read16(GUEST_DS_SELECTOR); 4019 vmcs12->guest_fs_selector = vmcs_read16(GUEST_FS_SELECTOR); 4020 vmcs12->guest_gs_selector = vmcs_read16(GUEST_GS_SELECTOR); 4021 vmcs12->guest_ldtr_selector = vmcs_read16(GUEST_LDTR_SELECTOR); 4022 vmcs12->guest_tr_selector = vmcs_read16(GUEST_TR_SELECTOR); 4023 vmcs12->guest_es_limit = vmcs_read32(GUEST_ES_LIMIT); 4024 vmcs12->guest_cs_limit = vmcs_read32(GUEST_CS_LIMIT); 4025 vmcs12->guest_ss_limit = vmcs_read32(GUEST_SS_LIMIT); 4026 vmcs12->guest_ds_limit = vmcs_read32(GUEST_DS_LIMIT); 4027 vmcs12->guest_fs_limit = vmcs_read32(GUEST_FS_LIMIT); 4028 vmcs12->guest_gs_limit = vmcs_read32(GUEST_GS_LIMIT); 4029 vmcs12->guest_ldtr_limit = vmcs_read32(GUEST_LDTR_LIMIT); 4030 vmcs12->guest_tr_limit = vmcs_read32(GUEST_TR_LIMIT); 4031 vmcs12->guest_gdtr_limit = vmcs_read32(GUEST_GDTR_LIMIT); 4032 vmcs12->guest_idtr_limit = vmcs_read32(GUEST_IDTR_LIMIT); 4033 vmcs12->guest_es_ar_bytes = vmcs_read32(GUEST_ES_AR_BYTES); 4034 vmcs12->guest_ds_ar_bytes = vmcs_read32(GUEST_DS_AR_BYTES); 4035 vmcs12->guest_fs_ar_bytes = vmcs_read32(GUEST_FS_AR_BYTES); 4036 vmcs12->guest_gs_ar_bytes = vmcs_read32(GUEST_GS_AR_BYTES); 4037 vmcs12->guest_ldtr_ar_bytes = vmcs_read32(GUEST_LDTR_AR_BYTES); 4038 vmcs12->guest_tr_ar_bytes = vmcs_read32(GUEST_TR_AR_BYTES); 4039 vmcs12->guest_es_base = vmcs_readl(GUEST_ES_BASE); 4040 vmcs12->guest_cs_base = vmcs_readl(GUEST_CS_BASE); 4041 vmcs12->guest_ss_base = vmcs_readl(GUEST_SS_BASE); 4042 vmcs12->guest_ds_base = vmcs_readl(GUEST_DS_BASE); 4043 vmcs12->guest_fs_base = vmcs_readl(GUEST_FS_BASE); 4044 vmcs12->guest_gs_base = vmcs_readl(GUEST_GS_BASE); 4045 vmcs12->guest_ldtr_base = vmcs_readl(GUEST_LDTR_BASE); 4046 vmcs12->guest_tr_base = vmcs_readl(GUEST_TR_BASE); 4047 vmcs12->guest_gdtr_base = vmcs_readl(GUEST_GDTR_BASE); 4048 vmcs12->guest_idtr_base = vmcs_readl(GUEST_IDTR_BASE); 4049 vmcs12->guest_pending_dbg_exceptions = 4050 vmcs_readl(GUEST_PENDING_DBG_EXCEPTIONS); 4051 if (kvm_mpx_supported()) 4052 vmcs12->guest_bndcfgs = vmcs_read64(GUEST_BNDCFGS); 4053 4054 vmx->nested.need_sync_vmcs02_to_vmcs12_rare = false; 4055 } 4056 4057 static void copy_vmcs02_to_vmcs12_rare(struct kvm_vcpu *vcpu, 4058 struct vmcs12 *vmcs12) 4059 { 4060 struct vcpu_vmx *vmx = to_vmx(vcpu); 4061 int cpu; 4062 4063 if (!vmx->nested.need_sync_vmcs02_to_vmcs12_rare) 4064 return; 4065 4066 4067 WARN_ON_ONCE(vmx->loaded_vmcs != &vmx->vmcs01); 4068 4069 cpu = get_cpu(); 4070 vmx->loaded_vmcs = &vmx->nested.vmcs02; 4071 vmx_vcpu_load_vmcs(vcpu, cpu, &vmx->vmcs01); 4072 4073 sync_vmcs02_to_vmcs12_rare(vcpu, vmcs12); 4074 4075 vmx->loaded_vmcs = &vmx->vmcs01; 4076 vmx_vcpu_load_vmcs(vcpu, cpu, &vmx->nested.vmcs02); 4077 put_cpu(); 4078 } 4079 4080 /* 4081 * Update the guest state fields of vmcs12 to reflect changes that 4082 * occurred while L2 was running. (The "IA-32e mode guest" bit of the 4083 * VM-entry controls is also updated, since this is really a guest 4084 * state bit.) 4085 */ 4086 static void sync_vmcs02_to_vmcs12(struct kvm_vcpu *vcpu, struct vmcs12 *vmcs12) 4087 { 4088 struct vcpu_vmx *vmx = to_vmx(vcpu); 4089 4090 if (evmptr_is_valid(vmx->nested.hv_evmcs_vmptr)) 4091 sync_vmcs02_to_vmcs12_rare(vcpu, vmcs12); 4092 4093 vmx->nested.need_sync_vmcs02_to_vmcs12_rare = 4094 !evmptr_is_valid(vmx->nested.hv_evmcs_vmptr); 4095 4096 vmcs12->guest_cr0 = vmcs12_guest_cr0(vcpu, vmcs12); 4097 vmcs12->guest_cr4 = vmcs12_guest_cr4(vcpu, vmcs12); 4098 4099 vmcs12->guest_rsp = kvm_rsp_read(vcpu); 4100 vmcs12->guest_rip = kvm_rip_read(vcpu); 4101 vmcs12->guest_rflags = vmcs_readl(GUEST_RFLAGS); 4102 4103 vmcs12->guest_cs_ar_bytes = vmcs_read32(GUEST_CS_AR_BYTES); 4104 vmcs12->guest_ss_ar_bytes = vmcs_read32(GUEST_SS_AR_BYTES); 4105 4106 vmcs12->guest_interruptibility_info = 4107 vmcs_read32(GUEST_INTERRUPTIBILITY_INFO); 4108 4109 if (vcpu->arch.mp_state == KVM_MP_STATE_HALTED) 4110 vmcs12->guest_activity_state = GUEST_ACTIVITY_HLT; 4111 else if (vcpu->arch.mp_state == KVM_MP_STATE_INIT_RECEIVED) 4112 vmcs12->guest_activity_state = GUEST_ACTIVITY_WAIT_SIPI; 4113 else 4114 vmcs12->guest_activity_state = GUEST_ACTIVITY_ACTIVE; 4115 4116 if (nested_cpu_has_preemption_timer(vmcs12) && 4117 vmcs12->vm_exit_controls & VM_EXIT_SAVE_VMX_PREEMPTION_TIMER && 4118 !vmx->nested.nested_run_pending) 4119 vmcs12->vmx_preemption_timer_value = 4120 vmx_get_preemption_timer_value(vcpu); 4121 4122 /* 4123 * In some cases (usually, nested EPT), L2 is allowed to change its 4124 * own CR3 without exiting. If it has changed it, we must keep it. 4125 * Of course, if L0 is using shadow page tables, GUEST_CR3 was defined 4126 * by L0, not L1 or L2, so we mustn't unconditionally copy it to vmcs12. 4127 * 4128 * Additionally, restore L2's PDPTR to vmcs12. 4129 */ 4130 if (enable_ept) { 4131 vmcs12->guest_cr3 = vmcs_readl(GUEST_CR3); 4132 if (nested_cpu_has_ept(vmcs12) && is_pae_paging(vcpu)) { 4133 vmcs12->guest_pdptr0 = vmcs_read64(GUEST_PDPTR0); 4134 vmcs12->guest_pdptr1 = vmcs_read64(GUEST_PDPTR1); 4135 vmcs12->guest_pdptr2 = vmcs_read64(GUEST_PDPTR2); 4136 vmcs12->guest_pdptr3 = vmcs_read64(GUEST_PDPTR3); 4137 } 4138 } 4139 4140 vmcs12->guest_linear_address = vmcs_readl(GUEST_LINEAR_ADDRESS); 4141 4142 if (nested_cpu_has_vid(vmcs12)) 4143 vmcs12->guest_intr_status = vmcs_read16(GUEST_INTR_STATUS); 4144 4145 vmcs12->vm_entry_controls = 4146 (vmcs12->vm_entry_controls & ~VM_ENTRY_IA32E_MODE) | 4147 (vm_entry_controls_get(to_vmx(vcpu)) & VM_ENTRY_IA32E_MODE); 4148 4149 if (vmcs12->vm_exit_controls & VM_EXIT_SAVE_DEBUG_CONTROLS) 4150 kvm_get_dr(vcpu, 7, (unsigned long *)&vmcs12->guest_dr7); 4151 4152 if (vmcs12->vm_exit_controls & VM_EXIT_SAVE_IA32_EFER) 4153 vmcs12->guest_ia32_efer = vcpu->arch.efer; 4154 } 4155 4156 /* 4157 * prepare_vmcs12 is part of what we need to do when the nested L2 guest exits 4158 * and we want to prepare to run its L1 parent. L1 keeps a vmcs for L2 (vmcs12), 4159 * and this function updates it to reflect the changes to the guest state while 4160 * L2 was running (and perhaps made some exits which were handled directly by L0 4161 * without going back to L1), and to reflect the exit reason. 4162 * Note that we do not have to copy here all VMCS fields, just those that 4163 * could have changed by the L2 guest or the exit - i.e., the guest-state and 4164 * exit-information fields only. Other fields are modified by L1 with VMWRITE, 4165 * which already writes to vmcs12 directly. 4166 */ 4167 static void prepare_vmcs12(struct kvm_vcpu *vcpu, struct vmcs12 *vmcs12, 4168 u32 vm_exit_reason, u32 exit_intr_info, 4169 unsigned long exit_qualification) 4170 { 4171 /* update exit information fields: */ 4172 vmcs12->vm_exit_reason = vm_exit_reason; 4173 if (to_vmx(vcpu)->exit_reason.enclave_mode) 4174 vmcs12->vm_exit_reason |= VMX_EXIT_REASONS_SGX_ENCLAVE_MODE; 4175 vmcs12->exit_qualification = exit_qualification; 4176 vmcs12->vm_exit_intr_info = exit_intr_info; 4177 4178 vmcs12->idt_vectoring_info_field = 0; 4179 vmcs12->vm_exit_instruction_len = vmcs_read32(VM_EXIT_INSTRUCTION_LEN); 4180 vmcs12->vmx_instruction_info = vmcs_read32(VMX_INSTRUCTION_INFO); 4181 4182 if (!(vmcs12->vm_exit_reason & VMX_EXIT_REASONS_FAILED_VMENTRY)) { 4183 vmcs12->launch_state = 1; 4184 4185 /* vm_entry_intr_info_field is cleared on exit. Emulate this 4186 * instead of reading the real value. */ 4187 vmcs12->vm_entry_intr_info_field &= ~INTR_INFO_VALID_MASK; 4188 4189 /* 4190 * Transfer the event that L0 or L1 may wanted to inject into 4191 * L2 to IDT_VECTORING_INFO_FIELD. 4192 */ 4193 vmcs12_save_pending_event(vcpu, vmcs12); 4194 4195 /* 4196 * According to spec, there's no need to store the guest's 4197 * MSRs if the exit is due to a VM-entry failure that occurs 4198 * during or after loading the guest state. Since this exit 4199 * does not fall in that category, we need to save the MSRs. 4200 */ 4201 if (nested_vmx_store_msr(vcpu, 4202 vmcs12->vm_exit_msr_store_addr, 4203 vmcs12->vm_exit_msr_store_count)) 4204 nested_vmx_abort(vcpu, 4205 VMX_ABORT_SAVE_GUEST_MSR_FAIL); 4206 } 4207 4208 /* 4209 * Drop what we picked up for L2 via vmx_complete_interrupts. It is 4210 * preserved above and would only end up incorrectly in L1. 4211 */ 4212 vcpu->arch.nmi_injected = false; 4213 kvm_clear_exception_queue(vcpu); 4214 kvm_clear_interrupt_queue(vcpu); 4215 } 4216 4217 /* 4218 * A part of what we need to when the nested L2 guest exits and we want to 4219 * run its L1 parent, is to reset L1's guest state to the host state specified 4220 * in vmcs12. 4221 * This function is to be called not only on normal nested exit, but also on 4222 * a nested entry failure, as explained in Intel's spec, 3B.23.7 ("VM-Entry 4223 * Failures During or After Loading Guest State"). 4224 * This function should be called when the active VMCS is L1's (vmcs01). 4225 */ 4226 static void load_vmcs12_host_state(struct kvm_vcpu *vcpu, 4227 struct vmcs12 *vmcs12) 4228 { 4229 enum vm_entry_failure_code ignored; 4230 struct kvm_segment seg; 4231 4232 if (vmcs12->vm_exit_controls & VM_EXIT_LOAD_IA32_EFER) 4233 vcpu->arch.efer = vmcs12->host_ia32_efer; 4234 else if (vmcs12->vm_exit_controls & VM_EXIT_HOST_ADDR_SPACE_SIZE) 4235 vcpu->arch.efer |= (EFER_LMA | EFER_LME); 4236 else 4237 vcpu->arch.efer &= ~(EFER_LMA | EFER_LME); 4238 vmx_set_efer(vcpu, vcpu->arch.efer); 4239 4240 kvm_rsp_write(vcpu, vmcs12->host_rsp); 4241 kvm_rip_write(vcpu, vmcs12->host_rip); 4242 vmx_set_rflags(vcpu, X86_EFLAGS_FIXED); 4243 vmx_set_interrupt_shadow(vcpu, 0); 4244 4245 /* 4246 * Note that calling vmx_set_cr0 is important, even if cr0 hasn't 4247 * actually changed, because vmx_set_cr0 refers to efer set above. 4248 * 4249 * CR0_GUEST_HOST_MASK is already set in the original vmcs01 4250 * (KVM doesn't change it); 4251 */ 4252 vcpu->arch.cr0_guest_owned_bits = KVM_POSSIBLE_CR0_GUEST_BITS; 4253 vmx_set_cr0(vcpu, vmcs12->host_cr0); 4254 4255 /* Same as above - no reason to call set_cr4_guest_host_mask(). */ 4256 vcpu->arch.cr4_guest_owned_bits = ~vmcs_readl(CR4_GUEST_HOST_MASK); 4257 vmx_set_cr4(vcpu, vmcs12->host_cr4); 4258 4259 nested_ept_uninit_mmu_context(vcpu); 4260 4261 /* 4262 * Only PDPTE load can fail as the value of cr3 was checked on entry and 4263 * couldn't have changed. 4264 */ 4265 if (nested_vmx_load_cr3(vcpu, vmcs12->host_cr3, false, true, &ignored)) 4266 nested_vmx_abort(vcpu, VMX_ABORT_LOAD_HOST_PDPTE_FAIL); 4267 4268 nested_vmx_transition_tlb_flush(vcpu, vmcs12, false); 4269 4270 vmcs_write32(GUEST_SYSENTER_CS, vmcs12->host_ia32_sysenter_cs); 4271 vmcs_writel(GUEST_SYSENTER_ESP, vmcs12->host_ia32_sysenter_esp); 4272 vmcs_writel(GUEST_SYSENTER_EIP, vmcs12->host_ia32_sysenter_eip); 4273 vmcs_writel(GUEST_IDTR_BASE, vmcs12->host_idtr_base); 4274 vmcs_writel(GUEST_GDTR_BASE, vmcs12->host_gdtr_base); 4275 vmcs_write32(GUEST_IDTR_LIMIT, 0xFFFF); 4276 vmcs_write32(GUEST_GDTR_LIMIT, 0xFFFF); 4277 4278 /* If not VM_EXIT_CLEAR_BNDCFGS, the L2 value propagates to L1. */ 4279 if (vmcs12->vm_exit_controls & VM_EXIT_CLEAR_BNDCFGS) 4280 vmcs_write64(GUEST_BNDCFGS, 0); 4281 4282 if (vmcs12->vm_exit_controls & VM_EXIT_LOAD_IA32_PAT) { 4283 vmcs_write64(GUEST_IA32_PAT, vmcs12->host_ia32_pat); 4284 vcpu->arch.pat = vmcs12->host_ia32_pat; 4285 } 4286 if (vmcs12->vm_exit_controls & VM_EXIT_LOAD_IA32_PERF_GLOBAL_CTRL) 4287 WARN_ON_ONCE(kvm_set_msr(vcpu, MSR_CORE_PERF_GLOBAL_CTRL, 4288 vmcs12->host_ia32_perf_global_ctrl)); 4289 4290 /* Set L1 segment info according to Intel SDM 4291 27.5.2 Loading Host Segment and Descriptor-Table Registers */ 4292 seg = (struct kvm_segment) { 4293 .base = 0, 4294 .limit = 0xFFFFFFFF, 4295 .selector = vmcs12->host_cs_selector, 4296 .type = 11, 4297 .present = 1, 4298 .s = 1, 4299 .g = 1 4300 }; 4301 if (vmcs12->vm_exit_controls & VM_EXIT_HOST_ADDR_SPACE_SIZE) 4302 seg.l = 1; 4303 else 4304 seg.db = 1; 4305 __vmx_set_segment(vcpu, &seg, VCPU_SREG_CS); 4306 seg = (struct kvm_segment) { 4307 .base = 0, 4308 .limit = 0xFFFFFFFF, 4309 .type = 3, 4310 .present = 1, 4311 .s = 1, 4312 .db = 1, 4313 .g = 1 4314 }; 4315 seg.selector = vmcs12->host_ds_selector; 4316 __vmx_set_segment(vcpu, &seg, VCPU_SREG_DS); 4317 seg.selector = vmcs12->host_es_selector; 4318 __vmx_set_segment(vcpu, &seg, VCPU_SREG_ES); 4319 seg.selector = vmcs12->host_ss_selector; 4320 __vmx_set_segment(vcpu, &seg, VCPU_SREG_SS); 4321 seg.selector = vmcs12->host_fs_selector; 4322 seg.base = vmcs12->host_fs_base; 4323 __vmx_set_segment(vcpu, &seg, VCPU_SREG_FS); 4324 seg.selector = vmcs12->host_gs_selector; 4325 seg.base = vmcs12->host_gs_base; 4326 __vmx_set_segment(vcpu, &seg, VCPU_SREG_GS); 4327 seg = (struct kvm_segment) { 4328 .base = vmcs12->host_tr_base, 4329 .limit = 0x67, 4330 .selector = vmcs12->host_tr_selector, 4331 .type = 11, 4332 .present = 1 4333 }; 4334 __vmx_set_segment(vcpu, &seg, VCPU_SREG_TR); 4335 4336 memset(&seg, 0, sizeof(seg)); 4337 seg.unusable = 1; 4338 __vmx_set_segment(vcpu, &seg, VCPU_SREG_LDTR); 4339 4340 kvm_set_dr(vcpu, 7, 0x400); 4341 vmcs_write64(GUEST_IA32_DEBUGCTL, 0); 4342 4343 if (nested_vmx_load_msr(vcpu, vmcs12->vm_exit_msr_load_addr, 4344 vmcs12->vm_exit_msr_load_count)) 4345 nested_vmx_abort(vcpu, VMX_ABORT_LOAD_HOST_MSR_FAIL); 4346 4347 to_vmx(vcpu)->emulation_required = vmx_emulation_required(vcpu); 4348 } 4349 4350 static inline u64 nested_vmx_get_vmcs01_guest_efer(struct vcpu_vmx *vmx) 4351 { 4352 struct vmx_uret_msr *efer_msr; 4353 unsigned int i; 4354 4355 if (vm_entry_controls_get(vmx) & VM_ENTRY_LOAD_IA32_EFER) 4356 return vmcs_read64(GUEST_IA32_EFER); 4357 4358 if (cpu_has_load_ia32_efer()) 4359 return host_efer; 4360 4361 for (i = 0; i < vmx->msr_autoload.guest.nr; ++i) { 4362 if (vmx->msr_autoload.guest.val[i].index == MSR_EFER) 4363 return vmx->msr_autoload.guest.val[i].value; 4364 } 4365 4366 efer_msr = vmx_find_uret_msr(vmx, MSR_EFER); 4367 if (efer_msr) 4368 return efer_msr->data; 4369 4370 return host_efer; 4371 } 4372 4373 static void nested_vmx_restore_host_state(struct kvm_vcpu *vcpu) 4374 { 4375 struct vmcs12 *vmcs12 = get_vmcs12(vcpu); 4376 struct vcpu_vmx *vmx = to_vmx(vcpu); 4377 struct vmx_msr_entry g, h; 4378 gpa_t gpa; 4379 u32 i, j; 4380 4381 vcpu->arch.pat = vmcs_read64(GUEST_IA32_PAT); 4382 4383 if (vmcs12->vm_entry_controls & VM_ENTRY_LOAD_DEBUG_CONTROLS) { 4384 /* 4385 * L1's host DR7 is lost if KVM_GUESTDBG_USE_HW_BP is set 4386 * as vmcs01.GUEST_DR7 contains a userspace defined value 4387 * and vcpu->arch.dr7 is not squirreled away before the 4388 * nested VMENTER (not worth adding a variable in nested_vmx). 4389 */ 4390 if (vcpu->guest_debug & KVM_GUESTDBG_USE_HW_BP) 4391 kvm_set_dr(vcpu, 7, DR7_FIXED_1); 4392 else 4393 WARN_ON(kvm_set_dr(vcpu, 7, vmcs_readl(GUEST_DR7))); 4394 } 4395 4396 /* 4397 * Note that calling vmx_set_{efer,cr0,cr4} is important as they 4398 * handle a variety of side effects to KVM's software model. 4399 */ 4400 vmx_set_efer(vcpu, nested_vmx_get_vmcs01_guest_efer(vmx)); 4401 4402 vcpu->arch.cr0_guest_owned_bits = KVM_POSSIBLE_CR0_GUEST_BITS; 4403 vmx_set_cr0(vcpu, vmcs_readl(CR0_READ_SHADOW)); 4404 4405 vcpu->arch.cr4_guest_owned_bits = ~vmcs_readl(CR4_GUEST_HOST_MASK); 4406 vmx_set_cr4(vcpu, vmcs_readl(CR4_READ_SHADOW)); 4407 4408 nested_ept_uninit_mmu_context(vcpu); 4409 vcpu->arch.cr3 = vmcs_readl(GUEST_CR3); 4410 kvm_register_mark_available(vcpu, VCPU_EXREG_CR3); 4411 4412 /* 4413 * Use ept_save_pdptrs(vcpu) to load the MMU's cached PDPTRs 4414 * from vmcs01 (if necessary). The PDPTRs are not loaded on 4415 * VMFail, like everything else we just need to ensure our 4416 * software model is up-to-date. 4417 */ 4418 if (enable_ept && is_pae_paging(vcpu)) 4419 ept_save_pdptrs(vcpu); 4420 4421 kvm_mmu_reset_context(vcpu); 4422 4423 /* 4424 * This nasty bit of open coding is a compromise between blindly 4425 * loading L1's MSRs using the exit load lists (incorrect emulation 4426 * of VMFail), leaving the nested VM's MSRs in the software model 4427 * (incorrect behavior) and snapshotting the modified MSRs (too 4428 * expensive since the lists are unbound by hardware). For each 4429 * MSR that was (prematurely) loaded from the nested VMEntry load 4430 * list, reload it from the exit load list if it exists and differs 4431 * from the guest value. The intent is to stuff host state as 4432 * silently as possible, not to fully process the exit load list. 4433 */ 4434 for (i = 0; i < vmcs12->vm_entry_msr_load_count; i++) { 4435 gpa = vmcs12->vm_entry_msr_load_addr + (i * sizeof(g)); 4436 if (kvm_vcpu_read_guest(vcpu, gpa, &g, sizeof(g))) { 4437 pr_debug_ratelimited( 4438 "%s read MSR index failed (%u, 0x%08llx)\n", 4439 __func__, i, gpa); 4440 goto vmabort; 4441 } 4442 4443 for (j = 0; j < vmcs12->vm_exit_msr_load_count; j++) { 4444 gpa = vmcs12->vm_exit_msr_load_addr + (j * sizeof(h)); 4445 if (kvm_vcpu_read_guest(vcpu, gpa, &h, sizeof(h))) { 4446 pr_debug_ratelimited( 4447 "%s read MSR failed (%u, 0x%08llx)\n", 4448 __func__, j, gpa); 4449 goto vmabort; 4450 } 4451 if (h.index != g.index) 4452 continue; 4453 if (h.value == g.value) 4454 break; 4455 4456 if (nested_vmx_load_msr_check(vcpu, &h)) { 4457 pr_debug_ratelimited( 4458 "%s check failed (%u, 0x%x, 0x%x)\n", 4459 __func__, j, h.index, h.reserved); 4460 goto vmabort; 4461 } 4462 4463 if (kvm_set_msr(vcpu, h.index, h.value)) { 4464 pr_debug_ratelimited( 4465 "%s WRMSR failed (%u, 0x%x, 0x%llx)\n", 4466 __func__, j, h.index, h.value); 4467 goto vmabort; 4468 } 4469 } 4470 } 4471 4472 return; 4473 4474 vmabort: 4475 nested_vmx_abort(vcpu, VMX_ABORT_LOAD_HOST_MSR_FAIL); 4476 } 4477 4478 /* 4479 * Emulate an exit from nested guest (L2) to L1, i.e., prepare to run L1 4480 * and modify vmcs12 to make it see what it would expect to see there if 4481 * L2 was its real guest. Must only be called when in L2 (is_guest_mode()) 4482 */ 4483 void nested_vmx_vmexit(struct kvm_vcpu *vcpu, u32 vm_exit_reason, 4484 u32 exit_intr_info, unsigned long exit_qualification) 4485 { 4486 struct vcpu_vmx *vmx = to_vmx(vcpu); 4487 struct vmcs12 *vmcs12 = get_vmcs12(vcpu); 4488 4489 /* trying to cancel vmlaunch/vmresume is a bug */ 4490 WARN_ON_ONCE(vmx->nested.nested_run_pending); 4491 4492 /* Similarly, triple faults in L2 should never escape. */ 4493 WARN_ON_ONCE(kvm_check_request(KVM_REQ_TRIPLE_FAULT, vcpu)); 4494 4495 if (kvm_check_request(KVM_REQ_GET_NESTED_STATE_PAGES, vcpu)) { 4496 /* 4497 * KVM_REQ_GET_NESTED_STATE_PAGES is also used to map 4498 * Enlightened VMCS after migration and we still need to 4499 * do that when something is forcing L2->L1 exit prior to 4500 * the first L2 run. 4501 */ 4502 (void)nested_get_evmcs_page(vcpu); 4503 } 4504 4505 /* Service the TLB flush request for L2 before switching to L1. */ 4506 if (kvm_check_request(KVM_REQ_TLB_FLUSH_CURRENT, vcpu)) 4507 kvm_vcpu_flush_tlb_current(vcpu); 4508 4509 /* 4510 * VCPU_EXREG_PDPTR will be clobbered in arch/x86/kvm/vmx/vmx.h between 4511 * now and the new vmentry. Ensure that the VMCS02 PDPTR fields are 4512 * up-to-date before switching to L1. 4513 */ 4514 if (enable_ept && is_pae_paging(vcpu)) 4515 vmx_ept_load_pdptrs(vcpu); 4516 4517 leave_guest_mode(vcpu); 4518 4519 if (nested_cpu_has_preemption_timer(vmcs12)) 4520 hrtimer_cancel(&to_vmx(vcpu)->nested.preemption_timer); 4521 4522 if (nested_cpu_has(vmcs12, CPU_BASED_USE_TSC_OFFSETTING)) { 4523 vcpu->arch.tsc_offset = vcpu->arch.l1_tsc_offset; 4524 if (nested_cpu_has2(vmcs12, SECONDARY_EXEC_TSC_SCALING)) 4525 vcpu->arch.tsc_scaling_ratio = vcpu->arch.l1_tsc_scaling_ratio; 4526 } 4527 4528 if (likely(!vmx->fail)) { 4529 sync_vmcs02_to_vmcs12(vcpu, vmcs12); 4530 4531 if (vm_exit_reason != -1) 4532 prepare_vmcs12(vcpu, vmcs12, vm_exit_reason, 4533 exit_intr_info, exit_qualification); 4534 4535 /* 4536 * Must happen outside of sync_vmcs02_to_vmcs12() as it will 4537 * also be used to capture vmcs12 cache as part of 4538 * capturing nVMX state for snapshot (migration). 4539 * 4540 * Otherwise, this flush will dirty guest memory at a 4541 * point it is already assumed by user-space to be 4542 * immutable. 4543 */ 4544 nested_flush_cached_shadow_vmcs12(vcpu, vmcs12); 4545 } else { 4546 /* 4547 * The only expected VM-instruction error is "VM entry with 4548 * invalid control field(s)." Anything else indicates a 4549 * problem with L0. And we should never get here with a 4550 * VMFail of any type if early consistency checks are enabled. 4551 */ 4552 WARN_ON_ONCE(vmcs_read32(VM_INSTRUCTION_ERROR) != 4553 VMXERR_ENTRY_INVALID_CONTROL_FIELD); 4554 WARN_ON_ONCE(nested_early_check); 4555 } 4556 4557 vmx_switch_vmcs(vcpu, &vmx->vmcs01); 4558 4559 /* Update any VMCS fields that might have changed while L2 ran */ 4560 vmcs_write32(VM_EXIT_MSR_LOAD_COUNT, vmx->msr_autoload.host.nr); 4561 vmcs_write32(VM_ENTRY_MSR_LOAD_COUNT, vmx->msr_autoload.guest.nr); 4562 vmcs_write64(TSC_OFFSET, vcpu->arch.tsc_offset); 4563 if (kvm_has_tsc_control) 4564 vmcs_write64(TSC_MULTIPLIER, vcpu->arch.tsc_scaling_ratio); 4565 4566 if (vmx->nested.l1_tpr_threshold != -1) 4567 vmcs_write32(TPR_THRESHOLD, vmx->nested.l1_tpr_threshold); 4568 4569 if (vmx->nested.change_vmcs01_virtual_apic_mode) { 4570 vmx->nested.change_vmcs01_virtual_apic_mode = false; 4571 vmx_set_virtual_apic_mode(vcpu); 4572 } 4573 4574 if (vmx->nested.update_vmcs01_cpu_dirty_logging) { 4575 vmx->nested.update_vmcs01_cpu_dirty_logging = false; 4576 vmx_update_cpu_dirty_logging(vcpu); 4577 } 4578 4579 /* Unpin physical memory we referred to in vmcs02 */ 4580 if (vmx->nested.apic_access_page) { 4581 kvm_release_page_clean(vmx->nested.apic_access_page); 4582 vmx->nested.apic_access_page = NULL; 4583 } 4584 kvm_vcpu_unmap(vcpu, &vmx->nested.virtual_apic_map, true); 4585 kvm_vcpu_unmap(vcpu, &vmx->nested.pi_desc_map, true); 4586 vmx->nested.pi_desc = NULL; 4587 4588 if (vmx->nested.reload_vmcs01_apic_access_page) { 4589 vmx->nested.reload_vmcs01_apic_access_page = false; 4590 kvm_make_request(KVM_REQ_APIC_PAGE_RELOAD, vcpu); 4591 } 4592 4593 if ((vm_exit_reason != -1) && 4594 (enable_shadow_vmcs || evmptr_is_valid(vmx->nested.hv_evmcs_vmptr))) 4595 vmx->nested.need_vmcs12_to_shadow_sync = true; 4596 4597 /* in case we halted in L2 */ 4598 vcpu->arch.mp_state = KVM_MP_STATE_RUNNABLE; 4599 4600 if (likely(!vmx->fail)) { 4601 if ((u16)vm_exit_reason == EXIT_REASON_EXTERNAL_INTERRUPT && 4602 nested_exit_intr_ack_set(vcpu)) { 4603 int irq = kvm_cpu_get_interrupt(vcpu); 4604 WARN_ON(irq < 0); 4605 vmcs12->vm_exit_intr_info = irq | 4606 INTR_INFO_VALID_MASK | INTR_TYPE_EXT_INTR; 4607 } 4608 4609 if (vm_exit_reason != -1) 4610 trace_kvm_nested_vmexit_inject(vmcs12->vm_exit_reason, 4611 vmcs12->exit_qualification, 4612 vmcs12->idt_vectoring_info_field, 4613 vmcs12->vm_exit_intr_info, 4614 vmcs12->vm_exit_intr_error_code, 4615 KVM_ISA_VMX); 4616 4617 load_vmcs12_host_state(vcpu, vmcs12); 4618 4619 return; 4620 } 4621 4622 /* 4623 * After an early L2 VM-entry failure, we're now back 4624 * in L1 which thinks it just finished a VMLAUNCH or 4625 * VMRESUME instruction, so we need to set the failure 4626 * flag and the VM-instruction error field of the VMCS 4627 * accordingly, and skip the emulated instruction. 4628 */ 4629 (void)nested_vmx_fail(vcpu, VMXERR_ENTRY_INVALID_CONTROL_FIELD); 4630 4631 /* 4632 * Restore L1's host state to KVM's software model. We're here 4633 * because a consistency check was caught by hardware, which 4634 * means some amount of guest state has been propagated to KVM's 4635 * model and needs to be unwound to the host's state. 4636 */ 4637 nested_vmx_restore_host_state(vcpu); 4638 4639 vmx->fail = 0; 4640 } 4641 4642 static void nested_vmx_triple_fault(struct kvm_vcpu *vcpu) 4643 { 4644 nested_vmx_vmexit(vcpu, EXIT_REASON_TRIPLE_FAULT, 0, 0); 4645 } 4646 4647 /* 4648 * Decode the memory-address operand of a vmx instruction, as recorded on an 4649 * exit caused by such an instruction (run by a guest hypervisor). 4650 * On success, returns 0. When the operand is invalid, returns 1 and throws 4651 * #UD, #GP, or #SS. 4652 */ 4653 int get_vmx_mem_address(struct kvm_vcpu *vcpu, unsigned long exit_qualification, 4654 u32 vmx_instruction_info, bool wr, int len, gva_t *ret) 4655 { 4656 gva_t off; 4657 bool exn; 4658 struct kvm_segment s; 4659 4660 /* 4661 * According to Vol. 3B, "Information for VM Exits Due to Instruction 4662 * Execution", on an exit, vmx_instruction_info holds most of the 4663 * addressing components of the operand. Only the displacement part 4664 * is put in exit_qualification (see 3B, "Basic VM-Exit Information"). 4665 * For how an actual address is calculated from all these components, 4666 * refer to Vol. 1, "Operand Addressing". 4667 */ 4668 int scaling = vmx_instruction_info & 3; 4669 int addr_size = (vmx_instruction_info >> 7) & 7; 4670 bool is_reg = vmx_instruction_info & (1u << 10); 4671 int seg_reg = (vmx_instruction_info >> 15) & 7; 4672 int index_reg = (vmx_instruction_info >> 18) & 0xf; 4673 bool index_is_valid = !(vmx_instruction_info & (1u << 22)); 4674 int base_reg = (vmx_instruction_info >> 23) & 0xf; 4675 bool base_is_valid = !(vmx_instruction_info & (1u << 27)); 4676 4677 if (is_reg) { 4678 kvm_queue_exception(vcpu, UD_VECTOR); 4679 return 1; 4680 } 4681 4682 /* Addr = segment_base + offset */ 4683 /* offset = base + [index * scale] + displacement */ 4684 off = exit_qualification; /* holds the displacement */ 4685 if (addr_size == 1) 4686 off = (gva_t)sign_extend64(off, 31); 4687 else if (addr_size == 0) 4688 off = (gva_t)sign_extend64(off, 15); 4689 if (base_is_valid) 4690 off += kvm_register_read(vcpu, base_reg); 4691 if (index_is_valid) 4692 off += kvm_register_read(vcpu, index_reg) << scaling; 4693 vmx_get_segment(vcpu, &s, seg_reg); 4694 4695 /* 4696 * The effective address, i.e. @off, of a memory operand is truncated 4697 * based on the address size of the instruction. Note that this is 4698 * the *effective address*, i.e. the address prior to accounting for 4699 * the segment's base. 4700 */ 4701 if (addr_size == 1) /* 32 bit */ 4702 off &= 0xffffffff; 4703 else if (addr_size == 0) /* 16 bit */ 4704 off &= 0xffff; 4705 4706 /* Checks for #GP/#SS exceptions. */ 4707 exn = false; 4708 if (is_long_mode(vcpu)) { 4709 /* 4710 * The virtual/linear address is never truncated in 64-bit 4711 * mode, e.g. a 32-bit address size can yield a 64-bit virtual 4712 * address when using FS/GS with a non-zero base. 4713 */ 4714 if (seg_reg == VCPU_SREG_FS || seg_reg == VCPU_SREG_GS) 4715 *ret = s.base + off; 4716 else 4717 *ret = off; 4718 4719 /* Long mode: #GP(0)/#SS(0) if the memory address is in a 4720 * non-canonical form. This is the only check on the memory 4721 * destination for long mode! 4722 */ 4723 exn = is_noncanonical_address(*ret, vcpu); 4724 } else { 4725 /* 4726 * When not in long mode, the virtual/linear address is 4727 * unconditionally truncated to 32 bits regardless of the 4728 * address size. 4729 */ 4730 *ret = (s.base + off) & 0xffffffff; 4731 4732 /* Protected mode: apply checks for segment validity in the 4733 * following order: 4734 * - segment type check (#GP(0) may be thrown) 4735 * - usability check (#GP(0)/#SS(0)) 4736 * - limit check (#GP(0)/#SS(0)) 4737 */ 4738 if (wr) 4739 /* #GP(0) if the destination operand is located in a 4740 * read-only data segment or any code segment. 4741 */ 4742 exn = ((s.type & 0xa) == 0 || (s.type & 8)); 4743 else 4744 /* #GP(0) if the source operand is located in an 4745 * execute-only code segment 4746 */ 4747 exn = ((s.type & 0xa) == 8); 4748 if (exn) { 4749 kvm_queue_exception_e(vcpu, GP_VECTOR, 0); 4750 return 1; 4751 } 4752 /* Protected mode: #GP(0)/#SS(0) if the segment is unusable. 4753 */ 4754 exn = (s.unusable != 0); 4755 4756 /* 4757 * Protected mode: #GP(0)/#SS(0) if the memory operand is 4758 * outside the segment limit. All CPUs that support VMX ignore 4759 * limit checks for flat segments, i.e. segments with base==0, 4760 * limit==0xffffffff and of type expand-up data or code. 4761 */ 4762 if (!(s.base == 0 && s.limit == 0xffffffff && 4763 ((s.type & 8) || !(s.type & 4)))) 4764 exn = exn || ((u64)off + len - 1 > s.limit); 4765 } 4766 if (exn) { 4767 kvm_queue_exception_e(vcpu, 4768 seg_reg == VCPU_SREG_SS ? 4769 SS_VECTOR : GP_VECTOR, 4770 0); 4771 return 1; 4772 } 4773 4774 return 0; 4775 } 4776 4777 void nested_vmx_pmu_entry_exit_ctls_update(struct kvm_vcpu *vcpu) 4778 { 4779 struct vcpu_vmx *vmx; 4780 4781 if (!nested_vmx_allowed(vcpu)) 4782 return; 4783 4784 vmx = to_vmx(vcpu); 4785 if (kvm_x86_ops.pmu_ops->is_valid_msr(vcpu, MSR_CORE_PERF_GLOBAL_CTRL)) { 4786 vmx->nested.msrs.entry_ctls_high |= 4787 VM_ENTRY_LOAD_IA32_PERF_GLOBAL_CTRL; 4788 vmx->nested.msrs.exit_ctls_high |= 4789 VM_EXIT_LOAD_IA32_PERF_GLOBAL_CTRL; 4790 } else { 4791 vmx->nested.msrs.entry_ctls_high &= 4792 ~VM_ENTRY_LOAD_IA32_PERF_GLOBAL_CTRL; 4793 vmx->nested.msrs.exit_ctls_high &= 4794 ~VM_EXIT_LOAD_IA32_PERF_GLOBAL_CTRL; 4795 } 4796 } 4797 4798 static int nested_vmx_get_vmptr(struct kvm_vcpu *vcpu, gpa_t *vmpointer, 4799 int *ret) 4800 { 4801 gva_t gva; 4802 struct x86_exception e; 4803 int r; 4804 4805 if (get_vmx_mem_address(vcpu, vmx_get_exit_qual(vcpu), 4806 vmcs_read32(VMX_INSTRUCTION_INFO), false, 4807 sizeof(*vmpointer), &gva)) { 4808 *ret = 1; 4809 return -EINVAL; 4810 } 4811 4812 r = kvm_read_guest_virt(vcpu, gva, vmpointer, sizeof(*vmpointer), &e); 4813 if (r != X86EMUL_CONTINUE) { 4814 *ret = kvm_handle_memory_failure(vcpu, r, &e); 4815 return -EINVAL; 4816 } 4817 4818 return 0; 4819 } 4820 4821 /* 4822 * Allocate a shadow VMCS and associate it with the currently loaded 4823 * VMCS, unless such a shadow VMCS already exists. The newly allocated 4824 * VMCS is also VMCLEARed, so that it is ready for use. 4825 */ 4826 static struct vmcs *alloc_shadow_vmcs(struct kvm_vcpu *vcpu) 4827 { 4828 struct vcpu_vmx *vmx = to_vmx(vcpu); 4829 struct loaded_vmcs *loaded_vmcs = vmx->loaded_vmcs; 4830 4831 /* 4832 * We should allocate a shadow vmcs for vmcs01 only when L1 4833 * executes VMXON and free it when L1 executes VMXOFF. 4834 * As it is invalid to execute VMXON twice, we shouldn't reach 4835 * here when vmcs01 already have an allocated shadow vmcs. 4836 */ 4837 WARN_ON(loaded_vmcs == &vmx->vmcs01 && loaded_vmcs->shadow_vmcs); 4838 4839 if (!loaded_vmcs->shadow_vmcs) { 4840 loaded_vmcs->shadow_vmcs = alloc_vmcs(true); 4841 if (loaded_vmcs->shadow_vmcs) 4842 vmcs_clear(loaded_vmcs->shadow_vmcs); 4843 } 4844 return loaded_vmcs->shadow_vmcs; 4845 } 4846 4847 static int enter_vmx_operation(struct kvm_vcpu *vcpu) 4848 { 4849 struct vcpu_vmx *vmx = to_vmx(vcpu); 4850 int r; 4851 4852 r = alloc_loaded_vmcs(&vmx->nested.vmcs02); 4853 if (r < 0) 4854 goto out_vmcs02; 4855 4856 vmx->nested.cached_vmcs12 = kzalloc(VMCS12_SIZE, GFP_KERNEL_ACCOUNT); 4857 if (!vmx->nested.cached_vmcs12) 4858 goto out_cached_vmcs12; 4859 4860 vmx->nested.cached_shadow_vmcs12 = kzalloc(VMCS12_SIZE, GFP_KERNEL_ACCOUNT); 4861 if (!vmx->nested.cached_shadow_vmcs12) 4862 goto out_cached_shadow_vmcs12; 4863 4864 if (enable_shadow_vmcs && !alloc_shadow_vmcs(vcpu)) 4865 goto out_shadow_vmcs; 4866 4867 hrtimer_init(&vmx->nested.preemption_timer, CLOCK_MONOTONIC, 4868 HRTIMER_MODE_ABS_PINNED); 4869 vmx->nested.preemption_timer.function = vmx_preemption_timer_fn; 4870 4871 vmx->nested.vpid02 = allocate_vpid(); 4872 4873 vmx->nested.vmcs02_initialized = false; 4874 vmx->nested.vmxon = true; 4875 4876 if (vmx_pt_mode_is_host_guest()) { 4877 vmx->pt_desc.guest.ctl = 0; 4878 pt_update_intercept_for_msr(vcpu); 4879 } 4880 4881 return 0; 4882 4883 out_shadow_vmcs: 4884 kfree(vmx->nested.cached_shadow_vmcs12); 4885 4886 out_cached_shadow_vmcs12: 4887 kfree(vmx->nested.cached_vmcs12); 4888 4889 out_cached_vmcs12: 4890 free_loaded_vmcs(&vmx->nested.vmcs02); 4891 4892 out_vmcs02: 4893 return -ENOMEM; 4894 } 4895 4896 /* Emulate the VMXON instruction. */ 4897 static int handle_vmon(struct kvm_vcpu *vcpu) 4898 { 4899 int ret; 4900 gpa_t vmptr; 4901 uint32_t revision; 4902 struct vcpu_vmx *vmx = to_vmx(vcpu); 4903 const u64 VMXON_NEEDED_FEATURES = FEAT_CTL_LOCKED 4904 | FEAT_CTL_VMX_ENABLED_OUTSIDE_SMX; 4905 4906 /* 4907 * The Intel VMX Instruction Reference lists a bunch of bits that are 4908 * prerequisite to running VMXON, most notably cr4.VMXE must be set to 4909 * 1 (see vmx_is_valid_cr4() for when we allow the guest to set this). 4910 * Otherwise, we should fail with #UD. But most faulting conditions 4911 * have already been checked by hardware, prior to the VM-exit for 4912 * VMXON. We do test guest cr4.VMXE because processor CR4 always has 4913 * that bit set to 1 in non-root mode. 4914 */ 4915 if (!kvm_read_cr4_bits(vcpu, X86_CR4_VMXE)) { 4916 kvm_queue_exception(vcpu, UD_VECTOR); 4917 return 1; 4918 } 4919 4920 /* CPL=0 must be checked manually. */ 4921 if (vmx_get_cpl(vcpu)) { 4922 kvm_inject_gp(vcpu, 0); 4923 return 1; 4924 } 4925 4926 if (vmx->nested.vmxon) 4927 return nested_vmx_fail(vcpu, VMXERR_VMXON_IN_VMX_ROOT_OPERATION); 4928 4929 if ((vmx->msr_ia32_feature_control & VMXON_NEEDED_FEATURES) 4930 != VMXON_NEEDED_FEATURES) { 4931 kvm_inject_gp(vcpu, 0); 4932 return 1; 4933 } 4934 4935 if (nested_vmx_get_vmptr(vcpu, &vmptr, &ret)) 4936 return ret; 4937 4938 /* 4939 * SDM 3: 24.11.5 4940 * The first 4 bytes of VMXON region contain the supported 4941 * VMCS revision identifier 4942 * 4943 * Note - IA32_VMX_BASIC[48] will never be 1 for the nested case; 4944 * which replaces physical address width with 32 4945 */ 4946 if (!page_address_valid(vcpu, vmptr)) 4947 return nested_vmx_failInvalid(vcpu); 4948 4949 if (kvm_read_guest(vcpu->kvm, vmptr, &revision, sizeof(revision)) || 4950 revision != VMCS12_REVISION) 4951 return nested_vmx_failInvalid(vcpu); 4952 4953 vmx->nested.vmxon_ptr = vmptr; 4954 ret = enter_vmx_operation(vcpu); 4955 if (ret) 4956 return ret; 4957 4958 return nested_vmx_succeed(vcpu); 4959 } 4960 4961 static inline void nested_release_vmcs12(struct kvm_vcpu *vcpu) 4962 { 4963 struct vcpu_vmx *vmx = to_vmx(vcpu); 4964 4965 if (vmx->nested.current_vmptr == INVALID_GPA) 4966 return; 4967 4968 copy_vmcs02_to_vmcs12_rare(vcpu, get_vmcs12(vcpu)); 4969 4970 if (enable_shadow_vmcs) { 4971 /* copy to memory all shadowed fields in case 4972 they were modified */ 4973 copy_shadow_to_vmcs12(vmx); 4974 vmx_disable_shadow_vmcs(vmx); 4975 } 4976 vmx->nested.posted_intr_nv = -1; 4977 4978 /* Flush VMCS12 to guest memory */ 4979 kvm_vcpu_write_guest_page(vcpu, 4980 vmx->nested.current_vmptr >> PAGE_SHIFT, 4981 vmx->nested.cached_vmcs12, 0, VMCS12_SIZE); 4982 4983 kvm_mmu_free_roots(vcpu, &vcpu->arch.guest_mmu, KVM_MMU_ROOTS_ALL); 4984 4985 vmx->nested.current_vmptr = INVALID_GPA; 4986 } 4987 4988 /* Emulate the VMXOFF instruction */ 4989 static int handle_vmoff(struct kvm_vcpu *vcpu) 4990 { 4991 if (!nested_vmx_check_permission(vcpu)) 4992 return 1; 4993 4994 free_nested(vcpu); 4995 4996 /* Process a latched INIT during time CPU was in VMX operation */ 4997 kvm_make_request(KVM_REQ_EVENT, vcpu); 4998 4999 return nested_vmx_succeed(vcpu); 5000 } 5001 5002 /* Emulate the VMCLEAR instruction */ 5003 static int handle_vmclear(struct kvm_vcpu *vcpu) 5004 { 5005 struct vcpu_vmx *vmx = to_vmx(vcpu); 5006 u32 zero = 0; 5007 gpa_t vmptr; 5008 u64 evmcs_gpa; 5009 int r; 5010 5011 if (!nested_vmx_check_permission(vcpu)) 5012 return 1; 5013 5014 if (nested_vmx_get_vmptr(vcpu, &vmptr, &r)) 5015 return r; 5016 5017 if (!page_address_valid(vcpu, vmptr)) 5018 return nested_vmx_fail(vcpu, VMXERR_VMCLEAR_INVALID_ADDRESS); 5019 5020 if (vmptr == vmx->nested.vmxon_ptr) 5021 return nested_vmx_fail(vcpu, VMXERR_VMCLEAR_VMXON_POINTER); 5022 5023 /* 5024 * When Enlightened VMEntry is enabled on the calling CPU we treat 5025 * memory area pointer by vmptr as Enlightened VMCS (as there's no good 5026 * way to distinguish it from VMCS12) and we must not corrupt it by 5027 * writing to the non-existent 'launch_state' field. The area doesn't 5028 * have to be the currently active EVMCS on the calling CPU and there's 5029 * nothing KVM has to do to transition it from 'active' to 'non-active' 5030 * state. It is possible that the area will stay mapped as 5031 * vmx->nested.hv_evmcs but this shouldn't be a problem. 5032 */ 5033 if (likely(!vmx->nested.enlightened_vmcs_enabled || 5034 !nested_enlightened_vmentry(vcpu, &evmcs_gpa))) { 5035 if (vmptr == vmx->nested.current_vmptr) 5036 nested_release_vmcs12(vcpu); 5037 5038 kvm_vcpu_write_guest(vcpu, 5039 vmptr + offsetof(struct vmcs12, 5040 launch_state), 5041 &zero, sizeof(zero)); 5042 } else if (vmx->nested.hv_evmcs && vmptr == vmx->nested.hv_evmcs_vmptr) { 5043 nested_release_evmcs(vcpu); 5044 } 5045 5046 return nested_vmx_succeed(vcpu); 5047 } 5048 5049 /* Emulate the VMLAUNCH instruction */ 5050 static int handle_vmlaunch(struct kvm_vcpu *vcpu) 5051 { 5052 return nested_vmx_run(vcpu, true); 5053 } 5054 5055 /* Emulate the VMRESUME instruction */ 5056 static int handle_vmresume(struct kvm_vcpu *vcpu) 5057 { 5058 5059 return nested_vmx_run(vcpu, false); 5060 } 5061 5062 static int handle_vmread(struct kvm_vcpu *vcpu) 5063 { 5064 struct vmcs12 *vmcs12 = is_guest_mode(vcpu) ? get_shadow_vmcs12(vcpu) 5065 : get_vmcs12(vcpu); 5066 unsigned long exit_qualification = vmx_get_exit_qual(vcpu); 5067 u32 instr_info = vmcs_read32(VMX_INSTRUCTION_INFO); 5068 struct vcpu_vmx *vmx = to_vmx(vcpu); 5069 struct x86_exception e; 5070 unsigned long field; 5071 u64 value; 5072 gva_t gva = 0; 5073 short offset; 5074 int len, r; 5075 5076 if (!nested_vmx_check_permission(vcpu)) 5077 return 1; 5078 5079 /* 5080 * In VMX non-root operation, when the VMCS-link pointer is INVALID_GPA, 5081 * any VMREAD sets the ALU flags for VMfailInvalid. 5082 */ 5083 if (vmx->nested.current_vmptr == INVALID_GPA || 5084 (is_guest_mode(vcpu) && 5085 get_vmcs12(vcpu)->vmcs_link_pointer == INVALID_GPA)) 5086 return nested_vmx_failInvalid(vcpu); 5087 5088 /* Decode instruction info and find the field to read */ 5089 field = kvm_register_read(vcpu, (((instr_info) >> 28) & 0xf)); 5090 5091 offset = vmcs_field_to_offset(field); 5092 if (offset < 0) 5093 return nested_vmx_fail(vcpu, VMXERR_UNSUPPORTED_VMCS_COMPONENT); 5094 5095 if (!is_guest_mode(vcpu) && is_vmcs12_ext_field(field)) 5096 copy_vmcs02_to_vmcs12_rare(vcpu, vmcs12); 5097 5098 /* Read the field, zero-extended to a u64 value */ 5099 value = vmcs12_read_any(vmcs12, field, offset); 5100 5101 /* 5102 * Now copy part of this value to register or memory, as requested. 5103 * Note that the number of bits actually copied is 32 or 64 depending 5104 * on the guest's mode (32 or 64 bit), not on the given field's length. 5105 */ 5106 if (instr_info & BIT(10)) { 5107 kvm_register_write(vcpu, (((instr_info) >> 3) & 0xf), value); 5108 } else { 5109 len = is_64_bit_mode(vcpu) ? 8 : 4; 5110 if (get_vmx_mem_address(vcpu, exit_qualification, 5111 instr_info, true, len, &gva)) 5112 return 1; 5113 /* _system ok, nested_vmx_check_permission has verified cpl=0 */ 5114 r = kvm_write_guest_virt_system(vcpu, gva, &value, len, &e); 5115 if (r != X86EMUL_CONTINUE) 5116 return kvm_handle_memory_failure(vcpu, r, &e); 5117 } 5118 5119 return nested_vmx_succeed(vcpu); 5120 } 5121 5122 static bool is_shadow_field_rw(unsigned long field) 5123 { 5124 switch (field) { 5125 #define SHADOW_FIELD_RW(x, y) case x: 5126 #include "vmcs_shadow_fields.h" 5127 return true; 5128 default: 5129 break; 5130 } 5131 return false; 5132 } 5133 5134 static bool is_shadow_field_ro(unsigned long field) 5135 { 5136 switch (field) { 5137 #define SHADOW_FIELD_RO(x, y) case x: 5138 #include "vmcs_shadow_fields.h" 5139 return true; 5140 default: 5141 break; 5142 } 5143 return false; 5144 } 5145 5146 static int handle_vmwrite(struct kvm_vcpu *vcpu) 5147 { 5148 struct vmcs12 *vmcs12 = is_guest_mode(vcpu) ? get_shadow_vmcs12(vcpu) 5149 : get_vmcs12(vcpu); 5150 unsigned long exit_qualification = vmx_get_exit_qual(vcpu); 5151 u32 instr_info = vmcs_read32(VMX_INSTRUCTION_INFO); 5152 struct vcpu_vmx *vmx = to_vmx(vcpu); 5153 struct x86_exception e; 5154 unsigned long field; 5155 short offset; 5156 gva_t gva; 5157 int len, r; 5158 5159 /* 5160 * The value to write might be 32 or 64 bits, depending on L1's long 5161 * mode, and eventually we need to write that into a field of several 5162 * possible lengths. The code below first zero-extends the value to 64 5163 * bit (value), and then copies only the appropriate number of 5164 * bits into the vmcs12 field. 5165 */ 5166 u64 value = 0; 5167 5168 if (!nested_vmx_check_permission(vcpu)) 5169 return 1; 5170 5171 /* 5172 * In VMX non-root operation, when the VMCS-link pointer is INVALID_GPA, 5173 * any VMWRITE sets the ALU flags for VMfailInvalid. 5174 */ 5175 if (vmx->nested.current_vmptr == INVALID_GPA || 5176 (is_guest_mode(vcpu) && 5177 get_vmcs12(vcpu)->vmcs_link_pointer == INVALID_GPA)) 5178 return nested_vmx_failInvalid(vcpu); 5179 5180 if (instr_info & BIT(10)) 5181 value = kvm_register_read(vcpu, (((instr_info) >> 3) & 0xf)); 5182 else { 5183 len = is_64_bit_mode(vcpu) ? 8 : 4; 5184 if (get_vmx_mem_address(vcpu, exit_qualification, 5185 instr_info, false, len, &gva)) 5186 return 1; 5187 r = kvm_read_guest_virt(vcpu, gva, &value, len, &e); 5188 if (r != X86EMUL_CONTINUE) 5189 return kvm_handle_memory_failure(vcpu, r, &e); 5190 } 5191 5192 field = kvm_register_read(vcpu, (((instr_info) >> 28) & 0xf)); 5193 5194 offset = vmcs_field_to_offset(field); 5195 if (offset < 0) 5196 return nested_vmx_fail(vcpu, VMXERR_UNSUPPORTED_VMCS_COMPONENT); 5197 5198 /* 5199 * If the vCPU supports "VMWRITE to any supported field in the 5200 * VMCS," then the "read-only" fields are actually read/write. 5201 */ 5202 if (vmcs_field_readonly(field) && 5203 !nested_cpu_has_vmwrite_any_field(vcpu)) 5204 return nested_vmx_fail(vcpu, VMXERR_VMWRITE_READ_ONLY_VMCS_COMPONENT); 5205 5206 /* 5207 * Ensure vmcs12 is up-to-date before any VMWRITE that dirties 5208 * vmcs12, else we may crush a field or consume a stale value. 5209 */ 5210 if (!is_guest_mode(vcpu) && !is_shadow_field_rw(field)) 5211 copy_vmcs02_to_vmcs12_rare(vcpu, vmcs12); 5212 5213 /* 5214 * Some Intel CPUs intentionally drop the reserved bits of the AR byte 5215 * fields on VMWRITE. Emulate this behavior to ensure consistent KVM 5216 * behavior regardless of the underlying hardware, e.g. if an AR_BYTE 5217 * field is intercepted for VMWRITE but not VMREAD (in L1), then VMREAD 5218 * from L1 will return a different value than VMREAD from L2 (L1 sees 5219 * the stripped down value, L2 sees the full value as stored by KVM). 5220 */ 5221 if (field >= GUEST_ES_AR_BYTES && field <= GUEST_TR_AR_BYTES) 5222 value &= 0x1f0ff; 5223 5224 vmcs12_write_any(vmcs12, field, offset, value); 5225 5226 /* 5227 * Do not track vmcs12 dirty-state if in guest-mode as we actually 5228 * dirty shadow vmcs12 instead of vmcs12. Fields that can be updated 5229 * by L1 without a vmexit are always updated in the vmcs02, i.e. don't 5230 * "dirty" vmcs12, all others go down the prepare_vmcs02() slow path. 5231 */ 5232 if (!is_guest_mode(vcpu) && !is_shadow_field_rw(field)) { 5233 /* 5234 * L1 can read these fields without exiting, ensure the 5235 * shadow VMCS is up-to-date. 5236 */ 5237 if (enable_shadow_vmcs && is_shadow_field_ro(field)) { 5238 preempt_disable(); 5239 vmcs_load(vmx->vmcs01.shadow_vmcs); 5240 5241 __vmcs_writel(field, value); 5242 5243 vmcs_clear(vmx->vmcs01.shadow_vmcs); 5244 vmcs_load(vmx->loaded_vmcs->vmcs); 5245 preempt_enable(); 5246 } 5247 vmx->nested.dirty_vmcs12 = true; 5248 } 5249 5250 return nested_vmx_succeed(vcpu); 5251 } 5252 5253 static void set_current_vmptr(struct vcpu_vmx *vmx, gpa_t vmptr) 5254 { 5255 vmx->nested.current_vmptr = vmptr; 5256 if (enable_shadow_vmcs) { 5257 secondary_exec_controls_setbit(vmx, SECONDARY_EXEC_SHADOW_VMCS); 5258 vmcs_write64(VMCS_LINK_POINTER, 5259 __pa(vmx->vmcs01.shadow_vmcs)); 5260 vmx->nested.need_vmcs12_to_shadow_sync = true; 5261 } 5262 vmx->nested.dirty_vmcs12 = true; 5263 } 5264 5265 /* Emulate the VMPTRLD instruction */ 5266 static int handle_vmptrld(struct kvm_vcpu *vcpu) 5267 { 5268 struct vcpu_vmx *vmx = to_vmx(vcpu); 5269 gpa_t vmptr; 5270 int r; 5271 5272 if (!nested_vmx_check_permission(vcpu)) 5273 return 1; 5274 5275 if (nested_vmx_get_vmptr(vcpu, &vmptr, &r)) 5276 return r; 5277 5278 if (!page_address_valid(vcpu, vmptr)) 5279 return nested_vmx_fail(vcpu, VMXERR_VMPTRLD_INVALID_ADDRESS); 5280 5281 if (vmptr == vmx->nested.vmxon_ptr) 5282 return nested_vmx_fail(vcpu, VMXERR_VMPTRLD_VMXON_POINTER); 5283 5284 /* Forbid normal VMPTRLD if Enlightened version was used */ 5285 if (evmptr_is_valid(vmx->nested.hv_evmcs_vmptr)) 5286 return 1; 5287 5288 if (vmx->nested.current_vmptr != vmptr) { 5289 struct gfn_to_hva_cache *ghc = &vmx->nested.vmcs12_cache; 5290 struct vmcs_hdr hdr; 5291 5292 if (ghc->gpa != vmptr && 5293 kvm_gfn_to_hva_cache_init(vcpu->kvm, ghc, vmptr, VMCS12_SIZE)) { 5294 /* 5295 * Reads from an unbacked page return all 1s, 5296 * which means that the 32 bits located at the 5297 * given physical address won't match the required 5298 * VMCS12_REVISION identifier. 5299 */ 5300 return nested_vmx_fail(vcpu, 5301 VMXERR_VMPTRLD_INCORRECT_VMCS_REVISION_ID); 5302 } 5303 5304 if (kvm_read_guest_offset_cached(vcpu->kvm, ghc, &hdr, 5305 offsetof(struct vmcs12, hdr), 5306 sizeof(hdr))) { 5307 return nested_vmx_fail(vcpu, 5308 VMXERR_VMPTRLD_INCORRECT_VMCS_REVISION_ID); 5309 } 5310 5311 if (hdr.revision_id != VMCS12_REVISION || 5312 (hdr.shadow_vmcs && 5313 !nested_cpu_has_vmx_shadow_vmcs(vcpu))) { 5314 return nested_vmx_fail(vcpu, 5315 VMXERR_VMPTRLD_INCORRECT_VMCS_REVISION_ID); 5316 } 5317 5318 nested_release_vmcs12(vcpu); 5319 5320 /* 5321 * Load VMCS12 from guest memory since it is not already 5322 * cached. 5323 */ 5324 if (kvm_read_guest_cached(vcpu->kvm, ghc, vmx->nested.cached_vmcs12, 5325 VMCS12_SIZE)) { 5326 return nested_vmx_fail(vcpu, 5327 VMXERR_VMPTRLD_INCORRECT_VMCS_REVISION_ID); 5328 } 5329 5330 set_current_vmptr(vmx, vmptr); 5331 } 5332 5333 return nested_vmx_succeed(vcpu); 5334 } 5335 5336 /* Emulate the VMPTRST instruction */ 5337 static int handle_vmptrst(struct kvm_vcpu *vcpu) 5338 { 5339 unsigned long exit_qual = vmx_get_exit_qual(vcpu); 5340 u32 instr_info = vmcs_read32(VMX_INSTRUCTION_INFO); 5341 gpa_t current_vmptr = to_vmx(vcpu)->nested.current_vmptr; 5342 struct x86_exception e; 5343 gva_t gva; 5344 int r; 5345 5346 if (!nested_vmx_check_permission(vcpu)) 5347 return 1; 5348 5349 if (unlikely(evmptr_is_valid(to_vmx(vcpu)->nested.hv_evmcs_vmptr))) 5350 return 1; 5351 5352 if (get_vmx_mem_address(vcpu, exit_qual, instr_info, 5353 true, sizeof(gpa_t), &gva)) 5354 return 1; 5355 /* *_system ok, nested_vmx_check_permission has verified cpl=0 */ 5356 r = kvm_write_guest_virt_system(vcpu, gva, (void *)¤t_vmptr, 5357 sizeof(gpa_t), &e); 5358 if (r != X86EMUL_CONTINUE) 5359 return kvm_handle_memory_failure(vcpu, r, &e); 5360 5361 return nested_vmx_succeed(vcpu); 5362 } 5363 5364 /* Emulate the INVEPT instruction */ 5365 static int handle_invept(struct kvm_vcpu *vcpu) 5366 { 5367 struct vcpu_vmx *vmx = to_vmx(vcpu); 5368 u32 vmx_instruction_info, types; 5369 unsigned long type, roots_to_free; 5370 struct kvm_mmu *mmu; 5371 gva_t gva; 5372 struct x86_exception e; 5373 struct { 5374 u64 eptp, gpa; 5375 } operand; 5376 int i, r, gpr_index; 5377 5378 if (!(vmx->nested.msrs.secondary_ctls_high & 5379 SECONDARY_EXEC_ENABLE_EPT) || 5380 !(vmx->nested.msrs.ept_caps & VMX_EPT_INVEPT_BIT)) { 5381 kvm_queue_exception(vcpu, UD_VECTOR); 5382 return 1; 5383 } 5384 5385 if (!nested_vmx_check_permission(vcpu)) 5386 return 1; 5387 5388 vmx_instruction_info = vmcs_read32(VMX_INSTRUCTION_INFO); 5389 gpr_index = vmx_get_instr_info_reg2(vmx_instruction_info); 5390 type = kvm_register_read(vcpu, gpr_index); 5391 5392 types = (vmx->nested.msrs.ept_caps >> VMX_EPT_EXTENT_SHIFT) & 6; 5393 5394 if (type >= 32 || !(types & (1 << type))) 5395 return nested_vmx_fail(vcpu, VMXERR_INVALID_OPERAND_TO_INVEPT_INVVPID); 5396 5397 /* According to the Intel VMX instruction reference, the memory 5398 * operand is read even if it isn't needed (e.g., for type==global) 5399 */ 5400 if (get_vmx_mem_address(vcpu, vmx_get_exit_qual(vcpu), 5401 vmx_instruction_info, false, sizeof(operand), &gva)) 5402 return 1; 5403 r = kvm_read_guest_virt(vcpu, gva, &operand, sizeof(operand), &e); 5404 if (r != X86EMUL_CONTINUE) 5405 return kvm_handle_memory_failure(vcpu, r, &e); 5406 5407 /* 5408 * Nested EPT roots are always held through guest_mmu, 5409 * not root_mmu. 5410 */ 5411 mmu = &vcpu->arch.guest_mmu; 5412 5413 switch (type) { 5414 case VMX_EPT_EXTENT_CONTEXT: 5415 if (!nested_vmx_check_eptp(vcpu, operand.eptp)) 5416 return nested_vmx_fail(vcpu, 5417 VMXERR_INVALID_OPERAND_TO_INVEPT_INVVPID); 5418 5419 roots_to_free = 0; 5420 if (nested_ept_root_matches(mmu->root_hpa, mmu->root_pgd, 5421 operand.eptp)) 5422 roots_to_free |= KVM_MMU_ROOT_CURRENT; 5423 5424 for (i = 0; i < KVM_MMU_NUM_PREV_ROOTS; i++) { 5425 if (nested_ept_root_matches(mmu->prev_roots[i].hpa, 5426 mmu->prev_roots[i].pgd, 5427 operand.eptp)) 5428 roots_to_free |= KVM_MMU_ROOT_PREVIOUS(i); 5429 } 5430 break; 5431 case VMX_EPT_EXTENT_GLOBAL: 5432 roots_to_free = KVM_MMU_ROOTS_ALL; 5433 break; 5434 default: 5435 BUG(); 5436 break; 5437 } 5438 5439 if (roots_to_free) 5440 kvm_mmu_free_roots(vcpu, mmu, roots_to_free); 5441 5442 return nested_vmx_succeed(vcpu); 5443 } 5444 5445 static int handle_invvpid(struct kvm_vcpu *vcpu) 5446 { 5447 struct vcpu_vmx *vmx = to_vmx(vcpu); 5448 u32 vmx_instruction_info; 5449 unsigned long type, types; 5450 gva_t gva; 5451 struct x86_exception e; 5452 struct { 5453 u64 vpid; 5454 u64 gla; 5455 } operand; 5456 u16 vpid02; 5457 int r, gpr_index; 5458 5459 if (!(vmx->nested.msrs.secondary_ctls_high & 5460 SECONDARY_EXEC_ENABLE_VPID) || 5461 !(vmx->nested.msrs.vpid_caps & VMX_VPID_INVVPID_BIT)) { 5462 kvm_queue_exception(vcpu, UD_VECTOR); 5463 return 1; 5464 } 5465 5466 if (!nested_vmx_check_permission(vcpu)) 5467 return 1; 5468 5469 vmx_instruction_info = vmcs_read32(VMX_INSTRUCTION_INFO); 5470 gpr_index = vmx_get_instr_info_reg2(vmx_instruction_info); 5471 type = kvm_register_read(vcpu, gpr_index); 5472 5473 types = (vmx->nested.msrs.vpid_caps & 5474 VMX_VPID_EXTENT_SUPPORTED_MASK) >> 8; 5475 5476 if (type >= 32 || !(types & (1 << type))) 5477 return nested_vmx_fail(vcpu, 5478 VMXERR_INVALID_OPERAND_TO_INVEPT_INVVPID); 5479 5480 /* according to the intel vmx instruction reference, the memory 5481 * operand is read even if it isn't needed (e.g., for type==global) 5482 */ 5483 if (get_vmx_mem_address(vcpu, vmx_get_exit_qual(vcpu), 5484 vmx_instruction_info, false, sizeof(operand), &gva)) 5485 return 1; 5486 r = kvm_read_guest_virt(vcpu, gva, &operand, sizeof(operand), &e); 5487 if (r != X86EMUL_CONTINUE) 5488 return kvm_handle_memory_failure(vcpu, r, &e); 5489 5490 if (operand.vpid >> 16) 5491 return nested_vmx_fail(vcpu, 5492 VMXERR_INVALID_OPERAND_TO_INVEPT_INVVPID); 5493 5494 vpid02 = nested_get_vpid02(vcpu); 5495 switch (type) { 5496 case VMX_VPID_EXTENT_INDIVIDUAL_ADDR: 5497 if (!operand.vpid || 5498 is_noncanonical_address(operand.gla, vcpu)) 5499 return nested_vmx_fail(vcpu, 5500 VMXERR_INVALID_OPERAND_TO_INVEPT_INVVPID); 5501 vpid_sync_vcpu_addr(vpid02, operand.gla); 5502 break; 5503 case VMX_VPID_EXTENT_SINGLE_CONTEXT: 5504 case VMX_VPID_EXTENT_SINGLE_NON_GLOBAL: 5505 if (!operand.vpid) 5506 return nested_vmx_fail(vcpu, 5507 VMXERR_INVALID_OPERAND_TO_INVEPT_INVVPID); 5508 vpid_sync_context(vpid02); 5509 break; 5510 case VMX_VPID_EXTENT_ALL_CONTEXT: 5511 vpid_sync_context(vpid02); 5512 break; 5513 default: 5514 WARN_ON_ONCE(1); 5515 return kvm_skip_emulated_instruction(vcpu); 5516 } 5517 5518 /* 5519 * Sync the shadow page tables if EPT is disabled, L1 is invalidating 5520 * linear mappings for L2 (tagged with L2's VPID). Free all guest 5521 * roots as VPIDs are not tracked in the MMU role. 5522 * 5523 * Note, this operates on root_mmu, not guest_mmu, as L1 and L2 share 5524 * an MMU when EPT is disabled. 5525 * 5526 * TODO: sync only the affected SPTEs for INVDIVIDUAL_ADDR. 5527 */ 5528 if (!enable_ept) 5529 kvm_mmu_free_guest_mode_roots(vcpu, &vcpu->arch.root_mmu); 5530 5531 return nested_vmx_succeed(vcpu); 5532 } 5533 5534 static int nested_vmx_eptp_switching(struct kvm_vcpu *vcpu, 5535 struct vmcs12 *vmcs12) 5536 { 5537 u32 index = kvm_rcx_read(vcpu); 5538 u64 new_eptp; 5539 5540 if (WARN_ON_ONCE(!nested_cpu_has_ept(vmcs12))) 5541 return 1; 5542 if (index >= VMFUNC_EPTP_ENTRIES) 5543 return 1; 5544 5545 if (kvm_vcpu_read_guest_page(vcpu, vmcs12->eptp_list_address >> PAGE_SHIFT, 5546 &new_eptp, index * 8, 8)) 5547 return 1; 5548 5549 /* 5550 * If the (L2) guest does a vmfunc to the currently 5551 * active ept pointer, we don't have to do anything else 5552 */ 5553 if (vmcs12->ept_pointer != new_eptp) { 5554 if (!nested_vmx_check_eptp(vcpu, new_eptp)) 5555 return 1; 5556 5557 vmcs12->ept_pointer = new_eptp; 5558 nested_ept_new_eptp(vcpu); 5559 5560 if (!nested_cpu_has_vpid(vmcs12)) 5561 kvm_make_request(KVM_REQ_TLB_FLUSH_GUEST, vcpu); 5562 } 5563 5564 return 0; 5565 } 5566 5567 static int handle_vmfunc(struct kvm_vcpu *vcpu) 5568 { 5569 struct vcpu_vmx *vmx = to_vmx(vcpu); 5570 struct vmcs12 *vmcs12; 5571 u32 function = kvm_rax_read(vcpu); 5572 5573 /* 5574 * VMFUNC is only supported for nested guests, but we always enable the 5575 * secondary control for simplicity; for non-nested mode, fake that we 5576 * didn't by injecting #UD. 5577 */ 5578 if (!is_guest_mode(vcpu)) { 5579 kvm_queue_exception(vcpu, UD_VECTOR); 5580 return 1; 5581 } 5582 5583 vmcs12 = get_vmcs12(vcpu); 5584 5585 /* 5586 * #UD on out-of-bounds function has priority over VM-Exit, and VMFUNC 5587 * is enabled in vmcs02 if and only if it's enabled in vmcs12. 5588 */ 5589 if (WARN_ON_ONCE((function > 63) || !nested_cpu_has_vmfunc(vmcs12))) { 5590 kvm_queue_exception(vcpu, UD_VECTOR); 5591 return 1; 5592 } 5593 5594 if (!(vmcs12->vm_function_control & BIT_ULL(function))) 5595 goto fail; 5596 5597 switch (function) { 5598 case 0: 5599 if (nested_vmx_eptp_switching(vcpu, vmcs12)) 5600 goto fail; 5601 break; 5602 default: 5603 goto fail; 5604 } 5605 return kvm_skip_emulated_instruction(vcpu); 5606 5607 fail: 5608 /* 5609 * This is effectively a reflected VM-Exit, as opposed to a synthesized 5610 * nested VM-Exit. Pass the original exit reason, i.e. don't hardcode 5611 * EXIT_REASON_VMFUNC as the exit reason. 5612 */ 5613 nested_vmx_vmexit(vcpu, vmx->exit_reason.full, 5614 vmx_get_intr_info(vcpu), 5615 vmx_get_exit_qual(vcpu)); 5616 return 1; 5617 } 5618 5619 /* 5620 * Return true if an IO instruction with the specified port and size should cause 5621 * a VM-exit into L1. 5622 */ 5623 bool nested_vmx_check_io_bitmaps(struct kvm_vcpu *vcpu, unsigned int port, 5624 int size) 5625 { 5626 struct vmcs12 *vmcs12 = get_vmcs12(vcpu); 5627 gpa_t bitmap, last_bitmap; 5628 u8 b; 5629 5630 last_bitmap = INVALID_GPA; 5631 b = -1; 5632 5633 while (size > 0) { 5634 if (port < 0x8000) 5635 bitmap = vmcs12->io_bitmap_a; 5636 else if (port < 0x10000) 5637 bitmap = vmcs12->io_bitmap_b; 5638 else 5639 return true; 5640 bitmap += (port & 0x7fff) / 8; 5641 5642 if (last_bitmap != bitmap) 5643 if (kvm_vcpu_read_guest(vcpu, bitmap, &b, 1)) 5644 return true; 5645 if (b & (1 << (port & 7))) 5646 return true; 5647 5648 port++; 5649 size--; 5650 last_bitmap = bitmap; 5651 } 5652 5653 return false; 5654 } 5655 5656 static bool nested_vmx_exit_handled_io(struct kvm_vcpu *vcpu, 5657 struct vmcs12 *vmcs12) 5658 { 5659 unsigned long exit_qualification; 5660 unsigned short port; 5661 int size; 5662 5663 if (!nested_cpu_has(vmcs12, CPU_BASED_USE_IO_BITMAPS)) 5664 return nested_cpu_has(vmcs12, CPU_BASED_UNCOND_IO_EXITING); 5665 5666 exit_qualification = vmx_get_exit_qual(vcpu); 5667 5668 port = exit_qualification >> 16; 5669 size = (exit_qualification & 7) + 1; 5670 5671 return nested_vmx_check_io_bitmaps(vcpu, port, size); 5672 } 5673 5674 /* 5675 * Return 1 if we should exit from L2 to L1 to handle an MSR access, 5676 * rather than handle it ourselves in L0. I.e., check whether L1 expressed 5677 * disinterest in the current event (read or write a specific MSR) by using an 5678 * MSR bitmap. This may be the case even when L0 doesn't use MSR bitmaps. 5679 */ 5680 static bool nested_vmx_exit_handled_msr(struct kvm_vcpu *vcpu, 5681 struct vmcs12 *vmcs12, 5682 union vmx_exit_reason exit_reason) 5683 { 5684 u32 msr_index = kvm_rcx_read(vcpu); 5685 gpa_t bitmap; 5686 5687 if (!nested_cpu_has(vmcs12, CPU_BASED_USE_MSR_BITMAPS)) 5688 return true; 5689 5690 /* 5691 * The MSR_BITMAP page is divided into four 1024-byte bitmaps, 5692 * for the four combinations of read/write and low/high MSR numbers. 5693 * First we need to figure out which of the four to use: 5694 */ 5695 bitmap = vmcs12->msr_bitmap; 5696 if (exit_reason.basic == EXIT_REASON_MSR_WRITE) 5697 bitmap += 2048; 5698 if (msr_index >= 0xc0000000) { 5699 msr_index -= 0xc0000000; 5700 bitmap += 1024; 5701 } 5702 5703 /* Then read the msr_index'th bit from this bitmap: */ 5704 if (msr_index < 1024*8) { 5705 unsigned char b; 5706 if (kvm_vcpu_read_guest(vcpu, bitmap + msr_index/8, &b, 1)) 5707 return true; 5708 return 1 & (b >> (msr_index & 7)); 5709 } else 5710 return true; /* let L1 handle the wrong parameter */ 5711 } 5712 5713 /* 5714 * Return 1 if we should exit from L2 to L1 to handle a CR access exit, 5715 * rather than handle it ourselves in L0. I.e., check if L1 wanted to 5716 * intercept (via guest_host_mask etc.) the current event. 5717 */ 5718 static bool nested_vmx_exit_handled_cr(struct kvm_vcpu *vcpu, 5719 struct vmcs12 *vmcs12) 5720 { 5721 unsigned long exit_qualification = vmx_get_exit_qual(vcpu); 5722 int cr = exit_qualification & 15; 5723 int reg; 5724 unsigned long val; 5725 5726 switch ((exit_qualification >> 4) & 3) { 5727 case 0: /* mov to cr */ 5728 reg = (exit_qualification >> 8) & 15; 5729 val = kvm_register_read(vcpu, reg); 5730 switch (cr) { 5731 case 0: 5732 if (vmcs12->cr0_guest_host_mask & 5733 (val ^ vmcs12->cr0_read_shadow)) 5734 return true; 5735 break; 5736 case 3: 5737 if (nested_cpu_has(vmcs12, CPU_BASED_CR3_LOAD_EXITING)) 5738 return true; 5739 break; 5740 case 4: 5741 if (vmcs12->cr4_guest_host_mask & 5742 (vmcs12->cr4_read_shadow ^ val)) 5743 return true; 5744 break; 5745 case 8: 5746 if (nested_cpu_has(vmcs12, CPU_BASED_CR8_LOAD_EXITING)) 5747 return true; 5748 break; 5749 } 5750 break; 5751 case 2: /* clts */ 5752 if ((vmcs12->cr0_guest_host_mask & X86_CR0_TS) && 5753 (vmcs12->cr0_read_shadow & X86_CR0_TS)) 5754 return true; 5755 break; 5756 case 1: /* mov from cr */ 5757 switch (cr) { 5758 case 3: 5759 if (vmcs12->cpu_based_vm_exec_control & 5760 CPU_BASED_CR3_STORE_EXITING) 5761 return true; 5762 break; 5763 case 8: 5764 if (vmcs12->cpu_based_vm_exec_control & 5765 CPU_BASED_CR8_STORE_EXITING) 5766 return true; 5767 break; 5768 } 5769 break; 5770 case 3: /* lmsw */ 5771 /* 5772 * lmsw can change bits 1..3 of cr0, and only set bit 0 of 5773 * cr0. Other attempted changes are ignored, with no exit. 5774 */ 5775 val = (exit_qualification >> LMSW_SOURCE_DATA_SHIFT) & 0x0f; 5776 if (vmcs12->cr0_guest_host_mask & 0xe & 5777 (val ^ vmcs12->cr0_read_shadow)) 5778 return true; 5779 if ((vmcs12->cr0_guest_host_mask & 0x1) && 5780 !(vmcs12->cr0_read_shadow & 0x1) && 5781 (val & 0x1)) 5782 return true; 5783 break; 5784 } 5785 return false; 5786 } 5787 5788 static bool nested_vmx_exit_handled_encls(struct kvm_vcpu *vcpu, 5789 struct vmcs12 *vmcs12) 5790 { 5791 u32 encls_leaf; 5792 5793 if (!guest_cpuid_has(vcpu, X86_FEATURE_SGX) || 5794 !nested_cpu_has2(vmcs12, SECONDARY_EXEC_ENCLS_EXITING)) 5795 return false; 5796 5797 encls_leaf = kvm_rax_read(vcpu); 5798 if (encls_leaf > 62) 5799 encls_leaf = 63; 5800 return vmcs12->encls_exiting_bitmap & BIT_ULL(encls_leaf); 5801 } 5802 5803 static bool nested_vmx_exit_handled_vmcs_access(struct kvm_vcpu *vcpu, 5804 struct vmcs12 *vmcs12, gpa_t bitmap) 5805 { 5806 u32 vmx_instruction_info; 5807 unsigned long field; 5808 u8 b; 5809 5810 if (!nested_cpu_has_shadow_vmcs(vmcs12)) 5811 return true; 5812 5813 /* Decode instruction info and find the field to access */ 5814 vmx_instruction_info = vmcs_read32(VMX_INSTRUCTION_INFO); 5815 field = kvm_register_read(vcpu, (((vmx_instruction_info) >> 28) & 0xf)); 5816 5817 /* Out-of-range fields always cause a VM exit from L2 to L1 */ 5818 if (field >> 15) 5819 return true; 5820 5821 if (kvm_vcpu_read_guest(vcpu, bitmap + field/8, &b, 1)) 5822 return true; 5823 5824 return 1 & (b >> (field & 7)); 5825 } 5826 5827 static bool nested_vmx_exit_handled_mtf(struct vmcs12 *vmcs12) 5828 { 5829 u32 entry_intr_info = vmcs12->vm_entry_intr_info_field; 5830 5831 if (nested_cpu_has_mtf(vmcs12)) 5832 return true; 5833 5834 /* 5835 * An MTF VM-exit may be injected into the guest by setting the 5836 * interruption-type to 7 (other event) and the vector field to 0. Such 5837 * is the case regardless of the 'monitor trap flag' VM-execution 5838 * control. 5839 */ 5840 return entry_intr_info == (INTR_INFO_VALID_MASK 5841 | INTR_TYPE_OTHER_EVENT); 5842 } 5843 5844 /* 5845 * Return true if L0 wants to handle an exit from L2 regardless of whether or not 5846 * L1 wants the exit. Only call this when in is_guest_mode (L2). 5847 */ 5848 static bool nested_vmx_l0_wants_exit(struct kvm_vcpu *vcpu, 5849 union vmx_exit_reason exit_reason) 5850 { 5851 u32 intr_info; 5852 5853 switch ((u16)exit_reason.basic) { 5854 case EXIT_REASON_EXCEPTION_NMI: 5855 intr_info = vmx_get_intr_info(vcpu); 5856 if (is_nmi(intr_info)) 5857 return true; 5858 else if (is_page_fault(intr_info)) 5859 return vcpu->arch.apf.host_apf_flags || 5860 vmx_need_pf_intercept(vcpu); 5861 else if (is_debug(intr_info) && 5862 vcpu->guest_debug & 5863 (KVM_GUESTDBG_SINGLESTEP | KVM_GUESTDBG_USE_HW_BP)) 5864 return true; 5865 else if (is_breakpoint(intr_info) && 5866 vcpu->guest_debug & KVM_GUESTDBG_USE_SW_BP) 5867 return true; 5868 else if (is_alignment_check(intr_info) && 5869 !vmx_guest_inject_ac(vcpu)) 5870 return true; 5871 return false; 5872 case EXIT_REASON_EXTERNAL_INTERRUPT: 5873 return true; 5874 case EXIT_REASON_MCE_DURING_VMENTRY: 5875 return true; 5876 case EXIT_REASON_EPT_VIOLATION: 5877 /* 5878 * L0 always deals with the EPT violation. If nested EPT is 5879 * used, and the nested mmu code discovers that the address is 5880 * missing in the guest EPT table (EPT12), the EPT violation 5881 * will be injected with nested_ept_inject_page_fault() 5882 */ 5883 return true; 5884 case EXIT_REASON_EPT_MISCONFIG: 5885 /* 5886 * L2 never uses directly L1's EPT, but rather L0's own EPT 5887 * table (shadow on EPT) or a merged EPT table that L0 built 5888 * (EPT on EPT). So any problems with the structure of the 5889 * table is L0's fault. 5890 */ 5891 return true; 5892 case EXIT_REASON_PREEMPTION_TIMER: 5893 return true; 5894 case EXIT_REASON_PML_FULL: 5895 /* 5896 * PML is emulated for an L1 VMM and should never be enabled in 5897 * vmcs02, always "handle" PML_FULL by exiting to userspace. 5898 */ 5899 return true; 5900 case EXIT_REASON_VMFUNC: 5901 /* VM functions are emulated through L2->L0 vmexits. */ 5902 return true; 5903 case EXIT_REASON_BUS_LOCK: 5904 /* 5905 * At present, bus lock VM exit is never exposed to L1. 5906 * Handle L2's bus locks in L0 directly. 5907 */ 5908 return true; 5909 default: 5910 break; 5911 } 5912 return false; 5913 } 5914 5915 /* 5916 * Return 1 if L1 wants to intercept an exit from L2. Only call this when in 5917 * is_guest_mode (L2). 5918 */ 5919 static bool nested_vmx_l1_wants_exit(struct kvm_vcpu *vcpu, 5920 union vmx_exit_reason exit_reason) 5921 { 5922 struct vmcs12 *vmcs12 = get_vmcs12(vcpu); 5923 u32 intr_info; 5924 5925 switch ((u16)exit_reason.basic) { 5926 case EXIT_REASON_EXCEPTION_NMI: 5927 intr_info = vmx_get_intr_info(vcpu); 5928 if (is_nmi(intr_info)) 5929 return true; 5930 else if (is_page_fault(intr_info)) 5931 return true; 5932 return vmcs12->exception_bitmap & 5933 (1u << (intr_info & INTR_INFO_VECTOR_MASK)); 5934 case EXIT_REASON_EXTERNAL_INTERRUPT: 5935 return nested_exit_on_intr(vcpu); 5936 case EXIT_REASON_TRIPLE_FAULT: 5937 return true; 5938 case EXIT_REASON_INTERRUPT_WINDOW: 5939 return nested_cpu_has(vmcs12, CPU_BASED_INTR_WINDOW_EXITING); 5940 case EXIT_REASON_NMI_WINDOW: 5941 return nested_cpu_has(vmcs12, CPU_BASED_NMI_WINDOW_EXITING); 5942 case EXIT_REASON_TASK_SWITCH: 5943 return true; 5944 case EXIT_REASON_CPUID: 5945 return true; 5946 case EXIT_REASON_HLT: 5947 return nested_cpu_has(vmcs12, CPU_BASED_HLT_EXITING); 5948 case EXIT_REASON_INVD: 5949 return true; 5950 case EXIT_REASON_INVLPG: 5951 return nested_cpu_has(vmcs12, CPU_BASED_INVLPG_EXITING); 5952 case EXIT_REASON_RDPMC: 5953 return nested_cpu_has(vmcs12, CPU_BASED_RDPMC_EXITING); 5954 case EXIT_REASON_RDRAND: 5955 return nested_cpu_has2(vmcs12, SECONDARY_EXEC_RDRAND_EXITING); 5956 case EXIT_REASON_RDSEED: 5957 return nested_cpu_has2(vmcs12, SECONDARY_EXEC_RDSEED_EXITING); 5958 case EXIT_REASON_RDTSC: case EXIT_REASON_RDTSCP: 5959 return nested_cpu_has(vmcs12, CPU_BASED_RDTSC_EXITING); 5960 case EXIT_REASON_VMREAD: 5961 return nested_vmx_exit_handled_vmcs_access(vcpu, vmcs12, 5962 vmcs12->vmread_bitmap); 5963 case EXIT_REASON_VMWRITE: 5964 return nested_vmx_exit_handled_vmcs_access(vcpu, vmcs12, 5965 vmcs12->vmwrite_bitmap); 5966 case EXIT_REASON_VMCALL: case EXIT_REASON_VMCLEAR: 5967 case EXIT_REASON_VMLAUNCH: case EXIT_REASON_VMPTRLD: 5968 case EXIT_REASON_VMPTRST: case EXIT_REASON_VMRESUME: 5969 case EXIT_REASON_VMOFF: case EXIT_REASON_VMON: 5970 case EXIT_REASON_INVEPT: case EXIT_REASON_INVVPID: 5971 /* 5972 * VMX instructions trap unconditionally. This allows L1 to 5973 * emulate them for its L2 guest, i.e., allows 3-level nesting! 5974 */ 5975 return true; 5976 case EXIT_REASON_CR_ACCESS: 5977 return nested_vmx_exit_handled_cr(vcpu, vmcs12); 5978 case EXIT_REASON_DR_ACCESS: 5979 return nested_cpu_has(vmcs12, CPU_BASED_MOV_DR_EXITING); 5980 case EXIT_REASON_IO_INSTRUCTION: 5981 return nested_vmx_exit_handled_io(vcpu, vmcs12); 5982 case EXIT_REASON_GDTR_IDTR: case EXIT_REASON_LDTR_TR: 5983 return nested_cpu_has2(vmcs12, SECONDARY_EXEC_DESC); 5984 case EXIT_REASON_MSR_READ: 5985 case EXIT_REASON_MSR_WRITE: 5986 return nested_vmx_exit_handled_msr(vcpu, vmcs12, exit_reason); 5987 case EXIT_REASON_INVALID_STATE: 5988 return true; 5989 case EXIT_REASON_MWAIT_INSTRUCTION: 5990 return nested_cpu_has(vmcs12, CPU_BASED_MWAIT_EXITING); 5991 case EXIT_REASON_MONITOR_TRAP_FLAG: 5992 return nested_vmx_exit_handled_mtf(vmcs12); 5993 case EXIT_REASON_MONITOR_INSTRUCTION: 5994 return nested_cpu_has(vmcs12, CPU_BASED_MONITOR_EXITING); 5995 case EXIT_REASON_PAUSE_INSTRUCTION: 5996 return nested_cpu_has(vmcs12, CPU_BASED_PAUSE_EXITING) || 5997 nested_cpu_has2(vmcs12, 5998 SECONDARY_EXEC_PAUSE_LOOP_EXITING); 5999 case EXIT_REASON_MCE_DURING_VMENTRY: 6000 return true; 6001 case EXIT_REASON_TPR_BELOW_THRESHOLD: 6002 return nested_cpu_has(vmcs12, CPU_BASED_TPR_SHADOW); 6003 case EXIT_REASON_APIC_ACCESS: 6004 case EXIT_REASON_APIC_WRITE: 6005 case EXIT_REASON_EOI_INDUCED: 6006 /* 6007 * The controls for "virtualize APIC accesses," "APIC- 6008 * register virtualization," and "virtual-interrupt 6009 * delivery" only come from vmcs12. 6010 */ 6011 return true; 6012 case EXIT_REASON_INVPCID: 6013 return 6014 nested_cpu_has2(vmcs12, SECONDARY_EXEC_ENABLE_INVPCID) && 6015 nested_cpu_has(vmcs12, CPU_BASED_INVLPG_EXITING); 6016 case EXIT_REASON_WBINVD: 6017 return nested_cpu_has2(vmcs12, SECONDARY_EXEC_WBINVD_EXITING); 6018 case EXIT_REASON_XSETBV: 6019 return true; 6020 case EXIT_REASON_XSAVES: case EXIT_REASON_XRSTORS: 6021 /* 6022 * This should never happen, since it is not possible to 6023 * set XSS to a non-zero value---neither in L1 nor in L2. 6024 * If if it were, XSS would have to be checked against 6025 * the XSS exit bitmap in vmcs12. 6026 */ 6027 return nested_cpu_has2(vmcs12, SECONDARY_EXEC_XSAVES); 6028 case EXIT_REASON_UMWAIT: 6029 case EXIT_REASON_TPAUSE: 6030 return nested_cpu_has2(vmcs12, 6031 SECONDARY_EXEC_ENABLE_USR_WAIT_PAUSE); 6032 case EXIT_REASON_ENCLS: 6033 return nested_vmx_exit_handled_encls(vcpu, vmcs12); 6034 default: 6035 return true; 6036 } 6037 } 6038 6039 /* 6040 * Conditionally reflect a VM-Exit into L1. Returns %true if the VM-Exit was 6041 * reflected into L1. 6042 */ 6043 bool nested_vmx_reflect_vmexit(struct kvm_vcpu *vcpu) 6044 { 6045 struct vcpu_vmx *vmx = to_vmx(vcpu); 6046 union vmx_exit_reason exit_reason = vmx->exit_reason; 6047 unsigned long exit_qual; 6048 u32 exit_intr_info; 6049 6050 WARN_ON_ONCE(vmx->nested.nested_run_pending); 6051 6052 /* 6053 * Late nested VM-Fail shares the same flow as nested VM-Exit since KVM 6054 * has already loaded L2's state. 6055 */ 6056 if (unlikely(vmx->fail)) { 6057 trace_kvm_nested_vmenter_failed( 6058 "hardware VM-instruction error: ", 6059 vmcs_read32(VM_INSTRUCTION_ERROR)); 6060 exit_intr_info = 0; 6061 exit_qual = 0; 6062 goto reflect_vmexit; 6063 } 6064 6065 trace_kvm_nested_vmexit(vcpu, KVM_ISA_VMX); 6066 6067 /* If L0 (KVM) wants the exit, it trumps L1's desires. */ 6068 if (nested_vmx_l0_wants_exit(vcpu, exit_reason)) 6069 return false; 6070 6071 /* If L1 doesn't want the exit, handle it in L0. */ 6072 if (!nested_vmx_l1_wants_exit(vcpu, exit_reason)) 6073 return false; 6074 6075 /* 6076 * vmcs.VM_EXIT_INTR_INFO is only valid for EXCEPTION_NMI exits. For 6077 * EXTERNAL_INTERRUPT, the value for vmcs12->vm_exit_intr_info would 6078 * need to be synthesized by querying the in-kernel LAPIC, but external 6079 * interrupts are never reflected to L1 so it's a non-issue. 6080 */ 6081 exit_intr_info = vmx_get_intr_info(vcpu); 6082 if (is_exception_with_error_code(exit_intr_info)) { 6083 struct vmcs12 *vmcs12 = get_vmcs12(vcpu); 6084 6085 vmcs12->vm_exit_intr_error_code = 6086 vmcs_read32(VM_EXIT_INTR_ERROR_CODE); 6087 } 6088 exit_qual = vmx_get_exit_qual(vcpu); 6089 6090 reflect_vmexit: 6091 nested_vmx_vmexit(vcpu, exit_reason.full, exit_intr_info, exit_qual); 6092 return true; 6093 } 6094 6095 static int vmx_get_nested_state(struct kvm_vcpu *vcpu, 6096 struct kvm_nested_state __user *user_kvm_nested_state, 6097 u32 user_data_size) 6098 { 6099 struct vcpu_vmx *vmx; 6100 struct vmcs12 *vmcs12; 6101 struct kvm_nested_state kvm_state = { 6102 .flags = 0, 6103 .format = KVM_STATE_NESTED_FORMAT_VMX, 6104 .size = sizeof(kvm_state), 6105 .hdr.vmx.flags = 0, 6106 .hdr.vmx.vmxon_pa = INVALID_GPA, 6107 .hdr.vmx.vmcs12_pa = INVALID_GPA, 6108 .hdr.vmx.preemption_timer_deadline = 0, 6109 }; 6110 struct kvm_vmx_nested_state_data __user *user_vmx_nested_state = 6111 &user_kvm_nested_state->data.vmx[0]; 6112 6113 if (!vcpu) 6114 return kvm_state.size + sizeof(*user_vmx_nested_state); 6115 6116 vmx = to_vmx(vcpu); 6117 vmcs12 = get_vmcs12(vcpu); 6118 6119 if (nested_vmx_allowed(vcpu) && 6120 (vmx->nested.vmxon || vmx->nested.smm.vmxon)) { 6121 kvm_state.hdr.vmx.vmxon_pa = vmx->nested.vmxon_ptr; 6122 kvm_state.hdr.vmx.vmcs12_pa = vmx->nested.current_vmptr; 6123 6124 if (vmx_has_valid_vmcs12(vcpu)) { 6125 kvm_state.size += sizeof(user_vmx_nested_state->vmcs12); 6126 6127 /* 'hv_evmcs_vmptr' can also be EVMPTR_MAP_PENDING here */ 6128 if (vmx->nested.hv_evmcs_vmptr != EVMPTR_INVALID) 6129 kvm_state.flags |= KVM_STATE_NESTED_EVMCS; 6130 6131 if (is_guest_mode(vcpu) && 6132 nested_cpu_has_shadow_vmcs(vmcs12) && 6133 vmcs12->vmcs_link_pointer != INVALID_GPA) 6134 kvm_state.size += sizeof(user_vmx_nested_state->shadow_vmcs12); 6135 } 6136 6137 if (vmx->nested.smm.vmxon) 6138 kvm_state.hdr.vmx.smm.flags |= KVM_STATE_NESTED_SMM_VMXON; 6139 6140 if (vmx->nested.smm.guest_mode) 6141 kvm_state.hdr.vmx.smm.flags |= KVM_STATE_NESTED_SMM_GUEST_MODE; 6142 6143 if (is_guest_mode(vcpu)) { 6144 kvm_state.flags |= KVM_STATE_NESTED_GUEST_MODE; 6145 6146 if (vmx->nested.nested_run_pending) 6147 kvm_state.flags |= KVM_STATE_NESTED_RUN_PENDING; 6148 6149 if (vmx->nested.mtf_pending) 6150 kvm_state.flags |= KVM_STATE_NESTED_MTF_PENDING; 6151 6152 if (nested_cpu_has_preemption_timer(vmcs12) && 6153 vmx->nested.has_preemption_timer_deadline) { 6154 kvm_state.hdr.vmx.flags |= 6155 KVM_STATE_VMX_PREEMPTION_TIMER_DEADLINE; 6156 kvm_state.hdr.vmx.preemption_timer_deadline = 6157 vmx->nested.preemption_timer_deadline; 6158 } 6159 } 6160 } 6161 6162 if (user_data_size < kvm_state.size) 6163 goto out; 6164 6165 if (copy_to_user(user_kvm_nested_state, &kvm_state, sizeof(kvm_state))) 6166 return -EFAULT; 6167 6168 if (!vmx_has_valid_vmcs12(vcpu)) 6169 goto out; 6170 6171 /* 6172 * When running L2, the authoritative vmcs12 state is in the 6173 * vmcs02. When running L1, the authoritative vmcs12 state is 6174 * in the shadow or enlightened vmcs linked to vmcs01, unless 6175 * need_vmcs12_to_shadow_sync is set, in which case, the authoritative 6176 * vmcs12 state is in the vmcs12 already. 6177 */ 6178 if (is_guest_mode(vcpu)) { 6179 sync_vmcs02_to_vmcs12(vcpu, vmcs12); 6180 sync_vmcs02_to_vmcs12_rare(vcpu, vmcs12); 6181 } else { 6182 copy_vmcs02_to_vmcs12_rare(vcpu, get_vmcs12(vcpu)); 6183 if (!vmx->nested.need_vmcs12_to_shadow_sync) { 6184 if (evmptr_is_valid(vmx->nested.hv_evmcs_vmptr)) 6185 /* 6186 * L1 hypervisor is not obliged to keep eVMCS 6187 * clean fields data always up-to-date while 6188 * not in guest mode, 'hv_clean_fields' is only 6189 * supposed to be actual upon vmentry so we need 6190 * to ignore it here and do full copy. 6191 */ 6192 copy_enlightened_to_vmcs12(vmx, 0); 6193 else if (enable_shadow_vmcs) 6194 copy_shadow_to_vmcs12(vmx); 6195 } 6196 } 6197 6198 BUILD_BUG_ON(sizeof(user_vmx_nested_state->vmcs12) < VMCS12_SIZE); 6199 BUILD_BUG_ON(sizeof(user_vmx_nested_state->shadow_vmcs12) < VMCS12_SIZE); 6200 6201 /* 6202 * Copy over the full allocated size of vmcs12 rather than just the size 6203 * of the struct. 6204 */ 6205 if (copy_to_user(user_vmx_nested_state->vmcs12, vmcs12, VMCS12_SIZE)) 6206 return -EFAULT; 6207 6208 if (nested_cpu_has_shadow_vmcs(vmcs12) && 6209 vmcs12->vmcs_link_pointer != INVALID_GPA) { 6210 if (copy_to_user(user_vmx_nested_state->shadow_vmcs12, 6211 get_shadow_vmcs12(vcpu), VMCS12_SIZE)) 6212 return -EFAULT; 6213 } 6214 out: 6215 return kvm_state.size; 6216 } 6217 6218 /* 6219 * Forcibly leave nested mode in order to be able to reset the VCPU later on. 6220 */ 6221 void vmx_leave_nested(struct kvm_vcpu *vcpu) 6222 { 6223 if (is_guest_mode(vcpu)) { 6224 to_vmx(vcpu)->nested.nested_run_pending = 0; 6225 nested_vmx_vmexit(vcpu, -1, 0, 0); 6226 } 6227 free_nested(vcpu); 6228 } 6229 6230 static int vmx_set_nested_state(struct kvm_vcpu *vcpu, 6231 struct kvm_nested_state __user *user_kvm_nested_state, 6232 struct kvm_nested_state *kvm_state) 6233 { 6234 struct vcpu_vmx *vmx = to_vmx(vcpu); 6235 struct vmcs12 *vmcs12; 6236 enum vm_entry_failure_code ignored; 6237 struct kvm_vmx_nested_state_data __user *user_vmx_nested_state = 6238 &user_kvm_nested_state->data.vmx[0]; 6239 int ret; 6240 6241 if (kvm_state->format != KVM_STATE_NESTED_FORMAT_VMX) 6242 return -EINVAL; 6243 6244 if (kvm_state->hdr.vmx.vmxon_pa == INVALID_GPA) { 6245 if (kvm_state->hdr.vmx.smm.flags) 6246 return -EINVAL; 6247 6248 if (kvm_state->hdr.vmx.vmcs12_pa != INVALID_GPA) 6249 return -EINVAL; 6250 6251 /* 6252 * KVM_STATE_NESTED_EVMCS used to signal that KVM should 6253 * enable eVMCS capability on vCPU. However, since then 6254 * code was changed such that flag signals vmcs12 should 6255 * be copied into eVMCS in guest memory. 6256 * 6257 * To preserve backwards compatability, allow user 6258 * to set this flag even when there is no VMXON region. 6259 */ 6260 if (kvm_state->flags & ~KVM_STATE_NESTED_EVMCS) 6261 return -EINVAL; 6262 } else { 6263 if (!nested_vmx_allowed(vcpu)) 6264 return -EINVAL; 6265 6266 if (!page_address_valid(vcpu, kvm_state->hdr.vmx.vmxon_pa)) 6267 return -EINVAL; 6268 } 6269 6270 if ((kvm_state->hdr.vmx.smm.flags & KVM_STATE_NESTED_SMM_GUEST_MODE) && 6271 (kvm_state->flags & KVM_STATE_NESTED_GUEST_MODE)) 6272 return -EINVAL; 6273 6274 if (kvm_state->hdr.vmx.smm.flags & 6275 ~(KVM_STATE_NESTED_SMM_GUEST_MODE | KVM_STATE_NESTED_SMM_VMXON)) 6276 return -EINVAL; 6277 6278 if (kvm_state->hdr.vmx.flags & ~KVM_STATE_VMX_PREEMPTION_TIMER_DEADLINE) 6279 return -EINVAL; 6280 6281 /* 6282 * SMM temporarily disables VMX, so we cannot be in guest mode, 6283 * nor can VMLAUNCH/VMRESUME be pending. Outside SMM, SMM flags 6284 * must be zero. 6285 */ 6286 if (is_smm(vcpu) ? 6287 (kvm_state->flags & 6288 (KVM_STATE_NESTED_GUEST_MODE | KVM_STATE_NESTED_RUN_PENDING)) 6289 : kvm_state->hdr.vmx.smm.flags) 6290 return -EINVAL; 6291 6292 if ((kvm_state->hdr.vmx.smm.flags & KVM_STATE_NESTED_SMM_GUEST_MODE) && 6293 !(kvm_state->hdr.vmx.smm.flags & KVM_STATE_NESTED_SMM_VMXON)) 6294 return -EINVAL; 6295 6296 if ((kvm_state->flags & KVM_STATE_NESTED_EVMCS) && 6297 (!nested_vmx_allowed(vcpu) || !vmx->nested.enlightened_vmcs_enabled)) 6298 return -EINVAL; 6299 6300 vmx_leave_nested(vcpu); 6301 6302 if (kvm_state->hdr.vmx.vmxon_pa == INVALID_GPA) 6303 return 0; 6304 6305 vmx->nested.vmxon_ptr = kvm_state->hdr.vmx.vmxon_pa; 6306 ret = enter_vmx_operation(vcpu); 6307 if (ret) 6308 return ret; 6309 6310 /* Empty 'VMXON' state is permitted if no VMCS loaded */ 6311 if (kvm_state->size < sizeof(*kvm_state) + sizeof(*vmcs12)) { 6312 /* See vmx_has_valid_vmcs12. */ 6313 if ((kvm_state->flags & KVM_STATE_NESTED_GUEST_MODE) || 6314 (kvm_state->flags & KVM_STATE_NESTED_EVMCS) || 6315 (kvm_state->hdr.vmx.vmcs12_pa != INVALID_GPA)) 6316 return -EINVAL; 6317 else 6318 return 0; 6319 } 6320 6321 if (kvm_state->hdr.vmx.vmcs12_pa != INVALID_GPA) { 6322 if (kvm_state->hdr.vmx.vmcs12_pa == kvm_state->hdr.vmx.vmxon_pa || 6323 !page_address_valid(vcpu, kvm_state->hdr.vmx.vmcs12_pa)) 6324 return -EINVAL; 6325 6326 set_current_vmptr(vmx, kvm_state->hdr.vmx.vmcs12_pa); 6327 } else if (kvm_state->flags & KVM_STATE_NESTED_EVMCS) { 6328 /* 6329 * nested_vmx_handle_enlightened_vmptrld() cannot be called 6330 * directly from here as HV_X64_MSR_VP_ASSIST_PAGE may not be 6331 * restored yet. EVMCS will be mapped from 6332 * nested_get_vmcs12_pages(). 6333 */ 6334 vmx->nested.hv_evmcs_vmptr = EVMPTR_MAP_PENDING; 6335 kvm_make_request(KVM_REQ_GET_NESTED_STATE_PAGES, vcpu); 6336 } else { 6337 return -EINVAL; 6338 } 6339 6340 if (kvm_state->hdr.vmx.smm.flags & KVM_STATE_NESTED_SMM_VMXON) { 6341 vmx->nested.smm.vmxon = true; 6342 vmx->nested.vmxon = false; 6343 6344 if (kvm_state->hdr.vmx.smm.flags & KVM_STATE_NESTED_SMM_GUEST_MODE) 6345 vmx->nested.smm.guest_mode = true; 6346 } 6347 6348 vmcs12 = get_vmcs12(vcpu); 6349 if (copy_from_user(vmcs12, user_vmx_nested_state->vmcs12, sizeof(*vmcs12))) 6350 return -EFAULT; 6351 6352 if (vmcs12->hdr.revision_id != VMCS12_REVISION) 6353 return -EINVAL; 6354 6355 if (!(kvm_state->flags & KVM_STATE_NESTED_GUEST_MODE)) 6356 return 0; 6357 6358 vmx->nested.nested_run_pending = 6359 !!(kvm_state->flags & KVM_STATE_NESTED_RUN_PENDING); 6360 6361 vmx->nested.mtf_pending = 6362 !!(kvm_state->flags & KVM_STATE_NESTED_MTF_PENDING); 6363 6364 ret = -EINVAL; 6365 if (nested_cpu_has_shadow_vmcs(vmcs12) && 6366 vmcs12->vmcs_link_pointer != INVALID_GPA) { 6367 struct vmcs12 *shadow_vmcs12 = get_shadow_vmcs12(vcpu); 6368 6369 if (kvm_state->size < 6370 sizeof(*kvm_state) + 6371 sizeof(user_vmx_nested_state->vmcs12) + sizeof(*shadow_vmcs12)) 6372 goto error_guest_mode; 6373 6374 if (copy_from_user(shadow_vmcs12, 6375 user_vmx_nested_state->shadow_vmcs12, 6376 sizeof(*shadow_vmcs12))) { 6377 ret = -EFAULT; 6378 goto error_guest_mode; 6379 } 6380 6381 if (shadow_vmcs12->hdr.revision_id != VMCS12_REVISION || 6382 !shadow_vmcs12->hdr.shadow_vmcs) 6383 goto error_guest_mode; 6384 } 6385 6386 vmx->nested.has_preemption_timer_deadline = false; 6387 if (kvm_state->hdr.vmx.flags & KVM_STATE_VMX_PREEMPTION_TIMER_DEADLINE) { 6388 vmx->nested.has_preemption_timer_deadline = true; 6389 vmx->nested.preemption_timer_deadline = 6390 kvm_state->hdr.vmx.preemption_timer_deadline; 6391 } 6392 6393 if (nested_vmx_check_controls(vcpu, vmcs12) || 6394 nested_vmx_check_host_state(vcpu, vmcs12) || 6395 nested_vmx_check_guest_state(vcpu, vmcs12, &ignored)) 6396 goto error_guest_mode; 6397 6398 vmx->nested.dirty_vmcs12 = true; 6399 ret = nested_vmx_enter_non_root_mode(vcpu, false); 6400 if (ret) 6401 goto error_guest_mode; 6402 6403 return 0; 6404 6405 error_guest_mode: 6406 vmx->nested.nested_run_pending = 0; 6407 return ret; 6408 } 6409 6410 void nested_vmx_set_vmcs_shadowing_bitmap(void) 6411 { 6412 if (enable_shadow_vmcs) { 6413 vmcs_write64(VMREAD_BITMAP, __pa(vmx_vmread_bitmap)); 6414 vmcs_write64(VMWRITE_BITMAP, __pa(vmx_vmwrite_bitmap)); 6415 } 6416 } 6417 6418 /* 6419 * Indexing into the vmcs12 uses the VMCS encoding rotated left by 6. Undo 6420 * that madness to get the encoding for comparison. 6421 */ 6422 #define VMCS12_IDX_TO_ENC(idx) ((u16)(((u16)(idx) >> 6) | ((u16)(idx) << 10))) 6423 6424 static u64 nested_vmx_calc_vmcs_enum_msr(void) 6425 { 6426 /* 6427 * Note these are the so called "index" of the VMCS field encoding, not 6428 * the index into vmcs12. 6429 */ 6430 unsigned int max_idx, idx; 6431 int i; 6432 6433 /* 6434 * For better or worse, KVM allows VMREAD/VMWRITE to all fields in 6435 * vmcs12, regardless of whether or not the associated feature is 6436 * exposed to L1. Simply find the field with the highest index. 6437 */ 6438 max_idx = 0; 6439 for (i = 0; i < nr_vmcs12_fields; i++) { 6440 /* The vmcs12 table is very, very sparsely populated. */ 6441 if (!vmcs_field_to_offset_table[i]) 6442 continue; 6443 6444 idx = vmcs_field_index(VMCS12_IDX_TO_ENC(i)); 6445 if (idx > max_idx) 6446 max_idx = idx; 6447 } 6448 6449 return (u64)max_idx << VMCS_FIELD_INDEX_SHIFT; 6450 } 6451 6452 /* 6453 * nested_vmx_setup_ctls_msrs() sets up variables containing the values to be 6454 * returned for the various VMX controls MSRs when nested VMX is enabled. 6455 * The same values should also be used to verify that vmcs12 control fields are 6456 * valid during nested entry from L1 to L2. 6457 * Each of these control msrs has a low and high 32-bit half: A low bit is on 6458 * if the corresponding bit in the (32-bit) control field *must* be on, and a 6459 * bit in the high half is on if the corresponding bit in the control field 6460 * may be on. See also vmx_control_verify(). 6461 */ 6462 void nested_vmx_setup_ctls_msrs(struct nested_vmx_msrs *msrs, u32 ept_caps) 6463 { 6464 /* 6465 * Note that as a general rule, the high half of the MSRs (bits in 6466 * the control fields which may be 1) should be initialized by the 6467 * intersection of the underlying hardware's MSR (i.e., features which 6468 * can be supported) and the list of features we want to expose - 6469 * because they are known to be properly supported in our code. 6470 * Also, usually, the low half of the MSRs (bits which must be 1) can 6471 * be set to 0, meaning that L1 may turn off any of these bits. The 6472 * reason is that if one of these bits is necessary, it will appear 6473 * in vmcs01 and prepare_vmcs02, when it bitwise-or's the control 6474 * fields of vmcs01 and vmcs02, will turn these bits off - and 6475 * nested_vmx_l1_wants_exit() will not pass related exits to L1. 6476 * These rules have exceptions below. 6477 */ 6478 6479 /* pin-based controls */ 6480 rdmsr(MSR_IA32_VMX_PINBASED_CTLS, 6481 msrs->pinbased_ctls_low, 6482 msrs->pinbased_ctls_high); 6483 msrs->pinbased_ctls_low |= 6484 PIN_BASED_ALWAYSON_WITHOUT_TRUE_MSR; 6485 msrs->pinbased_ctls_high &= 6486 PIN_BASED_EXT_INTR_MASK | 6487 PIN_BASED_NMI_EXITING | 6488 PIN_BASED_VIRTUAL_NMIS | 6489 (enable_apicv ? PIN_BASED_POSTED_INTR : 0); 6490 msrs->pinbased_ctls_high |= 6491 PIN_BASED_ALWAYSON_WITHOUT_TRUE_MSR | 6492 PIN_BASED_VMX_PREEMPTION_TIMER; 6493 6494 /* exit controls */ 6495 rdmsr(MSR_IA32_VMX_EXIT_CTLS, 6496 msrs->exit_ctls_low, 6497 msrs->exit_ctls_high); 6498 msrs->exit_ctls_low = 6499 VM_EXIT_ALWAYSON_WITHOUT_TRUE_MSR; 6500 6501 msrs->exit_ctls_high &= 6502 #ifdef CONFIG_X86_64 6503 VM_EXIT_HOST_ADDR_SPACE_SIZE | 6504 #endif 6505 VM_EXIT_LOAD_IA32_PAT | VM_EXIT_SAVE_IA32_PAT | 6506 VM_EXIT_CLEAR_BNDCFGS | VM_EXIT_LOAD_IA32_PERF_GLOBAL_CTRL; 6507 msrs->exit_ctls_high |= 6508 VM_EXIT_ALWAYSON_WITHOUT_TRUE_MSR | 6509 VM_EXIT_LOAD_IA32_EFER | VM_EXIT_SAVE_IA32_EFER | 6510 VM_EXIT_SAVE_VMX_PREEMPTION_TIMER | VM_EXIT_ACK_INTR_ON_EXIT; 6511 6512 /* We support free control of debug control saving. */ 6513 msrs->exit_ctls_low &= ~VM_EXIT_SAVE_DEBUG_CONTROLS; 6514 6515 /* entry controls */ 6516 rdmsr(MSR_IA32_VMX_ENTRY_CTLS, 6517 msrs->entry_ctls_low, 6518 msrs->entry_ctls_high); 6519 msrs->entry_ctls_low = 6520 VM_ENTRY_ALWAYSON_WITHOUT_TRUE_MSR; 6521 msrs->entry_ctls_high &= 6522 #ifdef CONFIG_X86_64 6523 VM_ENTRY_IA32E_MODE | 6524 #endif 6525 VM_ENTRY_LOAD_IA32_PAT | VM_ENTRY_LOAD_BNDCFGS | 6526 VM_ENTRY_LOAD_IA32_PERF_GLOBAL_CTRL; 6527 msrs->entry_ctls_high |= 6528 (VM_ENTRY_ALWAYSON_WITHOUT_TRUE_MSR | VM_ENTRY_LOAD_IA32_EFER); 6529 6530 /* We support free control of debug control loading. */ 6531 msrs->entry_ctls_low &= ~VM_ENTRY_LOAD_DEBUG_CONTROLS; 6532 6533 /* cpu-based controls */ 6534 rdmsr(MSR_IA32_VMX_PROCBASED_CTLS, 6535 msrs->procbased_ctls_low, 6536 msrs->procbased_ctls_high); 6537 msrs->procbased_ctls_low = 6538 CPU_BASED_ALWAYSON_WITHOUT_TRUE_MSR; 6539 msrs->procbased_ctls_high &= 6540 CPU_BASED_INTR_WINDOW_EXITING | 6541 CPU_BASED_NMI_WINDOW_EXITING | CPU_BASED_USE_TSC_OFFSETTING | 6542 CPU_BASED_HLT_EXITING | CPU_BASED_INVLPG_EXITING | 6543 CPU_BASED_MWAIT_EXITING | CPU_BASED_CR3_LOAD_EXITING | 6544 CPU_BASED_CR3_STORE_EXITING | 6545 #ifdef CONFIG_X86_64 6546 CPU_BASED_CR8_LOAD_EXITING | CPU_BASED_CR8_STORE_EXITING | 6547 #endif 6548 CPU_BASED_MOV_DR_EXITING | CPU_BASED_UNCOND_IO_EXITING | 6549 CPU_BASED_USE_IO_BITMAPS | CPU_BASED_MONITOR_TRAP_FLAG | 6550 CPU_BASED_MONITOR_EXITING | CPU_BASED_RDPMC_EXITING | 6551 CPU_BASED_RDTSC_EXITING | CPU_BASED_PAUSE_EXITING | 6552 CPU_BASED_TPR_SHADOW | CPU_BASED_ACTIVATE_SECONDARY_CONTROLS; 6553 /* 6554 * We can allow some features even when not supported by the 6555 * hardware. For example, L1 can specify an MSR bitmap - and we 6556 * can use it to avoid exits to L1 - even when L0 runs L2 6557 * without MSR bitmaps. 6558 */ 6559 msrs->procbased_ctls_high |= 6560 CPU_BASED_ALWAYSON_WITHOUT_TRUE_MSR | 6561 CPU_BASED_USE_MSR_BITMAPS; 6562 6563 /* We support free control of CR3 access interception. */ 6564 msrs->procbased_ctls_low &= 6565 ~(CPU_BASED_CR3_LOAD_EXITING | CPU_BASED_CR3_STORE_EXITING); 6566 6567 /* 6568 * secondary cpu-based controls. Do not include those that 6569 * depend on CPUID bits, they are added later by 6570 * vmx_vcpu_after_set_cpuid. 6571 */ 6572 if (msrs->procbased_ctls_high & CPU_BASED_ACTIVATE_SECONDARY_CONTROLS) 6573 rdmsr(MSR_IA32_VMX_PROCBASED_CTLS2, 6574 msrs->secondary_ctls_low, 6575 msrs->secondary_ctls_high); 6576 6577 msrs->secondary_ctls_low = 0; 6578 msrs->secondary_ctls_high &= 6579 SECONDARY_EXEC_DESC | 6580 SECONDARY_EXEC_ENABLE_RDTSCP | 6581 SECONDARY_EXEC_VIRTUALIZE_X2APIC_MODE | 6582 SECONDARY_EXEC_WBINVD_EXITING | 6583 SECONDARY_EXEC_APIC_REGISTER_VIRT | 6584 SECONDARY_EXEC_VIRTUAL_INTR_DELIVERY | 6585 SECONDARY_EXEC_RDRAND_EXITING | 6586 SECONDARY_EXEC_ENABLE_INVPCID | 6587 SECONDARY_EXEC_RDSEED_EXITING | 6588 SECONDARY_EXEC_XSAVES | 6589 SECONDARY_EXEC_TSC_SCALING; 6590 6591 /* 6592 * We can emulate "VMCS shadowing," even if the hardware 6593 * doesn't support it. 6594 */ 6595 msrs->secondary_ctls_high |= 6596 SECONDARY_EXEC_SHADOW_VMCS; 6597 6598 if (enable_ept) { 6599 /* nested EPT: emulate EPT also to L1 */ 6600 msrs->secondary_ctls_high |= 6601 SECONDARY_EXEC_ENABLE_EPT; 6602 msrs->ept_caps = 6603 VMX_EPT_PAGE_WALK_4_BIT | 6604 VMX_EPT_PAGE_WALK_5_BIT | 6605 VMX_EPTP_WB_BIT | 6606 VMX_EPT_INVEPT_BIT | 6607 VMX_EPT_EXECUTE_ONLY_BIT; 6608 6609 msrs->ept_caps &= ept_caps; 6610 msrs->ept_caps |= VMX_EPT_EXTENT_GLOBAL_BIT | 6611 VMX_EPT_EXTENT_CONTEXT_BIT | VMX_EPT_2MB_PAGE_BIT | 6612 VMX_EPT_1GB_PAGE_BIT; 6613 if (enable_ept_ad_bits) { 6614 msrs->secondary_ctls_high |= 6615 SECONDARY_EXEC_ENABLE_PML; 6616 msrs->ept_caps |= VMX_EPT_AD_BIT; 6617 } 6618 } 6619 6620 if (cpu_has_vmx_vmfunc()) { 6621 msrs->secondary_ctls_high |= 6622 SECONDARY_EXEC_ENABLE_VMFUNC; 6623 /* 6624 * Advertise EPTP switching unconditionally 6625 * since we emulate it 6626 */ 6627 if (enable_ept) 6628 msrs->vmfunc_controls = 6629 VMX_VMFUNC_EPTP_SWITCHING; 6630 } 6631 6632 /* 6633 * Old versions of KVM use the single-context version without 6634 * checking for support, so declare that it is supported even 6635 * though it is treated as global context. The alternative is 6636 * not failing the single-context invvpid, and it is worse. 6637 */ 6638 if (enable_vpid) { 6639 msrs->secondary_ctls_high |= 6640 SECONDARY_EXEC_ENABLE_VPID; 6641 msrs->vpid_caps = VMX_VPID_INVVPID_BIT | 6642 VMX_VPID_EXTENT_SUPPORTED_MASK; 6643 } 6644 6645 if (enable_unrestricted_guest) 6646 msrs->secondary_ctls_high |= 6647 SECONDARY_EXEC_UNRESTRICTED_GUEST; 6648 6649 if (flexpriority_enabled) 6650 msrs->secondary_ctls_high |= 6651 SECONDARY_EXEC_VIRTUALIZE_APIC_ACCESSES; 6652 6653 if (enable_sgx) 6654 msrs->secondary_ctls_high |= SECONDARY_EXEC_ENCLS_EXITING; 6655 6656 /* miscellaneous data */ 6657 rdmsr(MSR_IA32_VMX_MISC, 6658 msrs->misc_low, 6659 msrs->misc_high); 6660 msrs->misc_low &= VMX_MISC_SAVE_EFER_LMA; 6661 msrs->misc_low |= 6662 MSR_IA32_VMX_MISC_VMWRITE_SHADOW_RO_FIELDS | 6663 VMX_MISC_EMULATED_PREEMPTION_TIMER_RATE | 6664 VMX_MISC_ACTIVITY_HLT | 6665 VMX_MISC_ACTIVITY_WAIT_SIPI; 6666 msrs->misc_high = 0; 6667 6668 /* 6669 * This MSR reports some information about VMX support. We 6670 * should return information about the VMX we emulate for the 6671 * guest, and the VMCS structure we give it - not about the 6672 * VMX support of the underlying hardware. 6673 */ 6674 msrs->basic = 6675 VMCS12_REVISION | 6676 VMX_BASIC_TRUE_CTLS | 6677 ((u64)VMCS12_SIZE << VMX_BASIC_VMCS_SIZE_SHIFT) | 6678 (VMX_BASIC_MEM_TYPE_WB << VMX_BASIC_MEM_TYPE_SHIFT); 6679 6680 if (cpu_has_vmx_basic_inout()) 6681 msrs->basic |= VMX_BASIC_INOUT; 6682 6683 /* 6684 * These MSRs specify bits which the guest must keep fixed on 6685 * while L1 is in VMXON mode (in L1's root mode, or running an L2). 6686 * We picked the standard core2 setting. 6687 */ 6688 #define VMXON_CR0_ALWAYSON (X86_CR0_PE | X86_CR0_PG | X86_CR0_NE) 6689 #define VMXON_CR4_ALWAYSON X86_CR4_VMXE 6690 msrs->cr0_fixed0 = VMXON_CR0_ALWAYSON; 6691 msrs->cr4_fixed0 = VMXON_CR4_ALWAYSON; 6692 6693 /* These MSRs specify bits which the guest must keep fixed off. */ 6694 rdmsrl(MSR_IA32_VMX_CR0_FIXED1, msrs->cr0_fixed1); 6695 rdmsrl(MSR_IA32_VMX_CR4_FIXED1, msrs->cr4_fixed1); 6696 6697 msrs->vmcs_enum = nested_vmx_calc_vmcs_enum_msr(); 6698 } 6699 6700 void nested_vmx_hardware_unsetup(void) 6701 { 6702 int i; 6703 6704 if (enable_shadow_vmcs) { 6705 for (i = 0; i < VMX_BITMAP_NR; i++) 6706 free_page((unsigned long)vmx_bitmap[i]); 6707 } 6708 } 6709 6710 __init int nested_vmx_hardware_setup(int (*exit_handlers[])(struct kvm_vcpu *)) 6711 { 6712 int i; 6713 6714 if (!cpu_has_vmx_shadow_vmcs()) 6715 enable_shadow_vmcs = 0; 6716 if (enable_shadow_vmcs) { 6717 for (i = 0; i < VMX_BITMAP_NR; i++) { 6718 /* 6719 * The vmx_bitmap is not tied to a VM and so should 6720 * not be charged to a memcg. 6721 */ 6722 vmx_bitmap[i] = (unsigned long *) 6723 __get_free_page(GFP_KERNEL); 6724 if (!vmx_bitmap[i]) { 6725 nested_vmx_hardware_unsetup(); 6726 return -ENOMEM; 6727 } 6728 } 6729 6730 init_vmcs_shadow_fields(); 6731 } 6732 6733 exit_handlers[EXIT_REASON_VMCLEAR] = handle_vmclear; 6734 exit_handlers[EXIT_REASON_VMLAUNCH] = handle_vmlaunch; 6735 exit_handlers[EXIT_REASON_VMPTRLD] = handle_vmptrld; 6736 exit_handlers[EXIT_REASON_VMPTRST] = handle_vmptrst; 6737 exit_handlers[EXIT_REASON_VMREAD] = handle_vmread; 6738 exit_handlers[EXIT_REASON_VMRESUME] = handle_vmresume; 6739 exit_handlers[EXIT_REASON_VMWRITE] = handle_vmwrite; 6740 exit_handlers[EXIT_REASON_VMOFF] = handle_vmoff; 6741 exit_handlers[EXIT_REASON_VMON] = handle_vmon; 6742 exit_handlers[EXIT_REASON_INVEPT] = handle_invept; 6743 exit_handlers[EXIT_REASON_INVVPID] = handle_invvpid; 6744 exit_handlers[EXIT_REASON_VMFUNC] = handle_vmfunc; 6745 6746 return 0; 6747 } 6748 6749 struct kvm_x86_nested_ops vmx_nested_ops = { 6750 .check_events = vmx_check_nested_events, 6751 .hv_timer_pending = nested_vmx_preemption_timer_pending, 6752 .triple_fault = nested_vmx_triple_fault, 6753 .get_state = vmx_get_nested_state, 6754 .set_state = vmx_set_nested_state, 6755 .get_nested_state_pages = vmx_get_nested_state_pages, 6756 .write_log_dirty = nested_vmx_write_pml_buffer, 6757 .enable_evmcs = nested_enable_evmcs, 6758 .get_evmcs_version = nested_get_evmcs_version, 6759 }; 6760