1 // SPDX-License-Identifier: GPL-2.0 2 3 #include <linux/objtool.h> 4 #include <linux/percpu.h> 5 6 #include <asm/debugreg.h> 7 #include <asm/mmu_context.h> 8 9 #include "cpuid.h" 10 #include "evmcs.h" 11 #include "hyperv.h" 12 #include "mmu.h" 13 #include "nested.h" 14 #include "pmu.h" 15 #include "sgx.h" 16 #include "trace.h" 17 #include "vmx.h" 18 #include "x86.h" 19 20 static bool __read_mostly enable_shadow_vmcs = 1; 21 module_param_named(enable_shadow_vmcs, enable_shadow_vmcs, bool, S_IRUGO); 22 23 static bool __read_mostly nested_early_check = 0; 24 module_param(nested_early_check, bool, S_IRUGO); 25 26 #define CC KVM_NESTED_VMENTER_CONSISTENCY_CHECK 27 28 /* 29 * Hyper-V requires all of these, so mark them as supported even though 30 * they are just treated the same as all-context. 31 */ 32 #define VMX_VPID_EXTENT_SUPPORTED_MASK \ 33 (VMX_VPID_EXTENT_INDIVIDUAL_ADDR_BIT | \ 34 VMX_VPID_EXTENT_SINGLE_CONTEXT_BIT | \ 35 VMX_VPID_EXTENT_GLOBAL_CONTEXT_BIT | \ 36 VMX_VPID_EXTENT_SINGLE_NON_GLOBAL_BIT) 37 38 #define VMX_MISC_EMULATED_PREEMPTION_TIMER_RATE 5 39 40 enum { 41 VMX_VMREAD_BITMAP, 42 VMX_VMWRITE_BITMAP, 43 VMX_BITMAP_NR 44 }; 45 static unsigned long *vmx_bitmap[VMX_BITMAP_NR]; 46 47 #define vmx_vmread_bitmap (vmx_bitmap[VMX_VMREAD_BITMAP]) 48 #define vmx_vmwrite_bitmap (vmx_bitmap[VMX_VMWRITE_BITMAP]) 49 50 struct shadow_vmcs_field { 51 u16 encoding; 52 u16 offset; 53 }; 54 static struct shadow_vmcs_field shadow_read_only_fields[] = { 55 #define SHADOW_FIELD_RO(x, y) { x, offsetof(struct vmcs12, y) }, 56 #include "vmcs_shadow_fields.h" 57 }; 58 static int max_shadow_read_only_fields = 59 ARRAY_SIZE(shadow_read_only_fields); 60 61 static struct shadow_vmcs_field shadow_read_write_fields[] = { 62 #define SHADOW_FIELD_RW(x, y) { x, offsetof(struct vmcs12, y) }, 63 #include "vmcs_shadow_fields.h" 64 }; 65 static int max_shadow_read_write_fields = 66 ARRAY_SIZE(shadow_read_write_fields); 67 68 static void init_vmcs_shadow_fields(void) 69 { 70 int i, j; 71 72 memset(vmx_vmread_bitmap, 0xff, PAGE_SIZE); 73 memset(vmx_vmwrite_bitmap, 0xff, PAGE_SIZE); 74 75 for (i = j = 0; i < max_shadow_read_only_fields; i++) { 76 struct shadow_vmcs_field entry = shadow_read_only_fields[i]; 77 u16 field = entry.encoding; 78 79 if (vmcs_field_width(field) == VMCS_FIELD_WIDTH_U64 && 80 (i + 1 == max_shadow_read_only_fields || 81 shadow_read_only_fields[i + 1].encoding != field + 1)) 82 pr_err("Missing field from shadow_read_only_field %x\n", 83 field + 1); 84 85 clear_bit(field, vmx_vmread_bitmap); 86 if (field & 1) 87 #ifdef CONFIG_X86_64 88 continue; 89 #else 90 entry.offset += sizeof(u32); 91 #endif 92 shadow_read_only_fields[j++] = entry; 93 } 94 max_shadow_read_only_fields = j; 95 96 for (i = j = 0; i < max_shadow_read_write_fields; i++) { 97 struct shadow_vmcs_field entry = shadow_read_write_fields[i]; 98 u16 field = entry.encoding; 99 100 if (vmcs_field_width(field) == VMCS_FIELD_WIDTH_U64 && 101 (i + 1 == max_shadow_read_write_fields || 102 shadow_read_write_fields[i + 1].encoding != field + 1)) 103 pr_err("Missing field from shadow_read_write_field %x\n", 104 field + 1); 105 106 WARN_ONCE(field >= GUEST_ES_AR_BYTES && 107 field <= GUEST_TR_AR_BYTES, 108 "Update vmcs12_write_any() to drop reserved bits from AR_BYTES"); 109 110 /* 111 * PML and the preemption timer can be emulated, but the 112 * processor cannot vmwrite to fields that don't exist 113 * on bare metal. 114 */ 115 switch (field) { 116 case GUEST_PML_INDEX: 117 if (!cpu_has_vmx_pml()) 118 continue; 119 break; 120 case VMX_PREEMPTION_TIMER_VALUE: 121 if (!cpu_has_vmx_preemption_timer()) 122 continue; 123 break; 124 case GUEST_INTR_STATUS: 125 if (!cpu_has_vmx_apicv()) 126 continue; 127 break; 128 default: 129 break; 130 } 131 132 clear_bit(field, vmx_vmwrite_bitmap); 133 clear_bit(field, vmx_vmread_bitmap); 134 if (field & 1) 135 #ifdef CONFIG_X86_64 136 continue; 137 #else 138 entry.offset += sizeof(u32); 139 #endif 140 shadow_read_write_fields[j++] = entry; 141 } 142 max_shadow_read_write_fields = j; 143 } 144 145 /* 146 * The following 3 functions, nested_vmx_succeed()/failValid()/failInvalid(), 147 * set the success or error code of an emulated VMX instruction (as specified 148 * by Vol 2B, VMX Instruction Reference, "Conventions"), and skip the emulated 149 * instruction. 150 */ 151 static int nested_vmx_succeed(struct kvm_vcpu *vcpu) 152 { 153 vmx_set_rflags(vcpu, vmx_get_rflags(vcpu) 154 & ~(X86_EFLAGS_CF | X86_EFLAGS_PF | X86_EFLAGS_AF | 155 X86_EFLAGS_ZF | X86_EFLAGS_SF | X86_EFLAGS_OF)); 156 return kvm_skip_emulated_instruction(vcpu); 157 } 158 159 static int nested_vmx_failInvalid(struct kvm_vcpu *vcpu) 160 { 161 vmx_set_rflags(vcpu, (vmx_get_rflags(vcpu) 162 & ~(X86_EFLAGS_PF | X86_EFLAGS_AF | X86_EFLAGS_ZF | 163 X86_EFLAGS_SF | X86_EFLAGS_OF)) 164 | X86_EFLAGS_CF); 165 return kvm_skip_emulated_instruction(vcpu); 166 } 167 168 static int nested_vmx_failValid(struct kvm_vcpu *vcpu, 169 u32 vm_instruction_error) 170 { 171 vmx_set_rflags(vcpu, (vmx_get_rflags(vcpu) 172 & ~(X86_EFLAGS_CF | X86_EFLAGS_PF | X86_EFLAGS_AF | 173 X86_EFLAGS_SF | X86_EFLAGS_OF)) 174 | X86_EFLAGS_ZF); 175 get_vmcs12(vcpu)->vm_instruction_error = vm_instruction_error; 176 /* 177 * We don't need to force sync to shadow VMCS because 178 * VM_INSTRUCTION_ERROR is not shadowed. Enlightened VMCS 'shadows' all 179 * fields and thus must be synced. 180 */ 181 if (to_vmx(vcpu)->nested.hv_evmcs_vmptr != EVMPTR_INVALID) 182 to_vmx(vcpu)->nested.need_vmcs12_to_shadow_sync = true; 183 184 return kvm_skip_emulated_instruction(vcpu); 185 } 186 187 static int nested_vmx_fail(struct kvm_vcpu *vcpu, u32 vm_instruction_error) 188 { 189 struct vcpu_vmx *vmx = to_vmx(vcpu); 190 191 /* 192 * failValid writes the error number to the current VMCS, which 193 * can't be done if there isn't a current VMCS. 194 */ 195 if (vmx->nested.current_vmptr == INVALID_GPA && 196 !evmptr_is_valid(vmx->nested.hv_evmcs_vmptr)) 197 return nested_vmx_failInvalid(vcpu); 198 199 return nested_vmx_failValid(vcpu, vm_instruction_error); 200 } 201 202 static void nested_vmx_abort(struct kvm_vcpu *vcpu, u32 indicator) 203 { 204 /* TODO: not to reset guest simply here. */ 205 kvm_make_request(KVM_REQ_TRIPLE_FAULT, vcpu); 206 pr_debug_ratelimited("kvm: nested vmx abort, indicator %d\n", indicator); 207 } 208 209 static inline bool vmx_control_verify(u32 control, u32 low, u32 high) 210 { 211 return fixed_bits_valid(control, low, high); 212 } 213 214 static inline u64 vmx_control_msr(u32 low, u32 high) 215 { 216 return low | ((u64)high << 32); 217 } 218 219 static void vmx_disable_shadow_vmcs(struct vcpu_vmx *vmx) 220 { 221 secondary_exec_controls_clearbit(vmx, SECONDARY_EXEC_SHADOW_VMCS); 222 vmcs_write64(VMCS_LINK_POINTER, INVALID_GPA); 223 vmx->nested.need_vmcs12_to_shadow_sync = false; 224 } 225 226 static inline void nested_release_evmcs(struct kvm_vcpu *vcpu) 227 { 228 struct vcpu_vmx *vmx = to_vmx(vcpu); 229 230 if (evmptr_is_valid(vmx->nested.hv_evmcs_vmptr)) { 231 kvm_vcpu_unmap(vcpu, &vmx->nested.hv_evmcs_map, true); 232 vmx->nested.hv_evmcs = NULL; 233 } 234 235 vmx->nested.hv_evmcs_vmptr = EVMPTR_INVALID; 236 } 237 238 static void vmx_sync_vmcs_host_state(struct vcpu_vmx *vmx, 239 struct loaded_vmcs *prev) 240 { 241 struct vmcs_host_state *dest, *src; 242 243 if (unlikely(!vmx->guest_state_loaded)) 244 return; 245 246 src = &prev->host_state; 247 dest = &vmx->loaded_vmcs->host_state; 248 249 vmx_set_host_fs_gs(dest, src->fs_sel, src->gs_sel, src->fs_base, src->gs_base); 250 dest->ldt_sel = src->ldt_sel; 251 #ifdef CONFIG_X86_64 252 dest->ds_sel = src->ds_sel; 253 dest->es_sel = src->es_sel; 254 #endif 255 } 256 257 static void vmx_switch_vmcs(struct kvm_vcpu *vcpu, struct loaded_vmcs *vmcs) 258 { 259 struct vcpu_vmx *vmx = to_vmx(vcpu); 260 struct loaded_vmcs *prev; 261 int cpu; 262 263 if (WARN_ON_ONCE(vmx->loaded_vmcs == vmcs)) 264 return; 265 266 cpu = get_cpu(); 267 prev = vmx->loaded_vmcs; 268 vmx->loaded_vmcs = vmcs; 269 vmx_vcpu_load_vmcs(vcpu, cpu, prev); 270 vmx_sync_vmcs_host_state(vmx, prev); 271 put_cpu(); 272 273 vcpu->arch.regs_avail = ~VMX_REGS_LAZY_LOAD_SET; 274 275 /* 276 * All lazily updated registers will be reloaded from VMCS12 on both 277 * vmentry and vmexit. 278 */ 279 vcpu->arch.regs_dirty = 0; 280 } 281 282 /* 283 * Free whatever needs to be freed from vmx->nested when L1 goes down, or 284 * just stops using VMX. 285 */ 286 static void free_nested(struct kvm_vcpu *vcpu) 287 { 288 struct vcpu_vmx *vmx = to_vmx(vcpu); 289 290 if (WARN_ON_ONCE(vmx->loaded_vmcs != &vmx->vmcs01)) 291 vmx_switch_vmcs(vcpu, &vmx->vmcs01); 292 293 if (!vmx->nested.vmxon && !vmx->nested.smm.vmxon) 294 return; 295 296 kvm_clear_request(KVM_REQ_GET_NESTED_STATE_PAGES, vcpu); 297 298 vmx->nested.vmxon = false; 299 vmx->nested.smm.vmxon = false; 300 vmx->nested.vmxon_ptr = INVALID_GPA; 301 free_vpid(vmx->nested.vpid02); 302 vmx->nested.posted_intr_nv = -1; 303 vmx->nested.current_vmptr = INVALID_GPA; 304 if (enable_shadow_vmcs) { 305 vmx_disable_shadow_vmcs(vmx); 306 vmcs_clear(vmx->vmcs01.shadow_vmcs); 307 free_vmcs(vmx->vmcs01.shadow_vmcs); 308 vmx->vmcs01.shadow_vmcs = NULL; 309 } 310 kfree(vmx->nested.cached_vmcs12); 311 vmx->nested.cached_vmcs12 = NULL; 312 kfree(vmx->nested.cached_shadow_vmcs12); 313 vmx->nested.cached_shadow_vmcs12 = NULL; 314 /* Unpin physical memory we referred to in the vmcs02 */ 315 if (vmx->nested.apic_access_page) { 316 kvm_release_page_clean(vmx->nested.apic_access_page); 317 vmx->nested.apic_access_page = NULL; 318 } 319 kvm_vcpu_unmap(vcpu, &vmx->nested.virtual_apic_map, true); 320 kvm_vcpu_unmap(vcpu, &vmx->nested.pi_desc_map, true); 321 vmx->nested.pi_desc = NULL; 322 323 kvm_mmu_free_roots(vcpu->kvm, &vcpu->arch.guest_mmu, KVM_MMU_ROOTS_ALL); 324 325 nested_release_evmcs(vcpu); 326 327 free_loaded_vmcs(&vmx->nested.vmcs02); 328 } 329 330 /* 331 * Ensure that the current vmcs of the logical processor is the 332 * vmcs01 of the vcpu before calling free_nested(). 333 */ 334 void nested_vmx_free_vcpu(struct kvm_vcpu *vcpu) 335 { 336 vcpu_load(vcpu); 337 vmx_leave_nested(vcpu); 338 vcpu_put(vcpu); 339 } 340 341 #define EPTP_PA_MASK GENMASK_ULL(51, 12) 342 343 static bool nested_ept_root_matches(hpa_t root_hpa, u64 root_eptp, u64 eptp) 344 { 345 return VALID_PAGE(root_hpa) && 346 ((root_eptp & EPTP_PA_MASK) == (eptp & EPTP_PA_MASK)); 347 } 348 349 static void nested_ept_invalidate_addr(struct kvm_vcpu *vcpu, gpa_t eptp, 350 gpa_t addr) 351 { 352 uint i; 353 struct kvm_mmu_root_info *cached_root; 354 355 WARN_ON_ONCE(!mmu_is_nested(vcpu)); 356 357 for (i = 0; i < KVM_MMU_NUM_PREV_ROOTS; i++) { 358 cached_root = &vcpu->arch.mmu->prev_roots[i]; 359 360 if (nested_ept_root_matches(cached_root->hpa, cached_root->pgd, 361 eptp)) 362 vcpu->arch.mmu->invlpg(vcpu, addr, cached_root->hpa); 363 } 364 } 365 366 static void nested_ept_inject_page_fault(struct kvm_vcpu *vcpu, 367 struct x86_exception *fault) 368 { 369 struct vmcs12 *vmcs12 = get_vmcs12(vcpu); 370 struct vcpu_vmx *vmx = to_vmx(vcpu); 371 u32 vm_exit_reason; 372 unsigned long exit_qualification = vcpu->arch.exit_qualification; 373 374 if (vmx->nested.pml_full) { 375 vm_exit_reason = EXIT_REASON_PML_FULL; 376 vmx->nested.pml_full = false; 377 exit_qualification &= INTR_INFO_UNBLOCK_NMI; 378 } else { 379 if (fault->error_code & PFERR_RSVD_MASK) 380 vm_exit_reason = EXIT_REASON_EPT_MISCONFIG; 381 else 382 vm_exit_reason = EXIT_REASON_EPT_VIOLATION; 383 384 /* 385 * Although the caller (kvm_inject_emulated_page_fault) would 386 * have already synced the faulting address in the shadow EPT 387 * tables for the current EPTP12, we also need to sync it for 388 * any other cached EPTP02s based on the same EP4TA, since the 389 * TLB associates mappings to the EP4TA rather than the full EPTP. 390 */ 391 nested_ept_invalidate_addr(vcpu, vmcs12->ept_pointer, 392 fault->address); 393 } 394 395 nested_vmx_vmexit(vcpu, vm_exit_reason, 0, exit_qualification); 396 vmcs12->guest_physical_address = fault->address; 397 } 398 399 static void nested_ept_new_eptp(struct kvm_vcpu *vcpu) 400 { 401 struct vcpu_vmx *vmx = to_vmx(vcpu); 402 bool execonly = vmx->nested.msrs.ept_caps & VMX_EPT_EXECUTE_ONLY_BIT; 403 int ept_lpage_level = ept_caps_to_lpage_level(vmx->nested.msrs.ept_caps); 404 405 kvm_init_shadow_ept_mmu(vcpu, execonly, ept_lpage_level, 406 nested_ept_ad_enabled(vcpu), 407 nested_ept_get_eptp(vcpu)); 408 } 409 410 static void nested_ept_init_mmu_context(struct kvm_vcpu *vcpu) 411 { 412 WARN_ON(mmu_is_nested(vcpu)); 413 414 vcpu->arch.mmu = &vcpu->arch.guest_mmu; 415 nested_ept_new_eptp(vcpu); 416 vcpu->arch.mmu->get_guest_pgd = nested_ept_get_eptp; 417 vcpu->arch.mmu->inject_page_fault = nested_ept_inject_page_fault; 418 vcpu->arch.mmu->get_pdptr = kvm_pdptr_read; 419 420 vcpu->arch.walk_mmu = &vcpu->arch.nested_mmu; 421 } 422 423 static void nested_ept_uninit_mmu_context(struct kvm_vcpu *vcpu) 424 { 425 vcpu->arch.mmu = &vcpu->arch.root_mmu; 426 vcpu->arch.walk_mmu = &vcpu->arch.root_mmu; 427 } 428 429 static bool nested_vmx_is_page_fault_vmexit(struct vmcs12 *vmcs12, 430 u16 error_code) 431 { 432 bool inequality, bit; 433 434 bit = (vmcs12->exception_bitmap & (1u << PF_VECTOR)) != 0; 435 inequality = 436 (error_code & vmcs12->page_fault_error_code_mask) != 437 vmcs12->page_fault_error_code_match; 438 return inequality ^ bit; 439 } 440 441 442 /* 443 * KVM wants to inject page-faults which it got to the guest. This function 444 * checks whether in a nested guest, we need to inject them to L1 or L2. 445 */ 446 static int nested_vmx_check_exception(struct kvm_vcpu *vcpu, unsigned long *exit_qual) 447 { 448 struct vmcs12 *vmcs12 = get_vmcs12(vcpu); 449 unsigned int nr = vcpu->arch.exception.nr; 450 bool has_payload = vcpu->arch.exception.has_payload; 451 unsigned long payload = vcpu->arch.exception.payload; 452 453 if (nr == PF_VECTOR) { 454 if (vcpu->arch.exception.nested_apf) { 455 *exit_qual = vcpu->arch.apf.nested_apf_token; 456 return 1; 457 } 458 if (nested_vmx_is_page_fault_vmexit(vmcs12, 459 vcpu->arch.exception.error_code)) { 460 *exit_qual = has_payload ? payload : vcpu->arch.cr2; 461 return 1; 462 } 463 } else if (vmcs12->exception_bitmap & (1u << nr)) { 464 if (nr == DB_VECTOR) { 465 if (!has_payload) { 466 payload = vcpu->arch.dr6; 467 payload &= ~DR6_BT; 468 payload ^= DR6_ACTIVE_LOW; 469 } 470 *exit_qual = payload; 471 } else 472 *exit_qual = 0; 473 return 1; 474 } 475 476 return 0; 477 } 478 479 480 static void vmx_inject_page_fault_nested(struct kvm_vcpu *vcpu, 481 struct x86_exception *fault) 482 { 483 struct vmcs12 *vmcs12 = get_vmcs12(vcpu); 484 485 WARN_ON(!is_guest_mode(vcpu)); 486 487 if (nested_vmx_is_page_fault_vmexit(vmcs12, fault->error_code) && 488 !to_vmx(vcpu)->nested.nested_run_pending) { 489 vmcs12->vm_exit_intr_error_code = fault->error_code; 490 nested_vmx_vmexit(vcpu, EXIT_REASON_EXCEPTION_NMI, 491 PF_VECTOR | INTR_TYPE_HARD_EXCEPTION | 492 INTR_INFO_DELIVER_CODE_MASK | INTR_INFO_VALID_MASK, 493 fault->address); 494 } else { 495 kvm_inject_page_fault(vcpu, fault); 496 } 497 } 498 499 static int nested_vmx_check_io_bitmap_controls(struct kvm_vcpu *vcpu, 500 struct vmcs12 *vmcs12) 501 { 502 if (!nested_cpu_has(vmcs12, CPU_BASED_USE_IO_BITMAPS)) 503 return 0; 504 505 if (CC(!page_address_valid(vcpu, vmcs12->io_bitmap_a)) || 506 CC(!page_address_valid(vcpu, vmcs12->io_bitmap_b))) 507 return -EINVAL; 508 509 return 0; 510 } 511 512 static int nested_vmx_check_msr_bitmap_controls(struct kvm_vcpu *vcpu, 513 struct vmcs12 *vmcs12) 514 { 515 if (!nested_cpu_has(vmcs12, CPU_BASED_USE_MSR_BITMAPS)) 516 return 0; 517 518 if (CC(!page_address_valid(vcpu, vmcs12->msr_bitmap))) 519 return -EINVAL; 520 521 return 0; 522 } 523 524 static int nested_vmx_check_tpr_shadow_controls(struct kvm_vcpu *vcpu, 525 struct vmcs12 *vmcs12) 526 { 527 if (!nested_cpu_has(vmcs12, CPU_BASED_TPR_SHADOW)) 528 return 0; 529 530 if (CC(!page_address_valid(vcpu, vmcs12->virtual_apic_page_addr))) 531 return -EINVAL; 532 533 return 0; 534 } 535 536 /* 537 * For x2APIC MSRs, ignore the vmcs01 bitmap. L1 can enable x2APIC without L1 538 * itself utilizing x2APIC. All MSRs were previously set to be intercepted, 539 * only the "disable intercept" case needs to be handled. 540 */ 541 static void nested_vmx_disable_intercept_for_x2apic_msr(unsigned long *msr_bitmap_l1, 542 unsigned long *msr_bitmap_l0, 543 u32 msr, int type) 544 { 545 if (type & MSR_TYPE_R && !vmx_test_msr_bitmap_read(msr_bitmap_l1, msr)) 546 vmx_clear_msr_bitmap_read(msr_bitmap_l0, msr); 547 548 if (type & MSR_TYPE_W && !vmx_test_msr_bitmap_write(msr_bitmap_l1, msr)) 549 vmx_clear_msr_bitmap_write(msr_bitmap_l0, msr); 550 } 551 552 static inline void enable_x2apic_msr_intercepts(unsigned long *msr_bitmap) 553 { 554 int msr; 555 556 for (msr = 0x800; msr <= 0x8ff; msr += BITS_PER_LONG) { 557 unsigned word = msr / BITS_PER_LONG; 558 559 msr_bitmap[word] = ~0; 560 msr_bitmap[word + (0x800 / sizeof(long))] = ~0; 561 } 562 } 563 564 #define BUILD_NVMX_MSR_INTERCEPT_HELPER(rw) \ 565 static inline \ 566 void nested_vmx_set_msr_##rw##_intercept(struct vcpu_vmx *vmx, \ 567 unsigned long *msr_bitmap_l1, \ 568 unsigned long *msr_bitmap_l0, u32 msr) \ 569 { \ 570 if (vmx_test_msr_bitmap_##rw(vmx->vmcs01.msr_bitmap, msr) || \ 571 vmx_test_msr_bitmap_##rw(msr_bitmap_l1, msr)) \ 572 vmx_set_msr_bitmap_##rw(msr_bitmap_l0, msr); \ 573 else \ 574 vmx_clear_msr_bitmap_##rw(msr_bitmap_l0, msr); \ 575 } 576 BUILD_NVMX_MSR_INTERCEPT_HELPER(read) 577 BUILD_NVMX_MSR_INTERCEPT_HELPER(write) 578 579 static inline void nested_vmx_set_intercept_for_msr(struct vcpu_vmx *vmx, 580 unsigned long *msr_bitmap_l1, 581 unsigned long *msr_bitmap_l0, 582 u32 msr, int types) 583 { 584 if (types & MSR_TYPE_R) 585 nested_vmx_set_msr_read_intercept(vmx, msr_bitmap_l1, 586 msr_bitmap_l0, msr); 587 if (types & MSR_TYPE_W) 588 nested_vmx_set_msr_write_intercept(vmx, msr_bitmap_l1, 589 msr_bitmap_l0, msr); 590 } 591 592 /* 593 * Merge L0's and L1's MSR bitmap, return false to indicate that 594 * we do not use the hardware. 595 */ 596 static inline bool nested_vmx_prepare_msr_bitmap(struct kvm_vcpu *vcpu, 597 struct vmcs12 *vmcs12) 598 { 599 struct vcpu_vmx *vmx = to_vmx(vcpu); 600 int msr; 601 unsigned long *msr_bitmap_l1; 602 unsigned long *msr_bitmap_l0 = vmx->nested.vmcs02.msr_bitmap; 603 struct hv_enlightened_vmcs *evmcs = vmx->nested.hv_evmcs; 604 struct kvm_host_map *map = &vmx->nested.msr_bitmap_map; 605 606 /* Nothing to do if the MSR bitmap is not in use. */ 607 if (!cpu_has_vmx_msr_bitmap() || 608 !nested_cpu_has(vmcs12, CPU_BASED_USE_MSR_BITMAPS)) 609 return false; 610 611 /* 612 * MSR bitmap update can be skipped when: 613 * - MSR bitmap for L1 hasn't changed. 614 * - Nested hypervisor (L1) is attempting to launch the same L2 as 615 * before. 616 * - Nested hypervisor (L1) has enabled 'Enlightened MSR Bitmap' feature 617 * and tells KVM (L0) there were no changes in MSR bitmap for L2. 618 */ 619 if (!vmx->nested.force_msr_bitmap_recalc && evmcs && 620 evmcs->hv_enlightenments_control.msr_bitmap && 621 evmcs->hv_clean_fields & HV_VMX_ENLIGHTENED_CLEAN_FIELD_MSR_BITMAP) 622 return true; 623 624 if (kvm_vcpu_map(vcpu, gpa_to_gfn(vmcs12->msr_bitmap), map)) 625 return false; 626 627 msr_bitmap_l1 = (unsigned long *)map->hva; 628 629 /* 630 * To keep the control flow simple, pay eight 8-byte writes (sixteen 631 * 4-byte writes on 32-bit systems) up front to enable intercepts for 632 * the x2APIC MSR range and selectively toggle those relevant to L2. 633 */ 634 enable_x2apic_msr_intercepts(msr_bitmap_l0); 635 636 if (nested_cpu_has_virt_x2apic_mode(vmcs12)) { 637 if (nested_cpu_has_apic_reg_virt(vmcs12)) { 638 /* 639 * L0 need not intercept reads for MSRs between 0x800 640 * and 0x8ff, it just lets the processor take the value 641 * from the virtual-APIC page; take those 256 bits 642 * directly from the L1 bitmap. 643 */ 644 for (msr = 0x800; msr <= 0x8ff; msr += BITS_PER_LONG) { 645 unsigned word = msr / BITS_PER_LONG; 646 647 msr_bitmap_l0[word] = msr_bitmap_l1[word]; 648 } 649 } 650 651 nested_vmx_disable_intercept_for_x2apic_msr( 652 msr_bitmap_l1, msr_bitmap_l0, 653 X2APIC_MSR(APIC_TASKPRI), 654 MSR_TYPE_R | MSR_TYPE_W); 655 656 if (nested_cpu_has_vid(vmcs12)) { 657 nested_vmx_disable_intercept_for_x2apic_msr( 658 msr_bitmap_l1, msr_bitmap_l0, 659 X2APIC_MSR(APIC_EOI), 660 MSR_TYPE_W); 661 nested_vmx_disable_intercept_for_x2apic_msr( 662 msr_bitmap_l1, msr_bitmap_l0, 663 X2APIC_MSR(APIC_SELF_IPI), 664 MSR_TYPE_W); 665 } 666 } 667 668 /* 669 * Always check vmcs01's bitmap to honor userspace MSR filters and any 670 * other runtime changes to vmcs01's bitmap, e.g. dynamic pass-through. 671 */ 672 #ifdef CONFIG_X86_64 673 nested_vmx_set_intercept_for_msr(vmx, msr_bitmap_l1, msr_bitmap_l0, 674 MSR_FS_BASE, MSR_TYPE_RW); 675 676 nested_vmx_set_intercept_for_msr(vmx, msr_bitmap_l1, msr_bitmap_l0, 677 MSR_GS_BASE, MSR_TYPE_RW); 678 679 nested_vmx_set_intercept_for_msr(vmx, msr_bitmap_l1, msr_bitmap_l0, 680 MSR_KERNEL_GS_BASE, MSR_TYPE_RW); 681 #endif 682 nested_vmx_set_intercept_for_msr(vmx, msr_bitmap_l1, msr_bitmap_l0, 683 MSR_IA32_SPEC_CTRL, MSR_TYPE_RW); 684 685 nested_vmx_set_intercept_for_msr(vmx, msr_bitmap_l1, msr_bitmap_l0, 686 MSR_IA32_PRED_CMD, MSR_TYPE_W); 687 688 kvm_vcpu_unmap(vcpu, &vmx->nested.msr_bitmap_map, false); 689 690 vmx->nested.force_msr_bitmap_recalc = false; 691 692 return true; 693 } 694 695 static void nested_cache_shadow_vmcs12(struct kvm_vcpu *vcpu, 696 struct vmcs12 *vmcs12) 697 { 698 struct vcpu_vmx *vmx = to_vmx(vcpu); 699 struct gfn_to_hva_cache *ghc = &vmx->nested.shadow_vmcs12_cache; 700 701 if (!nested_cpu_has_shadow_vmcs(vmcs12) || 702 vmcs12->vmcs_link_pointer == INVALID_GPA) 703 return; 704 705 if (ghc->gpa != vmcs12->vmcs_link_pointer && 706 kvm_gfn_to_hva_cache_init(vcpu->kvm, ghc, 707 vmcs12->vmcs_link_pointer, VMCS12_SIZE)) 708 return; 709 710 kvm_read_guest_cached(vmx->vcpu.kvm, ghc, get_shadow_vmcs12(vcpu), 711 VMCS12_SIZE); 712 } 713 714 static void nested_flush_cached_shadow_vmcs12(struct kvm_vcpu *vcpu, 715 struct vmcs12 *vmcs12) 716 { 717 struct vcpu_vmx *vmx = to_vmx(vcpu); 718 struct gfn_to_hva_cache *ghc = &vmx->nested.shadow_vmcs12_cache; 719 720 if (!nested_cpu_has_shadow_vmcs(vmcs12) || 721 vmcs12->vmcs_link_pointer == INVALID_GPA) 722 return; 723 724 if (ghc->gpa != vmcs12->vmcs_link_pointer && 725 kvm_gfn_to_hva_cache_init(vcpu->kvm, ghc, 726 vmcs12->vmcs_link_pointer, VMCS12_SIZE)) 727 return; 728 729 kvm_write_guest_cached(vmx->vcpu.kvm, ghc, get_shadow_vmcs12(vcpu), 730 VMCS12_SIZE); 731 } 732 733 /* 734 * In nested virtualization, check if L1 has set 735 * VM_EXIT_ACK_INTR_ON_EXIT 736 */ 737 static bool nested_exit_intr_ack_set(struct kvm_vcpu *vcpu) 738 { 739 return get_vmcs12(vcpu)->vm_exit_controls & 740 VM_EXIT_ACK_INTR_ON_EXIT; 741 } 742 743 static int nested_vmx_check_apic_access_controls(struct kvm_vcpu *vcpu, 744 struct vmcs12 *vmcs12) 745 { 746 if (nested_cpu_has2(vmcs12, SECONDARY_EXEC_VIRTUALIZE_APIC_ACCESSES) && 747 CC(!page_address_valid(vcpu, vmcs12->apic_access_addr))) 748 return -EINVAL; 749 else 750 return 0; 751 } 752 753 static int nested_vmx_check_apicv_controls(struct kvm_vcpu *vcpu, 754 struct vmcs12 *vmcs12) 755 { 756 if (!nested_cpu_has_virt_x2apic_mode(vmcs12) && 757 !nested_cpu_has_apic_reg_virt(vmcs12) && 758 !nested_cpu_has_vid(vmcs12) && 759 !nested_cpu_has_posted_intr(vmcs12)) 760 return 0; 761 762 /* 763 * If virtualize x2apic mode is enabled, 764 * virtualize apic access must be disabled. 765 */ 766 if (CC(nested_cpu_has_virt_x2apic_mode(vmcs12) && 767 nested_cpu_has2(vmcs12, SECONDARY_EXEC_VIRTUALIZE_APIC_ACCESSES))) 768 return -EINVAL; 769 770 /* 771 * If virtual interrupt delivery is enabled, 772 * we must exit on external interrupts. 773 */ 774 if (CC(nested_cpu_has_vid(vmcs12) && !nested_exit_on_intr(vcpu))) 775 return -EINVAL; 776 777 /* 778 * bits 15:8 should be zero in posted_intr_nv, 779 * the descriptor address has been already checked 780 * in nested_get_vmcs12_pages. 781 * 782 * bits 5:0 of posted_intr_desc_addr should be zero. 783 */ 784 if (nested_cpu_has_posted_intr(vmcs12) && 785 (CC(!nested_cpu_has_vid(vmcs12)) || 786 CC(!nested_exit_intr_ack_set(vcpu)) || 787 CC((vmcs12->posted_intr_nv & 0xff00)) || 788 CC(!kvm_vcpu_is_legal_aligned_gpa(vcpu, vmcs12->posted_intr_desc_addr, 64)))) 789 return -EINVAL; 790 791 /* tpr shadow is needed by all apicv features. */ 792 if (CC(!nested_cpu_has(vmcs12, CPU_BASED_TPR_SHADOW))) 793 return -EINVAL; 794 795 return 0; 796 } 797 798 static int nested_vmx_check_msr_switch(struct kvm_vcpu *vcpu, 799 u32 count, u64 addr) 800 { 801 if (count == 0) 802 return 0; 803 804 if (!kvm_vcpu_is_legal_aligned_gpa(vcpu, addr, 16) || 805 !kvm_vcpu_is_legal_gpa(vcpu, (addr + count * sizeof(struct vmx_msr_entry) - 1))) 806 return -EINVAL; 807 808 return 0; 809 } 810 811 static int nested_vmx_check_exit_msr_switch_controls(struct kvm_vcpu *vcpu, 812 struct vmcs12 *vmcs12) 813 { 814 if (CC(nested_vmx_check_msr_switch(vcpu, 815 vmcs12->vm_exit_msr_load_count, 816 vmcs12->vm_exit_msr_load_addr)) || 817 CC(nested_vmx_check_msr_switch(vcpu, 818 vmcs12->vm_exit_msr_store_count, 819 vmcs12->vm_exit_msr_store_addr))) 820 return -EINVAL; 821 822 return 0; 823 } 824 825 static int nested_vmx_check_entry_msr_switch_controls(struct kvm_vcpu *vcpu, 826 struct vmcs12 *vmcs12) 827 { 828 if (CC(nested_vmx_check_msr_switch(vcpu, 829 vmcs12->vm_entry_msr_load_count, 830 vmcs12->vm_entry_msr_load_addr))) 831 return -EINVAL; 832 833 return 0; 834 } 835 836 static int nested_vmx_check_pml_controls(struct kvm_vcpu *vcpu, 837 struct vmcs12 *vmcs12) 838 { 839 if (!nested_cpu_has_pml(vmcs12)) 840 return 0; 841 842 if (CC(!nested_cpu_has_ept(vmcs12)) || 843 CC(!page_address_valid(vcpu, vmcs12->pml_address))) 844 return -EINVAL; 845 846 return 0; 847 } 848 849 static int nested_vmx_check_unrestricted_guest_controls(struct kvm_vcpu *vcpu, 850 struct vmcs12 *vmcs12) 851 { 852 if (CC(nested_cpu_has2(vmcs12, SECONDARY_EXEC_UNRESTRICTED_GUEST) && 853 !nested_cpu_has_ept(vmcs12))) 854 return -EINVAL; 855 return 0; 856 } 857 858 static int nested_vmx_check_mode_based_ept_exec_controls(struct kvm_vcpu *vcpu, 859 struct vmcs12 *vmcs12) 860 { 861 if (CC(nested_cpu_has2(vmcs12, SECONDARY_EXEC_MODE_BASED_EPT_EXEC) && 862 !nested_cpu_has_ept(vmcs12))) 863 return -EINVAL; 864 return 0; 865 } 866 867 static int nested_vmx_check_shadow_vmcs_controls(struct kvm_vcpu *vcpu, 868 struct vmcs12 *vmcs12) 869 { 870 if (!nested_cpu_has_shadow_vmcs(vmcs12)) 871 return 0; 872 873 if (CC(!page_address_valid(vcpu, vmcs12->vmread_bitmap)) || 874 CC(!page_address_valid(vcpu, vmcs12->vmwrite_bitmap))) 875 return -EINVAL; 876 877 return 0; 878 } 879 880 static int nested_vmx_msr_check_common(struct kvm_vcpu *vcpu, 881 struct vmx_msr_entry *e) 882 { 883 /* x2APIC MSR accesses are not allowed */ 884 if (CC(vcpu->arch.apic_base & X2APIC_ENABLE && e->index >> 8 == 0x8)) 885 return -EINVAL; 886 if (CC(e->index == MSR_IA32_UCODE_WRITE) || /* SDM Table 35-2 */ 887 CC(e->index == MSR_IA32_UCODE_REV)) 888 return -EINVAL; 889 if (CC(e->reserved != 0)) 890 return -EINVAL; 891 return 0; 892 } 893 894 static int nested_vmx_load_msr_check(struct kvm_vcpu *vcpu, 895 struct vmx_msr_entry *e) 896 { 897 if (CC(e->index == MSR_FS_BASE) || 898 CC(e->index == MSR_GS_BASE) || 899 CC(e->index == MSR_IA32_SMM_MONITOR_CTL) || /* SMM is not supported */ 900 nested_vmx_msr_check_common(vcpu, e)) 901 return -EINVAL; 902 return 0; 903 } 904 905 static int nested_vmx_store_msr_check(struct kvm_vcpu *vcpu, 906 struct vmx_msr_entry *e) 907 { 908 if (CC(e->index == MSR_IA32_SMBASE) || /* SMM is not supported */ 909 nested_vmx_msr_check_common(vcpu, e)) 910 return -EINVAL; 911 return 0; 912 } 913 914 static u32 nested_vmx_max_atomic_switch_msrs(struct kvm_vcpu *vcpu) 915 { 916 struct vcpu_vmx *vmx = to_vmx(vcpu); 917 u64 vmx_misc = vmx_control_msr(vmx->nested.msrs.misc_low, 918 vmx->nested.msrs.misc_high); 919 920 return (vmx_misc_max_msr(vmx_misc) + 1) * VMX_MISC_MSR_LIST_MULTIPLIER; 921 } 922 923 /* 924 * Load guest's/host's msr at nested entry/exit. 925 * return 0 for success, entry index for failure. 926 * 927 * One of the failure modes for MSR load/store is when a list exceeds the 928 * virtual hardware's capacity. To maintain compatibility with hardware inasmuch 929 * as possible, process all valid entries before failing rather than precheck 930 * for a capacity violation. 931 */ 932 static u32 nested_vmx_load_msr(struct kvm_vcpu *vcpu, u64 gpa, u32 count) 933 { 934 u32 i; 935 struct vmx_msr_entry e; 936 u32 max_msr_list_size = nested_vmx_max_atomic_switch_msrs(vcpu); 937 938 for (i = 0; i < count; i++) { 939 if (unlikely(i >= max_msr_list_size)) 940 goto fail; 941 942 if (kvm_vcpu_read_guest(vcpu, gpa + i * sizeof(e), 943 &e, sizeof(e))) { 944 pr_debug_ratelimited( 945 "%s cannot read MSR entry (%u, 0x%08llx)\n", 946 __func__, i, gpa + i * sizeof(e)); 947 goto fail; 948 } 949 if (nested_vmx_load_msr_check(vcpu, &e)) { 950 pr_debug_ratelimited( 951 "%s check failed (%u, 0x%x, 0x%x)\n", 952 __func__, i, e.index, e.reserved); 953 goto fail; 954 } 955 if (kvm_set_msr(vcpu, e.index, e.value)) { 956 pr_debug_ratelimited( 957 "%s cannot write MSR (%u, 0x%x, 0x%llx)\n", 958 __func__, i, e.index, e.value); 959 goto fail; 960 } 961 } 962 return 0; 963 fail: 964 /* Note, max_msr_list_size is at most 4096, i.e. this can't wrap. */ 965 return i + 1; 966 } 967 968 static bool nested_vmx_get_vmexit_msr_value(struct kvm_vcpu *vcpu, 969 u32 msr_index, 970 u64 *data) 971 { 972 struct vcpu_vmx *vmx = to_vmx(vcpu); 973 974 /* 975 * If the L0 hypervisor stored a more accurate value for the TSC that 976 * does not include the time taken for emulation of the L2->L1 977 * VM-exit in L0, use the more accurate value. 978 */ 979 if (msr_index == MSR_IA32_TSC) { 980 int i = vmx_find_loadstore_msr_slot(&vmx->msr_autostore.guest, 981 MSR_IA32_TSC); 982 983 if (i >= 0) { 984 u64 val = vmx->msr_autostore.guest.val[i].value; 985 986 *data = kvm_read_l1_tsc(vcpu, val); 987 return true; 988 } 989 } 990 991 if (kvm_get_msr(vcpu, msr_index, data)) { 992 pr_debug_ratelimited("%s cannot read MSR (0x%x)\n", __func__, 993 msr_index); 994 return false; 995 } 996 return true; 997 } 998 999 static bool read_and_check_msr_entry(struct kvm_vcpu *vcpu, u64 gpa, int i, 1000 struct vmx_msr_entry *e) 1001 { 1002 if (kvm_vcpu_read_guest(vcpu, 1003 gpa + i * sizeof(*e), 1004 e, 2 * sizeof(u32))) { 1005 pr_debug_ratelimited( 1006 "%s cannot read MSR entry (%u, 0x%08llx)\n", 1007 __func__, i, gpa + i * sizeof(*e)); 1008 return false; 1009 } 1010 if (nested_vmx_store_msr_check(vcpu, e)) { 1011 pr_debug_ratelimited( 1012 "%s check failed (%u, 0x%x, 0x%x)\n", 1013 __func__, i, e->index, e->reserved); 1014 return false; 1015 } 1016 return true; 1017 } 1018 1019 static int nested_vmx_store_msr(struct kvm_vcpu *vcpu, u64 gpa, u32 count) 1020 { 1021 u64 data; 1022 u32 i; 1023 struct vmx_msr_entry e; 1024 u32 max_msr_list_size = nested_vmx_max_atomic_switch_msrs(vcpu); 1025 1026 for (i = 0; i < count; i++) { 1027 if (unlikely(i >= max_msr_list_size)) 1028 return -EINVAL; 1029 1030 if (!read_and_check_msr_entry(vcpu, gpa, i, &e)) 1031 return -EINVAL; 1032 1033 if (!nested_vmx_get_vmexit_msr_value(vcpu, e.index, &data)) 1034 return -EINVAL; 1035 1036 if (kvm_vcpu_write_guest(vcpu, 1037 gpa + i * sizeof(e) + 1038 offsetof(struct vmx_msr_entry, value), 1039 &data, sizeof(data))) { 1040 pr_debug_ratelimited( 1041 "%s cannot write MSR (%u, 0x%x, 0x%llx)\n", 1042 __func__, i, e.index, data); 1043 return -EINVAL; 1044 } 1045 } 1046 return 0; 1047 } 1048 1049 static bool nested_msr_store_list_has_msr(struct kvm_vcpu *vcpu, u32 msr_index) 1050 { 1051 struct vmcs12 *vmcs12 = get_vmcs12(vcpu); 1052 u32 count = vmcs12->vm_exit_msr_store_count; 1053 u64 gpa = vmcs12->vm_exit_msr_store_addr; 1054 struct vmx_msr_entry e; 1055 u32 i; 1056 1057 for (i = 0; i < count; i++) { 1058 if (!read_and_check_msr_entry(vcpu, gpa, i, &e)) 1059 return false; 1060 1061 if (e.index == msr_index) 1062 return true; 1063 } 1064 return false; 1065 } 1066 1067 static void prepare_vmx_msr_autostore_list(struct kvm_vcpu *vcpu, 1068 u32 msr_index) 1069 { 1070 struct vcpu_vmx *vmx = to_vmx(vcpu); 1071 struct vmx_msrs *autostore = &vmx->msr_autostore.guest; 1072 bool in_vmcs12_store_list; 1073 int msr_autostore_slot; 1074 bool in_autostore_list; 1075 int last; 1076 1077 msr_autostore_slot = vmx_find_loadstore_msr_slot(autostore, msr_index); 1078 in_autostore_list = msr_autostore_slot >= 0; 1079 in_vmcs12_store_list = nested_msr_store_list_has_msr(vcpu, msr_index); 1080 1081 if (in_vmcs12_store_list && !in_autostore_list) { 1082 if (autostore->nr == MAX_NR_LOADSTORE_MSRS) { 1083 /* 1084 * Emulated VMEntry does not fail here. Instead a less 1085 * accurate value will be returned by 1086 * nested_vmx_get_vmexit_msr_value() using kvm_get_msr() 1087 * instead of reading the value from the vmcs02 VMExit 1088 * MSR-store area. 1089 */ 1090 pr_warn_ratelimited( 1091 "Not enough msr entries in msr_autostore. Can't add msr %x\n", 1092 msr_index); 1093 return; 1094 } 1095 last = autostore->nr++; 1096 autostore->val[last].index = msr_index; 1097 } else if (!in_vmcs12_store_list && in_autostore_list) { 1098 last = --autostore->nr; 1099 autostore->val[msr_autostore_slot] = autostore->val[last]; 1100 } 1101 } 1102 1103 /* 1104 * Load guest's/host's cr3 at nested entry/exit. @nested_ept is true if we are 1105 * emulating VM-Entry into a guest with EPT enabled. On failure, the expected 1106 * Exit Qualification (for a VM-Entry consistency check VM-Exit) is assigned to 1107 * @entry_failure_code. 1108 */ 1109 static int nested_vmx_load_cr3(struct kvm_vcpu *vcpu, unsigned long cr3, 1110 bool nested_ept, bool reload_pdptrs, 1111 enum vm_entry_failure_code *entry_failure_code) 1112 { 1113 if (CC(kvm_vcpu_is_illegal_gpa(vcpu, cr3))) { 1114 *entry_failure_code = ENTRY_FAIL_DEFAULT; 1115 return -EINVAL; 1116 } 1117 1118 /* 1119 * If PAE paging and EPT are both on, CR3 is not used by the CPU and 1120 * must not be dereferenced. 1121 */ 1122 if (reload_pdptrs && !nested_ept && is_pae_paging(vcpu) && 1123 CC(!load_pdptrs(vcpu, cr3))) { 1124 *entry_failure_code = ENTRY_FAIL_PDPTE; 1125 return -EINVAL; 1126 } 1127 1128 vcpu->arch.cr3 = cr3; 1129 kvm_register_mark_dirty(vcpu, VCPU_EXREG_CR3); 1130 1131 /* Re-initialize the MMU, e.g. to pick up CR4 MMU role changes. */ 1132 kvm_init_mmu(vcpu); 1133 1134 if (!nested_ept) 1135 kvm_mmu_new_pgd(vcpu, cr3); 1136 1137 return 0; 1138 } 1139 1140 /* 1141 * Returns if KVM is able to config CPU to tag TLB entries 1142 * populated by L2 differently than TLB entries populated 1143 * by L1. 1144 * 1145 * If L0 uses EPT, L1 and L2 run with different EPTP because 1146 * guest_mode is part of kvm_mmu_page_role. Thus, TLB entries 1147 * are tagged with different EPTP. 1148 * 1149 * If L1 uses VPID and we allocated a vpid02, TLB entries are tagged 1150 * with different VPID (L1 entries are tagged with vmx->vpid 1151 * while L2 entries are tagged with vmx->nested.vpid02). 1152 */ 1153 static bool nested_has_guest_tlb_tag(struct kvm_vcpu *vcpu) 1154 { 1155 struct vmcs12 *vmcs12 = get_vmcs12(vcpu); 1156 1157 return enable_ept || 1158 (nested_cpu_has_vpid(vmcs12) && to_vmx(vcpu)->nested.vpid02); 1159 } 1160 1161 static void nested_vmx_transition_tlb_flush(struct kvm_vcpu *vcpu, 1162 struct vmcs12 *vmcs12, 1163 bool is_vmenter) 1164 { 1165 struct vcpu_vmx *vmx = to_vmx(vcpu); 1166 1167 /* 1168 * If vmcs12 doesn't use VPID, L1 expects linear and combined mappings 1169 * for *all* contexts to be flushed on VM-Enter/VM-Exit, i.e. it's a 1170 * full TLB flush from the guest's perspective. This is required even 1171 * if VPID is disabled in the host as KVM may need to synchronize the 1172 * MMU in response to the guest TLB flush. 1173 * 1174 * Note, using TLB_FLUSH_GUEST is correct even if nested EPT is in use. 1175 * EPT is a special snowflake, as guest-physical mappings aren't 1176 * flushed on VPID invalidations, including VM-Enter or VM-Exit with 1177 * VPID disabled. As a result, KVM _never_ needs to sync nEPT 1178 * entries on VM-Enter because L1 can't rely on VM-Enter to flush 1179 * those mappings. 1180 */ 1181 if (!nested_cpu_has_vpid(vmcs12)) { 1182 kvm_make_request(KVM_REQ_TLB_FLUSH_GUEST, vcpu); 1183 return; 1184 } 1185 1186 /* L2 should never have a VPID if VPID is disabled. */ 1187 WARN_ON(!enable_vpid); 1188 1189 /* 1190 * VPID is enabled and in use by vmcs12. If vpid12 is changing, then 1191 * emulate a guest TLB flush as KVM does not track vpid12 history nor 1192 * is the VPID incorporated into the MMU context. I.e. KVM must assume 1193 * that the new vpid12 has never been used and thus represents a new 1194 * guest ASID that cannot have entries in the TLB. 1195 */ 1196 if (is_vmenter && vmcs12->virtual_processor_id != vmx->nested.last_vpid) { 1197 vmx->nested.last_vpid = vmcs12->virtual_processor_id; 1198 kvm_make_request(KVM_REQ_TLB_FLUSH_GUEST, vcpu); 1199 return; 1200 } 1201 1202 /* 1203 * If VPID is enabled, used by vmc12, and vpid12 is not changing but 1204 * does not have a unique TLB tag (ASID), i.e. EPT is disabled and 1205 * KVM was unable to allocate a VPID for L2, flush the current context 1206 * as the effective ASID is common to both L1 and L2. 1207 */ 1208 if (!nested_has_guest_tlb_tag(vcpu)) 1209 kvm_make_request(KVM_REQ_TLB_FLUSH_CURRENT, vcpu); 1210 } 1211 1212 static bool is_bitwise_subset(u64 superset, u64 subset, u64 mask) 1213 { 1214 superset &= mask; 1215 subset &= mask; 1216 1217 return (superset | subset) == superset; 1218 } 1219 1220 static int vmx_restore_vmx_basic(struct vcpu_vmx *vmx, u64 data) 1221 { 1222 const u64 feature_and_reserved = 1223 /* feature (except bit 48; see below) */ 1224 BIT_ULL(49) | BIT_ULL(54) | BIT_ULL(55) | 1225 /* reserved */ 1226 BIT_ULL(31) | GENMASK_ULL(47, 45) | GENMASK_ULL(63, 56); 1227 u64 vmx_basic = vmx->nested.msrs.basic; 1228 1229 if (!is_bitwise_subset(vmx_basic, data, feature_and_reserved)) 1230 return -EINVAL; 1231 1232 /* 1233 * KVM does not emulate a version of VMX that constrains physical 1234 * addresses of VMX structures (e.g. VMCS) to 32-bits. 1235 */ 1236 if (data & BIT_ULL(48)) 1237 return -EINVAL; 1238 1239 if (vmx_basic_vmcs_revision_id(vmx_basic) != 1240 vmx_basic_vmcs_revision_id(data)) 1241 return -EINVAL; 1242 1243 if (vmx_basic_vmcs_size(vmx_basic) > vmx_basic_vmcs_size(data)) 1244 return -EINVAL; 1245 1246 vmx->nested.msrs.basic = data; 1247 return 0; 1248 } 1249 1250 static int 1251 vmx_restore_control_msr(struct vcpu_vmx *vmx, u32 msr_index, u64 data) 1252 { 1253 u64 supported; 1254 u32 *lowp, *highp; 1255 1256 switch (msr_index) { 1257 case MSR_IA32_VMX_TRUE_PINBASED_CTLS: 1258 lowp = &vmx->nested.msrs.pinbased_ctls_low; 1259 highp = &vmx->nested.msrs.pinbased_ctls_high; 1260 break; 1261 case MSR_IA32_VMX_TRUE_PROCBASED_CTLS: 1262 lowp = &vmx->nested.msrs.procbased_ctls_low; 1263 highp = &vmx->nested.msrs.procbased_ctls_high; 1264 break; 1265 case MSR_IA32_VMX_TRUE_EXIT_CTLS: 1266 lowp = &vmx->nested.msrs.exit_ctls_low; 1267 highp = &vmx->nested.msrs.exit_ctls_high; 1268 break; 1269 case MSR_IA32_VMX_TRUE_ENTRY_CTLS: 1270 lowp = &vmx->nested.msrs.entry_ctls_low; 1271 highp = &vmx->nested.msrs.entry_ctls_high; 1272 break; 1273 case MSR_IA32_VMX_PROCBASED_CTLS2: 1274 lowp = &vmx->nested.msrs.secondary_ctls_low; 1275 highp = &vmx->nested.msrs.secondary_ctls_high; 1276 break; 1277 default: 1278 BUG(); 1279 } 1280 1281 supported = vmx_control_msr(*lowp, *highp); 1282 1283 /* Check must-be-1 bits are still 1. */ 1284 if (!is_bitwise_subset(data, supported, GENMASK_ULL(31, 0))) 1285 return -EINVAL; 1286 1287 /* Check must-be-0 bits are still 0. */ 1288 if (!is_bitwise_subset(supported, data, GENMASK_ULL(63, 32))) 1289 return -EINVAL; 1290 1291 *lowp = data; 1292 *highp = data >> 32; 1293 return 0; 1294 } 1295 1296 static int vmx_restore_vmx_misc(struct vcpu_vmx *vmx, u64 data) 1297 { 1298 const u64 feature_and_reserved_bits = 1299 /* feature */ 1300 BIT_ULL(5) | GENMASK_ULL(8, 6) | BIT_ULL(14) | BIT_ULL(15) | 1301 BIT_ULL(28) | BIT_ULL(29) | BIT_ULL(30) | 1302 /* reserved */ 1303 GENMASK_ULL(13, 9) | BIT_ULL(31); 1304 u64 vmx_misc; 1305 1306 vmx_misc = vmx_control_msr(vmx->nested.msrs.misc_low, 1307 vmx->nested.msrs.misc_high); 1308 1309 if (!is_bitwise_subset(vmx_misc, data, feature_and_reserved_bits)) 1310 return -EINVAL; 1311 1312 if ((vmx->nested.msrs.pinbased_ctls_high & 1313 PIN_BASED_VMX_PREEMPTION_TIMER) && 1314 vmx_misc_preemption_timer_rate(data) != 1315 vmx_misc_preemption_timer_rate(vmx_misc)) 1316 return -EINVAL; 1317 1318 if (vmx_misc_cr3_count(data) > vmx_misc_cr3_count(vmx_misc)) 1319 return -EINVAL; 1320 1321 if (vmx_misc_max_msr(data) > vmx_misc_max_msr(vmx_misc)) 1322 return -EINVAL; 1323 1324 if (vmx_misc_mseg_revid(data) != vmx_misc_mseg_revid(vmx_misc)) 1325 return -EINVAL; 1326 1327 vmx->nested.msrs.misc_low = data; 1328 vmx->nested.msrs.misc_high = data >> 32; 1329 1330 return 0; 1331 } 1332 1333 static int vmx_restore_vmx_ept_vpid_cap(struct vcpu_vmx *vmx, u64 data) 1334 { 1335 u64 vmx_ept_vpid_cap; 1336 1337 vmx_ept_vpid_cap = vmx_control_msr(vmx->nested.msrs.ept_caps, 1338 vmx->nested.msrs.vpid_caps); 1339 1340 /* Every bit is either reserved or a feature bit. */ 1341 if (!is_bitwise_subset(vmx_ept_vpid_cap, data, -1ULL)) 1342 return -EINVAL; 1343 1344 vmx->nested.msrs.ept_caps = data; 1345 vmx->nested.msrs.vpid_caps = data >> 32; 1346 return 0; 1347 } 1348 1349 static int vmx_restore_fixed0_msr(struct vcpu_vmx *vmx, u32 msr_index, u64 data) 1350 { 1351 u64 *msr; 1352 1353 switch (msr_index) { 1354 case MSR_IA32_VMX_CR0_FIXED0: 1355 msr = &vmx->nested.msrs.cr0_fixed0; 1356 break; 1357 case MSR_IA32_VMX_CR4_FIXED0: 1358 msr = &vmx->nested.msrs.cr4_fixed0; 1359 break; 1360 default: 1361 BUG(); 1362 } 1363 1364 /* 1365 * 1 bits (which indicates bits which "must-be-1" during VMX operation) 1366 * must be 1 in the restored value. 1367 */ 1368 if (!is_bitwise_subset(data, *msr, -1ULL)) 1369 return -EINVAL; 1370 1371 *msr = data; 1372 return 0; 1373 } 1374 1375 /* 1376 * Called when userspace is restoring VMX MSRs. 1377 * 1378 * Returns 0 on success, non-0 otherwise. 1379 */ 1380 int vmx_set_vmx_msr(struct kvm_vcpu *vcpu, u32 msr_index, u64 data) 1381 { 1382 struct vcpu_vmx *vmx = to_vmx(vcpu); 1383 1384 /* 1385 * Don't allow changes to the VMX capability MSRs while the vCPU 1386 * is in VMX operation. 1387 */ 1388 if (vmx->nested.vmxon) 1389 return -EBUSY; 1390 1391 switch (msr_index) { 1392 case MSR_IA32_VMX_BASIC: 1393 return vmx_restore_vmx_basic(vmx, data); 1394 case MSR_IA32_VMX_PINBASED_CTLS: 1395 case MSR_IA32_VMX_PROCBASED_CTLS: 1396 case MSR_IA32_VMX_EXIT_CTLS: 1397 case MSR_IA32_VMX_ENTRY_CTLS: 1398 /* 1399 * The "non-true" VMX capability MSRs are generated from the 1400 * "true" MSRs, so we do not support restoring them directly. 1401 * 1402 * If userspace wants to emulate VMX_BASIC[55]=0, userspace 1403 * should restore the "true" MSRs with the must-be-1 bits 1404 * set according to the SDM Vol 3. A.2 "RESERVED CONTROLS AND 1405 * DEFAULT SETTINGS". 1406 */ 1407 return -EINVAL; 1408 case MSR_IA32_VMX_TRUE_PINBASED_CTLS: 1409 case MSR_IA32_VMX_TRUE_PROCBASED_CTLS: 1410 case MSR_IA32_VMX_TRUE_EXIT_CTLS: 1411 case MSR_IA32_VMX_TRUE_ENTRY_CTLS: 1412 case MSR_IA32_VMX_PROCBASED_CTLS2: 1413 return vmx_restore_control_msr(vmx, msr_index, data); 1414 case MSR_IA32_VMX_MISC: 1415 return vmx_restore_vmx_misc(vmx, data); 1416 case MSR_IA32_VMX_CR0_FIXED0: 1417 case MSR_IA32_VMX_CR4_FIXED0: 1418 return vmx_restore_fixed0_msr(vmx, msr_index, data); 1419 case MSR_IA32_VMX_CR0_FIXED1: 1420 case MSR_IA32_VMX_CR4_FIXED1: 1421 /* 1422 * These MSRs are generated based on the vCPU's CPUID, so we 1423 * do not support restoring them directly. 1424 */ 1425 return -EINVAL; 1426 case MSR_IA32_VMX_EPT_VPID_CAP: 1427 return vmx_restore_vmx_ept_vpid_cap(vmx, data); 1428 case MSR_IA32_VMX_VMCS_ENUM: 1429 vmx->nested.msrs.vmcs_enum = data; 1430 return 0; 1431 case MSR_IA32_VMX_VMFUNC: 1432 if (data & ~vmx->nested.msrs.vmfunc_controls) 1433 return -EINVAL; 1434 vmx->nested.msrs.vmfunc_controls = data; 1435 return 0; 1436 default: 1437 /* 1438 * The rest of the VMX capability MSRs do not support restore. 1439 */ 1440 return -EINVAL; 1441 } 1442 } 1443 1444 /* Returns 0 on success, non-0 otherwise. */ 1445 int vmx_get_vmx_msr(struct nested_vmx_msrs *msrs, u32 msr_index, u64 *pdata) 1446 { 1447 switch (msr_index) { 1448 case MSR_IA32_VMX_BASIC: 1449 *pdata = msrs->basic; 1450 break; 1451 case MSR_IA32_VMX_TRUE_PINBASED_CTLS: 1452 case MSR_IA32_VMX_PINBASED_CTLS: 1453 *pdata = vmx_control_msr( 1454 msrs->pinbased_ctls_low, 1455 msrs->pinbased_ctls_high); 1456 if (msr_index == MSR_IA32_VMX_PINBASED_CTLS) 1457 *pdata |= PIN_BASED_ALWAYSON_WITHOUT_TRUE_MSR; 1458 break; 1459 case MSR_IA32_VMX_TRUE_PROCBASED_CTLS: 1460 case MSR_IA32_VMX_PROCBASED_CTLS: 1461 *pdata = vmx_control_msr( 1462 msrs->procbased_ctls_low, 1463 msrs->procbased_ctls_high); 1464 if (msr_index == MSR_IA32_VMX_PROCBASED_CTLS) 1465 *pdata |= CPU_BASED_ALWAYSON_WITHOUT_TRUE_MSR; 1466 break; 1467 case MSR_IA32_VMX_TRUE_EXIT_CTLS: 1468 case MSR_IA32_VMX_EXIT_CTLS: 1469 *pdata = vmx_control_msr( 1470 msrs->exit_ctls_low, 1471 msrs->exit_ctls_high); 1472 if (msr_index == MSR_IA32_VMX_EXIT_CTLS) 1473 *pdata |= VM_EXIT_ALWAYSON_WITHOUT_TRUE_MSR; 1474 break; 1475 case MSR_IA32_VMX_TRUE_ENTRY_CTLS: 1476 case MSR_IA32_VMX_ENTRY_CTLS: 1477 *pdata = vmx_control_msr( 1478 msrs->entry_ctls_low, 1479 msrs->entry_ctls_high); 1480 if (msr_index == MSR_IA32_VMX_ENTRY_CTLS) 1481 *pdata |= VM_ENTRY_ALWAYSON_WITHOUT_TRUE_MSR; 1482 break; 1483 case MSR_IA32_VMX_MISC: 1484 *pdata = vmx_control_msr( 1485 msrs->misc_low, 1486 msrs->misc_high); 1487 break; 1488 case MSR_IA32_VMX_CR0_FIXED0: 1489 *pdata = msrs->cr0_fixed0; 1490 break; 1491 case MSR_IA32_VMX_CR0_FIXED1: 1492 *pdata = msrs->cr0_fixed1; 1493 break; 1494 case MSR_IA32_VMX_CR4_FIXED0: 1495 *pdata = msrs->cr4_fixed0; 1496 break; 1497 case MSR_IA32_VMX_CR4_FIXED1: 1498 *pdata = msrs->cr4_fixed1; 1499 break; 1500 case MSR_IA32_VMX_VMCS_ENUM: 1501 *pdata = msrs->vmcs_enum; 1502 break; 1503 case MSR_IA32_VMX_PROCBASED_CTLS2: 1504 *pdata = vmx_control_msr( 1505 msrs->secondary_ctls_low, 1506 msrs->secondary_ctls_high); 1507 break; 1508 case MSR_IA32_VMX_EPT_VPID_CAP: 1509 *pdata = msrs->ept_caps | 1510 ((u64)msrs->vpid_caps << 32); 1511 break; 1512 case MSR_IA32_VMX_VMFUNC: 1513 *pdata = msrs->vmfunc_controls; 1514 break; 1515 default: 1516 return 1; 1517 } 1518 1519 return 0; 1520 } 1521 1522 /* 1523 * Copy the writable VMCS shadow fields back to the VMCS12, in case they have 1524 * been modified by the L1 guest. Note, "writable" in this context means 1525 * "writable by the guest", i.e. tagged SHADOW_FIELD_RW; the set of 1526 * fields tagged SHADOW_FIELD_RO may or may not align with the "read-only" 1527 * VM-exit information fields (which are actually writable if the vCPU is 1528 * configured to support "VMWRITE to any supported field in the VMCS"). 1529 */ 1530 static void copy_shadow_to_vmcs12(struct vcpu_vmx *vmx) 1531 { 1532 struct vmcs *shadow_vmcs = vmx->vmcs01.shadow_vmcs; 1533 struct vmcs12 *vmcs12 = get_vmcs12(&vmx->vcpu); 1534 struct shadow_vmcs_field field; 1535 unsigned long val; 1536 int i; 1537 1538 if (WARN_ON(!shadow_vmcs)) 1539 return; 1540 1541 preempt_disable(); 1542 1543 vmcs_load(shadow_vmcs); 1544 1545 for (i = 0; i < max_shadow_read_write_fields; i++) { 1546 field = shadow_read_write_fields[i]; 1547 val = __vmcs_readl(field.encoding); 1548 vmcs12_write_any(vmcs12, field.encoding, field.offset, val); 1549 } 1550 1551 vmcs_clear(shadow_vmcs); 1552 vmcs_load(vmx->loaded_vmcs->vmcs); 1553 1554 preempt_enable(); 1555 } 1556 1557 static void copy_vmcs12_to_shadow(struct vcpu_vmx *vmx) 1558 { 1559 const struct shadow_vmcs_field *fields[] = { 1560 shadow_read_write_fields, 1561 shadow_read_only_fields 1562 }; 1563 const int max_fields[] = { 1564 max_shadow_read_write_fields, 1565 max_shadow_read_only_fields 1566 }; 1567 struct vmcs *shadow_vmcs = vmx->vmcs01.shadow_vmcs; 1568 struct vmcs12 *vmcs12 = get_vmcs12(&vmx->vcpu); 1569 struct shadow_vmcs_field field; 1570 unsigned long val; 1571 int i, q; 1572 1573 if (WARN_ON(!shadow_vmcs)) 1574 return; 1575 1576 vmcs_load(shadow_vmcs); 1577 1578 for (q = 0; q < ARRAY_SIZE(fields); q++) { 1579 for (i = 0; i < max_fields[q]; i++) { 1580 field = fields[q][i]; 1581 val = vmcs12_read_any(vmcs12, field.encoding, 1582 field.offset); 1583 __vmcs_writel(field.encoding, val); 1584 } 1585 } 1586 1587 vmcs_clear(shadow_vmcs); 1588 vmcs_load(vmx->loaded_vmcs->vmcs); 1589 } 1590 1591 static void copy_enlightened_to_vmcs12(struct vcpu_vmx *vmx, u32 hv_clean_fields) 1592 { 1593 struct vmcs12 *vmcs12 = vmx->nested.cached_vmcs12; 1594 struct hv_enlightened_vmcs *evmcs = vmx->nested.hv_evmcs; 1595 1596 /* HV_VMX_ENLIGHTENED_CLEAN_FIELD_NONE */ 1597 vmcs12->tpr_threshold = evmcs->tpr_threshold; 1598 vmcs12->guest_rip = evmcs->guest_rip; 1599 1600 if (unlikely(!(hv_clean_fields & 1601 HV_VMX_ENLIGHTENED_CLEAN_FIELD_GUEST_BASIC))) { 1602 vmcs12->guest_rsp = evmcs->guest_rsp; 1603 vmcs12->guest_rflags = evmcs->guest_rflags; 1604 vmcs12->guest_interruptibility_info = 1605 evmcs->guest_interruptibility_info; 1606 } 1607 1608 if (unlikely(!(hv_clean_fields & 1609 HV_VMX_ENLIGHTENED_CLEAN_FIELD_CONTROL_PROC))) { 1610 vmcs12->cpu_based_vm_exec_control = 1611 evmcs->cpu_based_vm_exec_control; 1612 } 1613 1614 if (unlikely(!(hv_clean_fields & 1615 HV_VMX_ENLIGHTENED_CLEAN_FIELD_CONTROL_EXCPN))) { 1616 vmcs12->exception_bitmap = evmcs->exception_bitmap; 1617 } 1618 1619 if (unlikely(!(hv_clean_fields & 1620 HV_VMX_ENLIGHTENED_CLEAN_FIELD_CONTROL_ENTRY))) { 1621 vmcs12->vm_entry_controls = evmcs->vm_entry_controls; 1622 } 1623 1624 if (unlikely(!(hv_clean_fields & 1625 HV_VMX_ENLIGHTENED_CLEAN_FIELD_CONTROL_EVENT))) { 1626 vmcs12->vm_entry_intr_info_field = 1627 evmcs->vm_entry_intr_info_field; 1628 vmcs12->vm_entry_exception_error_code = 1629 evmcs->vm_entry_exception_error_code; 1630 vmcs12->vm_entry_instruction_len = 1631 evmcs->vm_entry_instruction_len; 1632 } 1633 1634 if (unlikely(!(hv_clean_fields & 1635 HV_VMX_ENLIGHTENED_CLEAN_FIELD_HOST_GRP1))) { 1636 vmcs12->host_ia32_pat = evmcs->host_ia32_pat; 1637 vmcs12->host_ia32_efer = evmcs->host_ia32_efer; 1638 vmcs12->host_cr0 = evmcs->host_cr0; 1639 vmcs12->host_cr3 = evmcs->host_cr3; 1640 vmcs12->host_cr4 = evmcs->host_cr4; 1641 vmcs12->host_ia32_sysenter_esp = evmcs->host_ia32_sysenter_esp; 1642 vmcs12->host_ia32_sysenter_eip = evmcs->host_ia32_sysenter_eip; 1643 vmcs12->host_rip = evmcs->host_rip; 1644 vmcs12->host_ia32_sysenter_cs = evmcs->host_ia32_sysenter_cs; 1645 vmcs12->host_es_selector = evmcs->host_es_selector; 1646 vmcs12->host_cs_selector = evmcs->host_cs_selector; 1647 vmcs12->host_ss_selector = evmcs->host_ss_selector; 1648 vmcs12->host_ds_selector = evmcs->host_ds_selector; 1649 vmcs12->host_fs_selector = evmcs->host_fs_selector; 1650 vmcs12->host_gs_selector = evmcs->host_gs_selector; 1651 vmcs12->host_tr_selector = evmcs->host_tr_selector; 1652 } 1653 1654 if (unlikely(!(hv_clean_fields & 1655 HV_VMX_ENLIGHTENED_CLEAN_FIELD_CONTROL_GRP1))) { 1656 vmcs12->pin_based_vm_exec_control = 1657 evmcs->pin_based_vm_exec_control; 1658 vmcs12->vm_exit_controls = evmcs->vm_exit_controls; 1659 vmcs12->secondary_vm_exec_control = 1660 evmcs->secondary_vm_exec_control; 1661 } 1662 1663 if (unlikely(!(hv_clean_fields & 1664 HV_VMX_ENLIGHTENED_CLEAN_FIELD_IO_BITMAP))) { 1665 vmcs12->io_bitmap_a = evmcs->io_bitmap_a; 1666 vmcs12->io_bitmap_b = evmcs->io_bitmap_b; 1667 } 1668 1669 if (unlikely(!(hv_clean_fields & 1670 HV_VMX_ENLIGHTENED_CLEAN_FIELD_MSR_BITMAP))) { 1671 vmcs12->msr_bitmap = evmcs->msr_bitmap; 1672 } 1673 1674 if (unlikely(!(hv_clean_fields & 1675 HV_VMX_ENLIGHTENED_CLEAN_FIELD_GUEST_GRP2))) { 1676 vmcs12->guest_es_base = evmcs->guest_es_base; 1677 vmcs12->guest_cs_base = evmcs->guest_cs_base; 1678 vmcs12->guest_ss_base = evmcs->guest_ss_base; 1679 vmcs12->guest_ds_base = evmcs->guest_ds_base; 1680 vmcs12->guest_fs_base = evmcs->guest_fs_base; 1681 vmcs12->guest_gs_base = evmcs->guest_gs_base; 1682 vmcs12->guest_ldtr_base = evmcs->guest_ldtr_base; 1683 vmcs12->guest_tr_base = evmcs->guest_tr_base; 1684 vmcs12->guest_gdtr_base = evmcs->guest_gdtr_base; 1685 vmcs12->guest_idtr_base = evmcs->guest_idtr_base; 1686 vmcs12->guest_es_limit = evmcs->guest_es_limit; 1687 vmcs12->guest_cs_limit = evmcs->guest_cs_limit; 1688 vmcs12->guest_ss_limit = evmcs->guest_ss_limit; 1689 vmcs12->guest_ds_limit = evmcs->guest_ds_limit; 1690 vmcs12->guest_fs_limit = evmcs->guest_fs_limit; 1691 vmcs12->guest_gs_limit = evmcs->guest_gs_limit; 1692 vmcs12->guest_ldtr_limit = evmcs->guest_ldtr_limit; 1693 vmcs12->guest_tr_limit = evmcs->guest_tr_limit; 1694 vmcs12->guest_gdtr_limit = evmcs->guest_gdtr_limit; 1695 vmcs12->guest_idtr_limit = evmcs->guest_idtr_limit; 1696 vmcs12->guest_es_ar_bytes = evmcs->guest_es_ar_bytes; 1697 vmcs12->guest_cs_ar_bytes = evmcs->guest_cs_ar_bytes; 1698 vmcs12->guest_ss_ar_bytes = evmcs->guest_ss_ar_bytes; 1699 vmcs12->guest_ds_ar_bytes = evmcs->guest_ds_ar_bytes; 1700 vmcs12->guest_fs_ar_bytes = evmcs->guest_fs_ar_bytes; 1701 vmcs12->guest_gs_ar_bytes = evmcs->guest_gs_ar_bytes; 1702 vmcs12->guest_ldtr_ar_bytes = evmcs->guest_ldtr_ar_bytes; 1703 vmcs12->guest_tr_ar_bytes = evmcs->guest_tr_ar_bytes; 1704 vmcs12->guest_es_selector = evmcs->guest_es_selector; 1705 vmcs12->guest_cs_selector = evmcs->guest_cs_selector; 1706 vmcs12->guest_ss_selector = evmcs->guest_ss_selector; 1707 vmcs12->guest_ds_selector = evmcs->guest_ds_selector; 1708 vmcs12->guest_fs_selector = evmcs->guest_fs_selector; 1709 vmcs12->guest_gs_selector = evmcs->guest_gs_selector; 1710 vmcs12->guest_ldtr_selector = evmcs->guest_ldtr_selector; 1711 vmcs12->guest_tr_selector = evmcs->guest_tr_selector; 1712 } 1713 1714 if (unlikely(!(hv_clean_fields & 1715 HV_VMX_ENLIGHTENED_CLEAN_FIELD_CONTROL_GRP2))) { 1716 vmcs12->tsc_offset = evmcs->tsc_offset; 1717 vmcs12->virtual_apic_page_addr = evmcs->virtual_apic_page_addr; 1718 vmcs12->xss_exit_bitmap = evmcs->xss_exit_bitmap; 1719 } 1720 1721 if (unlikely(!(hv_clean_fields & 1722 HV_VMX_ENLIGHTENED_CLEAN_FIELD_CRDR))) { 1723 vmcs12->cr0_guest_host_mask = evmcs->cr0_guest_host_mask; 1724 vmcs12->cr4_guest_host_mask = evmcs->cr4_guest_host_mask; 1725 vmcs12->cr0_read_shadow = evmcs->cr0_read_shadow; 1726 vmcs12->cr4_read_shadow = evmcs->cr4_read_shadow; 1727 vmcs12->guest_cr0 = evmcs->guest_cr0; 1728 vmcs12->guest_cr3 = evmcs->guest_cr3; 1729 vmcs12->guest_cr4 = evmcs->guest_cr4; 1730 vmcs12->guest_dr7 = evmcs->guest_dr7; 1731 } 1732 1733 if (unlikely(!(hv_clean_fields & 1734 HV_VMX_ENLIGHTENED_CLEAN_FIELD_HOST_POINTER))) { 1735 vmcs12->host_fs_base = evmcs->host_fs_base; 1736 vmcs12->host_gs_base = evmcs->host_gs_base; 1737 vmcs12->host_tr_base = evmcs->host_tr_base; 1738 vmcs12->host_gdtr_base = evmcs->host_gdtr_base; 1739 vmcs12->host_idtr_base = evmcs->host_idtr_base; 1740 vmcs12->host_rsp = evmcs->host_rsp; 1741 } 1742 1743 if (unlikely(!(hv_clean_fields & 1744 HV_VMX_ENLIGHTENED_CLEAN_FIELD_CONTROL_XLAT))) { 1745 vmcs12->ept_pointer = evmcs->ept_pointer; 1746 vmcs12->virtual_processor_id = evmcs->virtual_processor_id; 1747 } 1748 1749 if (unlikely(!(hv_clean_fields & 1750 HV_VMX_ENLIGHTENED_CLEAN_FIELD_GUEST_GRP1))) { 1751 vmcs12->vmcs_link_pointer = evmcs->vmcs_link_pointer; 1752 vmcs12->guest_ia32_debugctl = evmcs->guest_ia32_debugctl; 1753 vmcs12->guest_ia32_pat = evmcs->guest_ia32_pat; 1754 vmcs12->guest_ia32_efer = evmcs->guest_ia32_efer; 1755 vmcs12->guest_pdptr0 = evmcs->guest_pdptr0; 1756 vmcs12->guest_pdptr1 = evmcs->guest_pdptr1; 1757 vmcs12->guest_pdptr2 = evmcs->guest_pdptr2; 1758 vmcs12->guest_pdptr3 = evmcs->guest_pdptr3; 1759 vmcs12->guest_pending_dbg_exceptions = 1760 evmcs->guest_pending_dbg_exceptions; 1761 vmcs12->guest_sysenter_esp = evmcs->guest_sysenter_esp; 1762 vmcs12->guest_sysenter_eip = evmcs->guest_sysenter_eip; 1763 vmcs12->guest_bndcfgs = evmcs->guest_bndcfgs; 1764 vmcs12->guest_activity_state = evmcs->guest_activity_state; 1765 vmcs12->guest_sysenter_cs = evmcs->guest_sysenter_cs; 1766 } 1767 1768 /* 1769 * Not used? 1770 * vmcs12->vm_exit_msr_store_addr = evmcs->vm_exit_msr_store_addr; 1771 * vmcs12->vm_exit_msr_load_addr = evmcs->vm_exit_msr_load_addr; 1772 * vmcs12->vm_entry_msr_load_addr = evmcs->vm_entry_msr_load_addr; 1773 * vmcs12->page_fault_error_code_mask = 1774 * evmcs->page_fault_error_code_mask; 1775 * vmcs12->page_fault_error_code_match = 1776 * evmcs->page_fault_error_code_match; 1777 * vmcs12->cr3_target_count = evmcs->cr3_target_count; 1778 * vmcs12->vm_exit_msr_store_count = evmcs->vm_exit_msr_store_count; 1779 * vmcs12->vm_exit_msr_load_count = evmcs->vm_exit_msr_load_count; 1780 * vmcs12->vm_entry_msr_load_count = evmcs->vm_entry_msr_load_count; 1781 */ 1782 1783 /* 1784 * Read only fields: 1785 * vmcs12->guest_physical_address = evmcs->guest_physical_address; 1786 * vmcs12->vm_instruction_error = evmcs->vm_instruction_error; 1787 * vmcs12->vm_exit_reason = evmcs->vm_exit_reason; 1788 * vmcs12->vm_exit_intr_info = evmcs->vm_exit_intr_info; 1789 * vmcs12->vm_exit_intr_error_code = evmcs->vm_exit_intr_error_code; 1790 * vmcs12->idt_vectoring_info_field = evmcs->idt_vectoring_info_field; 1791 * vmcs12->idt_vectoring_error_code = evmcs->idt_vectoring_error_code; 1792 * vmcs12->vm_exit_instruction_len = evmcs->vm_exit_instruction_len; 1793 * vmcs12->vmx_instruction_info = evmcs->vmx_instruction_info; 1794 * vmcs12->exit_qualification = evmcs->exit_qualification; 1795 * vmcs12->guest_linear_address = evmcs->guest_linear_address; 1796 * 1797 * Not present in struct vmcs12: 1798 * vmcs12->exit_io_instruction_ecx = evmcs->exit_io_instruction_ecx; 1799 * vmcs12->exit_io_instruction_esi = evmcs->exit_io_instruction_esi; 1800 * vmcs12->exit_io_instruction_edi = evmcs->exit_io_instruction_edi; 1801 * vmcs12->exit_io_instruction_eip = evmcs->exit_io_instruction_eip; 1802 */ 1803 1804 return; 1805 } 1806 1807 static void copy_vmcs12_to_enlightened(struct vcpu_vmx *vmx) 1808 { 1809 struct vmcs12 *vmcs12 = vmx->nested.cached_vmcs12; 1810 struct hv_enlightened_vmcs *evmcs = vmx->nested.hv_evmcs; 1811 1812 /* 1813 * Should not be changed by KVM: 1814 * 1815 * evmcs->host_es_selector = vmcs12->host_es_selector; 1816 * evmcs->host_cs_selector = vmcs12->host_cs_selector; 1817 * evmcs->host_ss_selector = vmcs12->host_ss_selector; 1818 * evmcs->host_ds_selector = vmcs12->host_ds_selector; 1819 * evmcs->host_fs_selector = vmcs12->host_fs_selector; 1820 * evmcs->host_gs_selector = vmcs12->host_gs_selector; 1821 * evmcs->host_tr_selector = vmcs12->host_tr_selector; 1822 * evmcs->host_ia32_pat = vmcs12->host_ia32_pat; 1823 * evmcs->host_ia32_efer = vmcs12->host_ia32_efer; 1824 * evmcs->host_cr0 = vmcs12->host_cr0; 1825 * evmcs->host_cr3 = vmcs12->host_cr3; 1826 * evmcs->host_cr4 = vmcs12->host_cr4; 1827 * evmcs->host_ia32_sysenter_esp = vmcs12->host_ia32_sysenter_esp; 1828 * evmcs->host_ia32_sysenter_eip = vmcs12->host_ia32_sysenter_eip; 1829 * evmcs->host_rip = vmcs12->host_rip; 1830 * evmcs->host_ia32_sysenter_cs = vmcs12->host_ia32_sysenter_cs; 1831 * evmcs->host_fs_base = vmcs12->host_fs_base; 1832 * evmcs->host_gs_base = vmcs12->host_gs_base; 1833 * evmcs->host_tr_base = vmcs12->host_tr_base; 1834 * evmcs->host_gdtr_base = vmcs12->host_gdtr_base; 1835 * evmcs->host_idtr_base = vmcs12->host_idtr_base; 1836 * evmcs->host_rsp = vmcs12->host_rsp; 1837 * sync_vmcs02_to_vmcs12() doesn't read these: 1838 * evmcs->io_bitmap_a = vmcs12->io_bitmap_a; 1839 * evmcs->io_bitmap_b = vmcs12->io_bitmap_b; 1840 * evmcs->msr_bitmap = vmcs12->msr_bitmap; 1841 * evmcs->ept_pointer = vmcs12->ept_pointer; 1842 * evmcs->xss_exit_bitmap = vmcs12->xss_exit_bitmap; 1843 * evmcs->vm_exit_msr_store_addr = vmcs12->vm_exit_msr_store_addr; 1844 * evmcs->vm_exit_msr_load_addr = vmcs12->vm_exit_msr_load_addr; 1845 * evmcs->vm_entry_msr_load_addr = vmcs12->vm_entry_msr_load_addr; 1846 * evmcs->tpr_threshold = vmcs12->tpr_threshold; 1847 * evmcs->virtual_processor_id = vmcs12->virtual_processor_id; 1848 * evmcs->exception_bitmap = vmcs12->exception_bitmap; 1849 * evmcs->vmcs_link_pointer = vmcs12->vmcs_link_pointer; 1850 * evmcs->pin_based_vm_exec_control = vmcs12->pin_based_vm_exec_control; 1851 * evmcs->vm_exit_controls = vmcs12->vm_exit_controls; 1852 * evmcs->secondary_vm_exec_control = vmcs12->secondary_vm_exec_control; 1853 * evmcs->page_fault_error_code_mask = 1854 * vmcs12->page_fault_error_code_mask; 1855 * evmcs->page_fault_error_code_match = 1856 * vmcs12->page_fault_error_code_match; 1857 * evmcs->cr3_target_count = vmcs12->cr3_target_count; 1858 * evmcs->virtual_apic_page_addr = vmcs12->virtual_apic_page_addr; 1859 * evmcs->tsc_offset = vmcs12->tsc_offset; 1860 * evmcs->guest_ia32_debugctl = vmcs12->guest_ia32_debugctl; 1861 * evmcs->cr0_guest_host_mask = vmcs12->cr0_guest_host_mask; 1862 * evmcs->cr4_guest_host_mask = vmcs12->cr4_guest_host_mask; 1863 * evmcs->cr0_read_shadow = vmcs12->cr0_read_shadow; 1864 * evmcs->cr4_read_shadow = vmcs12->cr4_read_shadow; 1865 * evmcs->vm_exit_msr_store_count = vmcs12->vm_exit_msr_store_count; 1866 * evmcs->vm_exit_msr_load_count = vmcs12->vm_exit_msr_load_count; 1867 * evmcs->vm_entry_msr_load_count = vmcs12->vm_entry_msr_load_count; 1868 * 1869 * Not present in struct vmcs12: 1870 * evmcs->exit_io_instruction_ecx = vmcs12->exit_io_instruction_ecx; 1871 * evmcs->exit_io_instruction_esi = vmcs12->exit_io_instruction_esi; 1872 * evmcs->exit_io_instruction_edi = vmcs12->exit_io_instruction_edi; 1873 * evmcs->exit_io_instruction_eip = vmcs12->exit_io_instruction_eip; 1874 */ 1875 1876 evmcs->guest_es_selector = vmcs12->guest_es_selector; 1877 evmcs->guest_cs_selector = vmcs12->guest_cs_selector; 1878 evmcs->guest_ss_selector = vmcs12->guest_ss_selector; 1879 evmcs->guest_ds_selector = vmcs12->guest_ds_selector; 1880 evmcs->guest_fs_selector = vmcs12->guest_fs_selector; 1881 evmcs->guest_gs_selector = vmcs12->guest_gs_selector; 1882 evmcs->guest_ldtr_selector = vmcs12->guest_ldtr_selector; 1883 evmcs->guest_tr_selector = vmcs12->guest_tr_selector; 1884 1885 evmcs->guest_es_limit = vmcs12->guest_es_limit; 1886 evmcs->guest_cs_limit = vmcs12->guest_cs_limit; 1887 evmcs->guest_ss_limit = vmcs12->guest_ss_limit; 1888 evmcs->guest_ds_limit = vmcs12->guest_ds_limit; 1889 evmcs->guest_fs_limit = vmcs12->guest_fs_limit; 1890 evmcs->guest_gs_limit = vmcs12->guest_gs_limit; 1891 evmcs->guest_ldtr_limit = vmcs12->guest_ldtr_limit; 1892 evmcs->guest_tr_limit = vmcs12->guest_tr_limit; 1893 evmcs->guest_gdtr_limit = vmcs12->guest_gdtr_limit; 1894 evmcs->guest_idtr_limit = vmcs12->guest_idtr_limit; 1895 1896 evmcs->guest_es_ar_bytes = vmcs12->guest_es_ar_bytes; 1897 evmcs->guest_cs_ar_bytes = vmcs12->guest_cs_ar_bytes; 1898 evmcs->guest_ss_ar_bytes = vmcs12->guest_ss_ar_bytes; 1899 evmcs->guest_ds_ar_bytes = vmcs12->guest_ds_ar_bytes; 1900 evmcs->guest_fs_ar_bytes = vmcs12->guest_fs_ar_bytes; 1901 evmcs->guest_gs_ar_bytes = vmcs12->guest_gs_ar_bytes; 1902 evmcs->guest_ldtr_ar_bytes = vmcs12->guest_ldtr_ar_bytes; 1903 evmcs->guest_tr_ar_bytes = vmcs12->guest_tr_ar_bytes; 1904 1905 evmcs->guest_es_base = vmcs12->guest_es_base; 1906 evmcs->guest_cs_base = vmcs12->guest_cs_base; 1907 evmcs->guest_ss_base = vmcs12->guest_ss_base; 1908 evmcs->guest_ds_base = vmcs12->guest_ds_base; 1909 evmcs->guest_fs_base = vmcs12->guest_fs_base; 1910 evmcs->guest_gs_base = vmcs12->guest_gs_base; 1911 evmcs->guest_ldtr_base = vmcs12->guest_ldtr_base; 1912 evmcs->guest_tr_base = vmcs12->guest_tr_base; 1913 evmcs->guest_gdtr_base = vmcs12->guest_gdtr_base; 1914 evmcs->guest_idtr_base = vmcs12->guest_idtr_base; 1915 1916 evmcs->guest_ia32_pat = vmcs12->guest_ia32_pat; 1917 evmcs->guest_ia32_efer = vmcs12->guest_ia32_efer; 1918 1919 evmcs->guest_pdptr0 = vmcs12->guest_pdptr0; 1920 evmcs->guest_pdptr1 = vmcs12->guest_pdptr1; 1921 evmcs->guest_pdptr2 = vmcs12->guest_pdptr2; 1922 evmcs->guest_pdptr3 = vmcs12->guest_pdptr3; 1923 1924 evmcs->guest_pending_dbg_exceptions = 1925 vmcs12->guest_pending_dbg_exceptions; 1926 evmcs->guest_sysenter_esp = vmcs12->guest_sysenter_esp; 1927 evmcs->guest_sysenter_eip = vmcs12->guest_sysenter_eip; 1928 1929 evmcs->guest_activity_state = vmcs12->guest_activity_state; 1930 evmcs->guest_sysenter_cs = vmcs12->guest_sysenter_cs; 1931 1932 evmcs->guest_cr0 = vmcs12->guest_cr0; 1933 evmcs->guest_cr3 = vmcs12->guest_cr3; 1934 evmcs->guest_cr4 = vmcs12->guest_cr4; 1935 evmcs->guest_dr7 = vmcs12->guest_dr7; 1936 1937 evmcs->guest_physical_address = vmcs12->guest_physical_address; 1938 1939 evmcs->vm_instruction_error = vmcs12->vm_instruction_error; 1940 evmcs->vm_exit_reason = vmcs12->vm_exit_reason; 1941 evmcs->vm_exit_intr_info = vmcs12->vm_exit_intr_info; 1942 evmcs->vm_exit_intr_error_code = vmcs12->vm_exit_intr_error_code; 1943 evmcs->idt_vectoring_info_field = vmcs12->idt_vectoring_info_field; 1944 evmcs->idt_vectoring_error_code = vmcs12->idt_vectoring_error_code; 1945 evmcs->vm_exit_instruction_len = vmcs12->vm_exit_instruction_len; 1946 evmcs->vmx_instruction_info = vmcs12->vmx_instruction_info; 1947 1948 evmcs->exit_qualification = vmcs12->exit_qualification; 1949 1950 evmcs->guest_linear_address = vmcs12->guest_linear_address; 1951 evmcs->guest_rsp = vmcs12->guest_rsp; 1952 evmcs->guest_rflags = vmcs12->guest_rflags; 1953 1954 evmcs->guest_interruptibility_info = 1955 vmcs12->guest_interruptibility_info; 1956 evmcs->cpu_based_vm_exec_control = vmcs12->cpu_based_vm_exec_control; 1957 evmcs->vm_entry_controls = vmcs12->vm_entry_controls; 1958 evmcs->vm_entry_intr_info_field = vmcs12->vm_entry_intr_info_field; 1959 evmcs->vm_entry_exception_error_code = 1960 vmcs12->vm_entry_exception_error_code; 1961 evmcs->vm_entry_instruction_len = vmcs12->vm_entry_instruction_len; 1962 1963 evmcs->guest_rip = vmcs12->guest_rip; 1964 1965 evmcs->guest_bndcfgs = vmcs12->guest_bndcfgs; 1966 1967 return; 1968 } 1969 1970 /* 1971 * This is an equivalent of the nested hypervisor executing the vmptrld 1972 * instruction. 1973 */ 1974 static enum nested_evmptrld_status nested_vmx_handle_enlightened_vmptrld( 1975 struct kvm_vcpu *vcpu, bool from_launch) 1976 { 1977 struct vcpu_vmx *vmx = to_vmx(vcpu); 1978 bool evmcs_gpa_changed = false; 1979 u64 evmcs_gpa; 1980 1981 if (likely(!vmx->nested.enlightened_vmcs_enabled)) 1982 return EVMPTRLD_DISABLED; 1983 1984 if (!nested_enlightened_vmentry(vcpu, &evmcs_gpa)) { 1985 nested_release_evmcs(vcpu); 1986 return EVMPTRLD_DISABLED; 1987 } 1988 1989 if (unlikely(evmcs_gpa != vmx->nested.hv_evmcs_vmptr)) { 1990 vmx->nested.current_vmptr = INVALID_GPA; 1991 1992 nested_release_evmcs(vcpu); 1993 1994 if (kvm_vcpu_map(vcpu, gpa_to_gfn(evmcs_gpa), 1995 &vmx->nested.hv_evmcs_map)) 1996 return EVMPTRLD_ERROR; 1997 1998 vmx->nested.hv_evmcs = vmx->nested.hv_evmcs_map.hva; 1999 2000 /* 2001 * Currently, KVM only supports eVMCS version 1 2002 * (== KVM_EVMCS_VERSION) and thus we expect guest to set this 2003 * value to first u32 field of eVMCS which should specify eVMCS 2004 * VersionNumber. 2005 * 2006 * Guest should be aware of supported eVMCS versions by host by 2007 * examining CPUID.0x4000000A.EAX[0:15]. Host userspace VMM is 2008 * expected to set this CPUID leaf according to the value 2009 * returned in vmcs_version from nested_enable_evmcs(). 2010 * 2011 * However, it turns out that Microsoft Hyper-V fails to comply 2012 * to their own invented interface: When Hyper-V use eVMCS, it 2013 * just sets first u32 field of eVMCS to revision_id specified 2014 * in MSR_IA32_VMX_BASIC. Instead of used eVMCS version number 2015 * which is one of the supported versions specified in 2016 * CPUID.0x4000000A.EAX[0:15]. 2017 * 2018 * To overcome Hyper-V bug, we accept here either a supported 2019 * eVMCS version or VMCS12 revision_id as valid values for first 2020 * u32 field of eVMCS. 2021 */ 2022 if ((vmx->nested.hv_evmcs->revision_id != KVM_EVMCS_VERSION) && 2023 (vmx->nested.hv_evmcs->revision_id != VMCS12_REVISION)) { 2024 nested_release_evmcs(vcpu); 2025 return EVMPTRLD_VMFAIL; 2026 } 2027 2028 vmx->nested.hv_evmcs_vmptr = evmcs_gpa; 2029 2030 evmcs_gpa_changed = true; 2031 /* 2032 * Unlike normal vmcs12, enlightened vmcs12 is not fully 2033 * reloaded from guest's memory (read only fields, fields not 2034 * present in struct hv_enlightened_vmcs, ...). Make sure there 2035 * are no leftovers. 2036 */ 2037 if (from_launch) { 2038 struct vmcs12 *vmcs12 = get_vmcs12(vcpu); 2039 memset(vmcs12, 0, sizeof(*vmcs12)); 2040 vmcs12->hdr.revision_id = VMCS12_REVISION; 2041 } 2042 2043 } 2044 2045 /* 2046 * Clean fields data can't be used on VMLAUNCH and when we switch 2047 * between different L2 guests as KVM keeps a single VMCS12 per L1. 2048 */ 2049 if (from_launch || evmcs_gpa_changed) { 2050 vmx->nested.hv_evmcs->hv_clean_fields &= 2051 ~HV_VMX_ENLIGHTENED_CLEAN_FIELD_ALL; 2052 2053 vmx->nested.force_msr_bitmap_recalc = true; 2054 } 2055 2056 return EVMPTRLD_SUCCEEDED; 2057 } 2058 2059 void nested_sync_vmcs12_to_shadow(struct kvm_vcpu *vcpu) 2060 { 2061 struct vcpu_vmx *vmx = to_vmx(vcpu); 2062 2063 if (evmptr_is_valid(vmx->nested.hv_evmcs_vmptr)) 2064 copy_vmcs12_to_enlightened(vmx); 2065 else 2066 copy_vmcs12_to_shadow(vmx); 2067 2068 vmx->nested.need_vmcs12_to_shadow_sync = false; 2069 } 2070 2071 static enum hrtimer_restart vmx_preemption_timer_fn(struct hrtimer *timer) 2072 { 2073 struct vcpu_vmx *vmx = 2074 container_of(timer, struct vcpu_vmx, nested.preemption_timer); 2075 2076 vmx->nested.preemption_timer_expired = true; 2077 kvm_make_request(KVM_REQ_EVENT, &vmx->vcpu); 2078 kvm_vcpu_kick(&vmx->vcpu); 2079 2080 return HRTIMER_NORESTART; 2081 } 2082 2083 static u64 vmx_calc_preemption_timer_value(struct kvm_vcpu *vcpu) 2084 { 2085 struct vcpu_vmx *vmx = to_vmx(vcpu); 2086 struct vmcs12 *vmcs12 = get_vmcs12(vcpu); 2087 2088 u64 l1_scaled_tsc = kvm_read_l1_tsc(vcpu, rdtsc()) >> 2089 VMX_MISC_EMULATED_PREEMPTION_TIMER_RATE; 2090 2091 if (!vmx->nested.has_preemption_timer_deadline) { 2092 vmx->nested.preemption_timer_deadline = 2093 vmcs12->vmx_preemption_timer_value + l1_scaled_tsc; 2094 vmx->nested.has_preemption_timer_deadline = true; 2095 } 2096 return vmx->nested.preemption_timer_deadline - l1_scaled_tsc; 2097 } 2098 2099 static void vmx_start_preemption_timer(struct kvm_vcpu *vcpu, 2100 u64 preemption_timeout) 2101 { 2102 struct vcpu_vmx *vmx = to_vmx(vcpu); 2103 2104 /* 2105 * A timer value of zero is architecturally guaranteed to cause 2106 * a VMExit prior to executing any instructions in the guest. 2107 */ 2108 if (preemption_timeout == 0) { 2109 vmx_preemption_timer_fn(&vmx->nested.preemption_timer); 2110 return; 2111 } 2112 2113 if (vcpu->arch.virtual_tsc_khz == 0) 2114 return; 2115 2116 preemption_timeout <<= VMX_MISC_EMULATED_PREEMPTION_TIMER_RATE; 2117 preemption_timeout *= 1000000; 2118 do_div(preemption_timeout, vcpu->arch.virtual_tsc_khz); 2119 hrtimer_start(&vmx->nested.preemption_timer, 2120 ktime_add_ns(ktime_get(), preemption_timeout), 2121 HRTIMER_MODE_ABS_PINNED); 2122 } 2123 2124 static u64 nested_vmx_calc_efer(struct vcpu_vmx *vmx, struct vmcs12 *vmcs12) 2125 { 2126 if (vmx->nested.nested_run_pending && 2127 (vmcs12->vm_entry_controls & VM_ENTRY_LOAD_IA32_EFER)) 2128 return vmcs12->guest_ia32_efer; 2129 else if (vmcs12->vm_entry_controls & VM_ENTRY_IA32E_MODE) 2130 return vmx->vcpu.arch.efer | (EFER_LMA | EFER_LME); 2131 else 2132 return vmx->vcpu.arch.efer & ~(EFER_LMA | EFER_LME); 2133 } 2134 2135 static void prepare_vmcs02_constant_state(struct vcpu_vmx *vmx) 2136 { 2137 /* 2138 * If vmcs02 hasn't been initialized, set the constant vmcs02 state 2139 * according to L0's settings (vmcs12 is irrelevant here). Host 2140 * fields that come from L0 and are not constant, e.g. HOST_CR3, 2141 * will be set as needed prior to VMLAUNCH/VMRESUME. 2142 */ 2143 if (vmx->nested.vmcs02_initialized) 2144 return; 2145 vmx->nested.vmcs02_initialized = true; 2146 2147 /* 2148 * We don't care what the EPTP value is we just need to guarantee 2149 * it's valid so we don't get a false positive when doing early 2150 * consistency checks. 2151 */ 2152 if (enable_ept && nested_early_check) 2153 vmcs_write64(EPT_POINTER, 2154 construct_eptp(&vmx->vcpu, 0, PT64_ROOT_4LEVEL)); 2155 2156 /* All VMFUNCs are currently emulated through L0 vmexits. */ 2157 if (cpu_has_vmx_vmfunc()) 2158 vmcs_write64(VM_FUNCTION_CONTROL, 0); 2159 2160 if (cpu_has_vmx_posted_intr()) 2161 vmcs_write16(POSTED_INTR_NV, POSTED_INTR_NESTED_VECTOR); 2162 2163 if (cpu_has_vmx_msr_bitmap()) 2164 vmcs_write64(MSR_BITMAP, __pa(vmx->nested.vmcs02.msr_bitmap)); 2165 2166 /* 2167 * PML is emulated for L2, but never enabled in hardware as the MMU 2168 * handles A/D emulation. Disabling PML for L2 also avoids having to 2169 * deal with filtering out L2 GPAs from the buffer. 2170 */ 2171 if (enable_pml) { 2172 vmcs_write64(PML_ADDRESS, 0); 2173 vmcs_write16(GUEST_PML_INDEX, -1); 2174 } 2175 2176 if (cpu_has_vmx_encls_vmexit()) 2177 vmcs_write64(ENCLS_EXITING_BITMAP, INVALID_GPA); 2178 2179 /* 2180 * Set the MSR load/store lists to match L0's settings. Only the 2181 * addresses are constant (for vmcs02), the counts can change based 2182 * on L2's behavior, e.g. switching to/from long mode. 2183 */ 2184 vmcs_write64(VM_EXIT_MSR_STORE_ADDR, __pa(vmx->msr_autostore.guest.val)); 2185 vmcs_write64(VM_EXIT_MSR_LOAD_ADDR, __pa(vmx->msr_autoload.host.val)); 2186 vmcs_write64(VM_ENTRY_MSR_LOAD_ADDR, __pa(vmx->msr_autoload.guest.val)); 2187 2188 vmx_set_constant_host_state(vmx); 2189 } 2190 2191 static void prepare_vmcs02_early_rare(struct vcpu_vmx *vmx, 2192 struct vmcs12 *vmcs12) 2193 { 2194 prepare_vmcs02_constant_state(vmx); 2195 2196 vmcs_write64(VMCS_LINK_POINTER, INVALID_GPA); 2197 2198 if (enable_vpid) { 2199 if (nested_cpu_has_vpid(vmcs12) && vmx->nested.vpid02) 2200 vmcs_write16(VIRTUAL_PROCESSOR_ID, vmx->nested.vpid02); 2201 else 2202 vmcs_write16(VIRTUAL_PROCESSOR_ID, vmx->vpid); 2203 } 2204 } 2205 2206 static void prepare_vmcs02_early(struct vcpu_vmx *vmx, struct loaded_vmcs *vmcs01, 2207 struct vmcs12 *vmcs12) 2208 { 2209 u32 exec_control; 2210 u64 guest_efer = nested_vmx_calc_efer(vmx, vmcs12); 2211 2212 if (vmx->nested.dirty_vmcs12 || evmptr_is_valid(vmx->nested.hv_evmcs_vmptr)) 2213 prepare_vmcs02_early_rare(vmx, vmcs12); 2214 2215 /* 2216 * PIN CONTROLS 2217 */ 2218 exec_control = __pin_controls_get(vmcs01); 2219 exec_control |= (vmcs12->pin_based_vm_exec_control & 2220 ~PIN_BASED_VMX_PREEMPTION_TIMER); 2221 2222 /* Posted interrupts setting is only taken from vmcs12. */ 2223 vmx->nested.pi_pending = false; 2224 if (nested_cpu_has_posted_intr(vmcs12)) 2225 vmx->nested.posted_intr_nv = vmcs12->posted_intr_nv; 2226 else 2227 exec_control &= ~PIN_BASED_POSTED_INTR; 2228 pin_controls_set(vmx, exec_control); 2229 2230 /* 2231 * EXEC CONTROLS 2232 */ 2233 exec_control = __exec_controls_get(vmcs01); /* L0's desires */ 2234 exec_control &= ~CPU_BASED_INTR_WINDOW_EXITING; 2235 exec_control &= ~CPU_BASED_NMI_WINDOW_EXITING; 2236 exec_control &= ~CPU_BASED_TPR_SHADOW; 2237 exec_control |= vmcs12->cpu_based_vm_exec_control; 2238 2239 vmx->nested.l1_tpr_threshold = -1; 2240 if (exec_control & CPU_BASED_TPR_SHADOW) 2241 vmcs_write32(TPR_THRESHOLD, vmcs12->tpr_threshold); 2242 #ifdef CONFIG_X86_64 2243 else 2244 exec_control |= CPU_BASED_CR8_LOAD_EXITING | 2245 CPU_BASED_CR8_STORE_EXITING; 2246 #endif 2247 2248 /* 2249 * A vmexit (to either L1 hypervisor or L0 userspace) is always needed 2250 * for I/O port accesses. 2251 */ 2252 exec_control |= CPU_BASED_UNCOND_IO_EXITING; 2253 exec_control &= ~CPU_BASED_USE_IO_BITMAPS; 2254 2255 /* 2256 * This bit will be computed in nested_get_vmcs12_pages, because 2257 * we do not have access to L1's MSR bitmap yet. For now, keep 2258 * the same bit as before, hoping to avoid multiple VMWRITEs that 2259 * only set/clear this bit. 2260 */ 2261 exec_control &= ~CPU_BASED_USE_MSR_BITMAPS; 2262 exec_control |= exec_controls_get(vmx) & CPU_BASED_USE_MSR_BITMAPS; 2263 2264 exec_controls_set(vmx, exec_control); 2265 2266 /* 2267 * SECONDARY EXEC CONTROLS 2268 */ 2269 if (cpu_has_secondary_exec_ctrls()) { 2270 exec_control = __secondary_exec_controls_get(vmcs01); 2271 2272 /* Take the following fields only from vmcs12 */ 2273 exec_control &= ~(SECONDARY_EXEC_VIRTUALIZE_APIC_ACCESSES | 2274 SECONDARY_EXEC_VIRTUALIZE_X2APIC_MODE | 2275 SECONDARY_EXEC_ENABLE_INVPCID | 2276 SECONDARY_EXEC_ENABLE_RDTSCP | 2277 SECONDARY_EXEC_XSAVES | 2278 SECONDARY_EXEC_ENABLE_USR_WAIT_PAUSE | 2279 SECONDARY_EXEC_VIRTUAL_INTR_DELIVERY | 2280 SECONDARY_EXEC_APIC_REGISTER_VIRT | 2281 SECONDARY_EXEC_ENABLE_VMFUNC | 2282 SECONDARY_EXEC_TSC_SCALING | 2283 SECONDARY_EXEC_DESC); 2284 2285 if (nested_cpu_has(vmcs12, 2286 CPU_BASED_ACTIVATE_SECONDARY_CONTROLS)) 2287 exec_control |= vmcs12->secondary_vm_exec_control; 2288 2289 /* PML is emulated and never enabled in hardware for L2. */ 2290 exec_control &= ~SECONDARY_EXEC_ENABLE_PML; 2291 2292 /* VMCS shadowing for L2 is emulated for now */ 2293 exec_control &= ~SECONDARY_EXEC_SHADOW_VMCS; 2294 2295 /* 2296 * Preset *DT exiting when emulating UMIP, so that vmx_set_cr4() 2297 * will not have to rewrite the controls just for this bit. 2298 */ 2299 if (!boot_cpu_has(X86_FEATURE_UMIP) && vmx_umip_emulated() && 2300 (vmcs12->guest_cr4 & X86_CR4_UMIP)) 2301 exec_control |= SECONDARY_EXEC_DESC; 2302 2303 if (exec_control & SECONDARY_EXEC_VIRTUAL_INTR_DELIVERY) 2304 vmcs_write16(GUEST_INTR_STATUS, 2305 vmcs12->guest_intr_status); 2306 2307 if (!nested_cpu_has2(vmcs12, SECONDARY_EXEC_UNRESTRICTED_GUEST)) 2308 exec_control &= ~SECONDARY_EXEC_UNRESTRICTED_GUEST; 2309 2310 if (exec_control & SECONDARY_EXEC_ENCLS_EXITING) 2311 vmx_write_encls_bitmap(&vmx->vcpu, vmcs12); 2312 2313 secondary_exec_controls_set(vmx, exec_control); 2314 } 2315 2316 /* 2317 * ENTRY CONTROLS 2318 * 2319 * vmcs12's VM_{ENTRY,EXIT}_LOAD_IA32_EFER and VM_ENTRY_IA32E_MODE 2320 * are emulated by vmx_set_efer() in prepare_vmcs02(), but speculate 2321 * on the related bits (if supported by the CPU) in the hope that 2322 * we can avoid VMWrites during vmx_set_efer(). 2323 */ 2324 exec_control = __vm_entry_controls_get(vmcs01); 2325 exec_control |= vmcs12->vm_entry_controls; 2326 exec_control &= ~(VM_ENTRY_IA32E_MODE | VM_ENTRY_LOAD_IA32_EFER); 2327 if (cpu_has_load_ia32_efer()) { 2328 if (guest_efer & EFER_LMA) 2329 exec_control |= VM_ENTRY_IA32E_MODE; 2330 if (guest_efer != host_efer) 2331 exec_control |= VM_ENTRY_LOAD_IA32_EFER; 2332 } 2333 vm_entry_controls_set(vmx, exec_control); 2334 2335 /* 2336 * EXIT CONTROLS 2337 * 2338 * L2->L1 exit controls are emulated - the hardware exit is to L0 so 2339 * we should use its exit controls. Note that VM_EXIT_LOAD_IA32_EFER 2340 * bits may be modified by vmx_set_efer() in prepare_vmcs02(). 2341 */ 2342 exec_control = __vm_exit_controls_get(vmcs01); 2343 if (cpu_has_load_ia32_efer() && guest_efer != host_efer) 2344 exec_control |= VM_EXIT_LOAD_IA32_EFER; 2345 else 2346 exec_control &= ~VM_EXIT_LOAD_IA32_EFER; 2347 vm_exit_controls_set(vmx, exec_control); 2348 2349 /* 2350 * Interrupt/Exception Fields 2351 */ 2352 if (vmx->nested.nested_run_pending) { 2353 vmcs_write32(VM_ENTRY_INTR_INFO_FIELD, 2354 vmcs12->vm_entry_intr_info_field); 2355 vmcs_write32(VM_ENTRY_EXCEPTION_ERROR_CODE, 2356 vmcs12->vm_entry_exception_error_code); 2357 vmcs_write32(VM_ENTRY_INSTRUCTION_LEN, 2358 vmcs12->vm_entry_instruction_len); 2359 vmcs_write32(GUEST_INTERRUPTIBILITY_INFO, 2360 vmcs12->guest_interruptibility_info); 2361 vmx->loaded_vmcs->nmi_known_unmasked = 2362 !(vmcs12->guest_interruptibility_info & GUEST_INTR_STATE_NMI); 2363 } else { 2364 vmcs_write32(VM_ENTRY_INTR_INFO_FIELD, 0); 2365 } 2366 } 2367 2368 static void prepare_vmcs02_rare(struct vcpu_vmx *vmx, struct vmcs12 *vmcs12) 2369 { 2370 struct hv_enlightened_vmcs *hv_evmcs = vmx->nested.hv_evmcs; 2371 2372 if (!hv_evmcs || !(hv_evmcs->hv_clean_fields & 2373 HV_VMX_ENLIGHTENED_CLEAN_FIELD_GUEST_GRP2)) { 2374 vmcs_write16(GUEST_ES_SELECTOR, vmcs12->guest_es_selector); 2375 vmcs_write16(GUEST_CS_SELECTOR, vmcs12->guest_cs_selector); 2376 vmcs_write16(GUEST_SS_SELECTOR, vmcs12->guest_ss_selector); 2377 vmcs_write16(GUEST_DS_SELECTOR, vmcs12->guest_ds_selector); 2378 vmcs_write16(GUEST_FS_SELECTOR, vmcs12->guest_fs_selector); 2379 vmcs_write16(GUEST_GS_SELECTOR, vmcs12->guest_gs_selector); 2380 vmcs_write16(GUEST_LDTR_SELECTOR, vmcs12->guest_ldtr_selector); 2381 vmcs_write16(GUEST_TR_SELECTOR, vmcs12->guest_tr_selector); 2382 vmcs_write32(GUEST_ES_LIMIT, vmcs12->guest_es_limit); 2383 vmcs_write32(GUEST_CS_LIMIT, vmcs12->guest_cs_limit); 2384 vmcs_write32(GUEST_SS_LIMIT, vmcs12->guest_ss_limit); 2385 vmcs_write32(GUEST_DS_LIMIT, vmcs12->guest_ds_limit); 2386 vmcs_write32(GUEST_FS_LIMIT, vmcs12->guest_fs_limit); 2387 vmcs_write32(GUEST_GS_LIMIT, vmcs12->guest_gs_limit); 2388 vmcs_write32(GUEST_LDTR_LIMIT, vmcs12->guest_ldtr_limit); 2389 vmcs_write32(GUEST_TR_LIMIT, vmcs12->guest_tr_limit); 2390 vmcs_write32(GUEST_GDTR_LIMIT, vmcs12->guest_gdtr_limit); 2391 vmcs_write32(GUEST_IDTR_LIMIT, vmcs12->guest_idtr_limit); 2392 vmcs_write32(GUEST_CS_AR_BYTES, vmcs12->guest_cs_ar_bytes); 2393 vmcs_write32(GUEST_SS_AR_BYTES, vmcs12->guest_ss_ar_bytes); 2394 vmcs_write32(GUEST_ES_AR_BYTES, vmcs12->guest_es_ar_bytes); 2395 vmcs_write32(GUEST_DS_AR_BYTES, vmcs12->guest_ds_ar_bytes); 2396 vmcs_write32(GUEST_FS_AR_BYTES, vmcs12->guest_fs_ar_bytes); 2397 vmcs_write32(GUEST_GS_AR_BYTES, vmcs12->guest_gs_ar_bytes); 2398 vmcs_write32(GUEST_LDTR_AR_BYTES, vmcs12->guest_ldtr_ar_bytes); 2399 vmcs_write32(GUEST_TR_AR_BYTES, vmcs12->guest_tr_ar_bytes); 2400 vmcs_writel(GUEST_ES_BASE, vmcs12->guest_es_base); 2401 vmcs_writel(GUEST_CS_BASE, vmcs12->guest_cs_base); 2402 vmcs_writel(GUEST_SS_BASE, vmcs12->guest_ss_base); 2403 vmcs_writel(GUEST_DS_BASE, vmcs12->guest_ds_base); 2404 vmcs_writel(GUEST_FS_BASE, vmcs12->guest_fs_base); 2405 vmcs_writel(GUEST_GS_BASE, vmcs12->guest_gs_base); 2406 vmcs_writel(GUEST_LDTR_BASE, vmcs12->guest_ldtr_base); 2407 vmcs_writel(GUEST_TR_BASE, vmcs12->guest_tr_base); 2408 vmcs_writel(GUEST_GDTR_BASE, vmcs12->guest_gdtr_base); 2409 vmcs_writel(GUEST_IDTR_BASE, vmcs12->guest_idtr_base); 2410 2411 vmx->segment_cache.bitmask = 0; 2412 } 2413 2414 if (!hv_evmcs || !(hv_evmcs->hv_clean_fields & 2415 HV_VMX_ENLIGHTENED_CLEAN_FIELD_GUEST_GRP1)) { 2416 vmcs_write32(GUEST_SYSENTER_CS, vmcs12->guest_sysenter_cs); 2417 vmcs_writel(GUEST_PENDING_DBG_EXCEPTIONS, 2418 vmcs12->guest_pending_dbg_exceptions); 2419 vmcs_writel(GUEST_SYSENTER_ESP, vmcs12->guest_sysenter_esp); 2420 vmcs_writel(GUEST_SYSENTER_EIP, vmcs12->guest_sysenter_eip); 2421 2422 /* 2423 * L1 may access the L2's PDPTR, so save them to construct 2424 * vmcs12 2425 */ 2426 if (enable_ept) { 2427 vmcs_write64(GUEST_PDPTR0, vmcs12->guest_pdptr0); 2428 vmcs_write64(GUEST_PDPTR1, vmcs12->guest_pdptr1); 2429 vmcs_write64(GUEST_PDPTR2, vmcs12->guest_pdptr2); 2430 vmcs_write64(GUEST_PDPTR3, vmcs12->guest_pdptr3); 2431 } 2432 2433 if (kvm_mpx_supported() && vmx->nested.nested_run_pending && 2434 (vmcs12->vm_entry_controls & VM_ENTRY_LOAD_BNDCFGS)) 2435 vmcs_write64(GUEST_BNDCFGS, vmcs12->guest_bndcfgs); 2436 } 2437 2438 if (nested_cpu_has_xsaves(vmcs12)) 2439 vmcs_write64(XSS_EXIT_BITMAP, vmcs12->xss_exit_bitmap); 2440 2441 /* 2442 * Whether page-faults are trapped is determined by a combination of 2443 * 3 settings: PFEC_MASK, PFEC_MATCH and EXCEPTION_BITMAP.PF. If L0 2444 * doesn't care about page faults then we should set all of these to 2445 * L1's desires. However, if L0 does care about (some) page faults, it 2446 * is not easy (if at all possible?) to merge L0 and L1's desires, we 2447 * simply ask to exit on each and every L2 page fault. This is done by 2448 * setting MASK=MATCH=0 and (see below) EB.PF=1. 2449 * Note that below we don't need special code to set EB.PF beyond the 2450 * "or"ing of the EB of vmcs01 and vmcs12, because when enable_ept, 2451 * vmcs01's EB.PF is 0 so the "or" will take vmcs12's value, and when 2452 * !enable_ept, EB.PF is 1, so the "or" will always be 1. 2453 */ 2454 if (vmx_need_pf_intercept(&vmx->vcpu)) { 2455 /* 2456 * TODO: if both L0 and L1 need the same MASK and MATCH, 2457 * go ahead and use it? 2458 */ 2459 vmcs_write32(PAGE_FAULT_ERROR_CODE_MASK, 0); 2460 vmcs_write32(PAGE_FAULT_ERROR_CODE_MATCH, 0); 2461 } else { 2462 vmcs_write32(PAGE_FAULT_ERROR_CODE_MASK, vmcs12->page_fault_error_code_mask); 2463 vmcs_write32(PAGE_FAULT_ERROR_CODE_MATCH, vmcs12->page_fault_error_code_match); 2464 } 2465 2466 if (cpu_has_vmx_apicv()) { 2467 vmcs_write64(EOI_EXIT_BITMAP0, vmcs12->eoi_exit_bitmap0); 2468 vmcs_write64(EOI_EXIT_BITMAP1, vmcs12->eoi_exit_bitmap1); 2469 vmcs_write64(EOI_EXIT_BITMAP2, vmcs12->eoi_exit_bitmap2); 2470 vmcs_write64(EOI_EXIT_BITMAP3, vmcs12->eoi_exit_bitmap3); 2471 } 2472 2473 /* 2474 * Make sure the msr_autostore list is up to date before we set the 2475 * count in the vmcs02. 2476 */ 2477 prepare_vmx_msr_autostore_list(&vmx->vcpu, MSR_IA32_TSC); 2478 2479 vmcs_write32(VM_EXIT_MSR_STORE_COUNT, vmx->msr_autostore.guest.nr); 2480 vmcs_write32(VM_EXIT_MSR_LOAD_COUNT, vmx->msr_autoload.host.nr); 2481 vmcs_write32(VM_ENTRY_MSR_LOAD_COUNT, vmx->msr_autoload.guest.nr); 2482 2483 set_cr4_guest_host_mask(vmx); 2484 } 2485 2486 /* 2487 * prepare_vmcs02 is called when the L1 guest hypervisor runs its nested 2488 * L2 guest. L1 has a vmcs for L2 (vmcs12), and this function "merges" it 2489 * with L0's requirements for its guest (a.k.a. vmcs01), so we can run the L2 2490 * guest in a way that will both be appropriate to L1's requests, and our 2491 * needs. In addition to modifying the active vmcs (which is vmcs02), this 2492 * function also has additional necessary side-effects, like setting various 2493 * vcpu->arch fields. 2494 * Returns 0 on success, 1 on failure. Invalid state exit qualification code 2495 * is assigned to entry_failure_code on failure. 2496 */ 2497 static int prepare_vmcs02(struct kvm_vcpu *vcpu, struct vmcs12 *vmcs12, 2498 bool from_vmentry, 2499 enum vm_entry_failure_code *entry_failure_code) 2500 { 2501 struct vcpu_vmx *vmx = to_vmx(vcpu); 2502 bool load_guest_pdptrs_vmcs12 = false; 2503 2504 if (vmx->nested.dirty_vmcs12 || evmptr_is_valid(vmx->nested.hv_evmcs_vmptr)) { 2505 prepare_vmcs02_rare(vmx, vmcs12); 2506 vmx->nested.dirty_vmcs12 = false; 2507 2508 load_guest_pdptrs_vmcs12 = !evmptr_is_valid(vmx->nested.hv_evmcs_vmptr) || 2509 !(vmx->nested.hv_evmcs->hv_clean_fields & 2510 HV_VMX_ENLIGHTENED_CLEAN_FIELD_GUEST_GRP1); 2511 } 2512 2513 if (vmx->nested.nested_run_pending && 2514 (vmcs12->vm_entry_controls & VM_ENTRY_LOAD_DEBUG_CONTROLS)) { 2515 kvm_set_dr(vcpu, 7, vmcs12->guest_dr7); 2516 vmcs_write64(GUEST_IA32_DEBUGCTL, vmcs12->guest_ia32_debugctl); 2517 } else { 2518 kvm_set_dr(vcpu, 7, vcpu->arch.dr7); 2519 vmcs_write64(GUEST_IA32_DEBUGCTL, vmx->nested.vmcs01_debugctl); 2520 } 2521 if (kvm_mpx_supported() && (!vmx->nested.nested_run_pending || 2522 !(vmcs12->vm_entry_controls & VM_ENTRY_LOAD_BNDCFGS))) 2523 vmcs_write64(GUEST_BNDCFGS, vmx->nested.vmcs01_guest_bndcfgs); 2524 vmx_set_rflags(vcpu, vmcs12->guest_rflags); 2525 2526 /* EXCEPTION_BITMAP and CR0_GUEST_HOST_MASK should basically be the 2527 * bitwise-or of what L1 wants to trap for L2, and what we want to 2528 * trap. Note that CR0.TS also needs updating - we do this later. 2529 */ 2530 vmx_update_exception_bitmap(vcpu); 2531 vcpu->arch.cr0_guest_owned_bits &= ~vmcs12->cr0_guest_host_mask; 2532 vmcs_writel(CR0_GUEST_HOST_MASK, ~vcpu->arch.cr0_guest_owned_bits); 2533 2534 if (vmx->nested.nested_run_pending && 2535 (vmcs12->vm_entry_controls & VM_ENTRY_LOAD_IA32_PAT)) { 2536 vmcs_write64(GUEST_IA32_PAT, vmcs12->guest_ia32_pat); 2537 vcpu->arch.pat = vmcs12->guest_ia32_pat; 2538 } else if (vmcs_config.vmentry_ctrl & VM_ENTRY_LOAD_IA32_PAT) { 2539 vmcs_write64(GUEST_IA32_PAT, vmx->vcpu.arch.pat); 2540 } 2541 2542 vcpu->arch.tsc_offset = kvm_calc_nested_tsc_offset( 2543 vcpu->arch.l1_tsc_offset, 2544 vmx_get_l2_tsc_offset(vcpu), 2545 vmx_get_l2_tsc_multiplier(vcpu)); 2546 2547 vcpu->arch.tsc_scaling_ratio = kvm_calc_nested_tsc_multiplier( 2548 vcpu->arch.l1_tsc_scaling_ratio, 2549 vmx_get_l2_tsc_multiplier(vcpu)); 2550 2551 vmcs_write64(TSC_OFFSET, vcpu->arch.tsc_offset); 2552 if (kvm_has_tsc_control) 2553 vmcs_write64(TSC_MULTIPLIER, vcpu->arch.tsc_scaling_ratio); 2554 2555 nested_vmx_transition_tlb_flush(vcpu, vmcs12, true); 2556 2557 if (nested_cpu_has_ept(vmcs12)) 2558 nested_ept_init_mmu_context(vcpu); 2559 2560 /* 2561 * This sets GUEST_CR0 to vmcs12->guest_cr0, possibly modifying those 2562 * bits which we consider mandatory enabled. 2563 * The CR0_READ_SHADOW is what L2 should have expected to read given 2564 * the specifications by L1; It's not enough to take 2565 * vmcs12->cr0_read_shadow because on our cr0_guest_host_mask we we 2566 * have more bits than L1 expected. 2567 */ 2568 vmx_set_cr0(vcpu, vmcs12->guest_cr0); 2569 vmcs_writel(CR0_READ_SHADOW, nested_read_cr0(vmcs12)); 2570 2571 vmx_set_cr4(vcpu, vmcs12->guest_cr4); 2572 vmcs_writel(CR4_READ_SHADOW, nested_read_cr4(vmcs12)); 2573 2574 vcpu->arch.efer = nested_vmx_calc_efer(vmx, vmcs12); 2575 /* Note: may modify VM_ENTRY/EXIT_CONTROLS and GUEST/HOST_IA32_EFER */ 2576 vmx_set_efer(vcpu, vcpu->arch.efer); 2577 2578 /* 2579 * Guest state is invalid and unrestricted guest is disabled, 2580 * which means L1 attempted VMEntry to L2 with invalid state. 2581 * Fail the VMEntry. 2582 * 2583 * However when force loading the guest state (SMM exit or 2584 * loading nested state after migration, it is possible to 2585 * have invalid guest state now, which will be later fixed by 2586 * restoring L2 register state 2587 */ 2588 if (CC(from_vmentry && !vmx_guest_state_valid(vcpu))) { 2589 *entry_failure_code = ENTRY_FAIL_DEFAULT; 2590 return -EINVAL; 2591 } 2592 2593 /* Shadow page tables on either EPT or shadow page tables. */ 2594 if (nested_vmx_load_cr3(vcpu, vmcs12->guest_cr3, nested_cpu_has_ept(vmcs12), 2595 from_vmentry, entry_failure_code)) 2596 return -EINVAL; 2597 2598 /* 2599 * Immediately write vmcs02.GUEST_CR3. It will be propagated to vmcs12 2600 * on nested VM-Exit, which can occur without actually running L2 and 2601 * thus without hitting vmx_load_mmu_pgd(), e.g. if L1 is entering L2 with 2602 * vmcs12.GUEST_ACTIVITYSTATE=HLT, in which case KVM will intercept the 2603 * transition to HLT instead of running L2. 2604 */ 2605 if (enable_ept) 2606 vmcs_writel(GUEST_CR3, vmcs12->guest_cr3); 2607 2608 /* Late preparation of GUEST_PDPTRs now that EFER and CRs are set. */ 2609 if (load_guest_pdptrs_vmcs12 && nested_cpu_has_ept(vmcs12) && 2610 is_pae_paging(vcpu)) { 2611 vmcs_write64(GUEST_PDPTR0, vmcs12->guest_pdptr0); 2612 vmcs_write64(GUEST_PDPTR1, vmcs12->guest_pdptr1); 2613 vmcs_write64(GUEST_PDPTR2, vmcs12->guest_pdptr2); 2614 vmcs_write64(GUEST_PDPTR3, vmcs12->guest_pdptr3); 2615 } 2616 2617 if (!enable_ept) 2618 vcpu->arch.walk_mmu->inject_page_fault = vmx_inject_page_fault_nested; 2619 2620 if ((vmcs12->vm_entry_controls & VM_ENTRY_LOAD_IA32_PERF_GLOBAL_CTRL) && 2621 WARN_ON_ONCE(kvm_set_msr(vcpu, MSR_CORE_PERF_GLOBAL_CTRL, 2622 vmcs12->guest_ia32_perf_global_ctrl))) { 2623 *entry_failure_code = ENTRY_FAIL_DEFAULT; 2624 return -EINVAL; 2625 } 2626 2627 kvm_rsp_write(vcpu, vmcs12->guest_rsp); 2628 kvm_rip_write(vcpu, vmcs12->guest_rip); 2629 2630 /* 2631 * It was observed that genuine Hyper-V running in L1 doesn't reset 2632 * 'hv_clean_fields' by itself, it only sets the corresponding dirty 2633 * bits when it changes a field in eVMCS. Mark all fields as clean 2634 * here. 2635 */ 2636 if (evmptr_is_valid(vmx->nested.hv_evmcs_vmptr)) 2637 vmx->nested.hv_evmcs->hv_clean_fields |= 2638 HV_VMX_ENLIGHTENED_CLEAN_FIELD_ALL; 2639 2640 return 0; 2641 } 2642 2643 static int nested_vmx_check_nmi_controls(struct vmcs12 *vmcs12) 2644 { 2645 if (CC(!nested_cpu_has_nmi_exiting(vmcs12) && 2646 nested_cpu_has_virtual_nmis(vmcs12))) 2647 return -EINVAL; 2648 2649 if (CC(!nested_cpu_has_virtual_nmis(vmcs12) && 2650 nested_cpu_has(vmcs12, CPU_BASED_NMI_WINDOW_EXITING))) 2651 return -EINVAL; 2652 2653 return 0; 2654 } 2655 2656 static bool nested_vmx_check_eptp(struct kvm_vcpu *vcpu, u64 new_eptp) 2657 { 2658 struct vcpu_vmx *vmx = to_vmx(vcpu); 2659 2660 /* Check for memory type validity */ 2661 switch (new_eptp & VMX_EPTP_MT_MASK) { 2662 case VMX_EPTP_MT_UC: 2663 if (CC(!(vmx->nested.msrs.ept_caps & VMX_EPTP_UC_BIT))) 2664 return false; 2665 break; 2666 case VMX_EPTP_MT_WB: 2667 if (CC(!(vmx->nested.msrs.ept_caps & VMX_EPTP_WB_BIT))) 2668 return false; 2669 break; 2670 default: 2671 return false; 2672 } 2673 2674 /* Page-walk levels validity. */ 2675 switch (new_eptp & VMX_EPTP_PWL_MASK) { 2676 case VMX_EPTP_PWL_5: 2677 if (CC(!(vmx->nested.msrs.ept_caps & VMX_EPT_PAGE_WALK_5_BIT))) 2678 return false; 2679 break; 2680 case VMX_EPTP_PWL_4: 2681 if (CC(!(vmx->nested.msrs.ept_caps & VMX_EPT_PAGE_WALK_4_BIT))) 2682 return false; 2683 break; 2684 default: 2685 return false; 2686 } 2687 2688 /* Reserved bits should not be set */ 2689 if (CC(kvm_vcpu_is_illegal_gpa(vcpu, new_eptp) || ((new_eptp >> 7) & 0x1f))) 2690 return false; 2691 2692 /* AD, if set, should be supported */ 2693 if (new_eptp & VMX_EPTP_AD_ENABLE_BIT) { 2694 if (CC(!(vmx->nested.msrs.ept_caps & VMX_EPT_AD_BIT))) 2695 return false; 2696 } 2697 2698 return true; 2699 } 2700 2701 /* 2702 * Checks related to VM-Execution Control Fields 2703 */ 2704 static int nested_check_vm_execution_controls(struct kvm_vcpu *vcpu, 2705 struct vmcs12 *vmcs12) 2706 { 2707 struct vcpu_vmx *vmx = to_vmx(vcpu); 2708 2709 if (CC(!vmx_control_verify(vmcs12->pin_based_vm_exec_control, 2710 vmx->nested.msrs.pinbased_ctls_low, 2711 vmx->nested.msrs.pinbased_ctls_high)) || 2712 CC(!vmx_control_verify(vmcs12->cpu_based_vm_exec_control, 2713 vmx->nested.msrs.procbased_ctls_low, 2714 vmx->nested.msrs.procbased_ctls_high))) 2715 return -EINVAL; 2716 2717 if (nested_cpu_has(vmcs12, CPU_BASED_ACTIVATE_SECONDARY_CONTROLS) && 2718 CC(!vmx_control_verify(vmcs12->secondary_vm_exec_control, 2719 vmx->nested.msrs.secondary_ctls_low, 2720 vmx->nested.msrs.secondary_ctls_high))) 2721 return -EINVAL; 2722 2723 if (CC(vmcs12->cr3_target_count > nested_cpu_vmx_misc_cr3_count(vcpu)) || 2724 nested_vmx_check_io_bitmap_controls(vcpu, vmcs12) || 2725 nested_vmx_check_msr_bitmap_controls(vcpu, vmcs12) || 2726 nested_vmx_check_tpr_shadow_controls(vcpu, vmcs12) || 2727 nested_vmx_check_apic_access_controls(vcpu, vmcs12) || 2728 nested_vmx_check_apicv_controls(vcpu, vmcs12) || 2729 nested_vmx_check_nmi_controls(vmcs12) || 2730 nested_vmx_check_pml_controls(vcpu, vmcs12) || 2731 nested_vmx_check_unrestricted_guest_controls(vcpu, vmcs12) || 2732 nested_vmx_check_mode_based_ept_exec_controls(vcpu, vmcs12) || 2733 nested_vmx_check_shadow_vmcs_controls(vcpu, vmcs12) || 2734 CC(nested_cpu_has_vpid(vmcs12) && !vmcs12->virtual_processor_id)) 2735 return -EINVAL; 2736 2737 if (!nested_cpu_has_preemption_timer(vmcs12) && 2738 nested_cpu_has_save_preemption_timer(vmcs12)) 2739 return -EINVAL; 2740 2741 if (nested_cpu_has_ept(vmcs12) && 2742 CC(!nested_vmx_check_eptp(vcpu, vmcs12->ept_pointer))) 2743 return -EINVAL; 2744 2745 if (nested_cpu_has_vmfunc(vmcs12)) { 2746 if (CC(vmcs12->vm_function_control & 2747 ~vmx->nested.msrs.vmfunc_controls)) 2748 return -EINVAL; 2749 2750 if (nested_cpu_has_eptp_switching(vmcs12)) { 2751 if (CC(!nested_cpu_has_ept(vmcs12)) || 2752 CC(!page_address_valid(vcpu, vmcs12->eptp_list_address))) 2753 return -EINVAL; 2754 } 2755 } 2756 2757 return 0; 2758 } 2759 2760 /* 2761 * Checks related to VM-Exit Control Fields 2762 */ 2763 static int nested_check_vm_exit_controls(struct kvm_vcpu *vcpu, 2764 struct vmcs12 *vmcs12) 2765 { 2766 struct vcpu_vmx *vmx = to_vmx(vcpu); 2767 2768 if (CC(!vmx_control_verify(vmcs12->vm_exit_controls, 2769 vmx->nested.msrs.exit_ctls_low, 2770 vmx->nested.msrs.exit_ctls_high)) || 2771 CC(nested_vmx_check_exit_msr_switch_controls(vcpu, vmcs12))) 2772 return -EINVAL; 2773 2774 return 0; 2775 } 2776 2777 /* 2778 * Checks related to VM-Entry Control Fields 2779 */ 2780 static int nested_check_vm_entry_controls(struct kvm_vcpu *vcpu, 2781 struct vmcs12 *vmcs12) 2782 { 2783 struct vcpu_vmx *vmx = to_vmx(vcpu); 2784 2785 if (CC(!vmx_control_verify(vmcs12->vm_entry_controls, 2786 vmx->nested.msrs.entry_ctls_low, 2787 vmx->nested.msrs.entry_ctls_high))) 2788 return -EINVAL; 2789 2790 /* 2791 * From the Intel SDM, volume 3: 2792 * Fields relevant to VM-entry event injection must be set properly. 2793 * These fields are the VM-entry interruption-information field, the 2794 * VM-entry exception error code, and the VM-entry instruction length. 2795 */ 2796 if (vmcs12->vm_entry_intr_info_field & INTR_INFO_VALID_MASK) { 2797 u32 intr_info = vmcs12->vm_entry_intr_info_field; 2798 u8 vector = intr_info & INTR_INFO_VECTOR_MASK; 2799 u32 intr_type = intr_info & INTR_INFO_INTR_TYPE_MASK; 2800 bool has_error_code = intr_info & INTR_INFO_DELIVER_CODE_MASK; 2801 bool should_have_error_code; 2802 bool urg = nested_cpu_has2(vmcs12, 2803 SECONDARY_EXEC_UNRESTRICTED_GUEST); 2804 bool prot_mode = !urg || vmcs12->guest_cr0 & X86_CR0_PE; 2805 2806 /* VM-entry interruption-info field: interruption type */ 2807 if (CC(intr_type == INTR_TYPE_RESERVED) || 2808 CC(intr_type == INTR_TYPE_OTHER_EVENT && 2809 !nested_cpu_supports_monitor_trap_flag(vcpu))) 2810 return -EINVAL; 2811 2812 /* VM-entry interruption-info field: vector */ 2813 if (CC(intr_type == INTR_TYPE_NMI_INTR && vector != NMI_VECTOR) || 2814 CC(intr_type == INTR_TYPE_HARD_EXCEPTION && vector > 31) || 2815 CC(intr_type == INTR_TYPE_OTHER_EVENT && vector != 0)) 2816 return -EINVAL; 2817 2818 /* VM-entry interruption-info field: deliver error code */ 2819 should_have_error_code = 2820 intr_type == INTR_TYPE_HARD_EXCEPTION && prot_mode && 2821 x86_exception_has_error_code(vector); 2822 if (CC(has_error_code != should_have_error_code)) 2823 return -EINVAL; 2824 2825 /* VM-entry exception error code */ 2826 if (CC(has_error_code && 2827 vmcs12->vm_entry_exception_error_code & GENMASK(31, 16))) 2828 return -EINVAL; 2829 2830 /* VM-entry interruption-info field: reserved bits */ 2831 if (CC(intr_info & INTR_INFO_RESVD_BITS_MASK)) 2832 return -EINVAL; 2833 2834 /* VM-entry instruction length */ 2835 switch (intr_type) { 2836 case INTR_TYPE_SOFT_EXCEPTION: 2837 case INTR_TYPE_SOFT_INTR: 2838 case INTR_TYPE_PRIV_SW_EXCEPTION: 2839 if (CC(vmcs12->vm_entry_instruction_len > 15) || 2840 CC(vmcs12->vm_entry_instruction_len == 0 && 2841 CC(!nested_cpu_has_zero_length_injection(vcpu)))) 2842 return -EINVAL; 2843 } 2844 } 2845 2846 if (nested_vmx_check_entry_msr_switch_controls(vcpu, vmcs12)) 2847 return -EINVAL; 2848 2849 return 0; 2850 } 2851 2852 static int nested_vmx_check_controls(struct kvm_vcpu *vcpu, 2853 struct vmcs12 *vmcs12) 2854 { 2855 if (nested_check_vm_execution_controls(vcpu, vmcs12) || 2856 nested_check_vm_exit_controls(vcpu, vmcs12) || 2857 nested_check_vm_entry_controls(vcpu, vmcs12)) 2858 return -EINVAL; 2859 2860 if (to_vmx(vcpu)->nested.enlightened_vmcs_enabled) 2861 return nested_evmcs_check_controls(vmcs12); 2862 2863 return 0; 2864 } 2865 2866 static int nested_vmx_check_address_space_size(struct kvm_vcpu *vcpu, 2867 struct vmcs12 *vmcs12) 2868 { 2869 #ifdef CONFIG_X86_64 2870 if (CC(!!(vmcs12->vm_exit_controls & VM_EXIT_HOST_ADDR_SPACE_SIZE) != 2871 !!(vcpu->arch.efer & EFER_LMA))) 2872 return -EINVAL; 2873 #endif 2874 return 0; 2875 } 2876 2877 static int nested_vmx_check_host_state(struct kvm_vcpu *vcpu, 2878 struct vmcs12 *vmcs12) 2879 { 2880 bool ia32e; 2881 2882 if (CC(!nested_host_cr0_valid(vcpu, vmcs12->host_cr0)) || 2883 CC(!nested_host_cr4_valid(vcpu, vmcs12->host_cr4)) || 2884 CC(kvm_vcpu_is_illegal_gpa(vcpu, vmcs12->host_cr3))) 2885 return -EINVAL; 2886 2887 if (CC(is_noncanonical_address(vmcs12->host_ia32_sysenter_esp, vcpu)) || 2888 CC(is_noncanonical_address(vmcs12->host_ia32_sysenter_eip, vcpu))) 2889 return -EINVAL; 2890 2891 if ((vmcs12->vm_exit_controls & VM_EXIT_LOAD_IA32_PAT) && 2892 CC(!kvm_pat_valid(vmcs12->host_ia32_pat))) 2893 return -EINVAL; 2894 2895 if ((vmcs12->vm_exit_controls & VM_EXIT_LOAD_IA32_PERF_GLOBAL_CTRL) && 2896 CC(!kvm_valid_perf_global_ctrl(vcpu_to_pmu(vcpu), 2897 vmcs12->host_ia32_perf_global_ctrl))) 2898 return -EINVAL; 2899 2900 #ifdef CONFIG_X86_64 2901 ia32e = !!(vmcs12->vm_exit_controls & VM_EXIT_HOST_ADDR_SPACE_SIZE); 2902 #else 2903 ia32e = false; 2904 #endif 2905 2906 if (ia32e) { 2907 if (CC(!(vmcs12->host_cr4 & X86_CR4_PAE))) 2908 return -EINVAL; 2909 } else { 2910 if (CC(vmcs12->vm_entry_controls & VM_ENTRY_IA32E_MODE) || 2911 CC(vmcs12->host_cr4 & X86_CR4_PCIDE) || 2912 CC((vmcs12->host_rip) >> 32)) 2913 return -EINVAL; 2914 } 2915 2916 if (CC(vmcs12->host_cs_selector & (SEGMENT_RPL_MASK | SEGMENT_TI_MASK)) || 2917 CC(vmcs12->host_ss_selector & (SEGMENT_RPL_MASK | SEGMENT_TI_MASK)) || 2918 CC(vmcs12->host_ds_selector & (SEGMENT_RPL_MASK | SEGMENT_TI_MASK)) || 2919 CC(vmcs12->host_es_selector & (SEGMENT_RPL_MASK | SEGMENT_TI_MASK)) || 2920 CC(vmcs12->host_fs_selector & (SEGMENT_RPL_MASK | SEGMENT_TI_MASK)) || 2921 CC(vmcs12->host_gs_selector & (SEGMENT_RPL_MASK | SEGMENT_TI_MASK)) || 2922 CC(vmcs12->host_tr_selector & (SEGMENT_RPL_MASK | SEGMENT_TI_MASK)) || 2923 CC(vmcs12->host_cs_selector == 0) || 2924 CC(vmcs12->host_tr_selector == 0) || 2925 CC(vmcs12->host_ss_selector == 0 && !ia32e)) 2926 return -EINVAL; 2927 2928 if (CC(is_noncanonical_address(vmcs12->host_fs_base, vcpu)) || 2929 CC(is_noncanonical_address(vmcs12->host_gs_base, vcpu)) || 2930 CC(is_noncanonical_address(vmcs12->host_gdtr_base, vcpu)) || 2931 CC(is_noncanonical_address(vmcs12->host_idtr_base, vcpu)) || 2932 CC(is_noncanonical_address(vmcs12->host_tr_base, vcpu)) || 2933 CC(is_noncanonical_address(vmcs12->host_rip, vcpu))) 2934 return -EINVAL; 2935 2936 /* 2937 * If the load IA32_EFER VM-exit control is 1, bits reserved in the 2938 * IA32_EFER MSR must be 0 in the field for that register. In addition, 2939 * the values of the LMA and LME bits in the field must each be that of 2940 * the host address-space size VM-exit control. 2941 */ 2942 if (vmcs12->vm_exit_controls & VM_EXIT_LOAD_IA32_EFER) { 2943 if (CC(!kvm_valid_efer(vcpu, vmcs12->host_ia32_efer)) || 2944 CC(ia32e != !!(vmcs12->host_ia32_efer & EFER_LMA)) || 2945 CC(ia32e != !!(vmcs12->host_ia32_efer & EFER_LME))) 2946 return -EINVAL; 2947 } 2948 2949 return 0; 2950 } 2951 2952 static int nested_vmx_check_vmcs_link_ptr(struct kvm_vcpu *vcpu, 2953 struct vmcs12 *vmcs12) 2954 { 2955 struct vcpu_vmx *vmx = to_vmx(vcpu); 2956 struct gfn_to_hva_cache *ghc = &vmx->nested.shadow_vmcs12_cache; 2957 struct vmcs_hdr hdr; 2958 2959 if (vmcs12->vmcs_link_pointer == INVALID_GPA) 2960 return 0; 2961 2962 if (CC(!page_address_valid(vcpu, vmcs12->vmcs_link_pointer))) 2963 return -EINVAL; 2964 2965 if (ghc->gpa != vmcs12->vmcs_link_pointer && 2966 CC(kvm_gfn_to_hva_cache_init(vcpu->kvm, ghc, 2967 vmcs12->vmcs_link_pointer, VMCS12_SIZE))) 2968 return -EINVAL; 2969 2970 if (CC(kvm_read_guest_offset_cached(vcpu->kvm, ghc, &hdr, 2971 offsetof(struct vmcs12, hdr), 2972 sizeof(hdr)))) 2973 return -EINVAL; 2974 2975 if (CC(hdr.revision_id != VMCS12_REVISION) || 2976 CC(hdr.shadow_vmcs != nested_cpu_has_shadow_vmcs(vmcs12))) 2977 return -EINVAL; 2978 2979 return 0; 2980 } 2981 2982 /* 2983 * Checks related to Guest Non-register State 2984 */ 2985 static int nested_check_guest_non_reg_state(struct vmcs12 *vmcs12) 2986 { 2987 if (CC(vmcs12->guest_activity_state != GUEST_ACTIVITY_ACTIVE && 2988 vmcs12->guest_activity_state != GUEST_ACTIVITY_HLT && 2989 vmcs12->guest_activity_state != GUEST_ACTIVITY_WAIT_SIPI)) 2990 return -EINVAL; 2991 2992 return 0; 2993 } 2994 2995 static int nested_vmx_check_guest_state(struct kvm_vcpu *vcpu, 2996 struct vmcs12 *vmcs12, 2997 enum vm_entry_failure_code *entry_failure_code) 2998 { 2999 bool ia32e; 3000 3001 *entry_failure_code = ENTRY_FAIL_DEFAULT; 3002 3003 if (CC(!nested_guest_cr0_valid(vcpu, vmcs12->guest_cr0)) || 3004 CC(!nested_guest_cr4_valid(vcpu, vmcs12->guest_cr4))) 3005 return -EINVAL; 3006 3007 if ((vmcs12->vm_entry_controls & VM_ENTRY_LOAD_DEBUG_CONTROLS) && 3008 CC(!kvm_dr7_valid(vmcs12->guest_dr7))) 3009 return -EINVAL; 3010 3011 if ((vmcs12->vm_entry_controls & VM_ENTRY_LOAD_IA32_PAT) && 3012 CC(!kvm_pat_valid(vmcs12->guest_ia32_pat))) 3013 return -EINVAL; 3014 3015 if (nested_vmx_check_vmcs_link_ptr(vcpu, vmcs12)) { 3016 *entry_failure_code = ENTRY_FAIL_VMCS_LINK_PTR; 3017 return -EINVAL; 3018 } 3019 3020 if ((vmcs12->vm_entry_controls & VM_ENTRY_LOAD_IA32_PERF_GLOBAL_CTRL) && 3021 CC(!kvm_valid_perf_global_ctrl(vcpu_to_pmu(vcpu), 3022 vmcs12->guest_ia32_perf_global_ctrl))) 3023 return -EINVAL; 3024 3025 /* 3026 * If the load IA32_EFER VM-entry control is 1, the following checks 3027 * are performed on the field for the IA32_EFER MSR: 3028 * - Bits reserved in the IA32_EFER MSR must be 0. 3029 * - Bit 10 (corresponding to IA32_EFER.LMA) must equal the value of 3030 * the IA-32e mode guest VM-exit control. It must also be identical 3031 * to bit 8 (LME) if bit 31 in the CR0 field (corresponding to 3032 * CR0.PG) is 1. 3033 */ 3034 if (to_vmx(vcpu)->nested.nested_run_pending && 3035 (vmcs12->vm_entry_controls & VM_ENTRY_LOAD_IA32_EFER)) { 3036 ia32e = (vmcs12->vm_entry_controls & VM_ENTRY_IA32E_MODE) != 0; 3037 if (CC(!kvm_valid_efer(vcpu, vmcs12->guest_ia32_efer)) || 3038 CC(ia32e != !!(vmcs12->guest_ia32_efer & EFER_LMA)) || 3039 CC(((vmcs12->guest_cr0 & X86_CR0_PG) && 3040 ia32e != !!(vmcs12->guest_ia32_efer & EFER_LME)))) 3041 return -EINVAL; 3042 } 3043 3044 if ((vmcs12->vm_entry_controls & VM_ENTRY_LOAD_BNDCFGS) && 3045 (CC(is_noncanonical_address(vmcs12->guest_bndcfgs & PAGE_MASK, vcpu)) || 3046 CC((vmcs12->guest_bndcfgs & MSR_IA32_BNDCFGS_RSVD)))) 3047 return -EINVAL; 3048 3049 if (nested_check_guest_non_reg_state(vmcs12)) 3050 return -EINVAL; 3051 3052 return 0; 3053 } 3054 3055 static int nested_vmx_check_vmentry_hw(struct kvm_vcpu *vcpu) 3056 { 3057 struct vcpu_vmx *vmx = to_vmx(vcpu); 3058 unsigned long cr3, cr4; 3059 bool vm_fail; 3060 3061 if (!nested_early_check) 3062 return 0; 3063 3064 if (vmx->msr_autoload.host.nr) 3065 vmcs_write32(VM_EXIT_MSR_LOAD_COUNT, 0); 3066 if (vmx->msr_autoload.guest.nr) 3067 vmcs_write32(VM_ENTRY_MSR_LOAD_COUNT, 0); 3068 3069 preempt_disable(); 3070 3071 vmx_prepare_switch_to_guest(vcpu); 3072 3073 /* 3074 * Induce a consistency check VMExit by clearing bit 1 in GUEST_RFLAGS, 3075 * which is reserved to '1' by hardware. GUEST_RFLAGS is guaranteed to 3076 * be written (by prepare_vmcs02()) before the "real" VMEnter, i.e. 3077 * there is no need to preserve other bits or save/restore the field. 3078 */ 3079 vmcs_writel(GUEST_RFLAGS, 0); 3080 3081 cr3 = __get_current_cr3_fast(); 3082 if (unlikely(cr3 != vmx->loaded_vmcs->host_state.cr3)) { 3083 vmcs_writel(HOST_CR3, cr3); 3084 vmx->loaded_vmcs->host_state.cr3 = cr3; 3085 } 3086 3087 cr4 = cr4_read_shadow(); 3088 if (unlikely(cr4 != vmx->loaded_vmcs->host_state.cr4)) { 3089 vmcs_writel(HOST_CR4, cr4); 3090 vmx->loaded_vmcs->host_state.cr4 = cr4; 3091 } 3092 3093 vm_fail = __vmx_vcpu_run(vmx, (unsigned long *)&vcpu->arch.regs, 3094 vmx->loaded_vmcs->launched); 3095 3096 if (vmx->msr_autoload.host.nr) 3097 vmcs_write32(VM_EXIT_MSR_LOAD_COUNT, vmx->msr_autoload.host.nr); 3098 if (vmx->msr_autoload.guest.nr) 3099 vmcs_write32(VM_ENTRY_MSR_LOAD_COUNT, vmx->msr_autoload.guest.nr); 3100 3101 if (vm_fail) { 3102 u32 error = vmcs_read32(VM_INSTRUCTION_ERROR); 3103 3104 preempt_enable(); 3105 3106 trace_kvm_nested_vmenter_failed( 3107 "early hardware check VM-instruction error: ", error); 3108 WARN_ON_ONCE(error != VMXERR_ENTRY_INVALID_CONTROL_FIELD); 3109 return 1; 3110 } 3111 3112 /* 3113 * VMExit clears RFLAGS.IF and DR7, even on a consistency check. 3114 */ 3115 if (hw_breakpoint_active()) 3116 set_debugreg(__this_cpu_read(cpu_dr7), 7); 3117 local_irq_enable(); 3118 preempt_enable(); 3119 3120 /* 3121 * A non-failing VMEntry means we somehow entered guest mode with 3122 * an illegal RIP, and that's just the tip of the iceberg. There 3123 * is no telling what memory has been modified or what state has 3124 * been exposed to unknown code. Hitting this all but guarantees 3125 * a (very critical) hardware issue. 3126 */ 3127 WARN_ON(!(vmcs_read32(VM_EXIT_REASON) & 3128 VMX_EXIT_REASONS_FAILED_VMENTRY)); 3129 3130 return 0; 3131 } 3132 3133 static bool nested_get_evmcs_page(struct kvm_vcpu *vcpu) 3134 { 3135 struct vcpu_vmx *vmx = to_vmx(vcpu); 3136 3137 /* 3138 * hv_evmcs may end up being not mapped after migration (when 3139 * L2 was running), map it here to make sure vmcs12 changes are 3140 * properly reflected. 3141 */ 3142 if (vmx->nested.enlightened_vmcs_enabled && 3143 vmx->nested.hv_evmcs_vmptr == EVMPTR_MAP_PENDING) { 3144 enum nested_evmptrld_status evmptrld_status = 3145 nested_vmx_handle_enlightened_vmptrld(vcpu, false); 3146 3147 if (evmptrld_status == EVMPTRLD_VMFAIL || 3148 evmptrld_status == EVMPTRLD_ERROR) 3149 return false; 3150 3151 /* 3152 * Post migration VMCS12 always provides the most actual 3153 * information, copy it to eVMCS upon entry. 3154 */ 3155 vmx->nested.need_vmcs12_to_shadow_sync = true; 3156 } 3157 3158 return true; 3159 } 3160 3161 static bool nested_get_vmcs12_pages(struct kvm_vcpu *vcpu) 3162 { 3163 struct vmcs12 *vmcs12 = get_vmcs12(vcpu); 3164 struct vcpu_vmx *vmx = to_vmx(vcpu); 3165 struct kvm_host_map *map; 3166 struct page *page; 3167 u64 hpa; 3168 3169 if (!vcpu->arch.pdptrs_from_userspace && 3170 !nested_cpu_has_ept(vmcs12) && is_pae_paging(vcpu)) { 3171 /* 3172 * Reload the guest's PDPTRs since after a migration 3173 * the guest CR3 might be restored prior to setting the nested 3174 * state which can lead to a load of wrong PDPTRs. 3175 */ 3176 if (CC(!load_pdptrs(vcpu, vcpu->arch.cr3))) 3177 return false; 3178 } 3179 3180 3181 if (nested_cpu_has2(vmcs12, SECONDARY_EXEC_VIRTUALIZE_APIC_ACCESSES)) { 3182 /* 3183 * Translate L1 physical address to host physical 3184 * address for vmcs02. Keep the page pinned, so this 3185 * physical address remains valid. We keep a reference 3186 * to it so we can release it later. 3187 */ 3188 if (vmx->nested.apic_access_page) { /* shouldn't happen */ 3189 kvm_release_page_clean(vmx->nested.apic_access_page); 3190 vmx->nested.apic_access_page = NULL; 3191 } 3192 page = kvm_vcpu_gpa_to_page(vcpu, vmcs12->apic_access_addr); 3193 if (!is_error_page(page)) { 3194 vmx->nested.apic_access_page = page; 3195 hpa = page_to_phys(vmx->nested.apic_access_page); 3196 vmcs_write64(APIC_ACCESS_ADDR, hpa); 3197 } else { 3198 pr_debug_ratelimited("%s: no backing 'struct page' for APIC-access address in vmcs12\n", 3199 __func__); 3200 vcpu->run->exit_reason = KVM_EXIT_INTERNAL_ERROR; 3201 vcpu->run->internal.suberror = 3202 KVM_INTERNAL_ERROR_EMULATION; 3203 vcpu->run->internal.ndata = 0; 3204 return false; 3205 } 3206 } 3207 3208 if (nested_cpu_has(vmcs12, CPU_BASED_TPR_SHADOW)) { 3209 map = &vmx->nested.virtual_apic_map; 3210 3211 if (!kvm_vcpu_map(vcpu, gpa_to_gfn(vmcs12->virtual_apic_page_addr), map)) { 3212 vmcs_write64(VIRTUAL_APIC_PAGE_ADDR, pfn_to_hpa(map->pfn)); 3213 } else if (nested_cpu_has(vmcs12, CPU_BASED_CR8_LOAD_EXITING) && 3214 nested_cpu_has(vmcs12, CPU_BASED_CR8_STORE_EXITING) && 3215 !nested_cpu_has2(vmcs12, SECONDARY_EXEC_VIRTUALIZE_APIC_ACCESSES)) { 3216 /* 3217 * The processor will never use the TPR shadow, simply 3218 * clear the bit from the execution control. Such a 3219 * configuration is useless, but it happens in tests. 3220 * For any other configuration, failing the vm entry is 3221 * _not_ what the processor does but it's basically the 3222 * only possibility we have. 3223 */ 3224 exec_controls_clearbit(vmx, CPU_BASED_TPR_SHADOW); 3225 } else { 3226 /* 3227 * Write an illegal value to VIRTUAL_APIC_PAGE_ADDR to 3228 * force VM-Entry to fail. 3229 */ 3230 vmcs_write64(VIRTUAL_APIC_PAGE_ADDR, INVALID_GPA); 3231 } 3232 } 3233 3234 if (nested_cpu_has_posted_intr(vmcs12)) { 3235 map = &vmx->nested.pi_desc_map; 3236 3237 if (!kvm_vcpu_map(vcpu, gpa_to_gfn(vmcs12->posted_intr_desc_addr), map)) { 3238 vmx->nested.pi_desc = 3239 (struct pi_desc *)(((void *)map->hva) + 3240 offset_in_page(vmcs12->posted_intr_desc_addr)); 3241 vmcs_write64(POSTED_INTR_DESC_ADDR, 3242 pfn_to_hpa(map->pfn) + offset_in_page(vmcs12->posted_intr_desc_addr)); 3243 } else { 3244 /* 3245 * Defer the KVM_INTERNAL_EXIT until KVM tries to 3246 * access the contents of the VMCS12 posted interrupt 3247 * descriptor. (Note that KVM may do this when it 3248 * should not, per the architectural specification.) 3249 */ 3250 vmx->nested.pi_desc = NULL; 3251 pin_controls_clearbit(vmx, PIN_BASED_POSTED_INTR); 3252 } 3253 } 3254 if (nested_vmx_prepare_msr_bitmap(vcpu, vmcs12)) 3255 exec_controls_setbit(vmx, CPU_BASED_USE_MSR_BITMAPS); 3256 else 3257 exec_controls_clearbit(vmx, CPU_BASED_USE_MSR_BITMAPS); 3258 3259 return true; 3260 } 3261 3262 static bool vmx_get_nested_state_pages(struct kvm_vcpu *vcpu) 3263 { 3264 if (!nested_get_evmcs_page(vcpu)) { 3265 pr_debug_ratelimited("%s: enlightened vmptrld failed\n", 3266 __func__); 3267 vcpu->run->exit_reason = KVM_EXIT_INTERNAL_ERROR; 3268 vcpu->run->internal.suberror = 3269 KVM_INTERNAL_ERROR_EMULATION; 3270 vcpu->run->internal.ndata = 0; 3271 3272 return false; 3273 } 3274 3275 if (is_guest_mode(vcpu) && !nested_get_vmcs12_pages(vcpu)) 3276 return false; 3277 3278 return true; 3279 } 3280 3281 static int nested_vmx_write_pml_buffer(struct kvm_vcpu *vcpu, gpa_t gpa) 3282 { 3283 struct vmcs12 *vmcs12; 3284 struct vcpu_vmx *vmx = to_vmx(vcpu); 3285 gpa_t dst; 3286 3287 if (WARN_ON_ONCE(!is_guest_mode(vcpu))) 3288 return 0; 3289 3290 if (WARN_ON_ONCE(vmx->nested.pml_full)) 3291 return 1; 3292 3293 /* 3294 * Check if PML is enabled for the nested guest. Whether eptp bit 6 is 3295 * set is already checked as part of A/D emulation. 3296 */ 3297 vmcs12 = get_vmcs12(vcpu); 3298 if (!nested_cpu_has_pml(vmcs12)) 3299 return 0; 3300 3301 if (vmcs12->guest_pml_index >= PML_ENTITY_NUM) { 3302 vmx->nested.pml_full = true; 3303 return 1; 3304 } 3305 3306 gpa &= ~0xFFFull; 3307 dst = vmcs12->pml_address + sizeof(u64) * vmcs12->guest_pml_index; 3308 3309 if (kvm_write_guest_page(vcpu->kvm, gpa_to_gfn(dst), &gpa, 3310 offset_in_page(dst), sizeof(gpa))) 3311 return 0; 3312 3313 vmcs12->guest_pml_index--; 3314 3315 return 0; 3316 } 3317 3318 /* 3319 * Intel's VMX Instruction Reference specifies a common set of prerequisites 3320 * for running VMX instructions (except VMXON, whose prerequisites are 3321 * slightly different). It also specifies what exception to inject otherwise. 3322 * Note that many of these exceptions have priority over VM exits, so they 3323 * don't have to be checked again here. 3324 */ 3325 static int nested_vmx_check_permission(struct kvm_vcpu *vcpu) 3326 { 3327 if (!to_vmx(vcpu)->nested.vmxon) { 3328 kvm_queue_exception(vcpu, UD_VECTOR); 3329 return 0; 3330 } 3331 3332 if (vmx_get_cpl(vcpu)) { 3333 kvm_inject_gp(vcpu, 0); 3334 return 0; 3335 } 3336 3337 return 1; 3338 } 3339 3340 static u8 vmx_has_apicv_interrupt(struct kvm_vcpu *vcpu) 3341 { 3342 u8 rvi = vmx_get_rvi(); 3343 u8 vppr = kvm_lapic_get_reg(vcpu->arch.apic, APIC_PROCPRI); 3344 3345 return ((rvi & 0xf0) > (vppr & 0xf0)); 3346 } 3347 3348 static void load_vmcs12_host_state(struct kvm_vcpu *vcpu, 3349 struct vmcs12 *vmcs12); 3350 3351 /* 3352 * If from_vmentry is false, this is being called from state restore (either RSM 3353 * or KVM_SET_NESTED_STATE). Otherwise it's called from vmlaunch/vmresume. 3354 * 3355 * Returns: 3356 * NVMX_VMENTRY_SUCCESS: Entered VMX non-root mode 3357 * NVMX_VMENTRY_VMFAIL: Consistency check VMFail 3358 * NVMX_VMENTRY_VMEXIT: Consistency check VMExit 3359 * NVMX_VMENTRY_KVM_INTERNAL_ERROR: KVM internal error 3360 */ 3361 enum nvmx_vmentry_status nested_vmx_enter_non_root_mode(struct kvm_vcpu *vcpu, 3362 bool from_vmentry) 3363 { 3364 struct vcpu_vmx *vmx = to_vmx(vcpu); 3365 struct vmcs12 *vmcs12 = get_vmcs12(vcpu); 3366 enum vm_entry_failure_code entry_failure_code; 3367 bool evaluate_pending_interrupts; 3368 union vmx_exit_reason exit_reason = { 3369 .basic = EXIT_REASON_INVALID_STATE, 3370 .failed_vmentry = 1, 3371 }; 3372 u32 failed_index; 3373 3374 kvm_service_local_tlb_flush_requests(vcpu); 3375 3376 evaluate_pending_interrupts = exec_controls_get(vmx) & 3377 (CPU_BASED_INTR_WINDOW_EXITING | CPU_BASED_NMI_WINDOW_EXITING); 3378 if (likely(!evaluate_pending_interrupts) && kvm_vcpu_apicv_active(vcpu)) 3379 evaluate_pending_interrupts |= vmx_has_apicv_interrupt(vcpu); 3380 3381 if (!(vmcs12->vm_entry_controls & VM_ENTRY_LOAD_DEBUG_CONTROLS)) 3382 vmx->nested.vmcs01_debugctl = vmcs_read64(GUEST_IA32_DEBUGCTL); 3383 if (kvm_mpx_supported() && 3384 !(vmcs12->vm_entry_controls & VM_ENTRY_LOAD_BNDCFGS)) 3385 vmx->nested.vmcs01_guest_bndcfgs = vmcs_read64(GUEST_BNDCFGS); 3386 3387 /* 3388 * Overwrite vmcs01.GUEST_CR3 with L1's CR3 if EPT is disabled *and* 3389 * nested early checks are disabled. In the event of a "late" VM-Fail, 3390 * i.e. a VM-Fail detected by hardware but not KVM, KVM must unwind its 3391 * software model to the pre-VMEntry host state. When EPT is disabled, 3392 * GUEST_CR3 holds KVM's shadow CR3, not L1's "real" CR3, which causes 3393 * nested_vmx_restore_host_state() to corrupt vcpu->arch.cr3. Stuffing 3394 * vmcs01.GUEST_CR3 results in the unwind naturally setting arch.cr3 to 3395 * the correct value. Smashing vmcs01.GUEST_CR3 is safe because nested 3396 * VM-Exits, and the unwind, reset KVM's MMU, i.e. vmcs01.GUEST_CR3 is 3397 * guaranteed to be overwritten with a shadow CR3 prior to re-entering 3398 * L1. Don't stuff vmcs01.GUEST_CR3 when using nested early checks as 3399 * KVM modifies vcpu->arch.cr3 if and only if the early hardware checks 3400 * pass, and early VM-Fails do not reset KVM's MMU, i.e. the VM-Fail 3401 * path would need to manually save/restore vmcs01.GUEST_CR3. 3402 */ 3403 if (!enable_ept && !nested_early_check) 3404 vmcs_writel(GUEST_CR3, vcpu->arch.cr3); 3405 3406 vmx_switch_vmcs(vcpu, &vmx->nested.vmcs02); 3407 3408 prepare_vmcs02_early(vmx, &vmx->vmcs01, vmcs12); 3409 3410 if (from_vmentry) { 3411 if (unlikely(!nested_get_vmcs12_pages(vcpu))) { 3412 vmx_switch_vmcs(vcpu, &vmx->vmcs01); 3413 return NVMX_VMENTRY_KVM_INTERNAL_ERROR; 3414 } 3415 3416 if (nested_vmx_check_vmentry_hw(vcpu)) { 3417 vmx_switch_vmcs(vcpu, &vmx->vmcs01); 3418 return NVMX_VMENTRY_VMFAIL; 3419 } 3420 3421 if (nested_vmx_check_guest_state(vcpu, vmcs12, 3422 &entry_failure_code)) { 3423 exit_reason.basic = EXIT_REASON_INVALID_STATE; 3424 vmcs12->exit_qualification = entry_failure_code; 3425 goto vmentry_fail_vmexit; 3426 } 3427 } 3428 3429 enter_guest_mode(vcpu); 3430 3431 if (prepare_vmcs02(vcpu, vmcs12, from_vmentry, &entry_failure_code)) { 3432 exit_reason.basic = EXIT_REASON_INVALID_STATE; 3433 vmcs12->exit_qualification = entry_failure_code; 3434 goto vmentry_fail_vmexit_guest_mode; 3435 } 3436 3437 if (from_vmentry) { 3438 failed_index = nested_vmx_load_msr(vcpu, 3439 vmcs12->vm_entry_msr_load_addr, 3440 vmcs12->vm_entry_msr_load_count); 3441 if (failed_index) { 3442 exit_reason.basic = EXIT_REASON_MSR_LOAD_FAIL; 3443 vmcs12->exit_qualification = failed_index; 3444 goto vmentry_fail_vmexit_guest_mode; 3445 } 3446 } else { 3447 /* 3448 * The MMU is not initialized to point at the right entities yet and 3449 * "get pages" would need to read data from the guest (i.e. we will 3450 * need to perform gpa to hpa translation). Request a call 3451 * to nested_get_vmcs12_pages before the next VM-entry. The MSRs 3452 * have already been set at vmentry time and should not be reset. 3453 */ 3454 kvm_make_request(KVM_REQ_GET_NESTED_STATE_PAGES, vcpu); 3455 } 3456 3457 /* 3458 * If L1 had a pending IRQ/NMI until it executed 3459 * VMLAUNCH/VMRESUME which wasn't delivered because it was 3460 * disallowed (e.g. interrupts disabled), L0 needs to 3461 * evaluate if this pending event should cause an exit from L2 3462 * to L1 or delivered directly to L2 (e.g. In case L1 don't 3463 * intercept EXTERNAL_INTERRUPT). 3464 * 3465 * Usually this would be handled by the processor noticing an 3466 * IRQ/NMI window request, or checking RVI during evaluation of 3467 * pending virtual interrupts. However, this setting was done 3468 * on VMCS01 and now VMCS02 is active instead. Thus, we force L0 3469 * to perform pending event evaluation by requesting a KVM_REQ_EVENT. 3470 */ 3471 if (unlikely(evaluate_pending_interrupts)) 3472 kvm_make_request(KVM_REQ_EVENT, vcpu); 3473 3474 /* 3475 * Do not start the preemption timer hrtimer until after we know 3476 * we are successful, so that only nested_vmx_vmexit needs to cancel 3477 * the timer. 3478 */ 3479 vmx->nested.preemption_timer_expired = false; 3480 if (nested_cpu_has_preemption_timer(vmcs12)) { 3481 u64 timer_value = vmx_calc_preemption_timer_value(vcpu); 3482 vmx_start_preemption_timer(vcpu, timer_value); 3483 } 3484 3485 /* 3486 * Note no nested_vmx_succeed or nested_vmx_fail here. At this point 3487 * we are no longer running L1, and VMLAUNCH/VMRESUME has not yet 3488 * returned as far as L1 is concerned. It will only return (and set 3489 * the success flag) when L2 exits (see nested_vmx_vmexit()). 3490 */ 3491 return NVMX_VMENTRY_SUCCESS; 3492 3493 /* 3494 * A failed consistency check that leads to a VMExit during L1's 3495 * VMEnter to L2 is a variation of a normal VMexit, as explained in 3496 * 26.7 "VM-entry failures during or after loading guest state". 3497 */ 3498 vmentry_fail_vmexit_guest_mode: 3499 if (vmcs12->cpu_based_vm_exec_control & CPU_BASED_USE_TSC_OFFSETTING) 3500 vcpu->arch.tsc_offset -= vmcs12->tsc_offset; 3501 leave_guest_mode(vcpu); 3502 3503 vmentry_fail_vmexit: 3504 vmx_switch_vmcs(vcpu, &vmx->vmcs01); 3505 3506 if (!from_vmentry) 3507 return NVMX_VMENTRY_VMEXIT; 3508 3509 load_vmcs12_host_state(vcpu, vmcs12); 3510 vmcs12->vm_exit_reason = exit_reason.full; 3511 if (enable_shadow_vmcs || evmptr_is_valid(vmx->nested.hv_evmcs_vmptr)) 3512 vmx->nested.need_vmcs12_to_shadow_sync = true; 3513 return NVMX_VMENTRY_VMEXIT; 3514 } 3515 3516 /* 3517 * nested_vmx_run() handles a nested entry, i.e., a VMLAUNCH or VMRESUME on L1 3518 * for running an L2 nested guest. 3519 */ 3520 static int nested_vmx_run(struct kvm_vcpu *vcpu, bool launch) 3521 { 3522 struct vmcs12 *vmcs12; 3523 enum nvmx_vmentry_status status; 3524 struct vcpu_vmx *vmx = to_vmx(vcpu); 3525 u32 interrupt_shadow = vmx_get_interrupt_shadow(vcpu); 3526 enum nested_evmptrld_status evmptrld_status; 3527 3528 if (!nested_vmx_check_permission(vcpu)) 3529 return 1; 3530 3531 evmptrld_status = nested_vmx_handle_enlightened_vmptrld(vcpu, launch); 3532 if (evmptrld_status == EVMPTRLD_ERROR) { 3533 kvm_queue_exception(vcpu, UD_VECTOR); 3534 return 1; 3535 } 3536 3537 kvm_pmu_trigger_event(vcpu, PERF_COUNT_HW_BRANCH_INSTRUCTIONS); 3538 3539 if (CC(evmptrld_status == EVMPTRLD_VMFAIL)) 3540 return nested_vmx_failInvalid(vcpu); 3541 3542 if (CC(!evmptr_is_valid(vmx->nested.hv_evmcs_vmptr) && 3543 vmx->nested.current_vmptr == INVALID_GPA)) 3544 return nested_vmx_failInvalid(vcpu); 3545 3546 vmcs12 = get_vmcs12(vcpu); 3547 3548 /* 3549 * Can't VMLAUNCH or VMRESUME a shadow VMCS. Despite the fact 3550 * that there *is* a valid VMCS pointer, RFLAGS.CF is set 3551 * rather than RFLAGS.ZF, and no error number is stored to the 3552 * VM-instruction error field. 3553 */ 3554 if (CC(vmcs12->hdr.shadow_vmcs)) 3555 return nested_vmx_failInvalid(vcpu); 3556 3557 if (evmptr_is_valid(vmx->nested.hv_evmcs_vmptr)) { 3558 copy_enlightened_to_vmcs12(vmx, vmx->nested.hv_evmcs->hv_clean_fields); 3559 /* Enlightened VMCS doesn't have launch state */ 3560 vmcs12->launch_state = !launch; 3561 } else if (enable_shadow_vmcs) { 3562 copy_shadow_to_vmcs12(vmx); 3563 } 3564 3565 /* 3566 * The nested entry process starts with enforcing various prerequisites 3567 * on vmcs12 as required by the Intel SDM, and act appropriately when 3568 * they fail: As the SDM explains, some conditions should cause the 3569 * instruction to fail, while others will cause the instruction to seem 3570 * to succeed, but return an EXIT_REASON_INVALID_STATE. 3571 * To speed up the normal (success) code path, we should avoid checking 3572 * for misconfigurations which will anyway be caught by the processor 3573 * when using the merged vmcs02. 3574 */ 3575 if (CC(interrupt_shadow & KVM_X86_SHADOW_INT_MOV_SS)) 3576 return nested_vmx_fail(vcpu, VMXERR_ENTRY_EVENTS_BLOCKED_BY_MOV_SS); 3577 3578 if (CC(vmcs12->launch_state == launch)) 3579 return nested_vmx_fail(vcpu, 3580 launch ? VMXERR_VMLAUNCH_NONCLEAR_VMCS 3581 : VMXERR_VMRESUME_NONLAUNCHED_VMCS); 3582 3583 if (nested_vmx_check_controls(vcpu, vmcs12)) 3584 return nested_vmx_fail(vcpu, VMXERR_ENTRY_INVALID_CONTROL_FIELD); 3585 3586 if (nested_vmx_check_address_space_size(vcpu, vmcs12)) 3587 return nested_vmx_fail(vcpu, VMXERR_ENTRY_INVALID_HOST_STATE_FIELD); 3588 3589 if (nested_vmx_check_host_state(vcpu, vmcs12)) 3590 return nested_vmx_fail(vcpu, VMXERR_ENTRY_INVALID_HOST_STATE_FIELD); 3591 3592 /* 3593 * We're finally done with prerequisite checking, and can start with 3594 * the nested entry. 3595 */ 3596 vmx->nested.nested_run_pending = 1; 3597 vmx->nested.has_preemption_timer_deadline = false; 3598 status = nested_vmx_enter_non_root_mode(vcpu, true); 3599 if (unlikely(status != NVMX_VMENTRY_SUCCESS)) 3600 goto vmentry_failed; 3601 3602 /* Emulate processing of posted interrupts on VM-Enter. */ 3603 if (nested_cpu_has_posted_intr(vmcs12) && 3604 kvm_apic_has_interrupt(vcpu) == vmx->nested.posted_intr_nv) { 3605 vmx->nested.pi_pending = true; 3606 kvm_make_request(KVM_REQ_EVENT, vcpu); 3607 kvm_apic_clear_irr(vcpu, vmx->nested.posted_intr_nv); 3608 } 3609 3610 /* Hide L1D cache contents from the nested guest. */ 3611 vmx->vcpu.arch.l1tf_flush_l1d = true; 3612 3613 /* 3614 * Must happen outside of nested_vmx_enter_non_root_mode() as it will 3615 * also be used as part of restoring nVMX state for 3616 * snapshot restore (migration). 3617 * 3618 * In this flow, it is assumed that vmcs12 cache was 3619 * transferred as part of captured nVMX state and should 3620 * therefore not be read from guest memory (which may not 3621 * exist on destination host yet). 3622 */ 3623 nested_cache_shadow_vmcs12(vcpu, vmcs12); 3624 3625 switch (vmcs12->guest_activity_state) { 3626 case GUEST_ACTIVITY_HLT: 3627 /* 3628 * If we're entering a halted L2 vcpu and the L2 vcpu won't be 3629 * awakened by event injection or by an NMI-window VM-exit or 3630 * by an interrupt-window VM-exit, halt the vcpu. 3631 */ 3632 if (!(vmcs12->vm_entry_intr_info_field & INTR_INFO_VALID_MASK) && 3633 !nested_cpu_has(vmcs12, CPU_BASED_NMI_WINDOW_EXITING) && 3634 !(nested_cpu_has(vmcs12, CPU_BASED_INTR_WINDOW_EXITING) && 3635 (vmcs12->guest_rflags & X86_EFLAGS_IF))) { 3636 vmx->nested.nested_run_pending = 0; 3637 return kvm_emulate_halt_noskip(vcpu); 3638 } 3639 break; 3640 case GUEST_ACTIVITY_WAIT_SIPI: 3641 vmx->nested.nested_run_pending = 0; 3642 vcpu->arch.mp_state = KVM_MP_STATE_INIT_RECEIVED; 3643 break; 3644 default: 3645 break; 3646 } 3647 3648 return 1; 3649 3650 vmentry_failed: 3651 vmx->nested.nested_run_pending = 0; 3652 if (status == NVMX_VMENTRY_KVM_INTERNAL_ERROR) 3653 return 0; 3654 if (status == NVMX_VMENTRY_VMEXIT) 3655 return 1; 3656 WARN_ON_ONCE(status != NVMX_VMENTRY_VMFAIL); 3657 return nested_vmx_fail(vcpu, VMXERR_ENTRY_INVALID_CONTROL_FIELD); 3658 } 3659 3660 /* 3661 * On a nested exit from L2 to L1, vmcs12.guest_cr0 might not be up-to-date 3662 * because L2 may have changed some cr0 bits directly (CR0_GUEST_HOST_MASK). 3663 * This function returns the new value we should put in vmcs12.guest_cr0. 3664 * It's not enough to just return the vmcs02 GUEST_CR0. Rather, 3665 * 1. Bits that neither L0 nor L1 trapped, were set directly by L2 and are now 3666 * available in vmcs02 GUEST_CR0. (Note: It's enough to check that L0 3667 * didn't trap the bit, because if L1 did, so would L0). 3668 * 2. Bits that L1 asked to trap (and therefore L0 also did) could not have 3669 * been modified by L2, and L1 knows it. So just leave the old value of 3670 * the bit from vmcs12.guest_cr0. Note that the bit from vmcs02 GUEST_CR0 3671 * isn't relevant, because if L0 traps this bit it can set it to anything. 3672 * 3. Bits that L1 didn't trap, but L0 did. L1 believes the guest could have 3673 * changed these bits, and therefore they need to be updated, but L0 3674 * didn't necessarily allow them to be changed in GUEST_CR0 - and rather 3675 * put them in vmcs02 CR0_READ_SHADOW. So take these bits from there. 3676 */ 3677 static inline unsigned long 3678 vmcs12_guest_cr0(struct kvm_vcpu *vcpu, struct vmcs12 *vmcs12) 3679 { 3680 return 3681 /*1*/ (vmcs_readl(GUEST_CR0) & vcpu->arch.cr0_guest_owned_bits) | 3682 /*2*/ (vmcs12->guest_cr0 & vmcs12->cr0_guest_host_mask) | 3683 /*3*/ (vmcs_readl(CR0_READ_SHADOW) & ~(vmcs12->cr0_guest_host_mask | 3684 vcpu->arch.cr0_guest_owned_bits)); 3685 } 3686 3687 static inline unsigned long 3688 vmcs12_guest_cr4(struct kvm_vcpu *vcpu, struct vmcs12 *vmcs12) 3689 { 3690 return 3691 /*1*/ (vmcs_readl(GUEST_CR4) & vcpu->arch.cr4_guest_owned_bits) | 3692 /*2*/ (vmcs12->guest_cr4 & vmcs12->cr4_guest_host_mask) | 3693 /*3*/ (vmcs_readl(CR4_READ_SHADOW) & ~(vmcs12->cr4_guest_host_mask | 3694 vcpu->arch.cr4_guest_owned_bits)); 3695 } 3696 3697 static void vmcs12_save_pending_event(struct kvm_vcpu *vcpu, 3698 struct vmcs12 *vmcs12) 3699 { 3700 u32 idt_vectoring; 3701 unsigned int nr; 3702 3703 if (vcpu->arch.exception.injected) { 3704 nr = vcpu->arch.exception.nr; 3705 idt_vectoring = nr | VECTORING_INFO_VALID_MASK; 3706 3707 if (kvm_exception_is_soft(nr)) { 3708 vmcs12->vm_exit_instruction_len = 3709 vcpu->arch.event_exit_inst_len; 3710 idt_vectoring |= INTR_TYPE_SOFT_EXCEPTION; 3711 } else 3712 idt_vectoring |= INTR_TYPE_HARD_EXCEPTION; 3713 3714 if (vcpu->arch.exception.has_error_code) { 3715 idt_vectoring |= VECTORING_INFO_DELIVER_CODE_MASK; 3716 vmcs12->idt_vectoring_error_code = 3717 vcpu->arch.exception.error_code; 3718 } 3719 3720 vmcs12->idt_vectoring_info_field = idt_vectoring; 3721 } else if (vcpu->arch.nmi_injected) { 3722 vmcs12->idt_vectoring_info_field = 3723 INTR_TYPE_NMI_INTR | INTR_INFO_VALID_MASK | NMI_VECTOR; 3724 } else if (vcpu->arch.interrupt.injected) { 3725 nr = vcpu->arch.interrupt.nr; 3726 idt_vectoring = nr | VECTORING_INFO_VALID_MASK; 3727 3728 if (vcpu->arch.interrupt.soft) { 3729 idt_vectoring |= INTR_TYPE_SOFT_INTR; 3730 vmcs12->vm_entry_instruction_len = 3731 vcpu->arch.event_exit_inst_len; 3732 } else 3733 idt_vectoring |= INTR_TYPE_EXT_INTR; 3734 3735 vmcs12->idt_vectoring_info_field = idt_vectoring; 3736 } 3737 } 3738 3739 3740 void nested_mark_vmcs12_pages_dirty(struct kvm_vcpu *vcpu) 3741 { 3742 struct vmcs12 *vmcs12 = get_vmcs12(vcpu); 3743 gfn_t gfn; 3744 3745 /* 3746 * Don't need to mark the APIC access page dirty; it is never 3747 * written to by the CPU during APIC virtualization. 3748 */ 3749 3750 if (nested_cpu_has(vmcs12, CPU_BASED_TPR_SHADOW)) { 3751 gfn = vmcs12->virtual_apic_page_addr >> PAGE_SHIFT; 3752 kvm_vcpu_mark_page_dirty(vcpu, gfn); 3753 } 3754 3755 if (nested_cpu_has_posted_intr(vmcs12)) { 3756 gfn = vmcs12->posted_intr_desc_addr >> PAGE_SHIFT; 3757 kvm_vcpu_mark_page_dirty(vcpu, gfn); 3758 } 3759 } 3760 3761 static int vmx_complete_nested_posted_interrupt(struct kvm_vcpu *vcpu) 3762 { 3763 struct vcpu_vmx *vmx = to_vmx(vcpu); 3764 int max_irr; 3765 void *vapic_page; 3766 u16 status; 3767 3768 if (!vmx->nested.pi_pending) 3769 return 0; 3770 3771 if (!vmx->nested.pi_desc) 3772 goto mmio_needed; 3773 3774 vmx->nested.pi_pending = false; 3775 3776 if (!pi_test_and_clear_on(vmx->nested.pi_desc)) 3777 return 0; 3778 3779 max_irr = find_last_bit((unsigned long *)vmx->nested.pi_desc->pir, 256); 3780 if (max_irr != 256) { 3781 vapic_page = vmx->nested.virtual_apic_map.hva; 3782 if (!vapic_page) 3783 goto mmio_needed; 3784 3785 __kvm_apic_update_irr(vmx->nested.pi_desc->pir, 3786 vapic_page, &max_irr); 3787 status = vmcs_read16(GUEST_INTR_STATUS); 3788 if ((u8)max_irr > ((u8)status & 0xff)) { 3789 status &= ~0xff; 3790 status |= (u8)max_irr; 3791 vmcs_write16(GUEST_INTR_STATUS, status); 3792 } 3793 } 3794 3795 nested_mark_vmcs12_pages_dirty(vcpu); 3796 return 0; 3797 3798 mmio_needed: 3799 kvm_handle_memory_failure(vcpu, X86EMUL_IO_NEEDED, NULL); 3800 return -ENXIO; 3801 } 3802 3803 static void nested_vmx_inject_exception_vmexit(struct kvm_vcpu *vcpu, 3804 unsigned long exit_qual) 3805 { 3806 struct vmcs12 *vmcs12 = get_vmcs12(vcpu); 3807 unsigned int nr = vcpu->arch.exception.nr; 3808 u32 intr_info = nr | INTR_INFO_VALID_MASK; 3809 3810 if (vcpu->arch.exception.has_error_code) { 3811 vmcs12->vm_exit_intr_error_code = vcpu->arch.exception.error_code; 3812 intr_info |= INTR_INFO_DELIVER_CODE_MASK; 3813 } 3814 3815 if (kvm_exception_is_soft(nr)) 3816 intr_info |= INTR_TYPE_SOFT_EXCEPTION; 3817 else 3818 intr_info |= INTR_TYPE_HARD_EXCEPTION; 3819 3820 if (!(vmcs12->idt_vectoring_info_field & VECTORING_INFO_VALID_MASK) && 3821 vmx_get_nmi_mask(vcpu)) 3822 intr_info |= INTR_INFO_UNBLOCK_NMI; 3823 3824 nested_vmx_vmexit(vcpu, EXIT_REASON_EXCEPTION_NMI, intr_info, exit_qual); 3825 } 3826 3827 /* 3828 * Returns true if a debug trap is pending delivery. 3829 * 3830 * In KVM, debug traps bear an exception payload. As such, the class of a #DB 3831 * exception may be inferred from the presence of an exception payload. 3832 */ 3833 static inline bool vmx_pending_dbg_trap(struct kvm_vcpu *vcpu) 3834 { 3835 return vcpu->arch.exception.pending && 3836 vcpu->arch.exception.nr == DB_VECTOR && 3837 vcpu->arch.exception.payload; 3838 } 3839 3840 /* 3841 * Certain VM-exits set the 'pending debug exceptions' field to indicate a 3842 * recognized #DB (data or single-step) that has yet to be delivered. Since KVM 3843 * represents these debug traps with a payload that is said to be compatible 3844 * with the 'pending debug exceptions' field, write the payload to the VMCS 3845 * field if a VM-exit is delivered before the debug trap. 3846 */ 3847 static void nested_vmx_update_pending_dbg(struct kvm_vcpu *vcpu) 3848 { 3849 if (vmx_pending_dbg_trap(vcpu)) 3850 vmcs_writel(GUEST_PENDING_DBG_EXCEPTIONS, 3851 vcpu->arch.exception.payload); 3852 } 3853 3854 static bool nested_vmx_preemption_timer_pending(struct kvm_vcpu *vcpu) 3855 { 3856 return nested_cpu_has_preemption_timer(get_vmcs12(vcpu)) && 3857 to_vmx(vcpu)->nested.preemption_timer_expired; 3858 } 3859 3860 static int vmx_check_nested_events(struct kvm_vcpu *vcpu) 3861 { 3862 struct vcpu_vmx *vmx = to_vmx(vcpu); 3863 unsigned long exit_qual; 3864 bool block_nested_events = 3865 vmx->nested.nested_run_pending || kvm_event_needs_reinjection(vcpu); 3866 bool mtf_pending = vmx->nested.mtf_pending; 3867 struct kvm_lapic *apic = vcpu->arch.apic; 3868 3869 /* 3870 * Clear the MTF state. If a higher priority VM-exit is delivered first, 3871 * this state is discarded. 3872 */ 3873 if (!block_nested_events) 3874 vmx->nested.mtf_pending = false; 3875 3876 if (lapic_in_kernel(vcpu) && 3877 test_bit(KVM_APIC_INIT, &apic->pending_events)) { 3878 if (block_nested_events) 3879 return -EBUSY; 3880 nested_vmx_update_pending_dbg(vcpu); 3881 clear_bit(KVM_APIC_INIT, &apic->pending_events); 3882 if (vcpu->arch.mp_state != KVM_MP_STATE_INIT_RECEIVED) 3883 nested_vmx_vmexit(vcpu, EXIT_REASON_INIT_SIGNAL, 0, 0); 3884 return 0; 3885 } 3886 3887 if (lapic_in_kernel(vcpu) && 3888 test_bit(KVM_APIC_SIPI, &apic->pending_events)) { 3889 if (block_nested_events) 3890 return -EBUSY; 3891 3892 clear_bit(KVM_APIC_SIPI, &apic->pending_events); 3893 if (vcpu->arch.mp_state == KVM_MP_STATE_INIT_RECEIVED) 3894 nested_vmx_vmexit(vcpu, EXIT_REASON_SIPI_SIGNAL, 0, 3895 apic->sipi_vector & 0xFFUL); 3896 return 0; 3897 } 3898 3899 /* 3900 * Process any exceptions that are not debug traps before MTF. 3901 * 3902 * Note that only a pending nested run can block a pending exception. 3903 * Otherwise an injected NMI/interrupt should either be 3904 * lost or delivered to the nested hypervisor in the IDT_VECTORING_INFO, 3905 * while delivering the pending exception. 3906 */ 3907 3908 if (vcpu->arch.exception.pending && !vmx_pending_dbg_trap(vcpu)) { 3909 if (vmx->nested.nested_run_pending) 3910 return -EBUSY; 3911 if (!nested_vmx_check_exception(vcpu, &exit_qual)) 3912 goto no_vmexit; 3913 nested_vmx_inject_exception_vmexit(vcpu, exit_qual); 3914 return 0; 3915 } 3916 3917 if (mtf_pending) { 3918 if (block_nested_events) 3919 return -EBUSY; 3920 nested_vmx_update_pending_dbg(vcpu); 3921 nested_vmx_vmexit(vcpu, EXIT_REASON_MONITOR_TRAP_FLAG, 0, 0); 3922 return 0; 3923 } 3924 3925 if (vcpu->arch.exception.pending) { 3926 if (vmx->nested.nested_run_pending) 3927 return -EBUSY; 3928 if (!nested_vmx_check_exception(vcpu, &exit_qual)) 3929 goto no_vmexit; 3930 nested_vmx_inject_exception_vmexit(vcpu, exit_qual); 3931 return 0; 3932 } 3933 3934 if (nested_vmx_preemption_timer_pending(vcpu)) { 3935 if (block_nested_events) 3936 return -EBUSY; 3937 nested_vmx_vmexit(vcpu, EXIT_REASON_PREEMPTION_TIMER, 0, 0); 3938 return 0; 3939 } 3940 3941 if (vcpu->arch.smi_pending && !is_smm(vcpu)) { 3942 if (block_nested_events) 3943 return -EBUSY; 3944 goto no_vmexit; 3945 } 3946 3947 if (vcpu->arch.nmi_pending && !vmx_nmi_blocked(vcpu)) { 3948 if (block_nested_events) 3949 return -EBUSY; 3950 if (!nested_exit_on_nmi(vcpu)) 3951 goto no_vmexit; 3952 3953 nested_vmx_vmexit(vcpu, EXIT_REASON_EXCEPTION_NMI, 3954 NMI_VECTOR | INTR_TYPE_NMI_INTR | 3955 INTR_INFO_VALID_MASK, 0); 3956 /* 3957 * The NMI-triggered VM exit counts as injection: 3958 * clear this one and block further NMIs. 3959 */ 3960 vcpu->arch.nmi_pending = 0; 3961 vmx_set_nmi_mask(vcpu, true); 3962 return 0; 3963 } 3964 3965 if (kvm_cpu_has_interrupt(vcpu) && !vmx_interrupt_blocked(vcpu)) { 3966 if (block_nested_events) 3967 return -EBUSY; 3968 if (!nested_exit_on_intr(vcpu)) 3969 goto no_vmexit; 3970 nested_vmx_vmexit(vcpu, EXIT_REASON_EXTERNAL_INTERRUPT, 0, 0); 3971 return 0; 3972 } 3973 3974 no_vmexit: 3975 return vmx_complete_nested_posted_interrupt(vcpu); 3976 } 3977 3978 static u32 vmx_get_preemption_timer_value(struct kvm_vcpu *vcpu) 3979 { 3980 ktime_t remaining = 3981 hrtimer_get_remaining(&to_vmx(vcpu)->nested.preemption_timer); 3982 u64 value; 3983 3984 if (ktime_to_ns(remaining) <= 0) 3985 return 0; 3986 3987 value = ktime_to_ns(remaining) * vcpu->arch.virtual_tsc_khz; 3988 do_div(value, 1000000); 3989 return value >> VMX_MISC_EMULATED_PREEMPTION_TIMER_RATE; 3990 } 3991 3992 static bool is_vmcs12_ext_field(unsigned long field) 3993 { 3994 switch (field) { 3995 case GUEST_ES_SELECTOR: 3996 case GUEST_CS_SELECTOR: 3997 case GUEST_SS_SELECTOR: 3998 case GUEST_DS_SELECTOR: 3999 case GUEST_FS_SELECTOR: 4000 case GUEST_GS_SELECTOR: 4001 case GUEST_LDTR_SELECTOR: 4002 case GUEST_TR_SELECTOR: 4003 case GUEST_ES_LIMIT: 4004 case GUEST_CS_LIMIT: 4005 case GUEST_SS_LIMIT: 4006 case GUEST_DS_LIMIT: 4007 case GUEST_FS_LIMIT: 4008 case GUEST_GS_LIMIT: 4009 case GUEST_LDTR_LIMIT: 4010 case GUEST_TR_LIMIT: 4011 case GUEST_GDTR_LIMIT: 4012 case GUEST_IDTR_LIMIT: 4013 case GUEST_ES_AR_BYTES: 4014 case GUEST_DS_AR_BYTES: 4015 case GUEST_FS_AR_BYTES: 4016 case GUEST_GS_AR_BYTES: 4017 case GUEST_LDTR_AR_BYTES: 4018 case GUEST_TR_AR_BYTES: 4019 case GUEST_ES_BASE: 4020 case GUEST_CS_BASE: 4021 case GUEST_SS_BASE: 4022 case GUEST_DS_BASE: 4023 case GUEST_FS_BASE: 4024 case GUEST_GS_BASE: 4025 case GUEST_LDTR_BASE: 4026 case GUEST_TR_BASE: 4027 case GUEST_GDTR_BASE: 4028 case GUEST_IDTR_BASE: 4029 case GUEST_PENDING_DBG_EXCEPTIONS: 4030 case GUEST_BNDCFGS: 4031 return true; 4032 default: 4033 break; 4034 } 4035 4036 return false; 4037 } 4038 4039 static void sync_vmcs02_to_vmcs12_rare(struct kvm_vcpu *vcpu, 4040 struct vmcs12 *vmcs12) 4041 { 4042 struct vcpu_vmx *vmx = to_vmx(vcpu); 4043 4044 vmcs12->guest_es_selector = vmcs_read16(GUEST_ES_SELECTOR); 4045 vmcs12->guest_cs_selector = vmcs_read16(GUEST_CS_SELECTOR); 4046 vmcs12->guest_ss_selector = vmcs_read16(GUEST_SS_SELECTOR); 4047 vmcs12->guest_ds_selector = vmcs_read16(GUEST_DS_SELECTOR); 4048 vmcs12->guest_fs_selector = vmcs_read16(GUEST_FS_SELECTOR); 4049 vmcs12->guest_gs_selector = vmcs_read16(GUEST_GS_SELECTOR); 4050 vmcs12->guest_ldtr_selector = vmcs_read16(GUEST_LDTR_SELECTOR); 4051 vmcs12->guest_tr_selector = vmcs_read16(GUEST_TR_SELECTOR); 4052 vmcs12->guest_es_limit = vmcs_read32(GUEST_ES_LIMIT); 4053 vmcs12->guest_cs_limit = vmcs_read32(GUEST_CS_LIMIT); 4054 vmcs12->guest_ss_limit = vmcs_read32(GUEST_SS_LIMIT); 4055 vmcs12->guest_ds_limit = vmcs_read32(GUEST_DS_LIMIT); 4056 vmcs12->guest_fs_limit = vmcs_read32(GUEST_FS_LIMIT); 4057 vmcs12->guest_gs_limit = vmcs_read32(GUEST_GS_LIMIT); 4058 vmcs12->guest_ldtr_limit = vmcs_read32(GUEST_LDTR_LIMIT); 4059 vmcs12->guest_tr_limit = vmcs_read32(GUEST_TR_LIMIT); 4060 vmcs12->guest_gdtr_limit = vmcs_read32(GUEST_GDTR_LIMIT); 4061 vmcs12->guest_idtr_limit = vmcs_read32(GUEST_IDTR_LIMIT); 4062 vmcs12->guest_es_ar_bytes = vmcs_read32(GUEST_ES_AR_BYTES); 4063 vmcs12->guest_ds_ar_bytes = vmcs_read32(GUEST_DS_AR_BYTES); 4064 vmcs12->guest_fs_ar_bytes = vmcs_read32(GUEST_FS_AR_BYTES); 4065 vmcs12->guest_gs_ar_bytes = vmcs_read32(GUEST_GS_AR_BYTES); 4066 vmcs12->guest_ldtr_ar_bytes = vmcs_read32(GUEST_LDTR_AR_BYTES); 4067 vmcs12->guest_tr_ar_bytes = vmcs_read32(GUEST_TR_AR_BYTES); 4068 vmcs12->guest_es_base = vmcs_readl(GUEST_ES_BASE); 4069 vmcs12->guest_cs_base = vmcs_readl(GUEST_CS_BASE); 4070 vmcs12->guest_ss_base = vmcs_readl(GUEST_SS_BASE); 4071 vmcs12->guest_ds_base = vmcs_readl(GUEST_DS_BASE); 4072 vmcs12->guest_fs_base = vmcs_readl(GUEST_FS_BASE); 4073 vmcs12->guest_gs_base = vmcs_readl(GUEST_GS_BASE); 4074 vmcs12->guest_ldtr_base = vmcs_readl(GUEST_LDTR_BASE); 4075 vmcs12->guest_tr_base = vmcs_readl(GUEST_TR_BASE); 4076 vmcs12->guest_gdtr_base = vmcs_readl(GUEST_GDTR_BASE); 4077 vmcs12->guest_idtr_base = vmcs_readl(GUEST_IDTR_BASE); 4078 vmcs12->guest_pending_dbg_exceptions = 4079 vmcs_readl(GUEST_PENDING_DBG_EXCEPTIONS); 4080 if (kvm_mpx_supported()) 4081 vmcs12->guest_bndcfgs = vmcs_read64(GUEST_BNDCFGS); 4082 4083 vmx->nested.need_sync_vmcs02_to_vmcs12_rare = false; 4084 } 4085 4086 static void copy_vmcs02_to_vmcs12_rare(struct kvm_vcpu *vcpu, 4087 struct vmcs12 *vmcs12) 4088 { 4089 struct vcpu_vmx *vmx = to_vmx(vcpu); 4090 int cpu; 4091 4092 if (!vmx->nested.need_sync_vmcs02_to_vmcs12_rare) 4093 return; 4094 4095 4096 WARN_ON_ONCE(vmx->loaded_vmcs != &vmx->vmcs01); 4097 4098 cpu = get_cpu(); 4099 vmx->loaded_vmcs = &vmx->nested.vmcs02; 4100 vmx_vcpu_load_vmcs(vcpu, cpu, &vmx->vmcs01); 4101 4102 sync_vmcs02_to_vmcs12_rare(vcpu, vmcs12); 4103 4104 vmx->loaded_vmcs = &vmx->vmcs01; 4105 vmx_vcpu_load_vmcs(vcpu, cpu, &vmx->nested.vmcs02); 4106 put_cpu(); 4107 } 4108 4109 /* 4110 * Update the guest state fields of vmcs12 to reflect changes that 4111 * occurred while L2 was running. (The "IA-32e mode guest" bit of the 4112 * VM-entry controls is also updated, since this is really a guest 4113 * state bit.) 4114 */ 4115 static void sync_vmcs02_to_vmcs12(struct kvm_vcpu *vcpu, struct vmcs12 *vmcs12) 4116 { 4117 struct vcpu_vmx *vmx = to_vmx(vcpu); 4118 4119 if (evmptr_is_valid(vmx->nested.hv_evmcs_vmptr)) 4120 sync_vmcs02_to_vmcs12_rare(vcpu, vmcs12); 4121 4122 vmx->nested.need_sync_vmcs02_to_vmcs12_rare = 4123 !evmptr_is_valid(vmx->nested.hv_evmcs_vmptr); 4124 4125 vmcs12->guest_cr0 = vmcs12_guest_cr0(vcpu, vmcs12); 4126 vmcs12->guest_cr4 = vmcs12_guest_cr4(vcpu, vmcs12); 4127 4128 vmcs12->guest_rsp = kvm_rsp_read(vcpu); 4129 vmcs12->guest_rip = kvm_rip_read(vcpu); 4130 vmcs12->guest_rflags = vmcs_readl(GUEST_RFLAGS); 4131 4132 vmcs12->guest_cs_ar_bytes = vmcs_read32(GUEST_CS_AR_BYTES); 4133 vmcs12->guest_ss_ar_bytes = vmcs_read32(GUEST_SS_AR_BYTES); 4134 4135 vmcs12->guest_interruptibility_info = 4136 vmcs_read32(GUEST_INTERRUPTIBILITY_INFO); 4137 4138 if (vcpu->arch.mp_state == KVM_MP_STATE_HALTED) 4139 vmcs12->guest_activity_state = GUEST_ACTIVITY_HLT; 4140 else if (vcpu->arch.mp_state == KVM_MP_STATE_INIT_RECEIVED) 4141 vmcs12->guest_activity_state = GUEST_ACTIVITY_WAIT_SIPI; 4142 else 4143 vmcs12->guest_activity_state = GUEST_ACTIVITY_ACTIVE; 4144 4145 if (nested_cpu_has_preemption_timer(vmcs12) && 4146 vmcs12->vm_exit_controls & VM_EXIT_SAVE_VMX_PREEMPTION_TIMER && 4147 !vmx->nested.nested_run_pending) 4148 vmcs12->vmx_preemption_timer_value = 4149 vmx_get_preemption_timer_value(vcpu); 4150 4151 /* 4152 * In some cases (usually, nested EPT), L2 is allowed to change its 4153 * own CR3 without exiting. If it has changed it, we must keep it. 4154 * Of course, if L0 is using shadow page tables, GUEST_CR3 was defined 4155 * by L0, not L1 or L2, so we mustn't unconditionally copy it to vmcs12. 4156 * 4157 * Additionally, restore L2's PDPTR to vmcs12. 4158 */ 4159 if (enable_ept) { 4160 vmcs12->guest_cr3 = vmcs_readl(GUEST_CR3); 4161 if (nested_cpu_has_ept(vmcs12) && is_pae_paging(vcpu)) { 4162 vmcs12->guest_pdptr0 = vmcs_read64(GUEST_PDPTR0); 4163 vmcs12->guest_pdptr1 = vmcs_read64(GUEST_PDPTR1); 4164 vmcs12->guest_pdptr2 = vmcs_read64(GUEST_PDPTR2); 4165 vmcs12->guest_pdptr3 = vmcs_read64(GUEST_PDPTR3); 4166 } 4167 } 4168 4169 vmcs12->guest_linear_address = vmcs_readl(GUEST_LINEAR_ADDRESS); 4170 4171 if (nested_cpu_has_vid(vmcs12)) 4172 vmcs12->guest_intr_status = vmcs_read16(GUEST_INTR_STATUS); 4173 4174 vmcs12->vm_entry_controls = 4175 (vmcs12->vm_entry_controls & ~VM_ENTRY_IA32E_MODE) | 4176 (vm_entry_controls_get(to_vmx(vcpu)) & VM_ENTRY_IA32E_MODE); 4177 4178 if (vmcs12->vm_exit_controls & VM_EXIT_SAVE_DEBUG_CONTROLS) 4179 kvm_get_dr(vcpu, 7, (unsigned long *)&vmcs12->guest_dr7); 4180 4181 if (vmcs12->vm_exit_controls & VM_EXIT_SAVE_IA32_EFER) 4182 vmcs12->guest_ia32_efer = vcpu->arch.efer; 4183 } 4184 4185 /* 4186 * prepare_vmcs12 is part of what we need to do when the nested L2 guest exits 4187 * and we want to prepare to run its L1 parent. L1 keeps a vmcs for L2 (vmcs12), 4188 * and this function updates it to reflect the changes to the guest state while 4189 * L2 was running (and perhaps made some exits which were handled directly by L0 4190 * without going back to L1), and to reflect the exit reason. 4191 * Note that we do not have to copy here all VMCS fields, just those that 4192 * could have changed by the L2 guest or the exit - i.e., the guest-state and 4193 * exit-information fields only. Other fields are modified by L1 with VMWRITE, 4194 * which already writes to vmcs12 directly. 4195 */ 4196 static void prepare_vmcs12(struct kvm_vcpu *vcpu, struct vmcs12 *vmcs12, 4197 u32 vm_exit_reason, u32 exit_intr_info, 4198 unsigned long exit_qualification) 4199 { 4200 /* update exit information fields: */ 4201 vmcs12->vm_exit_reason = vm_exit_reason; 4202 if (to_vmx(vcpu)->exit_reason.enclave_mode) 4203 vmcs12->vm_exit_reason |= VMX_EXIT_REASONS_SGX_ENCLAVE_MODE; 4204 vmcs12->exit_qualification = exit_qualification; 4205 vmcs12->vm_exit_intr_info = exit_intr_info; 4206 4207 vmcs12->idt_vectoring_info_field = 0; 4208 vmcs12->vm_exit_instruction_len = vmcs_read32(VM_EXIT_INSTRUCTION_LEN); 4209 vmcs12->vmx_instruction_info = vmcs_read32(VMX_INSTRUCTION_INFO); 4210 4211 if (!(vmcs12->vm_exit_reason & VMX_EXIT_REASONS_FAILED_VMENTRY)) { 4212 vmcs12->launch_state = 1; 4213 4214 /* vm_entry_intr_info_field is cleared on exit. Emulate this 4215 * instead of reading the real value. */ 4216 vmcs12->vm_entry_intr_info_field &= ~INTR_INFO_VALID_MASK; 4217 4218 /* 4219 * Transfer the event that L0 or L1 may wanted to inject into 4220 * L2 to IDT_VECTORING_INFO_FIELD. 4221 */ 4222 vmcs12_save_pending_event(vcpu, vmcs12); 4223 4224 /* 4225 * According to spec, there's no need to store the guest's 4226 * MSRs if the exit is due to a VM-entry failure that occurs 4227 * during or after loading the guest state. Since this exit 4228 * does not fall in that category, we need to save the MSRs. 4229 */ 4230 if (nested_vmx_store_msr(vcpu, 4231 vmcs12->vm_exit_msr_store_addr, 4232 vmcs12->vm_exit_msr_store_count)) 4233 nested_vmx_abort(vcpu, 4234 VMX_ABORT_SAVE_GUEST_MSR_FAIL); 4235 } 4236 4237 /* 4238 * Drop what we picked up for L2 via vmx_complete_interrupts. It is 4239 * preserved above and would only end up incorrectly in L1. 4240 */ 4241 vcpu->arch.nmi_injected = false; 4242 kvm_clear_exception_queue(vcpu); 4243 kvm_clear_interrupt_queue(vcpu); 4244 } 4245 4246 /* 4247 * A part of what we need to when the nested L2 guest exits and we want to 4248 * run its L1 parent, is to reset L1's guest state to the host state specified 4249 * in vmcs12. 4250 * This function is to be called not only on normal nested exit, but also on 4251 * a nested entry failure, as explained in Intel's spec, 3B.23.7 ("VM-Entry 4252 * Failures During or After Loading Guest State"). 4253 * This function should be called when the active VMCS is L1's (vmcs01). 4254 */ 4255 static void load_vmcs12_host_state(struct kvm_vcpu *vcpu, 4256 struct vmcs12 *vmcs12) 4257 { 4258 enum vm_entry_failure_code ignored; 4259 struct kvm_segment seg; 4260 4261 if (vmcs12->vm_exit_controls & VM_EXIT_LOAD_IA32_EFER) 4262 vcpu->arch.efer = vmcs12->host_ia32_efer; 4263 else if (vmcs12->vm_exit_controls & VM_EXIT_HOST_ADDR_SPACE_SIZE) 4264 vcpu->arch.efer |= (EFER_LMA | EFER_LME); 4265 else 4266 vcpu->arch.efer &= ~(EFER_LMA | EFER_LME); 4267 vmx_set_efer(vcpu, vcpu->arch.efer); 4268 4269 kvm_rsp_write(vcpu, vmcs12->host_rsp); 4270 kvm_rip_write(vcpu, vmcs12->host_rip); 4271 vmx_set_rflags(vcpu, X86_EFLAGS_FIXED); 4272 vmx_set_interrupt_shadow(vcpu, 0); 4273 4274 /* 4275 * Note that calling vmx_set_cr0 is important, even if cr0 hasn't 4276 * actually changed, because vmx_set_cr0 refers to efer set above. 4277 * 4278 * CR0_GUEST_HOST_MASK is already set in the original vmcs01 4279 * (KVM doesn't change it); 4280 */ 4281 vcpu->arch.cr0_guest_owned_bits = KVM_POSSIBLE_CR0_GUEST_BITS; 4282 vmx_set_cr0(vcpu, vmcs12->host_cr0); 4283 4284 /* Same as above - no reason to call set_cr4_guest_host_mask(). */ 4285 vcpu->arch.cr4_guest_owned_bits = ~vmcs_readl(CR4_GUEST_HOST_MASK); 4286 vmx_set_cr4(vcpu, vmcs12->host_cr4); 4287 4288 nested_ept_uninit_mmu_context(vcpu); 4289 4290 /* 4291 * Only PDPTE load can fail as the value of cr3 was checked on entry and 4292 * couldn't have changed. 4293 */ 4294 if (nested_vmx_load_cr3(vcpu, vmcs12->host_cr3, false, true, &ignored)) 4295 nested_vmx_abort(vcpu, VMX_ABORT_LOAD_HOST_PDPTE_FAIL); 4296 4297 nested_vmx_transition_tlb_flush(vcpu, vmcs12, false); 4298 4299 vmcs_write32(GUEST_SYSENTER_CS, vmcs12->host_ia32_sysenter_cs); 4300 vmcs_writel(GUEST_SYSENTER_ESP, vmcs12->host_ia32_sysenter_esp); 4301 vmcs_writel(GUEST_SYSENTER_EIP, vmcs12->host_ia32_sysenter_eip); 4302 vmcs_writel(GUEST_IDTR_BASE, vmcs12->host_idtr_base); 4303 vmcs_writel(GUEST_GDTR_BASE, vmcs12->host_gdtr_base); 4304 vmcs_write32(GUEST_IDTR_LIMIT, 0xFFFF); 4305 vmcs_write32(GUEST_GDTR_LIMIT, 0xFFFF); 4306 4307 /* If not VM_EXIT_CLEAR_BNDCFGS, the L2 value propagates to L1. */ 4308 if (vmcs12->vm_exit_controls & VM_EXIT_CLEAR_BNDCFGS) 4309 vmcs_write64(GUEST_BNDCFGS, 0); 4310 4311 if (vmcs12->vm_exit_controls & VM_EXIT_LOAD_IA32_PAT) { 4312 vmcs_write64(GUEST_IA32_PAT, vmcs12->host_ia32_pat); 4313 vcpu->arch.pat = vmcs12->host_ia32_pat; 4314 } 4315 if (vmcs12->vm_exit_controls & VM_EXIT_LOAD_IA32_PERF_GLOBAL_CTRL) 4316 WARN_ON_ONCE(kvm_set_msr(vcpu, MSR_CORE_PERF_GLOBAL_CTRL, 4317 vmcs12->host_ia32_perf_global_ctrl)); 4318 4319 /* Set L1 segment info according to Intel SDM 4320 27.5.2 Loading Host Segment and Descriptor-Table Registers */ 4321 seg = (struct kvm_segment) { 4322 .base = 0, 4323 .limit = 0xFFFFFFFF, 4324 .selector = vmcs12->host_cs_selector, 4325 .type = 11, 4326 .present = 1, 4327 .s = 1, 4328 .g = 1 4329 }; 4330 if (vmcs12->vm_exit_controls & VM_EXIT_HOST_ADDR_SPACE_SIZE) 4331 seg.l = 1; 4332 else 4333 seg.db = 1; 4334 __vmx_set_segment(vcpu, &seg, VCPU_SREG_CS); 4335 seg = (struct kvm_segment) { 4336 .base = 0, 4337 .limit = 0xFFFFFFFF, 4338 .type = 3, 4339 .present = 1, 4340 .s = 1, 4341 .db = 1, 4342 .g = 1 4343 }; 4344 seg.selector = vmcs12->host_ds_selector; 4345 __vmx_set_segment(vcpu, &seg, VCPU_SREG_DS); 4346 seg.selector = vmcs12->host_es_selector; 4347 __vmx_set_segment(vcpu, &seg, VCPU_SREG_ES); 4348 seg.selector = vmcs12->host_ss_selector; 4349 __vmx_set_segment(vcpu, &seg, VCPU_SREG_SS); 4350 seg.selector = vmcs12->host_fs_selector; 4351 seg.base = vmcs12->host_fs_base; 4352 __vmx_set_segment(vcpu, &seg, VCPU_SREG_FS); 4353 seg.selector = vmcs12->host_gs_selector; 4354 seg.base = vmcs12->host_gs_base; 4355 __vmx_set_segment(vcpu, &seg, VCPU_SREG_GS); 4356 seg = (struct kvm_segment) { 4357 .base = vmcs12->host_tr_base, 4358 .limit = 0x67, 4359 .selector = vmcs12->host_tr_selector, 4360 .type = 11, 4361 .present = 1 4362 }; 4363 __vmx_set_segment(vcpu, &seg, VCPU_SREG_TR); 4364 4365 memset(&seg, 0, sizeof(seg)); 4366 seg.unusable = 1; 4367 __vmx_set_segment(vcpu, &seg, VCPU_SREG_LDTR); 4368 4369 kvm_set_dr(vcpu, 7, 0x400); 4370 vmcs_write64(GUEST_IA32_DEBUGCTL, 0); 4371 4372 if (nested_vmx_load_msr(vcpu, vmcs12->vm_exit_msr_load_addr, 4373 vmcs12->vm_exit_msr_load_count)) 4374 nested_vmx_abort(vcpu, VMX_ABORT_LOAD_HOST_MSR_FAIL); 4375 4376 to_vmx(vcpu)->emulation_required = vmx_emulation_required(vcpu); 4377 } 4378 4379 static inline u64 nested_vmx_get_vmcs01_guest_efer(struct vcpu_vmx *vmx) 4380 { 4381 struct vmx_uret_msr *efer_msr; 4382 unsigned int i; 4383 4384 if (vm_entry_controls_get(vmx) & VM_ENTRY_LOAD_IA32_EFER) 4385 return vmcs_read64(GUEST_IA32_EFER); 4386 4387 if (cpu_has_load_ia32_efer()) 4388 return host_efer; 4389 4390 for (i = 0; i < vmx->msr_autoload.guest.nr; ++i) { 4391 if (vmx->msr_autoload.guest.val[i].index == MSR_EFER) 4392 return vmx->msr_autoload.guest.val[i].value; 4393 } 4394 4395 efer_msr = vmx_find_uret_msr(vmx, MSR_EFER); 4396 if (efer_msr) 4397 return efer_msr->data; 4398 4399 return host_efer; 4400 } 4401 4402 static void nested_vmx_restore_host_state(struct kvm_vcpu *vcpu) 4403 { 4404 struct vmcs12 *vmcs12 = get_vmcs12(vcpu); 4405 struct vcpu_vmx *vmx = to_vmx(vcpu); 4406 struct vmx_msr_entry g, h; 4407 gpa_t gpa; 4408 u32 i, j; 4409 4410 vcpu->arch.pat = vmcs_read64(GUEST_IA32_PAT); 4411 4412 if (vmcs12->vm_entry_controls & VM_ENTRY_LOAD_DEBUG_CONTROLS) { 4413 /* 4414 * L1's host DR7 is lost if KVM_GUESTDBG_USE_HW_BP is set 4415 * as vmcs01.GUEST_DR7 contains a userspace defined value 4416 * and vcpu->arch.dr7 is not squirreled away before the 4417 * nested VMENTER (not worth adding a variable in nested_vmx). 4418 */ 4419 if (vcpu->guest_debug & KVM_GUESTDBG_USE_HW_BP) 4420 kvm_set_dr(vcpu, 7, DR7_FIXED_1); 4421 else 4422 WARN_ON(kvm_set_dr(vcpu, 7, vmcs_readl(GUEST_DR7))); 4423 } 4424 4425 /* 4426 * Note that calling vmx_set_{efer,cr0,cr4} is important as they 4427 * handle a variety of side effects to KVM's software model. 4428 */ 4429 vmx_set_efer(vcpu, nested_vmx_get_vmcs01_guest_efer(vmx)); 4430 4431 vcpu->arch.cr0_guest_owned_bits = KVM_POSSIBLE_CR0_GUEST_BITS; 4432 vmx_set_cr0(vcpu, vmcs_readl(CR0_READ_SHADOW)); 4433 4434 vcpu->arch.cr4_guest_owned_bits = ~vmcs_readl(CR4_GUEST_HOST_MASK); 4435 vmx_set_cr4(vcpu, vmcs_readl(CR4_READ_SHADOW)); 4436 4437 nested_ept_uninit_mmu_context(vcpu); 4438 vcpu->arch.cr3 = vmcs_readl(GUEST_CR3); 4439 kvm_register_mark_available(vcpu, VCPU_EXREG_CR3); 4440 4441 /* 4442 * Use ept_save_pdptrs(vcpu) to load the MMU's cached PDPTRs 4443 * from vmcs01 (if necessary). The PDPTRs are not loaded on 4444 * VMFail, like everything else we just need to ensure our 4445 * software model is up-to-date. 4446 */ 4447 if (enable_ept && is_pae_paging(vcpu)) 4448 ept_save_pdptrs(vcpu); 4449 4450 kvm_mmu_reset_context(vcpu); 4451 4452 /* 4453 * This nasty bit of open coding is a compromise between blindly 4454 * loading L1's MSRs using the exit load lists (incorrect emulation 4455 * of VMFail), leaving the nested VM's MSRs in the software model 4456 * (incorrect behavior) and snapshotting the modified MSRs (too 4457 * expensive since the lists are unbound by hardware). For each 4458 * MSR that was (prematurely) loaded from the nested VMEntry load 4459 * list, reload it from the exit load list if it exists and differs 4460 * from the guest value. The intent is to stuff host state as 4461 * silently as possible, not to fully process the exit load list. 4462 */ 4463 for (i = 0; i < vmcs12->vm_entry_msr_load_count; i++) { 4464 gpa = vmcs12->vm_entry_msr_load_addr + (i * sizeof(g)); 4465 if (kvm_vcpu_read_guest(vcpu, gpa, &g, sizeof(g))) { 4466 pr_debug_ratelimited( 4467 "%s read MSR index failed (%u, 0x%08llx)\n", 4468 __func__, i, gpa); 4469 goto vmabort; 4470 } 4471 4472 for (j = 0; j < vmcs12->vm_exit_msr_load_count; j++) { 4473 gpa = vmcs12->vm_exit_msr_load_addr + (j * sizeof(h)); 4474 if (kvm_vcpu_read_guest(vcpu, gpa, &h, sizeof(h))) { 4475 pr_debug_ratelimited( 4476 "%s read MSR failed (%u, 0x%08llx)\n", 4477 __func__, j, gpa); 4478 goto vmabort; 4479 } 4480 if (h.index != g.index) 4481 continue; 4482 if (h.value == g.value) 4483 break; 4484 4485 if (nested_vmx_load_msr_check(vcpu, &h)) { 4486 pr_debug_ratelimited( 4487 "%s check failed (%u, 0x%x, 0x%x)\n", 4488 __func__, j, h.index, h.reserved); 4489 goto vmabort; 4490 } 4491 4492 if (kvm_set_msr(vcpu, h.index, h.value)) { 4493 pr_debug_ratelimited( 4494 "%s WRMSR failed (%u, 0x%x, 0x%llx)\n", 4495 __func__, j, h.index, h.value); 4496 goto vmabort; 4497 } 4498 } 4499 } 4500 4501 return; 4502 4503 vmabort: 4504 nested_vmx_abort(vcpu, VMX_ABORT_LOAD_HOST_MSR_FAIL); 4505 } 4506 4507 /* 4508 * Emulate an exit from nested guest (L2) to L1, i.e., prepare to run L1 4509 * and modify vmcs12 to make it see what it would expect to see there if 4510 * L2 was its real guest. Must only be called when in L2 (is_guest_mode()) 4511 */ 4512 void nested_vmx_vmexit(struct kvm_vcpu *vcpu, u32 vm_exit_reason, 4513 u32 exit_intr_info, unsigned long exit_qualification) 4514 { 4515 struct vcpu_vmx *vmx = to_vmx(vcpu); 4516 struct vmcs12 *vmcs12 = get_vmcs12(vcpu); 4517 4518 /* trying to cancel vmlaunch/vmresume is a bug */ 4519 WARN_ON_ONCE(vmx->nested.nested_run_pending); 4520 4521 /* Similarly, triple faults in L2 should never escape. */ 4522 WARN_ON_ONCE(kvm_check_request(KVM_REQ_TRIPLE_FAULT, vcpu)); 4523 4524 if (kvm_check_request(KVM_REQ_GET_NESTED_STATE_PAGES, vcpu)) { 4525 /* 4526 * KVM_REQ_GET_NESTED_STATE_PAGES is also used to map 4527 * Enlightened VMCS after migration and we still need to 4528 * do that when something is forcing L2->L1 exit prior to 4529 * the first L2 run. 4530 */ 4531 (void)nested_get_evmcs_page(vcpu); 4532 } 4533 4534 /* Service pending TLB flush requests for L2 before switching to L1. */ 4535 kvm_service_local_tlb_flush_requests(vcpu); 4536 4537 /* 4538 * VCPU_EXREG_PDPTR will be clobbered in arch/x86/kvm/vmx/vmx.h between 4539 * now and the new vmentry. Ensure that the VMCS02 PDPTR fields are 4540 * up-to-date before switching to L1. 4541 */ 4542 if (enable_ept && is_pae_paging(vcpu)) 4543 vmx_ept_load_pdptrs(vcpu); 4544 4545 leave_guest_mode(vcpu); 4546 4547 if (nested_cpu_has_preemption_timer(vmcs12)) 4548 hrtimer_cancel(&to_vmx(vcpu)->nested.preemption_timer); 4549 4550 if (nested_cpu_has(vmcs12, CPU_BASED_USE_TSC_OFFSETTING)) { 4551 vcpu->arch.tsc_offset = vcpu->arch.l1_tsc_offset; 4552 if (nested_cpu_has2(vmcs12, SECONDARY_EXEC_TSC_SCALING)) 4553 vcpu->arch.tsc_scaling_ratio = vcpu->arch.l1_tsc_scaling_ratio; 4554 } 4555 4556 if (likely(!vmx->fail)) { 4557 sync_vmcs02_to_vmcs12(vcpu, vmcs12); 4558 4559 if (vm_exit_reason != -1) 4560 prepare_vmcs12(vcpu, vmcs12, vm_exit_reason, 4561 exit_intr_info, exit_qualification); 4562 4563 /* 4564 * Must happen outside of sync_vmcs02_to_vmcs12() as it will 4565 * also be used to capture vmcs12 cache as part of 4566 * capturing nVMX state for snapshot (migration). 4567 * 4568 * Otherwise, this flush will dirty guest memory at a 4569 * point it is already assumed by user-space to be 4570 * immutable. 4571 */ 4572 nested_flush_cached_shadow_vmcs12(vcpu, vmcs12); 4573 } else { 4574 /* 4575 * The only expected VM-instruction error is "VM entry with 4576 * invalid control field(s)." Anything else indicates a 4577 * problem with L0. And we should never get here with a 4578 * VMFail of any type if early consistency checks are enabled. 4579 */ 4580 WARN_ON_ONCE(vmcs_read32(VM_INSTRUCTION_ERROR) != 4581 VMXERR_ENTRY_INVALID_CONTROL_FIELD); 4582 WARN_ON_ONCE(nested_early_check); 4583 } 4584 4585 vmx_switch_vmcs(vcpu, &vmx->vmcs01); 4586 4587 /* Update any VMCS fields that might have changed while L2 ran */ 4588 vmcs_write32(VM_EXIT_MSR_LOAD_COUNT, vmx->msr_autoload.host.nr); 4589 vmcs_write32(VM_ENTRY_MSR_LOAD_COUNT, vmx->msr_autoload.guest.nr); 4590 vmcs_write64(TSC_OFFSET, vcpu->arch.tsc_offset); 4591 if (kvm_has_tsc_control) 4592 vmcs_write64(TSC_MULTIPLIER, vcpu->arch.tsc_scaling_ratio); 4593 4594 if (vmx->nested.l1_tpr_threshold != -1) 4595 vmcs_write32(TPR_THRESHOLD, vmx->nested.l1_tpr_threshold); 4596 4597 if (vmx->nested.change_vmcs01_virtual_apic_mode) { 4598 vmx->nested.change_vmcs01_virtual_apic_mode = false; 4599 vmx_set_virtual_apic_mode(vcpu); 4600 } 4601 4602 if (vmx->nested.update_vmcs01_cpu_dirty_logging) { 4603 vmx->nested.update_vmcs01_cpu_dirty_logging = false; 4604 vmx_update_cpu_dirty_logging(vcpu); 4605 } 4606 4607 /* Unpin physical memory we referred to in vmcs02 */ 4608 if (vmx->nested.apic_access_page) { 4609 kvm_release_page_clean(vmx->nested.apic_access_page); 4610 vmx->nested.apic_access_page = NULL; 4611 } 4612 kvm_vcpu_unmap(vcpu, &vmx->nested.virtual_apic_map, true); 4613 kvm_vcpu_unmap(vcpu, &vmx->nested.pi_desc_map, true); 4614 vmx->nested.pi_desc = NULL; 4615 4616 if (vmx->nested.reload_vmcs01_apic_access_page) { 4617 vmx->nested.reload_vmcs01_apic_access_page = false; 4618 kvm_make_request(KVM_REQ_APIC_PAGE_RELOAD, vcpu); 4619 } 4620 4621 if ((vm_exit_reason != -1) && 4622 (enable_shadow_vmcs || evmptr_is_valid(vmx->nested.hv_evmcs_vmptr))) 4623 vmx->nested.need_vmcs12_to_shadow_sync = true; 4624 4625 /* in case we halted in L2 */ 4626 vcpu->arch.mp_state = KVM_MP_STATE_RUNNABLE; 4627 4628 if (likely(!vmx->fail)) { 4629 if ((u16)vm_exit_reason == EXIT_REASON_EXTERNAL_INTERRUPT && 4630 nested_exit_intr_ack_set(vcpu)) { 4631 int irq = kvm_cpu_get_interrupt(vcpu); 4632 WARN_ON(irq < 0); 4633 vmcs12->vm_exit_intr_info = irq | 4634 INTR_INFO_VALID_MASK | INTR_TYPE_EXT_INTR; 4635 } 4636 4637 if (vm_exit_reason != -1) 4638 trace_kvm_nested_vmexit_inject(vmcs12->vm_exit_reason, 4639 vmcs12->exit_qualification, 4640 vmcs12->idt_vectoring_info_field, 4641 vmcs12->vm_exit_intr_info, 4642 vmcs12->vm_exit_intr_error_code, 4643 KVM_ISA_VMX); 4644 4645 load_vmcs12_host_state(vcpu, vmcs12); 4646 4647 return; 4648 } 4649 4650 /* 4651 * After an early L2 VM-entry failure, we're now back 4652 * in L1 which thinks it just finished a VMLAUNCH or 4653 * VMRESUME instruction, so we need to set the failure 4654 * flag and the VM-instruction error field of the VMCS 4655 * accordingly, and skip the emulated instruction. 4656 */ 4657 (void)nested_vmx_fail(vcpu, VMXERR_ENTRY_INVALID_CONTROL_FIELD); 4658 4659 /* 4660 * Restore L1's host state to KVM's software model. We're here 4661 * because a consistency check was caught by hardware, which 4662 * means some amount of guest state has been propagated to KVM's 4663 * model and needs to be unwound to the host's state. 4664 */ 4665 nested_vmx_restore_host_state(vcpu); 4666 4667 vmx->fail = 0; 4668 } 4669 4670 static void nested_vmx_triple_fault(struct kvm_vcpu *vcpu) 4671 { 4672 nested_vmx_vmexit(vcpu, EXIT_REASON_TRIPLE_FAULT, 0, 0); 4673 } 4674 4675 /* 4676 * Decode the memory-address operand of a vmx instruction, as recorded on an 4677 * exit caused by such an instruction (run by a guest hypervisor). 4678 * On success, returns 0. When the operand is invalid, returns 1 and throws 4679 * #UD, #GP, or #SS. 4680 */ 4681 int get_vmx_mem_address(struct kvm_vcpu *vcpu, unsigned long exit_qualification, 4682 u32 vmx_instruction_info, bool wr, int len, gva_t *ret) 4683 { 4684 gva_t off; 4685 bool exn; 4686 struct kvm_segment s; 4687 4688 /* 4689 * According to Vol. 3B, "Information for VM Exits Due to Instruction 4690 * Execution", on an exit, vmx_instruction_info holds most of the 4691 * addressing components of the operand. Only the displacement part 4692 * is put in exit_qualification (see 3B, "Basic VM-Exit Information"). 4693 * For how an actual address is calculated from all these components, 4694 * refer to Vol. 1, "Operand Addressing". 4695 */ 4696 int scaling = vmx_instruction_info & 3; 4697 int addr_size = (vmx_instruction_info >> 7) & 7; 4698 bool is_reg = vmx_instruction_info & (1u << 10); 4699 int seg_reg = (vmx_instruction_info >> 15) & 7; 4700 int index_reg = (vmx_instruction_info >> 18) & 0xf; 4701 bool index_is_valid = !(vmx_instruction_info & (1u << 22)); 4702 int base_reg = (vmx_instruction_info >> 23) & 0xf; 4703 bool base_is_valid = !(vmx_instruction_info & (1u << 27)); 4704 4705 if (is_reg) { 4706 kvm_queue_exception(vcpu, UD_VECTOR); 4707 return 1; 4708 } 4709 4710 /* Addr = segment_base + offset */ 4711 /* offset = base + [index * scale] + displacement */ 4712 off = exit_qualification; /* holds the displacement */ 4713 if (addr_size == 1) 4714 off = (gva_t)sign_extend64(off, 31); 4715 else if (addr_size == 0) 4716 off = (gva_t)sign_extend64(off, 15); 4717 if (base_is_valid) 4718 off += kvm_register_read(vcpu, base_reg); 4719 if (index_is_valid) 4720 off += kvm_register_read(vcpu, index_reg) << scaling; 4721 vmx_get_segment(vcpu, &s, seg_reg); 4722 4723 /* 4724 * The effective address, i.e. @off, of a memory operand is truncated 4725 * based on the address size of the instruction. Note that this is 4726 * the *effective address*, i.e. the address prior to accounting for 4727 * the segment's base. 4728 */ 4729 if (addr_size == 1) /* 32 bit */ 4730 off &= 0xffffffff; 4731 else if (addr_size == 0) /* 16 bit */ 4732 off &= 0xffff; 4733 4734 /* Checks for #GP/#SS exceptions. */ 4735 exn = false; 4736 if (is_long_mode(vcpu)) { 4737 /* 4738 * The virtual/linear address is never truncated in 64-bit 4739 * mode, e.g. a 32-bit address size can yield a 64-bit virtual 4740 * address when using FS/GS with a non-zero base. 4741 */ 4742 if (seg_reg == VCPU_SREG_FS || seg_reg == VCPU_SREG_GS) 4743 *ret = s.base + off; 4744 else 4745 *ret = off; 4746 4747 /* Long mode: #GP(0)/#SS(0) if the memory address is in a 4748 * non-canonical form. This is the only check on the memory 4749 * destination for long mode! 4750 */ 4751 exn = is_noncanonical_address(*ret, vcpu); 4752 } else { 4753 /* 4754 * When not in long mode, the virtual/linear address is 4755 * unconditionally truncated to 32 bits regardless of the 4756 * address size. 4757 */ 4758 *ret = (s.base + off) & 0xffffffff; 4759 4760 /* Protected mode: apply checks for segment validity in the 4761 * following order: 4762 * - segment type check (#GP(0) may be thrown) 4763 * - usability check (#GP(0)/#SS(0)) 4764 * - limit check (#GP(0)/#SS(0)) 4765 */ 4766 if (wr) 4767 /* #GP(0) if the destination operand is located in a 4768 * read-only data segment or any code segment. 4769 */ 4770 exn = ((s.type & 0xa) == 0 || (s.type & 8)); 4771 else 4772 /* #GP(0) if the source operand is located in an 4773 * execute-only code segment 4774 */ 4775 exn = ((s.type & 0xa) == 8); 4776 if (exn) { 4777 kvm_queue_exception_e(vcpu, GP_VECTOR, 0); 4778 return 1; 4779 } 4780 /* Protected mode: #GP(0)/#SS(0) if the segment is unusable. 4781 */ 4782 exn = (s.unusable != 0); 4783 4784 /* 4785 * Protected mode: #GP(0)/#SS(0) if the memory operand is 4786 * outside the segment limit. All CPUs that support VMX ignore 4787 * limit checks for flat segments, i.e. segments with base==0, 4788 * limit==0xffffffff and of type expand-up data or code. 4789 */ 4790 if (!(s.base == 0 && s.limit == 0xffffffff && 4791 ((s.type & 8) || !(s.type & 4)))) 4792 exn = exn || ((u64)off + len - 1 > s.limit); 4793 } 4794 if (exn) { 4795 kvm_queue_exception_e(vcpu, 4796 seg_reg == VCPU_SREG_SS ? 4797 SS_VECTOR : GP_VECTOR, 4798 0); 4799 return 1; 4800 } 4801 4802 return 0; 4803 } 4804 4805 void nested_vmx_pmu_refresh(struct kvm_vcpu *vcpu, 4806 bool vcpu_has_perf_global_ctrl) 4807 { 4808 struct vcpu_vmx *vmx; 4809 4810 if (!nested_vmx_allowed(vcpu)) 4811 return; 4812 4813 vmx = to_vmx(vcpu); 4814 if (vcpu_has_perf_global_ctrl) { 4815 vmx->nested.msrs.entry_ctls_high |= 4816 VM_ENTRY_LOAD_IA32_PERF_GLOBAL_CTRL; 4817 vmx->nested.msrs.exit_ctls_high |= 4818 VM_EXIT_LOAD_IA32_PERF_GLOBAL_CTRL; 4819 } else { 4820 vmx->nested.msrs.entry_ctls_high &= 4821 ~VM_ENTRY_LOAD_IA32_PERF_GLOBAL_CTRL; 4822 vmx->nested.msrs.exit_ctls_high &= 4823 ~VM_EXIT_LOAD_IA32_PERF_GLOBAL_CTRL; 4824 } 4825 } 4826 4827 static int nested_vmx_get_vmptr(struct kvm_vcpu *vcpu, gpa_t *vmpointer, 4828 int *ret) 4829 { 4830 gva_t gva; 4831 struct x86_exception e; 4832 int r; 4833 4834 if (get_vmx_mem_address(vcpu, vmx_get_exit_qual(vcpu), 4835 vmcs_read32(VMX_INSTRUCTION_INFO), false, 4836 sizeof(*vmpointer), &gva)) { 4837 *ret = 1; 4838 return -EINVAL; 4839 } 4840 4841 r = kvm_read_guest_virt(vcpu, gva, vmpointer, sizeof(*vmpointer), &e); 4842 if (r != X86EMUL_CONTINUE) { 4843 *ret = kvm_handle_memory_failure(vcpu, r, &e); 4844 return -EINVAL; 4845 } 4846 4847 return 0; 4848 } 4849 4850 /* 4851 * Allocate a shadow VMCS and associate it with the currently loaded 4852 * VMCS, unless such a shadow VMCS already exists. The newly allocated 4853 * VMCS is also VMCLEARed, so that it is ready for use. 4854 */ 4855 static struct vmcs *alloc_shadow_vmcs(struct kvm_vcpu *vcpu) 4856 { 4857 struct vcpu_vmx *vmx = to_vmx(vcpu); 4858 struct loaded_vmcs *loaded_vmcs = vmx->loaded_vmcs; 4859 4860 /* 4861 * KVM allocates a shadow VMCS only when L1 executes VMXON and frees it 4862 * when L1 executes VMXOFF or the vCPU is forced out of nested 4863 * operation. VMXON faults if the CPU is already post-VMXON, so it 4864 * should be impossible to already have an allocated shadow VMCS. KVM 4865 * doesn't support virtualization of VMCS shadowing, so vmcs01 should 4866 * always be the loaded VMCS. 4867 */ 4868 if (WARN_ON(loaded_vmcs != &vmx->vmcs01 || loaded_vmcs->shadow_vmcs)) 4869 return loaded_vmcs->shadow_vmcs; 4870 4871 loaded_vmcs->shadow_vmcs = alloc_vmcs(true); 4872 if (loaded_vmcs->shadow_vmcs) 4873 vmcs_clear(loaded_vmcs->shadow_vmcs); 4874 4875 return loaded_vmcs->shadow_vmcs; 4876 } 4877 4878 static int enter_vmx_operation(struct kvm_vcpu *vcpu) 4879 { 4880 struct vcpu_vmx *vmx = to_vmx(vcpu); 4881 int r; 4882 4883 r = alloc_loaded_vmcs(&vmx->nested.vmcs02); 4884 if (r < 0) 4885 goto out_vmcs02; 4886 4887 vmx->nested.cached_vmcs12 = kzalloc(VMCS12_SIZE, GFP_KERNEL_ACCOUNT); 4888 if (!vmx->nested.cached_vmcs12) 4889 goto out_cached_vmcs12; 4890 4891 vmx->nested.shadow_vmcs12_cache.gpa = INVALID_GPA; 4892 vmx->nested.cached_shadow_vmcs12 = kzalloc(VMCS12_SIZE, GFP_KERNEL_ACCOUNT); 4893 if (!vmx->nested.cached_shadow_vmcs12) 4894 goto out_cached_shadow_vmcs12; 4895 4896 if (enable_shadow_vmcs && !alloc_shadow_vmcs(vcpu)) 4897 goto out_shadow_vmcs; 4898 4899 hrtimer_init(&vmx->nested.preemption_timer, CLOCK_MONOTONIC, 4900 HRTIMER_MODE_ABS_PINNED); 4901 vmx->nested.preemption_timer.function = vmx_preemption_timer_fn; 4902 4903 vmx->nested.vpid02 = allocate_vpid(); 4904 4905 vmx->nested.vmcs02_initialized = false; 4906 vmx->nested.vmxon = true; 4907 4908 if (vmx_pt_mode_is_host_guest()) { 4909 vmx->pt_desc.guest.ctl = 0; 4910 pt_update_intercept_for_msr(vcpu); 4911 } 4912 4913 return 0; 4914 4915 out_shadow_vmcs: 4916 kfree(vmx->nested.cached_shadow_vmcs12); 4917 4918 out_cached_shadow_vmcs12: 4919 kfree(vmx->nested.cached_vmcs12); 4920 4921 out_cached_vmcs12: 4922 free_loaded_vmcs(&vmx->nested.vmcs02); 4923 4924 out_vmcs02: 4925 return -ENOMEM; 4926 } 4927 4928 /* Emulate the VMXON instruction. */ 4929 static int handle_vmon(struct kvm_vcpu *vcpu) 4930 { 4931 int ret; 4932 gpa_t vmptr; 4933 uint32_t revision; 4934 struct vcpu_vmx *vmx = to_vmx(vcpu); 4935 const u64 VMXON_NEEDED_FEATURES = FEAT_CTL_LOCKED 4936 | FEAT_CTL_VMX_ENABLED_OUTSIDE_SMX; 4937 4938 /* 4939 * The Intel VMX Instruction Reference lists a bunch of bits that are 4940 * prerequisite to running VMXON, most notably cr4.VMXE must be set to 4941 * 1 (see vmx_is_valid_cr4() for when we allow the guest to set this). 4942 * Otherwise, we should fail with #UD. But most faulting conditions 4943 * have already been checked by hardware, prior to the VM-exit for 4944 * VMXON. We do test guest cr4.VMXE because processor CR4 always has 4945 * that bit set to 1 in non-root mode. 4946 */ 4947 if (!kvm_read_cr4_bits(vcpu, X86_CR4_VMXE)) { 4948 kvm_queue_exception(vcpu, UD_VECTOR); 4949 return 1; 4950 } 4951 4952 /* CPL=0 must be checked manually. */ 4953 if (vmx_get_cpl(vcpu)) { 4954 kvm_inject_gp(vcpu, 0); 4955 return 1; 4956 } 4957 4958 if (vmx->nested.vmxon) 4959 return nested_vmx_fail(vcpu, VMXERR_VMXON_IN_VMX_ROOT_OPERATION); 4960 4961 if ((vmx->msr_ia32_feature_control & VMXON_NEEDED_FEATURES) 4962 != VMXON_NEEDED_FEATURES) { 4963 kvm_inject_gp(vcpu, 0); 4964 return 1; 4965 } 4966 4967 if (nested_vmx_get_vmptr(vcpu, &vmptr, &ret)) 4968 return ret; 4969 4970 /* 4971 * SDM 3: 24.11.5 4972 * The first 4 bytes of VMXON region contain the supported 4973 * VMCS revision identifier 4974 * 4975 * Note - IA32_VMX_BASIC[48] will never be 1 for the nested case; 4976 * which replaces physical address width with 32 4977 */ 4978 if (!page_address_valid(vcpu, vmptr)) 4979 return nested_vmx_failInvalid(vcpu); 4980 4981 if (kvm_read_guest(vcpu->kvm, vmptr, &revision, sizeof(revision)) || 4982 revision != VMCS12_REVISION) 4983 return nested_vmx_failInvalid(vcpu); 4984 4985 vmx->nested.vmxon_ptr = vmptr; 4986 ret = enter_vmx_operation(vcpu); 4987 if (ret) 4988 return ret; 4989 4990 return nested_vmx_succeed(vcpu); 4991 } 4992 4993 static inline void nested_release_vmcs12(struct kvm_vcpu *vcpu) 4994 { 4995 struct vcpu_vmx *vmx = to_vmx(vcpu); 4996 4997 if (vmx->nested.current_vmptr == INVALID_GPA) 4998 return; 4999 5000 copy_vmcs02_to_vmcs12_rare(vcpu, get_vmcs12(vcpu)); 5001 5002 if (enable_shadow_vmcs) { 5003 /* copy to memory all shadowed fields in case 5004 they were modified */ 5005 copy_shadow_to_vmcs12(vmx); 5006 vmx_disable_shadow_vmcs(vmx); 5007 } 5008 vmx->nested.posted_intr_nv = -1; 5009 5010 /* Flush VMCS12 to guest memory */ 5011 kvm_vcpu_write_guest_page(vcpu, 5012 vmx->nested.current_vmptr >> PAGE_SHIFT, 5013 vmx->nested.cached_vmcs12, 0, VMCS12_SIZE); 5014 5015 kvm_mmu_free_roots(vcpu->kvm, &vcpu->arch.guest_mmu, KVM_MMU_ROOTS_ALL); 5016 5017 vmx->nested.current_vmptr = INVALID_GPA; 5018 } 5019 5020 /* Emulate the VMXOFF instruction */ 5021 static int handle_vmoff(struct kvm_vcpu *vcpu) 5022 { 5023 if (!nested_vmx_check_permission(vcpu)) 5024 return 1; 5025 5026 free_nested(vcpu); 5027 5028 /* Process a latched INIT during time CPU was in VMX operation */ 5029 kvm_make_request(KVM_REQ_EVENT, vcpu); 5030 5031 return nested_vmx_succeed(vcpu); 5032 } 5033 5034 /* Emulate the VMCLEAR instruction */ 5035 static int handle_vmclear(struct kvm_vcpu *vcpu) 5036 { 5037 struct vcpu_vmx *vmx = to_vmx(vcpu); 5038 u32 zero = 0; 5039 gpa_t vmptr; 5040 u64 evmcs_gpa; 5041 int r; 5042 5043 if (!nested_vmx_check_permission(vcpu)) 5044 return 1; 5045 5046 if (nested_vmx_get_vmptr(vcpu, &vmptr, &r)) 5047 return r; 5048 5049 if (!page_address_valid(vcpu, vmptr)) 5050 return nested_vmx_fail(vcpu, VMXERR_VMCLEAR_INVALID_ADDRESS); 5051 5052 if (vmptr == vmx->nested.vmxon_ptr) 5053 return nested_vmx_fail(vcpu, VMXERR_VMCLEAR_VMXON_POINTER); 5054 5055 /* 5056 * When Enlightened VMEntry is enabled on the calling CPU we treat 5057 * memory area pointer by vmptr as Enlightened VMCS (as there's no good 5058 * way to distinguish it from VMCS12) and we must not corrupt it by 5059 * writing to the non-existent 'launch_state' field. The area doesn't 5060 * have to be the currently active EVMCS on the calling CPU and there's 5061 * nothing KVM has to do to transition it from 'active' to 'non-active' 5062 * state. It is possible that the area will stay mapped as 5063 * vmx->nested.hv_evmcs but this shouldn't be a problem. 5064 */ 5065 if (likely(!vmx->nested.enlightened_vmcs_enabled || 5066 !nested_enlightened_vmentry(vcpu, &evmcs_gpa))) { 5067 if (vmptr == vmx->nested.current_vmptr) 5068 nested_release_vmcs12(vcpu); 5069 5070 kvm_vcpu_write_guest(vcpu, 5071 vmptr + offsetof(struct vmcs12, 5072 launch_state), 5073 &zero, sizeof(zero)); 5074 } else if (vmx->nested.hv_evmcs && vmptr == vmx->nested.hv_evmcs_vmptr) { 5075 nested_release_evmcs(vcpu); 5076 } 5077 5078 return nested_vmx_succeed(vcpu); 5079 } 5080 5081 /* Emulate the VMLAUNCH instruction */ 5082 static int handle_vmlaunch(struct kvm_vcpu *vcpu) 5083 { 5084 return nested_vmx_run(vcpu, true); 5085 } 5086 5087 /* Emulate the VMRESUME instruction */ 5088 static int handle_vmresume(struct kvm_vcpu *vcpu) 5089 { 5090 5091 return nested_vmx_run(vcpu, false); 5092 } 5093 5094 static int handle_vmread(struct kvm_vcpu *vcpu) 5095 { 5096 struct vmcs12 *vmcs12 = is_guest_mode(vcpu) ? get_shadow_vmcs12(vcpu) 5097 : get_vmcs12(vcpu); 5098 unsigned long exit_qualification = vmx_get_exit_qual(vcpu); 5099 u32 instr_info = vmcs_read32(VMX_INSTRUCTION_INFO); 5100 struct vcpu_vmx *vmx = to_vmx(vcpu); 5101 struct x86_exception e; 5102 unsigned long field; 5103 u64 value; 5104 gva_t gva = 0; 5105 short offset; 5106 int len, r; 5107 5108 if (!nested_vmx_check_permission(vcpu)) 5109 return 1; 5110 5111 /* Decode instruction info and find the field to read */ 5112 field = kvm_register_read(vcpu, (((instr_info) >> 28) & 0xf)); 5113 5114 if (!evmptr_is_valid(vmx->nested.hv_evmcs_vmptr)) { 5115 /* 5116 * In VMX non-root operation, when the VMCS-link pointer is INVALID_GPA, 5117 * any VMREAD sets the ALU flags for VMfailInvalid. 5118 */ 5119 if (vmx->nested.current_vmptr == INVALID_GPA || 5120 (is_guest_mode(vcpu) && 5121 get_vmcs12(vcpu)->vmcs_link_pointer == INVALID_GPA)) 5122 return nested_vmx_failInvalid(vcpu); 5123 5124 offset = get_vmcs12_field_offset(field); 5125 if (offset < 0) 5126 return nested_vmx_fail(vcpu, VMXERR_UNSUPPORTED_VMCS_COMPONENT); 5127 5128 if (!is_guest_mode(vcpu) && is_vmcs12_ext_field(field)) 5129 copy_vmcs02_to_vmcs12_rare(vcpu, vmcs12); 5130 5131 /* Read the field, zero-extended to a u64 value */ 5132 value = vmcs12_read_any(vmcs12, field, offset); 5133 } else { 5134 /* 5135 * Hyper-V TLFS (as of 6.0b) explicitly states, that while an 5136 * enlightened VMCS is active VMREAD/VMWRITE instructions are 5137 * unsupported. Unfortunately, certain versions of Windows 11 5138 * don't comply with this requirement which is not enforced in 5139 * genuine Hyper-V. Allow VMREAD from an enlightened VMCS as a 5140 * workaround, as misbehaving guests will panic on VM-Fail. 5141 * Note, enlightened VMCS is incompatible with shadow VMCS so 5142 * all VMREADs from L2 should go to L1. 5143 */ 5144 if (WARN_ON_ONCE(is_guest_mode(vcpu))) 5145 return nested_vmx_failInvalid(vcpu); 5146 5147 offset = evmcs_field_offset(field, NULL); 5148 if (offset < 0) 5149 return nested_vmx_fail(vcpu, VMXERR_UNSUPPORTED_VMCS_COMPONENT); 5150 5151 /* Read the field, zero-extended to a u64 value */ 5152 value = evmcs_read_any(vmx->nested.hv_evmcs, field, offset); 5153 } 5154 5155 /* 5156 * Now copy part of this value to register or memory, as requested. 5157 * Note that the number of bits actually copied is 32 or 64 depending 5158 * on the guest's mode (32 or 64 bit), not on the given field's length. 5159 */ 5160 if (instr_info & BIT(10)) { 5161 kvm_register_write(vcpu, (((instr_info) >> 3) & 0xf), value); 5162 } else { 5163 len = is_64_bit_mode(vcpu) ? 8 : 4; 5164 if (get_vmx_mem_address(vcpu, exit_qualification, 5165 instr_info, true, len, &gva)) 5166 return 1; 5167 /* _system ok, nested_vmx_check_permission has verified cpl=0 */ 5168 r = kvm_write_guest_virt_system(vcpu, gva, &value, len, &e); 5169 if (r != X86EMUL_CONTINUE) 5170 return kvm_handle_memory_failure(vcpu, r, &e); 5171 } 5172 5173 return nested_vmx_succeed(vcpu); 5174 } 5175 5176 static bool is_shadow_field_rw(unsigned long field) 5177 { 5178 switch (field) { 5179 #define SHADOW_FIELD_RW(x, y) case x: 5180 #include "vmcs_shadow_fields.h" 5181 return true; 5182 default: 5183 break; 5184 } 5185 return false; 5186 } 5187 5188 static bool is_shadow_field_ro(unsigned long field) 5189 { 5190 switch (field) { 5191 #define SHADOW_FIELD_RO(x, y) case x: 5192 #include "vmcs_shadow_fields.h" 5193 return true; 5194 default: 5195 break; 5196 } 5197 return false; 5198 } 5199 5200 static int handle_vmwrite(struct kvm_vcpu *vcpu) 5201 { 5202 struct vmcs12 *vmcs12 = is_guest_mode(vcpu) ? get_shadow_vmcs12(vcpu) 5203 : get_vmcs12(vcpu); 5204 unsigned long exit_qualification = vmx_get_exit_qual(vcpu); 5205 u32 instr_info = vmcs_read32(VMX_INSTRUCTION_INFO); 5206 struct vcpu_vmx *vmx = to_vmx(vcpu); 5207 struct x86_exception e; 5208 unsigned long field; 5209 short offset; 5210 gva_t gva; 5211 int len, r; 5212 5213 /* 5214 * The value to write might be 32 or 64 bits, depending on L1's long 5215 * mode, and eventually we need to write that into a field of several 5216 * possible lengths. The code below first zero-extends the value to 64 5217 * bit (value), and then copies only the appropriate number of 5218 * bits into the vmcs12 field. 5219 */ 5220 u64 value = 0; 5221 5222 if (!nested_vmx_check_permission(vcpu)) 5223 return 1; 5224 5225 /* 5226 * In VMX non-root operation, when the VMCS-link pointer is INVALID_GPA, 5227 * any VMWRITE sets the ALU flags for VMfailInvalid. 5228 */ 5229 if (vmx->nested.current_vmptr == INVALID_GPA || 5230 (is_guest_mode(vcpu) && 5231 get_vmcs12(vcpu)->vmcs_link_pointer == INVALID_GPA)) 5232 return nested_vmx_failInvalid(vcpu); 5233 5234 if (instr_info & BIT(10)) 5235 value = kvm_register_read(vcpu, (((instr_info) >> 3) & 0xf)); 5236 else { 5237 len = is_64_bit_mode(vcpu) ? 8 : 4; 5238 if (get_vmx_mem_address(vcpu, exit_qualification, 5239 instr_info, false, len, &gva)) 5240 return 1; 5241 r = kvm_read_guest_virt(vcpu, gva, &value, len, &e); 5242 if (r != X86EMUL_CONTINUE) 5243 return kvm_handle_memory_failure(vcpu, r, &e); 5244 } 5245 5246 field = kvm_register_read(vcpu, (((instr_info) >> 28) & 0xf)); 5247 5248 offset = get_vmcs12_field_offset(field); 5249 if (offset < 0) 5250 return nested_vmx_fail(vcpu, VMXERR_UNSUPPORTED_VMCS_COMPONENT); 5251 5252 /* 5253 * If the vCPU supports "VMWRITE to any supported field in the 5254 * VMCS," then the "read-only" fields are actually read/write. 5255 */ 5256 if (vmcs_field_readonly(field) && 5257 !nested_cpu_has_vmwrite_any_field(vcpu)) 5258 return nested_vmx_fail(vcpu, VMXERR_VMWRITE_READ_ONLY_VMCS_COMPONENT); 5259 5260 /* 5261 * Ensure vmcs12 is up-to-date before any VMWRITE that dirties 5262 * vmcs12, else we may crush a field or consume a stale value. 5263 */ 5264 if (!is_guest_mode(vcpu) && !is_shadow_field_rw(field)) 5265 copy_vmcs02_to_vmcs12_rare(vcpu, vmcs12); 5266 5267 /* 5268 * Some Intel CPUs intentionally drop the reserved bits of the AR byte 5269 * fields on VMWRITE. Emulate this behavior to ensure consistent KVM 5270 * behavior regardless of the underlying hardware, e.g. if an AR_BYTE 5271 * field is intercepted for VMWRITE but not VMREAD (in L1), then VMREAD 5272 * from L1 will return a different value than VMREAD from L2 (L1 sees 5273 * the stripped down value, L2 sees the full value as stored by KVM). 5274 */ 5275 if (field >= GUEST_ES_AR_BYTES && field <= GUEST_TR_AR_BYTES) 5276 value &= 0x1f0ff; 5277 5278 vmcs12_write_any(vmcs12, field, offset, value); 5279 5280 /* 5281 * Do not track vmcs12 dirty-state if in guest-mode as we actually 5282 * dirty shadow vmcs12 instead of vmcs12. Fields that can be updated 5283 * by L1 without a vmexit are always updated in the vmcs02, i.e. don't 5284 * "dirty" vmcs12, all others go down the prepare_vmcs02() slow path. 5285 */ 5286 if (!is_guest_mode(vcpu) && !is_shadow_field_rw(field)) { 5287 /* 5288 * L1 can read these fields without exiting, ensure the 5289 * shadow VMCS is up-to-date. 5290 */ 5291 if (enable_shadow_vmcs && is_shadow_field_ro(field)) { 5292 preempt_disable(); 5293 vmcs_load(vmx->vmcs01.shadow_vmcs); 5294 5295 __vmcs_writel(field, value); 5296 5297 vmcs_clear(vmx->vmcs01.shadow_vmcs); 5298 vmcs_load(vmx->loaded_vmcs->vmcs); 5299 preempt_enable(); 5300 } 5301 vmx->nested.dirty_vmcs12 = true; 5302 } 5303 5304 return nested_vmx_succeed(vcpu); 5305 } 5306 5307 static void set_current_vmptr(struct vcpu_vmx *vmx, gpa_t vmptr) 5308 { 5309 vmx->nested.current_vmptr = vmptr; 5310 if (enable_shadow_vmcs) { 5311 secondary_exec_controls_setbit(vmx, SECONDARY_EXEC_SHADOW_VMCS); 5312 vmcs_write64(VMCS_LINK_POINTER, 5313 __pa(vmx->vmcs01.shadow_vmcs)); 5314 vmx->nested.need_vmcs12_to_shadow_sync = true; 5315 } 5316 vmx->nested.dirty_vmcs12 = true; 5317 vmx->nested.force_msr_bitmap_recalc = true; 5318 } 5319 5320 /* Emulate the VMPTRLD instruction */ 5321 static int handle_vmptrld(struct kvm_vcpu *vcpu) 5322 { 5323 struct vcpu_vmx *vmx = to_vmx(vcpu); 5324 gpa_t vmptr; 5325 int r; 5326 5327 if (!nested_vmx_check_permission(vcpu)) 5328 return 1; 5329 5330 if (nested_vmx_get_vmptr(vcpu, &vmptr, &r)) 5331 return r; 5332 5333 if (!page_address_valid(vcpu, vmptr)) 5334 return nested_vmx_fail(vcpu, VMXERR_VMPTRLD_INVALID_ADDRESS); 5335 5336 if (vmptr == vmx->nested.vmxon_ptr) 5337 return nested_vmx_fail(vcpu, VMXERR_VMPTRLD_VMXON_POINTER); 5338 5339 /* Forbid normal VMPTRLD if Enlightened version was used */ 5340 if (evmptr_is_valid(vmx->nested.hv_evmcs_vmptr)) 5341 return 1; 5342 5343 if (vmx->nested.current_vmptr != vmptr) { 5344 struct gfn_to_hva_cache *ghc = &vmx->nested.vmcs12_cache; 5345 struct vmcs_hdr hdr; 5346 5347 if (kvm_gfn_to_hva_cache_init(vcpu->kvm, ghc, vmptr, VMCS12_SIZE)) { 5348 /* 5349 * Reads from an unbacked page return all 1s, 5350 * which means that the 32 bits located at the 5351 * given physical address won't match the required 5352 * VMCS12_REVISION identifier. 5353 */ 5354 return nested_vmx_fail(vcpu, 5355 VMXERR_VMPTRLD_INCORRECT_VMCS_REVISION_ID); 5356 } 5357 5358 if (kvm_read_guest_offset_cached(vcpu->kvm, ghc, &hdr, 5359 offsetof(struct vmcs12, hdr), 5360 sizeof(hdr))) { 5361 return nested_vmx_fail(vcpu, 5362 VMXERR_VMPTRLD_INCORRECT_VMCS_REVISION_ID); 5363 } 5364 5365 if (hdr.revision_id != VMCS12_REVISION || 5366 (hdr.shadow_vmcs && 5367 !nested_cpu_has_vmx_shadow_vmcs(vcpu))) { 5368 return nested_vmx_fail(vcpu, 5369 VMXERR_VMPTRLD_INCORRECT_VMCS_REVISION_ID); 5370 } 5371 5372 nested_release_vmcs12(vcpu); 5373 5374 /* 5375 * Load VMCS12 from guest memory since it is not already 5376 * cached. 5377 */ 5378 if (kvm_read_guest_cached(vcpu->kvm, ghc, vmx->nested.cached_vmcs12, 5379 VMCS12_SIZE)) { 5380 return nested_vmx_fail(vcpu, 5381 VMXERR_VMPTRLD_INCORRECT_VMCS_REVISION_ID); 5382 } 5383 5384 set_current_vmptr(vmx, vmptr); 5385 } 5386 5387 return nested_vmx_succeed(vcpu); 5388 } 5389 5390 /* Emulate the VMPTRST instruction */ 5391 static int handle_vmptrst(struct kvm_vcpu *vcpu) 5392 { 5393 unsigned long exit_qual = vmx_get_exit_qual(vcpu); 5394 u32 instr_info = vmcs_read32(VMX_INSTRUCTION_INFO); 5395 gpa_t current_vmptr = to_vmx(vcpu)->nested.current_vmptr; 5396 struct x86_exception e; 5397 gva_t gva; 5398 int r; 5399 5400 if (!nested_vmx_check_permission(vcpu)) 5401 return 1; 5402 5403 if (unlikely(evmptr_is_valid(to_vmx(vcpu)->nested.hv_evmcs_vmptr))) 5404 return 1; 5405 5406 if (get_vmx_mem_address(vcpu, exit_qual, instr_info, 5407 true, sizeof(gpa_t), &gva)) 5408 return 1; 5409 /* *_system ok, nested_vmx_check_permission has verified cpl=0 */ 5410 r = kvm_write_guest_virt_system(vcpu, gva, (void *)¤t_vmptr, 5411 sizeof(gpa_t), &e); 5412 if (r != X86EMUL_CONTINUE) 5413 return kvm_handle_memory_failure(vcpu, r, &e); 5414 5415 return nested_vmx_succeed(vcpu); 5416 } 5417 5418 /* Emulate the INVEPT instruction */ 5419 static int handle_invept(struct kvm_vcpu *vcpu) 5420 { 5421 struct vcpu_vmx *vmx = to_vmx(vcpu); 5422 u32 vmx_instruction_info, types; 5423 unsigned long type, roots_to_free; 5424 struct kvm_mmu *mmu; 5425 gva_t gva; 5426 struct x86_exception e; 5427 struct { 5428 u64 eptp, gpa; 5429 } operand; 5430 int i, r, gpr_index; 5431 5432 if (!(vmx->nested.msrs.secondary_ctls_high & 5433 SECONDARY_EXEC_ENABLE_EPT) || 5434 !(vmx->nested.msrs.ept_caps & VMX_EPT_INVEPT_BIT)) { 5435 kvm_queue_exception(vcpu, UD_VECTOR); 5436 return 1; 5437 } 5438 5439 if (!nested_vmx_check_permission(vcpu)) 5440 return 1; 5441 5442 vmx_instruction_info = vmcs_read32(VMX_INSTRUCTION_INFO); 5443 gpr_index = vmx_get_instr_info_reg2(vmx_instruction_info); 5444 type = kvm_register_read(vcpu, gpr_index); 5445 5446 types = (vmx->nested.msrs.ept_caps >> VMX_EPT_EXTENT_SHIFT) & 6; 5447 5448 if (type >= 32 || !(types & (1 << type))) 5449 return nested_vmx_fail(vcpu, VMXERR_INVALID_OPERAND_TO_INVEPT_INVVPID); 5450 5451 /* According to the Intel VMX instruction reference, the memory 5452 * operand is read even if it isn't needed (e.g., for type==global) 5453 */ 5454 if (get_vmx_mem_address(vcpu, vmx_get_exit_qual(vcpu), 5455 vmx_instruction_info, false, sizeof(operand), &gva)) 5456 return 1; 5457 r = kvm_read_guest_virt(vcpu, gva, &operand, sizeof(operand), &e); 5458 if (r != X86EMUL_CONTINUE) 5459 return kvm_handle_memory_failure(vcpu, r, &e); 5460 5461 /* 5462 * Nested EPT roots are always held through guest_mmu, 5463 * not root_mmu. 5464 */ 5465 mmu = &vcpu->arch.guest_mmu; 5466 5467 switch (type) { 5468 case VMX_EPT_EXTENT_CONTEXT: 5469 if (!nested_vmx_check_eptp(vcpu, operand.eptp)) 5470 return nested_vmx_fail(vcpu, 5471 VMXERR_INVALID_OPERAND_TO_INVEPT_INVVPID); 5472 5473 roots_to_free = 0; 5474 if (nested_ept_root_matches(mmu->root.hpa, mmu->root.pgd, 5475 operand.eptp)) 5476 roots_to_free |= KVM_MMU_ROOT_CURRENT; 5477 5478 for (i = 0; i < KVM_MMU_NUM_PREV_ROOTS; i++) { 5479 if (nested_ept_root_matches(mmu->prev_roots[i].hpa, 5480 mmu->prev_roots[i].pgd, 5481 operand.eptp)) 5482 roots_to_free |= KVM_MMU_ROOT_PREVIOUS(i); 5483 } 5484 break; 5485 case VMX_EPT_EXTENT_GLOBAL: 5486 roots_to_free = KVM_MMU_ROOTS_ALL; 5487 break; 5488 default: 5489 BUG(); 5490 break; 5491 } 5492 5493 if (roots_to_free) 5494 kvm_mmu_free_roots(vcpu->kvm, mmu, roots_to_free); 5495 5496 return nested_vmx_succeed(vcpu); 5497 } 5498 5499 static int handle_invvpid(struct kvm_vcpu *vcpu) 5500 { 5501 struct vcpu_vmx *vmx = to_vmx(vcpu); 5502 u32 vmx_instruction_info; 5503 unsigned long type, types; 5504 gva_t gva; 5505 struct x86_exception e; 5506 struct { 5507 u64 vpid; 5508 u64 gla; 5509 } operand; 5510 u16 vpid02; 5511 int r, gpr_index; 5512 5513 if (!(vmx->nested.msrs.secondary_ctls_high & 5514 SECONDARY_EXEC_ENABLE_VPID) || 5515 !(vmx->nested.msrs.vpid_caps & VMX_VPID_INVVPID_BIT)) { 5516 kvm_queue_exception(vcpu, UD_VECTOR); 5517 return 1; 5518 } 5519 5520 if (!nested_vmx_check_permission(vcpu)) 5521 return 1; 5522 5523 vmx_instruction_info = vmcs_read32(VMX_INSTRUCTION_INFO); 5524 gpr_index = vmx_get_instr_info_reg2(vmx_instruction_info); 5525 type = kvm_register_read(vcpu, gpr_index); 5526 5527 types = (vmx->nested.msrs.vpid_caps & 5528 VMX_VPID_EXTENT_SUPPORTED_MASK) >> 8; 5529 5530 if (type >= 32 || !(types & (1 << type))) 5531 return nested_vmx_fail(vcpu, 5532 VMXERR_INVALID_OPERAND_TO_INVEPT_INVVPID); 5533 5534 /* according to the intel vmx instruction reference, the memory 5535 * operand is read even if it isn't needed (e.g., for type==global) 5536 */ 5537 if (get_vmx_mem_address(vcpu, vmx_get_exit_qual(vcpu), 5538 vmx_instruction_info, false, sizeof(operand), &gva)) 5539 return 1; 5540 r = kvm_read_guest_virt(vcpu, gva, &operand, sizeof(operand), &e); 5541 if (r != X86EMUL_CONTINUE) 5542 return kvm_handle_memory_failure(vcpu, r, &e); 5543 5544 if (operand.vpid >> 16) 5545 return nested_vmx_fail(vcpu, 5546 VMXERR_INVALID_OPERAND_TO_INVEPT_INVVPID); 5547 5548 vpid02 = nested_get_vpid02(vcpu); 5549 switch (type) { 5550 case VMX_VPID_EXTENT_INDIVIDUAL_ADDR: 5551 if (!operand.vpid || 5552 is_noncanonical_address(operand.gla, vcpu)) 5553 return nested_vmx_fail(vcpu, 5554 VMXERR_INVALID_OPERAND_TO_INVEPT_INVVPID); 5555 vpid_sync_vcpu_addr(vpid02, operand.gla); 5556 break; 5557 case VMX_VPID_EXTENT_SINGLE_CONTEXT: 5558 case VMX_VPID_EXTENT_SINGLE_NON_GLOBAL: 5559 if (!operand.vpid) 5560 return nested_vmx_fail(vcpu, 5561 VMXERR_INVALID_OPERAND_TO_INVEPT_INVVPID); 5562 vpid_sync_context(vpid02); 5563 break; 5564 case VMX_VPID_EXTENT_ALL_CONTEXT: 5565 vpid_sync_context(vpid02); 5566 break; 5567 default: 5568 WARN_ON_ONCE(1); 5569 return kvm_skip_emulated_instruction(vcpu); 5570 } 5571 5572 /* 5573 * Sync the shadow page tables if EPT is disabled, L1 is invalidating 5574 * linear mappings for L2 (tagged with L2's VPID). Free all guest 5575 * roots as VPIDs are not tracked in the MMU role. 5576 * 5577 * Note, this operates on root_mmu, not guest_mmu, as L1 and L2 share 5578 * an MMU when EPT is disabled. 5579 * 5580 * TODO: sync only the affected SPTEs for INVDIVIDUAL_ADDR. 5581 */ 5582 if (!enable_ept) 5583 kvm_mmu_free_guest_mode_roots(vcpu->kvm, &vcpu->arch.root_mmu); 5584 5585 return nested_vmx_succeed(vcpu); 5586 } 5587 5588 static int nested_vmx_eptp_switching(struct kvm_vcpu *vcpu, 5589 struct vmcs12 *vmcs12) 5590 { 5591 u32 index = kvm_rcx_read(vcpu); 5592 u64 new_eptp; 5593 5594 if (WARN_ON_ONCE(!nested_cpu_has_ept(vmcs12))) 5595 return 1; 5596 if (index >= VMFUNC_EPTP_ENTRIES) 5597 return 1; 5598 5599 if (kvm_vcpu_read_guest_page(vcpu, vmcs12->eptp_list_address >> PAGE_SHIFT, 5600 &new_eptp, index * 8, 8)) 5601 return 1; 5602 5603 /* 5604 * If the (L2) guest does a vmfunc to the currently 5605 * active ept pointer, we don't have to do anything else 5606 */ 5607 if (vmcs12->ept_pointer != new_eptp) { 5608 if (!nested_vmx_check_eptp(vcpu, new_eptp)) 5609 return 1; 5610 5611 vmcs12->ept_pointer = new_eptp; 5612 nested_ept_new_eptp(vcpu); 5613 5614 if (!nested_cpu_has_vpid(vmcs12)) 5615 kvm_make_request(KVM_REQ_TLB_FLUSH_GUEST, vcpu); 5616 } 5617 5618 return 0; 5619 } 5620 5621 static int handle_vmfunc(struct kvm_vcpu *vcpu) 5622 { 5623 struct vcpu_vmx *vmx = to_vmx(vcpu); 5624 struct vmcs12 *vmcs12; 5625 u32 function = kvm_rax_read(vcpu); 5626 5627 /* 5628 * VMFUNC is only supported for nested guests, but we always enable the 5629 * secondary control for simplicity; for non-nested mode, fake that we 5630 * didn't by injecting #UD. 5631 */ 5632 if (!is_guest_mode(vcpu)) { 5633 kvm_queue_exception(vcpu, UD_VECTOR); 5634 return 1; 5635 } 5636 5637 vmcs12 = get_vmcs12(vcpu); 5638 5639 /* 5640 * #UD on out-of-bounds function has priority over VM-Exit, and VMFUNC 5641 * is enabled in vmcs02 if and only if it's enabled in vmcs12. 5642 */ 5643 if (WARN_ON_ONCE((function > 63) || !nested_cpu_has_vmfunc(vmcs12))) { 5644 kvm_queue_exception(vcpu, UD_VECTOR); 5645 return 1; 5646 } 5647 5648 if (!(vmcs12->vm_function_control & BIT_ULL(function))) 5649 goto fail; 5650 5651 switch (function) { 5652 case 0: 5653 if (nested_vmx_eptp_switching(vcpu, vmcs12)) 5654 goto fail; 5655 break; 5656 default: 5657 goto fail; 5658 } 5659 return kvm_skip_emulated_instruction(vcpu); 5660 5661 fail: 5662 /* 5663 * This is effectively a reflected VM-Exit, as opposed to a synthesized 5664 * nested VM-Exit. Pass the original exit reason, i.e. don't hardcode 5665 * EXIT_REASON_VMFUNC as the exit reason. 5666 */ 5667 nested_vmx_vmexit(vcpu, vmx->exit_reason.full, 5668 vmx_get_intr_info(vcpu), 5669 vmx_get_exit_qual(vcpu)); 5670 return 1; 5671 } 5672 5673 /* 5674 * Return true if an IO instruction with the specified port and size should cause 5675 * a VM-exit into L1. 5676 */ 5677 bool nested_vmx_check_io_bitmaps(struct kvm_vcpu *vcpu, unsigned int port, 5678 int size) 5679 { 5680 struct vmcs12 *vmcs12 = get_vmcs12(vcpu); 5681 gpa_t bitmap, last_bitmap; 5682 u8 b; 5683 5684 last_bitmap = INVALID_GPA; 5685 b = -1; 5686 5687 while (size > 0) { 5688 if (port < 0x8000) 5689 bitmap = vmcs12->io_bitmap_a; 5690 else if (port < 0x10000) 5691 bitmap = vmcs12->io_bitmap_b; 5692 else 5693 return true; 5694 bitmap += (port & 0x7fff) / 8; 5695 5696 if (last_bitmap != bitmap) 5697 if (kvm_vcpu_read_guest(vcpu, bitmap, &b, 1)) 5698 return true; 5699 if (b & (1 << (port & 7))) 5700 return true; 5701 5702 port++; 5703 size--; 5704 last_bitmap = bitmap; 5705 } 5706 5707 return false; 5708 } 5709 5710 static bool nested_vmx_exit_handled_io(struct kvm_vcpu *vcpu, 5711 struct vmcs12 *vmcs12) 5712 { 5713 unsigned long exit_qualification; 5714 unsigned short port; 5715 int size; 5716 5717 if (!nested_cpu_has(vmcs12, CPU_BASED_USE_IO_BITMAPS)) 5718 return nested_cpu_has(vmcs12, CPU_BASED_UNCOND_IO_EXITING); 5719 5720 exit_qualification = vmx_get_exit_qual(vcpu); 5721 5722 port = exit_qualification >> 16; 5723 size = (exit_qualification & 7) + 1; 5724 5725 return nested_vmx_check_io_bitmaps(vcpu, port, size); 5726 } 5727 5728 /* 5729 * Return 1 if we should exit from L2 to L1 to handle an MSR access, 5730 * rather than handle it ourselves in L0. I.e., check whether L1 expressed 5731 * disinterest in the current event (read or write a specific MSR) by using an 5732 * MSR bitmap. This may be the case even when L0 doesn't use MSR bitmaps. 5733 */ 5734 static bool nested_vmx_exit_handled_msr(struct kvm_vcpu *vcpu, 5735 struct vmcs12 *vmcs12, 5736 union vmx_exit_reason exit_reason) 5737 { 5738 u32 msr_index = kvm_rcx_read(vcpu); 5739 gpa_t bitmap; 5740 5741 if (!nested_cpu_has(vmcs12, CPU_BASED_USE_MSR_BITMAPS)) 5742 return true; 5743 5744 /* 5745 * The MSR_BITMAP page is divided into four 1024-byte bitmaps, 5746 * for the four combinations of read/write and low/high MSR numbers. 5747 * First we need to figure out which of the four to use: 5748 */ 5749 bitmap = vmcs12->msr_bitmap; 5750 if (exit_reason.basic == EXIT_REASON_MSR_WRITE) 5751 bitmap += 2048; 5752 if (msr_index >= 0xc0000000) { 5753 msr_index -= 0xc0000000; 5754 bitmap += 1024; 5755 } 5756 5757 /* Then read the msr_index'th bit from this bitmap: */ 5758 if (msr_index < 1024*8) { 5759 unsigned char b; 5760 if (kvm_vcpu_read_guest(vcpu, bitmap + msr_index/8, &b, 1)) 5761 return true; 5762 return 1 & (b >> (msr_index & 7)); 5763 } else 5764 return true; /* let L1 handle the wrong parameter */ 5765 } 5766 5767 /* 5768 * Return 1 if we should exit from L2 to L1 to handle a CR access exit, 5769 * rather than handle it ourselves in L0. I.e., check if L1 wanted to 5770 * intercept (via guest_host_mask etc.) the current event. 5771 */ 5772 static bool nested_vmx_exit_handled_cr(struct kvm_vcpu *vcpu, 5773 struct vmcs12 *vmcs12) 5774 { 5775 unsigned long exit_qualification = vmx_get_exit_qual(vcpu); 5776 int cr = exit_qualification & 15; 5777 int reg; 5778 unsigned long val; 5779 5780 switch ((exit_qualification >> 4) & 3) { 5781 case 0: /* mov to cr */ 5782 reg = (exit_qualification >> 8) & 15; 5783 val = kvm_register_read(vcpu, reg); 5784 switch (cr) { 5785 case 0: 5786 if (vmcs12->cr0_guest_host_mask & 5787 (val ^ vmcs12->cr0_read_shadow)) 5788 return true; 5789 break; 5790 case 3: 5791 if (nested_cpu_has(vmcs12, CPU_BASED_CR3_LOAD_EXITING)) 5792 return true; 5793 break; 5794 case 4: 5795 if (vmcs12->cr4_guest_host_mask & 5796 (vmcs12->cr4_read_shadow ^ val)) 5797 return true; 5798 break; 5799 case 8: 5800 if (nested_cpu_has(vmcs12, CPU_BASED_CR8_LOAD_EXITING)) 5801 return true; 5802 break; 5803 } 5804 break; 5805 case 2: /* clts */ 5806 if ((vmcs12->cr0_guest_host_mask & X86_CR0_TS) && 5807 (vmcs12->cr0_read_shadow & X86_CR0_TS)) 5808 return true; 5809 break; 5810 case 1: /* mov from cr */ 5811 switch (cr) { 5812 case 3: 5813 if (vmcs12->cpu_based_vm_exec_control & 5814 CPU_BASED_CR3_STORE_EXITING) 5815 return true; 5816 break; 5817 case 8: 5818 if (vmcs12->cpu_based_vm_exec_control & 5819 CPU_BASED_CR8_STORE_EXITING) 5820 return true; 5821 break; 5822 } 5823 break; 5824 case 3: /* lmsw */ 5825 /* 5826 * lmsw can change bits 1..3 of cr0, and only set bit 0 of 5827 * cr0. Other attempted changes are ignored, with no exit. 5828 */ 5829 val = (exit_qualification >> LMSW_SOURCE_DATA_SHIFT) & 0x0f; 5830 if (vmcs12->cr0_guest_host_mask & 0xe & 5831 (val ^ vmcs12->cr0_read_shadow)) 5832 return true; 5833 if ((vmcs12->cr0_guest_host_mask & 0x1) && 5834 !(vmcs12->cr0_read_shadow & 0x1) && 5835 (val & 0x1)) 5836 return true; 5837 break; 5838 } 5839 return false; 5840 } 5841 5842 static bool nested_vmx_exit_handled_encls(struct kvm_vcpu *vcpu, 5843 struct vmcs12 *vmcs12) 5844 { 5845 u32 encls_leaf; 5846 5847 if (!guest_cpuid_has(vcpu, X86_FEATURE_SGX) || 5848 !nested_cpu_has2(vmcs12, SECONDARY_EXEC_ENCLS_EXITING)) 5849 return false; 5850 5851 encls_leaf = kvm_rax_read(vcpu); 5852 if (encls_leaf > 62) 5853 encls_leaf = 63; 5854 return vmcs12->encls_exiting_bitmap & BIT_ULL(encls_leaf); 5855 } 5856 5857 static bool nested_vmx_exit_handled_vmcs_access(struct kvm_vcpu *vcpu, 5858 struct vmcs12 *vmcs12, gpa_t bitmap) 5859 { 5860 u32 vmx_instruction_info; 5861 unsigned long field; 5862 u8 b; 5863 5864 if (!nested_cpu_has_shadow_vmcs(vmcs12)) 5865 return true; 5866 5867 /* Decode instruction info and find the field to access */ 5868 vmx_instruction_info = vmcs_read32(VMX_INSTRUCTION_INFO); 5869 field = kvm_register_read(vcpu, (((vmx_instruction_info) >> 28) & 0xf)); 5870 5871 /* Out-of-range fields always cause a VM exit from L2 to L1 */ 5872 if (field >> 15) 5873 return true; 5874 5875 if (kvm_vcpu_read_guest(vcpu, bitmap + field/8, &b, 1)) 5876 return true; 5877 5878 return 1 & (b >> (field & 7)); 5879 } 5880 5881 static bool nested_vmx_exit_handled_mtf(struct vmcs12 *vmcs12) 5882 { 5883 u32 entry_intr_info = vmcs12->vm_entry_intr_info_field; 5884 5885 if (nested_cpu_has_mtf(vmcs12)) 5886 return true; 5887 5888 /* 5889 * An MTF VM-exit may be injected into the guest by setting the 5890 * interruption-type to 7 (other event) and the vector field to 0. Such 5891 * is the case regardless of the 'monitor trap flag' VM-execution 5892 * control. 5893 */ 5894 return entry_intr_info == (INTR_INFO_VALID_MASK 5895 | INTR_TYPE_OTHER_EVENT); 5896 } 5897 5898 /* 5899 * Return true if L0 wants to handle an exit from L2 regardless of whether or not 5900 * L1 wants the exit. Only call this when in is_guest_mode (L2). 5901 */ 5902 static bool nested_vmx_l0_wants_exit(struct kvm_vcpu *vcpu, 5903 union vmx_exit_reason exit_reason) 5904 { 5905 u32 intr_info; 5906 5907 switch ((u16)exit_reason.basic) { 5908 case EXIT_REASON_EXCEPTION_NMI: 5909 intr_info = vmx_get_intr_info(vcpu); 5910 if (is_nmi(intr_info)) 5911 return true; 5912 else if (is_page_fault(intr_info)) 5913 return vcpu->arch.apf.host_apf_flags || 5914 vmx_need_pf_intercept(vcpu); 5915 else if (is_debug(intr_info) && 5916 vcpu->guest_debug & 5917 (KVM_GUESTDBG_SINGLESTEP | KVM_GUESTDBG_USE_HW_BP)) 5918 return true; 5919 else if (is_breakpoint(intr_info) && 5920 vcpu->guest_debug & KVM_GUESTDBG_USE_SW_BP) 5921 return true; 5922 else if (is_alignment_check(intr_info) && 5923 !vmx_guest_inject_ac(vcpu)) 5924 return true; 5925 return false; 5926 case EXIT_REASON_EXTERNAL_INTERRUPT: 5927 return true; 5928 case EXIT_REASON_MCE_DURING_VMENTRY: 5929 return true; 5930 case EXIT_REASON_EPT_VIOLATION: 5931 /* 5932 * L0 always deals with the EPT violation. If nested EPT is 5933 * used, and the nested mmu code discovers that the address is 5934 * missing in the guest EPT table (EPT12), the EPT violation 5935 * will be injected with nested_ept_inject_page_fault() 5936 */ 5937 return true; 5938 case EXIT_REASON_EPT_MISCONFIG: 5939 /* 5940 * L2 never uses directly L1's EPT, but rather L0's own EPT 5941 * table (shadow on EPT) or a merged EPT table that L0 built 5942 * (EPT on EPT). So any problems with the structure of the 5943 * table is L0's fault. 5944 */ 5945 return true; 5946 case EXIT_REASON_PREEMPTION_TIMER: 5947 return true; 5948 case EXIT_REASON_PML_FULL: 5949 /* 5950 * PML is emulated for an L1 VMM and should never be enabled in 5951 * vmcs02, always "handle" PML_FULL by exiting to userspace. 5952 */ 5953 return true; 5954 case EXIT_REASON_VMFUNC: 5955 /* VM functions are emulated through L2->L0 vmexits. */ 5956 return true; 5957 case EXIT_REASON_BUS_LOCK: 5958 /* 5959 * At present, bus lock VM exit is never exposed to L1. 5960 * Handle L2's bus locks in L0 directly. 5961 */ 5962 return true; 5963 default: 5964 break; 5965 } 5966 return false; 5967 } 5968 5969 /* 5970 * Return 1 if L1 wants to intercept an exit from L2. Only call this when in 5971 * is_guest_mode (L2). 5972 */ 5973 static bool nested_vmx_l1_wants_exit(struct kvm_vcpu *vcpu, 5974 union vmx_exit_reason exit_reason) 5975 { 5976 struct vmcs12 *vmcs12 = get_vmcs12(vcpu); 5977 u32 intr_info; 5978 5979 switch ((u16)exit_reason.basic) { 5980 case EXIT_REASON_EXCEPTION_NMI: 5981 intr_info = vmx_get_intr_info(vcpu); 5982 if (is_nmi(intr_info)) 5983 return true; 5984 else if (is_page_fault(intr_info)) 5985 return true; 5986 return vmcs12->exception_bitmap & 5987 (1u << (intr_info & INTR_INFO_VECTOR_MASK)); 5988 case EXIT_REASON_EXTERNAL_INTERRUPT: 5989 return nested_exit_on_intr(vcpu); 5990 case EXIT_REASON_TRIPLE_FAULT: 5991 return true; 5992 case EXIT_REASON_INTERRUPT_WINDOW: 5993 return nested_cpu_has(vmcs12, CPU_BASED_INTR_WINDOW_EXITING); 5994 case EXIT_REASON_NMI_WINDOW: 5995 return nested_cpu_has(vmcs12, CPU_BASED_NMI_WINDOW_EXITING); 5996 case EXIT_REASON_TASK_SWITCH: 5997 return true; 5998 case EXIT_REASON_CPUID: 5999 return true; 6000 case EXIT_REASON_HLT: 6001 return nested_cpu_has(vmcs12, CPU_BASED_HLT_EXITING); 6002 case EXIT_REASON_INVD: 6003 return true; 6004 case EXIT_REASON_INVLPG: 6005 return nested_cpu_has(vmcs12, CPU_BASED_INVLPG_EXITING); 6006 case EXIT_REASON_RDPMC: 6007 return nested_cpu_has(vmcs12, CPU_BASED_RDPMC_EXITING); 6008 case EXIT_REASON_RDRAND: 6009 return nested_cpu_has2(vmcs12, SECONDARY_EXEC_RDRAND_EXITING); 6010 case EXIT_REASON_RDSEED: 6011 return nested_cpu_has2(vmcs12, SECONDARY_EXEC_RDSEED_EXITING); 6012 case EXIT_REASON_RDTSC: case EXIT_REASON_RDTSCP: 6013 return nested_cpu_has(vmcs12, CPU_BASED_RDTSC_EXITING); 6014 case EXIT_REASON_VMREAD: 6015 return nested_vmx_exit_handled_vmcs_access(vcpu, vmcs12, 6016 vmcs12->vmread_bitmap); 6017 case EXIT_REASON_VMWRITE: 6018 return nested_vmx_exit_handled_vmcs_access(vcpu, vmcs12, 6019 vmcs12->vmwrite_bitmap); 6020 case EXIT_REASON_VMCALL: case EXIT_REASON_VMCLEAR: 6021 case EXIT_REASON_VMLAUNCH: case EXIT_REASON_VMPTRLD: 6022 case EXIT_REASON_VMPTRST: case EXIT_REASON_VMRESUME: 6023 case EXIT_REASON_VMOFF: case EXIT_REASON_VMON: 6024 case EXIT_REASON_INVEPT: case EXIT_REASON_INVVPID: 6025 /* 6026 * VMX instructions trap unconditionally. This allows L1 to 6027 * emulate them for its L2 guest, i.e., allows 3-level nesting! 6028 */ 6029 return true; 6030 case EXIT_REASON_CR_ACCESS: 6031 return nested_vmx_exit_handled_cr(vcpu, vmcs12); 6032 case EXIT_REASON_DR_ACCESS: 6033 return nested_cpu_has(vmcs12, CPU_BASED_MOV_DR_EXITING); 6034 case EXIT_REASON_IO_INSTRUCTION: 6035 return nested_vmx_exit_handled_io(vcpu, vmcs12); 6036 case EXIT_REASON_GDTR_IDTR: case EXIT_REASON_LDTR_TR: 6037 return nested_cpu_has2(vmcs12, SECONDARY_EXEC_DESC); 6038 case EXIT_REASON_MSR_READ: 6039 case EXIT_REASON_MSR_WRITE: 6040 return nested_vmx_exit_handled_msr(vcpu, vmcs12, exit_reason); 6041 case EXIT_REASON_INVALID_STATE: 6042 return true; 6043 case EXIT_REASON_MWAIT_INSTRUCTION: 6044 return nested_cpu_has(vmcs12, CPU_BASED_MWAIT_EXITING); 6045 case EXIT_REASON_MONITOR_TRAP_FLAG: 6046 return nested_vmx_exit_handled_mtf(vmcs12); 6047 case EXIT_REASON_MONITOR_INSTRUCTION: 6048 return nested_cpu_has(vmcs12, CPU_BASED_MONITOR_EXITING); 6049 case EXIT_REASON_PAUSE_INSTRUCTION: 6050 return nested_cpu_has(vmcs12, CPU_BASED_PAUSE_EXITING) || 6051 nested_cpu_has2(vmcs12, 6052 SECONDARY_EXEC_PAUSE_LOOP_EXITING); 6053 case EXIT_REASON_MCE_DURING_VMENTRY: 6054 return true; 6055 case EXIT_REASON_TPR_BELOW_THRESHOLD: 6056 return nested_cpu_has(vmcs12, CPU_BASED_TPR_SHADOW); 6057 case EXIT_REASON_APIC_ACCESS: 6058 case EXIT_REASON_APIC_WRITE: 6059 case EXIT_REASON_EOI_INDUCED: 6060 /* 6061 * The controls for "virtualize APIC accesses," "APIC- 6062 * register virtualization," and "virtual-interrupt 6063 * delivery" only come from vmcs12. 6064 */ 6065 return true; 6066 case EXIT_REASON_INVPCID: 6067 return 6068 nested_cpu_has2(vmcs12, SECONDARY_EXEC_ENABLE_INVPCID) && 6069 nested_cpu_has(vmcs12, CPU_BASED_INVLPG_EXITING); 6070 case EXIT_REASON_WBINVD: 6071 return nested_cpu_has2(vmcs12, SECONDARY_EXEC_WBINVD_EXITING); 6072 case EXIT_REASON_XSETBV: 6073 return true; 6074 case EXIT_REASON_XSAVES: case EXIT_REASON_XRSTORS: 6075 /* 6076 * This should never happen, since it is not possible to 6077 * set XSS to a non-zero value---neither in L1 nor in L2. 6078 * If if it were, XSS would have to be checked against 6079 * the XSS exit bitmap in vmcs12. 6080 */ 6081 return nested_cpu_has2(vmcs12, SECONDARY_EXEC_XSAVES); 6082 case EXIT_REASON_UMWAIT: 6083 case EXIT_REASON_TPAUSE: 6084 return nested_cpu_has2(vmcs12, 6085 SECONDARY_EXEC_ENABLE_USR_WAIT_PAUSE); 6086 case EXIT_REASON_ENCLS: 6087 return nested_vmx_exit_handled_encls(vcpu, vmcs12); 6088 default: 6089 return true; 6090 } 6091 } 6092 6093 /* 6094 * Conditionally reflect a VM-Exit into L1. Returns %true if the VM-Exit was 6095 * reflected into L1. 6096 */ 6097 bool nested_vmx_reflect_vmexit(struct kvm_vcpu *vcpu) 6098 { 6099 struct vcpu_vmx *vmx = to_vmx(vcpu); 6100 union vmx_exit_reason exit_reason = vmx->exit_reason; 6101 unsigned long exit_qual; 6102 u32 exit_intr_info; 6103 6104 WARN_ON_ONCE(vmx->nested.nested_run_pending); 6105 6106 /* 6107 * Late nested VM-Fail shares the same flow as nested VM-Exit since KVM 6108 * has already loaded L2's state. 6109 */ 6110 if (unlikely(vmx->fail)) { 6111 trace_kvm_nested_vmenter_failed( 6112 "hardware VM-instruction error: ", 6113 vmcs_read32(VM_INSTRUCTION_ERROR)); 6114 exit_intr_info = 0; 6115 exit_qual = 0; 6116 goto reflect_vmexit; 6117 } 6118 6119 trace_kvm_nested_vmexit(vcpu, KVM_ISA_VMX); 6120 6121 /* If L0 (KVM) wants the exit, it trumps L1's desires. */ 6122 if (nested_vmx_l0_wants_exit(vcpu, exit_reason)) 6123 return false; 6124 6125 /* If L1 doesn't want the exit, handle it in L0. */ 6126 if (!nested_vmx_l1_wants_exit(vcpu, exit_reason)) 6127 return false; 6128 6129 /* 6130 * vmcs.VM_EXIT_INTR_INFO is only valid for EXCEPTION_NMI exits. For 6131 * EXTERNAL_INTERRUPT, the value for vmcs12->vm_exit_intr_info would 6132 * need to be synthesized by querying the in-kernel LAPIC, but external 6133 * interrupts are never reflected to L1 so it's a non-issue. 6134 */ 6135 exit_intr_info = vmx_get_intr_info(vcpu); 6136 if (is_exception_with_error_code(exit_intr_info)) { 6137 struct vmcs12 *vmcs12 = get_vmcs12(vcpu); 6138 6139 vmcs12->vm_exit_intr_error_code = 6140 vmcs_read32(VM_EXIT_INTR_ERROR_CODE); 6141 } 6142 exit_qual = vmx_get_exit_qual(vcpu); 6143 6144 reflect_vmexit: 6145 nested_vmx_vmexit(vcpu, exit_reason.full, exit_intr_info, exit_qual); 6146 return true; 6147 } 6148 6149 static int vmx_get_nested_state(struct kvm_vcpu *vcpu, 6150 struct kvm_nested_state __user *user_kvm_nested_state, 6151 u32 user_data_size) 6152 { 6153 struct vcpu_vmx *vmx; 6154 struct vmcs12 *vmcs12; 6155 struct kvm_nested_state kvm_state = { 6156 .flags = 0, 6157 .format = KVM_STATE_NESTED_FORMAT_VMX, 6158 .size = sizeof(kvm_state), 6159 .hdr.vmx.flags = 0, 6160 .hdr.vmx.vmxon_pa = INVALID_GPA, 6161 .hdr.vmx.vmcs12_pa = INVALID_GPA, 6162 .hdr.vmx.preemption_timer_deadline = 0, 6163 }; 6164 struct kvm_vmx_nested_state_data __user *user_vmx_nested_state = 6165 &user_kvm_nested_state->data.vmx[0]; 6166 6167 if (!vcpu) 6168 return kvm_state.size + sizeof(*user_vmx_nested_state); 6169 6170 vmx = to_vmx(vcpu); 6171 vmcs12 = get_vmcs12(vcpu); 6172 6173 if (nested_vmx_allowed(vcpu) && 6174 (vmx->nested.vmxon || vmx->nested.smm.vmxon)) { 6175 kvm_state.hdr.vmx.vmxon_pa = vmx->nested.vmxon_ptr; 6176 kvm_state.hdr.vmx.vmcs12_pa = vmx->nested.current_vmptr; 6177 6178 if (vmx_has_valid_vmcs12(vcpu)) { 6179 kvm_state.size += sizeof(user_vmx_nested_state->vmcs12); 6180 6181 /* 'hv_evmcs_vmptr' can also be EVMPTR_MAP_PENDING here */ 6182 if (vmx->nested.hv_evmcs_vmptr != EVMPTR_INVALID) 6183 kvm_state.flags |= KVM_STATE_NESTED_EVMCS; 6184 6185 if (is_guest_mode(vcpu) && 6186 nested_cpu_has_shadow_vmcs(vmcs12) && 6187 vmcs12->vmcs_link_pointer != INVALID_GPA) 6188 kvm_state.size += sizeof(user_vmx_nested_state->shadow_vmcs12); 6189 } 6190 6191 if (vmx->nested.smm.vmxon) 6192 kvm_state.hdr.vmx.smm.flags |= KVM_STATE_NESTED_SMM_VMXON; 6193 6194 if (vmx->nested.smm.guest_mode) 6195 kvm_state.hdr.vmx.smm.flags |= KVM_STATE_NESTED_SMM_GUEST_MODE; 6196 6197 if (is_guest_mode(vcpu)) { 6198 kvm_state.flags |= KVM_STATE_NESTED_GUEST_MODE; 6199 6200 if (vmx->nested.nested_run_pending) 6201 kvm_state.flags |= KVM_STATE_NESTED_RUN_PENDING; 6202 6203 if (vmx->nested.mtf_pending) 6204 kvm_state.flags |= KVM_STATE_NESTED_MTF_PENDING; 6205 6206 if (nested_cpu_has_preemption_timer(vmcs12) && 6207 vmx->nested.has_preemption_timer_deadline) { 6208 kvm_state.hdr.vmx.flags |= 6209 KVM_STATE_VMX_PREEMPTION_TIMER_DEADLINE; 6210 kvm_state.hdr.vmx.preemption_timer_deadline = 6211 vmx->nested.preemption_timer_deadline; 6212 } 6213 } 6214 } 6215 6216 if (user_data_size < kvm_state.size) 6217 goto out; 6218 6219 if (copy_to_user(user_kvm_nested_state, &kvm_state, sizeof(kvm_state))) 6220 return -EFAULT; 6221 6222 if (!vmx_has_valid_vmcs12(vcpu)) 6223 goto out; 6224 6225 /* 6226 * When running L2, the authoritative vmcs12 state is in the 6227 * vmcs02. When running L1, the authoritative vmcs12 state is 6228 * in the shadow or enlightened vmcs linked to vmcs01, unless 6229 * need_vmcs12_to_shadow_sync is set, in which case, the authoritative 6230 * vmcs12 state is in the vmcs12 already. 6231 */ 6232 if (is_guest_mode(vcpu)) { 6233 sync_vmcs02_to_vmcs12(vcpu, vmcs12); 6234 sync_vmcs02_to_vmcs12_rare(vcpu, vmcs12); 6235 } else { 6236 copy_vmcs02_to_vmcs12_rare(vcpu, get_vmcs12(vcpu)); 6237 if (!vmx->nested.need_vmcs12_to_shadow_sync) { 6238 if (evmptr_is_valid(vmx->nested.hv_evmcs_vmptr)) 6239 /* 6240 * L1 hypervisor is not obliged to keep eVMCS 6241 * clean fields data always up-to-date while 6242 * not in guest mode, 'hv_clean_fields' is only 6243 * supposed to be actual upon vmentry so we need 6244 * to ignore it here and do full copy. 6245 */ 6246 copy_enlightened_to_vmcs12(vmx, 0); 6247 else if (enable_shadow_vmcs) 6248 copy_shadow_to_vmcs12(vmx); 6249 } 6250 } 6251 6252 BUILD_BUG_ON(sizeof(user_vmx_nested_state->vmcs12) < VMCS12_SIZE); 6253 BUILD_BUG_ON(sizeof(user_vmx_nested_state->shadow_vmcs12) < VMCS12_SIZE); 6254 6255 /* 6256 * Copy over the full allocated size of vmcs12 rather than just the size 6257 * of the struct. 6258 */ 6259 if (copy_to_user(user_vmx_nested_state->vmcs12, vmcs12, VMCS12_SIZE)) 6260 return -EFAULT; 6261 6262 if (nested_cpu_has_shadow_vmcs(vmcs12) && 6263 vmcs12->vmcs_link_pointer != INVALID_GPA) { 6264 if (copy_to_user(user_vmx_nested_state->shadow_vmcs12, 6265 get_shadow_vmcs12(vcpu), VMCS12_SIZE)) 6266 return -EFAULT; 6267 } 6268 out: 6269 return kvm_state.size; 6270 } 6271 6272 /* 6273 * Forcibly leave nested mode in order to be able to reset the VCPU later on. 6274 */ 6275 void vmx_leave_nested(struct kvm_vcpu *vcpu) 6276 { 6277 if (is_guest_mode(vcpu)) { 6278 to_vmx(vcpu)->nested.nested_run_pending = 0; 6279 nested_vmx_vmexit(vcpu, -1, 0, 0); 6280 } 6281 free_nested(vcpu); 6282 } 6283 6284 static int vmx_set_nested_state(struct kvm_vcpu *vcpu, 6285 struct kvm_nested_state __user *user_kvm_nested_state, 6286 struct kvm_nested_state *kvm_state) 6287 { 6288 struct vcpu_vmx *vmx = to_vmx(vcpu); 6289 struct vmcs12 *vmcs12; 6290 enum vm_entry_failure_code ignored; 6291 struct kvm_vmx_nested_state_data __user *user_vmx_nested_state = 6292 &user_kvm_nested_state->data.vmx[0]; 6293 int ret; 6294 6295 if (kvm_state->format != KVM_STATE_NESTED_FORMAT_VMX) 6296 return -EINVAL; 6297 6298 if (kvm_state->hdr.vmx.vmxon_pa == INVALID_GPA) { 6299 if (kvm_state->hdr.vmx.smm.flags) 6300 return -EINVAL; 6301 6302 if (kvm_state->hdr.vmx.vmcs12_pa != INVALID_GPA) 6303 return -EINVAL; 6304 6305 /* 6306 * KVM_STATE_NESTED_EVMCS used to signal that KVM should 6307 * enable eVMCS capability on vCPU. However, since then 6308 * code was changed such that flag signals vmcs12 should 6309 * be copied into eVMCS in guest memory. 6310 * 6311 * To preserve backwards compatability, allow user 6312 * to set this flag even when there is no VMXON region. 6313 */ 6314 if (kvm_state->flags & ~KVM_STATE_NESTED_EVMCS) 6315 return -EINVAL; 6316 } else { 6317 if (!nested_vmx_allowed(vcpu)) 6318 return -EINVAL; 6319 6320 if (!page_address_valid(vcpu, kvm_state->hdr.vmx.vmxon_pa)) 6321 return -EINVAL; 6322 } 6323 6324 if ((kvm_state->hdr.vmx.smm.flags & KVM_STATE_NESTED_SMM_GUEST_MODE) && 6325 (kvm_state->flags & KVM_STATE_NESTED_GUEST_MODE)) 6326 return -EINVAL; 6327 6328 if (kvm_state->hdr.vmx.smm.flags & 6329 ~(KVM_STATE_NESTED_SMM_GUEST_MODE | KVM_STATE_NESTED_SMM_VMXON)) 6330 return -EINVAL; 6331 6332 if (kvm_state->hdr.vmx.flags & ~KVM_STATE_VMX_PREEMPTION_TIMER_DEADLINE) 6333 return -EINVAL; 6334 6335 /* 6336 * SMM temporarily disables VMX, so we cannot be in guest mode, 6337 * nor can VMLAUNCH/VMRESUME be pending. Outside SMM, SMM flags 6338 * must be zero. 6339 */ 6340 if (is_smm(vcpu) ? 6341 (kvm_state->flags & 6342 (KVM_STATE_NESTED_GUEST_MODE | KVM_STATE_NESTED_RUN_PENDING)) 6343 : kvm_state->hdr.vmx.smm.flags) 6344 return -EINVAL; 6345 6346 if ((kvm_state->hdr.vmx.smm.flags & KVM_STATE_NESTED_SMM_GUEST_MODE) && 6347 !(kvm_state->hdr.vmx.smm.flags & KVM_STATE_NESTED_SMM_VMXON)) 6348 return -EINVAL; 6349 6350 if ((kvm_state->flags & KVM_STATE_NESTED_EVMCS) && 6351 (!nested_vmx_allowed(vcpu) || !vmx->nested.enlightened_vmcs_enabled)) 6352 return -EINVAL; 6353 6354 vmx_leave_nested(vcpu); 6355 6356 if (kvm_state->hdr.vmx.vmxon_pa == INVALID_GPA) 6357 return 0; 6358 6359 vmx->nested.vmxon_ptr = kvm_state->hdr.vmx.vmxon_pa; 6360 ret = enter_vmx_operation(vcpu); 6361 if (ret) 6362 return ret; 6363 6364 /* Empty 'VMXON' state is permitted if no VMCS loaded */ 6365 if (kvm_state->size < sizeof(*kvm_state) + sizeof(*vmcs12)) { 6366 /* See vmx_has_valid_vmcs12. */ 6367 if ((kvm_state->flags & KVM_STATE_NESTED_GUEST_MODE) || 6368 (kvm_state->flags & KVM_STATE_NESTED_EVMCS) || 6369 (kvm_state->hdr.vmx.vmcs12_pa != INVALID_GPA)) 6370 return -EINVAL; 6371 else 6372 return 0; 6373 } 6374 6375 if (kvm_state->hdr.vmx.vmcs12_pa != INVALID_GPA) { 6376 if (kvm_state->hdr.vmx.vmcs12_pa == kvm_state->hdr.vmx.vmxon_pa || 6377 !page_address_valid(vcpu, kvm_state->hdr.vmx.vmcs12_pa)) 6378 return -EINVAL; 6379 6380 set_current_vmptr(vmx, kvm_state->hdr.vmx.vmcs12_pa); 6381 } else if (kvm_state->flags & KVM_STATE_NESTED_EVMCS) { 6382 /* 6383 * nested_vmx_handle_enlightened_vmptrld() cannot be called 6384 * directly from here as HV_X64_MSR_VP_ASSIST_PAGE may not be 6385 * restored yet. EVMCS will be mapped from 6386 * nested_get_vmcs12_pages(). 6387 */ 6388 vmx->nested.hv_evmcs_vmptr = EVMPTR_MAP_PENDING; 6389 kvm_make_request(KVM_REQ_GET_NESTED_STATE_PAGES, vcpu); 6390 } else { 6391 return -EINVAL; 6392 } 6393 6394 if (kvm_state->hdr.vmx.smm.flags & KVM_STATE_NESTED_SMM_VMXON) { 6395 vmx->nested.smm.vmxon = true; 6396 vmx->nested.vmxon = false; 6397 6398 if (kvm_state->hdr.vmx.smm.flags & KVM_STATE_NESTED_SMM_GUEST_MODE) 6399 vmx->nested.smm.guest_mode = true; 6400 } 6401 6402 vmcs12 = get_vmcs12(vcpu); 6403 if (copy_from_user(vmcs12, user_vmx_nested_state->vmcs12, sizeof(*vmcs12))) 6404 return -EFAULT; 6405 6406 if (vmcs12->hdr.revision_id != VMCS12_REVISION) 6407 return -EINVAL; 6408 6409 if (!(kvm_state->flags & KVM_STATE_NESTED_GUEST_MODE)) 6410 return 0; 6411 6412 vmx->nested.nested_run_pending = 6413 !!(kvm_state->flags & KVM_STATE_NESTED_RUN_PENDING); 6414 6415 vmx->nested.mtf_pending = 6416 !!(kvm_state->flags & KVM_STATE_NESTED_MTF_PENDING); 6417 6418 ret = -EINVAL; 6419 if (nested_cpu_has_shadow_vmcs(vmcs12) && 6420 vmcs12->vmcs_link_pointer != INVALID_GPA) { 6421 struct vmcs12 *shadow_vmcs12 = get_shadow_vmcs12(vcpu); 6422 6423 if (kvm_state->size < 6424 sizeof(*kvm_state) + 6425 sizeof(user_vmx_nested_state->vmcs12) + sizeof(*shadow_vmcs12)) 6426 goto error_guest_mode; 6427 6428 if (copy_from_user(shadow_vmcs12, 6429 user_vmx_nested_state->shadow_vmcs12, 6430 sizeof(*shadow_vmcs12))) { 6431 ret = -EFAULT; 6432 goto error_guest_mode; 6433 } 6434 6435 if (shadow_vmcs12->hdr.revision_id != VMCS12_REVISION || 6436 !shadow_vmcs12->hdr.shadow_vmcs) 6437 goto error_guest_mode; 6438 } 6439 6440 vmx->nested.has_preemption_timer_deadline = false; 6441 if (kvm_state->hdr.vmx.flags & KVM_STATE_VMX_PREEMPTION_TIMER_DEADLINE) { 6442 vmx->nested.has_preemption_timer_deadline = true; 6443 vmx->nested.preemption_timer_deadline = 6444 kvm_state->hdr.vmx.preemption_timer_deadline; 6445 } 6446 6447 if (nested_vmx_check_controls(vcpu, vmcs12) || 6448 nested_vmx_check_host_state(vcpu, vmcs12) || 6449 nested_vmx_check_guest_state(vcpu, vmcs12, &ignored)) 6450 goto error_guest_mode; 6451 6452 vmx->nested.dirty_vmcs12 = true; 6453 vmx->nested.force_msr_bitmap_recalc = true; 6454 ret = nested_vmx_enter_non_root_mode(vcpu, false); 6455 if (ret) 6456 goto error_guest_mode; 6457 6458 return 0; 6459 6460 error_guest_mode: 6461 vmx->nested.nested_run_pending = 0; 6462 return ret; 6463 } 6464 6465 void nested_vmx_set_vmcs_shadowing_bitmap(void) 6466 { 6467 if (enable_shadow_vmcs) { 6468 vmcs_write64(VMREAD_BITMAP, __pa(vmx_vmread_bitmap)); 6469 vmcs_write64(VMWRITE_BITMAP, __pa(vmx_vmwrite_bitmap)); 6470 } 6471 } 6472 6473 /* 6474 * Indexing into the vmcs12 uses the VMCS encoding rotated left by 6. Undo 6475 * that madness to get the encoding for comparison. 6476 */ 6477 #define VMCS12_IDX_TO_ENC(idx) ((u16)(((u16)(idx) >> 6) | ((u16)(idx) << 10))) 6478 6479 static u64 nested_vmx_calc_vmcs_enum_msr(void) 6480 { 6481 /* 6482 * Note these are the so called "index" of the VMCS field encoding, not 6483 * the index into vmcs12. 6484 */ 6485 unsigned int max_idx, idx; 6486 int i; 6487 6488 /* 6489 * For better or worse, KVM allows VMREAD/VMWRITE to all fields in 6490 * vmcs12, regardless of whether or not the associated feature is 6491 * exposed to L1. Simply find the field with the highest index. 6492 */ 6493 max_idx = 0; 6494 for (i = 0; i < nr_vmcs12_fields; i++) { 6495 /* The vmcs12 table is very, very sparsely populated. */ 6496 if (!vmcs12_field_offsets[i]) 6497 continue; 6498 6499 idx = vmcs_field_index(VMCS12_IDX_TO_ENC(i)); 6500 if (idx > max_idx) 6501 max_idx = idx; 6502 } 6503 6504 return (u64)max_idx << VMCS_FIELD_INDEX_SHIFT; 6505 } 6506 6507 /* 6508 * nested_vmx_setup_ctls_msrs() sets up variables containing the values to be 6509 * returned for the various VMX controls MSRs when nested VMX is enabled. 6510 * The same values should also be used to verify that vmcs12 control fields are 6511 * valid during nested entry from L1 to L2. 6512 * Each of these control msrs has a low and high 32-bit half: A low bit is on 6513 * if the corresponding bit in the (32-bit) control field *must* be on, and a 6514 * bit in the high half is on if the corresponding bit in the control field 6515 * may be on. See also vmx_control_verify(). 6516 */ 6517 void nested_vmx_setup_ctls_msrs(struct nested_vmx_msrs *msrs, u32 ept_caps) 6518 { 6519 /* 6520 * Note that as a general rule, the high half of the MSRs (bits in 6521 * the control fields which may be 1) should be initialized by the 6522 * intersection of the underlying hardware's MSR (i.e., features which 6523 * can be supported) and the list of features we want to expose - 6524 * because they are known to be properly supported in our code. 6525 * Also, usually, the low half of the MSRs (bits which must be 1) can 6526 * be set to 0, meaning that L1 may turn off any of these bits. The 6527 * reason is that if one of these bits is necessary, it will appear 6528 * in vmcs01 and prepare_vmcs02, when it bitwise-or's the control 6529 * fields of vmcs01 and vmcs02, will turn these bits off - and 6530 * nested_vmx_l1_wants_exit() will not pass related exits to L1. 6531 * These rules have exceptions below. 6532 */ 6533 6534 /* pin-based controls */ 6535 rdmsr(MSR_IA32_VMX_PINBASED_CTLS, 6536 msrs->pinbased_ctls_low, 6537 msrs->pinbased_ctls_high); 6538 msrs->pinbased_ctls_low |= 6539 PIN_BASED_ALWAYSON_WITHOUT_TRUE_MSR; 6540 msrs->pinbased_ctls_high &= 6541 PIN_BASED_EXT_INTR_MASK | 6542 PIN_BASED_NMI_EXITING | 6543 PIN_BASED_VIRTUAL_NMIS | 6544 (enable_apicv ? PIN_BASED_POSTED_INTR : 0); 6545 msrs->pinbased_ctls_high |= 6546 PIN_BASED_ALWAYSON_WITHOUT_TRUE_MSR | 6547 PIN_BASED_VMX_PREEMPTION_TIMER; 6548 6549 /* exit controls */ 6550 rdmsr(MSR_IA32_VMX_EXIT_CTLS, 6551 msrs->exit_ctls_low, 6552 msrs->exit_ctls_high); 6553 msrs->exit_ctls_low = 6554 VM_EXIT_ALWAYSON_WITHOUT_TRUE_MSR; 6555 6556 msrs->exit_ctls_high &= 6557 #ifdef CONFIG_X86_64 6558 VM_EXIT_HOST_ADDR_SPACE_SIZE | 6559 #endif 6560 VM_EXIT_LOAD_IA32_PAT | VM_EXIT_SAVE_IA32_PAT | 6561 VM_EXIT_CLEAR_BNDCFGS | VM_EXIT_LOAD_IA32_PERF_GLOBAL_CTRL; 6562 msrs->exit_ctls_high |= 6563 VM_EXIT_ALWAYSON_WITHOUT_TRUE_MSR | 6564 VM_EXIT_LOAD_IA32_EFER | VM_EXIT_SAVE_IA32_EFER | 6565 VM_EXIT_SAVE_VMX_PREEMPTION_TIMER | VM_EXIT_ACK_INTR_ON_EXIT; 6566 6567 /* We support free control of debug control saving. */ 6568 msrs->exit_ctls_low &= ~VM_EXIT_SAVE_DEBUG_CONTROLS; 6569 6570 /* entry controls */ 6571 rdmsr(MSR_IA32_VMX_ENTRY_CTLS, 6572 msrs->entry_ctls_low, 6573 msrs->entry_ctls_high); 6574 msrs->entry_ctls_low = 6575 VM_ENTRY_ALWAYSON_WITHOUT_TRUE_MSR; 6576 msrs->entry_ctls_high &= 6577 #ifdef CONFIG_X86_64 6578 VM_ENTRY_IA32E_MODE | 6579 #endif 6580 VM_ENTRY_LOAD_IA32_PAT | VM_ENTRY_LOAD_BNDCFGS | 6581 VM_ENTRY_LOAD_IA32_PERF_GLOBAL_CTRL; 6582 msrs->entry_ctls_high |= 6583 (VM_ENTRY_ALWAYSON_WITHOUT_TRUE_MSR | VM_ENTRY_LOAD_IA32_EFER); 6584 6585 /* We support free control of debug control loading. */ 6586 msrs->entry_ctls_low &= ~VM_ENTRY_LOAD_DEBUG_CONTROLS; 6587 6588 /* cpu-based controls */ 6589 rdmsr(MSR_IA32_VMX_PROCBASED_CTLS, 6590 msrs->procbased_ctls_low, 6591 msrs->procbased_ctls_high); 6592 msrs->procbased_ctls_low = 6593 CPU_BASED_ALWAYSON_WITHOUT_TRUE_MSR; 6594 msrs->procbased_ctls_high &= 6595 CPU_BASED_INTR_WINDOW_EXITING | 6596 CPU_BASED_NMI_WINDOW_EXITING | CPU_BASED_USE_TSC_OFFSETTING | 6597 CPU_BASED_HLT_EXITING | CPU_BASED_INVLPG_EXITING | 6598 CPU_BASED_MWAIT_EXITING | CPU_BASED_CR3_LOAD_EXITING | 6599 CPU_BASED_CR3_STORE_EXITING | 6600 #ifdef CONFIG_X86_64 6601 CPU_BASED_CR8_LOAD_EXITING | CPU_BASED_CR8_STORE_EXITING | 6602 #endif 6603 CPU_BASED_MOV_DR_EXITING | CPU_BASED_UNCOND_IO_EXITING | 6604 CPU_BASED_USE_IO_BITMAPS | CPU_BASED_MONITOR_TRAP_FLAG | 6605 CPU_BASED_MONITOR_EXITING | CPU_BASED_RDPMC_EXITING | 6606 CPU_BASED_RDTSC_EXITING | CPU_BASED_PAUSE_EXITING | 6607 CPU_BASED_TPR_SHADOW | CPU_BASED_ACTIVATE_SECONDARY_CONTROLS; 6608 /* 6609 * We can allow some features even when not supported by the 6610 * hardware. For example, L1 can specify an MSR bitmap - and we 6611 * can use it to avoid exits to L1 - even when L0 runs L2 6612 * without MSR bitmaps. 6613 */ 6614 msrs->procbased_ctls_high |= 6615 CPU_BASED_ALWAYSON_WITHOUT_TRUE_MSR | 6616 CPU_BASED_USE_MSR_BITMAPS; 6617 6618 /* We support free control of CR3 access interception. */ 6619 msrs->procbased_ctls_low &= 6620 ~(CPU_BASED_CR3_LOAD_EXITING | CPU_BASED_CR3_STORE_EXITING); 6621 6622 /* 6623 * secondary cpu-based controls. Do not include those that 6624 * depend on CPUID bits, they are added later by 6625 * vmx_vcpu_after_set_cpuid. 6626 */ 6627 if (msrs->procbased_ctls_high & CPU_BASED_ACTIVATE_SECONDARY_CONTROLS) 6628 rdmsr(MSR_IA32_VMX_PROCBASED_CTLS2, 6629 msrs->secondary_ctls_low, 6630 msrs->secondary_ctls_high); 6631 6632 msrs->secondary_ctls_low = 0; 6633 msrs->secondary_ctls_high &= 6634 SECONDARY_EXEC_DESC | 6635 SECONDARY_EXEC_ENABLE_RDTSCP | 6636 SECONDARY_EXEC_VIRTUALIZE_X2APIC_MODE | 6637 SECONDARY_EXEC_WBINVD_EXITING | 6638 SECONDARY_EXEC_APIC_REGISTER_VIRT | 6639 SECONDARY_EXEC_VIRTUAL_INTR_DELIVERY | 6640 SECONDARY_EXEC_RDRAND_EXITING | 6641 SECONDARY_EXEC_ENABLE_INVPCID | 6642 SECONDARY_EXEC_RDSEED_EXITING | 6643 SECONDARY_EXEC_XSAVES | 6644 SECONDARY_EXEC_TSC_SCALING; 6645 6646 /* 6647 * We can emulate "VMCS shadowing," even if the hardware 6648 * doesn't support it. 6649 */ 6650 msrs->secondary_ctls_high |= 6651 SECONDARY_EXEC_SHADOW_VMCS; 6652 6653 if (enable_ept) { 6654 /* nested EPT: emulate EPT also to L1 */ 6655 msrs->secondary_ctls_high |= 6656 SECONDARY_EXEC_ENABLE_EPT; 6657 msrs->ept_caps = 6658 VMX_EPT_PAGE_WALK_4_BIT | 6659 VMX_EPT_PAGE_WALK_5_BIT | 6660 VMX_EPTP_WB_BIT | 6661 VMX_EPT_INVEPT_BIT | 6662 VMX_EPT_EXECUTE_ONLY_BIT; 6663 6664 msrs->ept_caps &= ept_caps; 6665 msrs->ept_caps |= VMX_EPT_EXTENT_GLOBAL_BIT | 6666 VMX_EPT_EXTENT_CONTEXT_BIT | VMX_EPT_2MB_PAGE_BIT | 6667 VMX_EPT_1GB_PAGE_BIT; 6668 if (enable_ept_ad_bits) { 6669 msrs->secondary_ctls_high |= 6670 SECONDARY_EXEC_ENABLE_PML; 6671 msrs->ept_caps |= VMX_EPT_AD_BIT; 6672 } 6673 } 6674 6675 if (cpu_has_vmx_vmfunc()) { 6676 msrs->secondary_ctls_high |= 6677 SECONDARY_EXEC_ENABLE_VMFUNC; 6678 /* 6679 * Advertise EPTP switching unconditionally 6680 * since we emulate it 6681 */ 6682 if (enable_ept) 6683 msrs->vmfunc_controls = 6684 VMX_VMFUNC_EPTP_SWITCHING; 6685 } 6686 6687 /* 6688 * Old versions of KVM use the single-context version without 6689 * checking for support, so declare that it is supported even 6690 * though it is treated as global context. The alternative is 6691 * not failing the single-context invvpid, and it is worse. 6692 */ 6693 if (enable_vpid) { 6694 msrs->secondary_ctls_high |= 6695 SECONDARY_EXEC_ENABLE_VPID; 6696 msrs->vpid_caps = VMX_VPID_INVVPID_BIT | 6697 VMX_VPID_EXTENT_SUPPORTED_MASK; 6698 } 6699 6700 if (enable_unrestricted_guest) 6701 msrs->secondary_ctls_high |= 6702 SECONDARY_EXEC_UNRESTRICTED_GUEST; 6703 6704 if (flexpriority_enabled) 6705 msrs->secondary_ctls_high |= 6706 SECONDARY_EXEC_VIRTUALIZE_APIC_ACCESSES; 6707 6708 if (enable_sgx) 6709 msrs->secondary_ctls_high |= SECONDARY_EXEC_ENCLS_EXITING; 6710 6711 /* miscellaneous data */ 6712 rdmsr(MSR_IA32_VMX_MISC, 6713 msrs->misc_low, 6714 msrs->misc_high); 6715 msrs->misc_low &= VMX_MISC_SAVE_EFER_LMA; 6716 msrs->misc_low |= 6717 MSR_IA32_VMX_MISC_VMWRITE_SHADOW_RO_FIELDS | 6718 VMX_MISC_EMULATED_PREEMPTION_TIMER_RATE | 6719 VMX_MISC_ACTIVITY_HLT | 6720 VMX_MISC_ACTIVITY_WAIT_SIPI; 6721 msrs->misc_high = 0; 6722 6723 /* 6724 * This MSR reports some information about VMX support. We 6725 * should return information about the VMX we emulate for the 6726 * guest, and the VMCS structure we give it - not about the 6727 * VMX support of the underlying hardware. 6728 */ 6729 msrs->basic = 6730 VMCS12_REVISION | 6731 VMX_BASIC_TRUE_CTLS | 6732 ((u64)VMCS12_SIZE << VMX_BASIC_VMCS_SIZE_SHIFT) | 6733 (VMX_BASIC_MEM_TYPE_WB << VMX_BASIC_MEM_TYPE_SHIFT); 6734 6735 if (cpu_has_vmx_basic_inout()) 6736 msrs->basic |= VMX_BASIC_INOUT; 6737 6738 /* 6739 * These MSRs specify bits which the guest must keep fixed on 6740 * while L1 is in VMXON mode (in L1's root mode, or running an L2). 6741 * We picked the standard core2 setting. 6742 */ 6743 #define VMXON_CR0_ALWAYSON (X86_CR0_PE | X86_CR0_PG | X86_CR0_NE) 6744 #define VMXON_CR4_ALWAYSON X86_CR4_VMXE 6745 msrs->cr0_fixed0 = VMXON_CR0_ALWAYSON; 6746 msrs->cr4_fixed0 = VMXON_CR4_ALWAYSON; 6747 6748 /* These MSRs specify bits which the guest must keep fixed off. */ 6749 rdmsrl(MSR_IA32_VMX_CR0_FIXED1, msrs->cr0_fixed1); 6750 rdmsrl(MSR_IA32_VMX_CR4_FIXED1, msrs->cr4_fixed1); 6751 6752 msrs->vmcs_enum = nested_vmx_calc_vmcs_enum_msr(); 6753 } 6754 6755 void nested_vmx_hardware_unsetup(void) 6756 { 6757 int i; 6758 6759 if (enable_shadow_vmcs) { 6760 for (i = 0; i < VMX_BITMAP_NR; i++) 6761 free_page((unsigned long)vmx_bitmap[i]); 6762 } 6763 } 6764 6765 __init int nested_vmx_hardware_setup(int (*exit_handlers[])(struct kvm_vcpu *)) 6766 { 6767 int i; 6768 6769 if (!cpu_has_vmx_shadow_vmcs()) 6770 enable_shadow_vmcs = 0; 6771 if (enable_shadow_vmcs) { 6772 for (i = 0; i < VMX_BITMAP_NR; i++) { 6773 /* 6774 * The vmx_bitmap is not tied to a VM and so should 6775 * not be charged to a memcg. 6776 */ 6777 vmx_bitmap[i] = (unsigned long *) 6778 __get_free_page(GFP_KERNEL); 6779 if (!vmx_bitmap[i]) { 6780 nested_vmx_hardware_unsetup(); 6781 return -ENOMEM; 6782 } 6783 } 6784 6785 init_vmcs_shadow_fields(); 6786 } 6787 6788 exit_handlers[EXIT_REASON_VMCLEAR] = handle_vmclear; 6789 exit_handlers[EXIT_REASON_VMLAUNCH] = handle_vmlaunch; 6790 exit_handlers[EXIT_REASON_VMPTRLD] = handle_vmptrld; 6791 exit_handlers[EXIT_REASON_VMPTRST] = handle_vmptrst; 6792 exit_handlers[EXIT_REASON_VMREAD] = handle_vmread; 6793 exit_handlers[EXIT_REASON_VMRESUME] = handle_vmresume; 6794 exit_handlers[EXIT_REASON_VMWRITE] = handle_vmwrite; 6795 exit_handlers[EXIT_REASON_VMOFF] = handle_vmoff; 6796 exit_handlers[EXIT_REASON_VMON] = handle_vmon; 6797 exit_handlers[EXIT_REASON_INVEPT] = handle_invept; 6798 exit_handlers[EXIT_REASON_INVVPID] = handle_invvpid; 6799 exit_handlers[EXIT_REASON_VMFUNC] = handle_vmfunc; 6800 6801 return 0; 6802 } 6803 6804 struct kvm_x86_nested_ops vmx_nested_ops = { 6805 .leave_nested = vmx_leave_nested, 6806 .check_events = vmx_check_nested_events, 6807 .hv_timer_pending = nested_vmx_preemption_timer_pending, 6808 .triple_fault = nested_vmx_triple_fault, 6809 .get_state = vmx_get_nested_state, 6810 .set_state = vmx_set_nested_state, 6811 .get_nested_state_pages = vmx_get_nested_state_pages, 6812 .write_log_dirty = nested_vmx_write_pml_buffer, 6813 .enable_evmcs = nested_enable_evmcs, 6814 .get_evmcs_version = nested_get_evmcs_version, 6815 }; 6816