1 // SPDX-License-Identifier: GPL-2.0 2 3 #include <linux/frame.h> 4 #include <linux/percpu.h> 5 6 #include <asm/debugreg.h> 7 #include <asm/mmu_context.h> 8 9 #include "cpuid.h" 10 #include "hyperv.h" 11 #include "mmu.h" 12 #include "nested.h" 13 #include "pmu.h" 14 #include "trace.h" 15 #include "x86.h" 16 17 static bool __read_mostly enable_shadow_vmcs = 1; 18 module_param_named(enable_shadow_vmcs, enable_shadow_vmcs, bool, S_IRUGO); 19 20 static bool __read_mostly nested_early_check = 0; 21 module_param(nested_early_check, bool, S_IRUGO); 22 23 #define CC(consistency_check) \ 24 ({ \ 25 bool failed = (consistency_check); \ 26 if (failed) \ 27 trace_kvm_nested_vmenter_failed(#consistency_check, 0); \ 28 failed; \ 29 }) 30 31 /* 32 * Hyper-V requires all of these, so mark them as supported even though 33 * they are just treated the same as all-context. 34 */ 35 #define VMX_VPID_EXTENT_SUPPORTED_MASK \ 36 (VMX_VPID_EXTENT_INDIVIDUAL_ADDR_BIT | \ 37 VMX_VPID_EXTENT_SINGLE_CONTEXT_BIT | \ 38 VMX_VPID_EXTENT_GLOBAL_CONTEXT_BIT | \ 39 VMX_VPID_EXTENT_SINGLE_NON_GLOBAL_BIT) 40 41 #define VMX_MISC_EMULATED_PREEMPTION_TIMER_RATE 5 42 43 enum { 44 VMX_VMREAD_BITMAP, 45 VMX_VMWRITE_BITMAP, 46 VMX_BITMAP_NR 47 }; 48 static unsigned long *vmx_bitmap[VMX_BITMAP_NR]; 49 50 #define vmx_vmread_bitmap (vmx_bitmap[VMX_VMREAD_BITMAP]) 51 #define vmx_vmwrite_bitmap (vmx_bitmap[VMX_VMWRITE_BITMAP]) 52 53 struct shadow_vmcs_field { 54 u16 encoding; 55 u16 offset; 56 }; 57 static struct shadow_vmcs_field shadow_read_only_fields[] = { 58 #define SHADOW_FIELD_RO(x, y) { x, offsetof(struct vmcs12, y) }, 59 #include "vmcs_shadow_fields.h" 60 }; 61 static int max_shadow_read_only_fields = 62 ARRAY_SIZE(shadow_read_only_fields); 63 64 static struct shadow_vmcs_field shadow_read_write_fields[] = { 65 #define SHADOW_FIELD_RW(x, y) { x, offsetof(struct vmcs12, y) }, 66 #include "vmcs_shadow_fields.h" 67 }; 68 static int max_shadow_read_write_fields = 69 ARRAY_SIZE(shadow_read_write_fields); 70 71 static void init_vmcs_shadow_fields(void) 72 { 73 int i, j; 74 75 memset(vmx_vmread_bitmap, 0xff, PAGE_SIZE); 76 memset(vmx_vmwrite_bitmap, 0xff, PAGE_SIZE); 77 78 for (i = j = 0; i < max_shadow_read_only_fields; i++) { 79 struct shadow_vmcs_field entry = shadow_read_only_fields[i]; 80 u16 field = entry.encoding; 81 82 if (vmcs_field_width(field) == VMCS_FIELD_WIDTH_U64 && 83 (i + 1 == max_shadow_read_only_fields || 84 shadow_read_only_fields[i + 1].encoding != field + 1)) 85 pr_err("Missing field from shadow_read_only_field %x\n", 86 field + 1); 87 88 clear_bit(field, vmx_vmread_bitmap); 89 if (field & 1) 90 #ifdef CONFIG_X86_64 91 continue; 92 #else 93 entry.offset += sizeof(u32); 94 #endif 95 shadow_read_only_fields[j++] = entry; 96 } 97 max_shadow_read_only_fields = j; 98 99 for (i = j = 0; i < max_shadow_read_write_fields; i++) { 100 struct shadow_vmcs_field entry = shadow_read_write_fields[i]; 101 u16 field = entry.encoding; 102 103 if (vmcs_field_width(field) == VMCS_FIELD_WIDTH_U64 && 104 (i + 1 == max_shadow_read_write_fields || 105 shadow_read_write_fields[i + 1].encoding != field + 1)) 106 pr_err("Missing field from shadow_read_write_field %x\n", 107 field + 1); 108 109 WARN_ONCE(field >= GUEST_ES_AR_BYTES && 110 field <= GUEST_TR_AR_BYTES, 111 "Update vmcs12_write_any() to drop reserved bits from AR_BYTES"); 112 113 /* 114 * PML and the preemption timer can be emulated, but the 115 * processor cannot vmwrite to fields that don't exist 116 * on bare metal. 117 */ 118 switch (field) { 119 case GUEST_PML_INDEX: 120 if (!cpu_has_vmx_pml()) 121 continue; 122 break; 123 case VMX_PREEMPTION_TIMER_VALUE: 124 if (!cpu_has_vmx_preemption_timer()) 125 continue; 126 break; 127 case GUEST_INTR_STATUS: 128 if (!cpu_has_vmx_apicv()) 129 continue; 130 break; 131 default: 132 break; 133 } 134 135 clear_bit(field, vmx_vmwrite_bitmap); 136 clear_bit(field, vmx_vmread_bitmap); 137 if (field & 1) 138 #ifdef CONFIG_X86_64 139 continue; 140 #else 141 entry.offset += sizeof(u32); 142 #endif 143 shadow_read_write_fields[j++] = entry; 144 } 145 max_shadow_read_write_fields = j; 146 } 147 148 /* 149 * The following 3 functions, nested_vmx_succeed()/failValid()/failInvalid(), 150 * set the success or error code of an emulated VMX instruction (as specified 151 * by Vol 2B, VMX Instruction Reference, "Conventions"), and skip the emulated 152 * instruction. 153 */ 154 static int nested_vmx_succeed(struct kvm_vcpu *vcpu) 155 { 156 vmx_set_rflags(vcpu, vmx_get_rflags(vcpu) 157 & ~(X86_EFLAGS_CF | X86_EFLAGS_PF | X86_EFLAGS_AF | 158 X86_EFLAGS_ZF | X86_EFLAGS_SF | X86_EFLAGS_OF)); 159 return kvm_skip_emulated_instruction(vcpu); 160 } 161 162 static int nested_vmx_failInvalid(struct kvm_vcpu *vcpu) 163 { 164 vmx_set_rflags(vcpu, (vmx_get_rflags(vcpu) 165 & ~(X86_EFLAGS_PF | X86_EFLAGS_AF | X86_EFLAGS_ZF | 166 X86_EFLAGS_SF | X86_EFLAGS_OF)) 167 | X86_EFLAGS_CF); 168 return kvm_skip_emulated_instruction(vcpu); 169 } 170 171 static int nested_vmx_failValid(struct kvm_vcpu *vcpu, 172 u32 vm_instruction_error) 173 { 174 struct vcpu_vmx *vmx = to_vmx(vcpu); 175 176 /* 177 * failValid writes the error number to the current VMCS, which 178 * can't be done if there isn't a current VMCS. 179 */ 180 if (vmx->nested.current_vmptr == -1ull && !vmx->nested.hv_evmcs) 181 return nested_vmx_failInvalid(vcpu); 182 183 vmx_set_rflags(vcpu, (vmx_get_rflags(vcpu) 184 & ~(X86_EFLAGS_CF | X86_EFLAGS_PF | X86_EFLAGS_AF | 185 X86_EFLAGS_SF | X86_EFLAGS_OF)) 186 | X86_EFLAGS_ZF); 187 get_vmcs12(vcpu)->vm_instruction_error = vm_instruction_error; 188 /* 189 * We don't need to force a shadow sync because 190 * VM_INSTRUCTION_ERROR is not shadowed 191 */ 192 return kvm_skip_emulated_instruction(vcpu); 193 } 194 195 static void nested_vmx_abort(struct kvm_vcpu *vcpu, u32 indicator) 196 { 197 /* TODO: not to reset guest simply here. */ 198 kvm_make_request(KVM_REQ_TRIPLE_FAULT, vcpu); 199 pr_debug_ratelimited("kvm: nested vmx abort, indicator %d\n", indicator); 200 } 201 202 static inline bool vmx_control_verify(u32 control, u32 low, u32 high) 203 { 204 return fixed_bits_valid(control, low, high); 205 } 206 207 static inline u64 vmx_control_msr(u32 low, u32 high) 208 { 209 return low | ((u64)high << 32); 210 } 211 212 static void vmx_disable_shadow_vmcs(struct vcpu_vmx *vmx) 213 { 214 secondary_exec_controls_clearbit(vmx, SECONDARY_EXEC_SHADOW_VMCS); 215 vmcs_write64(VMCS_LINK_POINTER, -1ull); 216 vmx->nested.need_vmcs12_to_shadow_sync = false; 217 } 218 219 static inline void nested_release_evmcs(struct kvm_vcpu *vcpu) 220 { 221 struct vcpu_vmx *vmx = to_vmx(vcpu); 222 223 if (!vmx->nested.hv_evmcs) 224 return; 225 226 kvm_vcpu_unmap(vcpu, &vmx->nested.hv_evmcs_map, true); 227 vmx->nested.hv_evmcs_vmptr = -1ull; 228 vmx->nested.hv_evmcs = NULL; 229 } 230 231 /* 232 * Free whatever needs to be freed from vmx->nested when L1 goes down, or 233 * just stops using VMX. 234 */ 235 static void free_nested(struct kvm_vcpu *vcpu) 236 { 237 struct vcpu_vmx *vmx = to_vmx(vcpu); 238 239 if (!vmx->nested.vmxon && !vmx->nested.smm.vmxon) 240 return; 241 242 kvm_clear_request(KVM_REQ_GET_VMCS12_PAGES, vcpu); 243 244 vmx->nested.vmxon = false; 245 vmx->nested.smm.vmxon = false; 246 free_vpid(vmx->nested.vpid02); 247 vmx->nested.posted_intr_nv = -1; 248 vmx->nested.current_vmptr = -1ull; 249 if (enable_shadow_vmcs) { 250 vmx_disable_shadow_vmcs(vmx); 251 vmcs_clear(vmx->vmcs01.shadow_vmcs); 252 free_vmcs(vmx->vmcs01.shadow_vmcs); 253 vmx->vmcs01.shadow_vmcs = NULL; 254 } 255 kfree(vmx->nested.cached_vmcs12); 256 vmx->nested.cached_vmcs12 = NULL; 257 kfree(vmx->nested.cached_shadow_vmcs12); 258 vmx->nested.cached_shadow_vmcs12 = NULL; 259 /* Unpin physical memory we referred to in the vmcs02 */ 260 if (vmx->nested.apic_access_page) { 261 kvm_release_page_clean(vmx->nested.apic_access_page); 262 vmx->nested.apic_access_page = NULL; 263 } 264 kvm_vcpu_unmap(vcpu, &vmx->nested.virtual_apic_map, true); 265 kvm_vcpu_unmap(vcpu, &vmx->nested.pi_desc_map, true); 266 vmx->nested.pi_desc = NULL; 267 268 kvm_mmu_free_roots(vcpu, &vcpu->arch.guest_mmu, KVM_MMU_ROOTS_ALL); 269 270 nested_release_evmcs(vcpu); 271 272 free_loaded_vmcs(&vmx->nested.vmcs02); 273 } 274 275 static void vmx_sync_vmcs_host_state(struct vcpu_vmx *vmx, 276 struct loaded_vmcs *prev) 277 { 278 struct vmcs_host_state *dest, *src; 279 280 if (unlikely(!vmx->guest_state_loaded)) 281 return; 282 283 src = &prev->host_state; 284 dest = &vmx->loaded_vmcs->host_state; 285 286 vmx_set_host_fs_gs(dest, src->fs_sel, src->gs_sel, src->fs_base, src->gs_base); 287 dest->ldt_sel = src->ldt_sel; 288 #ifdef CONFIG_X86_64 289 dest->ds_sel = src->ds_sel; 290 dest->es_sel = src->es_sel; 291 #endif 292 } 293 294 static void vmx_switch_vmcs(struct kvm_vcpu *vcpu, struct loaded_vmcs *vmcs) 295 { 296 struct vcpu_vmx *vmx = to_vmx(vcpu); 297 struct loaded_vmcs *prev; 298 int cpu; 299 300 if (vmx->loaded_vmcs == vmcs) 301 return; 302 303 cpu = get_cpu(); 304 prev = vmx->loaded_vmcs; 305 vmx->loaded_vmcs = vmcs; 306 vmx_vcpu_load_vmcs(vcpu, cpu); 307 vmx_sync_vmcs_host_state(vmx, prev); 308 put_cpu(); 309 310 vmx_segment_cache_clear(vmx); 311 } 312 313 /* 314 * Ensure that the current vmcs of the logical processor is the 315 * vmcs01 of the vcpu before calling free_nested(). 316 */ 317 void nested_vmx_free_vcpu(struct kvm_vcpu *vcpu) 318 { 319 vcpu_load(vcpu); 320 vmx_leave_nested(vcpu); 321 vmx_switch_vmcs(vcpu, &to_vmx(vcpu)->vmcs01); 322 free_nested(vcpu); 323 vcpu_put(vcpu); 324 } 325 326 static void nested_ept_inject_page_fault(struct kvm_vcpu *vcpu, 327 struct x86_exception *fault) 328 { 329 struct vmcs12 *vmcs12 = get_vmcs12(vcpu); 330 struct vcpu_vmx *vmx = to_vmx(vcpu); 331 u32 exit_reason; 332 unsigned long exit_qualification = vcpu->arch.exit_qualification; 333 334 if (vmx->nested.pml_full) { 335 exit_reason = EXIT_REASON_PML_FULL; 336 vmx->nested.pml_full = false; 337 exit_qualification &= INTR_INFO_UNBLOCK_NMI; 338 } else if (fault->error_code & PFERR_RSVD_MASK) 339 exit_reason = EXIT_REASON_EPT_MISCONFIG; 340 else 341 exit_reason = EXIT_REASON_EPT_VIOLATION; 342 343 nested_vmx_vmexit(vcpu, exit_reason, 0, exit_qualification); 344 vmcs12->guest_physical_address = fault->address; 345 } 346 347 static void nested_ept_init_mmu_context(struct kvm_vcpu *vcpu) 348 { 349 WARN_ON(mmu_is_nested(vcpu)); 350 351 vcpu->arch.mmu = &vcpu->arch.guest_mmu; 352 kvm_init_shadow_ept_mmu(vcpu, 353 to_vmx(vcpu)->nested.msrs.ept_caps & 354 VMX_EPT_EXECUTE_ONLY_BIT, 355 nested_ept_ad_enabled(vcpu), 356 nested_ept_get_cr3(vcpu)); 357 vcpu->arch.mmu->set_cr3 = vmx_set_cr3; 358 vcpu->arch.mmu->get_cr3 = nested_ept_get_cr3; 359 vcpu->arch.mmu->inject_page_fault = nested_ept_inject_page_fault; 360 vcpu->arch.mmu->get_pdptr = kvm_pdptr_read; 361 362 vcpu->arch.walk_mmu = &vcpu->arch.nested_mmu; 363 } 364 365 static void nested_ept_uninit_mmu_context(struct kvm_vcpu *vcpu) 366 { 367 vcpu->arch.mmu = &vcpu->arch.root_mmu; 368 vcpu->arch.walk_mmu = &vcpu->arch.root_mmu; 369 } 370 371 static bool nested_vmx_is_page_fault_vmexit(struct vmcs12 *vmcs12, 372 u16 error_code) 373 { 374 bool inequality, bit; 375 376 bit = (vmcs12->exception_bitmap & (1u << PF_VECTOR)) != 0; 377 inequality = 378 (error_code & vmcs12->page_fault_error_code_mask) != 379 vmcs12->page_fault_error_code_match; 380 return inequality ^ bit; 381 } 382 383 384 /* 385 * KVM wants to inject page-faults which it got to the guest. This function 386 * checks whether in a nested guest, we need to inject them to L1 or L2. 387 */ 388 static int nested_vmx_check_exception(struct kvm_vcpu *vcpu, unsigned long *exit_qual) 389 { 390 struct vmcs12 *vmcs12 = get_vmcs12(vcpu); 391 unsigned int nr = vcpu->arch.exception.nr; 392 bool has_payload = vcpu->arch.exception.has_payload; 393 unsigned long payload = vcpu->arch.exception.payload; 394 395 if (nr == PF_VECTOR) { 396 if (vcpu->arch.exception.nested_apf) { 397 *exit_qual = vcpu->arch.apf.nested_apf_token; 398 return 1; 399 } 400 if (nested_vmx_is_page_fault_vmexit(vmcs12, 401 vcpu->arch.exception.error_code)) { 402 *exit_qual = has_payload ? payload : vcpu->arch.cr2; 403 return 1; 404 } 405 } else if (vmcs12->exception_bitmap & (1u << nr)) { 406 if (nr == DB_VECTOR) { 407 if (!has_payload) { 408 payload = vcpu->arch.dr6; 409 payload &= ~(DR6_FIXED_1 | DR6_BT); 410 payload ^= DR6_RTM; 411 } 412 *exit_qual = payload; 413 } else 414 *exit_qual = 0; 415 return 1; 416 } 417 418 return 0; 419 } 420 421 422 static void vmx_inject_page_fault_nested(struct kvm_vcpu *vcpu, 423 struct x86_exception *fault) 424 { 425 struct vmcs12 *vmcs12 = get_vmcs12(vcpu); 426 427 WARN_ON(!is_guest_mode(vcpu)); 428 429 if (nested_vmx_is_page_fault_vmexit(vmcs12, fault->error_code) && 430 !to_vmx(vcpu)->nested.nested_run_pending) { 431 vmcs12->vm_exit_intr_error_code = fault->error_code; 432 nested_vmx_vmexit(vcpu, EXIT_REASON_EXCEPTION_NMI, 433 PF_VECTOR | INTR_TYPE_HARD_EXCEPTION | 434 INTR_INFO_DELIVER_CODE_MASK | INTR_INFO_VALID_MASK, 435 fault->address); 436 } else { 437 kvm_inject_page_fault(vcpu, fault); 438 } 439 } 440 441 static bool page_address_valid(struct kvm_vcpu *vcpu, gpa_t gpa) 442 { 443 return PAGE_ALIGNED(gpa) && !(gpa >> cpuid_maxphyaddr(vcpu)); 444 } 445 446 static int nested_vmx_check_io_bitmap_controls(struct kvm_vcpu *vcpu, 447 struct vmcs12 *vmcs12) 448 { 449 if (!nested_cpu_has(vmcs12, CPU_BASED_USE_IO_BITMAPS)) 450 return 0; 451 452 if (CC(!page_address_valid(vcpu, vmcs12->io_bitmap_a)) || 453 CC(!page_address_valid(vcpu, vmcs12->io_bitmap_b))) 454 return -EINVAL; 455 456 return 0; 457 } 458 459 static int nested_vmx_check_msr_bitmap_controls(struct kvm_vcpu *vcpu, 460 struct vmcs12 *vmcs12) 461 { 462 if (!nested_cpu_has(vmcs12, CPU_BASED_USE_MSR_BITMAPS)) 463 return 0; 464 465 if (CC(!page_address_valid(vcpu, vmcs12->msr_bitmap))) 466 return -EINVAL; 467 468 return 0; 469 } 470 471 static int nested_vmx_check_tpr_shadow_controls(struct kvm_vcpu *vcpu, 472 struct vmcs12 *vmcs12) 473 { 474 if (!nested_cpu_has(vmcs12, CPU_BASED_TPR_SHADOW)) 475 return 0; 476 477 if (CC(!page_address_valid(vcpu, vmcs12->virtual_apic_page_addr))) 478 return -EINVAL; 479 480 return 0; 481 } 482 483 /* 484 * Check if MSR is intercepted for L01 MSR bitmap. 485 */ 486 static bool msr_write_intercepted_l01(struct kvm_vcpu *vcpu, u32 msr) 487 { 488 unsigned long *msr_bitmap; 489 int f = sizeof(unsigned long); 490 491 if (!cpu_has_vmx_msr_bitmap()) 492 return true; 493 494 msr_bitmap = to_vmx(vcpu)->vmcs01.msr_bitmap; 495 496 if (msr <= 0x1fff) { 497 return !!test_bit(msr, msr_bitmap + 0x800 / f); 498 } else if ((msr >= 0xc0000000) && (msr <= 0xc0001fff)) { 499 msr &= 0x1fff; 500 return !!test_bit(msr, msr_bitmap + 0xc00 / f); 501 } 502 503 return true; 504 } 505 506 /* 507 * If a msr is allowed by L0, we should check whether it is allowed by L1. 508 * The corresponding bit will be cleared unless both of L0 and L1 allow it. 509 */ 510 static void nested_vmx_disable_intercept_for_msr(unsigned long *msr_bitmap_l1, 511 unsigned long *msr_bitmap_nested, 512 u32 msr, int type) 513 { 514 int f = sizeof(unsigned long); 515 516 /* 517 * See Intel PRM Vol. 3, 20.6.9 (MSR-Bitmap Address). Early manuals 518 * have the write-low and read-high bitmap offsets the wrong way round. 519 * We can control MSRs 0x00000000-0x00001fff and 0xc0000000-0xc0001fff. 520 */ 521 if (msr <= 0x1fff) { 522 if (type & MSR_TYPE_R && 523 !test_bit(msr, msr_bitmap_l1 + 0x000 / f)) 524 /* read-low */ 525 __clear_bit(msr, msr_bitmap_nested + 0x000 / f); 526 527 if (type & MSR_TYPE_W && 528 !test_bit(msr, msr_bitmap_l1 + 0x800 / f)) 529 /* write-low */ 530 __clear_bit(msr, msr_bitmap_nested + 0x800 / f); 531 532 } else if ((msr >= 0xc0000000) && (msr <= 0xc0001fff)) { 533 msr &= 0x1fff; 534 if (type & MSR_TYPE_R && 535 !test_bit(msr, msr_bitmap_l1 + 0x400 / f)) 536 /* read-high */ 537 __clear_bit(msr, msr_bitmap_nested + 0x400 / f); 538 539 if (type & MSR_TYPE_W && 540 !test_bit(msr, msr_bitmap_l1 + 0xc00 / f)) 541 /* write-high */ 542 __clear_bit(msr, msr_bitmap_nested + 0xc00 / f); 543 544 } 545 } 546 547 static inline void enable_x2apic_msr_intercepts(unsigned long *msr_bitmap) 548 { 549 int msr; 550 551 for (msr = 0x800; msr <= 0x8ff; msr += BITS_PER_LONG) { 552 unsigned word = msr / BITS_PER_LONG; 553 554 msr_bitmap[word] = ~0; 555 msr_bitmap[word + (0x800 / sizeof(long))] = ~0; 556 } 557 } 558 559 /* 560 * Merge L0's and L1's MSR bitmap, return false to indicate that 561 * we do not use the hardware. 562 */ 563 static inline bool nested_vmx_prepare_msr_bitmap(struct kvm_vcpu *vcpu, 564 struct vmcs12 *vmcs12) 565 { 566 int msr; 567 unsigned long *msr_bitmap_l1; 568 unsigned long *msr_bitmap_l0 = to_vmx(vcpu)->nested.vmcs02.msr_bitmap; 569 struct kvm_host_map *map = &to_vmx(vcpu)->nested.msr_bitmap_map; 570 571 /* Nothing to do if the MSR bitmap is not in use. */ 572 if (!cpu_has_vmx_msr_bitmap() || 573 !nested_cpu_has(vmcs12, CPU_BASED_USE_MSR_BITMAPS)) 574 return false; 575 576 if (kvm_vcpu_map(vcpu, gpa_to_gfn(vmcs12->msr_bitmap), map)) 577 return false; 578 579 msr_bitmap_l1 = (unsigned long *)map->hva; 580 581 /* 582 * To keep the control flow simple, pay eight 8-byte writes (sixteen 583 * 4-byte writes on 32-bit systems) up front to enable intercepts for 584 * the x2APIC MSR range and selectively disable them below. 585 */ 586 enable_x2apic_msr_intercepts(msr_bitmap_l0); 587 588 if (nested_cpu_has_virt_x2apic_mode(vmcs12)) { 589 if (nested_cpu_has_apic_reg_virt(vmcs12)) { 590 /* 591 * L0 need not intercept reads for MSRs between 0x800 592 * and 0x8ff, it just lets the processor take the value 593 * from the virtual-APIC page; take those 256 bits 594 * directly from the L1 bitmap. 595 */ 596 for (msr = 0x800; msr <= 0x8ff; msr += BITS_PER_LONG) { 597 unsigned word = msr / BITS_PER_LONG; 598 599 msr_bitmap_l0[word] = msr_bitmap_l1[word]; 600 } 601 } 602 603 nested_vmx_disable_intercept_for_msr( 604 msr_bitmap_l1, msr_bitmap_l0, 605 X2APIC_MSR(APIC_TASKPRI), 606 MSR_TYPE_R | MSR_TYPE_W); 607 608 if (nested_cpu_has_vid(vmcs12)) { 609 nested_vmx_disable_intercept_for_msr( 610 msr_bitmap_l1, msr_bitmap_l0, 611 X2APIC_MSR(APIC_EOI), 612 MSR_TYPE_W); 613 nested_vmx_disable_intercept_for_msr( 614 msr_bitmap_l1, msr_bitmap_l0, 615 X2APIC_MSR(APIC_SELF_IPI), 616 MSR_TYPE_W); 617 } 618 } 619 620 /* KVM unconditionally exposes the FS/GS base MSRs to L1. */ 621 nested_vmx_disable_intercept_for_msr(msr_bitmap_l1, msr_bitmap_l0, 622 MSR_FS_BASE, MSR_TYPE_RW); 623 624 nested_vmx_disable_intercept_for_msr(msr_bitmap_l1, msr_bitmap_l0, 625 MSR_GS_BASE, MSR_TYPE_RW); 626 627 nested_vmx_disable_intercept_for_msr(msr_bitmap_l1, msr_bitmap_l0, 628 MSR_KERNEL_GS_BASE, MSR_TYPE_RW); 629 630 /* 631 * Checking the L0->L1 bitmap is trying to verify two things: 632 * 633 * 1. L0 gave a permission to L1 to actually passthrough the MSR. This 634 * ensures that we do not accidentally generate an L02 MSR bitmap 635 * from the L12 MSR bitmap that is too permissive. 636 * 2. That L1 or L2s have actually used the MSR. This avoids 637 * unnecessarily merging of the bitmap if the MSR is unused. This 638 * works properly because we only update the L01 MSR bitmap lazily. 639 * So even if L0 should pass L1 these MSRs, the L01 bitmap is only 640 * updated to reflect this when L1 (or its L2s) actually write to 641 * the MSR. 642 */ 643 if (!msr_write_intercepted_l01(vcpu, MSR_IA32_SPEC_CTRL)) 644 nested_vmx_disable_intercept_for_msr( 645 msr_bitmap_l1, msr_bitmap_l0, 646 MSR_IA32_SPEC_CTRL, 647 MSR_TYPE_R | MSR_TYPE_W); 648 649 if (!msr_write_intercepted_l01(vcpu, MSR_IA32_PRED_CMD)) 650 nested_vmx_disable_intercept_for_msr( 651 msr_bitmap_l1, msr_bitmap_l0, 652 MSR_IA32_PRED_CMD, 653 MSR_TYPE_W); 654 655 kvm_vcpu_unmap(vcpu, &to_vmx(vcpu)->nested.msr_bitmap_map, false); 656 657 return true; 658 } 659 660 static void nested_cache_shadow_vmcs12(struct kvm_vcpu *vcpu, 661 struct vmcs12 *vmcs12) 662 { 663 struct kvm_host_map map; 664 struct vmcs12 *shadow; 665 666 if (!nested_cpu_has_shadow_vmcs(vmcs12) || 667 vmcs12->vmcs_link_pointer == -1ull) 668 return; 669 670 shadow = get_shadow_vmcs12(vcpu); 671 672 if (kvm_vcpu_map(vcpu, gpa_to_gfn(vmcs12->vmcs_link_pointer), &map)) 673 return; 674 675 memcpy(shadow, map.hva, VMCS12_SIZE); 676 kvm_vcpu_unmap(vcpu, &map, false); 677 } 678 679 static void nested_flush_cached_shadow_vmcs12(struct kvm_vcpu *vcpu, 680 struct vmcs12 *vmcs12) 681 { 682 struct vcpu_vmx *vmx = to_vmx(vcpu); 683 684 if (!nested_cpu_has_shadow_vmcs(vmcs12) || 685 vmcs12->vmcs_link_pointer == -1ull) 686 return; 687 688 kvm_write_guest(vmx->vcpu.kvm, vmcs12->vmcs_link_pointer, 689 get_shadow_vmcs12(vcpu), VMCS12_SIZE); 690 } 691 692 /* 693 * In nested virtualization, check if L1 has set 694 * VM_EXIT_ACK_INTR_ON_EXIT 695 */ 696 static bool nested_exit_intr_ack_set(struct kvm_vcpu *vcpu) 697 { 698 return get_vmcs12(vcpu)->vm_exit_controls & 699 VM_EXIT_ACK_INTR_ON_EXIT; 700 } 701 702 static bool nested_exit_on_nmi(struct kvm_vcpu *vcpu) 703 { 704 return nested_cpu_has_nmi_exiting(get_vmcs12(vcpu)); 705 } 706 707 static int nested_vmx_check_apic_access_controls(struct kvm_vcpu *vcpu, 708 struct vmcs12 *vmcs12) 709 { 710 if (nested_cpu_has2(vmcs12, SECONDARY_EXEC_VIRTUALIZE_APIC_ACCESSES) && 711 CC(!page_address_valid(vcpu, vmcs12->apic_access_addr))) 712 return -EINVAL; 713 else 714 return 0; 715 } 716 717 static int nested_vmx_check_apicv_controls(struct kvm_vcpu *vcpu, 718 struct vmcs12 *vmcs12) 719 { 720 if (!nested_cpu_has_virt_x2apic_mode(vmcs12) && 721 !nested_cpu_has_apic_reg_virt(vmcs12) && 722 !nested_cpu_has_vid(vmcs12) && 723 !nested_cpu_has_posted_intr(vmcs12)) 724 return 0; 725 726 /* 727 * If virtualize x2apic mode is enabled, 728 * virtualize apic access must be disabled. 729 */ 730 if (CC(nested_cpu_has_virt_x2apic_mode(vmcs12) && 731 nested_cpu_has2(vmcs12, SECONDARY_EXEC_VIRTUALIZE_APIC_ACCESSES))) 732 return -EINVAL; 733 734 /* 735 * If virtual interrupt delivery is enabled, 736 * we must exit on external interrupts. 737 */ 738 if (CC(nested_cpu_has_vid(vmcs12) && !nested_exit_on_intr(vcpu))) 739 return -EINVAL; 740 741 /* 742 * bits 15:8 should be zero in posted_intr_nv, 743 * the descriptor address has been already checked 744 * in nested_get_vmcs12_pages. 745 * 746 * bits 5:0 of posted_intr_desc_addr should be zero. 747 */ 748 if (nested_cpu_has_posted_intr(vmcs12) && 749 (CC(!nested_cpu_has_vid(vmcs12)) || 750 CC(!nested_exit_intr_ack_set(vcpu)) || 751 CC((vmcs12->posted_intr_nv & 0xff00)) || 752 CC((vmcs12->posted_intr_desc_addr & 0x3f)) || 753 CC((vmcs12->posted_intr_desc_addr >> cpuid_maxphyaddr(vcpu))))) 754 return -EINVAL; 755 756 /* tpr shadow is needed by all apicv features. */ 757 if (CC(!nested_cpu_has(vmcs12, CPU_BASED_TPR_SHADOW))) 758 return -EINVAL; 759 760 return 0; 761 } 762 763 static int nested_vmx_check_msr_switch(struct kvm_vcpu *vcpu, 764 u32 count, u64 addr) 765 { 766 int maxphyaddr; 767 768 if (count == 0) 769 return 0; 770 maxphyaddr = cpuid_maxphyaddr(vcpu); 771 if (!IS_ALIGNED(addr, 16) || addr >> maxphyaddr || 772 (addr + count * sizeof(struct vmx_msr_entry) - 1) >> maxphyaddr) 773 return -EINVAL; 774 775 return 0; 776 } 777 778 static int nested_vmx_check_exit_msr_switch_controls(struct kvm_vcpu *vcpu, 779 struct vmcs12 *vmcs12) 780 { 781 if (CC(nested_vmx_check_msr_switch(vcpu, 782 vmcs12->vm_exit_msr_load_count, 783 vmcs12->vm_exit_msr_load_addr)) || 784 CC(nested_vmx_check_msr_switch(vcpu, 785 vmcs12->vm_exit_msr_store_count, 786 vmcs12->vm_exit_msr_store_addr))) 787 return -EINVAL; 788 789 return 0; 790 } 791 792 static int nested_vmx_check_entry_msr_switch_controls(struct kvm_vcpu *vcpu, 793 struct vmcs12 *vmcs12) 794 { 795 if (CC(nested_vmx_check_msr_switch(vcpu, 796 vmcs12->vm_entry_msr_load_count, 797 vmcs12->vm_entry_msr_load_addr))) 798 return -EINVAL; 799 800 return 0; 801 } 802 803 static int nested_vmx_check_pml_controls(struct kvm_vcpu *vcpu, 804 struct vmcs12 *vmcs12) 805 { 806 if (!nested_cpu_has_pml(vmcs12)) 807 return 0; 808 809 if (CC(!nested_cpu_has_ept(vmcs12)) || 810 CC(!page_address_valid(vcpu, vmcs12->pml_address))) 811 return -EINVAL; 812 813 return 0; 814 } 815 816 static int nested_vmx_check_unrestricted_guest_controls(struct kvm_vcpu *vcpu, 817 struct vmcs12 *vmcs12) 818 { 819 if (CC(nested_cpu_has2(vmcs12, SECONDARY_EXEC_UNRESTRICTED_GUEST) && 820 !nested_cpu_has_ept(vmcs12))) 821 return -EINVAL; 822 return 0; 823 } 824 825 static int nested_vmx_check_mode_based_ept_exec_controls(struct kvm_vcpu *vcpu, 826 struct vmcs12 *vmcs12) 827 { 828 if (CC(nested_cpu_has2(vmcs12, SECONDARY_EXEC_MODE_BASED_EPT_EXEC) && 829 !nested_cpu_has_ept(vmcs12))) 830 return -EINVAL; 831 return 0; 832 } 833 834 static int nested_vmx_check_shadow_vmcs_controls(struct kvm_vcpu *vcpu, 835 struct vmcs12 *vmcs12) 836 { 837 if (!nested_cpu_has_shadow_vmcs(vmcs12)) 838 return 0; 839 840 if (CC(!page_address_valid(vcpu, vmcs12->vmread_bitmap)) || 841 CC(!page_address_valid(vcpu, vmcs12->vmwrite_bitmap))) 842 return -EINVAL; 843 844 return 0; 845 } 846 847 static int nested_vmx_msr_check_common(struct kvm_vcpu *vcpu, 848 struct vmx_msr_entry *e) 849 { 850 /* x2APIC MSR accesses are not allowed */ 851 if (CC(vcpu->arch.apic_base & X2APIC_ENABLE && e->index >> 8 == 0x8)) 852 return -EINVAL; 853 if (CC(e->index == MSR_IA32_UCODE_WRITE) || /* SDM Table 35-2 */ 854 CC(e->index == MSR_IA32_UCODE_REV)) 855 return -EINVAL; 856 if (CC(e->reserved != 0)) 857 return -EINVAL; 858 return 0; 859 } 860 861 static int nested_vmx_load_msr_check(struct kvm_vcpu *vcpu, 862 struct vmx_msr_entry *e) 863 { 864 if (CC(e->index == MSR_FS_BASE) || 865 CC(e->index == MSR_GS_BASE) || 866 CC(e->index == MSR_IA32_SMM_MONITOR_CTL) || /* SMM is not supported */ 867 nested_vmx_msr_check_common(vcpu, e)) 868 return -EINVAL; 869 return 0; 870 } 871 872 static int nested_vmx_store_msr_check(struct kvm_vcpu *vcpu, 873 struct vmx_msr_entry *e) 874 { 875 if (CC(e->index == MSR_IA32_SMBASE) || /* SMM is not supported */ 876 nested_vmx_msr_check_common(vcpu, e)) 877 return -EINVAL; 878 return 0; 879 } 880 881 static u32 nested_vmx_max_atomic_switch_msrs(struct kvm_vcpu *vcpu) 882 { 883 struct vcpu_vmx *vmx = to_vmx(vcpu); 884 u64 vmx_misc = vmx_control_msr(vmx->nested.msrs.misc_low, 885 vmx->nested.msrs.misc_high); 886 887 return (vmx_misc_max_msr(vmx_misc) + 1) * VMX_MISC_MSR_LIST_MULTIPLIER; 888 } 889 890 /* 891 * Load guest's/host's msr at nested entry/exit. 892 * return 0 for success, entry index for failure. 893 * 894 * One of the failure modes for MSR load/store is when a list exceeds the 895 * virtual hardware's capacity. To maintain compatibility with hardware inasmuch 896 * as possible, process all valid entries before failing rather than precheck 897 * for a capacity violation. 898 */ 899 static u32 nested_vmx_load_msr(struct kvm_vcpu *vcpu, u64 gpa, u32 count) 900 { 901 u32 i; 902 struct vmx_msr_entry e; 903 u32 max_msr_list_size = nested_vmx_max_atomic_switch_msrs(vcpu); 904 905 for (i = 0; i < count; i++) { 906 if (unlikely(i >= max_msr_list_size)) 907 goto fail; 908 909 if (kvm_vcpu_read_guest(vcpu, gpa + i * sizeof(e), 910 &e, sizeof(e))) { 911 pr_debug_ratelimited( 912 "%s cannot read MSR entry (%u, 0x%08llx)\n", 913 __func__, i, gpa + i * sizeof(e)); 914 goto fail; 915 } 916 if (nested_vmx_load_msr_check(vcpu, &e)) { 917 pr_debug_ratelimited( 918 "%s check failed (%u, 0x%x, 0x%x)\n", 919 __func__, i, e.index, e.reserved); 920 goto fail; 921 } 922 if (kvm_set_msr(vcpu, e.index, e.value)) { 923 pr_debug_ratelimited( 924 "%s cannot write MSR (%u, 0x%x, 0x%llx)\n", 925 __func__, i, e.index, e.value); 926 goto fail; 927 } 928 } 929 return 0; 930 fail: 931 return i + 1; 932 } 933 934 static bool nested_vmx_get_vmexit_msr_value(struct kvm_vcpu *vcpu, 935 u32 msr_index, 936 u64 *data) 937 { 938 struct vcpu_vmx *vmx = to_vmx(vcpu); 939 940 /* 941 * If the L0 hypervisor stored a more accurate value for the TSC that 942 * does not include the time taken for emulation of the L2->L1 943 * VM-exit in L0, use the more accurate value. 944 */ 945 if (msr_index == MSR_IA32_TSC) { 946 int index = vmx_find_msr_index(&vmx->msr_autostore.guest, 947 MSR_IA32_TSC); 948 949 if (index >= 0) { 950 u64 val = vmx->msr_autostore.guest.val[index].value; 951 952 *data = kvm_read_l1_tsc(vcpu, val); 953 return true; 954 } 955 } 956 957 if (kvm_get_msr(vcpu, msr_index, data)) { 958 pr_debug_ratelimited("%s cannot read MSR (0x%x)\n", __func__, 959 msr_index); 960 return false; 961 } 962 return true; 963 } 964 965 static bool read_and_check_msr_entry(struct kvm_vcpu *vcpu, u64 gpa, int i, 966 struct vmx_msr_entry *e) 967 { 968 if (kvm_vcpu_read_guest(vcpu, 969 gpa + i * sizeof(*e), 970 e, 2 * sizeof(u32))) { 971 pr_debug_ratelimited( 972 "%s cannot read MSR entry (%u, 0x%08llx)\n", 973 __func__, i, gpa + i * sizeof(*e)); 974 return false; 975 } 976 if (nested_vmx_store_msr_check(vcpu, e)) { 977 pr_debug_ratelimited( 978 "%s check failed (%u, 0x%x, 0x%x)\n", 979 __func__, i, e->index, e->reserved); 980 return false; 981 } 982 return true; 983 } 984 985 static int nested_vmx_store_msr(struct kvm_vcpu *vcpu, u64 gpa, u32 count) 986 { 987 u64 data; 988 u32 i; 989 struct vmx_msr_entry e; 990 u32 max_msr_list_size = nested_vmx_max_atomic_switch_msrs(vcpu); 991 992 for (i = 0; i < count; i++) { 993 if (unlikely(i >= max_msr_list_size)) 994 return -EINVAL; 995 996 if (!read_and_check_msr_entry(vcpu, gpa, i, &e)) 997 return -EINVAL; 998 999 if (!nested_vmx_get_vmexit_msr_value(vcpu, e.index, &data)) 1000 return -EINVAL; 1001 1002 if (kvm_vcpu_write_guest(vcpu, 1003 gpa + i * sizeof(e) + 1004 offsetof(struct vmx_msr_entry, value), 1005 &data, sizeof(data))) { 1006 pr_debug_ratelimited( 1007 "%s cannot write MSR (%u, 0x%x, 0x%llx)\n", 1008 __func__, i, e.index, data); 1009 return -EINVAL; 1010 } 1011 } 1012 return 0; 1013 } 1014 1015 static bool nested_msr_store_list_has_msr(struct kvm_vcpu *vcpu, u32 msr_index) 1016 { 1017 struct vmcs12 *vmcs12 = get_vmcs12(vcpu); 1018 u32 count = vmcs12->vm_exit_msr_store_count; 1019 u64 gpa = vmcs12->vm_exit_msr_store_addr; 1020 struct vmx_msr_entry e; 1021 u32 i; 1022 1023 for (i = 0; i < count; i++) { 1024 if (!read_and_check_msr_entry(vcpu, gpa, i, &e)) 1025 return false; 1026 1027 if (e.index == msr_index) 1028 return true; 1029 } 1030 return false; 1031 } 1032 1033 static void prepare_vmx_msr_autostore_list(struct kvm_vcpu *vcpu, 1034 u32 msr_index) 1035 { 1036 struct vcpu_vmx *vmx = to_vmx(vcpu); 1037 struct vmx_msrs *autostore = &vmx->msr_autostore.guest; 1038 bool in_vmcs12_store_list; 1039 int msr_autostore_index; 1040 bool in_autostore_list; 1041 int last; 1042 1043 msr_autostore_index = vmx_find_msr_index(autostore, msr_index); 1044 in_autostore_list = msr_autostore_index >= 0; 1045 in_vmcs12_store_list = nested_msr_store_list_has_msr(vcpu, msr_index); 1046 1047 if (in_vmcs12_store_list && !in_autostore_list) { 1048 if (autostore->nr == NR_LOADSTORE_MSRS) { 1049 /* 1050 * Emulated VMEntry does not fail here. Instead a less 1051 * accurate value will be returned by 1052 * nested_vmx_get_vmexit_msr_value() using kvm_get_msr() 1053 * instead of reading the value from the vmcs02 VMExit 1054 * MSR-store area. 1055 */ 1056 pr_warn_ratelimited( 1057 "Not enough msr entries in msr_autostore. Can't add msr %x\n", 1058 msr_index); 1059 return; 1060 } 1061 last = autostore->nr++; 1062 autostore->val[last].index = msr_index; 1063 } else if (!in_vmcs12_store_list && in_autostore_list) { 1064 last = --autostore->nr; 1065 autostore->val[msr_autostore_index] = autostore->val[last]; 1066 } 1067 } 1068 1069 static bool nested_cr3_valid(struct kvm_vcpu *vcpu, unsigned long val) 1070 { 1071 unsigned long invalid_mask; 1072 1073 invalid_mask = (~0ULL) << cpuid_maxphyaddr(vcpu); 1074 return (val & invalid_mask) == 0; 1075 } 1076 1077 /* 1078 * Load guest's/host's cr3 at nested entry/exit. @nested_ept is true if we are 1079 * emulating VM-Entry into a guest with EPT enabled. On failure, the expected 1080 * Exit Qualification (for a VM-Entry consistency check VM-Exit) is assigned to 1081 * @entry_failure_code. 1082 */ 1083 static int nested_vmx_load_cr3(struct kvm_vcpu *vcpu, unsigned long cr3, bool nested_ept, 1084 u32 *entry_failure_code) 1085 { 1086 if (cr3 != kvm_read_cr3(vcpu) || (!nested_ept && pdptrs_changed(vcpu))) { 1087 if (CC(!nested_cr3_valid(vcpu, cr3))) { 1088 *entry_failure_code = ENTRY_FAIL_DEFAULT; 1089 return -EINVAL; 1090 } 1091 1092 /* 1093 * If PAE paging and EPT are both on, CR3 is not used by the CPU and 1094 * must not be dereferenced. 1095 */ 1096 if (is_pae_paging(vcpu) && !nested_ept) { 1097 if (CC(!load_pdptrs(vcpu, vcpu->arch.walk_mmu, cr3))) { 1098 *entry_failure_code = ENTRY_FAIL_PDPTE; 1099 return -EINVAL; 1100 } 1101 } 1102 } 1103 1104 if (!nested_ept) 1105 kvm_mmu_new_cr3(vcpu, cr3, false); 1106 1107 vcpu->arch.cr3 = cr3; 1108 kvm_register_mark_available(vcpu, VCPU_EXREG_CR3); 1109 1110 kvm_init_mmu(vcpu, false); 1111 1112 return 0; 1113 } 1114 1115 /* 1116 * Returns if KVM is able to config CPU to tag TLB entries 1117 * populated by L2 differently than TLB entries populated 1118 * by L1. 1119 * 1120 * If L0 uses EPT, L1 and L2 run with different EPTP because 1121 * guest_mode is part of kvm_mmu_page_role. Thus, TLB entries 1122 * are tagged with different EPTP. 1123 * 1124 * If L1 uses VPID and we allocated a vpid02, TLB entries are tagged 1125 * with different VPID (L1 entries are tagged with vmx->vpid 1126 * while L2 entries are tagged with vmx->nested.vpid02). 1127 */ 1128 static bool nested_has_guest_tlb_tag(struct kvm_vcpu *vcpu) 1129 { 1130 struct vmcs12 *vmcs12 = get_vmcs12(vcpu); 1131 1132 return enable_ept || 1133 (nested_cpu_has_vpid(vmcs12) && to_vmx(vcpu)->nested.vpid02); 1134 } 1135 1136 static u16 nested_get_vpid02(struct kvm_vcpu *vcpu) 1137 { 1138 struct vcpu_vmx *vmx = to_vmx(vcpu); 1139 1140 return vmx->nested.vpid02 ? vmx->nested.vpid02 : vmx->vpid; 1141 } 1142 1143 static bool is_bitwise_subset(u64 superset, u64 subset, u64 mask) 1144 { 1145 superset &= mask; 1146 subset &= mask; 1147 1148 return (superset | subset) == superset; 1149 } 1150 1151 static int vmx_restore_vmx_basic(struct vcpu_vmx *vmx, u64 data) 1152 { 1153 const u64 feature_and_reserved = 1154 /* feature (except bit 48; see below) */ 1155 BIT_ULL(49) | BIT_ULL(54) | BIT_ULL(55) | 1156 /* reserved */ 1157 BIT_ULL(31) | GENMASK_ULL(47, 45) | GENMASK_ULL(63, 56); 1158 u64 vmx_basic = vmx->nested.msrs.basic; 1159 1160 if (!is_bitwise_subset(vmx_basic, data, feature_and_reserved)) 1161 return -EINVAL; 1162 1163 /* 1164 * KVM does not emulate a version of VMX that constrains physical 1165 * addresses of VMX structures (e.g. VMCS) to 32-bits. 1166 */ 1167 if (data & BIT_ULL(48)) 1168 return -EINVAL; 1169 1170 if (vmx_basic_vmcs_revision_id(vmx_basic) != 1171 vmx_basic_vmcs_revision_id(data)) 1172 return -EINVAL; 1173 1174 if (vmx_basic_vmcs_size(vmx_basic) > vmx_basic_vmcs_size(data)) 1175 return -EINVAL; 1176 1177 vmx->nested.msrs.basic = data; 1178 return 0; 1179 } 1180 1181 static int 1182 vmx_restore_control_msr(struct vcpu_vmx *vmx, u32 msr_index, u64 data) 1183 { 1184 u64 supported; 1185 u32 *lowp, *highp; 1186 1187 switch (msr_index) { 1188 case MSR_IA32_VMX_TRUE_PINBASED_CTLS: 1189 lowp = &vmx->nested.msrs.pinbased_ctls_low; 1190 highp = &vmx->nested.msrs.pinbased_ctls_high; 1191 break; 1192 case MSR_IA32_VMX_TRUE_PROCBASED_CTLS: 1193 lowp = &vmx->nested.msrs.procbased_ctls_low; 1194 highp = &vmx->nested.msrs.procbased_ctls_high; 1195 break; 1196 case MSR_IA32_VMX_TRUE_EXIT_CTLS: 1197 lowp = &vmx->nested.msrs.exit_ctls_low; 1198 highp = &vmx->nested.msrs.exit_ctls_high; 1199 break; 1200 case MSR_IA32_VMX_TRUE_ENTRY_CTLS: 1201 lowp = &vmx->nested.msrs.entry_ctls_low; 1202 highp = &vmx->nested.msrs.entry_ctls_high; 1203 break; 1204 case MSR_IA32_VMX_PROCBASED_CTLS2: 1205 lowp = &vmx->nested.msrs.secondary_ctls_low; 1206 highp = &vmx->nested.msrs.secondary_ctls_high; 1207 break; 1208 default: 1209 BUG(); 1210 } 1211 1212 supported = vmx_control_msr(*lowp, *highp); 1213 1214 /* Check must-be-1 bits are still 1. */ 1215 if (!is_bitwise_subset(data, supported, GENMASK_ULL(31, 0))) 1216 return -EINVAL; 1217 1218 /* Check must-be-0 bits are still 0. */ 1219 if (!is_bitwise_subset(supported, data, GENMASK_ULL(63, 32))) 1220 return -EINVAL; 1221 1222 *lowp = data; 1223 *highp = data >> 32; 1224 return 0; 1225 } 1226 1227 static int vmx_restore_vmx_misc(struct vcpu_vmx *vmx, u64 data) 1228 { 1229 const u64 feature_and_reserved_bits = 1230 /* feature */ 1231 BIT_ULL(5) | GENMASK_ULL(8, 6) | BIT_ULL(14) | BIT_ULL(15) | 1232 BIT_ULL(28) | BIT_ULL(29) | BIT_ULL(30) | 1233 /* reserved */ 1234 GENMASK_ULL(13, 9) | BIT_ULL(31); 1235 u64 vmx_misc; 1236 1237 vmx_misc = vmx_control_msr(vmx->nested.msrs.misc_low, 1238 vmx->nested.msrs.misc_high); 1239 1240 if (!is_bitwise_subset(vmx_misc, data, feature_and_reserved_bits)) 1241 return -EINVAL; 1242 1243 if ((vmx->nested.msrs.pinbased_ctls_high & 1244 PIN_BASED_VMX_PREEMPTION_TIMER) && 1245 vmx_misc_preemption_timer_rate(data) != 1246 vmx_misc_preemption_timer_rate(vmx_misc)) 1247 return -EINVAL; 1248 1249 if (vmx_misc_cr3_count(data) > vmx_misc_cr3_count(vmx_misc)) 1250 return -EINVAL; 1251 1252 if (vmx_misc_max_msr(data) > vmx_misc_max_msr(vmx_misc)) 1253 return -EINVAL; 1254 1255 if (vmx_misc_mseg_revid(data) != vmx_misc_mseg_revid(vmx_misc)) 1256 return -EINVAL; 1257 1258 vmx->nested.msrs.misc_low = data; 1259 vmx->nested.msrs.misc_high = data >> 32; 1260 1261 return 0; 1262 } 1263 1264 static int vmx_restore_vmx_ept_vpid_cap(struct vcpu_vmx *vmx, u64 data) 1265 { 1266 u64 vmx_ept_vpid_cap; 1267 1268 vmx_ept_vpid_cap = vmx_control_msr(vmx->nested.msrs.ept_caps, 1269 vmx->nested.msrs.vpid_caps); 1270 1271 /* Every bit is either reserved or a feature bit. */ 1272 if (!is_bitwise_subset(vmx_ept_vpid_cap, data, -1ULL)) 1273 return -EINVAL; 1274 1275 vmx->nested.msrs.ept_caps = data; 1276 vmx->nested.msrs.vpid_caps = data >> 32; 1277 return 0; 1278 } 1279 1280 static int vmx_restore_fixed0_msr(struct vcpu_vmx *vmx, u32 msr_index, u64 data) 1281 { 1282 u64 *msr; 1283 1284 switch (msr_index) { 1285 case MSR_IA32_VMX_CR0_FIXED0: 1286 msr = &vmx->nested.msrs.cr0_fixed0; 1287 break; 1288 case MSR_IA32_VMX_CR4_FIXED0: 1289 msr = &vmx->nested.msrs.cr4_fixed0; 1290 break; 1291 default: 1292 BUG(); 1293 } 1294 1295 /* 1296 * 1 bits (which indicates bits which "must-be-1" during VMX operation) 1297 * must be 1 in the restored value. 1298 */ 1299 if (!is_bitwise_subset(data, *msr, -1ULL)) 1300 return -EINVAL; 1301 1302 *msr = data; 1303 return 0; 1304 } 1305 1306 /* 1307 * Called when userspace is restoring VMX MSRs. 1308 * 1309 * Returns 0 on success, non-0 otherwise. 1310 */ 1311 int vmx_set_vmx_msr(struct kvm_vcpu *vcpu, u32 msr_index, u64 data) 1312 { 1313 struct vcpu_vmx *vmx = to_vmx(vcpu); 1314 1315 /* 1316 * Don't allow changes to the VMX capability MSRs while the vCPU 1317 * is in VMX operation. 1318 */ 1319 if (vmx->nested.vmxon) 1320 return -EBUSY; 1321 1322 switch (msr_index) { 1323 case MSR_IA32_VMX_BASIC: 1324 return vmx_restore_vmx_basic(vmx, data); 1325 case MSR_IA32_VMX_PINBASED_CTLS: 1326 case MSR_IA32_VMX_PROCBASED_CTLS: 1327 case MSR_IA32_VMX_EXIT_CTLS: 1328 case MSR_IA32_VMX_ENTRY_CTLS: 1329 /* 1330 * The "non-true" VMX capability MSRs are generated from the 1331 * "true" MSRs, so we do not support restoring them directly. 1332 * 1333 * If userspace wants to emulate VMX_BASIC[55]=0, userspace 1334 * should restore the "true" MSRs with the must-be-1 bits 1335 * set according to the SDM Vol 3. A.2 "RESERVED CONTROLS AND 1336 * DEFAULT SETTINGS". 1337 */ 1338 return -EINVAL; 1339 case MSR_IA32_VMX_TRUE_PINBASED_CTLS: 1340 case MSR_IA32_VMX_TRUE_PROCBASED_CTLS: 1341 case MSR_IA32_VMX_TRUE_EXIT_CTLS: 1342 case MSR_IA32_VMX_TRUE_ENTRY_CTLS: 1343 case MSR_IA32_VMX_PROCBASED_CTLS2: 1344 return vmx_restore_control_msr(vmx, msr_index, data); 1345 case MSR_IA32_VMX_MISC: 1346 return vmx_restore_vmx_misc(vmx, data); 1347 case MSR_IA32_VMX_CR0_FIXED0: 1348 case MSR_IA32_VMX_CR4_FIXED0: 1349 return vmx_restore_fixed0_msr(vmx, msr_index, data); 1350 case MSR_IA32_VMX_CR0_FIXED1: 1351 case MSR_IA32_VMX_CR4_FIXED1: 1352 /* 1353 * These MSRs are generated based on the vCPU's CPUID, so we 1354 * do not support restoring them directly. 1355 */ 1356 return -EINVAL; 1357 case MSR_IA32_VMX_EPT_VPID_CAP: 1358 return vmx_restore_vmx_ept_vpid_cap(vmx, data); 1359 case MSR_IA32_VMX_VMCS_ENUM: 1360 vmx->nested.msrs.vmcs_enum = data; 1361 return 0; 1362 case MSR_IA32_VMX_VMFUNC: 1363 if (data & ~vmx->nested.msrs.vmfunc_controls) 1364 return -EINVAL; 1365 vmx->nested.msrs.vmfunc_controls = data; 1366 return 0; 1367 default: 1368 /* 1369 * The rest of the VMX capability MSRs do not support restore. 1370 */ 1371 return -EINVAL; 1372 } 1373 } 1374 1375 /* Returns 0 on success, non-0 otherwise. */ 1376 int vmx_get_vmx_msr(struct nested_vmx_msrs *msrs, u32 msr_index, u64 *pdata) 1377 { 1378 switch (msr_index) { 1379 case MSR_IA32_VMX_BASIC: 1380 *pdata = msrs->basic; 1381 break; 1382 case MSR_IA32_VMX_TRUE_PINBASED_CTLS: 1383 case MSR_IA32_VMX_PINBASED_CTLS: 1384 *pdata = vmx_control_msr( 1385 msrs->pinbased_ctls_low, 1386 msrs->pinbased_ctls_high); 1387 if (msr_index == MSR_IA32_VMX_PINBASED_CTLS) 1388 *pdata |= PIN_BASED_ALWAYSON_WITHOUT_TRUE_MSR; 1389 break; 1390 case MSR_IA32_VMX_TRUE_PROCBASED_CTLS: 1391 case MSR_IA32_VMX_PROCBASED_CTLS: 1392 *pdata = vmx_control_msr( 1393 msrs->procbased_ctls_low, 1394 msrs->procbased_ctls_high); 1395 if (msr_index == MSR_IA32_VMX_PROCBASED_CTLS) 1396 *pdata |= CPU_BASED_ALWAYSON_WITHOUT_TRUE_MSR; 1397 break; 1398 case MSR_IA32_VMX_TRUE_EXIT_CTLS: 1399 case MSR_IA32_VMX_EXIT_CTLS: 1400 *pdata = vmx_control_msr( 1401 msrs->exit_ctls_low, 1402 msrs->exit_ctls_high); 1403 if (msr_index == MSR_IA32_VMX_EXIT_CTLS) 1404 *pdata |= VM_EXIT_ALWAYSON_WITHOUT_TRUE_MSR; 1405 break; 1406 case MSR_IA32_VMX_TRUE_ENTRY_CTLS: 1407 case MSR_IA32_VMX_ENTRY_CTLS: 1408 *pdata = vmx_control_msr( 1409 msrs->entry_ctls_low, 1410 msrs->entry_ctls_high); 1411 if (msr_index == MSR_IA32_VMX_ENTRY_CTLS) 1412 *pdata |= VM_ENTRY_ALWAYSON_WITHOUT_TRUE_MSR; 1413 break; 1414 case MSR_IA32_VMX_MISC: 1415 *pdata = vmx_control_msr( 1416 msrs->misc_low, 1417 msrs->misc_high); 1418 break; 1419 case MSR_IA32_VMX_CR0_FIXED0: 1420 *pdata = msrs->cr0_fixed0; 1421 break; 1422 case MSR_IA32_VMX_CR0_FIXED1: 1423 *pdata = msrs->cr0_fixed1; 1424 break; 1425 case MSR_IA32_VMX_CR4_FIXED0: 1426 *pdata = msrs->cr4_fixed0; 1427 break; 1428 case MSR_IA32_VMX_CR4_FIXED1: 1429 *pdata = msrs->cr4_fixed1; 1430 break; 1431 case MSR_IA32_VMX_VMCS_ENUM: 1432 *pdata = msrs->vmcs_enum; 1433 break; 1434 case MSR_IA32_VMX_PROCBASED_CTLS2: 1435 *pdata = vmx_control_msr( 1436 msrs->secondary_ctls_low, 1437 msrs->secondary_ctls_high); 1438 break; 1439 case MSR_IA32_VMX_EPT_VPID_CAP: 1440 *pdata = msrs->ept_caps | 1441 ((u64)msrs->vpid_caps << 32); 1442 break; 1443 case MSR_IA32_VMX_VMFUNC: 1444 *pdata = msrs->vmfunc_controls; 1445 break; 1446 default: 1447 return 1; 1448 } 1449 1450 return 0; 1451 } 1452 1453 /* 1454 * Copy the writable VMCS shadow fields back to the VMCS12, in case they have 1455 * been modified by the L1 guest. Note, "writable" in this context means 1456 * "writable by the guest", i.e. tagged SHADOW_FIELD_RW; the set of 1457 * fields tagged SHADOW_FIELD_RO may or may not align with the "read-only" 1458 * VM-exit information fields (which are actually writable if the vCPU is 1459 * configured to support "VMWRITE to any supported field in the VMCS"). 1460 */ 1461 static void copy_shadow_to_vmcs12(struct vcpu_vmx *vmx) 1462 { 1463 struct vmcs *shadow_vmcs = vmx->vmcs01.shadow_vmcs; 1464 struct vmcs12 *vmcs12 = get_vmcs12(&vmx->vcpu); 1465 struct shadow_vmcs_field field; 1466 unsigned long val; 1467 int i; 1468 1469 if (WARN_ON(!shadow_vmcs)) 1470 return; 1471 1472 preempt_disable(); 1473 1474 vmcs_load(shadow_vmcs); 1475 1476 for (i = 0; i < max_shadow_read_write_fields; i++) { 1477 field = shadow_read_write_fields[i]; 1478 val = __vmcs_readl(field.encoding); 1479 vmcs12_write_any(vmcs12, field.encoding, field.offset, val); 1480 } 1481 1482 vmcs_clear(shadow_vmcs); 1483 vmcs_load(vmx->loaded_vmcs->vmcs); 1484 1485 preempt_enable(); 1486 } 1487 1488 static void copy_vmcs12_to_shadow(struct vcpu_vmx *vmx) 1489 { 1490 const struct shadow_vmcs_field *fields[] = { 1491 shadow_read_write_fields, 1492 shadow_read_only_fields 1493 }; 1494 const int max_fields[] = { 1495 max_shadow_read_write_fields, 1496 max_shadow_read_only_fields 1497 }; 1498 struct vmcs *shadow_vmcs = vmx->vmcs01.shadow_vmcs; 1499 struct vmcs12 *vmcs12 = get_vmcs12(&vmx->vcpu); 1500 struct shadow_vmcs_field field; 1501 unsigned long val; 1502 int i, q; 1503 1504 if (WARN_ON(!shadow_vmcs)) 1505 return; 1506 1507 vmcs_load(shadow_vmcs); 1508 1509 for (q = 0; q < ARRAY_SIZE(fields); q++) { 1510 for (i = 0; i < max_fields[q]; i++) { 1511 field = fields[q][i]; 1512 val = vmcs12_read_any(vmcs12, field.encoding, 1513 field.offset); 1514 __vmcs_writel(field.encoding, val); 1515 } 1516 } 1517 1518 vmcs_clear(shadow_vmcs); 1519 vmcs_load(vmx->loaded_vmcs->vmcs); 1520 } 1521 1522 static int copy_enlightened_to_vmcs12(struct vcpu_vmx *vmx) 1523 { 1524 struct vmcs12 *vmcs12 = vmx->nested.cached_vmcs12; 1525 struct hv_enlightened_vmcs *evmcs = vmx->nested.hv_evmcs; 1526 1527 /* HV_VMX_ENLIGHTENED_CLEAN_FIELD_NONE */ 1528 vmcs12->tpr_threshold = evmcs->tpr_threshold; 1529 vmcs12->guest_rip = evmcs->guest_rip; 1530 1531 if (unlikely(!(evmcs->hv_clean_fields & 1532 HV_VMX_ENLIGHTENED_CLEAN_FIELD_GUEST_BASIC))) { 1533 vmcs12->guest_rsp = evmcs->guest_rsp; 1534 vmcs12->guest_rflags = evmcs->guest_rflags; 1535 vmcs12->guest_interruptibility_info = 1536 evmcs->guest_interruptibility_info; 1537 } 1538 1539 if (unlikely(!(evmcs->hv_clean_fields & 1540 HV_VMX_ENLIGHTENED_CLEAN_FIELD_CONTROL_PROC))) { 1541 vmcs12->cpu_based_vm_exec_control = 1542 evmcs->cpu_based_vm_exec_control; 1543 } 1544 1545 if (unlikely(!(evmcs->hv_clean_fields & 1546 HV_VMX_ENLIGHTENED_CLEAN_FIELD_CONTROL_EXCPN))) { 1547 vmcs12->exception_bitmap = evmcs->exception_bitmap; 1548 } 1549 1550 if (unlikely(!(evmcs->hv_clean_fields & 1551 HV_VMX_ENLIGHTENED_CLEAN_FIELD_CONTROL_ENTRY))) { 1552 vmcs12->vm_entry_controls = evmcs->vm_entry_controls; 1553 } 1554 1555 if (unlikely(!(evmcs->hv_clean_fields & 1556 HV_VMX_ENLIGHTENED_CLEAN_FIELD_CONTROL_EVENT))) { 1557 vmcs12->vm_entry_intr_info_field = 1558 evmcs->vm_entry_intr_info_field; 1559 vmcs12->vm_entry_exception_error_code = 1560 evmcs->vm_entry_exception_error_code; 1561 vmcs12->vm_entry_instruction_len = 1562 evmcs->vm_entry_instruction_len; 1563 } 1564 1565 if (unlikely(!(evmcs->hv_clean_fields & 1566 HV_VMX_ENLIGHTENED_CLEAN_FIELD_HOST_GRP1))) { 1567 vmcs12->host_ia32_pat = evmcs->host_ia32_pat; 1568 vmcs12->host_ia32_efer = evmcs->host_ia32_efer; 1569 vmcs12->host_cr0 = evmcs->host_cr0; 1570 vmcs12->host_cr3 = evmcs->host_cr3; 1571 vmcs12->host_cr4 = evmcs->host_cr4; 1572 vmcs12->host_ia32_sysenter_esp = evmcs->host_ia32_sysenter_esp; 1573 vmcs12->host_ia32_sysenter_eip = evmcs->host_ia32_sysenter_eip; 1574 vmcs12->host_rip = evmcs->host_rip; 1575 vmcs12->host_ia32_sysenter_cs = evmcs->host_ia32_sysenter_cs; 1576 vmcs12->host_es_selector = evmcs->host_es_selector; 1577 vmcs12->host_cs_selector = evmcs->host_cs_selector; 1578 vmcs12->host_ss_selector = evmcs->host_ss_selector; 1579 vmcs12->host_ds_selector = evmcs->host_ds_selector; 1580 vmcs12->host_fs_selector = evmcs->host_fs_selector; 1581 vmcs12->host_gs_selector = evmcs->host_gs_selector; 1582 vmcs12->host_tr_selector = evmcs->host_tr_selector; 1583 } 1584 1585 if (unlikely(!(evmcs->hv_clean_fields & 1586 HV_VMX_ENLIGHTENED_CLEAN_FIELD_CONTROL_GRP1))) { 1587 vmcs12->pin_based_vm_exec_control = 1588 evmcs->pin_based_vm_exec_control; 1589 vmcs12->vm_exit_controls = evmcs->vm_exit_controls; 1590 vmcs12->secondary_vm_exec_control = 1591 evmcs->secondary_vm_exec_control; 1592 } 1593 1594 if (unlikely(!(evmcs->hv_clean_fields & 1595 HV_VMX_ENLIGHTENED_CLEAN_FIELD_IO_BITMAP))) { 1596 vmcs12->io_bitmap_a = evmcs->io_bitmap_a; 1597 vmcs12->io_bitmap_b = evmcs->io_bitmap_b; 1598 } 1599 1600 if (unlikely(!(evmcs->hv_clean_fields & 1601 HV_VMX_ENLIGHTENED_CLEAN_FIELD_MSR_BITMAP))) { 1602 vmcs12->msr_bitmap = evmcs->msr_bitmap; 1603 } 1604 1605 if (unlikely(!(evmcs->hv_clean_fields & 1606 HV_VMX_ENLIGHTENED_CLEAN_FIELD_GUEST_GRP2))) { 1607 vmcs12->guest_es_base = evmcs->guest_es_base; 1608 vmcs12->guest_cs_base = evmcs->guest_cs_base; 1609 vmcs12->guest_ss_base = evmcs->guest_ss_base; 1610 vmcs12->guest_ds_base = evmcs->guest_ds_base; 1611 vmcs12->guest_fs_base = evmcs->guest_fs_base; 1612 vmcs12->guest_gs_base = evmcs->guest_gs_base; 1613 vmcs12->guest_ldtr_base = evmcs->guest_ldtr_base; 1614 vmcs12->guest_tr_base = evmcs->guest_tr_base; 1615 vmcs12->guest_gdtr_base = evmcs->guest_gdtr_base; 1616 vmcs12->guest_idtr_base = evmcs->guest_idtr_base; 1617 vmcs12->guest_es_limit = evmcs->guest_es_limit; 1618 vmcs12->guest_cs_limit = evmcs->guest_cs_limit; 1619 vmcs12->guest_ss_limit = evmcs->guest_ss_limit; 1620 vmcs12->guest_ds_limit = evmcs->guest_ds_limit; 1621 vmcs12->guest_fs_limit = evmcs->guest_fs_limit; 1622 vmcs12->guest_gs_limit = evmcs->guest_gs_limit; 1623 vmcs12->guest_ldtr_limit = evmcs->guest_ldtr_limit; 1624 vmcs12->guest_tr_limit = evmcs->guest_tr_limit; 1625 vmcs12->guest_gdtr_limit = evmcs->guest_gdtr_limit; 1626 vmcs12->guest_idtr_limit = evmcs->guest_idtr_limit; 1627 vmcs12->guest_es_ar_bytes = evmcs->guest_es_ar_bytes; 1628 vmcs12->guest_cs_ar_bytes = evmcs->guest_cs_ar_bytes; 1629 vmcs12->guest_ss_ar_bytes = evmcs->guest_ss_ar_bytes; 1630 vmcs12->guest_ds_ar_bytes = evmcs->guest_ds_ar_bytes; 1631 vmcs12->guest_fs_ar_bytes = evmcs->guest_fs_ar_bytes; 1632 vmcs12->guest_gs_ar_bytes = evmcs->guest_gs_ar_bytes; 1633 vmcs12->guest_ldtr_ar_bytes = evmcs->guest_ldtr_ar_bytes; 1634 vmcs12->guest_tr_ar_bytes = evmcs->guest_tr_ar_bytes; 1635 vmcs12->guest_es_selector = evmcs->guest_es_selector; 1636 vmcs12->guest_cs_selector = evmcs->guest_cs_selector; 1637 vmcs12->guest_ss_selector = evmcs->guest_ss_selector; 1638 vmcs12->guest_ds_selector = evmcs->guest_ds_selector; 1639 vmcs12->guest_fs_selector = evmcs->guest_fs_selector; 1640 vmcs12->guest_gs_selector = evmcs->guest_gs_selector; 1641 vmcs12->guest_ldtr_selector = evmcs->guest_ldtr_selector; 1642 vmcs12->guest_tr_selector = evmcs->guest_tr_selector; 1643 } 1644 1645 if (unlikely(!(evmcs->hv_clean_fields & 1646 HV_VMX_ENLIGHTENED_CLEAN_FIELD_CONTROL_GRP2))) { 1647 vmcs12->tsc_offset = evmcs->tsc_offset; 1648 vmcs12->virtual_apic_page_addr = evmcs->virtual_apic_page_addr; 1649 vmcs12->xss_exit_bitmap = evmcs->xss_exit_bitmap; 1650 } 1651 1652 if (unlikely(!(evmcs->hv_clean_fields & 1653 HV_VMX_ENLIGHTENED_CLEAN_FIELD_CRDR))) { 1654 vmcs12->cr0_guest_host_mask = evmcs->cr0_guest_host_mask; 1655 vmcs12->cr4_guest_host_mask = evmcs->cr4_guest_host_mask; 1656 vmcs12->cr0_read_shadow = evmcs->cr0_read_shadow; 1657 vmcs12->cr4_read_shadow = evmcs->cr4_read_shadow; 1658 vmcs12->guest_cr0 = evmcs->guest_cr0; 1659 vmcs12->guest_cr3 = evmcs->guest_cr3; 1660 vmcs12->guest_cr4 = evmcs->guest_cr4; 1661 vmcs12->guest_dr7 = evmcs->guest_dr7; 1662 } 1663 1664 if (unlikely(!(evmcs->hv_clean_fields & 1665 HV_VMX_ENLIGHTENED_CLEAN_FIELD_HOST_POINTER))) { 1666 vmcs12->host_fs_base = evmcs->host_fs_base; 1667 vmcs12->host_gs_base = evmcs->host_gs_base; 1668 vmcs12->host_tr_base = evmcs->host_tr_base; 1669 vmcs12->host_gdtr_base = evmcs->host_gdtr_base; 1670 vmcs12->host_idtr_base = evmcs->host_idtr_base; 1671 vmcs12->host_rsp = evmcs->host_rsp; 1672 } 1673 1674 if (unlikely(!(evmcs->hv_clean_fields & 1675 HV_VMX_ENLIGHTENED_CLEAN_FIELD_CONTROL_XLAT))) { 1676 vmcs12->ept_pointer = evmcs->ept_pointer; 1677 vmcs12->virtual_processor_id = evmcs->virtual_processor_id; 1678 } 1679 1680 if (unlikely(!(evmcs->hv_clean_fields & 1681 HV_VMX_ENLIGHTENED_CLEAN_FIELD_GUEST_GRP1))) { 1682 vmcs12->vmcs_link_pointer = evmcs->vmcs_link_pointer; 1683 vmcs12->guest_ia32_debugctl = evmcs->guest_ia32_debugctl; 1684 vmcs12->guest_ia32_pat = evmcs->guest_ia32_pat; 1685 vmcs12->guest_ia32_efer = evmcs->guest_ia32_efer; 1686 vmcs12->guest_pdptr0 = evmcs->guest_pdptr0; 1687 vmcs12->guest_pdptr1 = evmcs->guest_pdptr1; 1688 vmcs12->guest_pdptr2 = evmcs->guest_pdptr2; 1689 vmcs12->guest_pdptr3 = evmcs->guest_pdptr3; 1690 vmcs12->guest_pending_dbg_exceptions = 1691 evmcs->guest_pending_dbg_exceptions; 1692 vmcs12->guest_sysenter_esp = evmcs->guest_sysenter_esp; 1693 vmcs12->guest_sysenter_eip = evmcs->guest_sysenter_eip; 1694 vmcs12->guest_bndcfgs = evmcs->guest_bndcfgs; 1695 vmcs12->guest_activity_state = evmcs->guest_activity_state; 1696 vmcs12->guest_sysenter_cs = evmcs->guest_sysenter_cs; 1697 } 1698 1699 /* 1700 * Not used? 1701 * vmcs12->vm_exit_msr_store_addr = evmcs->vm_exit_msr_store_addr; 1702 * vmcs12->vm_exit_msr_load_addr = evmcs->vm_exit_msr_load_addr; 1703 * vmcs12->vm_entry_msr_load_addr = evmcs->vm_entry_msr_load_addr; 1704 * vmcs12->cr3_target_value0 = evmcs->cr3_target_value0; 1705 * vmcs12->cr3_target_value1 = evmcs->cr3_target_value1; 1706 * vmcs12->cr3_target_value2 = evmcs->cr3_target_value2; 1707 * vmcs12->cr3_target_value3 = evmcs->cr3_target_value3; 1708 * vmcs12->page_fault_error_code_mask = 1709 * evmcs->page_fault_error_code_mask; 1710 * vmcs12->page_fault_error_code_match = 1711 * evmcs->page_fault_error_code_match; 1712 * vmcs12->cr3_target_count = evmcs->cr3_target_count; 1713 * vmcs12->vm_exit_msr_store_count = evmcs->vm_exit_msr_store_count; 1714 * vmcs12->vm_exit_msr_load_count = evmcs->vm_exit_msr_load_count; 1715 * vmcs12->vm_entry_msr_load_count = evmcs->vm_entry_msr_load_count; 1716 */ 1717 1718 /* 1719 * Read only fields: 1720 * vmcs12->guest_physical_address = evmcs->guest_physical_address; 1721 * vmcs12->vm_instruction_error = evmcs->vm_instruction_error; 1722 * vmcs12->vm_exit_reason = evmcs->vm_exit_reason; 1723 * vmcs12->vm_exit_intr_info = evmcs->vm_exit_intr_info; 1724 * vmcs12->vm_exit_intr_error_code = evmcs->vm_exit_intr_error_code; 1725 * vmcs12->idt_vectoring_info_field = evmcs->idt_vectoring_info_field; 1726 * vmcs12->idt_vectoring_error_code = evmcs->idt_vectoring_error_code; 1727 * vmcs12->vm_exit_instruction_len = evmcs->vm_exit_instruction_len; 1728 * vmcs12->vmx_instruction_info = evmcs->vmx_instruction_info; 1729 * vmcs12->exit_qualification = evmcs->exit_qualification; 1730 * vmcs12->guest_linear_address = evmcs->guest_linear_address; 1731 * 1732 * Not present in struct vmcs12: 1733 * vmcs12->exit_io_instruction_ecx = evmcs->exit_io_instruction_ecx; 1734 * vmcs12->exit_io_instruction_esi = evmcs->exit_io_instruction_esi; 1735 * vmcs12->exit_io_instruction_edi = evmcs->exit_io_instruction_edi; 1736 * vmcs12->exit_io_instruction_eip = evmcs->exit_io_instruction_eip; 1737 */ 1738 1739 return 0; 1740 } 1741 1742 static int copy_vmcs12_to_enlightened(struct vcpu_vmx *vmx) 1743 { 1744 struct vmcs12 *vmcs12 = vmx->nested.cached_vmcs12; 1745 struct hv_enlightened_vmcs *evmcs = vmx->nested.hv_evmcs; 1746 1747 /* 1748 * Should not be changed by KVM: 1749 * 1750 * evmcs->host_es_selector = vmcs12->host_es_selector; 1751 * evmcs->host_cs_selector = vmcs12->host_cs_selector; 1752 * evmcs->host_ss_selector = vmcs12->host_ss_selector; 1753 * evmcs->host_ds_selector = vmcs12->host_ds_selector; 1754 * evmcs->host_fs_selector = vmcs12->host_fs_selector; 1755 * evmcs->host_gs_selector = vmcs12->host_gs_selector; 1756 * evmcs->host_tr_selector = vmcs12->host_tr_selector; 1757 * evmcs->host_ia32_pat = vmcs12->host_ia32_pat; 1758 * evmcs->host_ia32_efer = vmcs12->host_ia32_efer; 1759 * evmcs->host_cr0 = vmcs12->host_cr0; 1760 * evmcs->host_cr3 = vmcs12->host_cr3; 1761 * evmcs->host_cr4 = vmcs12->host_cr4; 1762 * evmcs->host_ia32_sysenter_esp = vmcs12->host_ia32_sysenter_esp; 1763 * evmcs->host_ia32_sysenter_eip = vmcs12->host_ia32_sysenter_eip; 1764 * evmcs->host_rip = vmcs12->host_rip; 1765 * evmcs->host_ia32_sysenter_cs = vmcs12->host_ia32_sysenter_cs; 1766 * evmcs->host_fs_base = vmcs12->host_fs_base; 1767 * evmcs->host_gs_base = vmcs12->host_gs_base; 1768 * evmcs->host_tr_base = vmcs12->host_tr_base; 1769 * evmcs->host_gdtr_base = vmcs12->host_gdtr_base; 1770 * evmcs->host_idtr_base = vmcs12->host_idtr_base; 1771 * evmcs->host_rsp = vmcs12->host_rsp; 1772 * sync_vmcs02_to_vmcs12() doesn't read these: 1773 * evmcs->io_bitmap_a = vmcs12->io_bitmap_a; 1774 * evmcs->io_bitmap_b = vmcs12->io_bitmap_b; 1775 * evmcs->msr_bitmap = vmcs12->msr_bitmap; 1776 * evmcs->ept_pointer = vmcs12->ept_pointer; 1777 * evmcs->xss_exit_bitmap = vmcs12->xss_exit_bitmap; 1778 * evmcs->vm_exit_msr_store_addr = vmcs12->vm_exit_msr_store_addr; 1779 * evmcs->vm_exit_msr_load_addr = vmcs12->vm_exit_msr_load_addr; 1780 * evmcs->vm_entry_msr_load_addr = vmcs12->vm_entry_msr_load_addr; 1781 * evmcs->cr3_target_value0 = vmcs12->cr3_target_value0; 1782 * evmcs->cr3_target_value1 = vmcs12->cr3_target_value1; 1783 * evmcs->cr3_target_value2 = vmcs12->cr3_target_value2; 1784 * evmcs->cr3_target_value3 = vmcs12->cr3_target_value3; 1785 * evmcs->tpr_threshold = vmcs12->tpr_threshold; 1786 * evmcs->virtual_processor_id = vmcs12->virtual_processor_id; 1787 * evmcs->exception_bitmap = vmcs12->exception_bitmap; 1788 * evmcs->vmcs_link_pointer = vmcs12->vmcs_link_pointer; 1789 * evmcs->pin_based_vm_exec_control = vmcs12->pin_based_vm_exec_control; 1790 * evmcs->vm_exit_controls = vmcs12->vm_exit_controls; 1791 * evmcs->secondary_vm_exec_control = vmcs12->secondary_vm_exec_control; 1792 * evmcs->page_fault_error_code_mask = 1793 * vmcs12->page_fault_error_code_mask; 1794 * evmcs->page_fault_error_code_match = 1795 * vmcs12->page_fault_error_code_match; 1796 * evmcs->cr3_target_count = vmcs12->cr3_target_count; 1797 * evmcs->virtual_apic_page_addr = vmcs12->virtual_apic_page_addr; 1798 * evmcs->tsc_offset = vmcs12->tsc_offset; 1799 * evmcs->guest_ia32_debugctl = vmcs12->guest_ia32_debugctl; 1800 * evmcs->cr0_guest_host_mask = vmcs12->cr0_guest_host_mask; 1801 * evmcs->cr4_guest_host_mask = vmcs12->cr4_guest_host_mask; 1802 * evmcs->cr0_read_shadow = vmcs12->cr0_read_shadow; 1803 * evmcs->cr4_read_shadow = vmcs12->cr4_read_shadow; 1804 * evmcs->vm_exit_msr_store_count = vmcs12->vm_exit_msr_store_count; 1805 * evmcs->vm_exit_msr_load_count = vmcs12->vm_exit_msr_load_count; 1806 * evmcs->vm_entry_msr_load_count = vmcs12->vm_entry_msr_load_count; 1807 * 1808 * Not present in struct vmcs12: 1809 * evmcs->exit_io_instruction_ecx = vmcs12->exit_io_instruction_ecx; 1810 * evmcs->exit_io_instruction_esi = vmcs12->exit_io_instruction_esi; 1811 * evmcs->exit_io_instruction_edi = vmcs12->exit_io_instruction_edi; 1812 * evmcs->exit_io_instruction_eip = vmcs12->exit_io_instruction_eip; 1813 */ 1814 1815 evmcs->guest_es_selector = vmcs12->guest_es_selector; 1816 evmcs->guest_cs_selector = vmcs12->guest_cs_selector; 1817 evmcs->guest_ss_selector = vmcs12->guest_ss_selector; 1818 evmcs->guest_ds_selector = vmcs12->guest_ds_selector; 1819 evmcs->guest_fs_selector = vmcs12->guest_fs_selector; 1820 evmcs->guest_gs_selector = vmcs12->guest_gs_selector; 1821 evmcs->guest_ldtr_selector = vmcs12->guest_ldtr_selector; 1822 evmcs->guest_tr_selector = vmcs12->guest_tr_selector; 1823 1824 evmcs->guest_es_limit = vmcs12->guest_es_limit; 1825 evmcs->guest_cs_limit = vmcs12->guest_cs_limit; 1826 evmcs->guest_ss_limit = vmcs12->guest_ss_limit; 1827 evmcs->guest_ds_limit = vmcs12->guest_ds_limit; 1828 evmcs->guest_fs_limit = vmcs12->guest_fs_limit; 1829 evmcs->guest_gs_limit = vmcs12->guest_gs_limit; 1830 evmcs->guest_ldtr_limit = vmcs12->guest_ldtr_limit; 1831 evmcs->guest_tr_limit = vmcs12->guest_tr_limit; 1832 evmcs->guest_gdtr_limit = vmcs12->guest_gdtr_limit; 1833 evmcs->guest_idtr_limit = vmcs12->guest_idtr_limit; 1834 1835 evmcs->guest_es_ar_bytes = vmcs12->guest_es_ar_bytes; 1836 evmcs->guest_cs_ar_bytes = vmcs12->guest_cs_ar_bytes; 1837 evmcs->guest_ss_ar_bytes = vmcs12->guest_ss_ar_bytes; 1838 evmcs->guest_ds_ar_bytes = vmcs12->guest_ds_ar_bytes; 1839 evmcs->guest_fs_ar_bytes = vmcs12->guest_fs_ar_bytes; 1840 evmcs->guest_gs_ar_bytes = vmcs12->guest_gs_ar_bytes; 1841 evmcs->guest_ldtr_ar_bytes = vmcs12->guest_ldtr_ar_bytes; 1842 evmcs->guest_tr_ar_bytes = vmcs12->guest_tr_ar_bytes; 1843 1844 evmcs->guest_es_base = vmcs12->guest_es_base; 1845 evmcs->guest_cs_base = vmcs12->guest_cs_base; 1846 evmcs->guest_ss_base = vmcs12->guest_ss_base; 1847 evmcs->guest_ds_base = vmcs12->guest_ds_base; 1848 evmcs->guest_fs_base = vmcs12->guest_fs_base; 1849 evmcs->guest_gs_base = vmcs12->guest_gs_base; 1850 evmcs->guest_ldtr_base = vmcs12->guest_ldtr_base; 1851 evmcs->guest_tr_base = vmcs12->guest_tr_base; 1852 evmcs->guest_gdtr_base = vmcs12->guest_gdtr_base; 1853 evmcs->guest_idtr_base = vmcs12->guest_idtr_base; 1854 1855 evmcs->guest_ia32_pat = vmcs12->guest_ia32_pat; 1856 evmcs->guest_ia32_efer = vmcs12->guest_ia32_efer; 1857 1858 evmcs->guest_pdptr0 = vmcs12->guest_pdptr0; 1859 evmcs->guest_pdptr1 = vmcs12->guest_pdptr1; 1860 evmcs->guest_pdptr2 = vmcs12->guest_pdptr2; 1861 evmcs->guest_pdptr3 = vmcs12->guest_pdptr3; 1862 1863 evmcs->guest_pending_dbg_exceptions = 1864 vmcs12->guest_pending_dbg_exceptions; 1865 evmcs->guest_sysenter_esp = vmcs12->guest_sysenter_esp; 1866 evmcs->guest_sysenter_eip = vmcs12->guest_sysenter_eip; 1867 1868 evmcs->guest_activity_state = vmcs12->guest_activity_state; 1869 evmcs->guest_sysenter_cs = vmcs12->guest_sysenter_cs; 1870 1871 evmcs->guest_cr0 = vmcs12->guest_cr0; 1872 evmcs->guest_cr3 = vmcs12->guest_cr3; 1873 evmcs->guest_cr4 = vmcs12->guest_cr4; 1874 evmcs->guest_dr7 = vmcs12->guest_dr7; 1875 1876 evmcs->guest_physical_address = vmcs12->guest_physical_address; 1877 1878 evmcs->vm_instruction_error = vmcs12->vm_instruction_error; 1879 evmcs->vm_exit_reason = vmcs12->vm_exit_reason; 1880 evmcs->vm_exit_intr_info = vmcs12->vm_exit_intr_info; 1881 evmcs->vm_exit_intr_error_code = vmcs12->vm_exit_intr_error_code; 1882 evmcs->idt_vectoring_info_field = vmcs12->idt_vectoring_info_field; 1883 evmcs->idt_vectoring_error_code = vmcs12->idt_vectoring_error_code; 1884 evmcs->vm_exit_instruction_len = vmcs12->vm_exit_instruction_len; 1885 evmcs->vmx_instruction_info = vmcs12->vmx_instruction_info; 1886 1887 evmcs->exit_qualification = vmcs12->exit_qualification; 1888 1889 evmcs->guest_linear_address = vmcs12->guest_linear_address; 1890 evmcs->guest_rsp = vmcs12->guest_rsp; 1891 evmcs->guest_rflags = vmcs12->guest_rflags; 1892 1893 evmcs->guest_interruptibility_info = 1894 vmcs12->guest_interruptibility_info; 1895 evmcs->cpu_based_vm_exec_control = vmcs12->cpu_based_vm_exec_control; 1896 evmcs->vm_entry_controls = vmcs12->vm_entry_controls; 1897 evmcs->vm_entry_intr_info_field = vmcs12->vm_entry_intr_info_field; 1898 evmcs->vm_entry_exception_error_code = 1899 vmcs12->vm_entry_exception_error_code; 1900 evmcs->vm_entry_instruction_len = vmcs12->vm_entry_instruction_len; 1901 1902 evmcs->guest_rip = vmcs12->guest_rip; 1903 1904 evmcs->guest_bndcfgs = vmcs12->guest_bndcfgs; 1905 1906 return 0; 1907 } 1908 1909 /* 1910 * This is an equivalent of the nested hypervisor executing the vmptrld 1911 * instruction. 1912 */ 1913 static int nested_vmx_handle_enlightened_vmptrld(struct kvm_vcpu *vcpu, 1914 bool from_launch) 1915 { 1916 struct vcpu_vmx *vmx = to_vmx(vcpu); 1917 bool evmcs_gpa_changed = false; 1918 u64 evmcs_gpa; 1919 1920 if (likely(!vmx->nested.enlightened_vmcs_enabled)) 1921 return 1; 1922 1923 if (!nested_enlightened_vmentry(vcpu, &evmcs_gpa)) 1924 return 1; 1925 1926 if (unlikely(evmcs_gpa != vmx->nested.hv_evmcs_vmptr)) { 1927 if (!vmx->nested.hv_evmcs) 1928 vmx->nested.current_vmptr = -1ull; 1929 1930 nested_release_evmcs(vcpu); 1931 1932 if (kvm_vcpu_map(vcpu, gpa_to_gfn(evmcs_gpa), 1933 &vmx->nested.hv_evmcs_map)) 1934 return 0; 1935 1936 vmx->nested.hv_evmcs = vmx->nested.hv_evmcs_map.hva; 1937 1938 /* 1939 * Currently, KVM only supports eVMCS version 1 1940 * (== KVM_EVMCS_VERSION) and thus we expect guest to set this 1941 * value to first u32 field of eVMCS which should specify eVMCS 1942 * VersionNumber. 1943 * 1944 * Guest should be aware of supported eVMCS versions by host by 1945 * examining CPUID.0x4000000A.EAX[0:15]. Host userspace VMM is 1946 * expected to set this CPUID leaf according to the value 1947 * returned in vmcs_version from nested_enable_evmcs(). 1948 * 1949 * However, it turns out that Microsoft Hyper-V fails to comply 1950 * to their own invented interface: When Hyper-V use eVMCS, it 1951 * just sets first u32 field of eVMCS to revision_id specified 1952 * in MSR_IA32_VMX_BASIC. Instead of used eVMCS version number 1953 * which is one of the supported versions specified in 1954 * CPUID.0x4000000A.EAX[0:15]. 1955 * 1956 * To overcome Hyper-V bug, we accept here either a supported 1957 * eVMCS version or VMCS12 revision_id as valid values for first 1958 * u32 field of eVMCS. 1959 */ 1960 if ((vmx->nested.hv_evmcs->revision_id != KVM_EVMCS_VERSION) && 1961 (vmx->nested.hv_evmcs->revision_id != VMCS12_REVISION)) { 1962 nested_release_evmcs(vcpu); 1963 return 0; 1964 } 1965 1966 vmx->nested.dirty_vmcs12 = true; 1967 vmx->nested.hv_evmcs_vmptr = evmcs_gpa; 1968 1969 evmcs_gpa_changed = true; 1970 /* 1971 * Unlike normal vmcs12, enlightened vmcs12 is not fully 1972 * reloaded from guest's memory (read only fields, fields not 1973 * present in struct hv_enlightened_vmcs, ...). Make sure there 1974 * are no leftovers. 1975 */ 1976 if (from_launch) { 1977 struct vmcs12 *vmcs12 = get_vmcs12(vcpu); 1978 memset(vmcs12, 0, sizeof(*vmcs12)); 1979 vmcs12->hdr.revision_id = VMCS12_REVISION; 1980 } 1981 1982 } 1983 1984 /* 1985 * Clean fields data can't be used on VMLAUNCH and when we switch 1986 * between different L2 guests as KVM keeps a single VMCS12 per L1. 1987 */ 1988 if (from_launch || evmcs_gpa_changed) 1989 vmx->nested.hv_evmcs->hv_clean_fields &= 1990 ~HV_VMX_ENLIGHTENED_CLEAN_FIELD_ALL; 1991 1992 return 1; 1993 } 1994 1995 void nested_sync_vmcs12_to_shadow(struct kvm_vcpu *vcpu) 1996 { 1997 struct vcpu_vmx *vmx = to_vmx(vcpu); 1998 1999 /* 2000 * hv_evmcs may end up being not mapped after migration (when 2001 * L2 was running), map it here to make sure vmcs12 changes are 2002 * properly reflected. 2003 */ 2004 if (vmx->nested.enlightened_vmcs_enabled && !vmx->nested.hv_evmcs) 2005 nested_vmx_handle_enlightened_vmptrld(vcpu, false); 2006 2007 if (vmx->nested.hv_evmcs) { 2008 copy_vmcs12_to_enlightened(vmx); 2009 /* All fields are clean */ 2010 vmx->nested.hv_evmcs->hv_clean_fields |= 2011 HV_VMX_ENLIGHTENED_CLEAN_FIELD_ALL; 2012 } else { 2013 copy_vmcs12_to_shadow(vmx); 2014 } 2015 2016 vmx->nested.need_vmcs12_to_shadow_sync = false; 2017 } 2018 2019 static enum hrtimer_restart vmx_preemption_timer_fn(struct hrtimer *timer) 2020 { 2021 struct vcpu_vmx *vmx = 2022 container_of(timer, struct vcpu_vmx, nested.preemption_timer); 2023 2024 vmx->nested.preemption_timer_expired = true; 2025 kvm_make_request(KVM_REQ_EVENT, &vmx->vcpu); 2026 kvm_vcpu_kick(&vmx->vcpu); 2027 2028 return HRTIMER_NORESTART; 2029 } 2030 2031 static void vmx_start_preemption_timer(struct kvm_vcpu *vcpu) 2032 { 2033 u64 preemption_timeout = get_vmcs12(vcpu)->vmx_preemption_timer_value; 2034 struct vcpu_vmx *vmx = to_vmx(vcpu); 2035 2036 /* 2037 * A timer value of zero is architecturally guaranteed to cause 2038 * a VMExit prior to executing any instructions in the guest. 2039 */ 2040 if (preemption_timeout == 0) { 2041 vmx_preemption_timer_fn(&vmx->nested.preemption_timer); 2042 return; 2043 } 2044 2045 if (vcpu->arch.virtual_tsc_khz == 0) 2046 return; 2047 2048 preemption_timeout <<= VMX_MISC_EMULATED_PREEMPTION_TIMER_RATE; 2049 preemption_timeout *= 1000000; 2050 do_div(preemption_timeout, vcpu->arch.virtual_tsc_khz); 2051 hrtimer_start(&vmx->nested.preemption_timer, 2052 ns_to_ktime(preemption_timeout), HRTIMER_MODE_REL); 2053 } 2054 2055 static u64 nested_vmx_calc_efer(struct vcpu_vmx *vmx, struct vmcs12 *vmcs12) 2056 { 2057 if (vmx->nested.nested_run_pending && 2058 (vmcs12->vm_entry_controls & VM_ENTRY_LOAD_IA32_EFER)) 2059 return vmcs12->guest_ia32_efer; 2060 else if (vmcs12->vm_entry_controls & VM_ENTRY_IA32E_MODE) 2061 return vmx->vcpu.arch.efer | (EFER_LMA | EFER_LME); 2062 else 2063 return vmx->vcpu.arch.efer & ~(EFER_LMA | EFER_LME); 2064 } 2065 2066 static void prepare_vmcs02_constant_state(struct vcpu_vmx *vmx) 2067 { 2068 /* 2069 * If vmcs02 hasn't been initialized, set the constant vmcs02 state 2070 * according to L0's settings (vmcs12 is irrelevant here). Host 2071 * fields that come from L0 and are not constant, e.g. HOST_CR3, 2072 * will be set as needed prior to VMLAUNCH/VMRESUME. 2073 */ 2074 if (vmx->nested.vmcs02_initialized) 2075 return; 2076 vmx->nested.vmcs02_initialized = true; 2077 2078 /* 2079 * We don't care what the EPTP value is we just need to guarantee 2080 * it's valid so we don't get a false positive when doing early 2081 * consistency checks. 2082 */ 2083 if (enable_ept && nested_early_check) 2084 vmcs_write64(EPT_POINTER, construct_eptp(&vmx->vcpu, 0)); 2085 2086 /* All VMFUNCs are currently emulated through L0 vmexits. */ 2087 if (cpu_has_vmx_vmfunc()) 2088 vmcs_write64(VM_FUNCTION_CONTROL, 0); 2089 2090 if (cpu_has_vmx_posted_intr()) 2091 vmcs_write16(POSTED_INTR_NV, POSTED_INTR_NESTED_VECTOR); 2092 2093 if (cpu_has_vmx_msr_bitmap()) 2094 vmcs_write64(MSR_BITMAP, __pa(vmx->nested.vmcs02.msr_bitmap)); 2095 2096 /* 2097 * The PML address never changes, so it is constant in vmcs02. 2098 * Conceptually we want to copy the PML index from vmcs01 here, 2099 * and then back to vmcs01 on nested vmexit. But since we flush 2100 * the log and reset GUEST_PML_INDEX on each vmexit, the PML 2101 * index is also effectively constant in vmcs02. 2102 */ 2103 if (enable_pml) { 2104 vmcs_write64(PML_ADDRESS, page_to_phys(vmx->pml_pg)); 2105 vmcs_write16(GUEST_PML_INDEX, PML_ENTITY_NUM - 1); 2106 } 2107 2108 if (cpu_has_vmx_encls_vmexit()) 2109 vmcs_write64(ENCLS_EXITING_BITMAP, -1ull); 2110 2111 /* 2112 * Set the MSR load/store lists to match L0's settings. Only the 2113 * addresses are constant (for vmcs02), the counts can change based 2114 * on L2's behavior, e.g. switching to/from long mode. 2115 */ 2116 vmcs_write64(VM_EXIT_MSR_STORE_ADDR, __pa(vmx->msr_autostore.guest.val)); 2117 vmcs_write64(VM_EXIT_MSR_LOAD_ADDR, __pa(vmx->msr_autoload.host.val)); 2118 vmcs_write64(VM_ENTRY_MSR_LOAD_ADDR, __pa(vmx->msr_autoload.guest.val)); 2119 2120 vmx_set_constant_host_state(vmx); 2121 } 2122 2123 static void prepare_vmcs02_early_rare(struct vcpu_vmx *vmx, 2124 struct vmcs12 *vmcs12) 2125 { 2126 prepare_vmcs02_constant_state(vmx); 2127 2128 vmcs_write64(VMCS_LINK_POINTER, -1ull); 2129 2130 if (enable_vpid) { 2131 if (nested_cpu_has_vpid(vmcs12) && vmx->nested.vpid02) 2132 vmcs_write16(VIRTUAL_PROCESSOR_ID, vmx->nested.vpid02); 2133 else 2134 vmcs_write16(VIRTUAL_PROCESSOR_ID, vmx->vpid); 2135 } 2136 } 2137 2138 static void prepare_vmcs02_early(struct vcpu_vmx *vmx, struct vmcs12 *vmcs12) 2139 { 2140 u32 exec_control, vmcs12_exec_ctrl; 2141 u64 guest_efer = nested_vmx_calc_efer(vmx, vmcs12); 2142 2143 if (vmx->nested.dirty_vmcs12 || vmx->nested.hv_evmcs) 2144 prepare_vmcs02_early_rare(vmx, vmcs12); 2145 2146 /* 2147 * PIN CONTROLS 2148 */ 2149 exec_control = vmx_pin_based_exec_ctrl(vmx); 2150 exec_control |= (vmcs12->pin_based_vm_exec_control & 2151 ~PIN_BASED_VMX_PREEMPTION_TIMER); 2152 2153 /* Posted interrupts setting is only taken from vmcs12. */ 2154 if (nested_cpu_has_posted_intr(vmcs12)) { 2155 vmx->nested.posted_intr_nv = vmcs12->posted_intr_nv; 2156 vmx->nested.pi_pending = false; 2157 } else { 2158 exec_control &= ~PIN_BASED_POSTED_INTR; 2159 } 2160 pin_controls_set(vmx, exec_control); 2161 2162 /* 2163 * EXEC CONTROLS 2164 */ 2165 exec_control = vmx_exec_control(vmx); /* L0's desires */ 2166 exec_control &= ~CPU_BASED_INTR_WINDOW_EXITING; 2167 exec_control &= ~CPU_BASED_NMI_WINDOW_EXITING; 2168 exec_control &= ~CPU_BASED_TPR_SHADOW; 2169 exec_control |= vmcs12->cpu_based_vm_exec_control; 2170 2171 vmx->nested.l1_tpr_threshold = -1; 2172 if (exec_control & CPU_BASED_TPR_SHADOW) 2173 vmcs_write32(TPR_THRESHOLD, vmcs12->tpr_threshold); 2174 #ifdef CONFIG_X86_64 2175 else 2176 exec_control |= CPU_BASED_CR8_LOAD_EXITING | 2177 CPU_BASED_CR8_STORE_EXITING; 2178 #endif 2179 2180 /* 2181 * A vmexit (to either L1 hypervisor or L0 userspace) is always needed 2182 * for I/O port accesses. 2183 */ 2184 exec_control |= CPU_BASED_UNCOND_IO_EXITING; 2185 exec_control &= ~CPU_BASED_USE_IO_BITMAPS; 2186 2187 /* 2188 * This bit will be computed in nested_get_vmcs12_pages, because 2189 * we do not have access to L1's MSR bitmap yet. For now, keep 2190 * the same bit as before, hoping to avoid multiple VMWRITEs that 2191 * only set/clear this bit. 2192 */ 2193 exec_control &= ~CPU_BASED_USE_MSR_BITMAPS; 2194 exec_control |= exec_controls_get(vmx) & CPU_BASED_USE_MSR_BITMAPS; 2195 2196 exec_controls_set(vmx, exec_control); 2197 2198 /* 2199 * SECONDARY EXEC CONTROLS 2200 */ 2201 if (cpu_has_secondary_exec_ctrls()) { 2202 exec_control = vmx->secondary_exec_control; 2203 2204 /* Take the following fields only from vmcs12 */ 2205 exec_control &= ~(SECONDARY_EXEC_VIRTUALIZE_APIC_ACCESSES | 2206 SECONDARY_EXEC_ENABLE_INVPCID | 2207 SECONDARY_EXEC_RDTSCP | 2208 SECONDARY_EXEC_XSAVES | 2209 SECONDARY_EXEC_ENABLE_USR_WAIT_PAUSE | 2210 SECONDARY_EXEC_VIRTUAL_INTR_DELIVERY | 2211 SECONDARY_EXEC_APIC_REGISTER_VIRT | 2212 SECONDARY_EXEC_ENABLE_VMFUNC); 2213 if (nested_cpu_has(vmcs12, 2214 CPU_BASED_ACTIVATE_SECONDARY_CONTROLS)) { 2215 vmcs12_exec_ctrl = vmcs12->secondary_vm_exec_control & 2216 ~SECONDARY_EXEC_ENABLE_PML; 2217 exec_control |= vmcs12_exec_ctrl; 2218 } 2219 2220 /* VMCS shadowing for L2 is emulated for now */ 2221 exec_control &= ~SECONDARY_EXEC_SHADOW_VMCS; 2222 2223 /* 2224 * Preset *DT exiting when emulating UMIP, so that vmx_set_cr4() 2225 * will not have to rewrite the controls just for this bit. 2226 */ 2227 if (!boot_cpu_has(X86_FEATURE_UMIP) && vmx_umip_emulated() && 2228 (vmcs12->guest_cr4 & X86_CR4_UMIP)) 2229 exec_control |= SECONDARY_EXEC_DESC; 2230 2231 if (exec_control & SECONDARY_EXEC_VIRTUAL_INTR_DELIVERY) 2232 vmcs_write16(GUEST_INTR_STATUS, 2233 vmcs12->guest_intr_status); 2234 2235 secondary_exec_controls_set(vmx, exec_control); 2236 } 2237 2238 /* 2239 * ENTRY CONTROLS 2240 * 2241 * vmcs12's VM_{ENTRY,EXIT}_LOAD_IA32_EFER and VM_ENTRY_IA32E_MODE 2242 * are emulated by vmx_set_efer() in prepare_vmcs02(), but speculate 2243 * on the related bits (if supported by the CPU) in the hope that 2244 * we can avoid VMWrites during vmx_set_efer(). 2245 */ 2246 exec_control = (vmcs12->vm_entry_controls | vmx_vmentry_ctrl()) & 2247 ~VM_ENTRY_IA32E_MODE & ~VM_ENTRY_LOAD_IA32_EFER; 2248 if (cpu_has_load_ia32_efer()) { 2249 if (guest_efer & EFER_LMA) 2250 exec_control |= VM_ENTRY_IA32E_MODE; 2251 if (guest_efer != host_efer) 2252 exec_control |= VM_ENTRY_LOAD_IA32_EFER; 2253 } 2254 vm_entry_controls_set(vmx, exec_control); 2255 2256 /* 2257 * EXIT CONTROLS 2258 * 2259 * L2->L1 exit controls are emulated - the hardware exit is to L0 so 2260 * we should use its exit controls. Note that VM_EXIT_LOAD_IA32_EFER 2261 * bits may be modified by vmx_set_efer() in prepare_vmcs02(). 2262 */ 2263 exec_control = vmx_vmexit_ctrl(); 2264 if (cpu_has_load_ia32_efer() && guest_efer != host_efer) 2265 exec_control |= VM_EXIT_LOAD_IA32_EFER; 2266 vm_exit_controls_set(vmx, exec_control); 2267 2268 /* 2269 * Interrupt/Exception Fields 2270 */ 2271 if (vmx->nested.nested_run_pending) { 2272 vmcs_write32(VM_ENTRY_INTR_INFO_FIELD, 2273 vmcs12->vm_entry_intr_info_field); 2274 vmcs_write32(VM_ENTRY_EXCEPTION_ERROR_CODE, 2275 vmcs12->vm_entry_exception_error_code); 2276 vmcs_write32(VM_ENTRY_INSTRUCTION_LEN, 2277 vmcs12->vm_entry_instruction_len); 2278 vmcs_write32(GUEST_INTERRUPTIBILITY_INFO, 2279 vmcs12->guest_interruptibility_info); 2280 vmx->loaded_vmcs->nmi_known_unmasked = 2281 !(vmcs12->guest_interruptibility_info & GUEST_INTR_STATE_NMI); 2282 } else { 2283 vmcs_write32(VM_ENTRY_INTR_INFO_FIELD, 0); 2284 } 2285 } 2286 2287 static void prepare_vmcs02_rare(struct vcpu_vmx *vmx, struct vmcs12 *vmcs12) 2288 { 2289 struct hv_enlightened_vmcs *hv_evmcs = vmx->nested.hv_evmcs; 2290 2291 if (!hv_evmcs || !(hv_evmcs->hv_clean_fields & 2292 HV_VMX_ENLIGHTENED_CLEAN_FIELD_GUEST_GRP2)) { 2293 vmcs_write16(GUEST_ES_SELECTOR, vmcs12->guest_es_selector); 2294 vmcs_write16(GUEST_CS_SELECTOR, vmcs12->guest_cs_selector); 2295 vmcs_write16(GUEST_SS_SELECTOR, vmcs12->guest_ss_selector); 2296 vmcs_write16(GUEST_DS_SELECTOR, vmcs12->guest_ds_selector); 2297 vmcs_write16(GUEST_FS_SELECTOR, vmcs12->guest_fs_selector); 2298 vmcs_write16(GUEST_GS_SELECTOR, vmcs12->guest_gs_selector); 2299 vmcs_write16(GUEST_LDTR_SELECTOR, vmcs12->guest_ldtr_selector); 2300 vmcs_write16(GUEST_TR_SELECTOR, vmcs12->guest_tr_selector); 2301 vmcs_write32(GUEST_ES_LIMIT, vmcs12->guest_es_limit); 2302 vmcs_write32(GUEST_CS_LIMIT, vmcs12->guest_cs_limit); 2303 vmcs_write32(GUEST_SS_LIMIT, vmcs12->guest_ss_limit); 2304 vmcs_write32(GUEST_DS_LIMIT, vmcs12->guest_ds_limit); 2305 vmcs_write32(GUEST_FS_LIMIT, vmcs12->guest_fs_limit); 2306 vmcs_write32(GUEST_GS_LIMIT, vmcs12->guest_gs_limit); 2307 vmcs_write32(GUEST_LDTR_LIMIT, vmcs12->guest_ldtr_limit); 2308 vmcs_write32(GUEST_TR_LIMIT, vmcs12->guest_tr_limit); 2309 vmcs_write32(GUEST_GDTR_LIMIT, vmcs12->guest_gdtr_limit); 2310 vmcs_write32(GUEST_IDTR_LIMIT, vmcs12->guest_idtr_limit); 2311 vmcs_write32(GUEST_CS_AR_BYTES, vmcs12->guest_cs_ar_bytes); 2312 vmcs_write32(GUEST_SS_AR_BYTES, vmcs12->guest_ss_ar_bytes); 2313 vmcs_write32(GUEST_ES_AR_BYTES, vmcs12->guest_es_ar_bytes); 2314 vmcs_write32(GUEST_DS_AR_BYTES, vmcs12->guest_ds_ar_bytes); 2315 vmcs_write32(GUEST_FS_AR_BYTES, vmcs12->guest_fs_ar_bytes); 2316 vmcs_write32(GUEST_GS_AR_BYTES, vmcs12->guest_gs_ar_bytes); 2317 vmcs_write32(GUEST_LDTR_AR_BYTES, vmcs12->guest_ldtr_ar_bytes); 2318 vmcs_write32(GUEST_TR_AR_BYTES, vmcs12->guest_tr_ar_bytes); 2319 vmcs_writel(GUEST_ES_BASE, vmcs12->guest_es_base); 2320 vmcs_writel(GUEST_CS_BASE, vmcs12->guest_cs_base); 2321 vmcs_writel(GUEST_SS_BASE, vmcs12->guest_ss_base); 2322 vmcs_writel(GUEST_DS_BASE, vmcs12->guest_ds_base); 2323 vmcs_writel(GUEST_FS_BASE, vmcs12->guest_fs_base); 2324 vmcs_writel(GUEST_GS_BASE, vmcs12->guest_gs_base); 2325 vmcs_writel(GUEST_LDTR_BASE, vmcs12->guest_ldtr_base); 2326 vmcs_writel(GUEST_TR_BASE, vmcs12->guest_tr_base); 2327 vmcs_writel(GUEST_GDTR_BASE, vmcs12->guest_gdtr_base); 2328 vmcs_writel(GUEST_IDTR_BASE, vmcs12->guest_idtr_base); 2329 } 2330 2331 if (!hv_evmcs || !(hv_evmcs->hv_clean_fields & 2332 HV_VMX_ENLIGHTENED_CLEAN_FIELD_GUEST_GRP1)) { 2333 vmcs_write32(GUEST_SYSENTER_CS, vmcs12->guest_sysenter_cs); 2334 vmcs_writel(GUEST_PENDING_DBG_EXCEPTIONS, 2335 vmcs12->guest_pending_dbg_exceptions); 2336 vmcs_writel(GUEST_SYSENTER_ESP, vmcs12->guest_sysenter_esp); 2337 vmcs_writel(GUEST_SYSENTER_EIP, vmcs12->guest_sysenter_eip); 2338 2339 /* 2340 * L1 may access the L2's PDPTR, so save them to construct 2341 * vmcs12 2342 */ 2343 if (enable_ept) { 2344 vmcs_write64(GUEST_PDPTR0, vmcs12->guest_pdptr0); 2345 vmcs_write64(GUEST_PDPTR1, vmcs12->guest_pdptr1); 2346 vmcs_write64(GUEST_PDPTR2, vmcs12->guest_pdptr2); 2347 vmcs_write64(GUEST_PDPTR3, vmcs12->guest_pdptr3); 2348 } 2349 2350 if (kvm_mpx_supported() && vmx->nested.nested_run_pending && 2351 (vmcs12->vm_entry_controls & VM_ENTRY_LOAD_BNDCFGS)) 2352 vmcs_write64(GUEST_BNDCFGS, vmcs12->guest_bndcfgs); 2353 } 2354 2355 if (nested_cpu_has_xsaves(vmcs12)) 2356 vmcs_write64(XSS_EXIT_BITMAP, vmcs12->xss_exit_bitmap); 2357 2358 /* 2359 * Whether page-faults are trapped is determined by a combination of 2360 * 3 settings: PFEC_MASK, PFEC_MATCH and EXCEPTION_BITMAP.PF. 2361 * If enable_ept, L0 doesn't care about page faults and we should 2362 * set all of these to L1's desires. However, if !enable_ept, L0 does 2363 * care about (at least some) page faults, and because it is not easy 2364 * (if at all possible?) to merge L0 and L1's desires, we simply ask 2365 * to exit on each and every L2 page fault. This is done by setting 2366 * MASK=MATCH=0 and (see below) EB.PF=1. 2367 * Note that below we don't need special code to set EB.PF beyond the 2368 * "or"ing of the EB of vmcs01 and vmcs12, because when enable_ept, 2369 * vmcs01's EB.PF is 0 so the "or" will take vmcs12's value, and when 2370 * !enable_ept, EB.PF is 1, so the "or" will always be 1. 2371 */ 2372 vmcs_write32(PAGE_FAULT_ERROR_CODE_MASK, 2373 enable_ept ? vmcs12->page_fault_error_code_mask : 0); 2374 vmcs_write32(PAGE_FAULT_ERROR_CODE_MATCH, 2375 enable_ept ? vmcs12->page_fault_error_code_match : 0); 2376 2377 if (cpu_has_vmx_apicv()) { 2378 vmcs_write64(EOI_EXIT_BITMAP0, vmcs12->eoi_exit_bitmap0); 2379 vmcs_write64(EOI_EXIT_BITMAP1, vmcs12->eoi_exit_bitmap1); 2380 vmcs_write64(EOI_EXIT_BITMAP2, vmcs12->eoi_exit_bitmap2); 2381 vmcs_write64(EOI_EXIT_BITMAP3, vmcs12->eoi_exit_bitmap3); 2382 } 2383 2384 /* 2385 * Make sure the msr_autostore list is up to date before we set the 2386 * count in the vmcs02. 2387 */ 2388 prepare_vmx_msr_autostore_list(&vmx->vcpu, MSR_IA32_TSC); 2389 2390 vmcs_write32(VM_EXIT_MSR_STORE_COUNT, vmx->msr_autostore.guest.nr); 2391 vmcs_write32(VM_EXIT_MSR_LOAD_COUNT, vmx->msr_autoload.host.nr); 2392 vmcs_write32(VM_ENTRY_MSR_LOAD_COUNT, vmx->msr_autoload.guest.nr); 2393 2394 set_cr4_guest_host_mask(vmx); 2395 } 2396 2397 /* 2398 * prepare_vmcs02 is called when the L1 guest hypervisor runs its nested 2399 * L2 guest. L1 has a vmcs for L2 (vmcs12), and this function "merges" it 2400 * with L0's requirements for its guest (a.k.a. vmcs01), so we can run the L2 2401 * guest in a way that will both be appropriate to L1's requests, and our 2402 * needs. In addition to modifying the active vmcs (which is vmcs02), this 2403 * function also has additional necessary side-effects, like setting various 2404 * vcpu->arch fields. 2405 * Returns 0 on success, 1 on failure. Invalid state exit qualification code 2406 * is assigned to entry_failure_code on failure. 2407 */ 2408 static int prepare_vmcs02(struct kvm_vcpu *vcpu, struct vmcs12 *vmcs12, 2409 u32 *entry_failure_code) 2410 { 2411 struct vcpu_vmx *vmx = to_vmx(vcpu); 2412 struct hv_enlightened_vmcs *hv_evmcs = vmx->nested.hv_evmcs; 2413 bool load_guest_pdptrs_vmcs12 = false; 2414 2415 if (vmx->nested.dirty_vmcs12 || hv_evmcs) { 2416 prepare_vmcs02_rare(vmx, vmcs12); 2417 vmx->nested.dirty_vmcs12 = false; 2418 2419 load_guest_pdptrs_vmcs12 = !hv_evmcs || 2420 !(hv_evmcs->hv_clean_fields & 2421 HV_VMX_ENLIGHTENED_CLEAN_FIELD_GUEST_GRP1); 2422 } 2423 2424 if (vmx->nested.nested_run_pending && 2425 (vmcs12->vm_entry_controls & VM_ENTRY_LOAD_DEBUG_CONTROLS)) { 2426 kvm_set_dr(vcpu, 7, vmcs12->guest_dr7); 2427 vmcs_write64(GUEST_IA32_DEBUGCTL, vmcs12->guest_ia32_debugctl); 2428 } else { 2429 kvm_set_dr(vcpu, 7, vcpu->arch.dr7); 2430 vmcs_write64(GUEST_IA32_DEBUGCTL, vmx->nested.vmcs01_debugctl); 2431 } 2432 if (kvm_mpx_supported() && (!vmx->nested.nested_run_pending || 2433 !(vmcs12->vm_entry_controls & VM_ENTRY_LOAD_BNDCFGS))) 2434 vmcs_write64(GUEST_BNDCFGS, vmx->nested.vmcs01_guest_bndcfgs); 2435 vmx_set_rflags(vcpu, vmcs12->guest_rflags); 2436 2437 /* EXCEPTION_BITMAP and CR0_GUEST_HOST_MASK should basically be the 2438 * bitwise-or of what L1 wants to trap for L2, and what we want to 2439 * trap. Note that CR0.TS also needs updating - we do this later. 2440 */ 2441 update_exception_bitmap(vcpu); 2442 vcpu->arch.cr0_guest_owned_bits &= ~vmcs12->cr0_guest_host_mask; 2443 vmcs_writel(CR0_GUEST_HOST_MASK, ~vcpu->arch.cr0_guest_owned_bits); 2444 2445 if (vmx->nested.nested_run_pending && 2446 (vmcs12->vm_entry_controls & VM_ENTRY_LOAD_IA32_PAT)) { 2447 vmcs_write64(GUEST_IA32_PAT, vmcs12->guest_ia32_pat); 2448 vcpu->arch.pat = vmcs12->guest_ia32_pat; 2449 } else if (vmcs_config.vmentry_ctrl & VM_ENTRY_LOAD_IA32_PAT) { 2450 vmcs_write64(GUEST_IA32_PAT, vmx->vcpu.arch.pat); 2451 } 2452 2453 vmcs_write64(TSC_OFFSET, vcpu->arch.tsc_offset); 2454 2455 if (kvm_has_tsc_control) 2456 decache_tsc_multiplier(vmx); 2457 2458 if (enable_vpid) { 2459 /* 2460 * There is no direct mapping between vpid02 and vpid12, the 2461 * vpid02 is per-vCPU for L0 and reused while the value of 2462 * vpid12 is changed w/ one invvpid during nested vmentry. 2463 * The vpid12 is allocated by L1 for L2, so it will not 2464 * influence global bitmap(for vpid01 and vpid02 allocation) 2465 * even if spawn a lot of nested vCPUs. 2466 */ 2467 if (nested_cpu_has_vpid(vmcs12) && nested_has_guest_tlb_tag(vcpu)) { 2468 if (vmcs12->virtual_processor_id != vmx->nested.last_vpid) { 2469 vmx->nested.last_vpid = vmcs12->virtual_processor_id; 2470 __vmx_flush_tlb(vcpu, nested_get_vpid02(vcpu), false); 2471 } 2472 } else { 2473 /* 2474 * If L1 use EPT, then L0 needs to execute INVEPT on 2475 * EPTP02 instead of EPTP01. Therefore, delay TLB 2476 * flush until vmcs02->eptp is fully updated by 2477 * KVM_REQ_LOAD_CR3. Note that this assumes 2478 * KVM_REQ_TLB_FLUSH is evaluated after 2479 * KVM_REQ_LOAD_CR3 in vcpu_enter_guest(). 2480 */ 2481 kvm_make_request(KVM_REQ_TLB_FLUSH, vcpu); 2482 } 2483 } 2484 2485 if (nested_cpu_has_ept(vmcs12)) 2486 nested_ept_init_mmu_context(vcpu); 2487 2488 /* 2489 * This sets GUEST_CR0 to vmcs12->guest_cr0, possibly modifying those 2490 * bits which we consider mandatory enabled. 2491 * The CR0_READ_SHADOW is what L2 should have expected to read given 2492 * the specifications by L1; It's not enough to take 2493 * vmcs12->cr0_read_shadow because on our cr0_guest_host_mask we we 2494 * have more bits than L1 expected. 2495 */ 2496 vmx_set_cr0(vcpu, vmcs12->guest_cr0); 2497 vmcs_writel(CR0_READ_SHADOW, nested_read_cr0(vmcs12)); 2498 2499 vmx_set_cr4(vcpu, vmcs12->guest_cr4); 2500 vmcs_writel(CR4_READ_SHADOW, nested_read_cr4(vmcs12)); 2501 2502 vcpu->arch.efer = nested_vmx_calc_efer(vmx, vmcs12); 2503 /* Note: may modify VM_ENTRY/EXIT_CONTROLS and GUEST/HOST_IA32_EFER */ 2504 vmx_set_efer(vcpu, vcpu->arch.efer); 2505 2506 /* 2507 * Guest state is invalid and unrestricted guest is disabled, 2508 * which means L1 attempted VMEntry to L2 with invalid state. 2509 * Fail the VMEntry. 2510 */ 2511 if (vmx->emulation_required) { 2512 *entry_failure_code = ENTRY_FAIL_DEFAULT; 2513 return -EINVAL; 2514 } 2515 2516 /* Shadow page tables on either EPT or shadow page tables. */ 2517 if (nested_vmx_load_cr3(vcpu, vmcs12->guest_cr3, nested_cpu_has_ept(vmcs12), 2518 entry_failure_code)) 2519 return -EINVAL; 2520 2521 /* 2522 * Immediately write vmcs02.GUEST_CR3. It will be propagated to vmcs12 2523 * on nested VM-Exit, which can occur without actually running L2 and 2524 * thus without hitting vmx_set_cr3(), e.g. if L1 is entering L2 with 2525 * vmcs12.GUEST_ACTIVITYSTATE=HLT, in which case KVM will intercept the 2526 * transition to HLT instead of running L2. 2527 */ 2528 if (enable_ept) 2529 vmcs_writel(GUEST_CR3, vmcs12->guest_cr3); 2530 2531 /* Late preparation of GUEST_PDPTRs now that EFER and CRs are set. */ 2532 if (load_guest_pdptrs_vmcs12 && nested_cpu_has_ept(vmcs12) && 2533 is_pae_paging(vcpu)) { 2534 vmcs_write64(GUEST_PDPTR0, vmcs12->guest_pdptr0); 2535 vmcs_write64(GUEST_PDPTR1, vmcs12->guest_pdptr1); 2536 vmcs_write64(GUEST_PDPTR2, vmcs12->guest_pdptr2); 2537 vmcs_write64(GUEST_PDPTR3, vmcs12->guest_pdptr3); 2538 } 2539 2540 if (!enable_ept) 2541 vcpu->arch.walk_mmu->inject_page_fault = vmx_inject_page_fault_nested; 2542 2543 if ((vmcs12->vm_entry_controls & VM_ENTRY_LOAD_IA32_PERF_GLOBAL_CTRL) && 2544 WARN_ON_ONCE(kvm_set_msr(vcpu, MSR_CORE_PERF_GLOBAL_CTRL, 2545 vmcs12->guest_ia32_perf_global_ctrl))) 2546 return -EINVAL; 2547 2548 kvm_rsp_write(vcpu, vmcs12->guest_rsp); 2549 kvm_rip_write(vcpu, vmcs12->guest_rip); 2550 return 0; 2551 } 2552 2553 static int nested_vmx_check_nmi_controls(struct vmcs12 *vmcs12) 2554 { 2555 if (CC(!nested_cpu_has_nmi_exiting(vmcs12) && 2556 nested_cpu_has_virtual_nmis(vmcs12))) 2557 return -EINVAL; 2558 2559 if (CC(!nested_cpu_has_virtual_nmis(vmcs12) && 2560 nested_cpu_has(vmcs12, CPU_BASED_NMI_WINDOW_EXITING))) 2561 return -EINVAL; 2562 2563 return 0; 2564 } 2565 2566 static bool valid_ept_address(struct kvm_vcpu *vcpu, u64 address) 2567 { 2568 struct vcpu_vmx *vmx = to_vmx(vcpu); 2569 int maxphyaddr = cpuid_maxphyaddr(vcpu); 2570 2571 /* Check for memory type validity */ 2572 switch (address & VMX_EPTP_MT_MASK) { 2573 case VMX_EPTP_MT_UC: 2574 if (CC(!(vmx->nested.msrs.ept_caps & VMX_EPTP_UC_BIT))) 2575 return false; 2576 break; 2577 case VMX_EPTP_MT_WB: 2578 if (CC(!(vmx->nested.msrs.ept_caps & VMX_EPTP_WB_BIT))) 2579 return false; 2580 break; 2581 default: 2582 return false; 2583 } 2584 2585 /* only 4 levels page-walk length are valid */ 2586 if (CC((address & VMX_EPTP_PWL_MASK) != VMX_EPTP_PWL_4)) 2587 return false; 2588 2589 /* Reserved bits should not be set */ 2590 if (CC(address >> maxphyaddr || ((address >> 7) & 0x1f))) 2591 return false; 2592 2593 /* AD, if set, should be supported */ 2594 if (address & VMX_EPTP_AD_ENABLE_BIT) { 2595 if (CC(!(vmx->nested.msrs.ept_caps & VMX_EPT_AD_BIT))) 2596 return false; 2597 } 2598 2599 return true; 2600 } 2601 2602 /* 2603 * Checks related to VM-Execution Control Fields 2604 */ 2605 static int nested_check_vm_execution_controls(struct kvm_vcpu *vcpu, 2606 struct vmcs12 *vmcs12) 2607 { 2608 struct vcpu_vmx *vmx = to_vmx(vcpu); 2609 2610 if (CC(!vmx_control_verify(vmcs12->pin_based_vm_exec_control, 2611 vmx->nested.msrs.pinbased_ctls_low, 2612 vmx->nested.msrs.pinbased_ctls_high)) || 2613 CC(!vmx_control_verify(vmcs12->cpu_based_vm_exec_control, 2614 vmx->nested.msrs.procbased_ctls_low, 2615 vmx->nested.msrs.procbased_ctls_high))) 2616 return -EINVAL; 2617 2618 if (nested_cpu_has(vmcs12, CPU_BASED_ACTIVATE_SECONDARY_CONTROLS) && 2619 CC(!vmx_control_verify(vmcs12->secondary_vm_exec_control, 2620 vmx->nested.msrs.secondary_ctls_low, 2621 vmx->nested.msrs.secondary_ctls_high))) 2622 return -EINVAL; 2623 2624 if (CC(vmcs12->cr3_target_count > nested_cpu_vmx_misc_cr3_count(vcpu)) || 2625 nested_vmx_check_io_bitmap_controls(vcpu, vmcs12) || 2626 nested_vmx_check_msr_bitmap_controls(vcpu, vmcs12) || 2627 nested_vmx_check_tpr_shadow_controls(vcpu, vmcs12) || 2628 nested_vmx_check_apic_access_controls(vcpu, vmcs12) || 2629 nested_vmx_check_apicv_controls(vcpu, vmcs12) || 2630 nested_vmx_check_nmi_controls(vmcs12) || 2631 nested_vmx_check_pml_controls(vcpu, vmcs12) || 2632 nested_vmx_check_unrestricted_guest_controls(vcpu, vmcs12) || 2633 nested_vmx_check_mode_based_ept_exec_controls(vcpu, vmcs12) || 2634 nested_vmx_check_shadow_vmcs_controls(vcpu, vmcs12) || 2635 CC(nested_cpu_has_vpid(vmcs12) && !vmcs12->virtual_processor_id)) 2636 return -EINVAL; 2637 2638 if (!nested_cpu_has_preemption_timer(vmcs12) && 2639 nested_cpu_has_save_preemption_timer(vmcs12)) 2640 return -EINVAL; 2641 2642 if (nested_cpu_has_ept(vmcs12) && 2643 CC(!valid_ept_address(vcpu, vmcs12->ept_pointer))) 2644 return -EINVAL; 2645 2646 if (nested_cpu_has_vmfunc(vmcs12)) { 2647 if (CC(vmcs12->vm_function_control & 2648 ~vmx->nested.msrs.vmfunc_controls)) 2649 return -EINVAL; 2650 2651 if (nested_cpu_has_eptp_switching(vmcs12)) { 2652 if (CC(!nested_cpu_has_ept(vmcs12)) || 2653 CC(!page_address_valid(vcpu, vmcs12->eptp_list_address))) 2654 return -EINVAL; 2655 } 2656 } 2657 2658 return 0; 2659 } 2660 2661 /* 2662 * Checks related to VM-Exit Control Fields 2663 */ 2664 static int nested_check_vm_exit_controls(struct kvm_vcpu *vcpu, 2665 struct vmcs12 *vmcs12) 2666 { 2667 struct vcpu_vmx *vmx = to_vmx(vcpu); 2668 2669 if (CC(!vmx_control_verify(vmcs12->vm_exit_controls, 2670 vmx->nested.msrs.exit_ctls_low, 2671 vmx->nested.msrs.exit_ctls_high)) || 2672 CC(nested_vmx_check_exit_msr_switch_controls(vcpu, vmcs12))) 2673 return -EINVAL; 2674 2675 return 0; 2676 } 2677 2678 /* 2679 * Checks related to VM-Entry Control Fields 2680 */ 2681 static int nested_check_vm_entry_controls(struct kvm_vcpu *vcpu, 2682 struct vmcs12 *vmcs12) 2683 { 2684 struct vcpu_vmx *vmx = to_vmx(vcpu); 2685 2686 if (CC(!vmx_control_verify(vmcs12->vm_entry_controls, 2687 vmx->nested.msrs.entry_ctls_low, 2688 vmx->nested.msrs.entry_ctls_high))) 2689 return -EINVAL; 2690 2691 /* 2692 * From the Intel SDM, volume 3: 2693 * Fields relevant to VM-entry event injection must be set properly. 2694 * These fields are the VM-entry interruption-information field, the 2695 * VM-entry exception error code, and the VM-entry instruction length. 2696 */ 2697 if (vmcs12->vm_entry_intr_info_field & INTR_INFO_VALID_MASK) { 2698 u32 intr_info = vmcs12->vm_entry_intr_info_field; 2699 u8 vector = intr_info & INTR_INFO_VECTOR_MASK; 2700 u32 intr_type = intr_info & INTR_INFO_INTR_TYPE_MASK; 2701 bool has_error_code = intr_info & INTR_INFO_DELIVER_CODE_MASK; 2702 bool should_have_error_code; 2703 bool urg = nested_cpu_has2(vmcs12, 2704 SECONDARY_EXEC_UNRESTRICTED_GUEST); 2705 bool prot_mode = !urg || vmcs12->guest_cr0 & X86_CR0_PE; 2706 2707 /* VM-entry interruption-info field: interruption type */ 2708 if (CC(intr_type == INTR_TYPE_RESERVED) || 2709 CC(intr_type == INTR_TYPE_OTHER_EVENT && 2710 !nested_cpu_supports_monitor_trap_flag(vcpu))) 2711 return -EINVAL; 2712 2713 /* VM-entry interruption-info field: vector */ 2714 if (CC(intr_type == INTR_TYPE_NMI_INTR && vector != NMI_VECTOR) || 2715 CC(intr_type == INTR_TYPE_HARD_EXCEPTION && vector > 31) || 2716 CC(intr_type == INTR_TYPE_OTHER_EVENT && vector != 0)) 2717 return -EINVAL; 2718 2719 /* VM-entry interruption-info field: deliver error code */ 2720 should_have_error_code = 2721 intr_type == INTR_TYPE_HARD_EXCEPTION && prot_mode && 2722 x86_exception_has_error_code(vector); 2723 if (CC(has_error_code != should_have_error_code)) 2724 return -EINVAL; 2725 2726 /* VM-entry exception error code */ 2727 if (CC(has_error_code && 2728 vmcs12->vm_entry_exception_error_code & GENMASK(31, 16))) 2729 return -EINVAL; 2730 2731 /* VM-entry interruption-info field: reserved bits */ 2732 if (CC(intr_info & INTR_INFO_RESVD_BITS_MASK)) 2733 return -EINVAL; 2734 2735 /* VM-entry instruction length */ 2736 switch (intr_type) { 2737 case INTR_TYPE_SOFT_EXCEPTION: 2738 case INTR_TYPE_SOFT_INTR: 2739 case INTR_TYPE_PRIV_SW_EXCEPTION: 2740 if (CC(vmcs12->vm_entry_instruction_len > 15) || 2741 CC(vmcs12->vm_entry_instruction_len == 0 && 2742 CC(!nested_cpu_has_zero_length_injection(vcpu)))) 2743 return -EINVAL; 2744 } 2745 } 2746 2747 if (nested_vmx_check_entry_msr_switch_controls(vcpu, vmcs12)) 2748 return -EINVAL; 2749 2750 return 0; 2751 } 2752 2753 static int nested_vmx_check_controls(struct kvm_vcpu *vcpu, 2754 struct vmcs12 *vmcs12) 2755 { 2756 if (nested_check_vm_execution_controls(vcpu, vmcs12) || 2757 nested_check_vm_exit_controls(vcpu, vmcs12) || 2758 nested_check_vm_entry_controls(vcpu, vmcs12)) 2759 return -EINVAL; 2760 2761 if (to_vmx(vcpu)->nested.enlightened_vmcs_enabled) 2762 return nested_evmcs_check_controls(vmcs12); 2763 2764 return 0; 2765 } 2766 2767 static int nested_vmx_check_host_state(struct kvm_vcpu *vcpu, 2768 struct vmcs12 *vmcs12) 2769 { 2770 bool ia32e; 2771 2772 if (CC(!nested_host_cr0_valid(vcpu, vmcs12->host_cr0)) || 2773 CC(!nested_host_cr4_valid(vcpu, vmcs12->host_cr4)) || 2774 CC(!nested_cr3_valid(vcpu, vmcs12->host_cr3))) 2775 return -EINVAL; 2776 2777 if (CC(is_noncanonical_address(vmcs12->host_ia32_sysenter_esp, vcpu)) || 2778 CC(is_noncanonical_address(vmcs12->host_ia32_sysenter_eip, vcpu))) 2779 return -EINVAL; 2780 2781 if ((vmcs12->vm_exit_controls & VM_EXIT_LOAD_IA32_PAT) && 2782 CC(!kvm_pat_valid(vmcs12->host_ia32_pat))) 2783 return -EINVAL; 2784 2785 if ((vmcs12->vm_exit_controls & VM_EXIT_LOAD_IA32_PERF_GLOBAL_CTRL) && 2786 CC(!kvm_valid_perf_global_ctrl(vcpu_to_pmu(vcpu), 2787 vmcs12->host_ia32_perf_global_ctrl))) 2788 return -EINVAL; 2789 2790 #ifdef CONFIG_X86_64 2791 ia32e = !!(vcpu->arch.efer & EFER_LMA); 2792 #else 2793 ia32e = false; 2794 #endif 2795 2796 if (ia32e) { 2797 if (CC(!(vmcs12->vm_exit_controls & VM_EXIT_HOST_ADDR_SPACE_SIZE)) || 2798 CC(!(vmcs12->host_cr4 & X86_CR4_PAE))) 2799 return -EINVAL; 2800 } else { 2801 if (CC(vmcs12->vm_exit_controls & VM_EXIT_HOST_ADDR_SPACE_SIZE) || 2802 CC(vmcs12->vm_entry_controls & VM_ENTRY_IA32E_MODE) || 2803 CC(vmcs12->host_cr4 & X86_CR4_PCIDE) || 2804 CC((vmcs12->host_rip) >> 32)) 2805 return -EINVAL; 2806 } 2807 2808 if (CC(vmcs12->host_cs_selector & (SEGMENT_RPL_MASK | SEGMENT_TI_MASK)) || 2809 CC(vmcs12->host_ss_selector & (SEGMENT_RPL_MASK | SEGMENT_TI_MASK)) || 2810 CC(vmcs12->host_ds_selector & (SEGMENT_RPL_MASK | SEGMENT_TI_MASK)) || 2811 CC(vmcs12->host_es_selector & (SEGMENT_RPL_MASK | SEGMENT_TI_MASK)) || 2812 CC(vmcs12->host_fs_selector & (SEGMENT_RPL_MASK | SEGMENT_TI_MASK)) || 2813 CC(vmcs12->host_gs_selector & (SEGMENT_RPL_MASK | SEGMENT_TI_MASK)) || 2814 CC(vmcs12->host_tr_selector & (SEGMENT_RPL_MASK | SEGMENT_TI_MASK)) || 2815 CC(vmcs12->host_cs_selector == 0) || 2816 CC(vmcs12->host_tr_selector == 0) || 2817 CC(vmcs12->host_ss_selector == 0 && !ia32e)) 2818 return -EINVAL; 2819 2820 if (CC(is_noncanonical_address(vmcs12->host_fs_base, vcpu)) || 2821 CC(is_noncanonical_address(vmcs12->host_gs_base, vcpu)) || 2822 CC(is_noncanonical_address(vmcs12->host_gdtr_base, vcpu)) || 2823 CC(is_noncanonical_address(vmcs12->host_idtr_base, vcpu)) || 2824 CC(is_noncanonical_address(vmcs12->host_tr_base, vcpu)) || 2825 CC(is_noncanonical_address(vmcs12->host_rip, vcpu))) 2826 return -EINVAL; 2827 2828 /* 2829 * If the load IA32_EFER VM-exit control is 1, bits reserved in the 2830 * IA32_EFER MSR must be 0 in the field for that register. In addition, 2831 * the values of the LMA and LME bits in the field must each be that of 2832 * the host address-space size VM-exit control. 2833 */ 2834 if (vmcs12->vm_exit_controls & VM_EXIT_LOAD_IA32_EFER) { 2835 if (CC(!kvm_valid_efer(vcpu, vmcs12->host_ia32_efer)) || 2836 CC(ia32e != !!(vmcs12->host_ia32_efer & EFER_LMA)) || 2837 CC(ia32e != !!(vmcs12->host_ia32_efer & EFER_LME))) 2838 return -EINVAL; 2839 } 2840 2841 return 0; 2842 } 2843 2844 static int nested_vmx_check_vmcs_link_ptr(struct kvm_vcpu *vcpu, 2845 struct vmcs12 *vmcs12) 2846 { 2847 int r = 0; 2848 struct vmcs12 *shadow; 2849 struct kvm_host_map map; 2850 2851 if (vmcs12->vmcs_link_pointer == -1ull) 2852 return 0; 2853 2854 if (CC(!page_address_valid(vcpu, vmcs12->vmcs_link_pointer))) 2855 return -EINVAL; 2856 2857 if (CC(kvm_vcpu_map(vcpu, gpa_to_gfn(vmcs12->vmcs_link_pointer), &map))) 2858 return -EINVAL; 2859 2860 shadow = map.hva; 2861 2862 if (CC(shadow->hdr.revision_id != VMCS12_REVISION) || 2863 CC(shadow->hdr.shadow_vmcs != nested_cpu_has_shadow_vmcs(vmcs12))) 2864 r = -EINVAL; 2865 2866 kvm_vcpu_unmap(vcpu, &map, false); 2867 return r; 2868 } 2869 2870 /* 2871 * Checks related to Guest Non-register State 2872 */ 2873 static int nested_check_guest_non_reg_state(struct vmcs12 *vmcs12) 2874 { 2875 if (CC(vmcs12->guest_activity_state != GUEST_ACTIVITY_ACTIVE && 2876 vmcs12->guest_activity_state != GUEST_ACTIVITY_HLT)) 2877 return -EINVAL; 2878 2879 return 0; 2880 } 2881 2882 static int nested_vmx_check_guest_state(struct kvm_vcpu *vcpu, 2883 struct vmcs12 *vmcs12, 2884 u32 *exit_qual) 2885 { 2886 bool ia32e; 2887 2888 *exit_qual = ENTRY_FAIL_DEFAULT; 2889 2890 if (CC(!nested_guest_cr0_valid(vcpu, vmcs12->guest_cr0)) || 2891 CC(!nested_guest_cr4_valid(vcpu, vmcs12->guest_cr4))) 2892 return -EINVAL; 2893 2894 if ((vmcs12->vm_entry_controls & VM_ENTRY_LOAD_DEBUG_CONTROLS) && 2895 CC(!kvm_dr7_valid(vmcs12->guest_dr7))) 2896 return -EINVAL; 2897 2898 if ((vmcs12->vm_entry_controls & VM_ENTRY_LOAD_IA32_PAT) && 2899 CC(!kvm_pat_valid(vmcs12->guest_ia32_pat))) 2900 return -EINVAL; 2901 2902 if (nested_vmx_check_vmcs_link_ptr(vcpu, vmcs12)) { 2903 *exit_qual = ENTRY_FAIL_VMCS_LINK_PTR; 2904 return -EINVAL; 2905 } 2906 2907 if ((vmcs12->vm_entry_controls & VM_ENTRY_LOAD_IA32_PERF_GLOBAL_CTRL) && 2908 CC(!kvm_valid_perf_global_ctrl(vcpu_to_pmu(vcpu), 2909 vmcs12->guest_ia32_perf_global_ctrl))) 2910 return -EINVAL; 2911 2912 /* 2913 * If the load IA32_EFER VM-entry control is 1, the following checks 2914 * are performed on the field for the IA32_EFER MSR: 2915 * - Bits reserved in the IA32_EFER MSR must be 0. 2916 * - Bit 10 (corresponding to IA32_EFER.LMA) must equal the value of 2917 * the IA-32e mode guest VM-exit control. It must also be identical 2918 * to bit 8 (LME) if bit 31 in the CR0 field (corresponding to 2919 * CR0.PG) is 1. 2920 */ 2921 if (to_vmx(vcpu)->nested.nested_run_pending && 2922 (vmcs12->vm_entry_controls & VM_ENTRY_LOAD_IA32_EFER)) { 2923 ia32e = (vmcs12->vm_entry_controls & VM_ENTRY_IA32E_MODE) != 0; 2924 if (CC(!kvm_valid_efer(vcpu, vmcs12->guest_ia32_efer)) || 2925 CC(ia32e != !!(vmcs12->guest_ia32_efer & EFER_LMA)) || 2926 CC(((vmcs12->guest_cr0 & X86_CR0_PG) && 2927 ia32e != !!(vmcs12->guest_ia32_efer & EFER_LME)))) 2928 return -EINVAL; 2929 } 2930 2931 if ((vmcs12->vm_entry_controls & VM_ENTRY_LOAD_BNDCFGS) && 2932 (CC(is_noncanonical_address(vmcs12->guest_bndcfgs & PAGE_MASK, vcpu)) || 2933 CC((vmcs12->guest_bndcfgs & MSR_IA32_BNDCFGS_RSVD)))) 2934 return -EINVAL; 2935 2936 if (nested_check_guest_non_reg_state(vmcs12)) 2937 return -EINVAL; 2938 2939 return 0; 2940 } 2941 2942 static int nested_vmx_check_vmentry_hw(struct kvm_vcpu *vcpu) 2943 { 2944 struct vcpu_vmx *vmx = to_vmx(vcpu); 2945 unsigned long cr3, cr4; 2946 bool vm_fail; 2947 2948 if (!nested_early_check) 2949 return 0; 2950 2951 if (vmx->msr_autoload.host.nr) 2952 vmcs_write32(VM_EXIT_MSR_LOAD_COUNT, 0); 2953 if (vmx->msr_autoload.guest.nr) 2954 vmcs_write32(VM_ENTRY_MSR_LOAD_COUNT, 0); 2955 2956 preempt_disable(); 2957 2958 vmx_prepare_switch_to_guest(vcpu); 2959 2960 /* 2961 * Induce a consistency check VMExit by clearing bit 1 in GUEST_RFLAGS, 2962 * which is reserved to '1' by hardware. GUEST_RFLAGS is guaranteed to 2963 * be written (by preparve_vmcs02()) before the "real" VMEnter, i.e. 2964 * there is no need to preserve other bits or save/restore the field. 2965 */ 2966 vmcs_writel(GUEST_RFLAGS, 0); 2967 2968 cr3 = __get_current_cr3_fast(); 2969 if (unlikely(cr3 != vmx->loaded_vmcs->host_state.cr3)) { 2970 vmcs_writel(HOST_CR3, cr3); 2971 vmx->loaded_vmcs->host_state.cr3 = cr3; 2972 } 2973 2974 cr4 = cr4_read_shadow(); 2975 if (unlikely(cr4 != vmx->loaded_vmcs->host_state.cr4)) { 2976 vmcs_writel(HOST_CR4, cr4); 2977 vmx->loaded_vmcs->host_state.cr4 = cr4; 2978 } 2979 2980 asm( 2981 "sub $%c[wordsize], %%" _ASM_SP "\n\t" /* temporarily adjust RSP for CALL */ 2982 "cmp %%" _ASM_SP ", %c[host_state_rsp](%[loaded_vmcs]) \n\t" 2983 "je 1f \n\t" 2984 __ex("vmwrite %%" _ASM_SP ", %[HOST_RSP]") "\n\t" 2985 "mov %%" _ASM_SP ", %c[host_state_rsp](%[loaded_vmcs]) \n\t" 2986 "1: \n\t" 2987 "add $%c[wordsize], %%" _ASM_SP "\n\t" /* un-adjust RSP */ 2988 2989 /* Check if vmlaunch or vmresume is needed */ 2990 "cmpb $0, %c[launched](%[loaded_vmcs])\n\t" 2991 2992 /* 2993 * VMLAUNCH and VMRESUME clear RFLAGS.{CF,ZF} on VM-Exit, set 2994 * RFLAGS.CF on VM-Fail Invalid and set RFLAGS.ZF on VM-Fail 2995 * Valid. vmx_vmenter() directly "returns" RFLAGS, and so the 2996 * results of VM-Enter is captured via CC_{SET,OUT} to vm_fail. 2997 */ 2998 "call vmx_vmenter\n\t" 2999 3000 CC_SET(be) 3001 : ASM_CALL_CONSTRAINT, CC_OUT(be) (vm_fail) 3002 : [HOST_RSP]"r"((unsigned long)HOST_RSP), 3003 [loaded_vmcs]"r"(vmx->loaded_vmcs), 3004 [launched]"i"(offsetof(struct loaded_vmcs, launched)), 3005 [host_state_rsp]"i"(offsetof(struct loaded_vmcs, host_state.rsp)), 3006 [wordsize]"i"(sizeof(ulong)) 3007 : "memory" 3008 ); 3009 3010 if (vmx->msr_autoload.host.nr) 3011 vmcs_write32(VM_EXIT_MSR_LOAD_COUNT, vmx->msr_autoload.host.nr); 3012 if (vmx->msr_autoload.guest.nr) 3013 vmcs_write32(VM_ENTRY_MSR_LOAD_COUNT, vmx->msr_autoload.guest.nr); 3014 3015 if (vm_fail) { 3016 u32 error = vmcs_read32(VM_INSTRUCTION_ERROR); 3017 3018 preempt_enable(); 3019 3020 trace_kvm_nested_vmenter_failed( 3021 "early hardware check VM-instruction error: ", error); 3022 WARN_ON_ONCE(error != VMXERR_ENTRY_INVALID_CONTROL_FIELD); 3023 return 1; 3024 } 3025 3026 /* 3027 * VMExit clears RFLAGS.IF and DR7, even on a consistency check. 3028 */ 3029 local_irq_enable(); 3030 if (hw_breakpoint_active()) 3031 set_debugreg(__this_cpu_read(cpu_dr7), 7); 3032 preempt_enable(); 3033 3034 /* 3035 * A non-failing VMEntry means we somehow entered guest mode with 3036 * an illegal RIP, and that's just the tip of the iceberg. There 3037 * is no telling what memory has been modified or what state has 3038 * been exposed to unknown code. Hitting this all but guarantees 3039 * a (very critical) hardware issue. 3040 */ 3041 WARN_ON(!(vmcs_read32(VM_EXIT_REASON) & 3042 VMX_EXIT_REASONS_FAILED_VMENTRY)); 3043 3044 return 0; 3045 } 3046 3047 static bool nested_get_vmcs12_pages(struct kvm_vcpu *vcpu) 3048 { 3049 struct vmcs12 *vmcs12 = get_vmcs12(vcpu); 3050 struct vcpu_vmx *vmx = to_vmx(vcpu); 3051 struct kvm_host_map *map; 3052 struct page *page; 3053 u64 hpa; 3054 3055 if (nested_cpu_has2(vmcs12, SECONDARY_EXEC_VIRTUALIZE_APIC_ACCESSES)) { 3056 /* 3057 * Translate L1 physical address to host physical 3058 * address for vmcs02. Keep the page pinned, so this 3059 * physical address remains valid. We keep a reference 3060 * to it so we can release it later. 3061 */ 3062 if (vmx->nested.apic_access_page) { /* shouldn't happen */ 3063 kvm_release_page_clean(vmx->nested.apic_access_page); 3064 vmx->nested.apic_access_page = NULL; 3065 } 3066 page = kvm_vcpu_gpa_to_page(vcpu, vmcs12->apic_access_addr); 3067 if (!is_error_page(page)) { 3068 vmx->nested.apic_access_page = page; 3069 hpa = page_to_phys(vmx->nested.apic_access_page); 3070 vmcs_write64(APIC_ACCESS_ADDR, hpa); 3071 } else { 3072 pr_debug_ratelimited("%s: no backing 'struct page' for APIC-access address in vmcs12\n", 3073 __func__); 3074 vcpu->run->exit_reason = KVM_EXIT_INTERNAL_ERROR; 3075 vcpu->run->internal.suberror = 3076 KVM_INTERNAL_ERROR_EMULATION; 3077 vcpu->run->internal.ndata = 0; 3078 return false; 3079 } 3080 } 3081 3082 if (nested_cpu_has(vmcs12, CPU_BASED_TPR_SHADOW)) { 3083 map = &vmx->nested.virtual_apic_map; 3084 3085 if (!kvm_vcpu_map(vcpu, gpa_to_gfn(vmcs12->virtual_apic_page_addr), map)) { 3086 vmcs_write64(VIRTUAL_APIC_PAGE_ADDR, pfn_to_hpa(map->pfn)); 3087 } else if (nested_cpu_has(vmcs12, CPU_BASED_CR8_LOAD_EXITING) && 3088 nested_cpu_has(vmcs12, CPU_BASED_CR8_STORE_EXITING) && 3089 !nested_cpu_has2(vmcs12, SECONDARY_EXEC_VIRTUALIZE_APIC_ACCESSES)) { 3090 /* 3091 * The processor will never use the TPR shadow, simply 3092 * clear the bit from the execution control. Such a 3093 * configuration is useless, but it happens in tests. 3094 * For any other configuration, failing the vm entry is 3095 * _not_ what the processor does but it's basically the 3096 * only possibility we have. 3097 */ 3098 exec_controls_clearbit(vmx, CPU_BASED_TPR_SHADOW); 3099 } else { 3100 /* 3101 * Write an illegal value to VIRTUAL_APIC_PAGE_ADDR to 3102 * force VM-Entry to fail. 3103 */ 3104 vmcs_write64(VIRTUAL_APIC_PAGE_ADDR, -1ull); 3105 } 3106 } 3107 3108 if (nested_cpu_has_posted_intr(vmcs12)) { 3109 map = &vmx->nested.pi_desc_map; 3110 3111 if (!kvm_vcpu_map(vcpu, gpa_to_gfn(vmcs12->posted_intr_desc_addr), map)) { 3112 vmx->nested.pi_desc = 3113 (struct pi_desc *)(((void *)map->hva) + 3114 offset_in_page(vmcs12->posted_intr_desc_addr)); 3115 vmcs_write64(POSTED_INTR_DESC_ADDR, 3116 pfn_to_hpa(map->pfn) + offset_in_page(vmcs12->posted_intr_desc_addr)); 3117 } 3118 } 3119 if (nested_vmx_prepare_msr_bitmap(vcpu, vmcs12)) 3120 exec_controls_setbit(vmx, CPU_BASED_USE_MSR_BITMAPS); 3121 else 3122 exec_controls_clearbit(vmx, CPU_BASED_USE_MSR_BITMAPS); 3123 return true; 3124 } 3125 3126 /* 3127 * Intel's VMX Instruction Reference specifies a common set of prerequisites 3128 * for running VMX instructions (except VMXON, whose prerequisites are 3129 * slightly different). It also specifies what exception to inject otherwise. 3130 * Note that many of these exceptions have priority over VM exits, so they 3131 * don't have to be checked again here. 3132 */ 3133 static int nested_vmx_check_permission(struct kvm_vcpu *vcpu) 3134 { 3135 if (!to_vmx(vcpu)->nested.vmxon) { 3136 kvm_queue_exception(vcpu, UD_VECTOR); 3137 return 0; 3138 } 3139 3140 if (vmx_get_cpl(vcpu)) { 3141 kvm_inject_gp(vcpu, 0); 3142 return 0; 3143 } 3144 3145 return 1; 3146 } 3147 3148 static u8 vmx_has_apicv_interrupt(struct kvm_vcpu *vcpu) 3149 { 3150 u8 rvi = vmx_get_rvi(); 3151 u8 vppr = kvm_lapic_get_reg(vcpu->arch.apic, APIC_PROCPRI); 3152 3153 return ((rvi & 0xf0) > (vppr & 0xf0)); 3154 } 3155 3156 static void load_vmcs12_host_state(struct kvm_vcpu *vcpu, 3157 struct vmcs12 *vmcs12); 3158 3159 /* 3160 * If from_vmentry is false, this is being called from state restore (either RSM 3161 * or KVM_SET_NESTED_STATE). Otherwise it's called from vmlaunch/vmresume. 3162 * 3163 * Returns: 3164 * NVMX_ENTRY_SUCCESS: Entered VMX non-root mode 3165 * NVMX_ENTRY_VMFAIL: Consistency check VMFail 3166 * NVMX_ENTRY_VMEXIT: Consistency check VMExit 3167 * NVMX_ENTRY_KVM_INTERNAL_ERROR: KVM internal error 3168 */ 3169 enum nvmx_vmentry_status nested_vmx_enter_non_root_mode(struct kvm_vcpu *vcpu, 3170 bool from_vmentry) 3171 { 3172 struct vcpu_vmx *vmx = to_vmx(vcpu); 3173 struct vmcs12 *vmcs12 = get_vmcs12(vcpu); 3174 bool evaluate_pending_interrupts; 3175 u32 exit_reason = EXIT_REASON_INVALID_STATE; 3176 u32 exit_qual; 3177 3178 evaluate_pending_interrupts = exec_controls_get(vmx) & 3179 (CPU_BASED_INTR_WINDOW_EXITING | CPU_BASED_NMI_WINDOW_EXITING); 3180 if (likely(!evaluate_pending_interrupts) && kvm_vcpu_apicv_active(vcpu)) 3181 evaluate_pending_interrupts |= vmx_has_apicv_interrupt(vcpu); 3182 3183 if (!(vmcs12->vm_entry_controls & VM_ENTRY_LOAD_DEBUG_CONTROLS)) 3184 vmx->nested.vmcs01_debugctl = vmcs_read64(GUEST_IA32_DEBUGCTL); 3185 if (kvm_mpx_supported() && 3186 !(vmcs12->vm_entry_controls & VM_ENTRY_LOAD_BNDCFGS)) 3187 vmx->nested.vmcs01_guest_bndcfgs = vmcs_read64(GUEST_BNDCFGS); 3188 3189 /* 3190 * Overwrite vmcs01.GUEST_CR3 with L1's CR3 if EPT is disabled *and* 3191 * nested early checks are disabled. In the event of a "late" VM-Fail, 3192 * i.e. a VM-Fail detected by hardware but not KVM, KVM must unwind its 3193 * software model to the pre-VMEntry host state. When EPT is disabled, 3194 * GUEST_CR3 holds KVM's shadow CR3, not L1's "real" CR3, which causes 3195 * nested_vmx_restore_host_state() to corrupt vcpu->arch.cr3. Stuffing 3196 * vmcs01.GUEST_CR3 results in the unwind naturally setting arch.cr3 to 3197 * the correct value. Smashing vmcs01.GUEST_CR3 is safe because nested 3198 * VM-Exits, and the unwind, reset KVM's MMU, i.e. vmcs01.GUEST_CR3 is 3199 * guaranteed to be overwritten with a shadow CR3 prior to re-entering 3200 * L1. Don't stuff vmcs01.GUEST_CR3 when using nested early checks as 3201 * KVM modifies vcpu->arch.cr3 if and only if the early hardware checks 3202 * pass, and early VM-Fails do not reset KVM's MMU, i.e. the VM-Fail 3203 * path would need to manually save/restore vmcs01.GUEST_CR3. 3204 */ 3205 if (!enable_ept && !nested_early_check) 3206 vmcs_writel(GUEST_CR3, vcpu->arch.cr3); 3207 3208 vmx_switch_vmcs(vcpu, &vmx->nested.vmcs02); 3209 3210 prepare_vmcs02_early(vmx, vmcs12); 3211 3212 if (from_vmentry) { 3213 if (unlikely(!nested_get_vmcs12_pages(vcpu))) 3214 return NVMX_VMENTRY_KVM_INTERNAL_ERROR; 3215 3216 if (nested_vmx_check_vmentry_hw(vcpu)) { 3217 vmx_switch_vmcs(vcpu, &vmx->vmcs01); 3218 return NVMX_VMENTRY_VMFAIL; 3219 } 3220 3221 if (nested_vmx_check_guest_state(vcpu, vmcs12, &exit_qual)) 3222 goto vmentry_fail_vmexit; 3223 } 3224 3225 enter_guest_mode(vcpu); 3226 if (vmcs12->cpu_based_vm_exec_control & CPU_BASED_USE_TSC_OFFSETTING) 3227 vcpu->arch.tsc_offset += vmcs12->tsc_offset; 3228 3229 if (prepare_vmcs02(vcpu, vmcs12, &exit_qual)) 3230 goto vmentry_fail_vmexit_guest_mode; 3231 3232 if (from_vmentry) { 3233 exit_reason = EXIT_REASON_MSR_LOAD_FAIL; 3234 exit_qual = nested_vmx_load_msr(vcpu, 3235 vmcs12->vm_entry_msr_load_addr, 3236 vmcs12->vm_entry_msr_load_count); 3237 if (exit_qual) 3238 goto vmentry_fail_vmexit_guest_mode; 3239 } else { 3240 /* 3241 * The MMU is not initialized to point at the right entities yet and 3242 * "get pages" would need to read data from the guest (i.e. we will 3243 * need to perform gpa to hpa translation). Request a call 3244 * to nested_get_vmcs12_pages before the next VM-entry. The MSRs 3245 * have already been set at vmentry time and should not be reset. 3246 */ 3247 kvm_make_request(KVM_REQ_GET_VMCS12_PAGES, vcpu); 3248 } 3249 3250 /* 3251 * If L1 had a pending IRQ/NMI until it executed 3252 * VMLAUNCH/VMRESUME which wasn't delivered because it was 3253 * disallowed (e.g. interrupts disabled), L0 needs to 3254 * evaluate if this pending event should cause an exit from L2 3255 * to L1 or delivered directly to L2 (e.g. In case L1 don't 3256 * intercept EXTERNAL_INTERRUPT). 3257 * 3258 * Usually this would be handled by the processor noticing an 3259 * IRQ/NMI window request, or checking RVI during evaluation of 3260 * pending virtual interrupts. However, this setting was done 3261 * on VMCS01 and now VMCS02 is active instead. Thus, we force L0 3262 * to perform pending event evaluation by requesting a KVM_REQ_EVENT. 3263 */ 3264 if (unlikely(evaluate_pending_interrupts)) 3265 kvm_make_request(KVM_REQ_EVENT, vcpu); 3266 3267 /* 3268 * Do not start the preemption timer hrtimer until after we know 3269 * we are successful, so that only nested_vmx_vmexit needs to cancel 3270 * the timer. 3271 */ 3272 vmx->nested.preemption_timer_expired = false; 3273 if (nested_cpu_has_preemption_timer(vmcs12)) 3274 vmx_start_preemption_timer(vcpu); 3275 3276 /* 3277 * Note no nested_vmx_succeed or nested_vmx_fail here. At this point 3278 * we are no longer running L1, and VMLAUNCH/VMRESUME has not yet 3279 * returned as far as L1 is concerned. It will only return (and set 3280 * the success flag) when L2 exits (see nested_vmx_vmexit()). 3281 */ 3282 return NVMX_VMENTRY_SUCCESS; 3283 3284 /* 3285 * A failed consistency check that leads to a VMExit during L1's 3286 * VMEnter to L2 is a variation of a normal VMexit, as explained in 3287 * 26.7 "VM-entry failures during or after loading guest state". 3288 */ 3289 vmentry_fail_vmexit_guest_mode: 3290 if (vmcs12->cpu_based_vm_exec_control & CPU_BASED_USE_TSC_OFFSETTING) 3291 vcpu->arch.tsc_offset -= vmcs12->tsc_offset; 3292 leave_guest_mode(vcpu); 3293 3294 vmentry_fail_vmexit: 3295 vmx_switch_vmcs(vcpu, &vmx->vmcs01); 3296 3297 if (!from_vmentry) 3298 return NVMX_VMENTRY_VMEXIT; 3299 3300 load_vmcs12_host_state(vcpu, vmcs12); 3301 vmcs12->vm_exit_reason = exit_reason | VMX_EXIT_REASONS_FAILED_VMENTRY; 3302 vmcs12->exit_qualification = exit_qual; 3303 if (enable_shadow_vmcs || vmx->nested.hv_evmcs) 3304 vmx->nested.need_vmcs12_to_shadow_sync = true; 3305 return NVMX_VMENTRY_VMEXIT; 3306 } 3307 3308 /* 3309 * nested_vmx_run() handles a nested entry, i.e., a VMLAUNCH or VMRESUME on L1 3310 * for running an L2 nested guest. 3311 */ 3312 static int nested_vmx_run(struct kvm_vcpu *vcpu, bool launch) 3313 { 3314 struct vmcs12 *vmcs12; 3315 enum nvmx_vmentry_status status; 3316 struct vcpu_vmx *vmx = to_vmx(vcpu); 3317 u32 interrupt_shadow = vmx_get_interrupt_shadow(vcpu); 3318 3319 if (!nested_vmx_check_permission(vcpu)) 3320 return 1; 3321 3322 if (!nested_vmx_handle_enlightened_vmptrld(vcpu, launch)) 3323 return 1; 3324 3325 if (!vmx->nested.hv_evmcs && vmx->nested.current_vmptr == -1ull) 3326 return nested_vmx_failInvalid(vcpu); 3327 3328 vmcs12 = get_vmcs12(vcpu); 3329 3330 /* 3331 * Can't VMLAUNCH or VMRESUME a shadow VMCS. Despite the fact 3332 * that there *is* a valid VMCS pointer, RFLAGS.CF is set 3333 * rather than RFLAGS.ZF, and no error number is stored to the 3334 * VM-instruction error field. 3335 */ 3336 if (vmcs12->hdr.shadow_vmcs) 3337 return nested_vmx_failInvalid(vcpu); 3338 3339 if (vmx->nested.hv_evmcs) { 3340 copy_enlightened_to_vmcs12(vmx); 3341 /* Enlightened VMCS doesn't have launch state */ 3342 vmcs12->launch_state = !launch; 3343 } else if (enable_shadow_vmcs) { 3344 copy_shadow_to_vmcs12(vmx); 3345 } 3346 3347 /* 3348 * The nested entry process starts with enforcing various prerequisites 3349 * on vmcs12 as required by the Intel SDM, and act appropriately when 3350 * they fail: As the SDM explains, some conditions should cause the 3351 * instruction to fail, while others will cause the instruction to seem 3352 * to succeed, but return an EXIT_REASON_INVALID_STATE. 3353 * To speed up the normal (success) code path, we should avoid checking 3354 * for misconfigurations which will anyway be caught by the processor 3355 * when using the merged vmcs02. 3356 */ 3357 if (interrupt_shadow & KVM_X86_SHADOW_INT_MOV_SS) 3358 return nested_vmx_failValid(vcpu, 3359 VMXERR_ENTRY_EVENTS_BLOCKED_BY_MOV_SS); 3360 3361 if (vmcs12->launch_state == launch) 3362 return nested_vmx_failValid(vcpu, 3363 launch ? VMXERR_VMLAUNCH_NONCLEAR_VMCS 3364 : VMXERR_VMRESUME_NONLAUNCHED_VMCS); 3365 3366 if (nested_vmx_check_controls(vcpu, vmcs12)) 3367 return nested_vmx_failValid(vcpu, VMXERR_ENTRY_INVALID_CONTROL_FIELD); 3368 3369 if (nested_vmx_check_host_state(vcpu, vmcs12)) 3370 return nested_vmx_failValid(vcpu, VMXERR_ENTRY_INVALID_HOST_STATE_FIELD); 3371 3372 /* 3373 * We're finally done with prerequisite checking, and can start with 3374 * the nested entry. 3375 */ 3376 vmx->nested.nested_run_pending = 1; 3377 status = nested_vmx_enter_non_root_mode(vcpu, true); 3378 if (unlikely(status != NVMX_VMENTRY_SUCCESS)) 3379 goto vmentry_failed; 3380 3381 /* Hide L1D cache contents from the nested guest. */ 3382 vmx->vcpu.arch.l1tf_flush_l1d = true; 3383 3384 /* 3385 * Must happen outside of nested_vmx_enter_non_root_mode() as it will 3386 * also be used as part of restoring nVMX state for 3387 * snapshot restore (migration). 3388 * 3389 * In this flow, it is assumed that vmcs12 cache was 3390 * trasferred as part of captured nVMX state and should 3391 * therefore not be read from guest memory (which may not 3392 * exist on destination host yet). 3393 */ 3394 nested_cache_shadow_vmcs12(vcpu, vmcs12); 3395 3396 /* 3397 * If we're entering a halted L2 vcpu and the L2 vcpu won't be 3398 * awakened by event injection or by an NMI-window VM-exit or 3399 * by an interrupt-window VM-exit, halt the vcpu. 3400 */ 3401 if ((vmcs12->guest_activity_state == GUEST_ACTIVITY_HLT) && 3402 !(vmcs12->vm_entry_intr_info_field & INTR_INFO_VALID_MASK) && 3403 !(vmcs12->cpu_based_vm_exec_control & CPU_BASED_NMI_WINDOW_EXITING) && 3404 !((vmcs12->cpu_based_vm_exec_control & CPU_BASED_INTR_WINDOW_EXITING) && 3405 (vmcs12->guest_rflags & X86_EFLAGS_IF))) { 3406 vmx->nested.nested_run_pending = 0; 3407 return kvm_vcpu_halt(vcpu); 3408 } 3409 return 1; 3410 3411 vmentry_failed: 3412 vmx->nested.nested_run_pending = 0; 3413 if (status == NVMX_VMENTRY_KVM_INTERNAL_ERROR) 3414 return 0; 3415 if (status == NVMX_VMENTRY_VMEXIT) 3416 return 1; 3417 WARN_ON_ONCE(status != NVMX_VMENTRY_VMFAIL); 3418 return nested_vmx_failValid(vcpu, VMXERR_ENTRY_INVALID_CONTROL_FIELD); 3419 } 3420 3421 /* 3422 * On a nested exit from L2 to L1, vmcs12.guest_cr0 might not be up-to-date 3423 * because L2 may have changed some cr0 bits directly (CR0_GUEST_HOST_MASK). 3424 * This function returns the new value we should put in vmcs12.guest_cr0. 3425 * It's not enough to just return the vmcs02 GUEST_CR0. Rather, 3426 * 1. Bits that neither L0 nor L1 trapped, were set directly by L2 and are now 3427 * available in vmcs02 GUEST_CR0. (Note: It's enough to check that L0 3428 * didn't trap the bit, because if L1 did, so would L0). 3429 * 2. Bits that L1 asked to trap (and therefore L0 also did) could not have 3430 * been modified by L2, and L1 knows it. So just leave the old value of 3431 * the bit from vmcs12.guest_cr0. Note that the bit from vmcs02 GUEST_CR0 3432 * isn't relevant, because if L0 traps this bit it can set it to anything. 3433 * 3. Bits that L1 didn't trap, but L0 did. L1 believes the guest could have 3434 * changed these bits, and therefore they need to be updated, but L0 3435 * didn't necessarily allow them to be changed in GUEST_CR0 - and rather 3436 * put them in vmcs02 CR0_READ_SHADOW. So take these bits from there. 3437 */ 3438 static inline unsigned long 3439 vmcs12_guest_cr0(struct kvm_vcpu *vcpu, struct vmcs12 *vmcs12) 3440 { 3441 return 3442 /*1*/ (vmcs_readl(GUEST_CR0) & vcpu->arch.cr0_guest_owned_bits) | 3443 /*2*/ (vmcs12->guest_cr0 & vmcs12->cr0_guest_host_mask) | 3444 /*3*/ (vmcs_readl(CR0_READ_SHADOW) & ~(vmcs12->cr0_guest_host_mask | 3445 vcpu->arch.cr0_guest_owned_bits)); 3446 } 3447 3448 static inline unsigned long 3449 vmcs12_guest_cr4(struct kvm_vcpu *vcpu, struct vmcs12 *vmcs12) 3450 { 3451 return 3452 /*1*/ (vmcs_readl(GUEST_CR4) & vcpu->arch.cr4_guest_owned_bits) | 3453 /*2*/ (vmcs12->guest_cr4 & vmcs12->cr4_guest_host_mask) | 3454 /*3*/ (vmcs_readl(CR4_READ_SHADOW) & ~(vmcs12->cr4_guest_host_mask | 3455 vcpu->arch.cr4_guest_owned_bits)); 3456 } 3457 3458 static void vmcs12_save_pending_event(struct kvm_vcpu *vcpu, 3459 struct vmcs12 *vmcs12) 3460 { 3461 u32 idt_vectoring; 3462 unsigned int nr; 3463 3464 if (vcpu->arch.exception.injected) { 3465 nr = vcpu->arch.exception.nr; 3466 idt_vectoring = nr | VECTORING_INFO_VALID_MASK; 3467 3468 if (kvm_exception_is_soft(nr)) { 3469 vmcs12->vm_exit_instruction_len = 3470 vcpu->arch.event_exit_inst_len; 3471 idt_vectoring |= INTR_TYPE_SOFT_EXCEPTION; 3472 } else 3473 idt_vectoring |= INTR_TYPE_HARD_EXCEPTION; 3474 3475 if (vcpu->arch.exception.has_error_code) { 3476 idt_vectoring |= VECTORING_INFO_DELIVER_CODE_MASK; 3477 vmcs12->idt_vectoring_error_code = 3478 vcpu->arch.exception.error_code; 3479 } 3480 3481 vmcs12->idt_vectoring_info_field = idt_vectoring; 3482 } else if (vcpu->arch.nmi_injected) { 3483 vmcs12->idt_vectoring_info_field = 3484 INTR_TYPE_NMI_INTR | INTR_INFO_VALID_MASK | NMI_VECTOR; 3485 } else if (vcpu->arch.interrupt.injected) { 3486 nr = vcpu->arch.interrupt.nr; 3487 idt_vectoring = nr | VECTORING_INFO_VALID_MASK; 3488 3489 if (vcpu->arch.interrupt.soft) { 3490 idt_vectoring |= INTR_TYPE_SOFT_INTR; 3491 vmcs12->vm_entry_instruction_len = 3492 vcpu->arch.event_exit_inst_len; 3493 } else 3494 idt_vectoring |= INTR_TYPE_EXT_INTR; 3495 3496 vmcs12->idt_vectoring_info_field = idt_vectoring; 3497 } 3498 } 3499 3500 3501 static void nested_mark_vmcs12_pages_dirty(struct kvm_vcpu *vcpu) 3502 { 3503 struct vmcs12 *vmcs12 = get_vmcs12(vcpu); 3504 gfn_t gfn; 3505 3506 /* 3507 * Don't need to mark the APIC access page dirty; it is never 3508 * written to by the CPU during APIC virtualization. 3509 */ 3510 3511 if (nested_cpu_has(vmcs12, CPU_BASED_TPR_SHADOW)) { 3512 gfn = vmcs12->virtual_apic_page_addr >> PAGE_SHIFT; 3513 kvm_vcpu_mark_page_dirty(vcpu, gfn); 3514 } 3515 3516 if (nested_cpu_has_posted_intr(vmcs12)) { 3517 gfn = vmcs12->posted_intr_desc_addr >> PAGE_SHIFT; 3518 kvm_vcpu_mark_page_dirty(vcpu, gfn); 3519 } 3520 } 3521 3522 static void vmx_complete_nested_posted_interrupt(struct kvm_vcpu *vcpu) 3523 { 3524 struct vcpu_vmx *vmx = to_vmx(vcpu); 3525 int max_irr; 3526 void *vapic_page; 3527 u16 status; 3528 3529 if (!vmx->nested.pi_desc || !vmx->nested.pi_pending) 3530 return; 3531 3532 vmx->nested.pi_pending = false; 3533 if (!pi_test_and_clear_on(vmx->nested.pi_desc)) 3534 return; 3535 3536 max_irr = find_last_bit((unsigned long *)vmx->nested.pi_desc->pir, 256); 3537 if (max_irr != 256) { 3538 vapic_page = vmx->nested.virtual_apic_map.hva; 3539 if (!vapic_page) 3540 return; 3541 3542 __kvm_apic_update_irr(vmx->nested.pi_desc->pir, 3543 vapic_page, &max_irr); 3544 status = vmcs_read16(GUEST_INTR_STATUS); 3545 if ((u8)max_irr > ((u8)status & 0xff)) { 3546 status &= ~0xff; 3547 status |= (u8)max_irr; 3548 vmcs_write16(GUEST_INTR_STATUS, status); 3549 } 3550 } 3551 3552 nested_mark_vmcs12_pages_dirty(vcpu); 3553 } 3554 3555 static void nested_vmx_inject_exception_vmexit(struct kvm_vcpu *vcpu, 3556 unsigned long exit_qual) 3557 { 3558 struct vmcs12 *vmcs12 = get_vmcs12(vcpu); 3559 unsigned int nr = vcpu->arch.exception.nr; 3560 u32 intr_info = nr | INTR_INFO_VALID_MASK; 3561 3562 if (vcpu->arch.exception.has_error_code) { 3563 vmcs12->vm_exit_intr_error_code = vcpu->arch.exception.error_code; 3564 intr_info |= INTR_INFO_DELIVER_CODE_MASK; 3565 } 3566 3567 if (kvm_exception_is_soft(nr)) 3568 intr_info |= INTR_TYPE_SOFT_EXCEPTION; 3569 else 3570 intr_info |= INTR_TYPE_HARD_EXCEPTION; 3571 3572 if (!(vmcs12->idt_vectoring_info_field & VECTORING_INFO_VALID_MASK) && 3573 vmx_get_nmi_mask(vcpu)) 3574 intr_info |= INTR_INFO_UNBLOCK_NMI; 3575 3576 nested_vmx_vmexit(vcpu, EXIT_REASON_EXCEPTION_NMI, intr_info, exit_qual); 3577 } 3578 3579 /* 3580 * Returns true if a debug trap is pending delivery. 3581 * 3582 * In KVM, debug traps bear an exception payload. As such, the class of a #DB 3583 * exception may be inferred from the presence of an exception payload. 3584 */ 3585 static inline bool vmx_pending_dbg_trap(struct kvm_vcpu *vcpu) 3586 { 3587 return vcpu->arch.exception.pending && 3588 vcpu->arch.exception.nr == DB_VECTOR && 3589 vcpu->arch.exception.payload; 3590 } 3591 3592 /* 3593 * Certain VM-exits set the 'pending debug exceptions' field to indicate a 3594 * recognized #DB (data or single-step) that has yet to be delivered. Since KVM 3595 * represents these debug traps with a payload that is said to be compatible 3596 * with the 'pending debug exceptions' field, write the payload to the VMCS 3597 * field if a VM-exit is delivered before the debug trap. 3598 */ 3599 static void nested_vmx_update_pending_dbg(struct kvm_vcpu *vcpu) 3600 { 3601 if (vmx_pending_dbg_trap(vcpu)) 3602 vmcs_writel(GUEST_PENDING_DBG_EXCEPTIONS, 3603 vcpu->arch.exception.payload); 3604 } 3605 3606 static int vmx_check_nested_events(struct kvm_vcpu *vcpu, bool external_intr) 3607 { 3608 struct vcpu_vmx *vmx = to_vmx(vcpu); 3609 unsigned long exit_qual; 3610 bool block_nested_events = 3611 vmx->nested.nested_run_pending || kvm_event_needs_reinjection(vcpu); 3612 struct kvm_lapic *apic = vcpu->arch.apic; 3613 3614 if (lapic_in_kernel(vcpu) && 3615 test_bit(KVM_APIC_INIT, &apic->pending_events)) { 3616 if (block_nested_events) 3617 return -EBUSY; 3618 nested_vmx_update_pending_dbg(vcpu); 3619 clear_bit(KVM_APIC_INIT, &apic->pending_events); 3620 nested_vmx_vmexit(vcpu, EXIT_REASON_INIT_SIGNAL, 0, 0); 3621 return 0; 3622 } 3623 3624 if (vcpu->arch.exception.pending && 3625 nested_vmx_check_exception(vcpu, &exit_qual)) { 3626 if (block_nested_events) 3627 return -EBUSY; 3628 nested_vmx_inject_exception_vmexit(vcpu, exit_qual); 3629 return 0; 3630 } 3631 3632 if (nested_cpu_has_preemption_timer(get_vmcs12(vcpu)) && 3633 vmx->nested.preemption_timer_expired) { 3634 if (block_nested_events) 3635 return -EBUSY; 3636 nested_vmx_vmexit(vcpu, EXIT_REASON_PREEMPTION_TIMER, 0, 0); 3637 return 0; 3638 } 3639 3640 if (vcpu->arch.nmi_pending && nested_exit_on_nmi(vcpu)) { 3641 if (block_nested_events) 3642 return -EBUSY; 3643 nested_vmx_vmexit(vcpu, EXIT_REASON_EXCEPTION_NMI, 3644 NMI_VECTOR | INTR_TYPE_NMI_INTR | 3645 INTR_INFO_VALID_MASK, 0); 3646 /* 3647 * The NMI-triggered VM exit counts as injection: 3648 * clear this one and block further NMIs. 3649 */ 3650 vcpu->arch.nmi_pending = 0; 3651 vmx_set_nmi_mask(vcpu, true); 3652 return 0; 3653 } 3654 3655 if ((kvm_cpu_has_interrupt(vcpu) || external_intr) && 3656 nested_exit_on_intr(vcpu)) { 3657 if (block_nested_events) 3658 return -EBUSY; 3659 nested_vmx_vmexit(vcpu, EXIT_REASON_EXTERNAL_INTERRUPT, 0, 0); 3660 return 0; 3661 } 3662 3663 vmx_complete_nested_posted_interrupt(vcpu); 3664 return 0; 3665 } 3666 3667 static u32 vmx_get_preemption_timer_value(struct kvm_vcpu *vcpu) 3668 { 3669 ktime_t remaining = 3670 hrtimer_get_remaining(&to_vmx(vcpu)->nested.preemption_timer); 3671 u64 value; 3672 3673 if (ktime_to_ns(remaining) <= 0) 3674 return 0; 3675 3676 value = ktime_to_ns(remaining) * vcpu->arch.virtual_tsc_khz; 3677 do_div(value, 1000000); 3678 return value >> VMX_MISC_EMULATED_PREEMPTION_TIMER_RATE; 3679 } 3680 3681 static bool is_vmcs12_ext_field(unsigned long field) 3682 { 3683 switch (field) { 3684 case GUEST_ES_SELECTOR: 3685 case GUEST_CS_SELECTOR: 3686 case GUEST_SS_SELECTOR: 3687 case GUEST_DS_SELECTOR: 3688 case GUEST_FS_SELECTOR: 3689 case GUEST_GS_SELECTOR: 3690 case GUEST_LDTR_SELECTOR: 3691 case GUEST_TR_SELECTOR: 3692 case GUEST_ES_LIMIT: 3693 case GUEST_CS_LIMIT: 3694 case GUEST_SS_LIMIT: 3695 case GUEST_DS_LIMIT: 3696 case GUEST_FS_LIMIT: 3697 case GUEST_GS_LIMIT: 3698 case GUEST_LDTR_LIMIT: 3699 case GUEST_TR_LIMIT: 3700 case GUEST_GDTR_LIMIT: 3701 case GUEST_IDTR_LIMIT: 3702 case GUEST_ES_AR_BYTES: 3703 case GUEST_DS_AR_BYTES: 3704 case GUEST_FS_AR_BYTES: 3705 case GUEST_GS_AR_BYTES: 3706 case GUEST_LDTR_AR_BYTES: 3707 case GUEST_TR_AR_BYTES: 3708 case GUEST_ES_BASE: 3709 case GUEST_CS_BASE: 3710 case GUEST_SS_BASE: 3711 case GUEST_DS_BASE: 3712 case GUEST_FS_BASE: 3713 case GUEST_GS_BASE: 3714 case GUEST_LDTR_BASE: 3715 case GUEST_TR_BASE: 3716 case GUEST_GDTR_BASE: 3717 case GUEST_IDTR_BASE: 3718 case GUEST_PENDING_DBG_EXCEPTIONS: 3719 case GUEST_BNDCFGS: 3720 return true; 3721 default: 3722 break; 3723 } 3724 3725 return false; 3726 } 3727 3728 static void sync_vmcs02_to_vmcs12_rare(struct kvm_vcpu *vcpu, 3729 struct vmcs12 *vmcs12) 3730 { 3731 struct vcpu_vmx *vmx = to_vmx(vcpu); 3732 3733 vmcs12->guest_es_selector = vmcs_read16(GUEST_ES_SELECTOR); 3734 vmcs12->guest_cs_selector = vmcs_read16(GUEST_CS_SELECTOR); 3735 vmcs12->guest_ss_selector = vmcs_read16(GUEST_SS_SELECTOR); 3736 vmcs12->guest_ds_selector = vmcs_read16(GUEST_DS_SELECTOR); 3737 vmcs12->guest_fs_selector = vmcs_read16(GUEST_FS_SELECTOR); 3738 vmcs12->guest_gs_selector = vmcs_read16(GUEST_GS_SELECTOR); 3739 vmcs12->guest_ldtr_selector = vmcs_read16(GUEST_LDTR_SELECTOR); 3740 vmcs12->guest_tr_selector = vmcs_read16(GUEST_TR_SELECTOR); 3741 vmcs12->guest_es_limit = vmcs_read32(GUEST_ES_LIMIT); 3742 vmcs12->guest_cs_limit = vmcs_read32(GUEST_CS_LIMIT); 3743 vmcs12->guest_ss_limit = vmcs_read32(GUEST_SS_LIMIT); 3744 vmcs12->guest_ds_limit = vmcs_read32(GUEST_DS_LIMIT); 3745 vmcs12->guest_fs_limit = vmcs_read32(GUEST_FS_LIMIT); 3746 vmcs12->guest_gs_limit = vmcs_read32(GUEST_GS_LIMIT); 3747 vmcs12->guest_ldtr_limit = vmcs_read32(GUEST_LDTR_LIMIT); 3748 vmcs12->guest_tr_limit = vmcs_read32(GUEST_TR_LIMIT); 3749 vmcs12->guest_gdtr_limit = vmcs_read32(GUEST_GDTR_LIMIT); 3750 vmcs12->guest_idtr_limit = vmcs_read32(GUEST_IDTR_LIMIT); 3751 vmcs12->guest_es_ar_bytes = vmcs_read32(GUEST_ES_AR_BYTES); 3752 vmcs12->guest_ds_ar_bytes = vmcs_read32(GUEST_DS_AR_BYTES); 3753 vmcs12->guest_fs_ar_bytes = vmcs_read32(GUEST_FS_AR_BYTES); 3754 vmcs12->guest_gs_ar_bytes = vmcs_read32(GUEST_GS_AR_BYTES); 3755 vmcs12->guest_ldtr_ar_bytes = vmcs_read32(GUEST_LDTR_AR_BYTES); 3756 vmcs12->guest_tr_ar_bytes = vmcs_read32(GUEST_TR_AR_BYTES); 3757 vmcs12->guest_es_base = vmcs_readl(GUEST_ES_BASE); 3758 vmcs12->guest_cs_base = vmcs_readl(GUEST_CS_BASE); 3759 vmcs12->guest_ss_base = vmcs_readl(GUEST_SS_BASE); 3760 vmcs12->guest_ds_base = vmcs_readl(GUEST_DS_BASE); 3761 vmcs12->guest_fs_base = vmcs_readl(GUEST_FS_BASE); 3762 vmcs12->guest_gs_base = vmcs_readl(GUEST_GS_BASE); 3763 vmcs12->guest_ldtr_base = vmcs_readl(GUEST_LDTR_BASE); 3764 vmcs12->guest_tr_base = vmcs_readl(GUEST_TR_BASE); 3765 vmcs12->guest_gdtr_base = vmcs_readl(GUEST_GDTR_BASE); 3766 vmcs12->guest_idtr_base = vmcs_readl(GUEST_IDTR_BASE); 3767 vmcs12->guest_pending_dbg_exceptions = 3768 vmcs_readl(GUEST_PENDING_DBG_EXCEPTIONS); 3769 if (kvm_mpx_supported()) 3770 vmcs12->guest_bndcfgs = vmcs_read64(GUEST_BNDCFGS); 3771 3772 vmx->nested.need_sync_vmcs02_to_vmcs12_rare = false; 3773 } 3774 3775 static void copy_vmcs02_to_vmcs12_rare(struct kvm_vcpu *vcpu, 3776 struct vmcs12 *vmcs12) 3777 { 3778 struct vcpu_vmx *vmx = to_vmx(vcpu); 3779 int cpu; 3780 3781 if (!vmx->nested.need_sync_vmcs02_to_vmcs12_rare) 3782 return; 3783 3784 3785 WARN_ON_ONCE(vmx->loaded_vmcs != &vmx->vmcs01); 3786 3787 cpu = get_cpu(); 3788 vmx->loaded_vmcs = &vmx->nested.vmcs02; 3789 vmx_vcpu_load(&vmx->vcpu, cpu); 3790 3791 sync_vmcs02_to_vmcs12_rare(vcpu, vmcs12); 3792 3793 vmx->loaded_vmcs = &vmx->vmcs01; 3794 vmx_vcpu_load(&vmx->vcpu, cpu); 3795 put_cpu(); 3796 } 3797 3798 /* 3799 * Update the guest state fields of vmcs12 to reflect changes that 3800 * occurred while L2 was running. (The "IA-32e mode guest" bit of the 3801 * VM-entry controls is also updated, since this is really a guest 3802 * state bit.) 3803 */ 3804 static void sync_vmcs02_to_vmcs12(struct kvm_vcpu *vcpu, struct vmcs12 *vmcs12) 3805 { 3806 struct vcpu_vmx *vmx = to_vmx(vcpu); 3807 3808 if (vmx->nested.hv_evmcs) 3809 sync_vmcs02_to_vmcs12_rare(vcpu, vmcs12); 3810 3811 vmx->nested.need_sync_vmcs02_to_vmcs12_rare = !vmx->nested.hv_evmcs; 3812 3813 vmcs12->guest_cr0 = vmcs12_guest_cr0(vcpu, vmcs12); 3814 vmcs12->guest_cr4 = vmcs12_guest_cr4(vcpu, vmcs12); 3815 3816 vmcs12->guest_rsp = kvm_rsp_read(vcpu); 3817 vmcs12->guest_rip = kvm_rip_read(vcpu); 3818 vmcs12->guest_rflags = vmcs_readl(GUEST_RFLAGS); 3819 3820 vmcs12->guest_cs_ar_bytes = vmcs_read32(GUEST_CS_AR_BYTES); 3821 vmcs12->guest_ss_ar_bytes = vmcs_read32(GUEST_SS_AR_BYTES); 3822 3823 vmcs12->guest_sysenter_cs = vmcs_read32(GUEST_SYSENTER_CS); 3824 vmcs12->guest_sysenter_esp = vmcs_readl(GUEST_SYSENTER_ESP); 3825 vmcs12->guest_sysenter_eip = vmcs_readl(GUEST_SYSENTER_EIP); 3826 3827 vmcs12->guest_interruptibility_info = 3828 vmcs_read32(GUEST_INTERRUPTIBILITY_INFO); 3829 3830 if (vcpu->arch.mp_state == KVM_MP_STATE_HALTED) 3831 vmcs12->guest_activity_state = GUEST_ACTIVITY_HLT; 3832 else 3833 vmcs12->guest_activity_state = GUEST_ACTIVITY_ACTIVE; 3834 3835 if (nested_cpu_has_preemption_timer(vmcs12) && 3836 vmcs12->vm_exit_controls & VM_EXIT_SAVE_VMX_PREEMPTION_TIMER) 3837 vmcs12->vmx_preemption_timer_value = 3838 vmx_get_preemption_timer_value(vcpu); 3839 3840 /* 3841 * In some cases (usually, nested EPT), L2 is allowed to change its 3842 * own CR3 without exiting. If it has changed it, we must keep it. 3843 * Of course, if L0 is using shadow page tables, GUEST_CR3 was defined 3844 * by L0, not L1 or L2, so we mustn't unconditionally copy it to vmcs12. 3845 * 3846 * Additionally, restore L2's PDPTR to vmcs12. 3847 */ 3848 if (enable_ept) { 3849 vmcs12->guest_cr3 = vmcs_readl(GUEST_CR3); 3850 if (nested_cpu_has_ept(vmcs12) && is_pae_paging(vcpu)) { 3851 vmcs12->guest_pdptr0 = vmcs_read64(GUEST_PDPTR0); 3852 vmcs12->guest_pdptr1 = vmcs_read64(GUEST_PDPTR1); 3853 vmcs12->guest_pdptr2 = vmcs_read64(GUEST_PDPTR2); 3854 vmcs12->guest_pdptr3 = vmcs_read64(GUEST_PDPTR3); 3855 } 3856 } 3857 3858 vmcs12->guest_linear_address = vmcs_readl(GUEST_LINEAR_ADDRESS); 3859 3860 if (nested_cpu_has_vid(vmcs12)) 3861 vmcs12->guest_intr_status = vmcs_read16(GUEST_INTR_STATUS); 3862 3863 vmcs12->vm_entry_controls = 3864 (vmcs12->vm_entry_controls & ~VM_ENTRY_IA32E_MODE) | 3865 (vm_entry_controls_get(to_vmx(vcpu)) & VM_ENTRY_IA32E_MODE); 3866 3867 if (vmcs12->vm_exit_controls & VM_EXIT_SAVE_DEBUG_CONTROLS) 3868 kvm_get_dr(vcpu, 7, (unsigned long *)&vmcs12->guest_dr7); 3869 3870 if (vmcs12->vm_exit_controls & VM_EXIT_SAVE_IA32_EFER) 3871 vmcs12->guest_ia32_efer = vcpu->arch.efer; 3872 } 3873 3874 /* 3875 * prepare_vmcs12 is part of what we need to do when the nested L2 guest exits 3876 * and we want to prepare to run its L1 parent. L1 keeps a vmcs for L2 (vmcs12), 3877 * and this function updates it to reflect the changes to the guest state while 3878 * L2 was running (and perhaps made some exits which were handled directly by L0 3879 * without going back to L1), and to reflect the exit reason. 3880 * Note that we do not have to copy here all VMCS fields, just those that 3881 * could have changed by the L2 guest or the exit - i.e., the guest-state and 3882 * exit-information fields only. Other fields are modified by L1 with VMWRITE, 3883 * which already writes to vmcs12 directly. 3884 */ 3885 static void prepare_vmcs12(struct kvm_vcpu *vcpu, struct vmcs12 *vmcs12, 3886 u32 exit_reason, u32 exit_intr_info, 3887 unsigned long exit_qualification) 3888 { 3889 /* update exit information fields: */ 3890 vmcs12->vm_exit_reason = exit_reason; 3891 vmcs12->exit_qualification = exit_qualification; 3892 vmcs12->vm_exit_intr_info = exit_intr_info; 3893 3894 vmcs12->idt_vectoring_info_field = 0; 3895 vmcs12->vm_exit_instruction_len = vmcs_read32(VM_EXIT_INSTRUCTION_LEN); 3896 vmcs12->vmx_instruction_info = vmcs_read32(VMX_INSTRUCTION_INFO); 3897 3898 if (!(vmcs12->vm_exit_reason & VMX_EXIT_REASONS_FAILED_VMENTRY)) { 3899 vmcs12->launch_state = 1; 3900 3901 /* vm_entry_intr_info_field is cleared on exit. Emulate this 3902 * instead of reading the real value. */ 3903 vmcs12->vm_entry_intr_info_field &= ~INTR_INFO_VALID_MASK; 3904 3905 /* 3906 * Transfer the event that L0 or L1 may wanted to inject into 3907 * L2 to IDT_VECTORING_INFO_FIELD. 3908 */ 3909 vmcs12_save_pending_event(vcpu, vmcs12); 3910 3911 /* 3912 * According to spec, there's no need to store the guest's 3913 * MSRs if the exit is due to a VM-entry failure that occurs 3914 * during or after loading the guest state. Since this exit 3915 * does not fall in that category, we need to save the MSRs. 3916 */ 3917 if (nested_vmx_store_msr(vcpu, 3918 vmcs12->vm_exit_msr_store_addr, 3919 vmcs12->vm_exit_msr_store_count)) 3920 nested_vmx_abort(vcpu, 3921 VMX_ABORT_SAVE_GUEST_MSR_FAIL); 3922 } 3923 3924 /* 3925 * Drop what we picked up for L2 via vmx_complete_interrupts. It is 3926 * preserved above and would only end up incorrectly in L1. 3927 */ 3928 vcpu->arch.nmi_injected = false; 3929 kvm_clear_exception_queue(vcpu); 3930 kvm_clear_interrupt_queue(vcpu); 3931 } 3932 3933 /* 3934 * A part of what we need to when the nested L2 guest exits and we want to 3935 * run its L1 parent, is to reset L1's guest state to the host state specified 3936 * in vmcs12. 3937 * This function is to be called not only on normal nested exit, but also on 3938 * a nested entry failure, as explained in Intel's spec, 3B.23.7 ("VM-Entry 3939 * Failures During or After Loading Guest State"). 3940 * This function should be called when the active VMCS is L1's (vmcs01). 3941 */ 3942 static void load_vmcs12_host_state(struct kvm_vcpu *vcpu, 3943 struct vmcs12 *vmcs12) 3944 { 3945 struct kvm_segment seg; 3946 u32 entry_failure_code; 3947 3948 if (vmcs12->vm_exit_controls & VM_EXIT_LOAD_IA32_EFER) 3949 vcpu->arch.efer = vmcs12->host_ia32_efer; 3950 else if (vmcs12->vm_exit_controls & VM_EXIT_HOST_ADDR_SPACE_SIZE) 3951 vcpu->arch.efer |= (EFER_LMA | EFER_LME); 3952 else 3953 vcpu->arch.efer &= ~(EFER_LMA | EFER_LME); 3954 vmx_set_efer(vcpu, vcpu->arch.efer); 3955 3956 kvm_rsp_write(vcpu, vmcs12->host_rsp); 3957 kvm_rip_write(vcpu, vmcs12->host_rip); 3958 vmx_set_rflags(vcpu, X86_EFLAGS_FIXED); 3959 vmx_set_interrupt_shadow(vcpu, 0); 3960 3961 /* 3962 * Note that calling vmx_set_cr0 is important, even if cr0 hasn't 3963 * actually changed, because vmx_set_cr0 refers to efer set above. 3964 * 3965 * CR0_GUEST_HOST_MASK is already set in the original vmcs01 3966 * (KVM doesn't change it); 3967 */ 3968 vcpu->arch.cr0_guest_owned_bits = X86_CR0_TS; 3969 vmx_set_cr0(vcpu, vmcs12->host_cr0); 3970 3971 /* Same as above - no reason to call set_cr4_guest_host_mask(). */ 3972 vcpu->arch.cr4_guest_owned_bits = ~vmcs_readl(CR4_GUEST_HOST_MASK); 3973 vmx_set_cr4(vcpu, vmcs12->host_cr4); 3974 3975 nested_ept_uninit_mmu_context(vcpu); 3976 3977 /* 3978 * Only PDPTE load can fail as the value of cr3 was checked on entry and 3979 * couldn't have changed. 3980 */ 3981 if (nested_vmx_load_cr3(vcpu, vmcs12->host_cr3, false, &entry_failure_code)) 3982 nested_vmx_abort(vcpu, VMX_ABORT_LOAD_HOST_PDPTE_FAIL); 3983 3984 if (!enable_ept) 3985 vcpu->arch.walk_mmu->inject_page_fault = kvm_inject_page_fault; 3986 3987 /* 3988 * If vmcs01 doesn't use VPID, CPU flushes TLB on every 3989 * VMEntry/VMExit. Thus, no need to flush TLB. 3990 * 3991 * If vmcs12 doesn't use VPID, L1 expects TLB to be 3992 * flushed on every VMEntry/VMExit. 3993 * 3994 * Otherwise, we can preserve TLB entries as long as we are 3995 * able to tag L1 TLB entries differently than L2 TLB entries. 3996 * 3997 * If vmcs12 uses EPT, we need to execute this flush on EPTP01 3998 * and therefore we request the TLB flush to happen only after VMCS EPTP 3999 * has been set by KVM_REQ_LOAD_CR3. 4000 */ 4001 if (enable_vpid && 4002 (!nested_cpu_has_vpid(vmcs12) || !nested_has_guest_tlb_tag(vcpu))) { 4003 kvm_make_request(KVM_REQ_TLB_FLUSH, vcpu); 4004 } 4005 4006 vmcs_write32(GUEST_SYSENTER_CS, vmcs12->host_ia32_sysenter_cs); 4007 vmcs_writel(GUEST_SYSENTER_ESP, vmcs12->host_ia32_sysenter_esp); 4008 vmcs_writel(GUEST_SYSENTER_EIP, vmcs12->host_ia32_sysenter_eip); 4009 vmcs_writel(GUEST_IDTR_BASE, vmcs12->host_idtr_base); 4010 vmcs_writel(GUEST_GDTR_BASE, vmcs12->host_gdtr_base); 4011 vmcs_write32(GUEST_IDTR_LIMIT, 0xFFFF); 4012 vmcs_write32(GUEST_GDTR_LIMIT, 0xFFFF); 4013 4014 /* If not VM_EXIT_CLEAR_BNDCFGS, the L2 value propagates to L1. */ 4015 if (vmcs12->vm_exit_controls & VM_EXIT_CLEAR_BNDCFGS) 4016 vmcs_write64(GUEST_BNDCFGS, 0); 4017 4018 if (vmcs12->vm_exit_controls & VM_EXIT_LOAD_IA32_PAT) { 4019 vmcs_write64(GUEST_IA32_PAT, vmcs12->host_ia32_pat); 4020 vcpu->arch.pat = vmcs12->host_ia32_pat; 4021 } 4022 if (vmcs12->vm_exit_controls & VM_EXIT_LOAD_IA32_PERF_GLOBAL_CTRL) 4023 WARN_ON_ONCE(kvm_set_msr(vcpu, MSR_CORE_PERF_GLOBAL_CTRL, 4024 vmcs12->host_ia32_perf_global_ctrl)); 4025 4026 /* Set L1 segment info according to Intel SDM 4027 27.5.2 Loading Host Segment and Descriptor-Table Registers */ 4028 seg = (struct kvm_segment) { 4029 .base = 0, 4030 .limit = 0xFFFFFFFF, 4031 .selector = vmcs12->host_cs_selector, 4032 .type = 11, 4033 .present = 1, 4034 .s = 1, 4035 .g = 1 4036 }; 4037 if (vmcs12->vm_exit_controls & VM_EXIT_HOST_ADDR_SPACE_SIZE) 4038 seg.l = 1; 4039 else 4040 seg.db = 1; 4041 vmx_set_segment(vcpu, &seg, VCPU_SREG_CS); 4042 seg = (struct kvm_segment) { 4043 .base = 0, 4044 .limit = 0xFFFFFFFF, 4045 .type = 3, 4046 .present = 1, 4047 .s = 1, 4048 .db = 1, 4049 .g = 1 4050 }; 4051 seg.selector = vmcs12->host_ds_selector; 4052 vmx_set_segment(vcpu, &seg, VCPU_SREG_DS); 4053 seg.selector = vmcs12->host_es_selector; 4054 vmx_set_segment(vcpu, &seg, VCPU_SREG_ES); 4055 seg.selector = vmcs12->host_ss_selector; 4056 vmx_set_segment(vcpu, &seg, VCPU_SREG_SS); 4057 seg.selector = vmcs12->host_fs_selector; 4058 seg.base = vmcs12->host_fs_base; 4059 vmx_set_segment(vcpu, &seg, VCPU_SREG_FS); 4060 seg.selector = vmcs12->host_gs_selector; 4061 seg.base = vmcs12->host_gs_base; 4062 vmx_set_segment(vcpu, &seg, VCPU_SREG_GS); 4063 seg = (struct kvm_segment) { 4064 .base = vmcs12->host_tr_base, 4065 .limit = 0x67, 4066 .selector = vmcs12->host_tr_selector, 4067 .type = 11, 4068 .present = 1 4069 }; 4070 vmx_set_segment(vcpu, &seg, VCPU_SREG_TR); 4071 4072 kvm_set_dr(vcpu, 7, 0x400); 4073 vmcs_write64(GUEST_IA32_DEBUGCTL, 0); 4074 4075 if (cpu_has_vmx_msr_bitmap()) 4076 vmx_update_msr_bitmap(vcpu); 4077 4078 if (nested_vmx_load_msr(vcpu, vmcs12->vm_exit_msr_load_addr, 4079 vmcs12->vm_exit_msr_load_count)) 4080 nested_vmx_abort(vcpu, VMX_ABORT_LOAD_HOST_MSR_FAIL); 4081 } 4082 4083 static inline u64 nested_vmx_get_vmcs01_guest_efer(struct vcpu_vmx *vmx) 4084 { 4085 struct shared_msr_entry *efer_msr; 4086 unsigned int i; 4087 4088 if (vm_entry_controls_get(vmx) & VM_ENTRY_LOAD_IA32_EFER) 4089 return vmcs_read64(GUEST_IA32_EFER); 4090 4091 if (cpu_has_load_ia32_efer()) 4092 return host_efer; 4093 4094 for (i = 0; i < vmx->msr_autoload.guest.nr; ++i) { 4095 if (vmx->msr_autoload.guest.val[i].index == MSR_EFER) 4096 return vmx->msr_autoload.guest.val[i].value; 4097 } 4098 4099 efer_msr = find_msr_entry(vmx, MSR_EFER); 4100 if (efer_msr) 4101 return efer_msr->data; 4102 4103 return host_efer; 4104 } 4105 4106 static void nested_vmx_restore_host_state(struct kvm_vcpu *vcpu) 4107 { 4108 struct vmcs12 *vmcs12 = get_vmcs12(vcpu); 4109 struct vcpu_vmx *vmx = to_vmx(vcpu); 4110 struct vmx_msr_entry g, h; 4111 gpa_t gpa; 4112 u32 i, j; 4113 4114 vcpu->arch.pat = vmcs_read64(GUEST_IA32_PAT); 4115 4116 if (vmcs12->vm_entry_controls & VM_ENTRY_LOAD_DEBUG_CONTROLS) { 4117 /* 4118 * L1's host DR7 is lost if KVM_GUESTDBG_USE_HW_BP is set 4119 * as vmcs01.GUEST_DR7 contains a userspace defined value 4120 * and vcpu->arch.dr7 is not squirreled away before the 4121 * nested VMENTER (not worth adding a variable in nested_vmx). 4122 */ 4123 if (vcpu->guest_debug & KVM_GUESTDBG_USE_HW_BP) 4124 kvm_set_dr(vcpu, 7, DR7_FIXED_1); 4125 else 4126 WARN_ON(kvm_set_dr(vcpu, 7, vmcs_readl(GUEST_DR7))); 4127 } 4128 4129 /* 4130 * Note that calling vmx_set_{efer,cr0,cr4} is important as they 4131 * handle a variety of side effects to KVM's software model. 4132 */ 4133 vmx_set_efer(vcpu, nested_vmx_get_vmcs01_guest_efer(vmx)); 4134 4135 vcpu->arch.cr0_guest_owned_bits = X86_CR0_TS; 4136 vmx_set_cr0(vcpu, vmcs_readl(CR0_READ_SHADOW)); 4137 4138 vcpu->arch.cr4_guest_owned_bits = ~vmcs_readl(CR4_GUEST_HOST_MASK); 4139 vmx_set_cr4(vcpu, vmcs_readl(CR4_READ_SHADOW)); 4140 4141 nested_ept_uninit_mmu_context(vcpu); 4142 vcpu->arch.cr3 = vmcs_readl(GUEST_CR3); 4143 kvm_register_mark_available(vcpu, VCPU_EXREG_CR3); 4144 4145 /* 4146 * Use ept_save_pdptrs(vcpu) to load the MMU's cached PDPTRs 4147 * from vmcs01 (if necessary). The PDPTRs are not loaded on 4148 * VMFail, like everything else we just need to ensure our 4149 * software model is up-to-date. 4150 */ 4151 if (enable_ept) 4152 ept_save_pdptrs(vcpu); 4153 4154 kvm_mmu_reset_context(vcpu); 4155 4156 if (cpu_has_vmx_msr_bitmap()) 4157 vmx_update_msr_bitmap(vcpu); 4158 4159 /* 4160 * This nasty bit of open coding is a compromise between blindly 4161 * loading L1's MSRs using the exit load lists (incorrect emulation 4162 * of VMFail), leaving the nested VM's MSRs in the software model 4163 * (incorrect behavior) and snapshotting the modified MSRs (too 4164 * expensive since the lists are unbound by hardware). For each 4165 * MSR that was (prematurely) loaded from the nested VMEntry load 4166 * list, reload it from the exit load list if it exists and differs 4167 * from the guest value. The intent is to stuff host state as 4168 * silently as possible, not to fully process the exit load list. 4169 */ 4170 for (i = 0; i < vmcs12->vm_entry_msr_load_count; i++) { 4171 gpa = vmcs12->vm_entry_msr_load_addr + (i * sizeof(g)); 4172 if (kvm_vcpu_read_guest(vcpu, gpa, &g, sizeof(g))) { 4173 pr_debug_ratelimited( 4174 "%s read MSR index failed (%u, 0x%08llx)\n", 4175 __func__, i, gpa); 4176 goto vmabort; 4177 } 4178 4179 for (j = 0; j < vmcs12->vm_exit_msr_load_count; j++) { 4180 gpa = vmcs12->vm_exit_msr_load_addr + (j * sizeof(h)); 4181 if (kvm_vcpu_read_guest(vcpu, gpa, &h, sizeof(h))) { 4182 pr_debug_ratelimited( 4183 "%s read MSR failed (%u, 0x%08llx)\n", 4184 __func__, j, gpa); 4185 goto vmabort; 4186 } 4187 if (h.index != g.index) 4188 continue; 4189 if (h.value == g.value) 4190 break; 4191 4192 if (nested_vmx_load_msr_check(vcpu, &h)) { 4193 pr_debug_ratelimited( 4194 "%s check failed (%u, 0x%x, 0x%x)\n", 4195 __func__, j, h.index, h.reserved); 4196 goto vmabort; 4197 } 4198 4199 if (kvm_set_msr(vcpu, h.index, h.value)) { 4200 pr_debug_ratelimited( 4201 "%s WRMSR failed (%u, 0x%x, 0x%llx)\n", 4202 __func__, j, h.index, h.value); 4203 goto vmabort; 4204 } 4205 } 4206 } 4207 4208 return; 4209 4210 vmabort: 4211 nested_vmx_abort(vcpu, VMX_ABORT_LOAD_HOST_MSR_FAIL); 4212 } 4213 4214 /* 4215 * Emulate an exit from nested guest (L2) to L1, i.e., prepare to run L1 4216 * and modify vmcs12 to make it see what it would expect to see there if 4217 * L2 was its real guest. Must only be called when in L2 (is_guest_mode()) 4218 */ 4219 void nested_vmx_vmexit(struct kvm_vcpu *vcpu, u32 exit_reason, 4220 u32 exit_intr_info, unsigned long exit_qualification) 4221 { 4222 struct vcpu_vmx *vmx = to_vmx(vcpu); 4223 struct vmcs12 *vmcs12 = get_vmcs12(vcpu); 4224 4225 /* trying to cancel vmlaunch/vmresume is a bug */ 4226 WARN_ON_ONCE(vmx->nested.nested_run_pending); 4227 4228 leave_guest_mode(vcpu); 4229 4230 if (nested_cpu_has_preemption_timer(vmcs12)) 4231 hrtimer_cancel(&to_vmx(vcpu)->nested.preemption_timer); 4232 4233 if (vmcs12->cpu_based_vm_exec_control & CPU_BASED_USE_TSC_OFFSETTING) 4234 vcpu->arch.tsc_offset -= vmcs12->tsc_offset; 4235 4236 if (likely(!vmx->fail)) { 4237 sync_vmcs02_to_vmcs12(vcpu, vmcs12); 4238 4239 if (exit_reason != -1) 4240 prepare_vmcs12(vcpu, vmcs12, exit_reason, exit_intr_info, 4241 exit_qualification); 4242 4243 /* 4244 * Must happen outside of sync_vmcs02_to_vmcs12() as it will 4245 * also be used to capture vmcs12 cache as part of 4246 * capturing nVMX state for snapshot (migration). 4247 * 4248 * Otherwise, this flush will dirty guest memory at a 4249 * point it is already assumed by user-space to be 4250 * immutable. 4251 */ 4252 nested_flush_cached_shadow_vmcs12(vcpu, vmcs12); 4253 } else { 4254 /* 4255 * The only expected VM-instruction error is "VM entry with 4256 * invalid control field(s)." Anything else indicates a 4257 * problem with L0. And we should never get here with a 4258 * VMFail of any type if early consistency checks are enabled. 4259 */ 4260 WARN_ON_ONCE(vmcs_read32(VM_INSTRUCTION_ERROR) != 4261 VMXERR_ENTRY_INVALID_CONTROL_FIELD); 4262 WARN_ON_ONCE(nested_early_check); 4263 } 4264 4265 vmx_switch_vmcs(vcpu, &vmx->vmcs01); 4266 4267 /* Update any VMCS fields that might have changed while L2 ran */ 4268 vmcs_write32(VM_EXIT_MSR_LOAD_COUNT, vmx->msr_autoload.host.nr); 4269 vmcs_write32(VM_ENTRY_MSR_LOAD_COUNT, vmx->msr_autoload.guest.nr); 4270 vmcs_write64(TSC_OFFSET, vcpu->arch.tsc_offset); 4271 if (vmx->nested.l1_tpr_threshold != -1) 4272 vmcs_write32(TPR_THRESHOLD, vmx->nested.l1_tpr_threshold); 4273 4274 if (kvm_has_tsc_control) 4275 decache_tsc_multiplier(vmx); 4276 4277 if (vmx->nested.change_vmcs01_virtual_apic_mode) { 4278 vmx->nested.change_vmcs01_virtual_apic_mode = false; 4279 vmx_set_virtual_apic_mode(vcpu); 4280 } 4281 4282 /* Unpin physical memory we referred to in vmcs02 */ 4283 if (vmx->nested.apic_access_page) { 4284 kvm_release_page_clean(vmx->nested.apic_access_page); 4285 vmx->nested.apic_access_page = NULL; 4286 } 4287 kvm_vcpu_unmap(vcpu, &vmx->nested.virtual_apic_map, true); 4288 kvm_vcpu_unmap(vcpu, &vmx->nested.pi_desc_map, true); 4289 vmx->nested.pi_desc = NULL; 4290 4291 /* 4292 * We are now running in L2, mmu_notifier will force to reload the 4293 * page's hpa for L2 vmcs. Need to reload it for L1 before entering L1. 4294 */ 4295 kvm_make_request(KVM_REQ_APIC_PAGE_RELOAD, vcpu); 4296 4297 if ((exit_reason != -1) && (enable_shadow_vmcs || vmx->nested.hv_evmcs)) 4298 vmx->nested.need_vmcs12_to_shadow_sync = true; 4299 4300 /* in case we halted in L2 */ 4301 vcpu->arch.mp_state = KVM_MP_STATE_RUNNABLE; 4302 4303 if (likely(!vmx->fail)) { 4304 /* 4305 * TODO: SDM says that with acknowledge interrupt on 4306 * exit, bit 31 of the VM-exit interrupt information 4307 * (valid interrupt) is always set to 1 on 4308 * EXIT_REASON_EXTERNAL_INTERRUPT, so we shouldn't 4309 * need kvm_cpu_has_interrupt(). See the commit 4310 * message for details. 4311 */ 4312 if (nested_exit_intr_ack_set(vcpu) && 4313 exit_reason == EXIT_REASON_EXTERNAL_INTERRUPT && 4314 kvm_cpu_has_interrupt(vcpu)) { 4315 int irq = kvm_cpu_get_interrupt(vcpu); 4316 WARN_ON(irq < 0); 4317 vmcs12->vm_exit_intr_info = irq | 4318 INTR_INFO_VALID_MASK | INTR_TYPE_EXT_INTR; 4319 } 4320 4321 if (exit_reason != -1) 4322 trace_kvm_nested_vmexit_inject(vmcs12->vm_exit_reason, 4323 vmcs12->exit_qualification, 4324 vmcs12->idt_vectoring_info_field, 4325 vmcs12->vm_exit_intr_info, 4326 vmcs12->vm_exit_intr_error_code, 4327 KVM_ISA_VMX); 4328 4329 load_vmcs12_host_state(vcpu, vmcs12); 4330 4331 return; 4332 } 4333 4334 /* 4335 * After an early L2 VM-entry failure, we're now back 4336 * in L1 which thinks it just finished a VMLAUNCH or 4337 * VMRESUME instruction, so we need to set the failure 4338 * flag and the VM-instruction error field of the VMCS 4339 * accordingly, and skip the emulated instruction. 4340 */ 4341 (void)nested_vmx_failValid(vcpu, VMXERR_ENTRY_INVALID_CONTROL_FIELD); 4342 4343 /* 4344 * Restore L1's host state to KVM's software model. We're here 4345 * because a consistency check was caught by hardware, which 4346 * means some amount of guest state has been propagated to KVM's 4347 * model and needs to be unwound to the host's state. 4348 */ 4349 nested_vmx_restore_host_state(vcpu); 4350 4351 vmx->fail = 0; 4352 } 4353 4354 /* 4355 * Decode the memory-address operand of a vmx instruction, as recorded on an 4356 * exit caused by such an instruction (run by a guest hypervisor). 4357 * On success, returns 0. When the operand is invalid, returns 1 and throws 4358 * #UD or #GP. 4359 */ 4360 int get_vmx_mem_address(struct kvm_vcpu *vcpu, unsigned long exit_qualification, 4361 u32 vmx_instruction_info, bool wr, int len, gva_t *ret) 4362 { 4363 gva_t off; 4364 bool exn; 4365 struct kvm_segment s; 4366 4367 /* 4368 * According to Vol. 3B, "Information for VM Exits Due to Instruction 4369 * Execution", on an exit, vmx_instruction_info holds most of the 4370 * addressing components of the operand. Only the displacement part 4371 * is put in exit_qualification (see 3B, "Basic VM-Exit Information"). 4372 * For how an actual address is calculated from all these components, 4373 * refer to Vol. 1, "Operand Addressing". 4374 */ 4375 int scaling = vmx_instruction_info & 3; 4376 int addr_size = (vmx_instruction_info >> 7) & 7; 4377 bool is_reg = vmx_instruction_info & (1u << 10); 4378 int seg_reg = (vmx_instruction_info >> 15) & 7; 4379 int index_reg = (vmx_instruction_info >> 18) & 0xf; 4380 bool index_is_valid = !(vmx_instruction_info & (1u << 22)); 4381 int base_reg = (vmx_instruction_info >> 23) & 0xf; 4382 bool base_is_valid = !(vmx_instruction_info & (1u << 27)); 4383 4384 if (is_reg) { 4385 kvm_queue_exception(vcpu, UD_VECTOR); 4386 return 1; 4387 } 4388 4389 /* Addr = segment_base + offset */ 4390 /* offset = base + [index * scale] + displacement */ 4391 off = exit_qualification; /* holds the displacement */ 4392 if (addr_size == 1) 4393 off = (gva_t)sign_extend64(off, 31); 4394 else if (addr_size == 0) 4395 off = (gva_t)sign_extend64(off, 15); 4396 if (base_is_valid) 4397 off += kvm_register_read(vcpu, base_reg); 4398 if (index_is_valid) 4399 off += kvm_register_read(vcpu, index_reg)<<scaling; 4400 vmx_get_segment(vcpu, &s, seg_reg); 4401 4402 /* 4403 * The effective address, i.e. @off, of a memory operand is truncated 4404 * based on the address size of the instruction. Note that this is 4405 * the *effective address*, i.e. the address prior to accounting for 4406 * the segment's base. 4407 */ 4408 if (addr_size == 1) /* 32 bit */ 4409 off &= 0xffffffff; 4410 else if (addr_size == 0) /* 16 bit */ 4411 off &= 0xffff; 4412 4413 /* Checks for #GP/#SS exceptions. */ 4414 exn = false; 4415 if (is_long_mode(vcpu)) { 4416 /* 4417 * The virtual/linear address is never truncated in 64-bit 4418 * mode, e.g. a 32-bit address size can yield a 64-bit virtual 4419 * address when using FS/GS with a non-zero base. 4420 */ 4421 if (seg_reg == VCPU_SREG_FS || seg_reg == VCPU_SREG_GS) 4422 *ret = s.base + off; 4423 else 4424 *ret = off; 4425 4426 /* Long mode: #GP(0)/#SS(0) if the memory address is in a 4427 * non-canonical form. This is the only check on the memory 4428 * destination for long mode! 4429 */ 4430 exn = is_noncanonical_address(*ret, vcpu); 4431 } else { 4432 /* 4433 * When not in long mode, the virtual/linear address is 4434 * unconditionally truncated to 32 bits regardless of the 4435 * address size. 4436 */ 4437 *ret = (s.base + off) & 0xffffffff; 4438 4439 /* Protected mode: apply checks for segment validity in the 4440 * following order: 4441 * - segment type check (#GP(0) may be thrown) 4442 * - usability check (#GP(0)/#SS(0)) 4443 * - limit check (#GP(0)/#SS(0)) 4444 */ 4445 if (wr) 4446 /* #GP(0) if the destination operand is located in a 4447 * read-only data segment or any code segment. 4448 */ 4449 exn = ((s.type & 0xa) == 0 || (s.type & 8)); 4450 else 4451 /* #GP(0) if the source operand is located in an 4452 * execute-only code segment 4453 */ 4454 exn = ((s.type & 0xa) == 8); 4455 if (exn) { 4456 kvm_queue_exception_e(vcpu, GP_VECTOR, 0); 4457 return 1; 4458 } 4459 /* Protected mode: #GP(0)/#SS(0) if the segment is unusable. 4460 */ 4461 exn = (s.unusable != 0); 4462 4463 /* 4464 * Protected mode: #GP(0)/#SS(0) if the memory operand is 4465 * outside the segment limit. All CPUs that support VMX ignore 4466 * limit checks for flat segments, i.e. segments with base==0, 4467 * limit==0xffffffff and of type expand-up data or code. 4468 */ 4469 if (!(s.base == 0 && s.limit == 0xffffffff && 4470 ((s.type & 8) || !(s.type & 4)))) 4471 exn = exn || ((u64)off + len - 1 > s.limit); 4472 } 4473 if (exn) { 4474 kvm_queue_exception_e(vcpu, 4475 seg_reg == VCPU_SREG_SS ? 4476 SS_VECTOR : GP_VECTOR, 4477 0); 4478 return 1; 4479 } 4480 4481 return 0; 4482 } 4483 4484 void nested_vmx_pmu_entry_exit_ctls_update(struct kvm_vcpu *vcpu) 4485 { 4486 struct vcpu_vmx *vmx; 4487 4488 if (!nested_vmx_allowed(vcpu)) 4489 return; 4490 4491 vmx = to_vmx(vcpu); 4492 if (kvm_x86_ops->pmu_ops->is_valid_msr(vcpu, MSR_CORE_PERF_GLOBAL_CTRL)) { 4493 vmx->nested.msrs.entry_ctls_high |= 4494 VM_ENTRY_LOAD_IA32_PERF_GLOBAL_CTRL; 4495 vmx->nested.msrs.exit_ctls_high |= 4496 VM_EXIT_LOAD_IA32_PERF_GLOBAL_CTRL; 4497 } else { 4498 vmx->nested.msrs.entry_ctls_high &= 4499 ~VM_ENTRY_LOAD_IA32_PERF_GLOBAL_CTRL; 4500 vmx->nested.msrs.exit_ctls_high &= 4501 ~VM_ENTRY_LOAD_IA32_PERF_GLOBAL_CTRL; 4502 } 4503 } 4504 4505 static int nested_vmx_get_vmptr(struct kvm_vcpu *vcpu, gpa_t *vmpointer) 4506 { 4507 gva_t gva; 4508 struct x86_exception e; 4509 4510 if (get_vmx_mem_address(vcpu, vmcs_readl(EXIT_QUALIFICATION), 4511 vmcs_read32(VMX_INSTRUCTION_INFO), false, 4512 sizeof(*vmpointer), &gva)) 4513 return 1; 4514 4515 if (kvm_read_guest_virt(vcpu, gva, vmpointer, sizeof(*vmpointer), &e)) { 4516 kvm_inject_page_fault(vcpu, &e); 4517 return 1; 4518 } 4519 4520 return 0; 4521 } 4522 4523 /* 4524 * Allocate a shadow VMCS and associate it with the currently loaded 4525 * VMCS, unless such a shadow VMCS already exists. The newly allocated 4526 * VMCS is also VMCLEARed, so that it is ready for use. 4527 */ 4528 static struct vmcs *alloc_shadow_vmcs(struct kvm_vcpu *vcpu) 4529 { 4530 struct vcpu_vmx *vmx = to_vmx(vcpu); 4531 struct loaded_vmcs *loaded_vmcs = vmx->loaded_vmcs; 4532 4533 /* 4534 * We should allocate a shadow vmcs for vmcs01 only when L1 4535 * executes VMXON and free it when L1 executes VMXOFF. 4536 * As it is invalid to execute VMXON twice, we shouldn't reach 4537 * here when vmcs01 already have an allocated shadow vmcs. 4538 */ 4539 WARN_ON(loaded_vmcs == &vmx->vmcs01 && loaded_vmcs->shadow_vmcs); 4540 4541 if (!loaded_vmcs->shadow_vmcs) { 4542 loaded_vmcs->shadow_vmcs = alloc_vmcs(true); 4543 if (loaded_vmcs->shadow_vmcs) 4544 vmcs_clear(loaded_vmcs->shadow_vmcs); 4545 } 4546 return loaded_vmcs->shadow_vmcs; 4547 } 4548 4549 static int enter_vmx_operation(struct kvm_vcpu *vcpu) 4550 { 4551 struct vcpu_vmx *vmx = to_vmx(vcpu); 4552 int r; 4553 4554 r = alloc_loaded_vmcs(&vmx->nested.vmcs02); 4555 if (r < 0) 4556 goto out_vmcs02; 4557 4558 vmx->nested.cached_vmcs12 = kzalloc(VMCS12_SIZE, GFP_KERNEL_ACCOUNT); 4559 if (!vmx->nested.cached_vmcs12) 4560 goto out_cached_vmcs12; 4561 4562 vmx->nested.cached_shadow_vmcs12 = kzalloc(VMCS12_SIZE, GFP_KERNEL_ACCOUNT); 4563 if (!vmx->nested.cached_shadow_vmcs12) 4564 goto out_cached_shadow_vmcs12; 4565 4566 if (enable_shadow_vmcs && !alloc_shadow_vmcs(vcpu)) 4567 goto out_shadow_vmcs; 4568 4569 hrtimer_init(&vmx->nested.preemption_timer, CLOCK_MONOTONIC, 4570 HRTIMER_MODE_REL_PINNED); 4571 vmx->nested.preemption_timer.function = vmx_preemption_timer_fn; 4572 4573 vmx->nested.vpid02 = allocate_vpid(); 4574 4575 vmx->nested.vmcs02_initialized = false; 4576 vmx->nested.vmxon = true; 4577 4578 if (pt_mode == PT_MODE_HOST_GUEST) { 4579 vmx->pt_desc.guest.ctl = 0; 4580 pt_update_intercept_for_msr(vmx); 4581 } 4582 4583 return 0; 4584 4585 out_shadow_vmcs: 4586 kfree(vmx->nested.cached_shadow_vmcs12); 4587 4588 out_cached_shadow_vmcs12: 4589 kfree(vmx->nested.cached_vmcs12); 4590 4591 out_cached_vmcs12: 4592 free_loaded_vmcs(&vmx->nested.vmcs02); 4593 4594 out_vmcs02: 4595 return -ENOMEM; 4596 } 4597 4598 /* 4599 * Emulate the VMXON instruction. 4600 * Currently, we just remember that VMX is active, and do not save or even 4601 * inspect the argument to VMXON (the so-called "VMXON pointer") because we 4602 * do not currently need to store anything in that guest-allocated memory 4603 * region. Consequently, VMCLEAR and VMPTRLD also do not verify that the their 4604 * argument is different from the VMXON pointer (which the spec says they do). 4605 */ 4606 static int handle_vmon(struct kvm_vcpu *vcpu) 4607 { 4608 int ret; 4609 gpa_t vmptr; 4610 uint32_t revision; 4611 struct vcpu_vmx *vmx = to_vmx(vcpu); 4612 const u64 VMXON_NEEDED_FEATURES = FEAT_CTL_LOCKED 4613 | FEAT_CTL_VMX_ENABLED_OUTSIDE_SMX; 4614 4615 /* 4616 * The Intel VMX Instruction Reference lists a bunch of bits that are 4617 * prerequisite to running VMXON, most notably cr4.VMXE must be set to 4618 * 1 (see vmx_set_cr4() for when we allow the guest to set this). 4619 * Otherwise, we should fail with #UD. But most faulting conditions 4620 * have already been checked by hardware, prior to the VM-exit for 4621 * VMXON. We do test guest cr4.VMXE because processor CR4 always has 4622 * that bit set to 1 in non-root mode. 4623 */ 4624 if (!kvm_read_cr4_bits(vcpu, X86_CR4_VMXE)) { 4625 kvm_queue_exception(vcpu, UD_VECTOR); 4626 return 1; 4627 } 4628 4629 /* CPL=0 must be checked manually. */ 4630 if (vmx_get_cpl(vcpu)) { 4631 kvm_inject_gp(vcpu, 0); 4632 return 1; 4633 } 4634 4635 if (vmx->nested.vmxon) 4636 return nested_vmx_failValid(vcpu, 4637 VMXERR_VMXON_IN_VMX_ROOT_OPERATION); 4638 4639 if ((vmx->msr_ia32_feature_control & VMXON_NEEDED_FEATURES) 4640 != VMXON_NEEDED_FEATURES) { 4641 kvm_inject_gp(vcpu, 0); 4642 return 1; 4643 } 4644 4645 if (nested_vmx_get_vmptr(vcpu, &vmptr)) 4646 return 1; 4647 4648 /* 4649 * SDM 3: 24.11.5 4650 * The first 4 bytes of VMXON region contain the supported 4651 * VMCS revision identifier 4652 * 4653 * Note - IA32_VMX_BASIC[48] will never be 1 for the nested case; 4654 * which replaces physical address width with 32 4655 */ 4656 if (!page_address_valid(vcpu, vmptr)) 4657 return nested_vmx_failInvalid(vcpu); 4658 4659 if (kvm_read_guest(vcpu->kvm, vmptr, &revision, sizeof(revision)) || 4660 revision != VMCS12_REVISION) 4661 return nested_vmx_failInvalid(vcpu); 4662 4663 vmx->nested.vmxon_ptr = vmptr; 4664 ret = enter_vmx_operation(vcpu); 4665 if (ret) 4666 return ret; 4667 4668 return nested_vmx_succeed(vcpu); 4669 } 4670 4671 static inline void nested_release_vmcs12(struct kvm_vcpu *vcpu) 4672 { 4673 struct vcpu_vmx *vmx = to_vmx(vcpu); 4674 4675 if (vmx->nested.current_vmptr == -1ull) 4676 return; 4677 4678 copy_vmcs02_to_vmcs12_rare(vcpu, get_vmcs12(vcpu)); 4679 4680 if (enable_shadow_vmcs) { 4681 /* copy to memory all shadowed fields in case 4682 they were modified */ 4683 copy_shadow_to_vmcs12(vmx); 4684 vmx_disable_shadow_vmcs(vmx); 4685 } 4686 vmx->nested.posted_intr_nv = -1; 4687 4688 /* Flush VMCS12 to guest memory */ 4689 kvm_vcpu_write_guest_page(vcpu, 4690 vmx->nested.current_vmptr >> PAGE_SHIFT, 4691 vmx->nested.cached_vmcs12, 0, VMCS12_SIZE); 4692 4693 kvm_mmu_free_roots(vcpu, &vcpu->arch.guest_mmu, KVM_MMU_ROOTS_ALL); 4694 4695 vmx->nested.current_vmptr = -1ull; 4696 } 4697 4698 /* Emulate the VMXOFF instruction */ 4699 static int handle_vmoff(struct kvm_vcpu *vcpu) 4700 { 4701 if (!nested_vmx_check_permission(vcpu)) 4702 return 1; 4703 4704 free_nested(vcpu); 4705 4706 /* Process a latched INIT during time CPU was in VMX operation */ 4707 kvm_make_request(KVM_REQ_EVENT, vcpu); 4708 4709 return nested_vmx_succeed(vcpu); 4710 } 4711 4712 /* Emulate the VMCLEAR instruction */ 4713 static int handle_vmclear(struct kvm_vcpu *vcpu) 4714 { 4715 struct vcpu_vmx *vmx = to_vmx(vcpu); 4716 u32 zero = 0; 4717 gpa_t vmptr; 4718 u64 evmcs_gpa; 4719 4720 if (!nested_vmx_check_permission(vcpu)) 4721 return 1; 4722 4723 if (nested_vmx_get_vmptr(vcpu, &vmptr)) 4724 return 1; 4725 4726 if (!page_address_valid(vcpu, vmptr)) 4727 return nested_vmx_failValid(vcpu, 4728 VMXERR_VMCLEAR_INVALID_ADDRESS); 4729 4730 if (vmptr == vmx->nested.vmxon_ptr) 4731 return nested_vmx_failValid(vcpu, 4732 VMXERR_VMCLEAR_VMXON_POINTER); 4733 4734 /* 4735 * When Enlightened VMEntry is enabled on the calling CPU we treat 4736 * memory area pointer by vmptr as Enlightened VMCS (as there's no good 4737 * way to distinguish it from VMCS12) and we must not corrupt it by 4738 * writing to the non-existent 'launch_state' field. The area doesn't 4739 * have to be the currently active EVMCS on the calling CPU and there's 4740 * nothing KVM has to do to transition it from 'active' to 'non-active' 4741 * state. It is possible that the area will stay mapped as 4742 * vmx->nested.hv_evmcs but this shouldn't be a problem. 4743 */ 4744 if (likely(!vmx->nested.enlightened_vmcs_enabled || 4745 !nested_enlightened_vmentry(vcpu, &evmcs_gpa))) { 4746 if (vmptr == vmx->nested.current_vmptr) 4747 nested_release_vmcs12(vcpu); 4748 4749 kvm_vcpu_write_guest(vcpu, 4750 vmptr + offsetof(struct vmcs12, 4751 launch_state), 4752 &zero, sizeof(zero)); 4753 } 4754 4755 return nested_vmx_succeed(vcpu); 4756 } 4757 4758 /* Emulate the VMLAUNCH instruction */ 4759 static int handle_vmlaunch(struct kvm_vcpu *vcpu) 4760 { 4761 return nested_vmx_run(vcpu, true); 4762 } 4763 4764 /* Emulate the VMRESUME instruction */ 4765 static int handle_vmresume(struct kvm_vcpu *vcpu) 4766 { 4767 4768 return nested_vmx_run(vcpu, false); 4769 } 4770 4771 static int handle_vmread(struct kvm_vcpu *vcpu) 4772 { 4773 struct vmcs12 *vmcs12 = is_guest_mode(vcpu) ? get_shadow_vmcs12(vcpu) 4774 : get_vmcs12(vcpu); 4775 unsigned long exit_qualification = vmcs_readl(EXIT_QUALIFICATION); 4776 u32 instr_info = vmcs_read32(VMX_INSTRUCTION_INFO); 4777 struct vcpu_vmx *vmx = to_vmx(vcpu); 4778 struct x86_exception e; 4779 unsigned long field; 4780 u64 value; 4781 gva_t gva = 0; 4782 short offset; 4783 int len; 4784 4785 if (!nested_vmx_check_permission(vcpu)) 4786 return 1; 4787 4788 /* 4789 * In VMX non-root operation, when the VMCS-link pointer is -1ull, 4790 * any VMREAD sets the ALU flags for VMfailInvalid. 4791 */ 4792 if (vmx->nested.current_vmptr == -1ull || 4793 (is_guest_mode(vcpu) && 4794 get_vmcs12(vcpu)->vmcs_link_pointer == -1ull)) 4795 return nested_vmx_failInvalid(vcpu); 4796 4797 /* Decode instruction info and find the field to read */ 4798 field = kvm_register_readl(vcpu, (((instr_info) >> 28) & 0xf)); 4799 4800 offset = vmcs_field_to_offset(field); 4801 if (offset < 0) 4802 return nested_vmx_failValid(vcpu, 4803 VMXERR_UNSUPPORTED_VMCS_COMPONENT); 4804 4805 if (!is_guest_mode(vcpu) && is_vmcs12_ext_field(field)) 4806 copy_vmcs02_to_vmcs12_rare(vcpu, vmcs12); 4807 4808 /* Read the field, zero-extended to a u64 value */ 4809 value = vmcs12_read_any(vmcs12, field, offset); 4810 4811 /* 4812 * Now copy part of this value to register or memory, as requested. 4813 * Note that the number of bits actually copied is 32 or 64 depending 4814 * on the guest's mode (32 or 64 bit), not on the given field's length. 4815 */ 4816 if (instr_info & BIT(10)) { 4817 kvm_register_writel(vcpu, (((instr_info) >> 3) & 0xf), value); 4818 } else { 4819 len = is_64_bit_mode(vcpu) ? 8 : 4; 4820 if (get_vmx_mem_address(vcpu, exit_qualification, 4821 instr_info, true, len, &gva)) 4822 return 1; 4823 /* _system ok, nested_vmx_check_permission has verified cpl=0 */ 4824 if (kvm_write_guest_virt_system(vcpu, gva, &value, len, &e)) { 4825 kvm_inject_page_fault(vcpu, &e); 4826 return 1; 4827 } 4828 } 4829 4830 return nested_vmx_succeed(vcpu); 4831 } 4832 4833 static bool is_shadow_field_rw(unsigned long field) 4834 { 4835 switch (field) { 4836 #define SHADOW_FIELD_RW(x, y) case x: 4837 #include "vmcs_shadow_fields.h" 4838 return true; 4839 default: 4840 break; 4841 } 4842 return false; 4843 } 4844 4845 static bool is_shadow_field_ro(unsigned long field) 4846 { 4847 switch (field) { 4848 #define SHADOW_FIELD_RO(x, y) case x: 4849 #include "vmcs_shadow_fields.h" 4850 return true; 4851 default: 4852 break; 4853 } 4854 return false; 4855 } 4856 4857 static int handle_vmwrite(struct kvm_vcpu *vcpu) 4858 { 4859 struct vmcs12 *vmcs12 = is_guest_mode(vcpu) ? get_shadow_vmcs12(vcpu) 4860 : get_vmcs12(vcpu); 4861 unsigned long exit_qualification = vmcs_readl(EXIT_QUALIFICATION); 4862 u32 instr_info = vmcs_read32(VMX_INSTRUCTION_INFO); 4863 struct vcpu_vmx *vmx = to_vmx(vcpu); 4864 struct x86_exception e; 4865 unsigned long field; 4866 short offset; 4867 gva_t gva; 4868 int len; 4869 4870 /* 4871 * The value to write might be 32 or 64 bits, depending on L1's long 4872 * mode, and eventually we need to write that into a field of several 4873 * possible lengths. The code below first zero-extends the value to 64 4874 * bit (value), and then copies only the appropriate number of 4875 * bits into the vmcs12 field. 4876 */ 4877 u64 value = 0; 4878 4879 if (!nested_vmx_check_permission(vcpu)) 4880 return 1; 4881 4882 /* 4883 * In VMX non-root operation, when the VMCS-link pointer is -1ull, 4884 * any VMWRITE sets the ALU flags for VMfailInvalid. 4885 */ 4886 if (vmx->nested.current_vmptr == -1ull || 4887 (is_guest_mode(vcpu) && 4888 get_vmcs12(vcpu)->vmcs_link_pointer == -1ull)) 4889 return nested_vmx_failInvalid(vcpu); 4890 4891 if (instr_info & BIT(10)) 4892 value = kvm_register_readl(vcpu, (((instr_info) >> 3) & 0xf)); 4893 else { 4894 len = is_64_bit_mode(vcpu) ? 8 : 4; 4895 if (get_vmx_mem_address(vcpu, exit_qualification, 4896 instr_info, false, len, &gva)) 4897 return 1; 4898 if (kvm_read_guest_virt(vcpu, gva, &value, len, &e)) { 4899 kvm_inject_page_fault(vcpu, &e); 4900 return 1; 4901 } 4902 } 4903 4904 field = kvm_register_readl(vcpu, (((instr_info) >> 28) & 0xf)); 4905 4906 offset = vmcs_field_to_offset(field); 4907 if (offset < 0) 4908 return nested_vmx_failValid(vcpu, 4909 VMXERR_UNSUPPORTED_VMCS_COMPONENT); 4910 4911 /* 4912 * If the vCPU supports "VMWRITE to any supported field in the 4913 * VMCS," then the "read-only" fields are actually read/write. 4914 */ 4915 if (vmcs_field_readonly(field) && 4916 !nested_cpu_has_vmwrite_any_field(vcpu)) 4917 return nested_vmx_failValid(vcpu, 4918 VMXERR_VMWRITE_READ_ONLY_VMCS_COMPONENT); 4919 4920 /* 4921 * Ensure vmcs12 is up-to-date before any VMWRITE that dirties 4922 * vmcs12, else we may crush a field or consume a stale value. 4923 */ 4924 if (!is_guest_mode(vcpu) && !is_shadow_field_rw(field)) 4925 copy_vmcs02_to_vmcs12_rare(vcpu, vmcs12); 4926 4927 /* 4928 * Some Intel CPUs intentionally drop the reserved bits of the AR byte 4929 * fields on VMWRITE. Emulate this behavior to ensure consistent KVM 4930 * behavior regardless of the underlying hardware, e.g. if an AR_BYTE 4931 * field is intercepted for VMWRITE but not VMREAD (in L1), then VMREAD 4932 * from L1 will return a different value than VMREAD from L2 (L1 sees 4933 * the stripped down value, L2 sees the full value as stored by KVM). 4934 */ 4935 if (field >= GUEST_ES_AR_BYTES && field <= GUEST_TR_AR_BYTES) 4936 value &= 0x1f0ff; 4937 4938 vmcs12_write_any(vmcs12, field, offset, value); 4939 4940 /* 4941 * Do not track vmcs12 dirty-state if in guest-mode as we actually 4942 * dirty shadow vmcs12 instead of vmcs12. Fields that can be updated 4943 * by L1 without a vmexit are always updated in the vmcs02, i.e. don't 4944 * "dirty" vmcs12, all others go down the prepare_vmcs02() slow path. 4945 */ 4946 if (!is_guest_mode(vcpu) && !is_shadow_field_rw(field)) { 4947 /* 4948 * L1 can read these fields without exiting, ensure the 4949 * shadow VMCS is up-to-date. 4950 */ 4951 if (enable_shadow_vmcs && is_shadow_field_ro(field)) { 4952 preempt_disable(); 4953 vmcs_load(vmx->vmcs01.shadow_vmcs); 4954 4955 __vmcs_writel(field, value); 4956 4957 vmcs_clear(vmx->vmcs01.shadow_vmcs); 4958 vmcs_load(vmx->loaded_vmcs->vmcs); 4959 preempt_enable(); 4960 } 4961 vmx->nested.dirty_vmcs12 = true; 4962 } 4963 4964 return nested_vmx_succeed(vcpu); 4965 } 4966 4967 static void set_current_vmptr(struct vcpu_vmx *vmx, gpa_t vmptr) 4968 { 4969 vmx->nested.current_vmptr = vmptr; 4970 if (enable_shadow_vmcs) { 4971 secondary_exec_controls_setbit(vmx, SECONDARY_EXEC_SHADOW_VMCS); 4972 vmcs_write64(VMCS_LINK_POINTER, 4973 __pa(vmx->vmcs01.shadow_vmcs)); 4974 vmx->nested.need_vmcs12_to_shadow_sync = true; 4975 } 4976 vmx->nested.dirty_vmcs12 = true; 4977 } 4978 4979 /* Emulate the VMPTRLD instruction */ 4980 static int handle_vmptrld(struct kvm_vcpu *vcpu) 4981 { 4982 struct vcpu_vmx *vmx = to_vmx(vcpu); 4983 gpa_t vmptr; 4984 4985 if (!nested_vmx_check_permission(vcpu)) 4986 return 1; 4987 4988 if (nested_vmx_get_vmptr(vcpu, &vmptr)) 4989 return 1; 4990 4991 if (!page_address_valid(vcpu, vmptr)) 4992 return nested_vmx_failValid(vcpu, 4993 VMXERR_VMPTRLD_INVALID_ADDRESS); 4994 4995 if (vmptr == vmx->nested.vmxon_ptr) 4996 return nested_vmx_failValid(vcpu, 4997 VMXERR_VMPTRLD_VMXON_POINTER); 4998 4999 /* Forbid normal VMPTRLD if Enlightened version was used */ 5000 if (vmx->nested.hv_evmcs) 5001 return 1; 5002 5003 if (vmx->nested.current_vmptr != vmptr) { 5004 struct kvm_host_map map; 5005 struct vmcs12 *new_vmcs12; 5006 5007 if (kvm_vcpu_map(vcpu, gpa_to_gfn(vmptr), &map)) { 5008 /* 5009 * Reads from an unbacked page return all 1s, 5010 * which means that the 32 bits located at the 5011 * given physical address won't match the required 5012 * VMCS12_REVISION identifier. 5013 */ 5014 return nested_vmx_failValid(vcpu, 5015 VMXERR_VMPTRLD_INCORRECT_VMCS_REVISION_ID); 5016 } 5017 5018 new_vmcs12 = map.hva; 5019 5020 if (new_vmcs12->hdr.revision_id != VMCS12_REVISION || 5021 (new_vmcs12->hdr.shadow_vmcs && 5022 !nested_cpu_has_vmx_shadow_vmcs(vcpu))) { 5023 kvm_vcpu_unmap(vcpu, &map, false); 5024 return nested_vmx_failValid(vcpu, 5025 VMXERR_VMPTRLD_INCORRECT_VMCS_REVISION_ID); 5026 } 5027 5028 nested_release_vmcs12(vcpu); 5029 5030 /* 5031 * Load VMCS12 from guest memory since it is not already 5032 * cached. 5033 */ 5034 memcpy(vmx->nested.cached_vmcs12, new_vmcs12, VMCS12_SIZE); 5035 kvm_vcpu_unmap(vcpu, &map, false); 5036 5037 set_current_vmptr(vmx, vmptr); 5038 } 5039 5040 return nested_vmx_succeed(vcpu); 5041 } 5042 5043 /* Emulate the VMPTRST instruction */ 5044 static int handle_vmptrst(struct kvm_vcpu *vcpu) 5045 { 5046 unsigned long exit_qual = vmcs_readl(EXIT_QUALIFICATION); 5047 u32 instr_info = vmcs_read32(VMX_INSTRUCTION_INFO); 5048 gpa_t current_vmptr = to_vmx(vcpu)->nested.current_vmptr; 5049 struct x86_exception e; 5050 gva_t gva; 5051 5052 if (!nested_vmx_check_permission(vcpu)) 5053 return 1; 5054 5055 if (unlikely(to_vmx(vcpu)->nested.hv_evmcs)) 5056 return 1; 5057 5058 if (get_vmx_mem_address(vcpu, exit_qual, instr_info, 5059 true, sizeof(gpa_t), &gva)) 5060 return 1; 5061 /* *_system ok, nested_vmx_check_permission has verified cpl=0 */ 5062 if (kvm_write_guest_virt_system(vcpu, gva, (void *)¤t_vmptr, 5063 sizeof(gpa_t), &e)) { 5064 kvm_inject_page_fault(vcpu, &e); 5065 return 1; 5066 } 5067 return nested_vmx_succeed(vcpu); 5068 } 5069 5070 /* Emulate the INVEPT instruction */ 5071 static int handle_invept(struct kvm_vcpu *vcpu) 5072 { 5073 struct vcpu_vmx *vmx = to_vmx(vcpu); 5074 u32 vmx_instruction_info, types; 5075 unsigned long type; 5076 gva_t gva; 5077 struct x86_exception e; 5078 struct { 5079 u64 eptp, gpa; 5080 } operand; 5081 5082 if (!(vmx->nested.msrs.secondary_ctls_high & 5083 SECONDARY_EXEC_ENABLE_EPT) || 5084 !(vmx->nested.msrs.ept_caps & VMX_EPT_INVEPT_BIT)) { 5085 kvm_queue_exception(vcpu, UD_VECTOR); 5086 return 1; 5087 } 5088 5089 if (!nested_vmx_check_permission(vcpu)) 5090 return 1; 5091 5092 vmx_instruction_info = vmcs_read32(VMX_INSTRUCTION_INFO); 5093 type = kvm_register_readl(vcpu, (vmx_instruction_info >> 28) & 0xf); 5094 5095 types = (vmx->nested.msrs.ept_caps >> VMX_EPT_EXTENT_SHIFT) & 6; 5096 5097 if (type >= 32 || !(types & (1 << type))) 5098 return nested_vmx_failValid(vcpu, 5099 VMXERR_INVALID_OPERAND_TO_INVEPT_INVVPID); 5100 5101 /* According to the Intel VMX instruction reference, the memory 5102 * operand is read even if it isn't needed (e.g., for type==global) 5103 */ 5104 if (get_vmx_mem_address(vcpu, vmcs_readl(EXIT_QUALIFICATION), 5105 vmx_instruction_info, false, sizeof(operand), &gva)) 5106 return 1; 5107 if (kvm_read_guest_virt(vcpu, gva, &operand, sizeof(operand), &e)) { 5108 kvm_inject_page_fault(vcpu, &e); 5109 return 1; 5110 } 5111 5112 switch (type) { 5113 case VMX_EPT_EXTENT_GLOBAL: 5114 case VMX_EPT_EXTENT_CONTEXT: 5115 /* 5116 * TODO: Sync the necessary shadow EPT roots here, rather than 5117 * at the next emulated VM-entry. 5118 */ 5119 break; 5120 default: 5121 BUG_ON(1); 5122 break; 5123 } 5124 5125 return nested_vmx_succeed(vcpu); 5126 } 5127 5128 static int handle_invvpid(struct kvm_vcpu *vcpu) 5129 { 5130 struct vcpu_vmx *vmx = to_vmx(vcpu); 5131 u32 vmx_instruction_info; 5132 unsigned long type, types; 5133 gva_t gva; 5134 struct x86_exception e; 5135 struct { 5136 u64 vpid; 5137 u64 gla; 5138 } operand; 5139 u16 vpid02; 5140 5141 if (!(vmx->nested.msrs.secondary_ctls_high & 5142 SECONDARY_EXEC_ENABLE_VPID) || 5143 !(vmx->nested.msrs.vpid_caps & VMX_VPID_INVVPID_BIT)) { 5144 kvm_queue_exception(vcpu, UD_VECTOR); 5145 return 1; 5146 } 5147 5148 if (!nested_vmx_check_permission(vcpu)) 5149 return 1; 5150 5151 vmx_instruction_info = vmcs_read32(VMX_INSTRUCTION_INFO); 5152 type = kvm_register_readl(vcpu, (vmx_instruction_info >> 28) & 0xf); 5153 5154 types = (vmx->nested.msrs.vpid_caps & 5155 VMX_VPID_EXTENT_SUPPORTED_MASK) >> 8; 5156 5157 if (type >= 32 || !(types & (1 << type))) 5158 return nested_vmx_failValid(vcpu, 5159 VMXERR_INVALID_OPERAND_TO_INVEPT_INVVPID); 5160 5161 /* according to the intel vmx instruction reference, the memory 5162 * operand is read even if it isn't needed (e.g., for type==global) 5163 */ 5164 if (get_vmx_mem_address(vcpu, vmcs_readl(EXIT_QUALIFICATION), 5165 vmx_instruction_info, false, sizeof(operand), &gva)) 5166 return 1; 5167 if (kvm_read_guest_virt(vcpu, gva, &operand, sizeof(operand), &e)) { 5168 kvm_inject_page_fault(vcpu, &e); 5169 return 1; 5170 } 5171 if (operand.vpid >> 16) 5172 return nested_vmx_failValid(vcpu, 5173 VMXERR_INVALID_OPERAND_TO_INVEPT_INVVPID); 5174 5175 vpid02 = nested_get_vpid02(vcpu); 5176 switch (type) { 5177 case VMX_VPID_EXTENT_INDIVIDUAL_ADDR: 5178 if (!operand.vpid || 5179 is_noncanonical_address(operand.gla, vcpu)) 5180 return nested_vmx_failValid(vcpu, 5181 VMXERR_INVALID_OPERAND_TO_INVEPT_INVVPID); 5182 if (cpu_has_vmx_invvpid_individual_addr()) { 5183 __invvpid(VMX_VPID_EXTENT_INDIVIDUAL_ADDR, 5184 vpid02, operand.gla); 5185 } else 5186 __vmx_flush_tlb(vcpu, vpid02, false); 5187 break; 5188 case VMX_VPID_EXTENT_SINGLE_CONTEXT: 5189 case VMX_VPID_EXTENT_SINGLE_NON_GLOBAL: 5190 if (!operand.vpid) 5191 return nested_vmx_failValid(vcpu, 5192 VMXERR_INVALID_OPERAND_TO_INVEPT_INVVPID); 5193 __vmx_flush_tlb(vcpu, vpid02, false); 5194 break; 5195 case VMX_VPID_EXTENT_ALL_CONTEXT: 5196 __vmx_flush_tlb(vcpu, vpid02, false); 5197 break; 5198 default: 5199 WARN_ON_ONCE(1); 5200 return kvm_skip_emulated_instruction(vcpu); 5201 } 5202 5203 return nested_vmx_succeed(vcpu); 5204 } 5205 5206 static int nested_vmx_eptp_switching(struct kvm_vcpu *vcpu, 5207 struct vmcs12 *vmcs12) 5208 { 5209 u32 index = kvm_rcx_read(vcpu); 5210 u64 address; 5211 bool accessed_dirty; 5212 struct kvm_mmu *mmu = vcpu->arch.walk_mmu; 5213 5214 if (!nested_cpu_has_eptp_switching(vmcs12) || 5215 !nested_cpu_has_ept(vmcs12)) 5216 return 1; 5217 5218 if (index >= VMFUNC_EPTP_ENTRIES) 5219 return 1; 5220 5221 5222 if (kvm_vcpu_read_guest_page(vcpu, vmcs12->eptp_list_address >> PAGE_SHIFT, 5223 &address, index * 8, 8)) 5224 return 1; 5225 5226 accessed_dirty = !!(address & VMX_EPTP_AD_ENABLE_BIT); 5227 5228 /* 5229 * If the (L2) guest does a vmfunc to the currently 5230 * active ept pointer, we don't have to do anything else 5231 */ 5232 if (vmcs12->ept_pointer != address) { 5233 if (!valid_ept_address(vcpu, address)) 5234 return 1; 5235 5236 kvm_mmu_unload(vcpu); 5237 mmu->ept_ad = accessed_dirty; 5238 mmu->mmu_role.base.ad_disabled = !accessed_dirty; 5239 vmcs12->ept_pointer = address; 5240 /* 5241 * TODO: Check what's the correct approach in case 5242 * mmu reload fails. Currently, we just let the next 5243 * reload potentially fail 5244 */ 5245 kvm_mmu_reload(vcpu); 5246 } 5247 5248 return 0; 5249 } 5250 5251 static int handle_vmfunc(struct kvm_vcpu *vcpu) 5252 { 5253 struct vcpu_vmx *vmx = to_vmx(vcpu); 5254 struct vmcs12 *vmcs12; 5255 u32 function = kvm_rax_read(vcpu); 5256 5257 /* 5258 * VMFUNC is only supported for nested guests, but we always enable the 5259 * secondary control for simplicity; for non-nested mode, fake that we 5260 * didn't by injecting #UD. 5261 */ 5262 if (!is_guest_mode(vcpu)) { 5263 kvm_queue_exception(vcpu, UD_VECTOR); 5264 return 1; 5265 } 5266 5267 vmcs12 = get_vmcs12(vcpu); 5268 if ((vmcs12->vm_function_control & (1 << function)) == 0) 5269 goto fail; 5270 5271 switch (function) { 5272 case 0: 5273 if (nested_vmx_eptp_switching(vcpu, vmcs12)) 5274 goto fail; 5275 break; 5276 default: 5277 goto fail; 5278 } 5279 return kvm_skip_emulated_instruction(vcpu); 5280 5281 fail: 5282 nested_vmx_vmexit(vcpu, vmx->exit_reason, 5283 vmcs_read32(VM_EXIT_INTR_INFO), 5284 vmcs_readl(EXIT_QUALIFICATION)); 5285 return 1; 5286 } 5287 5288 5289 static bool nested_vmx_exit_handled_io(struct kvm_vcpu *vcpu, 5290 struct vmcs12 *vmcs12) 5291 { 5292 unsigned long exit_qualification; 5293 gpa_t bitmap, last_bitmap; 5294 unsigned int port; 5295 int size; 5296 u8 b; 5297 5298 if (!nested_cpu_has(vmcs12, CPU_BASED_USE_IO_BITMAPS)) 5299 return nested_cpu_has(vmcs12, CPU_BASED_UNCOND_IO_EXITING); 5300 5301 exit_qualification = vmcs_readl(EXIT_QUALIFICATION); 5302 5303 port = exit_qualification >> 16; 5304 size = (exit_qualification & 7) + 1; 5305 5306 last_bitmap = (gpa_t)-1; 5307 b = -1; 5308 5309 while (size > 0) { 5310 if (port < 0x8000) 5311 bitmap = vmcs12->io_bitmap_a; 5312 else if (port < 0x10000) 5313 bitmap = vmcs12->io_bitmap_b; 5314 else 5315 return true; 5316 bitmap += (port & 0x7fff) / 8; 5317 5318 if (last_bitmap != bitmap) 5319 if (kvm_vcpu_read_guest(vcpu, bitmap, &b, 1)) 5320 return true; 5321 if (b & (1 << (port & 7))) 5322 return true; 5323 5324 port++; 5325 size--; 5326 last_bitmap = bitmap; 5327 } 5328 5329 return false; 5330 } 5331 5332 /* 5333 * Return 1 if we should exit from L2 to L1 to handle an MSR access access, 5334 * rather than handle it ourselves in L0. I.e., check whether L1 expressed 5335 * disinterest in the current event (read or write a specific MSR) by using an 5336 * MSR bitmap. This may be the case even when L0 doesn't use MSR bitmaps. 5337 */ 5338 static bool nested_vmx_exit_handled_msr(struct kvm_vcpu *vcpu, 5339 struct vmcs12 *vmcs12, u32 exit_reason) 5340 { 5341 u32 msr_index = kvm_rcx_read(vcpu); 5342 gpa_t bitmap; 5343 5344 if (!nested_cpu_has(vmcs12, CPU_BASED_USE_MSR_BITMAPS)) 5345 return true; 5346 5347 /* 5348 * The MSR_BITMAP page is divided into four 1024-byte bitmaps, 5349 * for the four combinations of read/write and low/high MSR numbers. 5350 * First we need to figure out which of the four to use: 5351 */ 5352 bitmap = vmcs12->msr_bitmap; 5353 if (exit_reason == EXIT_REASON_MSR_WRITE) 5354 bitmap += 2048; 5355 if (msr_index >= 0xc0000000) { 5356 msr_index -= 0xc0000000; 5357 bitmap += 1024; 5358 } 5359 5360 /* Then read the msr_index'th bit from this bitmap: */ 5361 if (msr_index < 1024*8) { 5362 unsigned char b; 5363 if (kvm_vcpu_read_guest(vcpu, bitmap + msr_index/8, &b, 1)) 5364 return true; 5365 return 1 & (b >> (msr_index & 7)); 5366 } else 5367 return true; /* let L1 handle the wrong parameter */ 5368 } 5369 5370 /* 5371 * Return 1 if we should exit from L2 to L1 to handle a CR access exit, 5372 * rather than handle it ourselves in L0. I.e., check if L1 wanted to 5373 * intercept (via guest_host_mask etc.) the current event. 5374 */ 5375 static bool nested_vmx_exit_handled_cr(struct kvm_vcpu *vcpu, 5376 struct vmcs12 *vmcs12) 5377 { 5378 unsigned long exit_qualification = vmcs_readl(EXIT_QUALIFICATION); 5379 int cr = exit_qualification & 15; 5380 int reg; 5381 unsigned long val; 5382 5383 switch ((exit_qualification >> 4) & 3) { 5384 case 0: /* mov to cr */ 5385 reg = (exit_qualification >> 8) & 15; 5386 val = kvm_register_readl(vcpu, reg); 5387 switch (cr) { 5388 case 0: 5389 if (vmcs12->cr0_guest_host_mask & 5390 (val ^ vmcs12->cr0_read_shadow)) 5391 return true; 5392 break; 5393 case 3: 5394 if ((vmcs12->cr3_target_count >= 1 && 5395 vmcs12->cr3_target_value0 == val) || 5396 (vmcs12->cr3_target_count >= 2 && 5397 vmcs12->cr3_target_value1 == val) || 5398 (vmcs12->cr3_target_count >= 3 && 5399 vmcs12->cr3_target_value2 == val) || 5400 (vmcs12->cr3_target_count >= 4 && 5401 vmcs12->cr3_target_value3 == val)) 5402 return false; 5403 if (nested_cpu_has(vmcs12, CPU_BASED_CR3_LOAD_EXITING)) 5404 return true; 5405 break; 5406 case 4: 5407 if (vmcs12->cr4_guest_host_mask & 5408 (vmcs12->cr4_read_shadow ^ val)) 5409 return true; 5410 break; 5411 case 8: 5412 if (nested_cpu_has(vmcs12, CPU_BASED_CR8_LOAD_EXITING)) 5413 return true; 5414 break; 5415 } 5416 break; 5417 case 2: /* clts */ 5418 if ((vmcs12->cr0_guest_host_mask & X86_CR0_TS) && 5419 (vmcs12->cr0_read_shadow & X86_CR0_TS)) 5420 return true; 5421 break; 5422 case 1: /* mov from cr */ 5423 switch (cr) { 5424 case 3: 5425 if (vmcs12->cpu_based_vm_exec_control & 5426 CPU_BASED_CR3_STORE_EXITING) 5427 return true; 5428 break; 5429 case 8: 5430 if (vmcs12->cpu_based_vm_exec_control & 5431 CPU_BASED_CR8_STORE_EXITING) 5432 return true; 5433 break; 5434 } 5435 break; 5436 case 3: /* lmsw */ 5437 /* 5438 * lmsw can change bits 1..3 of cr0, and only set bit 0 of 5439 * cr0. Other attempted changes are ignored, with no exit. 5440 */ 5441 val = (exit_qualification >> LMSW_SOURCE_DATA_SHIFT) & 0x0f; 5442 if (vmcs12->cr0_guest_host_mask & 0xe & 5443 (val ^ vmcs12->cr0_read_shadow)) 5444 return true; 5445 if ((vmcs12->cr0_guest_host_mask & 0x1) && 5446 !(vmcs12->cr0_read_shadow & 0x1) && 5447 (val & 0x1)) 5448 return true; 5449 break; 5450 } 5451 return false; 5452 } 5453 5454 static bool nested_vmx_exit_handled_vmcs_access(struct kvm_vcpu *vcpu, 5455 struct vmcs12 *vmcs12, gpa_t bitmap) 5456 { 5457 u32 vmx_instruction_info; 5458 unsigned long field; 5459 u8 b; 5460 5461 if (!nested_cpu_has_shadow_vmcs(vmcs12)) 5462 return true; 5463 5464 /* Decode instruction info and find the field to access */ 5465 vmx_instruction_info = vmcs_read32(VMX_INSTRUCTION_INFO); 5466 field = kvm_register_read(vcpu, (((vmx_instruction_info) >> 28) & 0xf)); 5467 5468 /* Out-of-range fields always cause a VM exit from L2 to L1 */ 5469 if (field >> 15) 5470 return true; 5471 5472 if (kvm_vcpu_read_guest(vcpu, bitmap + field/8, &b, 1)) 5473 return true; 5474 5475 return 1 & (b >> (field & 7)); 5476 } 5477 5478 /* 5479 * Return 1 if we should exit from L2 to L1 to handle an exit, or 0 if we 5480 * should handle it ourselves in L0 (and then continue L2). Only call this 5481 * when in is_guest_mode (L2). 5482 */ 5483 bool nested_vmx_exit_reflected(struct kvm_vcpu *vcpu, u32 exit_reason) 5484 { 5485 u32 intr_info = vmcs_read32(VM_EXIT_INTR_INFO); 5486 struct vcpu_vmx *vmx = to_vmx(vcpu); 5487 struct vmcs12 *vmcs12 = get_vmcs12(vcpu); 5488 5489 if (vmx->nested.nested_run_pending) 5490 return false; 5491 5492 if (unlikely(vmx->fail)) { 5493 trace_kvm_nested_vmenter_failed( 5494 "hardware VM-instruction error: ", 5495 vmcs_read32(VM_INSTRUCTION_ERROR)); 5496 return true; 5497 } 5498 5499 /* 5500 * The host physical addresses of some pages of guest memory 5501 * are loaded into the vmcs02 (e.g. vmcs12's Virtual APIC 5502 * Page). The CPU may write to these pages via their host 5503 * physical address while L2 is running, bypassing any 5504 * address-translation-based dirty tracking (e.g. EPT write 5505 * protection). 5506 * 5507 * Mark them dirty on every exit from L2 to prevent them from 5508 * getting out of sync with dirty tracking. 5509 */ 5510 nested_mark_vmcs12_pages_dirty(vcpu); 5511 5512 trace_kvm_nested_vmexit(kvm_rip_read(vcpu), exit_reason, 5513 vmcs_readl(EXIT_QUALIFICATION), 5514 vmx->idt_vectoring_info, 5515 intr_info, 5516 vmcs_read32(VM_EXIT_INTR_ERROR_CODE), 5517 KVM_ISA_VMX); 5518 5519 switch (exit_reason) { 5520 case EXIT_REASON_EXCEPTION_NMI: 5521 if (is_nmi(intr_info)) 5522 return false; 5523 else if (is_page_fault(intr_info)) 5524 return !vmx->vcpu.arch.apf.host_apf_reason && enable_ept; 5525 else if (is_debug(intr_info) && 5526 vcpu->guest_debug & 5527 (KVM_GUESTDBG_SINGLESTEP | KVM_GUESTDBG_USE_HW_BP)) 5528 return false; 5529 else if (is_breakpoint(intr_info) && 5530 vcpu->guest_debug & KVM_GUESTDBG_USE_SW_BP) 5531 return false; 5532 return vmcs12->exception_bitmap & 5533 (1u << (intr_info & INTR_INFO_VECTOR_MASK)); 5534 case EXIT_REASON_EXTERNAL_INTERRUPT: 5535 return false; 5536 case EXIT_REASON_TRIPLE_FAULT: 5537 return true; 5538 case EXIT_REASON_INTERRUPT_WINDOW: 5539 return nested_cpu_has(vmcs12, CPU_BASED_INTR_WINDOW_EXITING); 5540 case EXIT_REASON_NMI_WINDOW: 5541 return nested_cpu_has(vmcs12, CPU_BASED_NMI_WINDOW_EXITING); 5542 case EXIT_REASON_TASK_SWITCH: 5543 return true; 5544 case EXIT_REASON_CPUID: 5545 return true; 5546 case EXIT_REASON_HLT: 5547 return nested_cpu_has(vmcs12, CPU_BASED_HLT_EXITING); 5548 case EXIT_REASON_INVD: 5549 return true; 5550 case EXIT_REASON_INVLPG: 5551 return nested_cpu_has(vmcs12, CPU_BASED_INVLPG_EXITING); 5552 case EXIT_REASON_RDPMC: 5553 return nested_cpu_has(vmcs12, CPU_BASED_RDPMC_EXITING); 5554 case EXIT_REASON_RDRAND: 5555 return nested_cpu_has2(vmcs12, SECONDARY_EXEC_RDRAND_EXITING); 5556 case EXIT_REASON_RDSEED: 5557 return nested_cpu_has2(vmcs12, SECONDARY_EXEC_RDSEED_EXITING); 5558 case EXIT_REASON_RDTSC: case EXIT_REASON_RDTSCP: 5559 return nested_cpu_has(vmcs12, CPU_BASED_RDTSC_EXITING); 5560 case EXIT_REASON_VMREAD: 5561 return nested_vmx_exit_handled_vmcs_access(vcpu, vmcs12, 5562 vmcs12->vmread_bitmap); 5563 case EXIT_REASON_VMWRITE: 5564 return nested_vmx_exit_handled_vmcs_access(vcpu, vmcs12, 5565 vmcs12->vmwrite_bitmap); 5566 case EXIT_REASON_VMCALL: case EXIT_REASON_VMCLEAR: 5567 case EXIT_REASON_VMLAUNCH: case EXIT_REASON_VMPTRLD: 5568 case EXIT_REASON_VMPTRST: case EXIT_REASON_VMRESUME: 5569 case EXIT_REASON_VMOFF: case EXIT_REASON_VMON: 5570 case EXIT_REASON_INVEPT: case EXIT_REASON_INVVPID: 5571 /* 5572 * VMX instructions trap unconditionally. This allows L1 to 5573 * emulate them for its L2 guest, i.e., allows 3-level nesting! 5574 */ 5575 return true; 5576 case EXIT_REASON_CR_ACCESS: 5577 return nested_vmx_exit_handled_cr(vcpu, vmcs12); 5578 case EXIT_REASON_DR_ACCESS: 5579 return nested_cpu_has(vmcs12, CPU_BASED_MOV_DR_EXITING); 5580 case EXIT_REASON_IO_INSTRUCTION: 5581 return nested_vmx_exit_handled_io(vcpu, vmcs12); 5582 case EXIT_REASON_GDTR_IDTR: case EXIT_REASON_LDTR_TR: 5583 return nested_cpu_has2(vmcs12, SECONDARY_EXEC_DESC); 5584 case EXIT_REASON_MSR_READ: 5585 case EXIT_REASON_MSR_WRITE: 5586 return nested_vmx_exit_handled_msr(vcpu, vmcs12, exit_reason); 5587 case EXIT_REASON_INVALID_STATE: 5588 return true; 5589 case EXIT_REASON_MWAIT_INSTRUCTION: 5590 return nested_cpu_has(vmcs12, CPU_BASED_MWAIT_EXITING); 5591 case EXIT_REASON_MONITOR_TRAP_FLAG: 5592 return nested_cpu_has(vmcs12, CPU_BASED_MONITOR_TRAP_FLAG); 5593 case EXIT_REASON_MONITOR_INSTRUCTION: 5594 return nested_cpu_has(vmcs12, CPU_BASED_MONITOR_EXITING); 5595 case EXIT_REASON_PAUSE_INSTRUCTION: 5596 return nested_cpu_has(vmcs12, CPU_BASED_PAUSE_EXITING) || 5597 nested_cpu_has2(vmcs12, 5598 SECONDARY_EXEC_PAUSE_LOOP_EXITING); 5599 case EXIT_REASON_MCE_DURING_VMENTRY: 5600 return false; 5601 case EXIT_REASON_TPR_BELOW_THRESHOLD: 5602 return nested_cpu_has(vmcs12, CPU_BASED_TPR_SHADOW); 5603 case EXIT_REASON_APIC_ACCESS: 5604 case EXIT_REASON_APIC_WRITE: 5605 case EXIT_REASON_EOI_INDUCED: 5606 /* 5607 * The controls for "virtualize APIC accesses," "APIC- 5608 * register virtualization," and "virtual-interrupt 5609 * delivery" only come from vmcs12. 5610 */ 5611 return true; 5612 case EXIT_REASON_EPT_VIOLATION: 5613 /* 5614 * L0 always deals with the EPT violation. If nested EPT is 5615 * used, and the nested mmu code discovers that the address is 5616 * missing in the guest EPT table (EPT12), the EPT violation 5617 * will be injected with nested_ept_inject_page_fault() 5618 */ 5619 return false; 5620 case EXIT_REASON_EPT_MISCONFIG: 5621 /* 5622 * L2 never uses directly L1's EPT, but rather L0's own EPT 5623 * table (shadow on EPT) or a merged EPT table that L0 built 5624 * (EPT on EPT). So any problems with the structure of the 5625 * table is L0's fault. 5626 */ 5627 return false; 5628 case EXIT_REASON_INVPCID: 5629 return 5630 nested_cpu_has2(vmcs12, SECONDARY_EXEC_ENABLE_INVPCID) && 5631 nested_cpu_has(vmcs12, CPU_BASED_INVLPG_EXITING); 5632 case EXIT_REASON_WBINVD: 5633 return nested_cpu_has2(vmcs12, SECONDARY_EXEC_WBINVD_EXITING); 5634 case EXIT_REASON_XSETBV: 5635 return true; 5636 case EXIT_REASON_XSAVES: case EXIT_REASON_XRSTORS: 5637 /* 5638 * This should never happen, since it is not possible to 5639 * set XSS to a non-zero value---neither in L1 nor in L2. 5640 * If if it were, XSS would have to be checked against 5641 * the XSS exit bitmap in vmcs12. 5642 */ 5643 return nested_cpu_has2(vmcs12, SECONDARY_EXEC_XSAVES); 5644 case EXIT_REASON_PREEMPTION_TIMER: 5645 return false; 5646 case EXIT_REASON_PML_FULL: 5647 /* We emulate PML support to L1. */ 5648 return false; 5649 case EXIT_REASON_VMFUNC: 5650 /* VM functions are emulated through L2->L0 vmexits. */ 5651 return false; 5652 case EXIT_REASON_ENCLS: 5653 /* SGX is never exposed to L1 */ 5654 return false; 5655 case EXIT_REASON_UMWAIT: 5656 case EXIT_REASON_TPAUSE: 5657 return nested_cpu_has2(vmcs12, 5658 SECONDARY_EXEC_ENABLE_USR_WAIT_PAUSE); 5659 default: 5660 return true; 5661 } 5662 } 5663 5664 5665 static int vmx_get_nested_state(struct kvm_vcpu *vcpu, 5666 struct kvm_nested_state __user *user_kvm_nested_state, 5667 u32 user_data_size) 5668 { 5669 struct vcpu_vmx *vmx; 5670 struct vmcs12 *vmcs12; 5671 struct kvm_nested_state kvm_state = { 5672 .flags = 0, 5673 .format = KVM_STATE_NESTED_FORMAT_VMX, 5674 .size = sizeof(kvm_state), 5675 .hdr.vmx.vmxon_pa = -1ull, 5676 .hdr.vmx.vmcs12_pa = -1ull, 5677 }; 5678 struct kvm_vmx_nested_state_data __user *user_vmx_nested_state = 5679 &user_kvm_nested_state->data.vmx[0]; 5680 5681 if (!vcpu) 5682 return kvm_state.size + sizeof(*user_vmx_nested_state); 5683 5684 vmx = to_vmx(vcpu); 5685 vmcs12 = get_vmcs12(vcpu); 5686 5687 if (nested_vmx_allowed(vcpu) && 5688 (vmx->nested.vmxon || vmx->nested.smm.vmxon)) { 5689 kvm_state.hdr.vmx.vmxon_pa = vmx->nested.vmxon_ptr; 5690 kvm_state.hdr.vmx.vmcs12_pa = vmx->nested.current_vmptr; 5691 5692 if (vmx_has_valid_vmcs12(vcpu)) { 5693 kvm_state.size += sizeof(user_vmx_nested_state->vmcs12); 5694 5695 if (vmx->nested.hv_evmcs) 5696 kvm_state.flags |= KVM_STATE_NESTED_EVMCS; 5697 5698 if (is_guest_mode(vcpu) && 5699 nested_cpu_has_shadow_vmcs(vmcs12) && 5700 vmcs12->vmcs_link_pointer != -1ull) 5701 kvm_state.size += sizeof(user_vmx_nested_state->shadow_vmcs12); 5702 } 5703 5704 if (vmx->nested.smm.vmxon) 5705 kvm_state.hdr.vmx.smm.flags |= KVM_STATE_NESTED_SMM_VMXON; 5706 5707 if (vmx->nested.smm.guest_mode) 5708 kvm_state.hdr.vmx.smm.flags |= KVM_STATE_NESTED_SMM_GUEST_MODE; 5709 5710 if (is_guest_mode(vcpu)) { 5711 kvm_state.flags |= KVM_STATE_NESTED_GUEST_MODE; 5712 5713 if (vmx->nested.nested_run_pending) 5714 kvm_state.flags |= KVM_STATE_NESTED_RUN_PENDING; 5715 } 5716 } 5717 5718 if (user_data_size < kvm_state.size) 5719 goto out; 5720 5721 if (copy_to_user(user_kvm_nested_state, &kvm_state, sizeof(kvm_state))) 5722 return -EFAULT; 5723 5724 if (!vmx_has_valid_vmcs12(vcpu)) 5725 goto out; 5726 5727 /* 5728 * When running L2, the authoritative vmcs12 state is in the 5729 * vmcs02. When running L1, the authoritative vmcs12 state is 5730 * in the shadow or enlightened vmcs linked to vmcs01, unless 5731 * need_vmcs12_to_shadow_sync is set, in which case, the authoritative 5732 * vmcs12 state is in the vmcs12 already. 5733 */ 5734 if (is_guest_mode(vcpu)) { 5735 sync_vmcs02_to_vmcs12(vcpu, vmcs12); 5736 sync_vmcs02_to_vmcs12_rare(vcpu, vmcs12); 5737 } else if (!vmx->nested.need_vmcs12_to_shadow_sync) { 5738 if (vmx->nested.hv_evmcs) 5739 copy_enlightened_to_vmcs12(vmx); 5740 else if (enable_shadow_vmcs) 5741 copy_shadow_to_vmcs12(vmx); 5742 } 5743 5744 BUILD_BUG_ON(sizeof(user_vmx_nested_state->vmcs12) < VMCS12_SIZE); 5745 BUILD_BUG_ON(sizeof(user_vmx_nested_state->shadow_vmcs12) < VMCS12_SIZE); 5746 5747 /* 5748 * Copy over the full allocated size of vmcs12 rather than just the size 5749 * of the struct. 5750 */ 5751 if (copy_to_user(user_vmx_nested_state->vmcs12, vmcs12, VMCS12_SIZE)) 5752 return -EFAULT; 5753 5754 if (nested_cpu_has_shadow_vmcs(vmcs12) && 5755 vmcs12->vmcs_link_pointer != -1ull) { 5756 if (copy_to_user(user_vmx_nested_state->shadow_vmcs12, 5757 get_shadow_vmcs12(vcpu), VMCS12_SIZE)) 5758 return -EFAULT; 5759 } 5760 5761 out: 5762 return kvm_state.size; 5763 } 5764 5765 /* 5766 * Forcibly leave nested mode in order to be able to reset the VCPU later on. 5767 */ 5768 void vmx_leave_nested(struct kvm_vcpu *vcpu) 5769 { 5770 if (is_guest_mode(vcpu)) { 5771 to_vmx(vcpu)->nested.nested_run_pending = 0; 5772 nested_vmx_vmexit(vcpu, -1, 0, 0); 5773 } 5774 free_nested(vcpu); 5775 } 5776 5777 static int vmx_set_nested_state(struct kvm_vcpu *vcpu, 5778 struct kvm_nested_state __user *user_kvm_nested_state, 5779 struct kvm_nested_state *kvm_state) 5780 { 5781 struct vcpu_vmx *vmx = to_vmx(vcpu); 5782 struct vmcs12 *vmcs12; 5783 u32 exit_qual; 5784 struct kvm_vmx_nested_state_data __user *user_vmx_nested_state = 5785 &user_kvm_nested_state->data.vmx[0]; 5786 int ret; 5787 5788 if (kvm_state->format != KVM_STATE_NESTED_FORMAT_VMX) 5789 return -EINVAL; 5790 5791 if (kvm_state->hdr.vmx.vmxon_pa == -1ull) { 5792 if (kvm_state->hdr.vmx.smm.flags) 5793 return -EINVAL; 5794 5795 if (kvm_state->hdr.vmx.vmcs12_pa != -1ull) 5796 return -EINVAL; 5797 5798 /* 5799 * KVM_STATE_NESTED_EVMCS used to signal that KVM should 5800 * enable eVMCS capability on vCPU. However, since then 5801 * code was changed such that flag signals vmcs12 should 5802 * be copied into eVMCS in guest memory. 5803 * 5804 * To preserve backwards compatability, allow user 5805 * to set this flag even when there is no VMXON region. 5806 */ 5807 if (kvm_state->flags & ~KVM_STATE_NESTED_EVMCS) 5808 return -EINVAL; 5809 } else { 5810 if (!nested_vmx_allowed(vcpu)) 5811 return -EINVAL; 5812 5813 if (!page_address_valid(vcpu, kvm_state->hdr.vmx.vmxon_pa)) 5814 return -EINVAL; 5815 } 5816 5817 if ((kvm_state->hdr.vmx.smm.flags & KVM_STATE_NESTED_SMM_GUEST_MODE) && 5818 (kvm_state->flags & KVM_STATE_NESTED_GUEST_MODE)) 5819 return -EINVAL; 5820 5821 if (kvm_state->hdr.vmx.smm.flags & 5822 ~(KVM_STATE_NESTED_SMM_GUEST_MODE | KVM_STATE_NESTED_SMM_VMXON)) 5823 return -EINVAL; 5824 5825 /* 5826 * SMM temporarily disables VMX, so we cannot be in guest mode, 5827 * nor can VMLAUNCH/VMRESUME be pending. Outside SMM, SMM flags 5828 * must be zero. 5829 */ 5830 if (is_smm(vcpu) ? 5831 (kvm_state->flags & 5832 (KVM_STATE_NESTED_GUEST_MODE | KVM_STATE_NESTED_RUN_PENDING)) 5833 : kvm_state->hdr.vmx.smm.flags) 5834 return -EINVAL; 5835 5836 if ((kvm_state->hdr.vmx.smm.flags & KVM_STATE_NESTED_SMM_GUEST_MODE) && 5837 !(kvm_state->hdr.vmx.smm.flags & KVM_STATE_NESTED_SMM_VMXON)) 5838 return -EINVAL; 5839 5840 if ((kvm_state->flags & KVM_STATE_NESTED_EVMCS) && 5841 (!nested_vmx_allowed(vcpu) || !vmx->nested.enlightened_vmcs_enabled)) 5842 return -EINVAL; 5843 5844 vmx_leave_nested(vcpu); 5845 5846 if (kvm_state->hdr.vmx.vmxon_pa == -1ull) 5847 return 0; 5848 5849 vmx->nested.vmxon_ptr = kvm_state->hdr.vmx.vmxon_pa; 5850 ret = enter_vmx_operation(vcpu); 5851 if (ret) 5852 return ret; 5853 5854 /* Empty 'VMXON' state is permitted */ 5855 if (kvm_state->size < sizeof(*kvm_state) + sizeof(*vmcs12)) 5856 return 0; 5857 5858 if (kvm_state->hdr.vmx.vmcs12_pa != -1ull) { 5859 if (kvm_state->hdr.vmx.vmcs12_pa == kvm_state->hdr.vmx.vmxon_pa || 5860 !page_address_valid(vcpu, kvm_state->hdr.vmx.vmcs12_pa)) 5861 return -EINVAL; 5862 5863 set_current_vmptr(vmx, kvm_state->hdr.vmx.vmcs12_pa); 5864 } else if (kvm_state->flags & KVM_STATE_NESTED_EVMCS) { 5865 /* 5866 * Sync eVMCS upon entry as we may not have 5867 * HV_X64_MSR_VP_ASSIST_PAGE set up yet. 5868 */ 5869 vmx->nested.need_vmcs12_to_shadow_sync = true; 5870 } else { 5871 return -EINVAL; 5872 } 5873 5874 if (kvm_state->hdr.vmx.smm.flags & KVM_STATE_NESTED_SMM_VMXON) { 5875 vmx->nested.smm.vmxon = true; 5876 vmx->nested.vmxon = false; 5877 5878 if (kvm_state->hdr.vmx.smm.flags & KVM_STATE_NESTED_SMM_GUEST_MODE) 5879 vmx->nested.smm.guest_mode = true; 5880 } 5881 5882 vmcs12 = get_vmcs12(vcpu); 5883 if (copy_from_user(vmcs12, user_vmx_nested_state->vmcs12, sizeof(*vmcs12))) 5884 return -EFAULT; 5885 5886 if (vmcs12->hdr.revision_id != VMCS12_REVISION) 5887 return -EINVAL; 5888 5889 if (!(kvm_state->flags & KVM_STATE_NESTED_GUEST_MODE)) 5890 return 0; 5891 5892 vmx->nested.nested_run_pending = 5893 !!(kvm_state->flags & KVM_STATE_NESTED_RUN_PENDING); 5894 5895 ret = -EINVAL; 5896 if (nested_cpu_has_shadow_vmcs(vmcs12) && 5897 vmcs12->vmcs_link_pointer != -1ull) { 5898 struct vmcs12 *shadow_vmcs12 = get_shadow_vmcs12(vcpu); 5899 5900 if (kvm_state->size < 5901 sizeof(*kvm_state) + 5902 sizeof(user_vmx_nested_state->vmcs12) + sizeof(*shadow_vmcs12)) 5903 goto error_guest_mode; 5904 5905 if (copy_from_user(shadow_vmcs12, 5906 user_vmx_nested_state->shadow_vmcs12, 5907 sizeof(*shadow_vmcs12))) { 5908 ret = -EFAULT; 5909 goto error_guest_mode; 5910 } 5911 5912 if (shadow_vmcs12->hdr.revision_id != VMCS12_REVISION || 5913 !shadow_vmcs12->hdr.shadow_vmcs) 5914 goto error_guest_mode; 5915 } 5916 5917 if (nested_vmx_check_controls(vcpu, vmcs12) || 5918 nested_vmx_check_host_state(vcpu, vmcs12) || 5919 nested_vmx_check_guest_state(vcpu, vmcs12, &exit_qual)) 5920 goto error_guest_mode; 5921 5922 vmx->nested.dirty_vmcs12 = true; 5923 ret = nested_vmx_enter_non_root_mode(vcpu, false); 5924 if (ret) 5925 goto error_guest_mode; 5926 5927 return 0; 5928 5929 error_guest_mode: 5930 vmx->nested.nested_run_pending = 0; 5931 return ret; 5932 } 5933 5934 void nested_vmx_set_vmcs_shadowing_bitmap(void) 5935 { 5936 if (enable_shadow_vmcs) { 5937 vmcs_write64(VMREAD_BITMAP, __pa(vmx_vmread_bitmap)); 5938 vmcs_write64(VMWRITE_BITMAP, __pa(vmx_vmwrite_bitmap)); 5939 } 5940 } 5941 5942 /* 5943 * nested_vmx_setup_ctls_msrs() sets up variables containing the values to be 5944 * returned for the various VMX controls MSRs when nested VMX is enabled. 5945 * The same values should also be used to verify that vmcs12 control fields are 5946 * valid during nested entry from L1 to L2. 5947 * Each of these control msrs has a low and high 32-bit half: A low bit is on 5948 * if the corresponding bit in the (32-bit) control field *must* be on, and a 5949 * bit in the high half is on if the corresponding bit in the control field 5950 * may be on. See also vmx_control_verify(). 5951 */ 5952 void nested_vmx_setup_ctls_msrs(struct nested_vmx_msrs *msrs, u32 ept_caps, 5953 bool apicv) 5954 { 5955 /* 5956 * Note that as a general rule, the high half of the MSRs (bits in 5957 * the control fields which may be 1) should be initialized by the 5958 * intersection of the underlying hardware's MSR (i.e., features which 5959 * can be supported) and the list of features we want to expose - 5960 * because they are known to be properly supported in our code. 5961 * Also, usually, the low half of the MSRs (bits which must be 1) can 5962 * be set to 0, meaning that L1 may turn off any of these bits. The 5963 * reason is that if one of these bits is necessary, it will appear 5964 * in vmcs01 and prepare_vmcs02, when it bitwise-or's the control 5965 * fields of vmcs01 and vmcs02, will turn these bits off - and 5966 * nested_vmx_exit_reflected() will not pass related exits to L1. 5967 * These rules have exceptions below. 5968 */ 5969 5970 /* pin-based controls */ 5971 rdmsr(MSR_IA32_VMX_PINBASED_CTLS, 5972 msrs->pinbased_ctls_low, 5973 msrs->pinbased_ctls_high); 5974 msrs->pinbased_ctls_low |= 5975 PIN_BASED_ALWAYSON_WITHOUT_TRUE_MSR; 5976 msrs->pinbased_ctls_high &= 5977 PIN_BASED_EXT_INTR_MASK | 5978 PIN_BASED_NMI_EXITING | 5979 PIN_BASED_VIRTUAL_NMIS | 5980 (apicv ? PIN_BASED_POSTED_INTR : 0); 5981 msrs->pinbased_ctls_high |= 5982 PIN_BASED_ALWAYSON_WITHOUT_TRUE_MSR | 5983 PIN_BASED_VMX_PREEMPTION_TIMER; 5984 5985 /* exit controls */ 5986 rdmsr(MSR_IA32_VMX_EXIT_CTLS, 5987 msrs->exit_ctls_low, 5988 msrs->exit_ctls_high); 5989 msrs->exit_ctls_low = 5990 VM_EXIT_ALWAYSON_WITHOUT_TRUE_MSR; 5991 5992 msrs->exit_ctls_high &= 5993 #ifdef CONFIG_X86_64 5994 VM_EXIT_HOST_ADDR_SPACE_SIZE | 5995 #endif 5996 VM_EXIT_LOAD_IA32_PAT | VM_EXIT_SAVE_IA32_PAT; 5997 msrs->exit_ctls_high |= 5998 VM_EXIT_ALWAYSON_WITHOUT_TRUE_MSR | 5999 VM_EXIT_LOAD_IA32_EFER | VM_EXIT_SAVE_IA32_EFER | 6000 VM_EXIT_SAVE_VMX_PREEMPTION_TIMER | VM_EXIT_ACK_INTR_ON_EXIT; 6001 6002 /* We support free control of debug control saving. */ 6003 msrs->exit_ctls_low &= ~VM_EXIT_SAVE_DEBUG_CONTROLS; 6004 6005 /* entry controls */ 6006 rdmsr(MSR_IA32_VMX_ENTRY_CTLS, 6007 msrs->entry_ctls_low, 6008 msrs->entry_ctls_high); 6009 msrs->entry_ctls_low = 6010 VM_ENTRY_ALWAYSON_WITHOUT_TRUE_MSR; 6011 msrs->entry_ctls_high &= 6012 #ifdef CONFIG_X86_64 6013 VM_ENTRY_IA32E_MODE | 6014 #endif 6015 VM_ENTRY_LOAD_IA32_PAT; 6016 msrs->entry_ctls_high |= 6017 (VM_ENTRY_ALWAYSON_WITHOUT_TRUE_MSR | VM_ENTRY_LOAD_IA32_EFER); 6018 6019 /* We support free control of debug control loading. */ 6020 msrs->entry_ctls_low &= ~VM_ENTRY_LOAD_DEBUG_CONTROLS; 6021 6022 /* cpu-based controls */ 6023 rdmsr(MSR_IA32_VMX_PROCBASED_CTLS, 6024 msrs->procbased_ctls_low, 6025 msrs->procbased_ctls_high); 6026 msrs->procbased_ctls_low = 6027 CPU_BASED_ALWAYSON_WITHOUT_TRUE_MSR; 6028 msrs->procbased_ctls_high &= 6029 CPU_BASED_INTR_WINDOW_EXITING | 6030 CPU_BASED_NMI_WINDOW_EXITING | CPU_BASED_USE_TSC_OFFSETTING | 6031 CPU_BASED_HLT_EXITING | CPU_BASED_INVLPG_EXITING | 6032 CPU_BASED_MWAIT_EXITING | CPU_BASED_CR3_LOAD_EXITING | 6033 CPU_BASED_CR3_STORE_EXITING | 6034 #ifdef CONFIG_X86_64 6035 CPU_BASED_CR8_LOAD_EXITING | CPU_BASED_CR8_STORE_EXITING | 6036 #endif 6037 CPU_BASED_MOV_DR_EXITING | CPU_BASED_UNCOND_IO_EXITING | 6038 CPU_BASED_USE_IO_BITMAPS | CPU_BASED_MONITOR_TRAP_FLAG | 6039 CPU_BASED_MONITOR_EXITING | CPU_BASED_RDPMC_EXITING | 6040 CPU_BASED_RDTSC_EXITING | CPU_BASED_PAUSE_EXITING | 6041 CPU_BASED_TPR_SHADOW | CPU_BASED_ACTIVATE_SECONDARY_CONTROLS; 6042 /* 6043 * We can allow some features even when not supported by the 6044 * hardware. For example, L1 can specify an MSR bitmap - and we 6045 * can use it to avoid exits to L1 - even when L0 runs L2 6046 * without MSR bitmaps. 6047 */ 6048 msrs->procbased_ctls_high |= 6049 CPU_BASED_ALWAYSON_WITHOUT_TRUE_MSR | 6050 CPU_BASED_USE_MSR_BITMAPS; 6051 6052 /* We support free control of CR3 access interception. */ 6053 msrs->procbased_ctls_low &= 6054 ~(CPU_BASED_CR3_LOAD_EXITING | CPU_BASED_CR3_STORE_EXITING); 6055 6056 /* 6057 * secondary cpu-based controls. Do not include those that 6058 * depend on CPUID bits, they are added later by vmx_cpuid_update. 6059 */ 6060 if (msrs->procbased_ctls_high & CPU_BASED_ACTIVATE_SECONDARY_CONTROLS) 6061 rdmsr(MSR_IA32_VMX_PROCBASED_CTLS2, 6062 msrs->secondary_ctls_low, 6063 msrs->secondary_ctls_high); 6064 6065 msrs->secondary_ctls_low = 0; 6066 msrs->secondary_ctls_high &= 6067 SECONDARY_EXEC_DESC | 6068 SECONDARY_EXEC_RDTSCP | 6069 SECONDARY_EXEC_VIRTUALIZE_X2APIC_MODE | 6070 SECONDARY_EXEC_WBINVD_EXITING | 6071 SECONDARY_EXEC_APIC_REGISTER_VIRT | 6072 SECONDARY_EXEC_VIRTUAL_INTR_DELIVERY | 6073 SECONDARY_EXEC_RDRAND_EXITING | 6074 SECONDARY_EXEC_ENABLE_INVPCID | 6075 SECONDARY_EXEC_RDSEED_EXITING | 6076 SECONDARY_EXEC_XSAVES; 6077 6078 /* 6079 * We can emulate "VMCS shadowing," even if the hardware 6080 * doesn't support it. 6081 */ 6082 msrs->secondary_ctls_high |= 6083 SECONDARY_EXEC_SHADOW_VMCS; 6084 6085 if (enable_ept) { 6086 /* nested EPT: emulate EPT also to L1 */ 6087 msrs->secondary_ctls_high |= 6088 SECONDARY_EXEC_ENABLE_EPT; 6089 msrs->ept_caps = VMX_EPT_PAGE_WALK_4_BIT | 6090 VMX_EPTP_WB_BIT | VMX_EPT_INVEPT_BIT; 6091 if (cpu_has_vmx_ept_execute_only()) 6092 msrs->ept_caps |= 6093 VMX_EPT_EXECUTE_ONLY_BIT; 6094 msrs->ept_caps &= ept_caps; 6095 msrs->ept_caps |= VMX_EPT_EXTENT_GLOBAL_BIT | 6096 VMX_EPT_EXTENT_CONTEXT_BIT | VMX_EPT_2MB_PAGE_BIT | 6097 VMX_EPT_1GB_PAGE_BIT; 6098 if (enable_ept_ad_bits) { 6099 msrs->secondary_ctls_high |= 6100 SECONDARY_EXEC_ENABLE_PML; 6101 msrs->ept_caps |= VMX_EPT_AD_BIT; 6102 } 6103 } 6104 6105 if (cpu_has_vmx_vmfunc()) { 6106 msrs->secondary_ctls_high |= 6107 SECONDARY_EXEC_ENABLE_VMFUNC; 6108 /* 6109 * Advertise EPTP switching unconditionally 6110 * since we emulate it 6111 */ 6112 if (enable_ept) 6113 msrs->vmfunc_controls = 6114 VMX_VMFUNC_EPTP_SWITCHING; 6115 } 6116 6117 /* 6118 * Old versions of KVM use the single-context version without 6119 * checking for support, so declare that it is supported even 6120 * though it is treated as global context. The alternative is 6121 * not failing the single-context invvpid, and it is worse. 6122 */ 6123 if (enable_vpid) { 6124 msrs->secondary_ctls_high |= 6125 SECONDARY_EXEC_ENABLE_VPID; 6126 msrs->vpid_caps = VMX_VPID_INVVPID_BIT | 6127 VMX_VPID_EXTENT_SUPPORTED_MASK; 6128 } 6129 6130 if (enable_unrestricted_guest) 6131 msrs->secondary_ctls_high |= 6132 SECONDARY_EXEC_UNRESTRICTED_GUEST; 6133 6134 if (flexpriority_enabled) 6135 msrs->secondary_ctls_high |= 6136 SECONDARY_EXEC_VIRTUALIZE_APIC_ACCESSES; 6137 6138 /* miscellaneous data */ 6139 rdmsr(MSR_IA32_VMX_MISC, 6140 msrs->misc_low, 6141 msrs->misc_high); 6142 msrs->misc_low &= VMX_MISC_SAVE_EFER_LMA; 6143 msrs->misc_low |= 6144 MSR_IA32_VMX_MISC_VMWRITE_SHADOW_RO_FIELDS | 6145 VMX_MISC_EMULATED_PREEMPTION_TIMER_RATE | 6146 VMX_MISC_ACTIVITY_HLT; 6147 msrs->misc_high = 0; 6148 6149 /* 6150 * This MSR reports some information about VMX support. We 6151 * should return information about the VMX we emulate for the 6152 * guest, and the VMCS structure we give it - not about the 6153 * VMX support of the underlying hardware. 6154 */ 6155 msrs->basic = 6156 VMCS12_REVISION | 6157 VMX_BASIC_TRUE_CTLS | 6158 ((u64)VMCS12_SIZE << VMX_BASIC_VMCS_SIZE_SHIFT) | 6159 (VMX_BASIC_MEM_TYPE_WB << VMX_BASIC_MEM_TYPE_SHIFT); 6160 6161 if (cpu_has_vmx_basic_inout()) 6162 msrs->basic |= VMX_BASIC_INOUT; 6163 6164 /* 6165 * These MSRs specify bits which the guest must keep fixed on 6166 * while L1 is in VMXON mode (in L1's root mode, or running an L2). 6167 * We picked the standard core2 setting. 6168 */ 6169 #define VMXON_CR0_ALWAYSON (X86_CR0_PE | X86_CR0_PG | X86_CR0_NE) 6170 #define VMXON_CR4_ALWAYSON X86_CR4_VMXE 6171 msrs->cr0_fixed0 = VMXON_CR0_ALWAYSON; 6172 msrs->cr4_fixed0 = VMXON_CR4_ALWAYSON; 6173 6174 /* These MSRs specify bits which the guest must keep fixed off. */ 6175 rdmsrl(MSR_IA32_VMX_CR0_FIXED1, msrs->cr0_fixed1); 6176 rdmsrl(MSR_IA32_VMX_CR4_FIXED1, msrs->cr4_fixed1); 6177 6178 /* highest index: VMX_PREEMPTION_TIMER_VALUE */ 6179 msrs->vmcs_enum = VMCS12_MAX_FIELD_INDEX << 1; 6180 } 6181 6182 void nested_vmx_hardware_unsetup(void) 6183 { 6184 int i; 6185 6186 if (enable_shadow_vmcs) { 6187 for (i = 0; i < VMX_BITMAP_NR; i++) 6188 free_page((unsigned long)vmx_bitmap[i]); 6189 } 6190 } 6191 6192 __init int nested_vmx_hardware_setup(int (*exit_handlers[])(struct kvm_vcpu *)) 6193 { 6194 int i; 6195 6196 if (!cpu_has_vmx_shadow_vmcs()) 6197 enable_shadow_vmcs = 0; 6198 if (enable_shadow_vmcs) { 6199 for (i = 0; i < VMX_BITMAP_NR; i++) { 6200 /* 6201 * The vmx_bitmap is not tied to a VM and so should 6202 * not be charged to a memcg. 6203 */ 6204 vmx_bitmap[i] = (unsigned long *) 6205 __get_free_page(GFP_KERNEL); 6206 if (!vmx_bitmap[i]) { 6207 nested_vmx_hardware_unsetup(); 6208 return -ENOMEM; 6209 } 6210 } 6211 6212 init_vmcs_shadow_fields(); 6213 } 6214 6215 exit_handlers[EXIT_REASON_VMCLEAR] = handle_vmclear; 6216 exit_handlers[EXIT_REASON_VMLAUNCH] = handle_vmlaunch; 6217 exit_handlers[EXIT_REASON_VMPTRLD] = handle_vmptrld; 6218 exit_handlers[EXIT_REASON_VMPTRST] = handle_vmptrst; 6219 exit_handlers[EXIT_REASON_VMREAD] = handle_vmread; 6220 exit_handlers[EXIT_REASON_VMRESUME] = handle_vmresume; 6221 exit_handlers[EXIT_REASON_VMWRITE] = handle_vmwrite; 6222 exit_handlers[EXIT_REASON_VMOFF] = handle_vmoff; 6223 exit_handlers[EXIT_REASON_VMON] = handle_vmon; 6224 exit_handlers[EXIT_REASON_INVEPT] = handle_invept; 6225 exit_handlers[EXIT_REASON_INVVPID] = handle_invvpid; 6226 exit_handlers[EXIT_REASON_VMFUNC] = handle_vmfunc; 6227 6228 kvm_x86_ops->check_nested_events = vmx_check_nested_events; 6229 kvm_x86_ops->get_nested_state = vmx_get_nested_state; 6230 kvm_x86_ops->set_nested_state = vmx_set_nested_state; 6231 kvm_x86_ops->get_vmcs12_pages = nested_get_vmcs12_pages; 6232 kvm_x86_ops->nested_enable_evmcs = nested_enable_evmcs; 6233 kvm_x86_ops->nested_get_evmcs_version = nested_get_evmcs_version; 6234 6235 return 0; 6236 } 6237