1 // SPDX-License-Identifier: GPL-2.0 2 3 #include <linux/frame.h> 4 #include <linux/percpu.h> 5 6 #include <asm/debugreg.h> 7 #include <asm/mmu_context.h> 8 9 #include "cpuid.h" 10 #include "hyperv.h" 11 #include "mmu.h" 12 #include "nested.h" 13 #include "trace.h" 14 #include "x86.h" 15 16 static bool __read_mostly enable_shadow_vmcs = 1; 17 module_param_named(enable_shadow_vmcs, enable_shadow_vmcs, bool, S_IRUGO); 18 19 static bool __read_mostly nested_early_check = 0; 20 module_param(nested_early_check, bool, S_IRUGO); 21 22 /* 23 * Hyper-V requires all of these, so mark them as supported even though 24 * they are just treated the same as all-context. 25 */ 26 #define VMX_VPID_EXTENT_SUPPORTED_MASK \ 27 (VMX_VPID_EXTENT_INDIVIDUAL_ADDR_BIT | \ 28 VMX_VPID_EXTENT_SINGLE_CONTEXT_BIT | \ 29 VMX_VPID_EXTENT_GLOBAL_CONTEXT_BIT | \ 30 VMX_VPID_EXTENT_SINGLE_NON_GLOBAL_BIT) 31 32 #define VMX_MISC_EMULATED_PREEMPTION_TIMER_RATE 5 33 34 enum { 35 VMX_VMREAD_BITMAP, 36 VMX_VMWRITE_BITMAP, 37 VMX_BITMAP_NR 38 }; 39 static unsigned long *vmx_bitmap[VMX_BITMAP_NR]; 40 41 #define vmx_vmread_bitmap (vmx_bitmap[VMX_VMREAD_BITMAP]) 42 #define vmx_vmwrite_bitmap (vmx_bitmap[VMX_VMWRITE_BITMAP]) 43 44 static u16 shadow_read_only_fields[] = { 45 #define SHADOW_FIELD_RO(x) x, 46 #include "vmcs_shadow_fields.h" 47 }; 48 static int max_shadow_read_only_fields = 49 ARRAY_SIZE(shadow_read_only_fields); 50 51 static u16 shadow_read_write_fields[] = { 52 #define SHADOW_FIELD_RW(x) x, 53 #include "vmcs_shadow_fields.h" 54 }; 55 static int max_shadow_read_write_fields = 56 ARRAY_SIZE(shadow_read_write_fields); 57 58 static void init_vmcs_shadow_fields(void) 59 { 60 int i, j; 61 62 memset(vmx_vmread_bitmap, 0xff, PAGE_SIZE); 63 memset(vmx_vmwrite_bitmap, 0xff, PAGE_SIZE); 64 65 for (i = j = 0; i < max_shadow_read_only_fields; i++) { 66 u16 field = shadow_read_only_fields[i]; 67 68 if (vmcs_field_width(field) == VMCS_FIELD_WIDTH_U64 && 69 (i + 1 == max_shadow_read_only_fields || 70 shadow_read_only_fields[i + 1] != field + 1)) 71 pr_err("Missing field from shadow_read_only_field %x\n", 72 field + 1); 73 74 clear_bit(field, vmx_vmread_bitmap); 75 #ifdef CONFIG_X86_64 76 if (field & 1) 77 continue; 78 #endif 79 if (j < i) 80 shadow_read_only_fields[j] = field; 81 j++; 82 } 83 max_shadow_read_only_fields = j; 84 85 for (i = j = 0; i < max_shadow_read_write_fields; i++) { 86 u16 field = shadow_read_write_fields[i]; 87 88 if (vmcs_field_width(field) == VMCS_FIELD_WIDTH_U64 && 89 (i + 1 == max_shadow_read_write_fields || 90 shadow_read_write_fields[i + 1] != field + 1)) 91 pr_err("Missing field from shadow_read_write_field %x\n", 92 field + 1); 93 94 /* 95 * PML and the preemption timer can be emulated, but the 96 * processor cannot vmwrite to fields that don't exist 97 * on bare metal. 98 */ 99 switch (field) { 100 case GUEST_PML_INDEX: 101 if (!cpu_has_vmx_pml()) 102 continue; 103 break; 104 case VMX_PREEMPTION_TIMER_VALUE: 105 if (!cpu_has_vmx_preemption_timer()) 106 continue; 107 break; 108 case GUEST_INTR_STATUS: 109 if (!cpu_has_vmx_apicv()) 110 continue; 111 break; 112 default: 113 break; 114 } 115 116 clear_bit(field, vmx_vmwrite_bitmap); 117 clear_bit(field, vmx_vmread_bitmap); 118 #ifdef CONFIG_X86_64 119 if (field & 1) 120 continue; 121 #endif 122 if (j < i) 123 shadow_read_write_fields[j] = field; 124 j++; 125 } 126 max_shadow_read_write_fields = j; 127 } 128 129 /* 130 * The following 3 functions, nested_vmx_succeed()/failValid()/failInvalid(), 131 * set the success or error code of an emulated VMX instruction (as specified 132 * by Vol 2B, VMX Instruction Reference, "Conventions"), and skip the emulated 133 * instruction. 134 */ 135 static int nested_vmx_succeed(struct kvm_vcpu *vcpu) 136 { 137 vmx_set_rflags(vcpu, vmx_get_rflags(vcpu) 138 & ~(X86_EFLAGS_CF | X86_EFLAGS_PF | X86_EFLAGS_AF | 139 X86_EFLAGS_ZF | X86_EFLAGS_SF | X86_EFLAGS_OF)); 140 return kvm_skip_emulated_instruction(vcpu); 141 } 142 143 static int nested_vmx_failInvalid(struct kvm_vcpu *vcpu) 144 { 145 vmx_set_rflags(vcpu, (vmx_get_rflags(vcpu) 146 & ~(X86_EFLAGS_PF | X86_EFLAGS_AF | X86_EFLAGS_ZF | 147 X86_EFLAGS_SF | X86_EFLAGS_OF)) 148 | X86_EFLAGS_CF); 149 return kvm_skip_emulated_instruction(vcpu); 150 } 151 152 static int nested_vmx_failValid(struct kvm_vcpu *vcpu, 153 u32 vm_instruction_error) 154 { 155 struct vcpu_vmx *vmx = to_vmx(vcpu); 156 157 /* 158 * failValid writes the error number to the current VMCS, which 159 * can't be done if there isn't a current VMCS. 160 */ 161 if (vmx->nested.current_vmptr == -1ull && !vmx->nested.hv_evmcs) 162 return nested_vmx_failInvalid(vcpu); 163 164 vmx_set_rflags(vcpu, (vmx_get_rflags(vcpu) 165 & ~(X86_EFLAGS_CF | X86_EFLAGS_PF | X86_EFLAGS_AF | 166 X86_EFLAGS_SF | X86_EFLAGS_OF)) 167 | X86_EFLAGS_ZF); 168 get_vmcs12(vcpu)->vm_instruction_error = vm_instruction_error; 169 /* 170 * We don't need to force a shadow sync because 171 * VM_INSTRUCTION_ERROR is not shadowed 172 */ 173 return kvm_skip_emulated_instruction(vcpu); 174 } 175 176 static void nested_vmx_abort(struct kvm_vcpu *vcpu, u32 indicator) 177 { 178 /* TODO: not to reset guest simply here. */ 179 kvm_make_request(KVM_REQ_TRIPLE_FAULT, vcpu); 180 pr_debug_ratelimited("kvm: nested vmx abort, indicator %d\n", indicator); 181 } 182 183 static void vmx_disable_shadow_vmcs(struct vcpu_vmx *vmx) 184 { 185 vmcs_clear_bits(SECONDARY_VM_EXEC_CONTROL, SECONDARY_EXEC_SHADOW_VMCS); 186 vmcs_write64(VMCS_LINK_POINTER, -1ull); 187 } 188 189 static inline void nested_release_evmcs(struct kvm_vcpu *vcpu) 190 { 191 struct vcpu_vmx *vmx = to_vmx(vcpu); 192 193 if (!vmx->nested.hv_evmcs) 194 return; 195 196 kunmap(vmx->nested.hv_evmcs_page); 197 kvm_release_page_dirty(vmx->nested.hv_evmcs_page); 198 vmx->nested.hv_evmcs_vmptr = -1ull; 199 vmx->nested.hv_evmcs_page = NULL; 200 vmx->nested.hv_evmcs = NULL; 201 } 202 203 /* 204 * Free whatever needs to be freed from vmx->nested when L1 goes down, or 205 * just stops using VMX. 206 */ 207 static void free_nested(struct kvm_vcpu *vcpu) 208 { 209 struct vcpu_vmx *vmx = to_vmx(vcpu); 210 211 if (!vmx->nested.vmxon && !vmx->nested.smm.vmxon) 212 return; 213 214 hrtimer_cancel(&vmx->nested.preemption_timer); 215 vmx->nested.vmxon = false; 216 vmx->nested.smm.vmxon = false; 217 free_vpid(vmx->nested.vpid02); 218 vmx->nested.posted_intr_nv = -1; 219 vmx->nested.current_vmptr = -1ull; 220 if (enable_shadow_vmcs) { 221 vmx_disable_shadow_vmcs(vmx); 222 vmcs_clear(vmx->vmcs01.shadow_vmcs); 223 free_vmcs(vmx->vmcs01.shadow_vmcs); 224 vmx->vmcs01.shadow_vmcs = NULL; 225 } 226 kfree(vmx->nested.cached_vmcs12); 227 kfree(vmx->nested.cached_shadow_vmcs12); 228 /* Unpin physical memory we referred to in the vmcs02 */ 229 if (vmx->nested.apic_access_page) { 230 kvm_release_page_dirty(vmx->nested.apic_access_page); 231 vmx->nested.apic_access_page = NULL; 232 } 233 if (vmx->nested.virtual_apic_page) { 234 kvm_release_page_dirty(vmx->nested.virtual_apic_page); 235 vmx->nested.virtual_apic_page = NULL; 236 } 237 if (vmx->nested.pi_desc_page) { 238 kunmap(vmx->nested.pi_desc_page); 239 kvm_release_page_dirty(vmx->nested.pi_desc_page); 240 vmx->nested.pi_desc_page = NULL; 241 vmx->nested.pi_desc = NULL; 242 } 243 244 kvm_mmu_free_roots(vcpu, &vcpu->arch.guest_mmu, KVM_MMU_ROOTS_ALL); 245 246 nested_release_evmcs(vcpu); 247 248 free_loaded_vmcs(&vmx->nested.vmcs02); 249 } 250 251 static void vmx_switch_vmcs(struct kvm_vcpu *vcpu, struct loaded_vmcs *vmcs) 252 { 253 struct vcpu_vmx *vmx = to_vmx(vcpu); 254 int cpu; 255 256 if (vmx->loaded_vmcs == vmcs) 257 return; 258 259 cpu = get_cpu(); 260 vmx_vcpu_put(vcpu); 261 vmx->loaded_vmcs = vmcs; 262 vmx_vcpu_load(vcpu, cpu); 263 put_cpu(); 264 265 vm_entry_controls_reset_shadow(vmx); 266 vm_exit_controls_reset_shadow(vmx); 267 vmx_segment_cache_clear(vmx); 268 } 269 270 /* 271 * Ensure that the current vmcs of the logical processor is the 272 * vmcs01 of the vcpu before calling free_nested(). 273 */ 274 void nested_vmx_free_vcpu(struct kvm_vcpu *vcpu) 275 { 276 vcpu_load(vcpu); 277 vmx_switch_vmcs(vcpu, &to_vmx(vcpu)->vmcs01); 278 free_nested(vcpu); 279 vcpu_put(vcpu); 280 } 281 282 static void nested_ept_inject_page_fault(struct kvm_vcpu *vcpu, 283 struct x86_exception *fault) 284 { 285 struct vmcs12 *vmcs12 = get_vmcs12(vcpu); 286 struct vcpu_vmx *vmx = to_vmx(vcpu); 287 u32 exit_reason; 288 unsigned long exit_qualification = vcpu->arch.exit_qualification; 289 290 if (vmx->nested.pml_full) { 291 exit_reason = EXIT_REASON_PML_FULL; 292 vmx->nested.pml_full = false; 293 exit_qualification &= INTR_INFO_UNBLOCK_NMI; 294 } else if (fault->error_code & PFERR_RSVD_MASK) 295 exit_reason = EXIT_REASON_EPT_MISCONFIG; 296 else 297 exit_reason = EXIT_REASON_EPT_VIOLATION; 298 299 nested_vmx_vmexit(vcpu, exit_reason, 0, exit_qualification); 300 vmcs12->guest_physical_address = fault->address; 301 } 302 303 static void nested_ept_init_mmu_context(struct kvm_vcpu *vcpu) 304 { 305 WARN_ON(mmu_is_nested(vcpu)); 306 307 vcpu->arch.mmu = &vcpu->arch.guest_mmu; 308 kvm_init_shadow_ept_mmu(vcpu, 309 to_vmx(vcpu)->nested.msrs.ept_caps & 310 VMX_EPT_EXECUTE_ONLY_BIT, 311 nested_ept_ad_enabled(vcpu), 312 nested_ept_get_cr3(vcpu)); 313 vcpu->arch.mmu->set_cr3 = vmx_set_cr3; 314 vcpu->arch.mmu->get_cr3 = nested_ept_get_cr3; 315 vcpu->arch.mmu->inject_page_fault = nested_ept_inject_page_fault; 316 vcpu->arch.mmu->get_pdptr = kvm_pdptr_read; 317 318 vcpu->arch.walk_mmu = &vcpu->arch.nested_mmu; 319 } 320 321 static void nested_ept_uninit_mmu_context(struct kvm_vcpu *vcpu) 322 { 323 vcpu->arch.mmu = &vcpu->arch.root_mmu; 324 vcpu->arch.walk_mmu = &vcpu->arch.root_mmu; 325 } 326 327 static bool nested_vmx_is_page_fault_vmexit(struct vmcs12 *vmcs12, 328 u16 error_code) 329 { 330 bool inequality, bit; 331 332 bit = (vmcs12->exception_bitmap & (1u << PF_VECTOR)) != 0; 333 inequality = 334 (error_code & vmcs12->page_fault_error_code_mask) != 335 vmcs12->page_fault_error_code_match; 336 return inequality ^ bit; 337 } 338 339 340 /* 341 * KVM wants to inject page-faults which it got to the guest. This function 342 * checks whether in a nested guest, we need to inject them to L1 or L2. 343 */ 344 static int nested_vmx_check_exception(struct kvm_vcpu *vcpu, unsigned long *exit_qual) 345 { 346 struct vmcs12 *vmcs12 = get_vmcs12(vcpu); 347 unsigned int nr = vcpu->arch.exception.nr; 348 bool has_payload = vcpu->arch.exception.has_payload; 349 unsigned long payload = vcpu->arch.exception.payload; 350 351 if (nr == PF_VECTOR) { 352 if (vcpu->arch.exception.nested_apf) { 353 *exit_qual = vcpu->arch.apf.nested_apf_token; 354 return 1; 355 } 356 if (nested_vmx_is_page_fault_vmexit(vmcs12, 357 vcpu->arch.exception.error_code)) { 358 *exit_qual = has_payload ? payload : vcpu->arch.cr2; 359 return 1; 360 } 361 } else if (vmcs12->exception_bitmap & (1u << nr)) { 362 if (nr == DB_VECTOR) { 363 if (!has_payload) { 364 payload = vcpu->arch.dr6; 365 payload &= ~(DR6_FIXED_1 | DR6_BT); 366 payload ^= DR6_RTM; 367 } 368 *exit_qual = payload; 369 } else 370 *exit_qual = 0; 371 return 1; 372 } 373 374 return 0; 375 } 376 377 378 static void vmx_inject_page_fault_nested(struct kvm_vcpu *vcpu, 379 struct x86_exception *fault) 380 { 381 struct vmcs12 *vmcs12 = get_vmcs12(vcpu); 382 383 WARN_ON(!is_guest_mode(vcpu)); 384 385 if (nested_vmx_is_page_fault_vmexit(vmcs12, fault->error_code) && 386 !to_vmx(vcpu)->nested.nested_run_pending) { 387 vmcs12->vm_exit_intr_error_code = fault->error_code; 388 nested_vmx_vmexit(vcpu, EXIT_REASON_EXCEPTION_NMI, 389 PF_VECTOR | INTR_TYPE_HARD_EXCEPTION | 390 INTR_INFO_DELIVER_CODE_MASK | INTR_INFO_VALID_MASK, 391 fault->address); 392 } else { 393 kvm_inject_page_fault(vcpu, fault); 394 } 395 } 396 397 static bool page_address_valid(struct kvm_vcpu *vcpu, gpa_t gpa) 398 { 399 return PAGE_ALIGNED(gpa) && !(gpa >> cpuid_maxphyaddr(vcpu)); 400 } 401 402 static int nested_vmx_check_io_bitmap_controls(struct kvm_vcpu *vcpu, 403 struct vmcs12 *vmcs12) 404 { 405 if (!nested_cpu_has(vmcs12, CPU_BASED_USE_IO_BITMAPS)) 406 return 0; 407 408 if (!page_address_valid(vcpu, vmcs12->io_bitmap_a) || 409 !page_address_valid(vcpu, vmcs12->io_bitmap_b)) 410 return -EINVAL; 411 412 return 0; 413 } 414 415 static int nested_vmx_check_msr_bitmap_controls(struct kvm_vcpu *vcpu, 416 struct vmcs12 *vmcs12) 417 { 418 if (!nested_cpu_has(vmcs12, CPU_BASED_USE_MSR_BITMAPS)) 419 return 0; 420 421 if (!page_address_valid(vcpu, vmcs12->msr_bitmap)) 422 return -EINVAL; 423 424 return 0; 425 } 426 427 static int nested_vmx_check_tpr_shadow_controls(struct kvm_vcpu *vcpu, 428 struct vmcs12 *vmcs12) 429 { 430 if (!nested_cpu_has(vmcs12, CPU_BASED_TPR_SHADOW)) 431 return 0; 432 433 if (!page_address_valid(vcpu, vmcs12->virtual_apic_page_addr)) 434 return -EINVAL; 435 436 return 0; 437 } 438 439 /* 440 * Check if MSR is intercepted for L01 MSR bitmap. 441 */ 442 static bool msr_write_intercepted_l01(struct kvm_vcpu *vcpu, u32 msr) 443 { 444 unsigned long *msr_bitmap; 445 int f = sizeof(unsigned long); 446 447 if (!cpu_has_vmx_msr_bitmap()) 448 return true; 449 450 msr_bitmap = to_vmx(vcpu)->vmcs01.msr_bitmap; 451 452 if (msr <= 0x1fff) { 453 return !!test_bit(msr, msr_bitmap + 0x800 / f); 454 } else if ((msr >= 0xc0000000) && (msr <= 0xc0001fff)) { 455 msr &= 0x1fff; 456 return !!test_bit(msr, msr_bitmap + 0xc00 / f); 457 } 458 459 return true; 460 } 461 462 /* 463 * If a msr is allowed by L0, we should check whether it is allowed by L1. 464 * The corresponding bit will be cleared unless both of L0 and L1 allow it. 465 */ 466 static void nested_vmx_disable_intercept_for_msr(unsigned long *msr_bitmap_l1, 467 unsigned long *msr_bitmap_nested, 468 u32 msr, int type) 469 { 470 int f = sizeof(unsigned long); 471 472 /* 473 * See Intel PRM Vol. 3, 20.6.9 (MSR-Bitmap Address). Early manuals 474 * have the write-low and read-high bitmap offsets the wrong way round. 475 * We can control MSRs 0x00000000-0x00001fff and 0xc0000000-0xc0001fff. 476 */ 477 if (msr <= 0x1fff) { 478 if (type & MSR_TYPE_R && 479 !test_bit(msr, msr_bitmap_l1 + 0x000 / f)) 480 /* read-low */ 481 __clear_bit(msr, msr_bitmap_nested + 0x000 / f); 482 483 if (type & MSR_TYPE_W && 484 !test_bit(msr, msr_bitmap_l1 + 0x800 / f)) 485 /* write-low */ 486 __clear_bit(msr, msr_bitmap_nested + 0x800 / f); 487 488 } else if ((msr >= 0xc0000000) && (msr <= 0xc0001fff)) { 489 msr &= 0x1fff; 490 if (type & MSR_TYPE_R && 491 !test_bit(msr, msr_bitmap_l1 + 0x400 / f)) 492 /* read-high */ 493 __clear_bit(msr, msr_bitmap_nested + 0x400 / f); 494 495 if (type & MSR_TYPE_W && 496 !test_bit(msr, msr_bitmap_l1 + 0xc00 / f)) 497 /* write-high */ 498 __clear_bit(msr, msr_bitmap_nested + 0xc00 / f); 499 500 } 501 } 502 503 /* 504 * Merge L0's and L1's MSR bitmap, return false to indicate that 505 * we do not use the hardware. 506 */ 507 static inline bool nested_vmx_prepare_msr_bitmap(struct kvm_vcpu *vcpu, 508 struct vmcs12 *vmcs12) 509 { 510 int msr; 511 struct page *page; 512 unsigned long *msr_bitmap_l1; 513 unsigned long *msr_bitmap_l0 = to_vmx(vcpu)->nested.vmcs02.msr_bitmap; 514 /* 515 * pred_cmd & spec_ctrl are trying to verify two things: 516 * 517 * 1. L0 gave a permission to L1 to actually passthrough the MSR. This 518 * ensures that we do not accidentally generate an L02 MSR bitmap 519 * from the L12 MSR bitmap that is too permissive. 520 * 2. That L1 or L2s have actually used the MSR. This avoids 521 * unnecessarily merging of the bitmap if the MSR is unused. This 522 * works properly because we only update the L01 MSR bitmap lazily. 523 * So even if L0 should pass L1 these MSRs, the L01 bitmap is only 524 * updated to reflect this when L1 (or its L2s) actually write to 525 * the MSR. 526 */ 527 bool pred_cmd = !msr_write_intercepted_l01(vcpu, MSR_IA32_PRED_CMD); 528 bool spec_ctrl = !msr_write_intercepted_l01(vcpu, MSR_IA32_SPEC_CTRL); 529 530 /* Nothing to do if the MSR bitmap is not in use. */ 531 if (!cpu_has_vmx_msr_bitmap() || 532 !nested_cpu_has(vmcs12, CPU_BASED_USE_MSR_BITMAPS)) 533 return false; 534 535 if (!nested_cpu_has_virt_x2apic_mode(vmcs12) && 536 !pred_cmd && !spec_ctrl) 537 return false; 538 539 page = kvm_vcpu_gpa_to_page(vcpu, vmcs12->msr_bitmap); 540 if (is_error_page(page)) 541 return false; 542 543 msr_bitmap_l1 = (unsigned long *)kmap(page); 544 if (nested_cpu_has_apic_reg_virt(vmcs12)) { 545 /* 546 * L0 need not intercept reads for MSRs between 0x800 and 0x8ff, it 547 * just lets the processor take the value from the virtual-APIC page; 548 * take those 256 bits directly from the L1 bitmap. 549 */ 550 for (msr = 0x800; msr <= 0x8ff; msr += BITS_PER_LONG) { 551 unsigned word = msr / BITS_PER_LONG; 552 msr_bitmap_l0[word] = msr_bitmap_l1[word]; 553 msr_bitmap_l0[word + (0x800 / sizeof(long))] = ~0; 554 } 555 } else { 556 for (msr = 0x800; msr <= 0x8ff; msr += BITS_PER_LONG) { 557 unsigned word = msr / BITS_PER_LONG; 558 msr_bitmap_l0[word] = ~0; 559 msr_bitmap_l0[word + (0x800 / sizeof(long))] = ~0; 560 } 561 } 562 563 nested_vmx_disable_intercept_for_msr( 564 msr_bitmap_l1, msr_bitmap_l0, 565 X2APIC_MSR(APIC_TASKPRI), 566 MSR_TYPE_W); 567 568 if (nested_cpu_has_vid(vmcs12)) { 569 nested_vmx_disable_intercept_for_msr( 570 msr_bitmap_l1, msr_bitmap_l0, 571 X2APIC_MSR(APIC_EOI), 572 MSR_TYPE_W); 573 nested_vmx_disable_intercept_for_msr( 574 msr_bitmap_l1, msr_bitmap_l0, 575 X2APIC_MSR(APIC_SELF_IPI), 576 MSR_TYPE_W); 577 } 578 579 if (spec_ctrl) 580 nested_vmx_disable_intercept_for_msr( 581 msr_bitmap_l1, msr_bitmap_l0, 582 MSR_IA32_SPEC_CTRL, 583 MSR_TYPE_R | MSR_TYPE_W); 584 585 if (pred_cmd) 586 nested_vmx_disable_intercept_for_msr( 587 msr_bitmap_l1, msr_bitmap_l0, 588 MSR_IA32_PRED_CMD, 589 MSR_TYPE_W); 590 591 kunmap(page); 592 kvm_release_page_clean(page); 593 594 return true; 595 } 596 597 static void nested_cache_shadow_vmcs12(struct kvm_vcpu *vcpu, 598 struct vmcs12 *vmcs12) 599 { 600 struct vmcs12 *shadow; 601 struct page *page; 602 603 if (!nested_cpu_has_shadow_vmcs(vmcs12) || 604 vmcs12->vmcs_link_pointer == -1ull) 605 return; 606 607 shadow = get_shadow_vmcs12(vcpu); 608 page = kvm_vcpu_gpa_to_page(vcpu, vmcs12->vmcs_link_pointer); 609 610 memcpy(shadow, kmap(page), VMCS12_SIZE); 611 612 kunmap(page); 613 kvm_release_page_clean(page); 614 } 615 616 static void nested_flush_cached_shadow_vmcs12(struct kvm_vcpu *vcpu, 617 struct vmcs12 *vmcs12) 618 { 619 struct vcpu_vmx *vmx = to_vmx(vcpu); 620 621 if (!nested_cpu_has_shadow_vmcs(vmcs12) || 622 vmcs12->vmcs_link_pointer == -1ull) 623 return; 624 625 kvm_write_guest(vmx->vcpu.kvm, vmcs12->vmcs_link_pointer, 626 get_shadow_vmcs12(vcpu), VMCS12_SIZE); 627 } 628 629 /* 630 * In nested virtualization, check if L1 has set 631 * VM_EXIT_ACK_INTR_ON_EXIT 632 */ 633 static bool nested_exit_intr_ack_set(struct kvm_vcpu *vcpu) 634 { 635 return get_vmcs12(vcpu)->vm_exit_controls & 636 VM_EXIT_ACK_INTR_ON_EXIT; 637 } 638 639 static bool nested_exit_on_nmi(struct kvm_vcpu *vcpu) 640 { 641 return nested_cpu_has_nmi_exiting(get_vmcs12(vcpu)); 642 } 643 644 static int nested_vmx_check_apic_access_controls(struct kvm_vcpu *vcpu, 645 struct vmcs12 *vmcs12) 646 { 647 if (nested_cpu_has2(vmcs12, SECONDARY_EXEC_VIRTUALIZE_APIC_ACCESSES) && 648 !page_address_valid(vcpu, vmcs12->apic_access_addr)) 649 return -EINVAL; 650 else 651 return 0; 652 } 653 654 static int nested_vmx_check_apicv_controls(struct kvm_vcpu *vcpu, 655 struct vmcs12 *vmcs12) 656 { 657 if (!nested_cpu_has_virt_x2apic_mode(vmcs12) && 658 !nested_cpu_has_apic_reg_virt(vmcs12) && 659 !nested_cpu_has_vid(vmcs12) && 660 !nested_cpu_has_posted_intr(vmcs12)) 661 return 0; 662 663 /* 664 * If virtualize x2apic mode is enabled, 665 * virtualize apic access must be disabled. 666 */ 667 if (nested_cpu_has_virt_x2apic_mode(vmcs12) && 668 nested_cpu_has2(vmcs12, SECONDARY_EXEC_VIRTUALIZE_APIC_ACCESSES)) 669 return -EINVAL; 670 671 /* 672 * If virtual interrupt delivery is enabled, 673 * we must exit on external interrupts. 674 */ 675 if (nested_cpu_has_vid(vmcs12) && 676 !nested_exit_on_intr(vcpu)) 677 return -EINVAL; 678 679 /* 680 * bits 15:8 should be zero in posted_intr_nv, 681 * the descriptor address has been already checked 682 * in nested_get_vmcs12_pages. 683 * 684 * bits 5:0 of posted_intr_desc_addr should be zero. 685 */ 686 if (nested_cpu_has_posted_intr(vmcs12) && 687 (!nested_cpu_has_vid(vmcs12) || 688 !nested_exit_intr_ack_set(vcpu) || 689 (vmcs12->posted_intr_nv & 0xff00) || 690 (vmcs12->posted_intr_desc_addr & 0x3f) || 691 (vmcs12->posted_intr_desc_addr >> cpuid_maxphyaddr(vcpu)))) 692 return -EINVAL; 693 694 /* tpr shadow is needed by all apicv features. */ 695 if (!nested_cpu_has(vmcs12, CPU_BASED_TPR_SHADOW)) 696 return -EINVAL; 697 698 return 0; 699 } 700 701 static int nested_vmx_check_msr_switch(struct kvm_vcpu *vcpu, 702 u32 count, u64 addr) 703 { 704 int maxphyaddr; 705 706 if (count == 0) 707 return 0; 708 maxphyaddr = cpuid_maxphyaddr(vcpu); 709 if (!IS_ALIGNED(addr, 16) || addr >> maxphyaddr || 710 (addr + count * sizeof(struct vmx_msr_entry) - 1) >> maxphyaddr) 711 return -EINVAL; 712 713 return 0; 714 } 715 716 static int nested_vmx_check_exit_msr_switch_controls(struct kvm_vcpu *vcpu, 717 struct vmcs12 *vmcs12) 718 { 719 if (nested_vmx_check_msr_switch(vcpu, vmcs12->vm_exit_msr_load_count, 720 vmcs12->vm_exit_msr_load_addr) || 721 nested_vmx_check_msr_switch(vcpu, vmcs12->vm_exit_msr_store_count, 722 vmcs12->vm_exit_msr_store_addr)) 723 return -EINVAL; 724 725 return 0; 726 } 727 728 static int nested_vmx_check_entry_msr_switch_controls(struct kvm_vcpu *vcpu, 729 struct vmcs12 *vmcs12) 730 { 731 if (nested_vmx_check_msr_switch(vcpu, vmcs12->vm_entry_msr_load_count, 732 vmcs12->vm_entry_msr_load_addr)) 733 return -EINVAL; 734 735 return 0; 736 } 737 738 static int nested_vmx_check_pml_controls(struct kvm_vcpu *vcpu, 739 struct vmcs12 *vmcs12) 740 { 741 if (!nested_cpu_has_pml(vmcs12)) 742 return 0; 743 744 if (!nested_cpu_has_ept(vmcs12) || 745 !page_address_valid(vcpu, vmcs12->pml_address)) 746 return -EINVAL; 747 748 return 0; 749 } 750 751 static int nested_vmx_check_unrestricted_guest_controls(struct kvm_vcpu *vcpu, 752 struct vmcs12 *vmcs12) 753 { 754 if (nested_cpu_has2(vmcs12, SECONDARY_EXEC_UNRESTRICTED_GUEST) && 755 !nested_cpu_has_ept(vmcs12)) 756 return -EINVAL; 757 return 0; 758 } 759 760 static int nested_vmx_check_mode_based_ept_exec_controls(struct kvm_vcpu *vcpu, 761 struct vmcs12 *vmcs12) 762 { 763 if (nested_cpu_has2(vmcs12, SECONDARY_EXEC_MODE_BASED_EPT_EXEC) && 764 !nested_cpu_has_ept(vmcs12)) 765 return -EINVAL; 766 return 0; 767 } 768 769 static int nested_vmx_check_shadow_vmcs_controls(struct kvm_vcpu *vcpu, 770 struct vmcs12 *vmcs12) 771 { 772 if (!nested_cpu_has_shadow_vmcs(vmcs12)) 773 return 0; 774 775 if (!page_address_valid(vcpu, vmcs12->vmread_bitmap) || 776 !page_address_valid(vcpu, vmcs12->vmwrite_bitmap)) 777 return -EINVAL; 778 779 return 0; 780 } 781 782 static int nested_vmx_msr_check_common(struct kvm_vcpu *vcpu, 783 struct vmx_msr_entry *e) 784 { 785 /* x2APIC MSR accesses are not allowed */ 786 if (vcpu->arch.apic_base & X2APIC_ENABLE && e->index >> 8 == 0x8) 787 return -EINVAL; 788 if (e->index == MSR_IA32_UCODE_WRITE || /* SDM Table 35-2 */ 789 e->index == MSR_IA32_UCODE_REV) 790 return -EINVAL; 791 if (e->reserved != 0) 792 return -EINVAL; 793 return 0; 794 } 795 796 static int nested_vmx_load_msr_check(struct kvm_vcpu *vcpu, 797 struct vmx_msr_entry *e) 798 { 799 if (e->index == MSR_FS_BASE || 800 e->index == MSR_GS_BASE || 801 e->index == MSR_IA32_SMM_MONITOR_CTL || /* SMM is not supported */ 802 nested_vmx_msr_check_common(vcpu, e)) 803 return -EINVAL; 804 return 0; 805 } 806 807 static int nested_vmx_store_msr_check(struct kvm_vcpu *vcpu, 808 struct vmx_msr_entry *e) 809 { 810 if (e->index == MSR_IA32_SMBASE || /* SMM is not supported */ 811 nested_vmx_msr_check_common(vcpu, e)) 812 return -EINVAL; 813 return 0; 814 } 815 816 /* 817 * Load guest's/host's msr at nested entry/exit. 818 * return 0 for success, entry index for failure. 819 */ 820 static u32 nested_vmx_load_msr(struct kvm_vcpu *vcpu, u64 gpa, u32 count) 821 { 822 u32 i; 823 struct vmx_msr_entry e; 824 struct msr_data msr; 825 826 msr.host_initiated = false; 827 for (i = 0; i < count; i++) { 828 if (kvm_vcpu_read_guest(vcpu, gpa + i * sizeof(e), 829 &e, sizeof(e))) { 830 pr_debug_ratelimited( 831 "%s cannot read MSR entry (%u, 0x%08llx)\n", 832 __func__, i, gpa + i * sizeof(e)); 833 goto fail; 834 } 835 if (nested_vmx_load_msr_check(vcpu, &e)) { 836 pr_debug_ratelimited( 837 "%s check failed (%u, 0x%x, 0x%x)\n", 838 __func__, i, e.index, e.reserved); 839 goto fail; 840 } 841 msr.index = e.index; 842 msr.data = e.value; 843 if (kvm_set_msr(vcpu, &msr)) { 844 pr_debug_ratelimited( 845 "%s cannot write MSR (%u, 0x%x, 0x%llx)\n", 846 __func__, i, e.index, e.value); 847 goto fail; 848 } 849 } 850 return 0; 851 fail: 852 return i + 1; 853 } 854 855 static int nested_vmx_store_msr(struct kvm_vcpu *vcpu, u64 gpa, u32 count) 856 { 857 u32 i; 858 struct vmx_msr_entry e; 859 860 for (i = 0; i < count; i++) { 861 struct msr_data msr_info; 862 if (kvm_vcpu_read_guest(vcpu, 863 gpa + i * sizeof(e), 864 &e, 2 * sizeof(u32))) { 865 pr_debug_ratelimited( 866 "%s cannot read MSR entry (%u, 0x%08llx)\n", 867 __func__, i, gpa + i * sizeof(e)); 868 return -EINVAL; 869 } 870 if (nested_vmx_store_msr_check(vcpu, &e)) { 871 pr_debug_ratelimited( 872 "%s check failed (%u, 0x%x, 0x%x)\n", 873 __func__, i, e.index, e.reserved); 874 return -EINVAL; 875 } 876 msr_info.host_initiated = false; 877 msr_info.index = e.index; 878 if (kvm_get_msr(vcpu, &msr_info)) { 879 pr_debug_ratelimited( 880 "%s cannot read MSR (%u, 0x%x)\n", 881 __func__, i, e.index); 882 return -EINVAL; 883 } 884 if (kvm_vcpu_write_guest(vcpu, 885 gpa + i * sizeof(e) + 886 offsetof(struct vmx_msr_entry, value), 887 &msr_info.data, sizeof(msr_info.data))) { 888 pr_debug_ratelimited( 889 "%s cannot write MSR (%u, 0x%x, 0x%llx)\n", 890 __func__, i, e.index, msr_info.data); 891 return -EINVAL; 892 } 893 } 894 return 0; 895 } 896 897 static bool nested_cr3_valid(struct kvm_vcpu *vcpu, unsigned long val) 898 { 899 unsigned long invalid_mask; 900 901 invalid_mask = (~0ULL) << cpuid_maxphyaddr(vcpu); 902 return (val & invalid_mask) == 0; 903 } 904 905 /* 906 * Load guest's/host's cr3 at nested entry/exit. nested_ept is true if we are 907 * emulating VM entry into a guest with EPT enabled. 908 * Returns 0 on success, 1 on failure. Invalid state exit qualification code 909 * is assigned to entry_failure_code on failure. 910 */ 911 static int nested_vmx_load_cr3(struct kvm_vcpu *vcpu, unsigned long cr3, bool nested_ept, 912 u32 *entry_failure_code) 913 { 914 if (cr3 != kvm_read_cr3(vcpu) || (!nested_ept && pdptrs_changed(vcpu))) { 915 if (!nested_cr3_valid(vcpu, cr3)) { 916 *entry_failure_code = ENTRY_FAIL_DEFAULT; 917 return 1; 918 } 919 920 /* 921 * If PAE paging and EPT are both on, CR3 is not used by the CPU and 922 * must not be dereferenced. 923 */ 924 if (!is_long_mode(vcpu) && is_pae(vcpu) && is_paging(vcpu) && 925 !nested_ept) { 926 if (!load_pdptrs(vcpu, vcpu->arch.walk_mmu, cr3)) { 927 *entry_failure_code = ENTRY_FAIL_PDPTE; 928 return 1; 929 } 930 } 931 } 932 933 if (!nested_ept) 934 kvm_mmu_new_cr3(vcpu, cr3, false); 935 936 vcpu->arch.cr3 = cr3; 937 __set_bit(VCPU_EXREG_CR3, (ulong *)&vcpu->arch.regs_avail); 938 939 kvm_init_mmu(vcpu, false); 940 941 return 0; 942 } 943 944 /* 945 * Returns if KVM is able to config CPU to tag TLB entries 946 * populated by L2 differently than TLB entries populated 947 * by L1. 948 * 949 * If L1 uses EPT, then TLB entries are tagged with different EPTP. 950 * 951 * If L1 uses VPID and we allocated a vpid02, TLB entries are tagged 952 * with different VPID (L1 entries are tagged with vmx->vpid 953 * while L2 entries are tagged with vmx->nested.vpid02). 954 */ 955 static bool nested_has_guest_tlb_tag(struct kvm_vcpu *vcpu) 956 { 957 struct vmcs12 *vmcs12 = get_vmcs12(vcpu); 958 959 return nested_cpu_has_ept(vmcs12) || 960 (nested_cpu_has_vpid(vmcs12) && to_vmx(vcpu)->nested.vpid02); 961 } 962 963 static u16 nested_get_vpid02(struct kvm_vcpu *vcpu) 964 { 965 struct vcpu_vmx *vmx = to_vmx(vcpu); 966 967 return vmx->nested.vpid02 ? vmx->nested.vpid02 : vmx->vpid; 968 } 969 970 971 static inline bool vmx_control_verify(u32 control, u32 low, u32 high) 972 { 973 return fixed_bits_valid(control, low, high); 974 } 975 976 static inline u64 vmx_control_msr(u32 low, u32 high) 977 { 978 return low | ((u64)high << 32); 979 } 980 981 static bool is_bitwise_subset(u64 superset, u64 subset, u64 mask) 982 { 983 superset &= mask; 984 subset &= mask; 985 986 return (superset | subset) == superset; 987 } 988 989 static int vmx_restore_vmx_basic(struct vcpu_vmx *vmx, u64 data) 990 { 991 const u64 feature_and_reserved = 992 /* feature (except bit 48; see below) */ 993 BIT_ULL(49) | BIT_ULL(54) | BIT_ULL(55) | 994 /* reserved */ 995 BIT_ULL(31) | GENMASK_ULL(47, 45) | GENMASK_ULL(63, 56); 996 u64 vmx_basic = vmx->nested.msrs.basic; 997 998 if (!is_bitwise_subset(vmx_basic, data, feature_and_reserved)) 999 return -EINVAL; 1000 1001 /* 1002 * KVM does not emulate a version of VMX that constrains physical 1003 * addresses of VMX structures (e.g. VMCS) to 32-bits. 1004 */ 1005 if (data & BIT_ULL(48)) 1006 return -EINVAL; 1007 1008 if (vmx_basic_vmcs_revision_id(vmx_basic) != 1009 vmx_basic_vmcs_revision_id(data)) 1010 return -EINVAL; 1011 1012 if (vmx_basic_vmcs_size(vmx_basic) > vmx_basic_vmcs_size(data)) 1013 return -EINVAL; 1014 1015 vmx->nested.msrs.basic = data; 1016 return 0; 1017 } 1018 1019 static int 1020 vmx_restore_control_msr(struct vcpu_vmx *vmx, u32 msr_index, u64 data) 1021 { 1022 u64 supported; 1023 u32 *lowp, *highp; 1024 1025 switch (msr_index) { 1026 case MSR_IA32_VMX_TRUE_PINBASED_CTLS: 1027 lowp = &vmx->nested.msrs.pinbased_ctls_low; 1028 highp = &vmx->nested.msrs.pinbased_ctls_high; 1029 break; 1030 case MSR_IA32_VMX_TRUE_PROCBASED_CTLS: 1031 lowp = &vmx->nested.msrs.procbased_ctls_low; 1032 highp = &vmx->nested.msrs.procbased_ctls_high; 1033 break; 1034 case MSR_IA32_VMX_TRUE_EXIT_CTLS: 1035 lowp = &vmx->nested.msrs.exit_ctls_low; 1036 highp = &vmx->nested.msrs.exit_ctls_high; 1037 break; 1038 case MSR_IA32_VMX_TRUE_ENTRY_CTLS: 1039 lowp = &vmx->nested.msrs.entry_ctls_low; 1040 highp = &vmx->nested.msrs.entry_ctls_high; 1041 break; 1042 case MSR_IA32_VMX_PROCBASED_CTLS2: 1043 lowp = &vmx->nested.msrs.secondary_ctls_low; 1044 highp = &vmx->nested.msrs.secondary_ctls_high; 1045 break; 1046 default: 1047 BUG(); 1048 } 1049 1050 supported = vmx_control_msr(*lowp, *highp); 1051 1052 /* Check must-be-1 bits are still 1. */ 1053 if (!is_bitwise_subset(data, supported, GENMASK_ULL(31, 0))) 1054 return -EINVAL; 1055 1056 /* Check must-be-0 bits are still 0. */ 1057 if (!is_bitwise_subset(supported, data, GENMASK_ULL(63, 32))) 1058 return -EINVAL; 1059 1060 *lowp = data; 1061 *highp = data >> 32; 1062 return 0; 1063 } 1064 1065 static int vmx_restore_vmx_misc(struct vcpu_vmx *vmx, u64 data) 1066 { 1067 const u64 feature_and_reserved_bits = 1068 /* feature */ 1069 BIT_ULL(5) | GENMASK_ULL(8, 6) | BIT_ULL(14) | BIT_ULL(15) | 1070 BIT_ULL(28) | BIT_ULL(29) | BIT_ULL(30) | 1071 /* reserved */ 1072 GENMASK_ULL(13, 9) | BIT_ULL(31); 1073 u64 vmx_misc; 1074 1075 vmx_misc = vmx_control_msr(vmx->nested.msrs.misc_low, 1076 vmx->nested.msrs.misc_high); 1077 1078 if (!is_bitwise_subset(vmx_misc, data, feature_and_reserved_bits)) 1079 return -EINVAL; 1080 1081 if ((vmx->nested.msrs.pinbased_ctls_high & 1082 PIN_BASED_VMX_PREEMPTION_TIMER) && 1083 vmx_misc_preemption_timer_rate(data) != 1084 vmx_misc_preemption_timer_rate(vmx_misc)) 1085 return -EINVAL; 1086 1087 if (vmx_misc_cr3_count(data) > vmx_misc_cr3_count(vmx_misc)) 1088 return -EINVAL; 1089 1090 if (vmx_misc_max_msr(data) > vmx_misc_max_msr(vmx_misc)) 1091 return -EINVAL; 1092 1093 if (vmx_misc_mseg_revid(data) != vmx_misc_mseg_revid(vmx_misc)) 1094 return -EINVAL; 1095 1096 vmx->nested.msrs.misc_low = data; 1097 vmx->nested.msrs.misc_high = data >> 32; 1098 1099 /* 1100 * If L1 has read-only VM-exit information fields, use the 1101 * less permissive vmx_vmwrite_bitmap to specify write 1102 * permissions for the shadow VMCS. 1103 */ 1104 if (enable_shadow_vmcs && !nested_cpu_has_vmwrite_any_field(&vmx->vcpu)) 1105 vmcs_write64(VMWRITE_BITMAP, __pa(vmx_vmwrite_bitmap)); 1106 1107 return 0; 1108 } 1109 1110 static int vmx_restore_vmx_ept_vpid_cap(struct vcpu_vmx *vmx, u64 data) 1111 { 1112 u64 vmx_ept_vpid_cap; 1113 1114 vmx_ept_vpid_cap = vmx_control_msr(vmx->nested.msrs.ept_caps, 1115 vmx->nested.msrs.vpid_caps); 1116 1117 /* Every bit is either reserved or a feature bit. */ 1118 if (!is_bitwise_subset(vmx_ept_vpid_cap, data, -1ULL)) 1119 return -EINVAL; 1120 1121 vmx->nested.msrs.ept_caps = data; 1122 vmx->nested.msrs.vpid_caps = data >> 32; 1123 return 0; 1124 } 1125 1126 static int vmx_restore_fixed0_msr(struct vcpu_vmx *vmx, u32 msr_index, u64 data) 1127 { 1128 u64 *msr; 1129 1130 switch (msr_index) { 1131 case MSR_IA32_VMX_CR0_FIXED0: 1132 msr = &vmx->nested.msrs.cr0_fixed0; 1133 break; 1134 case MSR_IA32_VMX_CR4_FIXED0: 1135 msr = &vmx->nested.msrs.cr4_fixed0; 1136 break; 1137 default: 1138 BUG(); 1139 } 1140 1141 /* 1142 * 1 bits (which indicates bits which "must-be-1" during VMX operation) 1143 * must be 1 in the restored value. 1144 */ 1145 if (!is_bitwise_subset(data, *msr, -1ULL)) 1146 return -EINVAL; 1147 1148 *msr = data; 1149 return 0; 1150 } 1151 1152 /* 1153 * Called when userspace is restoring VMX MSRs. 1154 * 1155 * Returns 0 on success, non-0 otherwise. 1156 */ 1157 int vmx_set_vmx_msr(struct kvm_vcpu *vcpu, u32 msr_index, u64 data) 1158 { 1159 struct vcpu_vmx *vmx = to_vmx(vcpu); 1160 1161 /* 1162 * Don't allow changes to the VMX capability MSRs while the vCPU 1163 * is in VMX operation. 1164 */ 1165 if (vmx->nested.vmxon) 1166 return -EBUSY; 1167 1168 switch (msr_index) { 1169 case MSR_IA32_VMX_BASIC: 1170 return vmx_restore_vmx_basic(vmx, data); 1171 case MSR_IA32_VMX_PINBASED_CTLS: 1172 case MSR_IA32_VMX_PROCBASED_CTLS: 1173 case MSR_IA32_VMX_EXIT_CTLS: 1174 case MSR_IA32_VMX_ENTRY_CTLS: 1175 /* 1176 * The "non-true" VMX capability MSRs are generated from the 1177 * "true" MSRs, so we do not support restoring them directly. 1178 * 1179 * If userspace wants to emulate VMX_BASIC[55]=0, userspace 1180 * should restore the "true" MSRs with the must-be-1 bits 1181 * set according to the SDM Vol 3. A.2 "RESERVED CONTROLS AND 1182 * DEFAULT SETTINGS". 1183 */ 1184 return -EINVAL; 1185 case MSR_IA32_VMX_TRUE_PINBASED_CTLS: 1186 case MSR_IA32_VMX_TRUE_PROCBASED_CTLS: 1187 case MSR_IA32_VMX_TRUE_EXIT_CTLS: 1188 case MSR_IA32_VMX_TRUE_ENTRY_CTLS: 1189 case MSR_IA32_VMX_PROCBASED_CTLS2: 1190 return vmx_restore_control_msr(vmx, msr_index, data); 1191 case MSR_IA32_VMX_MISC: 1192 return vmx_restore_vmx_misc(vmx, data); 1193 case MSR_IA32_VMX_CR0_FIXED0: 1194 case MSR_IA32_VMX_CR4_FIXED0: 1195 return vmx_restore_fixed0_msr(vmx, msr_index, data); 1196 case MSR_IA32_VMX_CR0_FIXED1: 1197 case MSR_IA32_VMX_CR4_FIXED1: 1198 /* 1199 * These MSRs are generated based on the vCPU's CPUID, so we 1200 * do not support restoring them directly. 1201 */ 1202 return -EINVAL; 1203 case MSR_IA32_VMX_EPT_VPID_CAP: 1204 return vmx_restore_vmx_ept_vpid_cap(vmx, data); 1205 case MSR_IA32_VMX_VMCS_ENUM: 1206 vmx->nested.msrs.vmcs_enum = data; 1207 return 0; 1208 default: 1209 /* 1210 * The rest of the VMX capability MSRs do not support restore. 1211 */ 1212 return -EINVAL; 1213 } 1214 } 1215 1216 /* Returns 0 on success, non-0 otherwise. */ 1217 int vmx_get_vmx_msr(struct nested_vmx_msrs *msrs, u32 msr_index, u64 *pdata) 1218 { 1219 switch (msr_index) { 1220 case MSR_IA32_VMX_BASIC: 1221 *pdata = msrs->basic; 1222 break; 1223 case MSR_IA32_VMX_TRUE_PINBASED_CTLS: 1224 case MSR_IA32_VMX_PINBASED_CTLS: 1225 *pdata = vmx_control_msr( 1226 msrs->pinbased_ctls_low, 1227 msrs->pinbased_ctls_high); 1228 if (msr_index == MSR_IA32_VMX_PINBASED_CTLS) 1229 *pdata |= PIN_BASED_ALWAYSON_WITHOUT_TRUE_MSR; 1230 break; 1231 case MSR_IA32_VMX_TRUE_PROCBASED_CTLS: 1232 case MSR_IA32_VMX_PROCBASED_CTLS: 1233 *pdata = vmx_control_msr( 1234 msrs->procbased_ctls_low, 1235 msrs->procbased_ctls_high); 1236 if (msr_index == MSR_IA32_VMX_PROCBASED_CTLS) 1237 *pdata |= CPU_BASED_ALWAYSON_WITHOUT_TRUE_MSR; 1238 break; 1239 case MSR_IA32_VMX_TRUE_EXIT_CTLS: 1240 case MSR_IA32_VMX_EXIT_CTLS: 1241 *pdata = vmx_control_msr( 1242 msrs->exit_ctls_low, 1243 msrs->exit_ctls_high); 1244 if (msr_index == MSR_IA32_VMX_EXIT_CTLS) 1245 *pdata |= VM_EXIT_ALWAYSON_WITHOUT_TRUE_MSR; 1246 break; 1247 case MSR_IA32_VMX_TRUE_ENTRY_CTLS: 1248 case MSR_IA32_VMX_ENTRY_CTLS: 1249 *pdata = vmx_control_msr( 1250 msrs->entry_ctls_low, 1251 msrs->entry_ctls_high); 1252 if (msr_index == MSR_IA32_VMX_ENTRY_CTLS) 1253 *pdata |= VM_ENTRY_ALWAYSON_WITHOUT_TRUE_MSR; 1254 break; 1255 case MSR_IA32_VMX_MISC: 1256 *pdata = vmx_control_msr( 1257 msrs->misc_low, 1258 msrs->misc_high); 1259 break; 1260 case MSR_IA32_VMX_CR0_FIXED0: 1261 *pdata = msrs->cr0_fixed0; 1262 break; 1263 case MSR_IA32_VMX_CR0_FIXED1: 1264 *pdata = msrs->cr0_fixed1; 1265 break; 1266 case MSR_IA32_VMX_CR4_FIXED0: 1267 *pdata = msrs->cr4_fixed0; 1268 break; 1269 case MSR_IA32_VMX_CR4_FIXED1: 1270 *pdata = msrs->cr4_fixed1; 1271 break; 1272 case MSR_IA32_VMX_VMCS_ENUM: 1273 *pdata = msrs->vmcs_enum; 1274 break; 1275 case MSR_IA32_VMX_PROCBASED_CTLS2: 1276 *pdata = vmx_control_msr( 1277 msrs->secondary_ctls_low, 1278 msrs->secondary_ctls_high); 1279 break; 1280 case MSR_IA32_VMX_EPT_VPID_CAP: 1281 *pdata = msrs->ept_caps | 1282 ((u64)msrs->vpid_caps << 32); 1283 break; 1284 case MSR_IA32_VMX_VMFUNC: 1285 *pdata = msrs->vmfunc_controls; 1286 break; 1287 default: 1288 return 1; 1289 } 1290 1291 return 0; 1292 } 1293 1294 /* 1295 * Copy the writable VMCS shadow fields back to the VMCS12, in case 1296 * they have been modified by the L1 guest. Note that the "read-only" 1297 * VM-exit information fields are actually writable if the vCPU is 1298 * configured to support "VMWRITE to any supported field in the VMCS." 1299 */ 1300 static void copy_shadow_to_vmcs12(struct vcpu_vmx *vmx) 1301 { 1302 const u16 *fields[] = { 1303 shadow_read_write_fields, 1304 shadow_read_only_fields 1305 }; 1306 const int max_fields[] = { 1307 max_shadow_read_write_fields, 1308 max_shadow_read_only_fields 1309 }; 1310 int i, q; 1311 unsigned long field; 1312 u64 field_value; 1313 struct vmcs *shadow_vmcs = vmx->vmcs01.shadow_vmcs; 1314 1315 preempt_disable(); 1316 1317 vmcs_load(shadow_vmcs); 1318 1319 for (q = 0; q < ARRAY_SIZE(fields); q++) { 1320 for (i = 0; i < max_fields[q]; i++) { 1321 field = fields[q][i]; 1322 field_value = __vmcs_readl(field); 1323 vmcs12_write_any(get_vmcs12(&vmx->vcpu), field, field_value); 1324 } 1325 /* 1326 * Skip the VM-exit information fields if they are read-only. 1327 */ 1328 if (!nested_cpu_has_vmwrite_any_field(&vmx->vcpu)) 1329 break; 1330 } 1331 1332 vmcs_clear(shadow_vmcs); 1333 vmcs_load(vmx->loaded_vmcs->vmcs); 1334 1335 preempt_enable(); 1336 } 1337 1338 static void copy_vmcs12_to_shadow(struct vcpu_vmx *vmx) 1339 { 1340 const u16 *fields[] = { 1341 shadow_read_write_fields, 1342 shadow_read_only_fields 1343 }; 1344 const int max_fields[] = { 1345 max_shadow_read_write_fields, 1346 max_shadow_read_only_fields 1347 }; 1348 int i, q; 1349 unsigned long field; 1350 u64 field_value = 0; 1351 struct vmcs *shadow_vmcs = vmx->vmcs01.shadow_vmcs; 1352 1353 vmcs_load(shadow_vmcs); 1354 1355 for (q = 0; q < ARRAY_SIZE(fields); q++) { 1356 for (i = 0; i < max_fields[q]; i++) { 1357 field = fields[q][i]; 1358 vmcs12_read_any(get_vmcs12(&vmx->vcpu), field, &field_value); 1359 __vmcs_writel(field, field_value); 1360 } 1361 } 1362 1363 vmcs_clear(shadow_vmcs); 1364 vmcs_load(vmx->loaded_vmcs->vmcs); 1365 } 1366 1367 static int copy_enlightened_to_vmcs12(struct vcpu_vmx *vmx) 1368 { 1369 struct vmcs12 *vmcs12 = vmx->nested.cached_vmcs12; 1370 struct hv_enlightened_vmcs *evmcs = vmx->nested.hv_evmcs; 1371 1372 /* HV_VMX_ENLIGHTENED_CLEAN_FIELD_NONE */ 1373 vmcs12->tpr_threshold = evmcs->tpr_threshold; 1374 vmcs12->guest_rip = evmcs->guest_rip; 1375 1376 if (unlikely(!(evmcs->hv_clean_fields & 1377 HV_VMX_ENLIGHTENED_CLEAN_FIELD_GUEST_BASIC))) { 1378 vmcs12->guest_rsp = evmcs->guest_rsp; 1379 vmcs12->guest_rflags = evmcs->guest_rflags; 1380 vmcs12->guest_interruptibility_info = 1381 evmcs->guest_interruptibility_info; 1382 } 1383 1384 if (unlikely(!(evmcs->hv_clean_fields & 1385 HV_VMX_ENLIGHTENED_CLEAN_FIELD_CONTROL_PROC))) { 1386 vmcs12->cpu_based_vm_exec_control = 1387 evmcs->cpu_based_vm_exec_control; 1388 } 1389 1390 if (unlikely(!(evmcs->hv_clean_fields & 1391 HV_VMX_ENLIGHTENED_CLEAN_FIELD_CONTROL_PROC))) { 1392 vmcs12->exception_bitmap = evmcs->exception_bitmap; 1393 } 1394 1395 if (unlikely(!(evmcs->hv_clean_fields & 1396 HV_VMX_ENLIGHTENED_CLEAN_FIELD_CONTROL_ENTRY))) { 1397 vmcs12->vm_entry_controls = evmcs->vm_entry_controls; 1398 } 1399 1400 if (unlikely(!(evmcs->hv_clean_fields & 1401 HV_VMX_ENLIGHTENED_CLEAN_FIELD_CONTROL_EVENT))) { 1402 vmcs12->vm_entry_intr_info_field = 1403 evmcs->vm_entry_intr_info_field; 1404 vmcs12->vm_entry_exception_error_code = 1405 evmcs->vm_entry_exception_error_code; 1406 vmcs12->vm_entry_instruction_len = 1407 evmcs->vm_entry_instruction_len; 1408 } 1409 1410 if (unlikely(!(evmcs->hv_clean_fields & 1411 HV_VMX_ENLIGHTENED_CLEAN_FIELD_HOST_GRP1))) { 1412 vmcs12->host_ia32_pat = evmcs->host_ia32_pat; 1413 vmcs12->host_ia32_efer = evmcs->host_ia32_efer; 1414 vmcs12->host_cr0 = evmcs->host_cr0; 1415 vmcs12->host_cr3 = evmcs->host_cr3; 1416 vmcs12->host_cr4 = evmcs->host_cr4; 1417 vmcs12->host_ia32_sysenter_esp = evmcs->host_ia32_sysenter_esp; 1418 vmcs12->host_ia32_sysenter_eip = evmcs->host_ia32_sysenter_eip; 1419 vmcs12->host_rip = evmcs->host_rip; 1420 vmcs12->host_ia32_sysenter_cs = evmcs->host_ia32_sysenter_cs; 1421 vmcs12->host_es_selector = evmcs->host_es_selector; 1422 vmcs12->host_cs_selector = evmcs->host_cs_selector; 1423 vmcs12->host_ss_selector = evmcs->host_ss_selector; 1424 vmcs12->host_ds_selector = evmcs->host_ds_selector; 1425 vmcs12->host_fs_selector = evmcs->host_fs_selector; 1426 vmcs12->host_gs_selector = evmcs->host_gs_selector; 1427 vmcs12->host_tr_selector = evmcs->host_tr_selector; 1428 } 1429 1430 if (unlikely(!(evmcs->hv_clean_fields & 1431 HV_VMX_ENLIGHTENED_CLEAN_FIELD_HOST_GRP1))) { 1432 vmcs12->pin_based_vm_exec_control = 1433 evmcs->pin_based_vm_exec_control; 1434 vmcs12->vm_exit_controls = evmcs->vm_exit_controls; 1435 vmcs12->secondary_vm_exec_control = 1436 evmcs->secondary_vm_exec_control; 1437 } 1438 1439 if (unlikely(!(evmcs->hv_clean_fields & 1440 HV_VMX_ENLIGHTENED_CLEAN_FIELD_IO_BITMAP))) { 1441 vmcs12->io_bitmap_a = evmcs->io_bitmap_a; 1442 vmcs12->io_bitmap_b = evmcs->io_bitmap_b; 1443 } 1444 1445 if (unlikely(!(evmcs->hv_clean_fields & 1446 HV_VMX_ENLIGHTENED_CLEAN_FIELD_MSR_BITMAP))) { 1447 vmcs12->msr_bitmap = evmcs->msr_bitmap; 1448 } 1449 1450 if (unlikely(!(evmcs->hv_clean_fields & 1451 HV_VMX_ENLIGHTENED_CLEAN_FIELD_GUEST_GRP2))) { 1452 vmcs12->guest_es_base = evmcs->guest_es_base; 1453 vmcs12->guest_cs_base = evmcs->guest_cs_base; 1454 vmcs12->guest_ss_base = evmcs->guest_ss_base; 1455 vmcs12->guest_ds_base = evmcs->guest_ds_base; 1456 vmcs12->guest_fs_base = evmcs->guest_fs_base; 1457 vmcs12->guest_gs_base = evmcs->guest_gs_base; 1458 vmcs12->guest_ldtr_base = evmcs->guest_ldtr_base; 1459 vmcs12->guest_tr_base = evmcs->guest_tr_base; 1460 vmcs12->guest_gdtr_base = evmcs->guest_gdtr_base; 1461 vmcs12->guest_idtr_base = evmcs->guest_idtr_base; 1462 vmcs12->guest_es_limit = evmcs->guest_es_limit; 1463 vmcs12->guest_cs_limit = evmcs->guest_cs_limit; 1464 vmcs12->guest_ss_limit = evmcs->guest_ss_limit; 1465 vmcs12->guest_ds_limit = evmcs->guest_ds_limit; 1466 vmcs12->guest_fs_limit = evmcs->guest_fs_limit; 1467 vmcs12->guest_gs_limit = evmcs->guest_gs_limit; 1468 vmcs12->guest_ldtr_limit = evmcs->guest_ldtr_limit; 1469 vmcs12->guest_tr_limit = evmcs->guest_tr_limit; 1470 vmcs12->guest_gdtr_limit = evmcs->guest_gdtr_limit; 1471 vmcs12->guest_idtr_limit = evmcs->guest_idtr_limit; 1472 vmcs12->guest_es_ar_bytes = evmcs->guest_es_ar_bytes; 1473 vmcs12->guest_cs_ar_bytes = evmcs->guest_cs_ar_bytes; 1474 vmcs12->guest_ss_ar_bytes = evmcs->guest_ss_ar_bytes; 1475 vmcs12->guest_ds_ar_bytes = evmcs->guest_ds_ar_bytes; 1476 vmcs12->guest_fs_ar_bytes = evmcs->guest_fs_ar_bytes; 1477 vmcs12->guest_gs_ar_bytes = evmcs->guest_gs_ar_bytes; 1478 vmcs12->guest_ldtr_ar_bytes = evmcs->guest_ldtr_ar_bytes; 1479 vmcs12->guest_tr_ar_bytes = evmcs->guest_tr_ar_bytes; 1480 vmcs12->guest_es_selector = evmcs->guest_es_selector; 1481 vmcs12->guest_cs_selector = evmcs->guest_cs_selector; 1482 vmcs12->guest_ss_selector = evmcs->guest_ss_selector; 1483 vmcs12->guest_ds_selector = evmcs->guest_ds_selector; 1484 vmcs12->guest_fs_selector = evmcs->guest_fs_selector; 1485 vmcs12->guest_gs_selector = evmcs->guest_gs_selector; 1486 vmcs12->guest_ldtr_selector = evmcs->guest_ldtr_selector; 1487 vmcs12->guest_tr_selector = evmcs->guest_tr_selector; 1488 } 1489 1490 if (unlikely(!(evmcs->hv_clean_fields & 1491 HV_VMX_ENLIGHTENED_CLEAN_FIELD_CONTROL_GRP2))) { 1492 vmcs12->tsc_offset = evmcs->tsc_offset; 1493 vmcs12->virtual_apic_page_addr = evmcs->virtual_apic_page_addr; 1494 vmcs12->xss_exit_bitmap = evmcs->xss_exit_bitmap; 1495 } 1496 1497 if (unlikely(!(evmcs->hv_clean_fields & 1498 HV_VMX_ENLIGHTENED_CLEAN_FIELD_CRDR))) { 1499 vmcs12->cr0_guest_host_mask = evmcs->cr0_guest_host_mask; 1500 vmcs12->cr4_guest_host_mask = evmcs->cr4_guest_host_mask; 1501 vmcs12->cr0_read_shadow = evmcs->cr0_read_shadow; 1502 vmcs12->cr4_read_shadow = evmcs->cr4_read_shadow; 1503 vmcs12->guest_cr0 = evmcs->guest_cr0; 1504 vmcs12->guest_cr3 = evmcs->guest_cr3; 1505 vmcs12->guest_cr4 = evmcs->guest_cr4; 1506 vmcs12->guest_dr7 = evmcs->guest_dr7; 1507 } 1508 1509 if (unlikely(!(evmcs->hv_clean_fields & 1510 HV_VMX_ENLIGHTENED_CLEAN_FIELD_HOST_POINTER))) { 1511 vmcs12->host_fs_base = evmcs->host_fs_base; 1512 vmcs12->host_gs_base = evmcs->host_gs_base; 1513 vmcs12->host_tr_base = evmcs->host_tr_base; 1514 vmcs12->host_gdtr_base = evmcs->host_gdtr_base; 1515 vmcs12->host_idtr_base = evmcs->host_idtr_base; 1516 vmcs12->host_rsp = evmcs->host_rsp; 1517 } 1518 1519 if (unlikely(!(evmcs->hv_clean_fields & 1520 HV_VMX_ENLIGHTENED_CLEAN_FIELD_CONTROL_XLAT))) { 1521 vmcs12->ept_pointer = evmcs->ept_pointer; 1522 vmcs12->virtual_processor_id = evmcs->virtual_processor_id; 1523 } 1524 1525 if (unlikely(!(evmcs->hv_clean_fields & 1526 HV_VMX_ENLIGHTENED_CLEAN_FIELD_GUEST_GRP1))) { 1527 vmcs12->vmcs_link_pointer = evmcs->vmcs_link_pointer; 1528 vmcs12->guest_ia32_debugctl = evmcs->guest_ia32_debugctl; 1529 vmcs12->guest_ia32_pat = evmcs->guest_ia32_pat; 1530 vmcs12->guest_ia32_efer = evmcs->guest_ia32_efer; 1531 vmcs12->guest_pdptr0 = evmcs->guest_pdptr0; 1532 vmcs12->guest_pdptr1 = evmcs->guest_pdptr1; 1533 vmcs12->guest_pdptr2 = evmcs->guest_pdptr2; 1534 vmcs12->guest_pdptr3 = evmcs->guest_pdptr3; 1535 vmcs12->guest_pending_dbg_exceptions = 1536 evmcs->guest_pending_dbg_exceptions; 1537 vmcs12->guest_sysenter_esp = evmcs->guest_sysenter_esp; 1538 vmcs12->guest_sysenter_eip = evmcs->guest_sysenter_eip; 1539 vmcs12->guest_bndcfgs = evmcs->guest_bndcfgs; 1540 vmcs12->guest_activity_state = evmcs->guest_activity_state; 1541 vmcs12->guest_sysenter_cs = evmcs->guest_sysenter_cs; 1542 } 1543 1544 /* 1545 * Not used? 1546 * vmcs12->vm_exit_msr_store_addr = evmcs->vm_exit_msr_store_addr; 1547 * vmcs12->vm_exit_msr_load_addr = evmcs->vm_exit_msr_load_addr; 1548 * vmcs12->vm_entry_msr_load_addr = evmcs->vm_entry_msr_load_addr; 1549 * vmcs12->cr3_target_value0 = evmcs->cr3_target_value0; 1550 * vmcs12->cr3_target_value1 = evmcs->cr3_target_value1; 1551 * vmcs12->cr3_target_value2 = evmcs->cr3_target_value2; 1552 * vmcs12->cr3_target_value3 = evmcs->cr3_target_value3; 1553 * vmcs12->page_fault_error_code_mask = 1554 * evmcs->page_fault_error_code_mask; 1555 * vmcs12->page_fault_error_code_match = 1556 * evmcs->page_fault_error_code_match; 1557 * vmcs12->cr3_target_count = evmcs->cr3_target_count; 1558 * vmcs12->vm_exit_msr_store_count = evmcs->vm_exit_msr_store_count; 1559 * vmcs12->vm_exit_msr_load_count = evmcs->vm_exit_msr_load_count; 1560 * vmcs12->vm_entry_msr_load_count = evmcs->vm_entry_msr_load_count; 1561 */ 1562 1563 /* 1564 * Read only fields: 1565 * vmcs12->guest_physical_address = evmcs->guest_physical_address; 1566 * vmcs12->vm_instruction_error = evmcs->vm_instruction_error; 1567 * vmcs12->vm_exit_reason = evmcs->vm_exit_reason; 1568 * vmcs12->vm_exit_intr_info = evmcs->vm_exit_intr_info; 1569 * vmcs12->vm_exit_intr_error_code = evmcs->vm_exit_intr_error_code; 1570 * vmcs12->idt_vectoring_info_field = evmcs->idt_vectoring_info_field; 1571 * vmcs12->idt_vectoring_error_code = evmcs->idt_vectoring_error_code; 1572 * vmcs12->vm_exit_instruction_len = evmcs->vm_exit_instruction_len; 1573 * vmcs12->vmx_instruction_info = evmcs->vmx_instruction_info; 1574 * vmcs12->exit_qualification = evmcs->exit_qualification; 1575 * vmcs12->guest_linear_address = evmcs->guest_linear_address; 1576 * 1577 * Not present in struct vmcs12: 1578 * vmcs12->exit_io_instruction_ecx = evmcs->exit_io_instruction_ecx; 1579 * vmcs12->exit_io_instruction_esi = evmcs->exit_io_instruction_esi; 1580 * vmcs12->exit_io_instruction_edi = evmcs->exit_io_instruction_edi; 1581 * vmcs12->exit_io_instruction_eip = evmcs->exit_io_instruction_eip; 1582 */ 1583 1584 return 0; 1585 } 1586 1587 static int copy_vmcs12_to_enlightened(struct vcpu_vmx *vmx) 1588 { 1589 struct vmcs12 *vmcs12 = vmx->nested.cached_vmcs12; 1590 struct hv_enlightened_vmcs *evmcs = vmx->nested.hv_evmcs; 1591 1592 /* 1593 * Should not be changed by KVM: 1594 * 1595 * evmcs->host_es_selector = vmcs12->host_es_selector; 1596 * evmcs->host_cs_selector = vmcs12->host_cs_selector; 1597 * evmcs->host_ss_selector = vmcs12->host_ss_selector; 1598 * evmcs->host_ds_selector = vmcs12->host_ds_selector; 1599 * evmcs->host_fs_selector = vmcs12->host_fs_selector; 1600 * evmcs->host_gs_selector = vmcs12->host_gs_selector; 1601 * evmcs->host_tr_selector = vmcs12->host_tr_selector; 1602 * evmcs->host_ia32_pat = vmcs12->host_ia32_pat; 1603 * evmcs->host_ia32_efer = vmcs12->host_ia32_efer; 1604 * evmcs->host_cr0 = vmcs12->host_cr0; 1605 * evmcs->host_cr3 = vmcs12->host_cr3; 1606 * evmcs->host_cr4 = vmcs12->host_cr4; 1607 * evmcs->host_ia32_sysenter_esp = vmcs12->host_ia32_sysenter_esp; 1608 * evmcs->host_ia32_sysenter_eip = vmcs12->host_ia32_sysenter_eip; 1609 * evmcs->host_rip = vmcs12->host_rip; 1610 * evmcs->host_ia32_sysenter_cs = vmcs12->host_ia32_sysenter_cs; 1611 * evmcs->host_fs_base = vmcs12->host_fs_base; 1612 * evmcs->host_gs_base = vmcs12->host_gs_base; 1613 * evmcs->host_tr_base = vmcs12->host_tr_base; 1614 * evmcs->host_gdtr_base = vmcs12->host_gdtr_base; 1615 * evmcs->host_idtr_base = vmcs12->host_idtr_base; 1616 * evmcs->host_rsp = vmcs12->host_rsp; 1617 * sync_vmcs12() doesn't read these: 1618 * evmcs->io_bitmap_a = vmcs12->io_bitmap_a; 1619 * evmcs->io_bitmap_b = vmcs12->io_bitmap_b; 1620 * evmcs->msr_bitmap = vmcs12->msr_bitmap; 1621 * evmcs->ept_pointer = vmcs12->ept_pointer; 1622 * evmcs->xss_exit_bitmap = vmcs12->xss_exit_bitmap; 1623 * evmcs->vm_exit_msr_store_addr = vmcs12->vm_exit_msr_store_addr; 1624 * evmcs->vm_exit_msr_load_addr = vmcs12->vm_exit_msr_load_addr; 1625 * evmcs->vm_entry_msr_load_addr = vmcs12->vm_entry_msr_load_addr; 1626 * evmcs->cr3_target_value0 = vmcs12->cr3_target_value0; 1627 * evmcs->cr3_target_value1 = vmcs12->cr3_target_value1; 1628 * evmcs->cr3_target_value2 = vmcs12->cr3_target_value2; 1629 * evmcs->cr3_target_value3 = vmcs12->cr3_target_value3; 1630 * evmcs->tpr_threshold = vmcs12->tpr_threshold; 1631 * evmcs->virtual_processor_id = vmcs12->virtual_processor_id; 1632 * evmcs->exception_bitmap = vmcs12->exception_bitmap; 1633 * evmcs->vmcs_link_pointer = vmcs12->vmcs_link_pointer; 1634 * evmcs->pin_based_vm_exec_control = vmcs12->pin_based_vm_exec_control; 1635 * evmcs->vm_exit_controls = vmcs12->vm_exit_controls; 1636 * evmcs->secondary_vm_exec_control = vmcs12->secondary_vm_exec_control; 1637 * evmcs->page_fault_error_code_mask = 1638 * vmcs12->page_fault_error_code_mask; 1639 * evmcs->page_fault_error_code_match = 1640 * vmcs12->page_fault_error_code_match; 1641 * evmcs->cr3_target_count = vmcs12->cr3_target_count; 1642 * evmcs->virtual_apic_page_addr = vmcs12->virtual_apic_page_addr; 1643 * evmcs->tsc_offset = vmcs12->tsc_offset; 1644 * evmcs->guest_ia32_debugctl = vmcs12->guest_ia32_debugctl; 1645 * evmcs->cr0_guest_host_mask = vmcs12->cr0_guest_host_mask; 1646 * evmcs->cr4_guest_host_mask = vmcs12->cr4_guest_host_mask; 1647 * evmcs->cr0_read_shadow = vmcs12->cr0_read_shadow; 1648 * evmcs->cr4_read_shadow = vmcs12->cr4_read_shadow; 1649 * evmcs->vm_exit_msr_store_count = vmcs12->vm_exit_msr_store_count; 1650 * evmcs->vm_exit_msr_load_count = vmcs12->vm_exit_msr_load_count; 1651 * evmcs->vm_entry_msr_load_count = vmcs12->vm_entry_msr_load_count; 1652 * 1653 * Not present in struct vmcs12: 1654 * evmcs->exit_io_instruction_ecx = vmcs12->exit_io_instruction_ecx; 1655 * evmcs->exit_io_instruction_esi = vmcs12->exit_io_instruction_esi; 1656 * evmcs->exit_io_instruction_edi = vmcs12->exit_io_instruction_edi; 1657 * evmcs->exit_io_instruction_eip = vmcs12->exit_io_instruction_eip; 1658 */ 1659 1660 evmcs->guest_es_selector = vmcs12->guest_es_selector; 1661 evmcs->guest_cs_selector = vmcs12->guest_cs_selector; 1662 evmcs->guest_ss_selector = vmcs12->guest_ss_selector; 1663 evmcs->guest_ds_selector = vmcs12->guest_ds_selector; 1664 evmcs->guest_fs_selector = vmcs12->guest_fs_selector; 1665 evmcs->guest_gs_selector = vmcs12->guest_gs_selector; 1666 evmcs->guest_ldtr_selector = vmcs12->guest_ldtr_selector; 1667 evmcs->guest_tr_selector = vmcs12->guest_tr_selector; 1668 1669 evmcs->guest_es_limit = vmcs12->guest_es_limit; 1670 evmcs->guest_cs_limit = vmcs12->guest_cs_limit; 1671 evmcs->guest_ss_limit = vmcs12->guest_ss_limit; 1672 evmcs->guest_ds_limit = vmcs12->guest_ds_limit; 1673 evmcs->guest_fs_limit = vmcs12->guest_fs_limit; 1674 evmcs->guest_gs_limit = vmcs12->guest_gs_limit; 1675 evmcs->guest_ldtr_limit = vmcs12->guest_ldtr_limit; 1676 evmcs->guest_tr_limit = vmcs12->guest_tr_limit; 1677 evmcs->guest_gdtr_limit = vmcs12->guest_gdtr_limit; 1678 evmcs->guest_idtr_limit = vmcs12->guest_idtr_limit; 1679 1680 evmcs->guest_es_ar_bytes = vmcs12->guest_es_ar_bytes; 1681 evmcs->guest_cs_ar_bytes = vmcs12->guest_cs_ar_bytes; 1682 evmcs->guest_ss_ar_bytes = vmcs12->guest_ss_ar_bytes; 1683 evmcs->guest_ds_ar_bytes = vmcs12->guest_ds_ar_bytes; 1684 evmcs->guest_fs_ar_bytes = vmcs12->guest_fs_ar_bytes; 1685 evmcs->guest_gs_ar_bytes = vmcs12->guest_gs_ar_bytes; 1686 evmcs->guest_ldtr_ar_bytes = vmcs12->guest_ldtr_ar_bytes; 1687 evmcs->guest_tr_ar_bytes = vmcs12->guest_tr_ar_bytes; 1688 1689 evmcs->guest_es_base = vmcs12->guest_es_base; 1690 evmcs->guest_cs_base = vmcs12->guest_cs_base; 1691 evmcs->guest_ss_base = vmcs12->guest_ss_base; 1692 evmcs->guest_ds_base = vmcs12->guest_ds_base; 1693 evmcs->guest_fs_base = vmcs12->guest_fs_base; 1694 evmcs->guest_gs_base = vmcs12->guest_gs_base; 1695 evmcs->guest_ldtr_base = vmcs12->guest_ldtr_base; 1696 evmcs->guest_tr_base = vmcs12->guest_tr_base; 1697 evmcs->guest_gdtr_base = vmcs12->guest_gdtr_base; 1698 evmcs->guest_idtr_base = vmcs12->guest_idtr_base; 1699 1700 evmcs->guest_ia32_pat = vmcs12->guest_ia32_pat; 1701 evmcs->guest_ia32_efer = vmcs12->guest_ia32_efer; 1702 1703 evmcs->guest_pdptr0 = vmcs12->guest_pdptr0; 1704 evmcs->guest_pdptr1 = vmcs12->guest_pdptr1; 1705 evmcs->guest_pdptr2 = vmcs12->guest_pdptr2; 1706 evmcs->guest_pdptr3 = vmcs12->guest_pdptr3; 1707 1708 evmcs->guest_pending_dbg_exceptions = 1709 vmcs12->guest_pending_dbg_exceptions; 1710 evmcs->guest_sysenter_esp = vmcs12->guest_sysenter_esp; 1711 evmcs->guest_sysenter_eip = vmcs12->guest_sysenter_eip; 1712 1713 evmcs->guest_activity_state = vmcs12->guest_activity_state; 1714 evmcs->guest_sysenter_cs = vmcs12->guest_sysenter_cs; 1715 1716 evmcs->guest_cr0 = vmcs12->guest_cr0; 1717 evmcs->guest_cr3 = vmcs12->guest_cr3; 1718 evmcs->guest_cr4 = vmcs12->guest_cr4; 1719 evmcs->guest_dr7 = vmcs12->guest_dr7; 1720 1721 evmcs->guest_physical_address = vmcs12->guest_physical_address; 1722 1723 evmcs->vm_instruction_error = vmcs12->vm_instruction_error; 1724 evmcs->vm_exit_reason = vmcs12->vm_exit_reason; 1725 evmcs->vm_exit_intr_info = vmcs12->vm_exit_intr_info; 1726 evmcs->vm_exit_intr_error_code = vmcs12->vm_exit_intr_error_code; 1727 evmcs->idt_vectoring_info_field = vmcs12->idt_vectoring_info_field; 1728 evmcs->idt_vectoring_error_code = vmcs12->idt_vectoring_error_code; 1729 evmcs->vm_exit_instruction_len = vmcs12->vm_exit_instruction_len; 1730 evmcs->vmx_instruction_info = vmcs12->vmx_instruction_info; 1731 1732 evmcs->exit_qualification = vmcs12->exit_qualification; 1733 1734 evmcs->guest_linear_address = vmcs12->guest_linear_address; 1735 evmcs->guest_rsp = vmcs12->guest_rsp; 1736 evmcs->guest_rflags = vmcs12->guest_rflags; 1737 1738 evmcs->guest_interruptibility_info = 1739 vmcs12->guest_interruptibility_info; 1740 evmcs->cpu_based_vm_exec_control = vmcs12->cpu_based_vm_exec_control; 1741 evmcs->vm_entry_controls = vmcs12->vm_entry_controls; 1742 evmcs->vm_entry_intr_info_field = vmcs12->vm_entry_intr_info_field; 1743 evmcs->vm_entry_exception_error_code = 1744 vmcs12->vm_entry_exception_error_code; 1745 evmcs->vm_entry_instruction_len = vmcs12->vm_entry_instruction_len; 1746 1747 evmcs->guest_rip = vmcs12->guest_rip; 1748 1749 evmcs->guest_bndcfgs = vmcs12->guest_bndcfgs; 1750 1751 return 0; 1752 } 1753 1754 /* 1755 * This is an equivalent of the nested hypervisor executing the vmptrld 1756 * instruction. 1757 */ 1758 static int nested_vmx_handle_enlightened_vmptrld(struct kvm_vcpu *vcpu, 1759 bool from_launch) 1760 { 1761 struct vcpu_vmx *vmx = to_vmx(vcpu); 1762 struct hv_vp_assist_page assist_page; 1763 1764 if (likely(!vmx->nested.enlightened_vmcs_enabled)) 1765 return 1; 1766 1767 if (unlikely(!kvm_hv_get_assist_page(vcpu, &assist_page))) 1768 return 1; 1769 1770 if (unlikely(!assist_page.enlighten_vmentry)) 1771 return 1; 1772 1773 if (unlikely(assist_page.current_nested_vmcs != 1774 vmx->nested.hv_evmcs_vmptr)) { 1775 1776 if (!vmx->nested.hv_evmcs) 1777 vmx->nested.current_vmptr = -1ull; 1778 1779 nested_release_evmcs(vcpu); 1780 1781 vmx->nested.hv_evmcs_page = kvm_vcpu_gpa_to_page( 1782 vcpu, assist_page.current_nested_vmcs); 1783 1784 if (unlikely(is_error_page(vmx->nested.hv_evmcs_page))) 1785 return 0; 1786 1787 vmx->nested.hv_evmcs = kmap(vmx->nested.hv_evmcs_page); 1788 1789 /* 1790 * Currently, KVM only supports eVMCS version 1 1791 * (== KVM_EVMCS_VERSION) and thus we expect guest to set this 1792 * value to first u32 field of eVMCS which should specify eVMCS 1793 * VersionNumber. 1794 * 1795 * Guest should be aware of supported eVMCS versions by host by 1796 * examining CPUID.0x4000000A.EAX[0:15]. Host userspace VMM is 1797 * expected to set this CPUID leaf according to the value 1798 * returned in vmcs_version from nested_enable_evmcs(). 1799 * 1800 * However, it turns out that Microsoft Hyper-V fails to comply 1801 * to their own invented interface: When Hyper-V use eVMCS, it 1802 * just sets first u32 field of eVMCS to revision_id specified 1803 * in MSR_IA32_VMX_BASIC. Instead of used eVMCS version number 1804 * which is one of the supported versions specified in 1805 * CPUID.0x4000000A.EAX[0:15]. 1806 * 1807 * To overcome Hyper-V bug, we accept here either a supported 1808 * eVMCS version or VMCS12 revision_id as valid values for first 1809 * u32 field of eVMCS. 1810 */ 1811 if ((vmx->nested.hv_evmcs->revision_id != KVM_EVMCS_VERSION) && 1812 (vmx->nested.hv_evmcs->revision_id != VMCS12_REVISION)) { 1813 nested_release_evmcs(vcpu); 1814 return 0; 1815 } 1816 1817 vmx->nested.dirty_vmcs12 = true; 1818 /* 1819 * As we keep L2 state for one guest only 'hv_clean_fields' mask 1820 * can't be used when we switch between them. Reset it here for 1821 * simplicity. 1822 */ 1823 vmx->nested.hv_evmcs->hv_clean_fields &= 1824 ~HV_VMX_ENLIGHTENED_CLEAN_FIELD_ALL; 1825 vmx->nested.hv_evmcs_vmptr = assist_page.current_nested_vmcs; 1826 1827 /* 1828 * Unlike normal vmcs12, enlightened vmcs12 is not fully 1829 * reloaded from guest's memory (read only fields, fields not 1830 * present in struct hv_enlightened_vmcs, ...). Make sure there 1831 * are no leftovers. 1832 */ 1833 if (from_launch) { 1834 struct vmcs12 *vmcs12 = get_vmcs12(vcpu); 1835 memset(vmcs12, 0, sizeof(*vmcs12)); 1836 vmcs12->hdr.revision_id = VMCS12_REVISION; 1837 } 1838 1839 } 1840 return 1; 1841 } 1842 1843 void nested_sync_from_vmcs12(struct kvm_vcpu *vcpu) 1844 { 1845 struct vcpu_vmx *vmx = to_vmx(vcpu); 1846 1847 /* 1848 * hv_evmcs may end up being not mapped after migration (when 1849 * L2 was running), map it here to make sure vmcs12 changes are 1850 * properly reflected. 1851 */ 1852 if (vmx->nested.enlightened_vmcs_enabled && !vmx->nested.hv_evmcs) 1853 nested_vmx_handle_enlightened_vmptrld(vcpu, false); 1854 1855 if (vmx->nested.hv_evmcs) { 1856 copy_vmcs12_to_enlightened(vmx); 1857 /* All fields are clean */ 1858 vmx->nested.hv_evmcs->hv_clean_fields |= 1859 HV_VMX_ENLIGHTENED_CLEAN_FIELD_ALL; 1860 } else { 1861 copy_vmcs12_to_shadow(vmx); 1862 } 1863 1864 vmx->nested.need_vmcs12_sync = false; 1865 } 1866 1867 static enum hrtimer_restart vmx_preemption_timer_fn(struct hrtimer *timer) 1868 { 1869 struct vcpu_vmx *vmx = 1870 container_of(timer, struct vcpu_vmx, nested.preemption_timer); 1871 1872 vmx->nested.preemption_timer_expired = true; 1873 kvm_make_request(KVM_REQ_EVENT, &vmx->vcpu); 1874 kvm_vcpu_kick(&vmx->vcpu); 1875 1876 return HRTIMER_NORESTART; 1877 } 1878 1879 static void vmx_start_preemption_timer(struct kvm_vcpu *vcpu) 1880 { 1881 u64 preemption_timeout = get_vmcs12(vcpu)->vmx_preemption_timer_value; 1882 struct vcpu_vmx *vmx = to_vmx(vcpu); 1883 1884 /* 1885 * A timer value of zero is architecturally guaranteed to cause 1886 * a VMExit prior to executing any instructions in the guest. 1887 */ 1888 if (preemption_timeout == 0) { 1889 vmx_preemption_timer_fn(&vmx->nested.preemption_timer); 1890 return; 1891 } 1892 1893 if (vcpu->arch.virtual_tsc_khz == 0) 1894 return; 1895 1896 preemption_timeout <<= VMX_MISC_EMULATED_PREEMPTION_TIMER_RATE; 1897 preemption_timeout *= 1000000; 1898 do_div(preemption_timeout, vcpu->arch.virtual_tsc_khz); 1899 hrtimer_start(&vmx->nested.preemption_timer, 1900 ns_to_ktime(preemption_timeout), HRTIMER_MODE_REL); 1901 } 1902 1903 static u64 nested_vmx_calc_efer(struct vcpu_vmx *vmx, struct vmcs12 *vmcs12) 1904 { 1905 if (vmx->nested.nested_run_pending && 1906 (vmcs12->vm_entry_controls & VM_ENTRY_LOAD_IA32_EFER)) 1907 return vmcs12->guest_ia32_efer; 1908 else if (vmcs12->vm_entry_controls & VM_ENTRY_IA32E_MODE) 1909 return vmx->vcpu.arch.efer | (EFER_LMA | EFER_LME); 1910 else 1911 return vmx->vcpu.arch.efer & ~(EFER_LMA | EFER_LME); 1912 } 1913 1914 static void prepare_vmcs02_constant_state(struct vcpu_vmx *vmx) 1915 { 1916 /* 1917 * If vmcs02 hasn't been initialized, set the constant vmcs02 state 1918 * according to L0's settings (vmcs12 is irrelevant here). Host 1919 * fields that come from L0 and are not constant, e.g. HOST_CR3, 1920 * will be set as needed prior to VMLAUNCH/VMRESUME. 1921 */ 1922 if (vmx->nested.vmcs02_initialized) 1923 return; 1924 vmx->nested.vmcs02_initialized = true; 1925 1926 /* 1927 * We don't care what the EPTP value is we just need to guarantee 1928 * it's valid so we don't get a false positive when doing early 1929 * consistency checks. 1930 */ 1931 if (enable_ept && nested_early_check) 1932 vmcs_write64(EPT_POINTER, construct_eptp(&vmx->vcpu, 0)); 1933 1934 /* All VMFUNCs are currently emulated through L0 vmexits. */ 1935 if (cpu_has_vmx_vmfunc()) 1936 vmcs_write64(VM_FUNCTION_CONTROL, 0); 1937 1938 if (cpu_has_vmx_posted_intr()) 1939 vmcs_write16(POSTED_INTR_NV, POSTED_INTR_NESTED_VECTOR); 1940 1941 if (cpu_has_vmx_msr_bitmap()) 1942 vmcs_write64(MSR_BITMAP, __pa(vmx->nested.vmcs02.msr_bitmap)); 1943 1944 if (enable_pml) 1945 vmcs_write64(PML_ADDRESS, page_to_phys(vmx->pml_pg)); 1946 1947 /* 1948 * Set the MSR load/store lists to match L0's settings. Only the 1949 * addresses are constant (for vmcs02), the counts can change based 1950 * on L2's behavior, e.g. switching to/from long mode. 1951 */ 1952 vmcs_write32(VM_EXIT_MSR_STORE_COUNT, 0); 1953 vmcs_write64(VM_EXIT_MSR_LOAD_ADDR, __pa(vmx->msr_autoload.host.val)); 1954 vmcs_write64(VM_ENTRY_MSR_LOAD_ADDR, __pa(vmx->msr_autoload.guest.val)); 1955 1956 vmx_set_constant_host_state(vmx); 1957 } 1958 1959 static void prepare_vmcs02_early_full(struct vcpu_vmx *vmx, 1960 struct vmcs12 *vmcs12) 1961 { 1962 prepare_vmcs02_constant_state(vmx); 1963 1964 vmcs_write64(VMCS_LINK_POINTER, -1ull); 1965 1966 if (enable_vpid) { 1967 if (nested_cpu_has_vpid(vmcs12) && vmx->nested.vpid02) 1968 vmcs_write16(VIRTUAL_PROCESSOR_ID, vmx->nested.vpid02); 1969 else 1970 vmcs_write16(VIRTUAL_PROCESSOR_ID, vmx->vpid); 1971 } 1972 } 1973 1974 static void prepare_vmcs02_early(struct vcpu_vmx *vmx, struct vmcs12 *vmcs12) 1975 { 1976 u32 exec_control, vmcs12_exec_ctrl; 1977 u64 guest_efer = nested_vmx_calc_efer(vmx, vmcs12); 1978 1979 if (vmx->nested.dirty_vmcs12 || vmx->nested.hv_evmcs) 1980 prepare_vmcs02_early_full(vmx, vmcs12); 1981 1982 /* 1983 * HOST_RSP is normally set correctly in vmx_vcpu_run() just before 1984 * entry, but only if the current (host) sp changed from the value 1985 * we wrote last (vmx->host_rsp). This cache is no longer relevant 1986 * if we switch vmcs, and rather than hold a separate cache per vmcs, 1987 * here we just force the write to happen on entry. host_rsp will 1988 * also be written unconditionally by nested_vmx_check_vmentry_hw() 1989 * if we are doing early consistency checks via hardware. 1990 */ 1991 vmx->host_rsp = 0; 1992 1993 /* 1994 * PIN CONTROLS 1995 */ 1996 exec_control = vmcs12->pin_based_vm_exec_control; 1997 1998 /* Preemption timer setting is computed directly in vmx_vcpu_run. */ 1999 exec_control |= vmcs_config.pin_based_exec_ctrl; 2000 exec_control &= ~PIN_BASED_VMX_PREEMPTION_TIMER; 2001 vmx->loaded_vmcs->hv_timer_armed = false; 2002 2003 /* Posted interrupts setting is only taken from vmcs12. */ 2004 if (nested_cpu_has_posted_intr(vmcs12)) { 2005 vmx->nested.posted_intr_nv = vmcs12->posted_intr_nv; 2006 vmx->nested.pi_pending = false; 2007 } else { 2008 exec_control &= ~PIN_BASED_POSTED_INTR; 2009 } 2010 vmcs_write32(PIN_BASED_VM_EXEC_CONTROL, exec_control); 2011 2012 /* 2013 * EXEC CONTROLS 2014 */ 2015 exec_control = vmx_exec_control(vmx); /* L0's desires */ 2016 exec_control &= ~CPU_BASED_VIRTUAL_INTR_PENDING; 2017 exec_control &= ~CPU_BASED_VIRTUAL_NMI_PENDING; 2018 exec_control &= ~CPU_BASED_TPR_SHADOW; 2019 exec_control |= vmcs12->cpu_based_vm_exec_control; 2020 2021 /* 2022 * Write an illegal value to VIRTUAL_APIC_PAGE_ADDR. Later, if 2023 * nested_get_vmcs12_pages can't fix it up, the illegal value 2024 * will result in a VM entry failure. 2025 */ 2026 if (exec_control & CPU_BASED_TPR_SHADOW) { 2027 vmcs_write64(VIRTUAL_APIC_PAGE_ADDR, -1ull); 2028 vmcs_write32(TPR_THRESHOLD, vmcs12->tpr_threshold); 2029 } else { 2030 #ifdef CONFIG_X86_64 2031 exec_control |= CPU_BASED_CR8_LOAD_EXITING | 2032 CPU_BASED_CR8_STORE_EXITING; 2033 #endif 2034 } 2035 2036 /* 2037 * A vmexit (to either L1 hypervisor or L0 userspace) is always needed 2038 * for I/O port accesses. 2039 */ 2040 exec_control &= ~CPU_BASED_USE_IO_BITMAPS; 2041 exec_control |= CPU_BASED_UNCOND_IO_EXITING; 2042 vmcs_write32(CPU_BASED_VM_EXEC_CONTROL, exec_control); 2043 2044 /* 2045 * SECONDARY EXEC CONTROLS 2046 */ 2047 if (cpu_has_secondary_exec_ctrls()) { 2048 exec_control = vmx->secondary_exec_control; 2049 2050 /* Take the following fields only from vmcs12 */ 2051 exec_control &= ~(SECONDARY_EXEC_VIRTUALIZE_APIC_ACCESSES | 2052 SECONDARY_EXEC_ENABLE_INVPCID | 2053 SECONDARY_EXEC_RDTSCP | 2054 SECONDARY_EXEC_XSAVES | 2055 SECONDARY_EXEC_VIRTUAL_INTR_DELIVERY | 2056 SECONDARY_EXEC_APIC_REGISTER_VIRT | 2057 SECONDARY_EXEC_ENABLE_VMFUNC); 2058 if (nested_cpu_has(vmcs12, 2059 CPU_BASED_ACTIVATE_SECONDARY_CONTROLS)) { 2060 vmcs12_exec_ctrl = vmcs12->secondary_vm_exec_control & 2061 ~SECONDARY_EXEC_ENABLE_PML; 2062 exec_control |= vmcs12_exec_ctrl; 2063 } 2064 2065 /* VMCS shadowing for L2 is emulated for now */ 2066 exec_control &= ~SECONDARY_EXEC_SHADOW_VMCS; 2067 2068 if (exec_control & SECONDARY_EXEC_VIRTUAL_INTR_DELIVERY) 2069 vmcs_write16(GUEST_INTR_STATUS, 2070 vmcs12->guest_intr_status); 2071 2072 /* 2073 * Write an illegal value to APIC_ACCESS_ADDR. Later, 2074 * nested_get_vmcs12_pages will either fix it up or 2075 * remove the VM execution control. 2076 */ 2077 if (exec_control & SECONDARY_EXEC_VIRTUALIZE_APIC_ACCESSES) 2078 vmcs_write64(APIC_ACCESS_ADDR, -1ull); 2079 2080 if (exec_control & SECONDARY_EXEC_ENCLS_EXITING) 2081 vmcs_write64(ENCLS_EXITING_BITMAP, -1ull); 2082 2083 vmcs_write32(SECONDARY_VM_EXEC_CONTROL, exec_control); 2084 } 2085 2086 /* 2087 * ENTRY CONTROLS 2088 * 2089 * vmcs12's VM_{ENTRY,EXIT}_LOAD_IA32_EFER and VM_ENTRY_IA32E_MODE 2090 * are emulated by vmx_set_efer() in prepare_vmcs02(), but speculate 2091 * on the related bits (if supported by the CPU) in the hope that 2092 * we can avoid VMWrites during vmx_set_efer(). 2093 */ 2094 exec_control = (vmcs12->vm_entry_controls | vmx_vmentry_ctrl()) & 2095 ~VM_ENTRY_IA32E_MODE & ~VM_ENTRY_LOAD_IA32_EFER; 2096 if (cpu_has_load_ia32_efer()) { 2097 if (guest_efer & EFER_LMA) 2098 exec_control |= VM_ENTRY_IA32E_MODE; 2099 if (guest_efer != host_efer) 2100 exec_control |= VM_ENTRY_LOAD_IA32_EFER; 2101 } 2102 vm_entry_controls_init(vmx, exec_control); 2103 2104 /* 2105 * EXIT CONTROLS 2106 * 2107 * L2->L1 exit controls are emulated - the hardware exit is to L0 so 2108 * we should use its exit controls. Note that VM_EXIT_LOAD_IA32_EFER 2109 * bits may be modified by vmx_set_efer() in prepare_vmcs02(). 2110 */ 2111 exec_control = vmx_vmexit_ctrl(); 2112 if (cpu_has_load_ia32_efer() && guest_efer != host_efer) 2113 exec_control |= VM_EXIT_LOAD_IA32_EFER; 2114 vm_exit_controls_init(vmx, exec_control); 2115 2116 /* 2117 * Conceptually we want to copy the PML address and index from 2118 * vmcs01 here, and then back to vmcs01 on nested vmexit. But, 2119 * since we always flush the log on each vmexit and never change 2120 * the PML address (once set), this happens to be equivalent to 2121 * simply resetting the index in vmcs02. 2122 */ 2123 if (enable_pml) 2124 vmcs_write16(GUEST_PML_INDEX, PML_ENTITY_NUM - 1); 2125 2126 /* 2127 * Interrupt/Exception Fields 2128 */ 2129 if (vmx->nested.nested_run_pending) { 2130 vmcs_write32(VM_ENTRY_INTR_INFO_FIELD, 2131 vmcs12->vm_entry_intr_info_field); 2132 vmcs_write32(VM_ENTRY_EXCEPTION_ERROR_CODE, 2133 vmcs12->vm_entry_exception_error_code); 2134 vmcs_write32(VM_ENTRY_INSTRUCTION_LEN, 2135 vmcs12->vm_entry_instruction_len); 2136 vmcs_write32(GUEST_INTERRUPTIBILITY_INFO, 2137 vmcs12->guest_interruptibility_info); 2138 vmx->loaded_vmcs->nmi_known_unmasked = 2139 !(vmcs12->guest_interruptibility_info & GUEST_INTR_STATE_NMI); 2140 } else { 2141 vmcs_write32(VM_ENTRY_INTR_INFO_FIELD, 0); 2142 } 2143 } 2144 2145 static void prepare_vmcs02_full(struct vcpu_vmx *vmx, struct vmcs12 *vmcs12) 2146 { 2147 struct hv_enlightened_vmcs *hv_evmcs = vmx->nested.hv_evmcs; 2148 2149 if (!hv_evmcs || !(hv_evmcs->hv_clean_fields & 2150 HV_VMX_ENLIGHTENED_CLEAN_FIELD_GUEST_GRP2)) { 2151 vmcs_write16(GUEST_ES_SELECTOR, vmcs12->guest_es_selector); 2152 vmcs_write16(GUEST_CS_SELECTOR, vmcs12->guest_cs_selector); 2153 vmcs_write16(GUEST_SS_SELECTOR, vmcs12->guest_ss_selector); 2154 vmcs_write16(GUEST_DS_SELECTOR, vmcs12->guest_ds_selector); 2155 vmcs_write16(GUEST_FS_SELECTOR, vmcs12->guest_fs_selector); 2156 vmcs_write16(GUEST_GS_SELECTOR, vmcs12->guest_gs_selector); 2157 vmcs_write16(GUEST_LDTR_SELECTOR, vmcs12->guest_ldtr_selector); 2158 vmcs_write16(GUEST_TR_SELECTOR, vmcs12->guest_tr_selector); 2159 vmcs_write32(GUEST_ES_LIMIT, vmcs12->guest_es_limit); 2160 vmcs_write32(GUEST_CS_LIMIT, vmcs12->guest_cs_limit); 2161 vmcs_write32(GUEST_SS_LIMIT, vmcs12->guest_ss_limit); 2162 vmcs_write32(GUEST_DS_LIMIT, vmcs12->guest_ds_limit); 2163 vmcs_write32(GUEST_FS_LIMIT, vmcs12->guest_fs_limit); 2164 vmcs_write32(GUEST_GS_LIMIT, vmcs12->guest_gs_limit); 2165 vmcs_write32(GUEST_LDTR_LIMIT, vmcs12->guest_ldtr_limit); 2166 vmcs_write32(GUEST_TR_LIMIT, vmcs12->guest_tr_limit); 2167 vmcs_write32(GUEST_GDTR_LIMIT, vmcs12->guest_gdtr_limit); 2168 vmcs_write32(GUEST_IDTR_LIMIT, vmcs12->guest_idtr_limit); 2169 vmcs_write32(GUEST_ES_AR_BYTES, vmcs12->guest_es_ar_bytes); 2170 vmcs_write32(GUEST_DS_AR_BYTES, vmcs12->guest_ds_ar_bytes); 2171 vmcs_write32(GUEST_FS_AR_BYTES, vmcs12->guest_fs_ar_bytes); 2172 vmcs_write32(GUEST_GS_AR_BYTES, vmcs12->guest_gs_ar_bytes); 2173 vmcs_write32(GUEST_LDTR_AR_BYTES, vmcs12->guest_ldtr_ar_bytes); 2174 vmcs_write32(GUEST_TR_AR_BYTES, vmcs12->guest_tr_ar_bytes); 2175 vmcs_writel(GUEST_ES_BASE, vmcs12->guest_es_base); 2176 vmcs_writel(GUEST_CS_BASE, vmcs12->guest_cs_base); 2177 vmcs_writel(GUEST_SS_BASE, vmcs12->guest_ss_base); 2178 vmcs_writel(GUEST_DS_BASE, vmcs12->guest_ds_base); 2179 vmcs_writel(GUEST_FS_BASE, vmcs12->guest_fs_base); 2180 vmcs_writel(GUEST_GS_BASE, vmcs12->guest_gs_base); 2181 vmcs_writel(GUEST_LDTR_BASE, vmcs12->guest_ldtr_base); 2182 vmcs_writel(GUEST_TR_BASE, vmcs12->guest_tr_base); 2183 vmcs_writel(GUEST_GDTR_BASE, vmcs12->guest_gdtr_base); 2184 vmcs_writel(GUEST_IDTR_BASE, vmcs12->guest_idtr_base); 2185 } 2186 2187 if (!hv_evmcs || !(hv_evmcs->hv_clean_fields & 2188 HV_VMX_ENLIGHTENED_CLEAN_FIELD_GUEST_GRP1)) { 2189 vmcs_write32(GUEST_SYSENTER_CS, vmcs12->guest_sysenter_cs); 2190 vmcs_writel(GUEST_PENDING_DBG_EXCEPTIONS, 2191 vmcs12->guest_pending_dbg_exceptions); 2192 vmcs_writel(GUEST_SYSENTER_ESP, vmcs12->guest_sysenter_esp); 2193 vmcs_writel(GUEST_SYSENTER_EIP, vmcs12->guest_sysenter_eip); 2194 2195 /* 2196 * L1 may access the L2's PDPTR, so save them to construct 2197 * vmcs12 2198 */ 2199 if (enable_ept) { 2200 vmcs_write64(GUEST_PDPTR0, vmcs12->guest_pdptr0); 2201 vmcs_write64(GUEST_PDPTR1, vmcs12->guest_pdptr1); 2202 vmcs_write64(GUEST_PDPTR2, vmcs12->guest_pdptr2); 2203 vmcs_write64(GUEST_PDPTR3, vmcs12->guest_pdptr3); 2204 } 2205 } 2206 2207 if (nested_cpu_has_xsaves(vmcs12)) 2208 vmcs_write64(XSS_EXIT_BITMAP, vmcs12->xss_exit_bitmap); 2209 2210 /* 2211 * Whether page-faults are trapped is determined by a combination of 2212 * 3 settings: PFEC_MASK, PFEC_MATCH and EXCEPTION_BITMAP.PF. 2213 * If enable_ept, L0 doesn't care about page faults and we should 2214 * set all of these to L1's desires. However, if !enable_ept, L0 does 2215 * care about (at least some) page faults, and because it is not easy 2216 * (if at all possible?) to merge L0 and L1's desires, we simply ask 2217 * to exit on each and every L2 page fault. This is done by setting 2218 * MASK=MATCH=0 and (see below) EB.PF=1. 2219 * Note that below we don't need special code to set EB.PF beyond the 2220 * "or"ing of the EB of vmcs01 and vmcs12, because when enable_ept, 2221 * vmcs01's EB.PF is 0 so the "or" will take vmcs12's value, and when 2222 * !enable_ept, EB.PF is 1, so the "or" will always be 1. 2223 */ 2224 vmcs_write32(PAGE_FAULT_ERROR_CODE_MASK, 2225 enable_ept ? vmcs12->page_fault_error_code_mask : 0); 2226 vmcs_write32(PAGE_FAULT_ERROR_CODE_MATCH, 2227 enable_ept ? vmcs12->page_fault_error_code_match : 0); 2228 2229 if (cpu_has_vmx_apicv()) { 2230 vmcs_write64(EOI_EXIT_BITMAP0, vmcs12->eoi_exit_bitmap0); 2231 vmcs_write64(EOI_EXIT_BITMAP1, vmcs12->eoi_exit_bitmap1); 2232 vmcs_write64(EOI_EXIT_BITMAP2, vmcs12->eoi_exit_bitmap2); 2233 vmcs_write64(EOI_EXIT_BITMAP3, vmcs12->eoi_exit_bitmap3); 2234 } 2235 2236 vmcs_write32(VM_EXIT_MSR_LOAD_COUNT, vmx->msr_autoload.host.nr); 2237 vmcs_write32(VM_ENTRY_MSR_LOAD_COUNT, vmx->msr_autoload.guest.nr); 2238 2239 set_cr4_guest_host_mask(vmx); 2240 2241 if (kvm_mpx_supported()) { 2242 if (vmx->nested.nested_run_pending && 2243 (vmcs12->vm_entry_controls & VM_ENTRY_LOAD_BNDCFGS)) 2244 vmcs_write64(GUEST_BNDCFGS, vmcs12->guest_bndcfgs); 2245 else 2246 vmcs_write64(GUEST_BNDCFGS, vmx->nested.vmcs01_guest_bndcfgs); 2247 } 2248 } 2249 2250 /* 2251 * prepare_vmcs02 is called when the L1 guest hypervisor runs its nested 2252 * L2 guest. L1 has a vmcs for L2 (vmcs12), and this function "merges" it 2253 * with L0's requirements for its guest (a.k.a. vmcs01), so we can run the L2 2254 * guest in a way that will both be appropriate to L1's requests, and our 2255 * needs. In addition to modifying the active vmcs (which is vmcs02), this 2256 * function also has additional necessary side-effects, like setting various 2257 * vcpu->arch fields. 2258 * Returns 0 on success, 1 on failure. Invalid state exit qualification code 2259 * is assigned to entry_failure_code on failure. 2260 */ 2261 static int prepare_vmcs02(struct kvm_vcpu *vcpu, struct vmcs12 *vmcs12, 2262 u32 *entry_failure_code) 2263 { 2264 struct vcpu_vmx *vmx = to_vmx(vcpu); 2265 struct hv_enlightened_vmcs *hv_evmcs = vmx->nested.hv_evmcs; 2266 2267 if (vmx->nested.dirty_vmcs12 || vmx->nested.hv_evmcs) { 2268 prepare_vmcs02_full(vmx, vmcs12); 2269 vmx->nested.dirty_vmcs12 = false; 2270 } 2271 2272 /* 2273 * First, the fields that are shadowed. This must be kept in sync 2274 * with vmcs_shadow_fields.h. 2275 */ 2276 if (!hv_evmcs || !(hv_evmcs->hv_clean_fields & 2277 HV_VMX_ENLIGHTENED_CLEAN_FIELD_GUEST_GRP2)) { 2278 vmcs_write32(GUEST_CS_AR_BYTES, vmcs12->guest_cs_ar_bytes); 2279 vmcs_write32(GUEST_SS_AR_BYTES, vmcs12->guest_ss_ar_bytes); 2280 } 2281 2282 if (vmx->nested.nested_run_pending && 2283 (vmcs12->vm_entry_controls & VM_ENTRY_LOAD_DEBUG_CONTROLS)) { 2284 kvm_set_dr(vcpu, 7, vmcs12->guest_dr7); 2285 vmcs_write64(GUEST_IA32_DEBUGCTL, vmcs12->guest_ia32_debugctl); 2286 } else { 2287 kvm_set_dr(vcpu, 7, vcpu->arch.dr7); 2288 vmcs_write64(GUEST_IA32_DEBUGCTL, vmx->nested.vmcs01_debugctl); 2289 } 2290 vmx_set_rflags(vcpu, vmcs12->guest_rflags); 2291 2292 vmx->nested.preemption_timer_expired = false; 2293 if (nested_cpu_has_preemption_timer(vmcs12)) 2294 vmx_start_preemption_timer(vcpu); 2295 2296 /* EXCEPTION_BITMAP and CR0_GUEST_HOST_MASK should basically be the 2297 * bitwise-or of what L1 wants to trap for L2, and what we want to 2298 * trap. Note that CR0.TS also needs updating - we do this later. 2299 */ 2300 update_exception_bitmap(vcpu); 2301 vcpu->arch.cr0_guest_owned_bits &= ~vmcs12->cr0_guest_host_mask; 2302 vmcs_writel(CR0_GUEST_HOST_MASK, ~vcpu->arch.cr0_guest_owned_bits); 2303 2304 if (vmx->nested.nested_run_pending && 2305 (vmcs12->vm_entry_controls & VM_ENTRY_LOAD_IA32_PAT)) { 2306 vmcs_write64(GUEST_IA32_PAT, vmcs12->guest_ia32_pat); 2307 vcpu->arch.pat = vmcs12->guest_ia32_pat; 2308 } else if (vmcs_config.vmentry_ctrl & VM_ENTRY_LOAD_IA32_PAT) { 2309 vmcs_write64(GUEST_IA32_PAT, vmx->vcpu.arch.pat); 2310 } 2311 2312 vmcs_write64(TSC_OFFSET, vcpu->arch.tsc_offset); 2313 2314 if (kvm_has_tsc_control) 2315 decache_tsc_multiplier(vmx); 2316 2317 if (enable_vpid) { 2318 /* 2319 * There is no direct mapping between vpid02 and vpid12, the 2320 * vpid02 is per-vCPU for L0 and reused while the value of 2321 * vpid12 is changed w/ one invvpid during nested vmentry. 2322 * The vpid12 is allocated by L1 for L2, so it will not 2323 * influence global bitmap(for vpid01 and vpid02 allocation) 2324 * even if spawn a lot of nested vCPUs. 2325 */ 2326 if (nested_cpu_has_vpid(vmcs12) && nested_has_guest_tlb_tag(vcpu)) { 2327 if (vmcs12->virtual_processor_id != vmx->nested.last_vpid) { 2328 vmx->nested.last_vpid = vmcs12->virtual_processor_id; 2329 __vmx_flush_tlb(vcpu, nested_get_vpid02(vcpu), false); 2330 } 2331 } else { 2332 /* 2333 * If L1 use EPT, then L0 needs to execute INVEPT on 2334 * EPTP02 instead of EPTP01. Therefore, delay TLB 2335 * flush until vmcs02->eptp is fully updated by 2336 * KVM_REQ_LOAD_CR3. Note that this assumes 2337 * KVM_REQ_TLB_FLUSH is evaluated after 2338 * KVM_REQ_LOAD_CR3 in vcpu_enter_guest(). 2339 */ 2340 kvm_make_request(KVM_REQ_TLB_FLUSH, vcpu); 2341 } 2342 } 2343 2344 if (nested_cpu_has_ept(vmcs12)) 2345 nested_ept_init_mmu_context(vcpu); 2346 else if (nested_cpu_has2(vmcs12, 2347 SECONDARY_EXEC_VIRTUALIZE_APIC_ACCESSES)) 2348 vmx_flush_tlb(vcpu, true); 2349 2350 /* 2351 * This sets GUEST_CR0 to vmcs12->guest_cr0, possibly modifying those 2352 * bits which we consider mandatory enabled. 2353 * The CR0_READ_SHADOW is what L2 should have expected to read given 2354 * the specifications by L1; It's not enough to take 2355 * vmcs12->cr0_read_shadow because on our cr0_guest_host_mask we we 2356 * have more bits than L1 expected. 2357 */ 2358 vmx_set_cr0(vcpu, vmcs12->guest_cr0); 2359 vmcs_writel(CR0_READ_SHADOW, nested_read_cr0(vmcs12)); 2360 2361 vmx_set_cr4(vcpu, vmcs12->guest_cr4); 2362 vmcs_writel(CR4_READ_SHADOW, nested_read_cr4(vmcs12)); 2363 2364 vcpu->arch.efer = nested_vmx_calc_efer(vmx, vmcs12); 2365 /* Note: may modify VM_ENTRY/EXIT_CONTROLS and GUEST/HOST_IA32_EFER */ 2366 vmx_set_efer(vcpu, vcpu->arch.efer); 2367 2368 /* 2369 * Guest state is invalid and unrestricted guest is disabled, 2370 * which means L1 attempted VMEntry to L2 with invalid state. 2371 * Fail the VMEntry. 2372 */ 2373 if (vmx->emulation_required) { 2374 *entry_failure_code = ENTRY_FAIL_DEFAULT; 2375 return 1; 2376 } 2377 2378 /* Shadow page tables on either EPT or shadow page tables. */ 2379 if (nested_vmx_load_cr3(vcpu, vmcs12->guest_cr3, nested_cpu_has_ept(vmcs12), 2380 entry_failure_code)) 2381 return 1; 2382 2383 if (!enable_ept) 2384 vcpu->arch.walk_mmu->inject_page_fault = vmx_inject_page_fault_nested; 2385 2386 kvm_register_write(vcpu, VCPU_REGS_RSP, vmcs12->guest_rsp); 2387 kvm_register_write(vcpu, VCPU_REGS_RIP, vmcs12->guest_rip); 2388 return 0; 2389 } 2390 2391 static int nested_vmx_check_nmi_controls(struct vmcs12 *vmcs12) 2392 { 2393 if (!nested_cpu_has_nmi_exiting(vmcs12) && 2394 nested_cpu_has_virtual_nmis(vmcs12)) 2395 return -EINVAL; 2396 2397 if (!nested_cpu_has_virtual_nmis(vmcs12) && 2398 nested_cpu_has(vmcs12, CPU_BASED_VIRTUAL_NMI_PENDING)) 2399 return -EINVAL; 2400 2401 return 0; 2402 } 2403 2404 static bool valid_ept_address(struct kvm_vcpu *vcpu, u64 address) 2405 { 2406 struct vcpu_vmx *vmx = to_vmx(vcpu); 2407 int maxphyaddr = cpuid_maxphyaddr(vcpu); 2408 2409 /* Check for memory type validity */ 2410 switch (address & VMX_EPTP_MT_MASK) { 2411 case VMX_EPTP_MT_UC: 2412 if (!(vmx->nested.msrs.ept_caps & VMX_EPTP_UC_BIT)) 2413 return false; 2414 break; 2415 case VMX_EPTP_MT_WB: 2416 if (!(vmx->nested.msrs.ept_caps & VMX_EPTP_WB_BIT)) 2417 return false; 2418 break; 2419 default: 2420 return false; 2421 } 2422 2423 /* only 4 levels page-walk length are valid */ 2424 if ((address & VMX_EPTP_PWL_MASK) != VMX_EPTP_PWL_4) 2425 return false; 2426 2427 /* Reserved bits should not be set */ 2428 if (address >> maxphyaddr || ((address >> 7) & 0x1f)) 2429 return false; 2430 2431 /* AD, if set, should be supported */ 2432 if (address & VMX_EPTP_AD_ENABLE_BIT) { 2433 if (!(vmx->nested.msrs.ept_caps & VMX_EPT_AD_BIT)) 2434 return false; 2435 } 2436 2437 return true; 2438 } 2439 2440 /* 2441 * Checks related to VM-Execution Control Fields 2442 */ 2443 static int nested_check_vm_execution_controls(struct kvm_vcpu *vcpu, 2444 struct vmcs12 *vmcs12) 2445 { 2446 struct vcpu_vmx *vmx = to_vmx(vcpu); 2447 2448 if (!vmx_control_verify(vmcs12->pin_based_vm_exec_control, 2449 vmx->nested.msrs.pinbased_ctls_low, 2450 vmx->nested.msrs.pinbased_ctls_high) || 2451 !vmx_control_verify(vmcs12->cpu_based_vm_exec_control, 2452 vmx->nested.msrs.procbased_ctls_low, 2453 vmx->nested.msrs.procbased_ctls_high)) 2454 return -EINVAL; 2455 2456 if (nested_cpu_has(vmcs12, CPU_BASED_ACTIVATE_SECONDARY_CONTROLS) && 2457 !vmx_control_verify(vmcs12->secondary_vm_exec_control, 2458 vmx->nested.msrs.secondary_ctls_low, 2459 vmx->nested.msrs.secondary_ctls_high)) 2460 return -EINVAL; 2461 2462 if (vmcs12->cr3_target_count > nested_cpu_vmx_misc_cr3_count(vcpu) || 2463 nested_vmx_check_io_bitmap_controls(vcpu, vmcs12) || 2464 nested_vmx_check_msr_bitmap_controls(vcpu, vmcs12) || 2465 nested_vmx_check_tpr_shadow_controls(vcpu, vmcs12) || 2466 nested_vmx_check_apic_access_controls(vcpu, vmcs12) || 2467 nested_vmx_check_apicv_controls(vcpu, vmcs12) || 2468 nested_vmx_check_nmi_controls(vmcs12) || 2469 nested_vmx_check_pml_controls(vcpu, vmcs12) || 2470 nested_vmx_check_unrestricted_guest_controls(vcpu, vmcs12) || 2471 nested_vmx_check_mode_based_ept_exec_controls(vcpu, vmcs12) || 2472 nested_vmx_check_shadow_vmcs_controls(vcpu, vmcs12) || 2473 (nested_cpu_has_vpid(vmcs12) && !vmcs12->virtual_processor_id)) 2474 return -EINVAL; 2475 2476 if (nested_cpu_has_ept(vmcs12) && 2477 !valid_ept_address(vcpu, vmcs12->ept_pointer)) 2478 return -EINVAL; 2479 2480 if (nested_cpu_has_vmfunc(vmcs12)) { 2481 if (vmcs12->vm_function_control & 2482 ~vmx->nested.msrs.vmfunc_controls) 2483 return -EINVAL; 2484 2485 if (nested_cpu_has_eptp_switching(vmcs12)) { 2486 if (!nested_cpu_has_ept(vmcs12) || 2487 !page_address_valid(vcpu, vmcs12->eptp_list_address)) 2488 return -EINVAL; 2489 } 2490 } 2491 2492 return 0; 2493 } 2494 2495 /* 2496 * Checks related to VM-Exit Control Fields 2497 */ 2498 static int nested_check_vm_exit_controls(struct kvm_vcpu *vcpu, 2499 struct vmcs12 *vmcs12) 2500 { 2501 struct vcpu_vmx *vmx = to_vmx(vcpu); 2502 2503 if (!vmx_control_verify(vmcs12->vm_exit_controls, 2504 vmx->nested.msrs.exit_ctls_low, 2505 vmx->nested.msrs.exit_ctls_high) || 2506 nested_vmx_check_exit_msr_switch_controls(vcpu, vmcs12)) 2507 return -EINVAL; 2508 2509 return 0; 2510 } 2511 2512 /* 2513 * Checks related to VM-Entry Control Fields 2514 */ 2515 static int nested_check_vm_entry_controls(struct kvm_vcpu *vcpu, 2516 struct vmcs12 *vmcs12) 2517 { 2518 struct vcpu_vmx *vmx = to_vmx(vcpu); 2519 2520 if (!vmx_control_verify(vmcs12->vm_entry_controls, 2521 vmx->nested.msrs.entry_ctls_low, 2522 vmx->nested.msrs.entry_ctls_high)) 2523 return -EINVAL; 2524 2525 /* 2526 * From the Intel SDM, volume 3: 2527 * Fields relevant to VM-entry event injection must be set properly. 2528 * These fields are the VM-entry interruption-information field, the 2529 * VM-entry exception error code, and the VM-entry instruction length. 2530 */ 2531 if (vmcs12->vm_entry_intr_info_field & INTR_INFO_VALID_MASK) { 2532 u32 intr_info = vmcs12->vm_entry_intr_info_field; 2533 u8 vector = intr_info & INTR_INFO_VECTOR_MASK; 2534 u32 intr_type = intr_info & INTR_INFO_INTR_TYPE_MASK; 2535 bool has_error_code = intr_info & INTR_INFO_DELIVER_CODE_MASK; 2536 bool should_have_error_code; 2537 bool urg = nested_cpu_has2(vmcs12, 2538 SECONDARY_EXEC_UNRESTRICTED_GUEST); 2539 bool prot_mode = !urg || vmcs12->guest_cr0 & X86_CR0_PE; 2540 2541 /* VM-entry interruption-info field: interruption type */ 2542 if (intr_type == INTR_TYPE_RESERVED || 2543 (intr_type == INTR_TYPE_OTHER_EVENT && 2544 !nested_cpu_supports_monitor_trap_flag(vcpu))) 2545 return -EINVAL; 2546 2547 /* VM-entry interruption-info field: vector */ 2548 if ((intr_type == INTR_TYPE_NMI_INTR && vector != NMI_VECTOR) || 2549 (intr_type == INTR_TYPE_HARD_EXCEPTION && vector > 31) || 2550 (intr_type == INTR_TYPE_OTHER_EVENT && vector != 0)) 2551 return -EINVAL; 2552 2553 /* VM-entry interruption-info field: deliver error code */ 2554 should_have_error_code = 2555 intr_type == INTR_TYPE_HARD_EXCEPTION && prot_mode && 2556 x86_exception_has_error_code(vector); 2557 if (has_error_code != should_have_error_code) 2558 return -EINVAL; 2559 2560 /* VM-entry exception error code */ 2561 if (has_error_code && 2562 vmcs12->vm_entry_exception_error_code & GENMASK(31, 15)) 2563 return -EINVAL; 2564 2565 /* VM-entry interruption-info field: reserved bits */ 2566 if (intr_info & INTR_INFO_RESVD_BITS_MASK) 2567 return -EINVAL; 2568 2569 /* VM-entry instruction length */ 2570 switch (intr_type) { 2571 case INTR_TYPE_SOFT_EXCEPTION: 2572 case INTR_TYPE_SOFT_INTR: 2573 case INTR_TYPE_PRIV_SW_EXCEPTION: 2574 if ((vmcs12->vm_entry_instruction_len > 15) || 2575 (vmcs12->vm_entry_instruction_len == 0 && 2576 !nested_cpu_has_zero_length_injection(vcpu))) 2577 return -EINVAL; 2578 } 2579 } 2580 2581 if (nested_vmx_check_entry_msr_switch_controls(vcpu, vmcs12)) 2582 return -EINVAL; 2583 2584 return 0; 2585 } 2586 2587 /* 2588 * Checks related to Host Control Registers and MSRs 2589 */ 2590 static int nested_check_host_control_regs(struct kvm_vcpu *vcpu, 2591 struct vmcs12 *vmcs12) 2592 { 2593 bool ia32e; 2594 2595 if (!nested_host_cr0_valid(vcpu, vmcs12->host_cr0) || 2596 !nested_host_cr4_valid(vcpu, vmcs12->host_cr4) || 2597 !nested_cr3_valid(vcpu, vmcs12->host_cr3)) 2598 return -EINVAL; 2599 /* 2600 * If the load IA32_EFER VM-exit control is 1, bits reserved in the 2601 * IA32_EFER MSR must be 0 in the field for that register. In addition, 2602 * the values of the LMA and LME bits in the field must each be that of 2603 * the host address-space size VM-exit control. 2604 */ 2605 if (vmcs12->vm_exit_controls & VM_EXIT_LOAD_IA32_EFER) { 2606 ia32e = (vmcs12->vm_exit_controls & 2607 VM_EXIT_HOST_ADDR_SPACE_SIZE) != 0; 2608 if (!kvm_valid_efer(vcpu, vmcs12->host_ia32_efer) || 2609 ia32e != !!(vmcs12->host_ia32_efer & EFER_LMA) || 2610 ia32e != !!(vmcs12->host_ia32_efer & EFER_LME)) 2611 return -EINVAL; 2612 } 2613 2614 return 0; 2615 } 2616 2617 /* 2618 * Checks related to Guest Non-register State 2619 */ 2620 static int nested_check_guest_non_reg_state(struct vmcs12 *vmcs12) 2621 { 2622 if (vmcs12->guest_activity_state != GUEST_ACTIVITY_ACTIVE && 2623 vmcs12->guest_activity_state != GUEST_ACTIVITY_HLT) 2624 return -EINVAL; 2625 2626 return 0; 2627 } 2628 2629 static int nested_vmx_check_vmentry_prereqs(struct kvm_vcpu *vcpu, 2630 struct vmcs12 *vmcs12) 2631 { 2632 if (nested_check_vm_execution_controls(vcpu, vmcs12) || 2633 nested_check_vm_exit_controls(vcpu, vmcs12) || 2634 nested_check_vm_entry_controls(vcpu, vmcs12)) 2635 return VMXERR_ENTRY_INVALID_CONTROL_FIELD; 2636 2637 if (nested_check_host_control_regs(vcpu, vmcs12)) 2638 return VMXERR_ENTRY_INVALID_HOST_STATE_FIELD; 2639 2640 if (nested_check_guest_non_reg_state(vmcs12)) 2641 return VMXERR_ENTRY_INVALID_CONTROL_FIELD; 2642 2643 return 0; 2644 } 2645 2646 static int nested_vmx_check_vmcs_link_ptr(struct kvm_vcpu *vcpu, 2647 struct vmcs12 *vmcs12) 2648 { 2649 int r; 2650 struct page *page; 2651 struct vmcs12 *shadow; 2652 2653 if (vmcs12->vmcs_link_pointer == -1ull) 2654 return 0; 2655 2656 if (!page_address_valid(vcpu, vmcs12->vmcs_link_pointer)) 2657 return -EINVAL; 2658 2659 page = kvm_vcpu_gpa_to_page(vcpu, vmcs12->vmcs_link_pointer); 2660 if (is_error_page(page)) 2661 return -EINVAL; 2662 2663 r = 0; 2664 shadow = kmap(page); 2665 if (shadow->hdr.revision_id != VMCS12_REVISION || 2666 shadow->hdr.shadow_vmcs != nested_cpu_has_shadow_vmcs(vmcs12)) 2667 r = -EINVAL; 2668 kunmap(page); 2669 kvm_release_page_clean(page); 2670 return r; 2671 } 2672 2673 static int nested_vmx_check_vmentry_postreqs(struct kvm_vcpu *vcpu, 2674 struct vmcs12 *vmcs12, 2675 u32 *exit_qual) 2676 { 2677 bool ia32e; 2678 2679 *exit_qual = ENTRY_FAIL_DEFAULT; 2680 2681 if (!nested_guest_cr0_valid(vcpu, vmcs12->guest_cr0) || 2682 !nested_guest_cr4_valid(vcpu, vmcs12->guest_cr4)) 2683 return 1; 2684 2685 if (nested_vmx_check_vmcs_link_ptr(vcpu, vmcs12)) { 2686 *exit_qual = ENTRY_FAIL_VMCS_LINK_PTR; 2687 return 1; 2688 } 2689 2690 /* 2691 * If the load IA32_EFER VM-entry control is 1, the following checks 2692 * are performed on the field for the IA32_EFER MSR: 2693 * - Bits reserved in the IA32_EFER MSR must be 0. 2694 * - Bit 10 (corresponding to IA32_EFER.LMA) must equal the value of 2695 * the IA-32e mode guest VM-exit control. It must also be identical 2696 * to bit 8 (LME) if bit 31 in the CR0 field (corresponding to 2697 * CR0.PG) is 1. 2698 */ 2699 if (to_vmx(vcpu)->nested.nested_run_pending && 2700 (vmcs12->vm_entry_controls & VM_ENTRY_LOAD_IA32_EFER)) { 2701 ia32e = (vmcs12->vm_entry_controls & VM_ENTRY_IA32E_MODE) != 0; 2702 if (!kvm_valid_efer(vcpu, vmcs12->guest_ia32_efer) || 2703 ia32e != !!(vmcs12->guest_ia32_efer & EFER_LMA) || 2704 ((vmcs12->guest_cr0 & X86_CR0_PG) && 2705 ia32e != !!(vmcs12->guest_ia32_efer & EFER_LME))) 2706 return 1; 2707 } 2708 2709 if ((vmcs12->vm_entry_controls & VM_ENTRY_LOAD_BNDCFGS) && 2710 (is_noncanonical_address(vmcs12->guest_bndcfgs & PAGE_MASK, vcpu) || 2711 (vmcs12->guest_bndcfgs & MSR_IA32_BNDCFGS_RSVD))) 2712 return 1; 2713 2714 return 0; 2715 } 2716 2717 static int nested_vmx_check_vmentry_hw(struct kvm_vcpu *vcpu) 2718 { 2719 struct vcpu_vmx *vmx = to_vmx(vcpu); 2720 unsigned long cr3, cr4; 2721 2722 if (!nested_early_check) 2723 return 0; 2724 2725 if (vmx->msr_autoload.host.nr) 2726 vmcs_write32(VM_EXIT_MSR_LOAD_COUNT, 0); 2727 if (vmx->msr_autoload.guest.nr) 2728 vmcs_write32(VM_ENTRY_MSR_LOAD_COUNT, 0); 2729 2730 preempt_disable(); 2731 2732 vmx_prepare_switch_to_guest(vcpu); 2733 2734 /* 2735 * Induce a consistency check VMExit by clearing bit 1 in GUEST_RFLAGS, 2736 * which is reserved to '1' by hardware. GUEST_RFLAGS is guaranteed to 2737 * be written (by preparve_vmcs02()) before the "real" VMEnter, i.e. 2738 * there is no need to preserve other bits or save/restore the field. 2739 */ 2740 vmcs_writel(GUEST_RFLAGS, 0); 2741 2742 cr3 = __get_current_cr3_fast(); 2743 if (unlikely(cr3 != vmx->loaded_vmcs->host_state.cr3)) { 2744 vmcs_writel(HOST_CR3, cr3); 2745 vmx->loaded_vmcs->host_state.cr3 = cr3; 2746 } 2747 2748 cr4 = cr4_read_shadow(); 2749 if (unlikely(cr4 != vmx->loaded_vmcs->host_state.cr4)) { 2750 vmcs_writel(HOST_CR4, cr4); 2751 vmx->loaded_vmcs->host_state.cr4 = cr4; 2752 } 2753 2754 vmx->__launched = vmx->loaded_vmcs->launched; 2755 2756 asm( 2757 /* Set HOST_RSP */ 2758 "sub $%c[wordsize], %%" _ASM_SP "\n\t" /* temporarily adjust RSP for CALL */ 2759 __ex("vmwrite %%" _ASM_SP ", %%" _ASM_DX) "\n\t" 2760 "mov %%" _ASM_SP ", %c[host_rsp](%1)\n\t" 2761 "add $%c[wordsize], %%" _ASM_SP "\n\t" /* un-adjust RSP */ 2762 2763 /* Check if vmlaunch or vmresume is needed */ 2764 "cmpl $0, %c[launched](%% " _ASM_CX")\n\t" 2765 2766 "call vmx_vmenter\n\t" 2767 2768 /* Set vmx->fail accordingly */ 2769 "setbe %c[fail](%% " _ASM_CX")\n\t" 2770 : ASM_CALL_CONSTRAINT 2771 : "c"(vmx), "d"((unsigned long)HOST_RSP), 2772 [launched]"i"(offsetof(struct vcpu_vmx, __launched)), 2773 [fail]"i"(offsetof(struct vcpu_vmx, fail)), 2774 [host_rsp]"i"(offsetof(struct vcpu_vmx, host_rsp)), 2775 [wordsize]"i"(sizeof(ulong)) 2776 : "rax", "cc", "memory" 2777 ); 2778 2779 preempt_enable(); 2780 2781 if (vmx->msr_autoload.host.nr) 2782 vmcs_write32(VM_EXIT_MSR_LOAD_COUNT, vmx->msr_autoload.host.nr); 2783 if (vmx->msr_autoload.guest.nr) 2784 vmcs_write32(VM_ENTRY_MSR_LOAD_COUNT, vmx->msr_autoload.guest.nr); 2785 2786 if (vmx->fail) { 2787 WARN_ON_ONCE(vmcs_read32(VM_INSTRUCTION_ERROR) != 2788 VMXERR_ENTRY_INVALID_CONTROL_FIELD); 2789 vmx->fail = 0; 2790 return 1; 2791 } 2792 2793 /* 2794 * VMExit clears RFLAGS.IF and DR7, even on a consistency check. 2795 */ 2796 local_irq_enable(); 2797 if (hw_breakpoint_active()) 2798 set_debugreg(__this_cpu_read(cpu_dr7), 7); 2799 2800 /* 2801 * A non-failing VMEntry means we somehow entered guest mode with 2802 * an illegal RIP, and that's just the tip of the iceberg. There 2803 * is no telling what memory has been modified or what state has 2804 * been exposed to unknown code. Hitting this all but guarantees 2805 * a (very critical) hardware issue. 2806 */ 2807 WARN_ON(!(vmcs_read32(VM_EXIT_REASON) & 2808 VMX_EXIT_REASONS_FAILED_VMENTRY)); 2809 2810 return 0; 2811 } 2812 STACK_FRAME_NON_STANDARD(nested_vmx_check_vmentry_hw); 2813 2814 2815 static inline bool nested_vmx_prepare_msr_bitmap(struct kvm_vcpu *vcpu, 2816 struct vmcs12 *vmcs12); 2817 2818 static void nested_get_vmcs12_pages(struct kvm_vcpu *vcpu) 2819 { 2820 struct vmcs12 *vmcs12 = get_vmcs12(vcpu); 2821 struct vcpu_vmx *vmx = to_vmx(vcpu); 2822 struct page *page; 2823 u64 hpa; 2824 2825 if (nested_cpu_has2(vmcs12, SECONDARY_EXEC_VIRTUALIZE_APIC_ACCESSES)) { 2826 /* 2827 * Translate L1 physical address to host physical 2828 * address for vmcs02. Keep the page pinned, so this 2829 * physical address remains valid. We keep a reference 2830 * to it so we can release it later. 2831 */ 2832 if (vmx->nested.apic_access_page) { /* shouldn't happen */ 2833 kvm_release_page_dirty(vmx->nested.apic_access_page); 2834 vmx->nested.apic_access_page = NULL; 2835 } 2836 page = kvm_vcpu_gpa_to_page(vcpu, vmcs12->apic_access_addr); 2837 /* 2838 * If translation failed, no matter: This feature asks 2839 * to exit when accessing the given address, and if it 2840 * can never be accessed, this feature won't do 2841 * anything anyway. 2842 */ 2843 if (!is_error_page(page)) { 2844 vmx->nested.apic_access_page = page; 2845 hpa = page_to_phys(vmx->nested.apic_access_page); 2846 vmcs_write64(APIC_ACCESS_ADDR, hpa); 2847 } else { 2848 vmcs_clear_bits(SECONDARY_VM_EXEC_CONTROL, 2849 SECONDARY_EXEC_VIRTUALIZE_APIC_ACCESSES); 2850 } 2851 } 2852 2853 if (nested_cpu_has(vmcs12, CPU_BASED_TPR_SHADOW)) { 2854 if (vmx->nested.virtual_apic_page) { /* shouldn't happen */ 2855 kvm_release_page_dirty(vmx->nested.virtual_apic_page); 2856 vmx->nested.virtual_apic_page = NULL; 2857 } 2858 page = kvm_vcpu_gpa_to_page(vcpu, vmcs12->virtual_apic_page_addr); 2859 2860 /* 2861 * If translation failed, VM entry will fail because 2862 * prepare_vmcs02 set VIRTUAL_APIC_PAGE_ADDR to -1ull. 2863 * Failing the vm entry is _not_ what the processor 2864 * does but it's basically the only possibility we 2865 * have. We could still enter the guest if CR8 load 2866 * exits are enabled, CR8 store exits are enabled, and 2867 * virtualize APIC access is disabled; in this case 2868 * the processor would never use the TPR shadow and we 2869 * could simply clear the bit from the execution 2870 * control. But such a configuration is useless, so 2871 * let's keep the code simple. 2872 */ 2873 if (!is_error_page(page)) { 2874 vmx->nested.virtual_apic_page = page; 2875 hpa = page_to_phys(vmx->nested.virtual_apic_page); 2876 vmcs_write64(VIRTUAL_APIC_PAGE_ADDR, hpa); 2877 } 2878 } 2879 2880 if (nested_cpu_has_posted_intr(vmcs12)) { 2881 if (vmx->nested.pi_desc_page) { /* shouldn't happen */ 2882 kunmap(vmx->nested.pi_desc_page); 2883 kvm_release_page_dirty(vmx->nested.pi_desc_page); 2884 vmx->nested.pi_desc_page = NULL; 2885 vmx->nested.pi_desc = NULL; 2886 vmcs_write64(POSTED_INTR_DESC_ADDR, -1ull); 2887 } 2888 page = kvm_vcpu_gpa_to_page(vcpu, vmcs12->posted_intr_desc_addr); 2889 if (is_error_page(page)) 2890 return; 2891 vmx->nested.pi_desc_page = page; 2892 vmx->nested.pi_desc = kmap(vmx->nested.pi_desc_page); 2893 vmx->nested.pi_desc = 2894 (struct pi_desc *)((void *)vmx->nested.pi_desc + 2895 (unsigned long)(vmcs12->posted_intr_desc_addr & 2896 (PAGE_SIZE - 1))); 2897 vmcs_write64(POSTED_INTR_DESC_ADDR, 2898 page_to_phys(vmx->nested.pi_desc_page) + 2899 (unsigned long)(vmcs12->posted_intr_desc_addr & 2900 (PAGE_SIZE - 1))); 2901 } 2902 if (nested_vmx_prepare_msr_bitmap(vcpu, vmcs12)) 2903 vmcs_set_bits(CPU_BASED_VM_EXEC_CONTROL, 2904 CPU_BASED_USE_MSR_BITMAPS); 2905 else 2906 vmcs_clear_bits(CPU_BASED_VM_EXEC_CONTROL, 2907 CPU_BASED_USE_MSR_BITMAPS); 2908 } 2909 2910 /* 2911 * Intel's VMX Instruction Reference specifies a common set of prerequisites 2912 * for running VMX instructions (except VMXON, whose prerequisites are 2913 * slightly different). It also specifies what exception to inject otherwise. 2914 * Note that many of these exceptions have priority over VM exits, so they 2915 * don't have to be checked again here. 2916 */ 2917 static int nested_vmx_check_permission(struct kvm_vcpu *vcpu) 2918 { 2919 if (!to_vmx(vcpu)->nested.vmxon) { 2920 kvm_queue_exception(vcpu, UD_VECTOR); 2921 return 0; 2922 } 2923 2924 if (vmx_get_cpl(vcpu)) { 2925 kvm_inject_gp(vcpu, 0); 2926 return 0; 2927 } 2928 2929 return 1; 2930 } 2931 2932 static u8 vmx_has_apicv_interrupt(struct kvm_vcpu *vcpu) 2933 { 2934 u8 rvi = vmx_get_rvi(); 2935 u8 vppr = kvm_lapic_get_reg(vcpu->arch.apic, APIC_PROCPRI); 2936 2937 return ((rvi & 0xf0) > (vppr & 0xf0)); 2938 } 2939 2940 static void load_vmcs12_host_state(struct kvm_vcpu *vcpu, 2941 struct vmcs12 *vmcs12); 2942 2943 /* 2944 * If from_vmentry is false, this is being called from state restore (either RSM 2945 * or KVM_SET_NESTED_STATE). Otherwise it's called from vmlaunch/vmresume. 2946 + * 2947 + * Returns: 2948 + * 0 - success, i.e. proceed with actual VMEnter 2949 + * 1 - consistency check VMExit 2950 + * -1 - consistency check VMFail 2951 */ 2952 int nested_vmx_enter_non_root_mode(struct kvm_vcpu *vcpu, bool from_vmentry) 2953 { 2954 struct vcpu_vmx *vmx = to_vmx(vcpu); 2955 struct vmcs12 *vmcs12 = get_vmcs12(vcpu); 2956 bool evaluate_pending_interrupts; 2957 u32 exit_reason = EXIT_REASON_INVALID_STATE; 2958 u32 exit_qual; 2959 2960 evaluate_pending_interrupts = vmcs_read32(CPU_BASED_VM_EXEC_CONTROL) & 2961 (CPU_BASED_VIRTUAL_INTR_PENDING | CPU_BASED_VIRTUAL_NMI_PENDING); 2962 if (likely(!evaluate_pending_interrupts) && kvm_vcpu_apicv_active(vcpu)) 2963 evaluate_pending_interrupts |= vmx_has_apicv_interrupt(vcpu); 2964 2965 if (!(vmcs12->vm_entry_controls & VM_ENTRY_LOAD_DEBUG_CONTROLS)) 2966 vmx->nested.vmcs01_debugctl = vmcs_read64(GUEST_IA32_DEBUGCTL); 2967 if (kvm_mpx_supported() && 2968 !(vmcs12->vm_entry_controls & VM_ENTRY_LOAD_BNDCFGS)) 2969 vmx->nested.vmcs01_guest_bndcfgs = vmcs_read64(GUEST_BNDCFGS); 2970 2971 vmx_switch_vmcs(vcpu, &vmx->nested.vmcs02); 2972 2973 prepare_vmcs02_early(vmx, vmcs12); 2974 2975 if (from_vmentry) { 2976 nested_get_vmcs12_pages(vcpu); 2977 2978 if (nested_vmx_check_vmentry_hw(vcpu)) { 2979 vmx_switch_vmcs(vcpu, &vmx->vmcs01); 2980 return -1; 2981 } 2982 2983 if (nested_vmx_check_vmentry_postreqs(vcpu, vmcs12, &exit_qual)) 2984 goto vmentry_fail_vmexit; 2985 } 2986 2987 enter_guest_mode(vcpu); 2988 if (vmcs12->cpu_based_vm_exec_control & CPU_BASED_USE_TSC_OFFSETING) 2989 vcpu->arch.tsc_offset += vmcs12->tsc_offset; 2990 2991 if (prepare_vmcs02(vcpu, vmcs12, &exit_qual)) 2992 goto vmentry_fail_vmexit_guest_mode; 2993 2994 if (from_vmentry) { 2995 exit_reason = EXIT_REASON_MSR_LOAD_FAIL; 2996 exit_qual = nested_vmx_load_msr(vcpu, 2997 vmcs12->vm_entry_msr_load_addr, 2998 vmcs12->vm_entry_msr_load_count); 2999 if (exit_qual) 3000 goto vmentry_fail_vmexit_guest_mode; 3001 } else { 3002 /* 3003 * The MMU is not initialized to point at the right entities yet and 3004 * "get pages" would need to read data from the guest (i.e. we will 3005 * need to perform gpa to hpa translation). Request a call 3006 * to nested_get_vmcs12_pages before the next VM-entry. The MSRs 3007 * have already been set at vmentry time and should not be reset. 3008 */ 3009 kvm_make_request(KVM_REQ_GET_VMCS12_PAGES, vcpu); 3010 } 3011 3012 /* 3013 * If L1 had a pending IRQ/NMI until it executed 3014 * VMLAUNCH/VMRESUME which wasn't delivered because it was 3015 * disallowed (e.g. interrupts disabled), L0 needs to 3016 * evaluate if this pending event should cause an exit from L2 3017 * to L1 or delivered directly to L2 (e.g. In case L1 don't 3018 * intercept EXTERNAL_INTERRUPT). 3019 * 3020 * Usually this would be handled by the processor noticing an 3021 * IRQ/NMI window request, or checking RVI during evaluation of 3022 * pending virtual interrupts. However, this setting was done 3023 * on VMCS01 and now VMCS02 is active instead. Thus, we force L0 3024 * to perform pending event evaluation by requesting a KVM_REQ_EVENT. 3025 */ 3026 if (unlikely(evaluate_pending_interrupts)) 3027 kvm_make_request(KVM_REQ_EVENT, vcpu); 3028 3029 /* 3030 * Note no nested_vmx_succeed or nested_vmx_fail here. At this point 3031 * we are no longer running L1, and VMLAUNCH/VMRESUME has not yet 3032 * returned as far as L1 is concerned. It will only return (and set 3033 * the success flag) when L2 exits (see nested_vmx_vmexit()). 3034 */ 3035 return 0; 3036 3037 /* 3038 * A failed consistency check that leads to a VMExit during L1's 3039 * VMEnter to L2 is a variation of a normal VMexit, as explained in 3040 * 26.7 "VM-entry failures during or after loading guest state". 3041 */ 3042 vmentry_fail_vmexit_guest_mode: 3043 if (vmcs12->cpu_based_vm_exec_control & CPU_BASED_USE_TSC_OFFSETING) 3044 vcpu->arch.tsc_offset -= vmcs12->tsc_offset; 3045 leave_guest_mode(vcpu); 3046 3047 vmentry_fail_vmexit: 3048 vmx_switch_vmcs(vcpu, &vmx->vmcs01); 3049 3050 if (!from_vmentry) 3051 return 1; 3052 3053 load_vmcs12_host_state(vcpu, vmcs12); 3054 vmcs12->vm_exit_reason = exit_reason | VMX_EXIT_REASONS_FAILED_VMENTRY; 3055 vmcs12->exit_qualification = exit_qual; 3056 if (enable_shadow_vmcs || vmx->nested.hv_evmcs) 3057 vmx->nested.need_vmcs12_sync = true; 3058 return 1; 3059 } 3060 3061 /* 3062 * nested_vmx_run() handles a nested entry, i.e., a VMLAUNCH or VMRESUME on L1 3063 * for running an L2 nested guest. 3064 */ 3065 static int nested_vmx_run(struct kvm_vcpu *vcpu, bool launch) 3066 { 3067 struct vmcs12 *vmcs12; 3068 struct vcpu_vmx *vmx = to_vmx(vcpu); 3069 u32 interrupt_shadow = vmx_get_interrupt_shadow(vcpu); 3070 int ret; 3071 3072 if (!nested_vmx_check_permission(vcpu)) 3073 return 1; 3074 3075 if (!nested_vmx_handle_enlightened_vmptrld(vcpu, true)) 3076 return 1; 3077 3078 if (!vmx->nested.hv_evmcs && vmx->nested.current_vmptr == -1ull) 3079 return nested_vmx_failInvalid(vcpu); 3080 3081 vmcs12 = get_vmcs12(vcpu); 3082 3083 /* 3084 * Can't VMLAUNCH or VMRESUME a shadow VMCS. Despite the fact 3085 * that there *is* a valid VMCS pointer, RFLAGS.CF is set 3086 * rather than RFLAGS.ZF, and no error number is stored to the 3087 * VM-instruction error field. 3088 */ 3089 if (vmcs12->hdr.shadow_vmcs) 3090 return nested_vmx_failInvalid(vcpu); 3091 3092 if (vmx->nested.hv_evmcs) { 3093 copy_enlightened_to_vmcs12(vmx); 3094 /* Enlightened VMCS doesn't have launch state */ 3095 vmcs12->launch_state = !launch; 3096 } else if (enable_shadow_vmcs) { 3097 copy_shadow_to_vmcs12(vmx); 3098 } 3099 3100 /* 3101 * The nested entry process starts with enforcing various prerequisites 3102 * on vmcs12 as required by the Intel SDM, and act appropriately when 3103 * they fail: As the SDM explains, some conditions should cause the 3104 * instruction to fail, while others will cause the instruction to seem 3105 * to succeed, but return an EXIT_REASON_INVALID_STATE. 3106 * To speed up the normal (success) code path, we should avoid checking 3107 * for misconfigurations which will anyway be caught by the processor 3108 * when using the merged vmcs02. 3109 */ 3110 if (interrupt_shadow & KVM_X86_SHADOW_INT_MOV_SS) 3111 return nested_vmx_failValid(vcpu, 3112 VMXERR_ENTRY_EVENTS_BLOCKED_BY_MOV_SS); 3113 3114 if (vmcs12->launch_state == launch) 3115 return nested_vmx_failValid(vcpu, 3116 launch ? VMXERR_VMLAUNCH_NONCLEAR_VMCS 3117 : VMXERR_VMRESUME_NONLAUNCHED_VMCS); 3118 3119 ret = nested_vmx_check_vmentry_prereqs(vcpu, vmcs12); 3120 if (ret) 3121 return nested_vmx_failValid(vcpu, ret); 3122 3123 /* 3124 * We're finally done with prerequisite checking, and can start with 3125 * the nested entry. 3126 */ 3127 vmx->nested.nested_run_pending = 1; 3128 ret = nested_vmx_enter_non_root_mode(vcpu, true); 3129 vmx->nested.nested_run_pending = !ret; 3130 if (ret > 0) 3131 return 1; 3132 else if (ret) 3133 return nested_vmx_failValid(vcpu, 3134 VMXERR_ENTRY_INVALID_CONTROL_FIELD); 3135 3136 /* Hide L1D cache contents from the nested guest. */ 3137 vmx->vcpu.arch.l1tf_flush_l1d = true; 3138 3139 /* 3140 * Must happen outside of nested_vmx_enter_non_root_mode() as it will 3141 * also be used as part of restoring nVMX state for 3142 * snapshot restore (migration). 3143 * 3144 * In this flow, it is assumed that vmcs12 cache was 3145 * trasferred as part of captured nVMX state and should 3146 * therefore not be read from guest memory (which may not 3147 * exist on destination host yet). 3148 */ 3149 nested_cache_shadow_vmcs12(vcpu, vmcs12); 3150 3151 /* 3152 * If we're entering a halted L2 vcpu and the L2 vcpu won't be 3153 * awakened by event injection or by an NMI-window VM-exit or 3154 * by an interrupt-window VM-exit, halt the vcpu. 3155 */ 3156 if ((vmcs12->guest_activity_state == GUEST_ACTIVITY_HLT) && 3157 !(vmcs12->vm_entry_intr_info_field & INTR_INFO_VALID_MASK) && 3158 !(vmcs12->cpu_based_vm_exec_control & CPU_BASED_VIRTUAL_NMI_PENDING) && 3159 !((vmcs12->cpu_based_vm_exec_control & CPU_BASED_VIRTUAL_INTR_PENDING) && 3160 (vmcs12->guest_rflags & X86_EFLAGS_IF))) { 3161 vmx->nested.nested_run_pending = 0; 3162 return kvm_vcpu_halt(vcpu); 3163 } 3164 return 1; 3165 } 3166 3167 /* 3168 * On a nested exit from L2 to L1, vmcs12.guest_cr0 might not be up-to-date 3169 * because L2 may have changed some cr0 bits directly (CRO_GUEST_HOST_MASK). 3170 * This function returns the new value we should put in vmcs12.guest_cr0. 3171 * It's not enough to just return the vmcs02 GUEST_CR0. Rather, 3172 * 1. Bits that neither L0 nor L1 trapped, were set directly by L2 and are now 3173 * available in vmcs02 GUEST_CR0. (Note: It's enough to check that L0 3174 * didn't trap the bit, because if L1 did, so would L0). 3175 * 2. Bits that L1 asked to trap (and therefore L0 also did) could not have 3176 * been modified by L2, and L1 knows it. So just leave the old value of 3177 * the bit from vmcs12.guest_cr0. Note that the bit from vmcs02 GUEST_CR0 3178 * isn't relevant, because if L0 traps this bit it can set it to anything. 3179 * 3. Bits that L1 didn't trap, but L0 did. L1 believes the guest could have 3180 * changed these bits, and therefore they need to be updated, but L0 3181 * didn't necessarily allow them to be changed in GUEST_CR0 - and rather 3182 * put them in vmcs02 CR0_READ_SHADOW. So take these bits from there. 3183 */ 3184 static inline unsigned long 3185 vmcs12_guest_cr0(struct kvm_vcpu *vcpu, struct vmcs12 *vmcs12) 3186 { 3187 return 3188 /*1*/ (vmcs_readl(GUEST_CR0) & vcpu->arch.cr0_guest_owned_bits) | 3189 /*2*/ (vmcs12->guest_cr0 & vmcs12->cr0_guest_host_mask) | 3190 /*3*/ (vmcs_readl(CR0_READ_SHADOW) & ~(vmcs12->cr0_guest_host_mask | 3191 vcpu->arch.cr0_guest_owned_bits)); 3192 } 3193 3194 static inline unsigned long 3195 vmcs12_guest_cr4(struct kvm_vcpu *vcpu, struct vmcs12 *vmcs12) 3196 { 3197 return 3198 /*1*/ (vmcs_readl(GUEST_CR4) & vcpu->arch.cr4_guest_owned_bits) | 3199 /*2*/ (vmcs12->guest_cr4 & vmcs12->cr4_guest_host_mask) | 3200 /*3*/ (vmcs_readl(CR4_READ_SHADOW) & ~(vmcs12->cr4_guest_host_mask | 3201 vcpu->arch.cr4_guest_owned_bits)); 3202 } 3203 3204 static void vmcs12_save_pending_event(struct kvm_vcpu *vcpu, 3205 struct vmcs12 *vmcs12) 3206 { 3207 u32 idt_vectoring; 3208 unsigned int nr; 3209 3210 if (vcpu->arch.exception.injected) { 3211 nr = vcpu->arch.exception.nr; 3212 idt_vectoring = nr | VECTORING_INFO_VALID_MASK; 3213 3214 if (kvm_exception_is_soft(nr)) { 3215 vmcs12->vm_exit_instruction_len = 3216 vcpu->arch.event_exit_inst_len; 3217 idt_vectoring |= INTR_TYPE_SOFT_EXCEPTION; 3218 } else 3219 idt_vectoring |= INTR_TYPE_HARD_EXCEPTION; 3220 3221 if (vcpu->arch.exception.has_error_code) { 3222 idt_vectoring |= VECTORING_INFO_DELIVER_CODE_MASK; 3223 vmcs12->idt_vectoring_error_code = 3224 vcpu->arch.exception.error_code; 3225 } 3226 3227 vmcs12->idt_vectoring_info_field = idt_vectoring; 3228 } else if (vcpu->arch.nmi_injected) { 3229 vmcs12->idt_vectoring_info_field = 3230 INTR_TYPE_NMI_INTR | INTR_INFO_VALID_MASK | NMI_VECTOR; 3231 } else if (vcpu->arch.interrupt.injected) { 3232 nr = vcpu->arch.interrupt.nr; 3233 idt_vectoring = nr | VECTORING_INFO_VALID_MASK; 3234 3235 if (vcpu->arch.interrupt.soft) { 3236 idt_vectoring |= INTR_TYPE_SOFT_INTR; 3237 vmcs12->vm_entry_instruction_len = 3238 vcpu->arch.event_exit_inst_len; 3239 } else 3240 idt_vectoring |= INTR_TYPE_EXT_INTR; 3241 3242 vmcs12->idt_vectoring_info_field = idt_vectoring; 3243 } 3244 } 3245 3246 3247 static void nested_mark_vmcs12_pages_dirty(struct kvm_vcpu *vcpu) 3248 { 3249 struct vmcs12 *vmcs12 = get_vmcs12(vcpu); 3250 gfn_t gfn; 3251 3252 /* 3253 * Don't need to mark the APIC access page dirty; it is never 3254 * written to by the CPU during APIC virtualization. 3255 */ 3256 3257 if (nested_cpu_has(vmcs12, CPU_BASED_TPR_SHADOW)) { 3258 gfn = vmcs12->virtual_apic_page_addr >> PAGE_SHIFT; 3259 kvm_vcpu_mark_page_dirty(vcpu, gfn); 3260 } 3261 3262 if (nested_cpu_has_posted_intr(vmcs12)) { 3263 gfn = vmcs12->posted_intr_desc_addr >> PAGE_SHIFT; 3264 kvm_vcpu_mark_page_dirty(vcpu, gfn); 3265 } 3266 } 3267 3268 static void vmx_complete_nested_posted_interrupt(struct kvm_vcpu *vcpu) 3269 { 3270 struct vcpu_vmx *vmx = to_vmx(vcpu); 3271 int max_irr; 3272 void *vapic_page; 3273 u16 status; 3274 3275 if (!vmx->nested.pi_desc || !vmx->nested.pi_pending) 3276 return; 3277 3278 vmx->nested.pi_pending = false; 3279 if (!pi_test_and_clear_on(vmx->nested.pi_desc)) 3280 return; 3281 3282 max_irr = find_last_bit((unsigned long *)vmx->nested.pi_desc->pir, 256); 3283 if (max_irr != 256) { 3284 vapic_page = kmap(vmx->nested.virtual_apic_page); 3285 __kvm_apic_update_irr(vmx->nested.pi_desc->pir, 3286 vapic_page, &max_irr); 3287 kunmap(vmx->nested.virtual_apic_page); 3288 3289 status = vmcs_read16(GUEST_INTR_STATUS); 3290 if ((u8)max_irr > ((u8)status & 0xff)) { 3291 status &= ~0xff; 3292 status |= (u8)max_irr; 3293 vmcs_write16(GUEST_INTR_STATUS, status); 3294 } 3295 } 3296 3297 nested_mark_vmcs12_pages_dirty(vcpu); 3298 } 3299 3300 static void nested_vmx_inject_exception_vmexit(struct kvm_vcpu *vcpu, 3301 unsigned long exit_qual) 3302 { 3303 struct vmcs12 *vmcs12 = get_vmcs12(vcpu); 3304 unsigned int nr = vcpu->arch.exception.nr; 3305 u32 intr_info = nr | INTR_INFO_VALID_MASK; 3306 3307 if (vcpu->arch.exception.has_error_code) { 3308 vmcs12->vm_exit_intr_error_code = vcpu->arch.exception.error_code; 3309 intr_info |= INTR_INFO_DELIVER_CODE_MASK; 3310 } 3311 3312 if (kvm_exception_is_soft(nr)) 3313 intr_info |= INTR_TYPE_SOFT_EXCEPTION; 3314 else 3315 intr_info |= INTR_TYPE_HARD_EXCEPTION; 3316 3317 if (!(vmcs12->idt_vectoring_info_field & VECTORING_INFO_VALID_MASK) && 3318 vmx_get_nmi_mask(vcpu)) 3319 intr_info |= INTR_INFO_UNBLOCK_NMI; 3320 3321 nested_vmx_vmexit(vcpu, EXIT_REASON_EXCEPTION_NMI, intr_info, exit_qual); 3322 } 3323 3324 static int vmx_check_nested_events(struct kvm_vcpu *vcpu, bool external_intr) 3325 { 3326 struct vcpu_vmx *vmx = to_vmx(vcpu); 3327 unsigned long exit_qual; 3328 bool block_nested_events = 3329 vmx->nested.nested_run_pending || kvm_event_needs_reinjection(vcpu); 3330 3331 if (vcpu->arch.exception.pending && 3332 nested_vmx_check_exception(vcpu, &exit_qual)) { 3333 if (block_nested_events) 3334 return -EBUSY; 3335 nested_vmx_inject_exception_vmexit(vcpu, exit_qual); 3336 return 0; 3337 } 3338 3339 if (nested_cpu_has_preemption_timer(get_vmcs12(vcpu)) && 3340 vmx->nested.preemption_timer_expired) { 3341 if (block_nested_events) 3342 return -EBUSY; 3343 nested_vmx_vmexit(vcpu, EXIT_REASON_PREEMPTION_TIMER, 0, 0); 3344 return 0; 3345 } 3346 3347 if (vcpu->arch.nmi_pending && nested_exit_on_nmi(vcpu)) { 3348 if (block_nested_events) 3349 return -EBUSY; 3350 nested_vmx_vmexit(vcpu, EXIT_REASON_EXCEPTION_NMI, 3351 NMI_VECTOR | INTR_TYPE_NMI_INTR | 3352 INTR_INFO_VALID_MASK, 0); 3353 /* 3354 * The NMI-triggered VM exit counts as injection: 3355 * clear this one and block further NMIs. 3356 */ 3357 vcpu->arch.nmi_pending = 0; 3358 vmx_set_nmi_mask(vcpu, true); 3359 return 0; 3360 } 3361 3362 if ((kvm_cpu_has_interrupt(vcpu) || external_intr) && 3363 nested_exit_on_intr(vcpu)) { 3364 if (block_nested_events) 3365 return -EBUSY; 3366 nested_vmx_vmexit(vcpu, EXIT_REASON_EXTERNAL_INTERRUPT, 0, 0); 3367 return 0; 3368 } 3369 3370 vmx_complete_nested_posted_interrupt(vcpu); 3371 return 0; 3372 } 3373 3374 static u32 vmx_get_preemption_timer_value(struct kvm_vcpu *vcpu) 3375 { 3376 ktime_t remaining = 3377 hrtimer_get_remaining(&to_vmx(vcpu)->nested.preemption_timer); 3378 u64 value; 3379 3380 if (ktime_to_ns(remaining) <= 0) 3381 return 0; 3382 3383 value = ktime_to_ns(remaining) * vcpu->arch.virtual_tsc_khz; 3384 do_div(value, 1000000); 3385 return value >> VMX_MISC_EMULATED_PREEMPTION_TIMER_RATE; 3386 } 3387 3388 /* 3389 * Update the guest state fields of vmcs12 to reflect changes that 3390 * occurred while L2 was running. (The "IA-32e mode guest" bit of the 3391 * VM-entry controls is also updated, since this is really a guest 3392 * state bit.) 3393 */ 3394 static void sync_vmcs12(struct kvm_vcpu *vcpu, struct vmcs12 *vmcs12) 3395 { 3396 vmcs12->guest_cr0 = vmcs12_guest_cr0(vcpu, vmcs12); 3397 vmcs12->guest_cr4 = vmcs12_guest_cr4(vcpu, vmcs12); 3398 3399 vmcs12->guest_rsp = kvm_register_read(vcpu, VCPU_REGS_RSP); 3400 vmcs12->guest_rip = kvm_register_read(vcpu, VCPU_REGS_RIP); 3401 vmcs12->guest_rflags = vmcs_readl(GUEST_RFLAGS); 3402 3403 vmcs12->guest_es_selector = vmcs_read16(GUEST_ES_SELECTOR); 3404 vmcs12->guest_cs_selector = vmcs_read16(GUEST_CS_SELECTOR); 3405 vmcs12->guest_ss_selector = vmcs_read16(GUEST_SS_SELECTOR); 3406 vmcs12->guest_ds_selector = vmcs_read16(GUEST_DS_SELECTOR); 3407 vmcs12->guest_fs_selector = vmcs_read16(GUEST_FS_SELECTOR); 3408 vmcs12->guest_gs_selector = vmcs_read16(GUEST_GS_SELECTOR); 3409 vmcs12->guest_ldtr_selector = vmcs_read16(GUEST_LDTR_SELECTOR); 3410 vmcs12->guest_tr_selector = vmcs_read16(GUEST_TR_SELECTOR); 3411 vmcs12->guest_es_limit = vmcs_read32(GUEST_ES_LIMIT); 3412 vmcs12->guest_cs_limit = vmcs_read32(GUEST_CS_LIMIT); 3413 vmcs12->guest_ss_limit = vmcs_read32(GUEST_SS_LIMIT); 3414 vmcs12->guest_ds_limit = vmcs_read32(GUEST_DS_LIMIT); 3415 vmcs12->guest_fs_limit = vmcs_read32(GUEST_FS_LIMIT); 3416 vmcs12->guest_gs_limit = vmcs_read32(GUEST_GS_LIMIT); 3417 vmcs12->guest_ldtr_limit = vmcs_read32(GUEST_LDTR_LIMIT); 3418 vmcs12->guest_tr_limit = vmcs_read32(GUEST_TR_LIMIT); 3419 vmcs12->guest_gdtr_limit = vmcs_read32(GUEST_GDTR_LIMIT); 3420 vmcs12->guest_idtr_limit = vmcs_read32(GUEST_IDTR_LIMIT); 3421 vmcs12->guest_es_ar_bytes = vmcs_read32(GUEST_ES_AR_BYTES); 3422 vmcs12->guest_cs_ar_bytes = vmcs_read32(GUEST_CS_AR_BYTES); 3423 vmcs12->guest_ss_ar_bytes = vmcs_read32(GUEST_SS_AR_BYTES); 3424 vmcs12->guest_ds_ar_bytes = vmcs_read32(GUEST_DS_AR_BYTES); 3425 vmcs12->guest_fs_ar_bytes = vmcs_read32(GUEST_FS_AR_BYTES); 3426 vmcs12->guest_gs_ar_bytes = vmcs_read32(GUEST_GS_AR_BYTES); 3427 vmcs12->guest_ldtr_ar_bytes = vmcs_read32(GUEST_LDTR_AR_BYTES); 3428 vmcs12->guest_tr_ar_bytes = vmcs_read32(GUEST_TR_AR_BYTES); 3429 vmcs12->guest_es_base = vmcs_readl(GUEST_ES_BASE); 3430 vmcs12->guest_cs_base = vmcs_readl(GUEST_CS_BASE); 3431 vmcs12->guest_ss_base = vmcs_readl(GUEST_SS_BASE); 3432 vmcs12->guest_ds_base = vmcs_readl(GUEST_DS_BASE); 3433 vmcs12->guest_fs_base = vmcs_readl(GUEST_FS_BASE); 3434 vmcs12->guest_gs_base = vmcs_readl(GUEST_GS_BASE); 3435 vmcs12->guest_ldtr_base = vmcs_readl(GUEST_LDTR_BASE); 3436 vmcs12->guest_tr_base = vmcs_readl(GUEST_TR_BASE); 3437 vmcs12->guest_gdtr_base = vmcs_readl(GUEST_GDTR_BASE); 3438 vmcs12->guest_idtr_base = vmcs_readl(GUEST_IDTR_BASE); 3439 3440 vmcs12->guest_interruptibility_info = 3441 vmcs_read32(GUEST_INTERRUPTIBILITY_INFO); 3442 vmcs12->guest_pending_dbg_exceptions = 3443 vmcs_readl(GUEST_PENDING_DBG_EXCEPTIONS); 3444 if (vcpu->arch.mp_state == KVM_MP_STATE_HALTED) 3445 vmcs12->guest_activity_state = GUEST_ACTIVITY_HLT; 3446 else 3447 vmcs12->guest_activity_state = GUEST_ACTIVITY_ACTIVE; 3448 3449 if (nested_cpu_has_preemption_timer(vmcs12)) { 3450 if (vmcs12->vm_exit_controls & 3451 VM_EXIT_SAVE_VMX_PREEMPTION_TIMER) 3452 vmcs12->vmx_preemption_timer_value = 3453 vmx_get_preemption_timer_value(vcpu); 3454 hrtimer_cancel(&to_vmx(vcpu)->nested.preemption_timer); 3455 } 3456 3457 /* 3458 * In some cases (usually, nested EPT), L2 is allowed to change its 3459 * own CR3 without exiting. If it has changed it, we must keep it. 3460 * Of course, if L0 is using shadow page tables, GUEST_CR3 was defined 3461 * by L0, not L1 or L2, so we mustn't unconditionally copy it to vmcs12. 3462 * 3463 * Additionally, restore L2's PDPTR to vmcs12. 3464 */ 3465 if (enable_ept) { 3466 vmcs12->guest_cr3 = vmcs_readl(GUEST_CR3); 3467 vmcs12->guest_pdptr0 = vmcs_read64(GUEST_PDPTR0); 3468 vmcs12->guest_pdptr1 = vmcs_read64(GUEST_PDPTR1); 3469 vmcs12->guest_pdptr2 = vmcs_read64(GUEST_PDPTR2); 3470 vmcs12->guest_pdptr3 = vmcs_read64(GUEST_PDPTR3); 3471 } 3472 3473 vmcs12->guest_linear_address = vmcs_readl(GUEST_LINEAR_ADDRESS); 3474 3475 if (nested_cpu_has_vid(vmcs12)) 3476 vmcs12->guest_intr_status = vmcs_read16(GUEST_INTR_STATUS); 3477 3478 vmcs12->vm_entry_controls = 3479 (vmcs12->vm_entry_controls & ~VM_ENTRY_IA32E_MODE) | 3480 (vm_entry_controls_get(to_vmx(vcpu)) & VM_ENTRY_IA32E_MODE); 3481 3482 if (vmcs12->vm_exit_controls & VM_EXIT_SAVE_DEBUG_CONTROLS) { 3483 kvm_get_dr(vcpu, 7, (unsigned long *)&vmcs12->guest_dr7); 3484 vmcs12->guest_ia32_debugctl = vmcs_read64(GUEST_IA32_DEBUGCTL); 3485 } 3486 3487 /* TODO: These cannot have changed unless we have MSR bitmaps and 3488 * the relevant bit asks not to trap the change */ 3489 if (vmcs12->vm_exit_controls & VM_EXIT_SAVE_IA32_PAT) 3490 vmcs12->guest_ia32_pat = vmcs_read64(GUEST_IA32_PAT); 3491 if (vmcs12->vm_exit_controls & VM_EXIT_SAVE_IA32_EFER) 3492 vmcs12->guest_ia32_efer = vcpu->arch.efer; 3493 vmcs12->guest_sysenter_cs = vmcs_read32(GUEST_SYSENTER_CS); 3494 vmcs12->guest_sysenter_esp = vmcs_readl(GUEST_SYSENTER_ESP); 3495 vmcs12->guest_sysenter_eip = vmcs_readl(GUEST_SYSENTER_EIP); 3496 if (kvm_mpx_supported()) 3497 vmcs12->guest_bndcfgs = vmcs_read64(GUEST_BNDCFGS); 3498 } 3499 3500 /* 3501 * prepare_vmcs12 is part of what we need to do when the nested L2 guest exits 3502 * and we want to prepare to run its L1 parent. L1 keeps a vmcs for L2 (vmcs12), 3503 * and this function updates it to reflect the changes to the guest state while 3504 * L2 was running (and perhaps made some exits which were handled directly by L0 3505 * without going back to L1), and to reflect the exit reason. 3506 * Note that we do not have to copy here all VMCS fields, just those that 3507 * could have changed by the L2 guest or the exit - i.e., the guest-state and 3508 * exit-information fields only. Other fields are modified by L1 with VMWRITE, 3509 * which already writes to vmcs12 directly. 3510 */ 3511 static void prepare_vmcs12(struct kvm_vcpu *vcpu, struct vmcs12 *vmcs12, 3512 u32 exit_reason, u32 exit_intr_info, 3513 unsigned long exit_qualification) 3514 { 3515 /* update guest state fields: */ 3516 sync_vmcs12(vcpu, vmcs12); 3517 3518 /* update exit information fields: */ 3519 3520 vmcs12->vm_exit_reason = exit_reason; 3521 vmcs12->exit_qualification = exit_qualification; 3522 vmcs12->vm_exit_intr_info = exit_intr_info; 3523 3524 vmcs12->idt_vectoring_info_field = 0; 3525 vmcs12->vm_exit_instruction_len = vmcs_read32(VM_EXIT_INSTRUCTION_LEN); 3526 vmcs12->vmx_instruction_info = vmcs_read32(VMX_INSTRUCTION_INFO); 3527 3528 if (!(vmcs12->vm_exit_reason & VMX_EXIT_REASONS_FAILED_VMENTRY)) { 3529 vmcs12->launch_state = 1; 3530 3531 /* vm_entry_intr_info_field is cleared on exit. Emulate this 3532 * instead of reading the real value. */ 3533 vmcs12->vm_entry_intr_info_field &= ~INTR_INFO_VALID_MASK; 3534 3535 /* 3536 * Transfer the event that L0 or L1 may wanted to inject into 3537 * L2 to IDT_VECTORING_INFO_FIELD. 3538 */ 3539 vmcs12_save_pending_event(vcpu, vmcs12); 3540 3541 /* 3542 * According to spec, there's no need to store the guest's 3543 * MSRs if the exit is due to a VM-entry failure that occurs 3544 * during or after loading the guest state. Since this exit 3545 * does not fall in that category, we need to save the MSRs. 3546 */ 3547 if (nested_vmx_store_msr(vcpu, 3548 vmcs12->vm_exit_msr_store_addr, 3549 vmcs12->vm_exit_msr_store_count)) 3550 nested_vmx_abort(vcpu, 3551 VMX_ABORT_SAVE_GUEST_MSR_FAIL); 3552 } 3553 3554 /* 3555 * Drop what we picked up for L2 via vmx_complete_interrupts. It is 3556 * preserved above and would only end up incorrectly in L1. 3557 */ 3558 vcpu->arch.nmi_injected = false; 3559 kvm_clear_exception_queue(vcpu); 3560 kvm_clear_interrupt_queue(vcpu); 3561 } 3562 3563 /* 3564 * A part of what we need to when the nested L2 guest exits and we want to 3565 * run its L1 parent, is to reset L1's guest state to the host state specified 3566 * in vmcs12. 3567 * This function is to be called not only on normal nested exit, but also on 3568 * a nested entry failure, as explained in Intel's spec, 3B.23.7 ("VM-Entry 3569 * Failures During or After Loading Guest State"). 3570 * This function should be called when the active VMCS is L1's (vmcs01). 3571 */ 3572 static void load_vmcs12_host_state(struct kvm_vcpu *vcpu, 3573 struct vmcs12 *vmcs12) 3574 { 3575 struct kvm_segment seg; 3576 u32 entry_failure_code; 3577 3578 if (vmcs12->vm_exit_controls & VM_EXIT_LOAD_IA32_EFER) 3579 vcpu->arch.efer = vmcs12->host_ia32_efer; 3580 else if (vmcs12->vm_exit_controls & VM_EXIT_HOST_ADDR_SPACE_SIZE) 3581 vcpu->arch.efer |= (EFER_LMA | EFER_LME); 3582 else 3583 vcpu->arch.efer &= ~(EFER_LMA | EFER_LME); 3584 vmx_set_efer(vcpu, vcpu->arch.efer); 3585 3586 kvm_register_write(vcpu, VCPU_REGS_RSP, vmcs12->host_rsp); 3587 kvm_register_write(vcpu, VCPU_REGS_RIP, vmcs12->host_rip); 3588 vmx_set_rflags(vcpu, X86_EFLAGS_FIXED); 3589 vmx_set_interrupt_shadow(vcpu, 0); 3590 3591 /* 3592 * Note that calling vmx_set_cr0 is important, even if cr0 hasn't 3593 * actually changed, because vmx_set_cr0 refers to efer set above. 3594 * 3595 * CR0_GUEST_HOST_MASK is already set in the original vmcs01 3596 * (KVM doesn't change it); 3597 */ 3598 vcpu->arch.cr0_guest_owned_bits = X86_CR0_TS; 3599 vmx_set_cr0(vcpu, vmcs12->host_cr0); 3600 3601 /* Same as above - no reason to call set_cr4_guest_host_mask(). */ 3602 vcpu->arch.cr4_guest_owned_bits = ~vmcs_readl(CR4_GUEST_HOST_MASK); 3603 vmx_set_cr4(vcpu, vmcs12->host_cr4); 3604 3605 nested_ept_uninit_mmu_context(vcpu); 3606 3607 /* 3608 * Only PDPTE load can fail as the value of cr3 was checked on entry and 3609 * couldn't have changed. 3610 */ 3611 if (nested_vmx_load_cr3(vcpu, vmcs12->host_cr3, false, &entry_failure_code)) 3612 nested_vmx_abort(vcpu, VMX_ABORT_LOAD_HOST_PDPTE_FAIL); 3613 3614 if (!enable_ept) 3615 vcpu->arch.walk_mmu->inject_page_fault = kvm_inject_page_fault; 3616 3617 /* 3618 * If vmcs01 doesn't use VPID, CPU flushes TLB on every 3619 * VMEntry/VMExit. Thus, no need to flush TLB. 3620 * 3621 * If vmcs12 doesn't use VPID, L1 expects TLB to be 3622 * flushed on every VMEntry/VMExit. 3623 * 3624 * Otherwise, we can preserve TLB entries as long as we are 3625 * able to tag L1 TLB entries differently than L2 TLB entries. 3626 * 3627 * If vmcs12 uses EPT, we need to execute this flush on EPTP01 3628 * and therefore we request the TLB flush to happen only after VMCS EPTP 3629 * has been set by KVM_REQ_LOAD_CR3. 3630 */ 3631 if (enable_vpid && 3632 (!nested_cpu_has_vpid(vmcs12) || !nested_has_guest_tlb_tag(vcpu))) { 3633 kvm_make_request(KVM_REQ_TLB_FLUSH, vcpu); 3634 } 3635 3636 vmcs_write32(GUEST_SYSENTER_CS, vmcs12->host_ia32_sysenter_cs); 3637 vmcs_writel(GUEST_SYSENTER_ESP, vmcs12->host_ia32_sysenter_esp); 3638 vmcs_writel(GUEST_SYSENTER_EIP, vmcs12->host_ia32_sysenter_eip); 3639 vmcs_writel(GUEST_IDTR_BASE, vmcs12->host_idtr_base); 3640 vmcs_writel(GUEST_GDTR_BASE, vmcs12->host_gdtr_base); 3641 vmcs_write32(GUEST_IDTR_LIMIT, 0xFFFF); 3642 vmcs_write32(GUEST_GDTR_LIMIT, 0xFFFF); 3643 3644 /* If not VM_EXIT_CLEAR_BNDCFGS, the L2 value propagates to L1. */ 3645 if (vmcs12->vm_exit_controls & VM_EXIT_CLEAR_BNDCFGS) 3646 vmcs_write64(GUEST_BNDCFGS, 0); 3647 3648 if (vmcs12->vm_exit_controls & VM_EXIT_LOAD_IA32_PAT) { 3649 vmcs_write64(GUEST_IA32_PAT, vmcs12->host_ia32_pat); 3650 vcpu->arch.pat = vmcs12->host_ia32_pat; 3651 } 3652 if (vmcs12->vm_exit_controls & VM_EXIT_LOAD_IA32_PERF_GLOBAL_CTRL) 3653 vmcs_write64(GUEST_IA32_PERF_GLOBAL_CTRL, 3654 vmcs12->host_ia32_perf_global_ctrl); 3655 3656 /* Set L1 segment info according to Intel SDM 3657 27.5.2 Loading Host Segment and Descriptor-Table Registers */ 3658 seg = (struct kvm_segment) { 3659 .base = 0, 3660 .limit = 0xFFFFFFFF, 3661 .selector = vmcs12->host_cs_selector, 3662 .type = 11, 3663 .present = 1, 3664 .s = 1, 3665 .g = 1 3666 }; 3667 if (vmcs12->vm_exit_controls & VM_EXIT_HOST_ADDR_SPACE_SIZE) 3668 seg.l = 1; 3669 else 3670 seg.db = 1; 3671 vmx_set_segment(vcpu, &seg, VCPU_SREG_CS); 3672 seg = (struct kvm_segment) { 3673 .base = 0, 3674 .limit = 0xFFFFFFFF, 3675 .type = 3, 3676 .present = 1, 3677 .s = 1, 3678 .db = 1, 3679 .g = 1 3680 }; 3681 seg.selector = vmcs12->host_ds_selector; 3682 vmx_set_segment(vcpu, &seg, VCPU_SREG_DS); 3683 seg.selector = vmcs12->host_es_selector; 3684 vmx_set_segment(vcpu, &seg, VCPU_SREG_ES); 3685 seg.selector = vmcs12->host_ss_selector; 3686 vmx_set_segment(vcpu, &seg, VCPU_SREG_SS); 3687 seg.selector = vmcs12->host_fs_selector; 3688 seg.base = vmcs12->host_fs_base; 3689 vmx_set_segment(vcpu, &seg, VCPU_SREG_FS); 3690 seg.selector = vmcs12->host_gs_selector; 3691 seg.base = vmcs12->host_gs_base; 3692 vmx_set_segment(vcpu, &seg, VCPU_SREG_GS); 3693 seg = (struct kvm_segment) { 3694 .base = vmcs12->host_tr_base, 3695 .limit = 0x67, 3696 .selector = vmcs12->host_tr_selector, 3697 .type = 11, 3698 .present = 1 3699 }; 3700 vmx_set_segment(vcpu, &seg, VCPU_SREG_TR); 3701 3702 kvm_set_dr(vcpu, 7, 0x400); 3703 vmcs_write64(GUEST_IA32_DEBUGCTL, 0); 3704 3705 if (cpu_has_vmx_msr_bitmap()) 3706 vmx_update_msr_bitmap(vcpu); 3707 3708 if (nested_vmx_load_msr(vcpu, vmcs12->vm_exit_msr_load_addr, 3709 vmcs12->vm_exit_msr_load_count)) 3710 nested_vmx_abort(vcpu, VMX_ABORT_LOAD_HOST_MSR_FAIL); 3711 } 3712 3713 static inline u64 nested_vmx_get_vmcs01_guest_efer(struct vcpu_vmx *vmx) 3714 { 3715 struct shared_msr_entry *efer_msr; 3716 unsigned int i; 3717 3718 if (vm_entry_controls_get(vmx) & VM_ENTRY_LOAD_IA32_EFER) 3719 return vmcs_read64(GUEST_IA32_EFER); 3720 3721 if (cpu_has_load_ia32_efer()) 3722 return host_efer; 3723 3724 for (i = 0; i < vmx->msr_autoload.guest.nr; ++i) { 3725 if (vmx->msr_autoload.guest.val[i].index == MSR_EFER) 3726 return vmx->msr_autoload.guest.val[i].value; 3727 } 3728 3729 efer_msr = find_msr_entry(vmx, MSR_EFER); 3730 if (efer_msr) 3731 return efer_msr->data; 3732 3733 return host_efer; 3734 } 3735 3736 static void nested_vmx_restore_host_state(struct kvm_vcpu *vcpu) 3737 { 3738 struct vmcs12 *vmcs12 = get_vmcs12(vcpu); 3739 struct vcpu_vmx *vmx = to_vmx(vcpu); 3740 struct vmx_msr_entry g, h; 3741 struct msr_data msr; 3742 gpa_t gpa; 3743 u32 i, j; 3744 3745 vcpu->arch.pat = vmcs_read64(GUEST_IA32_PAT); 3746 3747 if (vmcs12->vm_entry_controls & VM_ENTRY_LOAD_DEBUG_CONTROLS) { 3748 /* 3749 * L1's host DR7 is lost if KVM_GUESTDBG_USE_HW_BP is set 3750 * as vmcs01.GUEST_DR7 contains a userspace defined value 3751 * and vcpu->arch.dr7 is not squirreled away before the 3752 * nested VMENTER (not worth adding a variable in nested_vmx). 3753 */ 3754 if (vcpu->guest_debug & KVM_GUESTDBG_USE_HW_BP) 3755 kvm_set_dr(vcpu, 7, DR7_FIXED_1); 3756 else 3757 WARN_ON(kvm_set_dr(vcpu, 7, vmcs_readl(GUEST_DR7))); 3758 } 3759 3760 /* 3761 * Note that calling vmx_set_{efer,cr0,cr4} is important as they 3762 * handle a variety of side effects to KVM's software model. 3763 */ 3764 vmx_set_efer(vcpu, nested_vmx_get_vmcs01_guest_efer(vmx)); 3765 3766 vcpu->arch.cr0_guest_owned_bits = X86_CR0_TS; 3767 vmx_set_cr0(vcpu, vmcs_readl(CR0_READ_SHADOW)); 3768 3769 vcpu->arch.cr4_guest_owned_bits = ~vmcs_readl(CR4_GUEST_HOST_MASK); 3770 vmx_set_cr4(vcpu, vmcs_readl(CR4_READ_SHADOW)); 3771 3772 nested_ept_uninit_mmu_context(vcpu); 3773 vcpu->arch.cr3 = vmcs_readl(GUEST_CR3); 3774 __set_bit(VCPU_EXREG_CR3, (ulong *)&vcpu->arch.regs_avail); 3775 3776 /* 3777 * Use ept_save_pdptrs(vcpu) to load the MMU's cached PDPTRs 3778 * from vmcs01 (if necessary). The PDPTRs are not loaded on 3779 * VMFail, like everything else we just need to ensure our 3780 * software model is up-to-date. 3781 */ 3782 ept_save_pdptrs(vcpu); 3783 3784 kvm_mmu_reset_context(vcpu); 3785 3786 if (cpu_has_vmx_msr_bitmap()) 3787 vmx_update_msr_bitmap(vcpu); 3788 3789 /* 3790 * This nasty bit of open coding is a compromise between blindly 3791 * loading L1's MSRs using the exit load lists (incorrect emulation 3792 * of VMFail), leaving the nested VM's MSRs in the software model 3793 * (incorrect behavior) and snapshotting the modified MSRs (too 3794 * expensive since the lists are unbound by hardware). For each 3795 * MSR that was (prematurely) loaded from the nested VMEntry load 3796 * list, reload it from the exit load list if it exists and differs 3797 * from the guest value. The intent is to stuff host state as 3798 * silently as possible, not to fully process the exit load list. 3799 */ 3800 msr.host_initiated = false; 3801 for (i = 0; i < vmcs12->vm_entry_msr_load_count; i++) { 3802 gpa = vmcs12->vm_entry_msr_load_addr + (i * sizeof(g)); 3803 if (kvm_vcpu_read_guest(vcpu, gpa, &g, sizeof(g))) { 3804 pr_debug_ratelimited( 3805 "%s read MSR index failed (%u, 0x%08llx)\n", 3806 __func__, i, gpa); 3807 goto vmabort; 3808 } 3809 3810 for (j = 0; j < vmcs12->vm_exit_msr_load_count; j++) { 3811 gpa = vmcs12->vm_exit_msr_load_addr + (j * sizeof(h)); 3812 if (kvm_vcpu_read_guest(vcpu, gpa, &h, sizeof(h))) { 3813 pr_debug_ratelimited( 3814 "%s read MSR failed (%u, 0x%08llx)\n", 3815 __func__, j, gpa); 3816 goto vmabort; 3817 } 3818 if (h.index != g.index) 3819 continue; 3820 if (h.value == g.value) 3821 break; 3822 3823 if (nested_vmx_load_msr_check(vcpu, &h)) { 3824 pr_debug_ratelimited( 3825 "%s check failed (%u, 0x%x, 0x%x)\n", 3826 __func__, j, h.index, h.reserved); 3827 goto vmabort; 3828 } 3829 3830 msr.index = h.index; 3831 msr.data = h.value; 3832 if (kvm_set_msr(vcpu, &msr)) { 3833 pr_debug_ratelimited( 3834 "%s WRMSR failed (%u, 0x%x, 0x%llx)\n", 3835 __func__, j, h.index, h.value); 3836 goto vmabort; 3837 } 3838 } 3839 } 3840 3841 return; 3842 3843 vmabort: 3844 nested_vmx_abort(vcpu, VMX_ABORT_LOAD_HOST_MSR_FAIL); 3845 } 3846 3847 /* 3848 * Emulate an exit from nested guest (L2) to L1, i.e., prepare to run L1 3849 * and modify vmcs12 to make it see what it would expect to see there if 3850 * L2 was its real guest. Must only be called when in L2 (is_guest_mode()) 3851 */ 3852 void nested_vmx_vmexit(struct kvm_vcpu *vcpu, u32 exit_reason, 3853 u32 exit_intr_info, unsigned long exit_qualification) 3854 { 3855 struct vcpu_vmx *vmx = to_vmx(vcpu); 3856 struct vmcs12 *vmcs12 = get_vmcs12(vcpu); 3857 3858 /* trying to cancel vmlaunch/vmresume is a bug */ 3859 WARN_ON_ONCE(vmx->nested.nested_run_pending); 3860 3861 leave_guest_mode(vcpu); 3862 3863 if (vmcs12->cpu_based_vm_exec_control & CPU_BASED_USE_TSC_OFFSETING) 3864 vcpu->arch.tsc_offset -= vmcs12->tsc_offset; 3865 3866 if (likely(!vmx->fail)) { 3867 if (exit_reason == -1) 3868 sync_vmcs12(vcpu, vmcs12); 3869 else 3870 prepare_vmcs12(vcpu, vmcs12, exit_reason, exit_intr_info, 3871 exit_qualification); 3872 3873 /* 3874 * Must happen outside of sync_vmcs12() as it will 3875 * also be used to capture vmcs12 cache as part of 3876 * capturing nVMX state for snapshot (migration). 3877 * 3878 * Otherwise, this flush will dirty guest memory at a 3879 * point it is already assumed by user-space to be 3880 * immutable. 3881 */ 3882 nested_flush_cached_shadow_vmcs12(vcpu, vmcs12); 3883 } else { 3884 /* 3885 * The only expected VM-instruction error is "VM entry with 3886 * invalid control field(s)." Anything else indicates a 3887 * problem with L0. And we should never get here with a 3888 * VMFail of any type if early consistency checks are enabled. 3889 */ 3890 WARN_ON_ONCE(vmcs_read32(VM_INSTRUCTION_ERROR) != 3891 VMXERR_ENTRY_INVALID_CONTROL_FIELD); 3892 WARN_ON_ONCE(nested_early_check); 3893 } 3894 3895 vmx_switch_vmcs(vcpu, &vmx->vmcs01); 3896 3897 /* Update any VMCS fields that might have changed while L2 ran */ 3898 vmcs_write32(VM_EXIT_MSR_LOAD_COUNT, vmx->msr_autoload.host.nr); 3899 vmcs_write32(VM_ENTRY_MSR_LOAD_COUNT, vmx->msr_autoload.guest.nr); 3900 vmcs_write64(TSC_OFFSET, vcpu->arch.tsc_offset); 3901 3902 if (kvm_has_tsc_control) 3903 decache_tsc_multiplier(vmx); 3904 3905 if (vmx->nested.change_vmcs01_virtual_apic_mode) { 3906 vmx->nested.change_vmcs01_virtual_apic_mode = false; 3907 vmx_set_virtual_apic_mode(vcpu); 3908 } else if (!nested_cpu_has_ept(vmcs12) && 3909 nested_cpu_has2(vmcs12, 3910 SECONDARY_EXEC_VIRTUALIZE_APIC_ACCESSES)) { 3911 vmx_flush_tlb(vcpu, true); 3912 } 3913 3914 /* This is needed for same reason as it was needed in prepare_vmcs02 */ 3915 vmx->host_rsp = 0; 3916 3917 /* Unpin physical memory we referred to in vmcs02 */ 3918 if (vmx->nested.apic_access_page) { 3919 kvm_release_page_dirty(vmx->nested.apic_access_page); 3920 vmx->nested.apic_access_page = NULL; 3921 } 3922 if (vmx->nested.virtual_apic_page) { 3923 kvm_release_page_dirty(vmx->nested.virtual_apic_page); 3924 vmx->nested.virtual_apic_page = NULL; 3925 } 3926 if (vmx->nested.pi_desc_page) { 3927 kunmap(vmx->nested.pi_desc_page); 3928 kvm_release_page_dirty(vmx->nested.pi_desc_page); 3929 vmx->nested.pi_desc_page = NULL; 3930 vmx->nested.pi_desc = NULL; 3931 } 3932 3933 /* 3934 * We are now running in L2, mmu_notifier will force to reload the 3935 * page's hpa for L2 vmcs. Need to reload it for L1 before entering L1. 3936 */ 3937 kvm_make_request(KVM_REQ_APIC_PAGE_RELOAD, vcpu); 3938 3939 if ((exit_reason != -1) && (enable_shadow_vmcs || vmx->nested.hv_evmcs)) 3940 vmx->nested.need_vmcs12_sync = true; 3941 3942 /* in case we halted in L2 */ 3943 vcpu->arch.mp_state = KVM_MP_STATE_RUNNABLE; 3944 3945 if (likely(!vmx->fail)) { 3946 /* 3947 * TODO: SDM says that with acknowledge interrupt on 3948 * exit, bit 31 of the VM-exit interrupt information 3949 * (valid interrupt) is always set to 1 on 3950 * EXIT_REASON_EXTERNAL_INTERRUPT, so we shouldn't 3951 * need kvm_cpu_has_interrupt(). See the commit 3952 * message for details. 3953 */ 3954 if (nested_exit_intr_ack_set(vcpu) && 3955 exit_reason == EXIT_REASON_EXTERNAL_INTERRUPT && 3956 kvm_cpu_has_interrupt(vcpu)) { 3957 int irq = kvm_cpu_get_interrupt(vcpu); 3958 WARN_ON(irq < 0); 3959 vmcs12->vm_exit_intr_info = irq | 3960 INTR_INFO_VALID_MASK | INTR_TYPE_EXT_INTR; 3961 } 3962 3963 if (exit_reason != -1) 3964 trace_kvm_nested_vmexit_inject(vmcs12->vm_exit_reason, 3965 vmcs12->exit_qualification, 3966 vmcs12->idt_vectoring_info_field, 3967 vmcs12->vm_exit_intr_info, 3968 vmcs12->vm_exit_intr_error_code, 3969 KVM_ISA_VMX); 3970 3971 load_vmcs12_host_state(vcpu, vmcs12); 3972 3973 return; 3974 } 3975 3976 /* 3977 * After an early L2 VM-entry failure, we're now back 3978 * in L1 which thinks it just finished a VMLAUNCH or 3979 * VMRESUME instruction, so we need to set the failure 3980 * flag and the VM-instruction error field of the VMCS 3981 * accordingly, and skip the emulated instruction. 3982 */ 3983 (void)nested_vmx_failValid(vcpu, VMXERR_ENTRY_INVALID_CONTROL_FIELD); 3984 3985 /* 3986 * Restore L1's host state to KVM's software model. We're here 3987 * because a consistency check was caught by hardware, which 3988 * means some amount of guest state has been propagated to KVM's 3989 * model and needs to be unwound to the host's state. 3990 */ 3991 nested_vmx_restore_host_state(vcpu); 3992 3993 vmx->fail = 0; 3994 } 3995 3996 /* 3997 * Decode the memory-address operand of a vmx instruction, as recorded on an 3998 * exit caused by such an instruction (run by a guest hypervisor). 3999 * On success, returns 0. When the operand is invalid, returns 1 and throws 4000 * #UD or #GP. 4001 */ 4002 int get_vmx_mem_address(struct kvm_vcpu *vcpu, unsigned long exit_qualification, 4003 u32 vmx_instruction_info, bool wr, gva_t *ret) 4004 { 4005 gva_t off; 4006 bool exn; 4007 struct kvm_segment s; 4008 4009 /* 4010 * According to Vol. 3B, "Information for VM Exits Due to Instruction 4011 * Execution", on an exit, vmx_instruction_info holds most of the 4012 * addressing components of the operand. Only the displacement part 4013 * is put in exit_qualification (see 3B, "Basic VM-Exit Information"). 4014 * For how an actual address is calculated from all these components, 4015 * refer to Vol. 1, "Operand Addressing". 4016 */ 4017 int scaling = vmx_instruction_info & 3; 4018 int addr_size = (vmx_instruction_info >> 7) & 7; 4019 bool is_reg = vmx_instruction_info & (1u << 10); 4020 int seg_reg = (vmx_instruction_info >> 15) & 7; 4021 int index_reg = (vmx_instruction_info >> 18) & 0xf; 4022 bool index_is_valid = !(vmx_instruction_info & (1u << 22)); 4023 int base_reg = (vmx_instruction_info >> 23) & 0xf; 4024 bool base_is_valid = !(vmx_instruction_info & (1u << 27)); 4025 4026 if (is_reg) { 4027 kvm_queue_exception(vcpu, UD_VECTOR); 4028 return 1; 4029 } 4030 4031 /* Addr = segment_base + offset */ 4032 /* offset = base + [index * scale] + displacement */ 4033 off = exit_qualification; /* holds the displacement */ 4034 if (base_is_valid) 4035 off += kvm_register_read(vcpu, base_reg); 4036 if (index_is_valid) 4037 off += kvm_register_read(vcpu, index_reg)<<scaling; 4038 vmx_get_segment(vcpu, &s, seg_reg); 4039 *ret = s.base + off; 4040 4041 if (addr_size == 1) /* 32 bit */ 4042 *ret &= 0xffffffff; 4043 4044 /* Checks for #GP/#SS exceptions. */ 4045 exn = false; 4046 if (is_long_mode(vcpu)) { 4047 /* Long mode: #GP(0)/#SS(0) if the memory address is in a 4048 * non-canonical form. This is the only check on the memory 4049 * destination for long mode! 4050 */ 4051 exn = is_noncanonical_address(*ret, vcpu); 4052 } else if (is_protmode(vcpu)) { 4053 /* Protected mode: apply checks for segment validity in the 4054 * following order: 4055 * - segment type check (#GP(0) may be thrown) 4056 * - usability check (#GP(0)/#SS(0)) 4057 * - limit check (#GP(0)/#SS(0)) 4058 */ 4059 if (wr) 4060 /* #GP(0) if the destination operand is located in a 4061 * read-only data segment or any code segment. 4062 */ 4063 exn = ((s.type & 0xa) == 0 || (s.type & 8)); 4064 else 4065 /* #GP(0) if the source operand is located in an 4066 * execute-only code segment 4067 */ 4068 exn = ((s.type & 0xa) == 8); 4069 if (exn) { 4070 kvm_queue_exception_e(vcpu, GP_VECTOR, 0); 4071 return 1; 4072 } 4073 /* Protected mode: #GP(0)/#SS(0) if the segment is unusable. 4074 */ 4075 exn = (s.unusable != 0); 4076 /* Protected mode: #GP(0)/#SS(0) if the memory 4077 * operand is outside the segment limit. 4078 */ 4079 exn = exn || (off + sizeof(u64) > s.limit); 4080 } 4081 if (exn) { 4082 kvm_queue_exception_e(vcpu, 4083 seg_reg == VCPU_SREG_SS ? 4084 SS_VECTOR : GP_VECTOR, 4085 0); 4086 return 1; 4087 } 4088 4089 return 0; 4090 } 4091 4092 static int nested_vmx_get_vmptr(struct kvm_vcpu *vcpu, gpa_t *vmpointer) 4093 { 4094 gva_t gva; 4095 struct x86_exception e; 4096 4097 if (get_vmx_mem_address(vcpu, vmcs_readl(EXIT_QUALIFICATION), 4098 vmcs_read32(VMX_INSTRUCTION_INFO), false, &gva)) 4099 return 1; 4100 4101 if (kvm_read_guest_virt(vcpu, gva, vmpointer, sizeof(*vmpointer), &e)) { 4102 kvm_inject_page_fault(vcpu, &e); 4103 return 1; 4104 } 4105 4106 return 0; 4107 } 4108 4109 /* 4110 * Allocate a shadow VMCS and associate it with the currently loaded 4111 * VMCS, unless such a shadow VMCS already exists. The newly allocated 4112 * VMCS is also VMCLEARed, so that it is ready for use. 4113 */ 4114 static struct vmcs *alloc_shadow_vmcs(struct kvm_vcpu *vcpu) 4115 { 4116 struct vcpu_vmx *vmx = to_vmx(vcpu); 4117 struct loaded_vmcs *loaded_vmcs = vmx->loaded_vmcs; 4118 4119 /* 4120 * We should allocate a shadow vmcs for vmcs01 only when L1 4121 * executes VMXON and free it when L1 executes VMXOFF. 4122 * As it is invalid to execute VMXON twice, we shouldn't reach 4123 * here when vmcs01 already have an allocated shadow vmcs. 4124 */ 4125 WARN_ON(loaded_vmcs == &vmx->vmcs01 && loaded_vmcs->shadow_vmcs); 4126 4127 if (!loaded_vmcs->shadow_vmcs) { 4128 loaded_vmcs->shadow_vmcs = alloc_vmcs(true); 4129 if (loaded_vmcs->shadow_vmcs) 4130 vmcs_clear(loaded_vmcs->shadow_vmcs); 4131 } 4132 return loaded_vmcs->shadow_vmcs; 4133 } 4134 4135 static int enter_vmx_operation(struct kvm_vcpu *vcpu) 4136 { 4137 struct vcpu_vmx *vmx = to_vmx(vcpu); 4138 int r; 4139 4140 r = alloc_loaded_vmcs(&vmx->nested.vmcs02); 4141 if (r < 0) 4142 goto out_vmcs02; 4143 4144 vmx->nested.cached_vmcs12 = kzalloc(VMCS12_SIZE, GFP_KERNEL); 4145 if (!vmx->nested.cached_vmcs12) 4146 goto out_cached_vmcs12; 4147 4148 vmx->nested.cached_shadow_vmcs12 = kzalloc(VMCS12_SIZE, GFP_KERNEL); 4149 if (!vmx->nested.cached_shadow_vmcs12) 4150 goto out_cached_shadow_vmcs12; 4151 4152 if (enable_shadow_vmcs && !alloc_shadow_vmcs(vcpu)) 4153 goto out_shadow_vmcs; 4154 4155 hrtimer_init(&vmx->nested.preemption_timer, CLOCK_MONOTONIC, 4156 HRTIMER_MODE_REL_PINNED); 4157 vmx->nested.preemption_timer.function = vmx_preemption_timer_fn; 4158 4159 vmx->nested.vpid02 = allocate_vpid(); 4160 4161 vmx->nested.vmcs02_initialized = false; 4162 vmx->nested.vmxon = true; 4163 4164 if (pt_mode == PT_MODE_HOST_GUEST) { 4165 vmx->pt_desc.guest.ctl = 0; 4166 pt_update_intercept_for_msr(vmx); 4167 } 4168 4169 return 0; 4170 4171 out_shadow_vmcs: 4172 kfree(vmx->nested.cached_shadow_vmcs12); 4173 4174 out_cached_shadow_vmcs12: 4175 kfree(vmx->nested.cached_vmcs12); 4176 4177 out_cached_vmcs12: 4178 free_loaded_vmcs(&vmx->nested.vmcs02); 4179 4180 out_vmcs02: 4181 return -ENOMEM; 4182 } 4183 4184 /* 4185 * Emulate the VMXON instruction. 4186 * Currently, we just remember that VMX is active, and do not save or even 4187 * inspect the argument to VMXON (the so-called "VMXON pointer") because we 4188 * do not currently need to store anything in that guest-allocated memory 4189 * region. Consequently, VMCLEAR and VMPTRLD also do not verify that the their 4190 * argument is different from the VMXON pointer (which the spec says they do). 4191 */ 4192 static int handle_vmon(struct kvm_vcpu *vcpu) 4193 { 4194 int ret; 4195 gpa_t vmptr; 4196 struct page *page; 4197 struct vcpu_vmx *vmx = to_vmx(vcpu); 4198 const u64 VMXON_NEEDED_FEATURES = FEATURE_CONTROL_LOCKED 4199 | FEATURE_CONTROL_VMXON_ENABLED_OUTSIDE_SMX; 4200 4201 /* 4202 * The Intel VMX Instruction Reference lists a bunch of bits that are 4203 * prerequisite to running VMXON, most notably cr4.VMXE must be set to 4204 * 1 (see vmx_set_cr4() for when we allow the guest to set this). 4205 * Otherwise, we should fail with #UD. But most faulting conditions 4206 * have already been checked by hardware, prior to the VM-exit for 4207 * VMXON. We do test guest cr4.VMXE because processor CR4 always has 4208 * that bit set to 1 in non-root mode. 4209 */ 4210 if (!kvm_read_cr4_bits(vcpu, X86_CR4_VMXE)) { 4211 kvm_queue_exception(vcpu, UD_VECTOR); 4212 return 1; 4213 } 4214 4215 /* CPL=0 must be checked manually. */ 4216 if (vmx_get_cpl(vcpu)) { 4217 kvm_inject_gp(vcpu, 0); 4218 return 1; 4219 } 4220 4221 if (vmx->nested.vmxon) 4222 return nested_vmx_failValid(vcpu, 4223 VMXERR_VMXON_IN_VMX_ROOT_OPERATION); 4224 4225 if ((vmx->msr_ia32_feature_control & VMXON_NEEDED_FEATURES) 4226 != VMXON_NEEDED_FEATURES) { 4227 kvm_inject_gp(vcpu, 0); 4228 return 1; 4229 } 4230 4231 if (nested_vmx_get_vmptr(vcpu, &vmptr)) 4232 return 1; 4233 4234 /* 4235 * SDM 3: 24.11.5 4236 * The first 4 bytes of VMXON region contain the supported 4237 * VMCS revision identifier 4238 * 4239 * Note - IA32_VMX_BASIC[48] will never be 1 for the nested case; 4240 * which replaces physical address width with 32 4241 */ 4242 if (!PAGE_ALIGNED(vmptr) || (vmptr >> cpuid_maxphyaddr(vcpu))) 4243 return nested_vmx_failInvalid(vcpu); 4244 4245 page = kvm_vcpu_gpa_to_page(vcpu, vmptr); 4246 if (is_error_page(page)) 4247 return nested_vmx_failInvalid(vcpu); 4248 4249 if (*(u32 *)kmap(page) != VMCS12_REVISION) { 4250 kunmap(page); 4251 kvm_release_page_clean(page); 4252 return nested_vmx_failInvalid(vcpu); 4253 } 4254 kunmap(page); 4255 kvm_release_page_clean(page); 4256 4257 vmx->nested.vmxon_ptr = vmptr; 4258 ret = enter_vmx_operation(vcpu); 4259 if (ret) 4260 return ret; 4261 4262 return nested_vmx_succeed(vcpu); 4263 } 4264 4265 static inline void nested_release_vmcs12(struct kvm_vcpu *vcpu) 4266 { 4267 struct vcpu_vmx *vmx = to_vmx(vcpu); 4268 4269 if (vmx->nested.current_vmptr == -1ull) 4270 return; 4271 4272 if (enable_shadow_vmcs) { 4273 /* copy to memory all shadowed fields in case 4274 they were modified */ 4275 copy_shadow_to_vmcs12(vmx); 4276 vmx->nested.need_vmcs12_sync = false; 4277 vmx_disable_shadow_vmcs(vmx); 4278 } 4279 vmx->nested.posted_intr_nv = -1; 4280 4281 /* Flush VMCS12 to guest memory */ 4282 kvm_vcpu_write_guest_page(vcpu, 4283 vmx->nested.current_vmptr >> PAGE_SHIFT, 4284 vmx->nested.cached_vmcs12, 0, VMCS12_SIZE); 4285 4286 kvm_mmu_free_roots(vcpu, &vcpu->arch.guest_mmu, KVM_MMU_ROOTS_ALL); 4287 4288 vmx->nested.current_vmptr = -1ull; 4289 } 4290 4291 /* Emulate the VMXOFF instruction */ 4292 static int handle_vmoff(struct kvm_vcpu *vcpu) 4293 { 4294 if (!nested_vmx_check_permission(vcpu)) 4295 return 1; 4296 free_nested(vcpu); 4297 return nested_vmx_succeed(vcpu); 4298 } 4299 4300 /* Emulate the VMCLEAR instruction */ 4301 static int handle_vmclear(struct kvm_vcpu *vcpu) 4302 { 4303 struct vcpu_vmx *vmx = to_vmx(vcpu); 4304 u32 zero = 0; 4305 gpa_t vmptr; 4306 4307 if (!nested_vmx_check_permission(vcpu)) 4308 return 1; 4309 4310 if (nested_vmx_get_vmptr(vcpu, &vmptr)) 4311 return 1; 4312 4313 if (!PAGE_ALIGNED(vmptr) || (vmptr >> cpuid_maxphyaddr(vcpu))) 4314 return nested_vmx_failValid(vcpu, 4315 VMXERR_VMCLEAR_INVALID_ADDRESS); 4316 4317 if (vmptr == vmx->nested.vmxon_ptr) 4318 return nested_vmx_failValid(vcpu, 4319 VMXERR_VMCLEAR_VMXON_POINTER); 4320 4321 if (vmx->nested.hv_evmcs_page) { 4322 if (vmptr == vmx->nested.hv_evmcs_vmptr) 4323 nested_release_evmcs(vcpu); 4324 } else { 4325 if (vmptr == vmx->nested.current_vmptr) 4326 nested_release_vmcs12(vcpu); 4327 4328 kvm_vcpu_write_guest(vcpu, 4329 vmptr + offsetof(struct vmcs12, 4330 launch_state), 4331 &zero, sizeof(zero)); 4332 } 4333 4334 return nested_vmx_succeed(vcpu); 4335 } 4336 4337 static int nested_vmx_run(struct kvm_vcpu *vcpu, bool launch); 4338 4339 /* Emulate the VMLAUNCH instruction */ 4340 static int handle_vmlaunch(struct kvm_vcpu *vcpu) 4341 { 4342 return nested_vmx_run(vcpu, true); 4343 } 4344 4345 /* Emulate the VMRESUME instruction */ 4346 static int handle_vmresume(struct kvm_vcpu *vcpu) 4347 { 4348 4349 return nested_vmx_run(vcpu, false); 4350 } 4351 4352 static int handle_vmread(struct kvm_vcpu *vcpu) 4353 { 4354 unsigned long field; 4355 u64 field_value; 4356 unsigned long exit_qualification = vmcs_readl(EXIT_QUALIFICATION); 4357 u32 vmx_instruction_info = vmcs_read32(VMX_INSTRUCTION_INFO); 4358 gva_t gva = 0; 4359 struct vmcs12 *vmcs12; 4360 4361 if (!nested_vmx_check_permission(vcpu)) 4362 return 1; 4363 4364 if (to_vmx(vcpu)->nested.current_vmptr == -1ull) 4365 return nested_vmx_failInvalid(vcpu); 4366 4367 if (!is_guest_mode(vcpu)) 4368 vmcs12 = get_vmcs12(vcpu); 4369 else { 4370 /* 4371 * When vmcs->vmcs_link_pointer is -1ull, any VMREAD 4372 * to shadowed-field sets the ALU flags for VMfailInvalid. 4373 */ 4374 if (get_vmcs12(vcpu)->vmcs_link_pointer == -1ull) 4375 return nested_vmx_failInvalid(vcpu); 4376 vmcs12 = get_shadow_vmcs12(vcpu); 4377 } 4378 4379 /* Decode instruction info and find the field to read */ 4380 field = kvm_register_readl(vcpu, (((vmx_instruction_info) >> 28) & 0xf)); 4381 /* Read the field, zero-extended to a u64 field_value */ 4382 if (vmcs12_read_any(vmcs12, field, &field_value) < 0) 4383 return nested_vmx_failValid(vcpu, 4384 VMXERR_UNSUPPORTED_VMCS_COMPONENT); 4385 4386 /* 4387 * Now copy part of this value to register or memory, as requested. 4388 * Note that the number of bits actually copied is 32 or 64 depending 4389 * on the guest's mode (32 or 64 bit), not on the given field's length. 4390 */ 4391 if (vmx_instruction_info & (1u << 10)) { 4392 kvm_register_writel(vcpu, (((vmx_instruction_info) >> 3) & 0xf), 4393 field_value); 4394 } else { 4395 if (get_vmx_mem_address(vcpu, exit_qualification, 4396 vmx_instruction_info, true, &gva)) 4397 return 1; 4398 /* _system ok, nested_vmx_check_permission has verified cpl=0 */ 4399 kvm_write_guest_virt_system(vcpu, gva, &field_value, 4400 (is_long_mode(vcpu) ? 8 : 4), NULL); 4401 } 4402 4403 return nested_vmx_succeed(vcpu); 4404 } 4405 4406 4407 static int handle_vmwrite(struct kvm_vcpu *vcpu) 4408 { 4409 unsigned long field; 4410 gva_t gva; 4411 struct vcpu_vmx *vmx = to_vmx(vcpu); 4412 unsigned long exit_qualification = vmcs_readl(EXIT_QUALIFICATION); 4413 u32 vmx_instruction_info = vmcs_read32(VMX_INSTRUCTION_INFO); 4414 4415 /* The value to write might be 32 or 64 bits, depending on L1's long 4416 * mode, and eventually we need to write that into a field of several 4417 * possible lengths. The code below first zero-extends the value to 64 4418 * bit (field_value), and then copies only the appropriate number of 4419 * bits into the vmcs12 field. 4420 */ 4421 u64 field_value = 0; 4422 struct x86_exception e; 4423 struct vmcs12 *vmcs12; 4424 4425 if (!nested_vmx_check_permission(vcpu)) 4426 return 1; 4427 4428 if (vmx->nested.current_vmptr == -1ull) 4429 return nested_vmx_failInvalid(vcpu); 4430 4431 if (vmx_instruction_info & (1u << 10)) 4432 field_value = kvm_register_readl(vcpu, 4433 (((vmx_instruction_info) >> 3) & 0xf)); 4434 else { 4435 if (get_vmx_mem_address(vcpu, exit_qualification, 4436 vmx_instruction_info, false, &gva)) 4437 return 1; 4438 if (kvm_read_guest_virt(vcpu, gva, &field_value, 4439 (is_64_bit_mode(vcpu) ? 8 : 4), &e)) { 4440 kvm_inject_page_fault(vcpu, &e); 4441 return 1; 4442 } 4443 } 4444 4445 4446 field = kvm_register_readl(vcpu, (((vmx_instruction_info) >> 28) & 0xf)); 4447 /* 4448 * If the vCPU supports "VMWRITE to any supported field in the 4449 * VMCS," then the "read-only" fields are actually read/write. 4450 */ 4451 if (vmcs_field_readonly(field) && 4452 !nested_cpu_has_vmwrite_any_field(vcpu)) 4453 return nested_vmx_failValid(vcpu, 4454 VMXERR_VMWRITE_READ_ONLY_VMCS_COMPONENT); 4455 4456 if (!is_guest_mode(vcpu)) 4457 vmcs12 = get_vmcs12(vcpu); 4458 else { 4459 /* 4460 * When vmcs->vmcs_link_pointer is -1ull, any VMWRITE 4461 * to shadowed-field sets the ALU flags for VMfailInvalid. 4462 */ 4463 if (get_vmcs12(vcpu)->vmcs_link_pointer == -1ull) 4464 return nested_vmx_failInvalid(vcpu); 4465 vmcs12 = get_shadow_vmcs12(vcpu); 4466 } 4467 4468 if (vmcs12_write_any(vmcs12, field, field_value) < 0) 4469 return nested_vmx_failValid(vcpu, 4470 VMXERR_UNSUPPORTED_VMCS_COMPONENT); 4471 4472 /* 4473 * Do not track vmcs12 dirty-state if in guest-mode 4474 * as we actually dirty shadow vmcs12 instead of vmcs12. 4475 */ 4476 if (!is_guest_mode(vcpu)) { 4477 switch (field) { 4478 #define SHADOW_FIELD_RW(x) case x: 4479 #include "vmcs_shadow_fields.h" 4480 /* 4481 * The fields that can be updated by L1 without a vmexit are 4482 * always updated in the vmcs02, the others go down the slow 4483 * path of prepare_vmcs02. 4484 */ 4485 break; 4486 default: 4487 vmx->nested.dirty_vmcs12 = true; 4488 break; 4489 } 4490 } 4491 4492 return nested_vmx_succeed(vcpu); 4493 } 4494 4495 static void set_current_vmptr(struct vcpu_vmx *vmx, gpa_t vmptr) 4496 { 4497 vmx->nested.current_vmptr = vmptr; 4498 if (enable_shadow_vmcs) { 4499 vmcs_set_bits(SECONDARY_VM_EXEC_CONTROL, 4500 SECONDARY_EXEC_SHADOW_VMCS); 4501 vmcs_write64(VMCS_LINK_POINTER, 4502 __pa(vmx->vmcs01.shadow_vmcs)); 4503 vmx->nested.need_vmcs12_sync = true; 4504 } 4505 vmx->nested.dirty_vmcs12 = true; 4506 } 4507 4508 /* Emulate the VMPTRLD instruction */ 4509 static int handle_vmptrld(struct kvm_vcpu *vcpu) 4510 { 4511 struct vcpu_vmx *vmx = to_vmx(vcpu); 4512 gpa_t vmptr; 4513 4514 if (!nested_vmx_check_permission(vcpu)) 4515 return 1; 4516 4517 if (nested_vmx_get_vmptr(vcpu, &vmptr)) 4518 return 1; 4519 4520 if (!PAGE_ALIGNED(vmptr) || (vmptr >> cpuid_maxphyaddr(vcpu))) 4521 return nested_vmx_failValid(vcpu, 4522 VMXERR_VMPTRLD_INVALID_ADDRESS); 4523 4524 if (vmptr == vmx->nested.vmxon_ptr) 4525 return nested_vmx_failValid(vcpu, 4526 VMXERR_VMPTRLD_VMXON_POINTER); 4527 4528 /* Forbid normal VMPTRLD if Enlightened version was used */ 4529 if (vmx->nested.hv_evmcs) 4530 return 1; 4531 4532 if (vmx->nested.current_vmptr != vmptr) { 4533 struct vmcs12 *new_vmcs12; 4534 struct page *page; 4535 4536 page = kvm_vcpu_gpa_to_page(vcpu, vmptr); 4537 if (is_error_page(page)) { 4538 /* 4539 * Reads from an unbacked page return all 1s, 4540 * which means that the 32 bits located at the 4541 * given physical address won't match the required 4542 * VMCS12_REVISION identifier. 4543 */ 4544 return nested_vmx_failValid(vcpu, 4545 VMXERR_VMPTRLD_INCORRECT_VMCS_REVISION_ID); 4546 } 4547 new_vmcs12 = kmap(page); 4548 if (new_vmcs12->hdr.revision_id != VMCS12_REVISION || 4549 (new_vmcs12->hdr.shadow_vmcs && 4550 !nested_cpu_has_vmx_shadow_vmcs(vcpu))) { 4551 kunmap(page); 4552 kvm_release_page_clean(page); 4553 return nested_vmx_failValid(vcpu, 4554 VMXERR_VMPTRLD_INCORRECT_VMCS_REVISION_ID); 4555 } 4556 4557 nested_release_vmcs12(vcpu); 4558 4559 /* 4560 * Load VMCS12 from guest memory since it is not already 4561 * cached. 4562 */ 4563 memcpy(vmx->nested.cached_vmcs12, new_vmcs12, VMCS12_SIZE); 4564 kunmap(page); 4565 kvm_release_page_clean(page); 4566 4567 set_current_vmptr(vmx, vmptr); 4568 } 4569 4570 return nested_vmx_succeed(vcpu); 4571 } 4572 4573 /* Emulate the VMPTRST instruction */ 4574 static int handle_vmptrst(struct kvm_vcpu *vcpu) 4575 { 4576 unsigned long exit_qual = vmcs_readl(EXIT_QUALIFICATION); 4577 u32 instr_info = vmcs_read32(VMX_INSTRUCTION_INFO); 4578 gpa_t current_vmptr = to_vmx(vcpu)->nested.current_vmptr; 4579 struct x86_exception e; 4580 gva_t gva; 4581 4582 if (!nested_vmx_check_permission(vcpu)) 4583 return 1; 4584 4585 if (unlikely(to_vmx(vcpu)->nested.hv_evmcs)) 4586 return 1; 4587 4588 if (get_vmx_mem_address(vcpu, exit_qual, instr_info, true, &gva)) 4589 return 1; 4590 /* *_system ok, nested_vmx_check_permission has verified cpl=0 */ 4591 if (kvm_write_guest_virt_system(vcpu, gva, (void *)¤t_vmptr, 4592 sizeof(gpa_t), &e)) { 4593 kvm_inject_page_fault(vcpu, &e); 4594 return 1; 4595 } 4596 return nested_vmx_succeed(vcpu); 4597 } 4598 4599 /* Emulate the INVEPT instruction */ 4600 static int handle_invept(struct kvm_vcpu *vcpu) 4601 { 4602 struct vcpu_vmx *vmx = to_vmx(vcpu); 4603 u32 vmx_instruction_info, types; 4604 unsigned long type; 4605 gva_t gva; 4606 struct x86_exception e; 4607 struct { 4608 u64 eptp, gpa; 4609 } operand; 4610 4611 if (!(vmx->nested.msrs.secondary_ctls_high & 4612 SECONDARY_EXEC_ENABLE_EPT) || 4613 !(vmx->nested.msrs.ept_caps & VMX_EPT_INVEPT_BIT)) { 4614 kvm_queue_exception(vcpu, UD_VECTOR); 4615 return 1; 4616 } 4617 4618 if (!nested_vmx_check_permission(vcpu)) 4619 return 1; 4620 4621 vmx_instruction_info = vmcs_read32(VMX_INSTRUCTION_INFO); 4622 type = kvm_register_readl(vcpu, (vmx_instruction_info >> 28) & 0xf); 4623 4624 types = (vmx->nested.msrs.ept_caps >> VMX_EPT_EXTENT_SHIFT) & 6; 4625 4626 if (type >= 32 || !(types & (1 << type))) 4627 return nested_vmx_failValid(vcpu, 4628 VMXERR_INVALID_OPERAND_TO_INVEPT_INVVPID); 4629 4630 /* According to the Intel VMX instruction reference, the memory 4631 * operand is read even if it isn't needed (e.g., for type==global) 4632 */ 4633 if (get_vmx_mem_address(vcpu, vmcs_readl(EXIT_QUALIFICATION), 4634 vmx_instruction_info, false, &gva)) 4635 return 1; 4636 if (kvm_read_guest_virt(vcpu, gva, &operand, sizeof(operand), &e)) { 4637 kvm_inject_page_fault(vcpu, &e); 4638 return 1; 4639 } 4640 4641 switch (type) { 4642 case VMX_EPT_EXTENT_GLOBAL: 4643 /* 4644 * TODO: track mappings and invalidate 4645 * single context requests appropriately 4646 */ 4647 case VMX_EPT_EXTENT_CONTEXT: 4648 kvm_mmu_sync_roots(vcpu); 4649 kvm_make_request(KVM_REQ_TLB_FLUSH, vcpu); 4650 break; 4651 default: 4652 BUG_ON(1); 4653 break; 4654 } 4655 4656 return nested_vmx_succeed(vcpu); 4657 } 4658 4659 static int handle_invvpid(struct kvm_vcpu *vcpu) 4660 { 4661 struct vcpu_vmx *vmx = to_vmx(vcpu); 4662 u32 vmx_instruction_info; 4663 unsigned long type, types; 4664 gva_t gva; 4665 struct x86_exception e; 4666 struct { 4667 u64 vpid; 4668 u64 gla; 4669 } operand; 4670 u16 vpid02; 4671 4672 if (!(vmx->nested.msrs.secondary_ctls_high & 4673 SECONDARY_EXEC_ENABLE_VPID) || 4674 !(vmx->nested.msrs.vpid_caps & VMX_VPID_INVVPID_BIT)) { 4675 kvm_queue_exception(vcpu, UD_VECTOR); 4676 return 1; 4677 } 4678 4679 if (!nested_vmx_check_permission(vcpu)) 4680 return 1; 4681 4682 vmx_instruction_info = vmcs_read32(VMX_INSTRUCTION_INFO); 4683 type = kvm_register_readl(vcpu, (vmx_instruction_info >> 28) & 0xf); 4684 4685 types = (vmx->nested.msrs.vpid_caps & 4686 VMX_VPID_EXTENT_SUPPORTED_MASK) >> 8; 4687 4688 if (type >= 32 || !(types & (1 << type))) 4689 return nested_vmx_failValid(vcpu, 4690 VMXERR_INVALID_OPERAND_TO_INVEPT_INVVPID); 4691 4692 /* according to the intel vmx instruction reference, the memory 4693 * operand is read even if it isn't needed (e.g., for type==global) 4694 */ 4695 if (get_vmx_mem_address(vcpu, vmcs_readl(EXIT_QUALIFICATION), 4696 vmx_instruction_info, false, &gva)) 4697 return 1; 4698 if (kvm_read_guest_virt(vcpu, gva, &operand, sizeof(operand), &e)) { 4699 kvm_inject_page_fault(vcpu, &e); 4700 return 1; 4701 } 4702 if (operand.vpid >> 16) 4703 return nested_vmx_failValid(vcpu, 4704 VMXERR_INVALID_OPERAND_TO_INVEPT_INVVPID); 4705 4706 vpid02 = nested_get_vpid02(vcpu); 4707 switch (type) { 4708 case VMX_VPID_EXTENT_INDIVIDUAL_ADDR: 4709 if (!operand.vpid || 4710 is_noncanonical_address(operand.gla, vcpu)) 4711 return nested_vmx_failValid(vcpu, 4712 VMXERR_INVALID_OPERAND_TO_INVEPT_INVVPID); 4713 if (cpu_has_vmx_invvpid_individual_addr()) { 4714 __invvpid(VMX_VPID_EXTENT_INDIVIDUAL_ADDR, 4715 vpid02, operand.gla); 4716 } else 4717 __vmx_flush_tlb(vcpu, vpid02, false); 4718 break; 4719 case VMX_VPID_EXTENT_SINGLE_CONTEXT: 4720 case VMX_VPID_EXTENT_SINGLE_NON_GLOBAL: 4721 if (!operand.vpid) 4722 return nested_vmx_failValid(vcpu, 4723 VMXERR_INVALID_OPERAND_TO_INVEPT_INVVPID); 4724 __vmx_flush_tlb(vcpu, vpid02, false); 4725 break; 4726 case VMX_VPID_EXTENT_ALL_CONTEXT: 4727 __vmx_flush_tlb(vcpu, vpid02, false); 4728 break; 4729 default: 4730 WARN_ON_ONCE(1); 4731 return kvm_skip_emulated_instruction(vcpu); 4732 } 4733 4734 return nested_vmx_succeed(vcpu); 4735 } 4736 4737 static int nested_vmx_eptp_switching(struct kvm_vcpu *vcpu, 4738 struct vmcs12 *vmcs12) 4739 { 4740 u32 index = vcpu->arch.regs[VCPU_REGS_RCX]; 4741 u64 address; 4742 bool accessed_dirty; 4743 struct kvm_mmu *mmu = vcpu->arch.walk_mmu; 4744 4745 if (!nested_cpu_has_eptp_switching(vmcs12) || 4746 !nested_cpu_has_ept(vmcs12)) 4747 return 1; 4748 4749 if (index >= VMFUNC_EPTP_ENTRIES) 4750 return 1; 4751 4752 4753 if (kvm_vcpu_read_guest_page(vcpu, vmcs12->eptp_list_address >> PAGE_SHIFT, 4754 &address, index * 8, 8)) 4755 return 1; 4756 4757 accessed_dirty = !!(address & VMX_EPTP_AD_ENABLE_BIT); 4758 4759 /* 4760 * If the (L2) guest does a vmfunc to the currently 4761 * active ept pointer, we don't have to do anything else 4762 */ 4763 if (vmcs12->ept_pointer != address) { 4764 if (!valid_ept_address(vcpu, address)) 4765 return 1; 4766 4767 kvm_mmu_unload(vcpu); 4768 mmu->ept_ad = accessed_dirty; 4769 mmu->mmu_role.base.ad_disabled = !accessed_dirty; 4770 vmcs12->ept_pointer = address; 4771 /* 4772 * TODO: Check what's the correct approach in case 4773 * mmu reload fails. Currently, we just let the next 4774 * reload potentially fail 4775 */ 4776 kvm_mmu_reload(vcpu); 4777 } 4778 4779 return 0; 4780 } 4781 4782 static int handle_vmfunc(struct kvm_vcpu *vcpu) 4783 { 4784 struct vcpu_vmx *vmx = to_vmx(vcpu); 4785 struct vmcs12 *vmcs12; 4786 u32 function = vcpu->arch.regs[VCPU_REGS_RAX]; 4787 4788 /* 4789 * VMFUNC is only supported for nested guests, but we always enable the 4790 * secondary control for simplicity; for non-nested mode, fake that we 4791 * didn't by injecting #UD. 4792 */ 4793 if (!is_guest_mode(vcpu)) { 4794 kvm_queue_exception(vcpu, UD_VECTOR); 4795 return 1; 4796 } 4797 4798 vmcs12 = get_vmcs12(vcpu); 4799 if ((vmcs12->vm_function_control & (1 << function)) == 0) 4800 goto fail; 4801 4802 switch (function) { 4803 case 0: 4804 if (nested_vmx_eptp_switching(vcpu, vmcs12)) 4805 goto fail; 4806 break; 4807 default: 4808 goto fail; 4809 } 4810 return kvm_skip_emulated_instruction(vcpu); 4811 4812 fail: 4813 nested_vmx_vmexit(vcpu, vmx->exit_reason, 4814 vmcs_read32(VM_EXIT_INTR_INFO), 4815 vmcs_readl(EXIT_QUALIFICATION)); 4816 return 1; 4817 } 4818 4819 4820 static bool nested_vmx_exit_handled_io(struct kvm_vcpu *vcpu, 4821 struct vmcs12 *vmcs12) 4822 { 4823 unsigned long exit_qualification; 4824 gpa_t bitmap, last_bitmap; 4825 unsigned int port; 4826 int size; 4827 u8 b; 4828 4829 if (!nested_cpu_has(vmcs12, CPU_BASED_USE_IO_BITMAPS)) 4830 return nested_cpu_has(vmcs12, CPU_BASED_UNCOND_IO_EXITING); 4831 4832 exit_qualification = vmcs_readl(EXIT_QUALIFICATION); 4833 4834 port = exit_qualification >> 16; 4835 size = (exit_qualification & 7) + 1; 4836 4837 last_bitmap = (gpa_t)-1; 4838 b = -1; 4839 4840 while (size > 0) { 4841 if (port < 0x8000) 4842 bitmap = vmcs12->io_bitmap_a; 4843 else if (port < 0x10000) 4844 bitmap = vmcs12->io_bitmap_b; 4845 else 4846 return true; 4847 bitmap += (port & 0x7fff) / 8; 4848 4849 if (last_bitmap != bitmap) 4850 if (kvm_vcpu_read_guest(vcpu, bitmap, &b, 1)) 4851 return true; 4852 if (b & (1 << (port & 7))) 4853 return true; 4854 4855 port++; 4856 size--; 4857 last_bitmap = bitmap; 4858 } 4859 4860 return false; 4861 } 4862 4863 /* 4864 * Return 1 if we should exit from L2 to L1 to handle an MSR access access, 4865 * rather than handle it ourselves in L0. I.e., check whether L1 expressed 4866 * disinterest in the current event (read or write a specific MSR) by using an 4867 * MSR bitmap. This may be the case even when L0 doesn't use MSR bitmaps. 4868 */ 4869 static bool nested_vmx_exit_handled_msr(struct kvm_vcpu *vcpu, 4870 struct vmcs12 *vmcs12, u32 exit_reason) 4871 { 4872 u32 msr_index = vcpu->arch.regs[VCPU_REGS_RCX]; 4873 gpa_t bitmap; 4874 4875 if (!nested_cpu_has(vmcs12, CPU_BASED_USE_MSR_BITMAPS)) 4876 return true; 4877 4878 /* 4879 * The MSR_BITMAP page is divided into four 1024-byte bitmaps, 4880 * for the four combinations of read/write and low/high MSR numbers. 4881 * First we need to figure out which of the four to use: 4882 */ 4883 bitmap = vmcs12->msr_bitmap; 4884 if (exit_reason == EXIT_REASON_MSR_WRITE) 4885 bitmap += 2048; 4886 if (msr_index >= 0xc0000000) { 4887 msr_index -= 0xc0000000; 4888 bitmap += 1024; 4889 } 4890 4891 /* Then read the msr_index'th bit from this bitmap: */ 4892 if (msr_index < 1024*8) { 4893 unsigned char b; 4894 if (kvm_vcpu_read_guest(vcpu, bitmap + msr_index/8, &b, 1)) 4895 return true; 4896 return 1 & (b >> (msr_index & 7)); 4897 } else 4898 return true; /* let L1 handle the wrong parameter */ 4899 } 4900 4901 /* 4902 * Return 1 if we should exit from L2 to L1 to handle a CR access exit, 4903 * rather than handle it ourselves in L0. I.e., check if L1 wanted to 4904 * intercept (via guest_host_mask etc.) the current event. 4905 */ 4906 static bool nested_vmx_exit_handled_cr(struct kvm_vcpu *vcpu, 4907 struct vmcs12 *vmcs12) 4908 { 4909 unsigned long exit_qualification = vmcs_readl(EXIT_QUALIFICATION); 4910 int cr = exit_qualification & 15; 4911 int reg; 4912 unsigned long val; 4913 4914 switch ((exit_qualification >> 4) & 3) { 4915 case 0: /* mov to cr */ 4916 reg = (exit_qualification >> 8) & 15; 4917 val = kvm_register_readl(vcpu, reg); 4918 switch (cr) { 4919 case 0: 4920 if (vmcs12->cr0_guest_host_mask & 4921 (val ^ vmcs12->cr0_read_shadow)) 4922 return true; 4923 break; 4924 case 3: 4925 if ((vmcs12->cr3_target_count >= 1 && 4926 vmcs12->cr3_target_value0 == val) || 4927 (vmcs12->cr3_target_count >= 2 && 4928 vmcs12->cr3_target_value1 == val) || 4929 (vmcs12->cr3_target_count >= 3 && 4930 vmcs12->cr3_target_value2 == val) || 4931 (vmcs12->cr3_target_count >= 4 && 4932 vmcs12->cr3_target_value3 == val)) 4933 return false; 4934 if (nested_cpu_has(vmcs12, CPU_BASED_CR3_LOAD_EXITING)) 4935 return true; 4936 break; 4937 case 4: 4938 if (vmcs12->cr4_guest_host_mask & 4939 (vmcs12->cr4_read_shadow ^ val)) 4940 return true; 4941 break; 4942 case 8: 4943 if (nested_cpu_has(vmcs12, CPU_BASED_CR8_LOAD_EXITING)) 4944 return true; 4945 break; 4946 } 4947 break; 4948 case 2: /* clts */ 4949 if ((vmcs12->cr0_guest_host_mask & X86_CR0_TS) && 4950 (vmcs12->cr0_read_shadow & X86_CR0_TS)) 4951 return true; 4952 break; 4953 case 1: /* mov from cr */ 4954 switch (cr) { 4955 case 3: 4956 if (vmcs12->cpu_based_vm_exec_control & 4957 CPU_BASED_CR3_STORE_EXITING) 4958 return true; 4959 break; 4960 case 8: 4961 if (vmcs12->cpu_based_vm_exec_control & 4962 CPU_BASED_CR8_STORE_EXITING) 4963 return true; 4964 break; 4965 } 4966 break; 4967 case 3: /* lmsw */ 4968 /* 4969 * lmsw can change bits 1..3 of cr0, and only set bit 0 of 4970 * cr0. Other attempted changes are ignored, with no exit. 4971 */ 4972 val = (exit_qualification >> LMSW_SOURCE_DATA_SHIFT) & 0x0f; 4973 if (vmcs12->cr0_guest_host_mask & 0xe & 4974 (val ^ vmcs12->cr0_read_shadow)) 4975 return true; 4976 if ((vmcs12->cr0_guest_host_mask & 0x1) && 4977 !(vmcs12->cr0_read_shadow & 0x1) && 4978 (val & 0x1)) 4979 return true; 4980 break; 4981 } 4982 return false; 4983 } 4984 4985 static bool nested_vmx_exit_handled_vmcs_access(struct kvm_vcpu *vcpu, 4986 struct vmcs12 *vmcs12, gpa_t bitmap) 4987 { 4988 u32 vmx_instruction_info; 4989 unsigned long field; 4990 u8 b; 4991 4992 if (!nested_cpu_has_shadow_vmcs(vmcs12)) 4993 return true; 4994 4995 /* Decode instruction info and find the field to access */ 4996 vmx_instruction_info = vmcs_read32(VMX_INSTRUCTION_INFO); 4997 field = kvm_register_read(vcpu, (((vmx_instruction_info) >> 28) & 0xf)); 4998 4999 /* Out-of-range fields always cause a VM exit from L2 to L1 */ 5000 if (field >> 15) 5001 return true; 5002 5003 if (kvm_vcpu_read_guest(vcpu, bitmap + field/8, &b, 1)) 5004 return true; 5005 5006 return 1 & (b >> (field & 7)); 5007 } 5008 5009 /* 5010 * Return 1 if we should exit from L2 to L1 to handle an exit, or 0 if we 5011 * should handle it ourselves in L0 (and then continue L2). Only call this 5012 * when in is_guest_mode (L2). 5013 */ 5014 bool nested_vmx_exit_reflected(struct kvm_vcpu *vcpu, u32 exit_reason) 5015 { 5016 u32 intr_info = vmcs_read32(VM_EXIT_INTR_INFO); 5017 struct vcpu_vmx *vmx = to_vmx(vcpu); 5018 struct vmcs12 *vmcs12 = get_vmcs12(vcpu); 5019 5020 if (vmx->nested.nested_run_pending) 5021 return false; 5022 5023 if (unlikely(vmx->fail)) { 5024 pr_info_ratelimited("%s failed vm entry %x\n", __func__, 5025 vmcs_read32(VM_INSTRUCTION_ERROR)); 5026 return true; 5027 } 5028 5029 /* 5030 * The host physical addresses of some pages of guest memory 5031 * are loaded into the vmcs02 (e.g. vmcs12's Virtual APIC 5032 * Page). The CPU may write to these pages via their host 5033 * physical address while L2 is running, bypassing any 5034 * address-translation-based dirty tracking (e.g. EPT write 5035 * protection). 5036 * 5037 * Mark them dirty on every exit from L2 to prevent them from 5038 * getting out of sync with dirty tracking. 5039 */ 5040 nested_mark_vmcs12_pages_dirty(vcpu); 5041 5042 trace_kvm_nested_vmexit(kvm_rip_read(vcpu), exit_reason, 5043 vmcs_readl(EXIT_QUALIFICATION), 5044 vmx->idt_vectoring_info, 5045 intr_info, 5046 vmcs_read32(VM_EXIT_INTR_ERROR_CODE), 5047 KVM_ISA_VMX); 5048 5049 switch (exit_reason) { 5050 case EXIT_REASON_EXCEPTION_NMI: 5051 if (is_nmi(intr_info)) 5052 return false; 5053 else if (is_page_fault(intr_info)) 5054 return !vmx->vcpu.arch.apf.host_apf_reason && enable_ept; 5055 else if (is_debug(intr_info) && 5056 vcpu->guest_debug & 5057 (KVM_GUESTDBG_SINGLESTEP | KVM_GUESTDBG_USE_HW_BP)) 5058 return false; 5059 else if (is_breakpoint(intr_info) && 5060 vcpu->guest_debug & KVM_GUESTDBG_USE_SW_BP) 5061 return false; 5062 return vmcs12->exception_bitmap & 5063 (1u << (intr_info & INTR_INFO_VECTOR_MASK)); 5064 case EXIT_REASON_EXTERNAL_INTERRUPT: 5065 return false; 5066 case EXIT_REASON_TRIPLE_FAULT: 5067 return true; 5068 case EXIT_REASON_PENDING_INTERRUPT: 5069 return nested_cpu_has(vmcs12, CPU_BASED_VIRTUAL_INTR_PENDING); 5070 case EXIT_REASON_NMI_WINDOW: 5071 return nested_cpu_has(vmcs12, CPU_BASED_VIRTUAL_NMI_PENDING); 5072 case EXIT_REASON_TASK_SWITCH: 5073 return true; 5074 case EXIT_REASON_CPUID: 5075 return true; 5076 case EXIT_REASON_HLT: 5077 return nested_cpu_has(vmcs12, CPU_BASED_HLT_EXITING); 5078 case EXIT_REASON_INVD: 5079 return true; 5080 case EXIT_REASON_INVLPG: 5081 return nested_cpu_has(vmcs12, CPU_BASED_INVLPG_EXITING); 5082 case EXIT_REASON_RDPMC: 5083 return nested_cpu_has(vmcs12, CPU_BASED_RDPMC_EXITING); 5084 case EXIT_REASON_RDRAND: 5085 return nested_cpu_has2(vmcs12, SECONDARY_EXEC_RDRAND_EXITING); 5086 case EXIT_REASON_RDSEED: 5087 return nested_cpu_has2(vmcs12, SECONDARY_EXEC_RDSEED_EXITING); 5088 case EXIT_REASON_RDTSC: case EXIT_REASON_RDTSCP: 5089 return nested_cpu_has(vmcs12, CPU_BASED_RDTSC_EXITING); 5090 case EXIT_REASON_VMREAD: 5091 return nested_vmx_exit_handled_vmcs_access(vcpu, vmcs12, 5092 vmcs12->vmread_bitmap); 5093 case EXIT_REASON_VMWRITE: 5094 return nested_vmx_exit_handled_vmcs_access(vcpu, vmcs12, 5095 vmcs12->vmwrite_bitmap); 5096 case EXIT_REASON_VMCALL: case EXIT_REASON_VMCLEAR: 5097 case EXIT_REASON_VMLAUNCH: case EXIT_REASON_VMPTRLD: 5098 case EXIT_REASON_VMPTRST: case EXIT_REASON_VMRESUME: 5099 case EXIT_REASON_VMOFF: case EXIT_REASON_VMON: 5100 case EXIT_REASON_INVEPT: case EXIT_REASON_INVVPID: 5101 /* 5102 * VMX instructions trap unconditionally. This allows L1 to 5103 * emulate them for its L2 guest, i.e., allows 3-level nesting! 5104 */ 5105 return true; 5106 case EXIT_REASON_CR_ACCESS: 5107 return nested_vmx_exit_handled_cr(vcpu, vmcs12); 5108 case EXIT_REASON_DR_ACCESS: 5109 return nested_cpu_has(vmcs12, CPU_BASED_MOV_DR_EXITING); 5110 case EXIT_REASON_IO_INSTRUCTION: 5111 return nested_vmx_exit_handled_io(vcpu, vmcs12); 5112 case EXIT_REASON_GDTR_IDTR: case EXIT_REASON_LDTR_TR: 5113 return nested_cpu_has2(vmcs12, SECONDARY_EXEC_DESC); 5114 case EXIT_REASON_MSR_READ: 5115 case EXIT_REASON_MSR_WRITE: 5116 return nested_vmx_exit_handled_msr(vcpu, vmcs12, exit_reason); 5117 case EXIT_REASON_INVALID_STATE: 5118 return true; 5119 case EXIT_REASON_MWAIT_INSTRUCTION: 5120 return nested_cpu_has(vmcs12, CPU_BASED_MWAIT_EXITING); 5121 case EXIT_REASON_MONITOR_TRAP_FLAG: 5122 return nested_cpu_has(vmcs12, CPU_BASED_MONITOR_TRAP_FLAG); 5123 case EXIT_REASON_MONITOR_INSTRUCTION: 5124 return nested_cpu_has(vmcs12, CPU_BASED_MONITOR_EXITING); 5125 case EXIT_REASON_PAUSE_INSTRUCTION: 5126 return nested_cpu_has(vmcs12, CPU_BASED_PAUSE_EXITING) || 5127 nested_cpu_has2(vmcs12, 5128 SECONDARY_EXEC_PAUSE_LOOP_EXITING); 5129 case EXIT_REASON_MCE_DURING_VMENTRY: 5130 return false; 5131 case EXIT_REASON_TPR_BELOW_THRESHOLD: 5132 return nested_cpu_has(vmcs12, CPU_BASED_TPR_SHADOW); 5133 case EXIT_REASON_APIC_ACCESS: 5134 case EXIT_REASON_APIC_WRITE: 5135 case EXIT_REASON_EOI_INDUCED: 5136 /* 5137 * The controls for "virtualize APIC accesses," "APIC- 5138 * register virtualization," and "virtual-interrupt 5139 * delivery" only come from vmcs12. 5140 */ 5141 return true; 5142 case EXIT_REASON_EPT_VIOLATION: 5143 /* 5144 * L0 always deals with the EPT violation. If nested EPT is 5145 * used, and the nested mmu code discovers that the address is 5146 * missing in the guest EPT table (EPT12), the EPT violation 5147 * will be injected with nested_ept_inject_page_fault() 5148 */ 5149 return false; 5150 case EXIT_REASON_EPT_MISCONFIG: 5151 /* 5152 * L2 never uses directly L1's EPT, but rather L0's own EPT 5153 * table (shadow on EPT) or a merged EPT table that L0 built 5154 * (EPT on EPT). So any problems with the structure of the 5155 * table is L0's fault. 5156 */ 5157 return false; 5158 case EXIT_REASON_INVPCID: 5159 return 5160 nested_cpu_has2(vmcs12, SECONDARY_EXEC_ENABLE_INVPCID) && 5161 nested_cpu_has(vmcs12, CPU_BASED_INVLPG_EXITING); 5162 case EXIT_REASON_WBINVD: 5163 return nested_cpu_has2(vmcs12, SECONDARY_EXEC_WBINVD_EXITING); 5164 case EXIT_REASON_XSETBV: 5165 return true; 5166 case EXIT_REASON_XSAVES: case EXIT_REASON_XRSTORS: 5167 /* 5168 * This should never happen, since it is not possible to 5169 * set XSS to a non-zero value---neither in L1 nor in L2. 5170 * If if it were, XSS would have to be checked against 5171 * the XSS exit bitmap in vmcs12. 5172 */ 5173 return nested_cpu_has2(vmcs12, SECONDARY_EXEC_XSAVES); 5174 case EXIT_REASON_PREEMPTION_TIMER: 5175 return false; 5176 case EXIT_REASON_PML_FULL: 5177 /* We emulate PML support to L1. */ 5178 return false; 5179 case EXIT_REASON_VMFUNC: 5180 /* VM functions are emulated through L2->L0 vmexits. */ 5181 return false; 5182 case EXIT_REASON_ENCLS: 5183 /* SGX is never exposed to L1 */ 5184 return false; 5185 default: 5186 return true; 5187 } 5188 } 5189 5190 5191 static int vmx_get_nested_state(struct kvm_vcpu *vcpu, 5192 struct kvm_nested_state __user *user_kvm_nested_state, 5193 u32 user_data_size) 5194 { 5195 struct vcpu_vmx *vmx; 5196 struct vmcs12 *vmcs12; 5197 struct kvm_nested_state kvm_state = { 5198 .flags = 0, 5199 .format = 0, 5200 .size = sizeof(kvm_state), 5201 .vmx.vmxon_pa = -1ull, 5202 .vmx.vmcs_pa = -1ull, 5203 }; 5204 5205 if (!vcpu) 5206 return kvm_state.size + 2 * VMCS12_SIZE; 5207 5208 vmx = to_vmx(vcpu); 5209 vmcs12 = get_vmcs12(vcpu); 5210 5211 if (nested_vmx_allowed(vcpu) && vmx->nested.enlightened_vmcs_enabled) 5212 kvm_state.flags |= KVM_STATE_NESTED_EVMCS; 5213 5214 if (nested_vmx_allowed(vcpu) && 5215 (vmx->nested.vmxon || vmx->nested.smm.vmxon)) { 5216 kvm_state.vmx.vmxon_pa = vmx->nested.vmxon_ptr; 5217 kvm_state.vmx.vmcs_pa = vmx->nested.current_vmptr; 5218 5219 if (vmx_has_valid_vmcs12(vcpu)) { 5220 kvm_state.size += VMCS12_SIZE; 5221 5222 if (is_guest_mode(vcpu) && 5223 nested_cpu_has_shadow_vmcs(vmcs12) && 5224 vmcs12->vmcs_link_pointer != -1ull) 5225 kvm_state.size += VMCS12_SIZE; 5226 } 5227 5228 if (vmx->nested.smm.vmxon) 5229 kvm_state.vmx.smm.flags |= KVM_STATE_NESTED_SMM_VMXON; 5230 5231 if (vmx->nested.smm.guest_mode) 5232 kvm_state.vmx.smm.flags |= KVM_STATE_NESTED_SMM_GUEST_MODE; 5233 5234 if (is_guest_mode(vcpu)) { 5235 kvm_state.flags |= KVM_STATE_NESTED_GUEST_MODE; 5236 5237 if (vmx->nested.nested_run_pending) 5238 kvm_state.flags |= KVM_STATE_NESTED_RUN_PENDING; 5239 } 5240 } 5241 5242 if (user_data_size < kvm_state.size) 5243 goto out; 5244 5245 if (copy_to_user(user_kvm_nested_state, &kvm_state, sizeof(kvm_state))) 5246 return -EFAULT; 5247 5248 if (!vmx_has_valid_vmcs12(vcpu)) 5249 goto out; 5250 5251 /* 5252 * When running L2, the authoritative vmcs12 state is in the 5253 * vmcs02. When running L1, the authoritative vmcs12 state is 5254 * in the shadow or enlightened vmcs linked to vmcs01, unless 5255 * need_vmcs12_sync is set, in which case, the authoritative 5256 * vmcs12 state is in the vmcs12 already. 5257 */ 5258 if (is_guest_mode(vcpu)) { 5259 sync_vmcs12(vcpu, vmcs12); 5260 } else if (!vmx->nested.need_vmcs12_sync) { 5261 if (vmx->nested.hv_evmcs) 5262 copy_enlightened_to_vmcs12(vmx); 5263 else if (enable_shadow_vmcs) 5264 copy_shadow_to_vmcs12(vmx); 5265 } 5266 5267 /* 5268 * Copy over the full allocated size of vmcs12 rather than just the size 5269 * of the struct. 5270 */ 5271 if (copy_to_user(user_kvm_nested_state->data, vmcs12, VMCS12_SIZE)) 5272 return -EFAULT; 5273 5274 if (nested_cpu_has_shadow_vmcs(vmcs12) && 5275 vmcs12->vmcs_link_pointer != -1ull) { 5276 if (copy_to_user(user_kvm_nested_state->data + VMCS12_SIZE, 5277 get_shadow_vmcs12(vcpu), VMCS12_SIZE)) 5278 return -EFAULT; 5279 } 5280 5281 out: 5282 return kvm_state.size; 5283 } 5284 5285 /* 5286 * Forcibly leave nested mode in order to be able to reset the VCPU later on. 5287 */ 5288 void vmx_leave_nested(struct kvm_vcpu *vcpu) 5289 { 5290 if (is_guest_mode(vcpu)) { 5291 to_vmx(vcpu)->nested.nested_run_pending = 0; 5292 nested_vmx_vmexit(vcpu, -1, 0, 0); 5293 } 5294 free_nested(vcpu); 5295 } 5296 5297 static int vmx_set_nested_state(struct kvm_vcpu *vcpu, 5298 struct kvm_nested_state __user *user_kvm_nested_state, 5299 struct kvm_nested_state *kvm_state) 5300 { 5301 struct vcpu_vmx *vmx = to_vmx(vcpu); 5302 struct vmcs12 *vmcs12; 5303 u32 exit_qual; 5304 int ret; 5305 5306 if (kvm_state->format != 0) 5307 return -EINVAL; 5308 5309 if (kvm_state->flags & KVM_STATE_NESTED_EVMCS) 5310 nested_enable_evmcs(vcpu, NULL); 5311 5312 if (!nested_vmx_allowed(vcpu)) 5313 return kvm_state->vmx.vmxon_pa == -1ull ? 0 : -EINVAL; 5314 5315 if (kvm_state->vmx.vmxon_pa == -1ull) { 5316 if (kvm_state->vmx.smm.flags) 5317 return -EINVAL; 5318 5319 if (kvm_state->vmx.vmcs_pa != -1ull) 5320 return -EINVAL; 5321 5322 vmx_leave_nested(vcpu); 5323 return 0; 5324 } 5325 5326 if (!page_address_valid(vcpu, kvm_state->vmx.vmxon_pa)) 5327 return -EINVAL; 5328 5329 if ((kvm_state->vmx.smm.flags & KVM_STATE_NESTED_SMM_GUEST_MODE) && 5330 (kvm_state->flags & KVM_STATE_NESTED_GUEST_MODE)) 5331 return -EINVAL; 5332 5333 if (kvm_state->vmx.smm.flags & 5334 ~(KVM_STATE_NESTED_SMM_GUEST_MODE | KVM_STATE_NESTED_SMM_VMXON)) 5335 return -EINVAL; 5336 5337 /* 5338 * SMM temporarily disables VMX, so we cannot be in guest mode, 5339 * nor can VMLAUNCH/VMRESUME be pending. Outside SMM, SMM flags 5340 * must be zero. 5341 */ 5342 if (is_smm(vcpu) ? kvm_state->flags : kvm_state->vmx.smm.flags) 5343 return -EINVAL; 5344 5345 if ((kvm_state->vmx.smm.flags & KVM_STATE_NESTED_SMM_GUEST_MODE) && 5346 !(kvm_state->vmx.smm.flags & KVM_STATE_NESTED_SMM_VMXON)) 5347 return -EINVAL; 5348 5349 vmx_leave_nested(vcpu); 5350 if (kvm_state->vmx.vmxon_pa == -1ull) 5351 return 0; 5352 5353 vmx->nested.vmxon_ptr = kvm_state->vmx.vmxon_pa; 5354 ret = enter_vmx_operation(vcpu); 5355 if (ret) 5356 return ret; 5357 5358 /* Empty 'VMXON' state is permitted */ 5359 if (kvm_state->size < sizeof(kvm_state) + sizeof(*vmcs12)) 5360 return 0; 5361 5362 if (kvm_state->vmx.vmcs_pa != -1ull) { 5363 if (kvm_state->vmx.vmcs_pa == kvm_state->vmx.vmxon_pa || 5364 !page_address_valid(vcpu, kvm_state->vmx.vmcs_pa)) 5365 return -EINVAL; 5366 5367 set_current_vmptr(vmx, kvm_state->vmx.vmcs_pa); 5368 } else if (kvm_state->flags & KVM_STATE_NESTED_EVMCS) { 5369 /* 5370 * Sync eVMCS upon entry as we may not have 5371 * HV_X64_MSR_VP_ASSIST_PAGE set up yet. 5372 */ 5373 vmx->nested.need_vmcs12_sync = true; 5374 } else { 5375 return -EINVAL; 5376 } 5377 5378 if (kvm_state->vmx.smm.flags & KVM_STATE_NESTED_SMM_VMXON) { 5379 vmx->nested.smm.vmxon = true; 5380 vmx->nested.vmxon = false; 5381 5382 if (kvm_state->vmx.smm.flags & KVM_STATE_NESTED_SMM_GUEST_MODE) 5383 vmx->nested.smm.guest_mode = true; 5384 } 5385 5386 vmcs12 = get_vmcs12(vcpu); 5387 if (copy_from_user(vmcs12, user_kvm_nested_state->data, sizeof(*vmcs12))) 5388 return -EFAULT; 5389 5390 if (vmcs12->hdr.revision_id != VMCS12_REVISION) 5391 return -EINVAL; 5392 5393 if (!(kvm_state->flags & KVM_STATE_NESTED_GUEST_MODE)) 5394 return 0; 5395 5396 vmx->nested.nested_run_pending = 5397 !!(kvm_state->flags & KVM_STATE_NESTED_RUN_PENDING); 5398 5399 if (nested_cpu_has_shadow_vmcs(vmcs12) && 5400 vmcs12->vmcs_link_pointer != -1ull) { 5401 struct vmcs12 *shadow_vmcs12 = get_shadow_vmcs12(vcpu); 5402 5403 if (kvm_state->size < sizeof(kvm_state) + 2 * sizeof(*vmcs12)) 5404 return -EINVAL; 5405 5406 if (copy_from_user(shadow_vmcs12, 5407 user_kvm_nested_state->data + VMCS12_SIZE, 5408 sizeof(*vmcs12))) 5409 return -EFAULT; 5410 5411 if (shadow_vmcs12->hdr.revision_id != VMCS12_REVISION || 5412 !shadow_vmcs12->hdr.shadow_vmcs) 5413 return -EINVAL; 5414 } 5415 5416 if (nested_vmx_check_vmentry_prereqs(vcpu, vmcs12) || 5417 nested_vmx_check_vmentry_postreqs(vcpu, vmcs12, &exit_qual)) 5418 return -EINVAL; 5419 5420 vmx->nested.dirty_vmcs12 = true; 5421 ret = nested_vmx_enter_non_root_mode(vcpu, false); 5422 if (ret) 5423 return -EINVAL; 5424 5425 return 0; 5426 } 5427 5428 void nested_vmx_vcpu_setup(void) 5429 { 5430 if (enable_shadow_vmcs) { 5431 /* 5432 * At vCPU creation, "VMWRITE to any supported field 5433 * in the VMCS" is supported, so use the more 5434 * permissive vmx_vmread_bitmap to specify both read 5435 * and write permissions for the shadow VMCS. 5436 */ 5437 vmcs_write64(VMREAD_BITMAP, __pa(vmx_vmread_bitmap)); 5438 vmcs_write64(VMWRITE_BITMAP, __pa(vmx_vmread_bitmap)); 5439 } 5440 } 5441 5442 /* 5443 * nested_vmx_setup_ctls_msrs() sets up variables containing the values to be 5444 * returned for the various VMX controls MSRs when nested VMX is enabled. 5445 * The same values should also be used to verify that vmcs12 control fields are 5446 * valid during nested entry from L1 to L2. 5447 * Each of these control msrs has a low and high 32-bit half: A low bit is on 5448 * if the corresponding bit in the (32-bit) control field *must* be on, and a 5449 * bit in the high half is on if the corresponding bit in the control field 5450 * may be on. See also vmx_control_verify(). 5451 */ 5452 void nested_vmx_setup_ctls_msrs(struct nested_vmx_msrs *msrs, u32 ept_caps, 5453 bool apicv) 5454 { 5455 /* 5456 * Note that as a general rule, the high half of the MSRs (bits in 5457 * the control fields which may be 1) should be initialized by the 5458 * intersection of the underlying hardware's MSR (i.e., features which 5459 * can be supported) and the list of features we want to expose - 5460 * because they are known to be properly supported in our code. 5461 * Also, usually, the low half of the MSRs (bits which must be 1) can 5462 * be set to 0, meaning that L1 may turn off any of these bits. The 5463 * reason is that if one of these bits is necessary, it will appear 5464 * in vmcs01 and prepare_vmcs02, when it bitwise-or's the control 5465 * fields of vmcs01 and vmcs02, will turn these bits off - and 5466 * nested_vmx_exit_reflected() will not pass related exits to L1. 5467 * These rules have exceptions below. 5468 */ 5469 5470 /* pin-based controls */ 5471 rdmsr(MSR_IA32_VMX_PINBASED_CTLS, 5472 msrs->pinbased_ctls_low, 5473 msrs->pinbased_ctls_high); 5474 msrs->pinbased_ctls_low |= 5475 PIN_BASED_ALWAYSON_WITHOUT_TRUE_MSR; 5476 msrs->pinbased_ctls_high &= 5477 PIN_BASED_EXT_INTR_MASK | 5478 PIN_BASED_NMI_EXITING | 5479 PIN_BASED_VIRTUAL_NMIS | 5480 (apicv ? PIN_BASED_POSTED_INTR : 0); 5481 msrs->pinbased_ctls_high |= 5482 PIN_BASED_ALWAYSON_WITHOUT_TRUE_MSR | 5483 PIN_BASED_VMX_PREEMPTION_TIMER; 5484 5485 /* exit controls */ 5486 rdmsr(MSR_IA32_VMX_EXIT_CTLS, 5487 msrs->exit_ctls_low, 5488 msrs->exit_ctls_high); 5489 msrs->exit_ctls_low = 5490 VM_EXIT_ALWAYSON_WITHOUT_TRUE_MSR; 5491 5492 msrs->exit_ctls_high &= 5493 #ifdef CONFIG_X86_64 5494 VM_EXIT_HOST_ADDR_SPACE_SIZE | 5495 #endif 5496 VM_EXIT_LOAD_IA32_PAT | VM_EXIT_SAVE_IA32_PAT; 5497 msrs->exit_ctls_high |= 5498 VM_EXIT_ALWAYSON_WITHOUT_TRUE_MSR | 5499 VM_EXIT_LOAD_IA32_EFER | VM_EXIT_SAVE_IA32_EFER | 5500 VM_EXIT_SAVE_VMX_PREEMPTION_TIMER | VM_EXIT_ACK_INTR_ON_EXIT; 5501 5502 /* We support free control of debug control saving. */ 5503 msrs->exit_ctls_low &= ~VM_EXIT_SAVE_DEBUG_CONTROLS; 5504 5505 /* entry controls */ 5506 rdmsr(MSR_IA32_VMX_ENTRY_CTLS, 5507 msrs->entry_ctls_low, 5508 msrs->entry_ctls_high); 5509 msrs->entry_ctls_low = 5510 VM_ENTRY_ALWAYSON_WITHOUT_TRUE_MSR; 5511 msrs->entry_ctls_high &= 5512 #ifdef CONFIG_X86_64 5513 VM_ENTRY_IA32E_MODE | 5514 #endif 5515 VM_ENTRY_LOAD_IA32_PAT; 5516 msrs->entry_ctls_high |= 5517 (VM_ENTRY_ALWAYSON_WITHOUT_TRUE_MSR | VM_ENTRY_LOAD_IA32_EFER); 5518 5519 /* We support free control of debug control loading. */ 5520 msrs->entry_ctls_low &= ~VM_ENTRY_LOAD_DEBUG_CONTROLS; 5521 5522 /* cpu-based controls */ 5523 rdmsr(MSR_IA32_VMX_PROCBASED_CTLS, 5524 msrs->procbased_ctls_low, 5525 msrs->procbased_ctls_high); 5526 msrs->procbased_ctls_low = 5527 CPU_BASED_ALWAYSON_WITHOUT_TRUE_MSR; 5528 msrs->procbased_ctls_high &= 5529 CPU_BASED_VIRTUAL_INTR_PENDING | 5530 CPU_BASED_VIRTUAL_NMI_PENDING | CPU_BASED_USE_TSC_OFFSETING | 5531 CPU_BASED_HLT_EXITING | CPU_BASED_INVLPG_EXITING | 5532 CPU_BASED_MWAIT_EXITING | CPU_BASED_CR3_LOAD_EXITING | 5533 CPU_BASED_CR3_STORE_EXITING | 5534 #ifdef CONFIG_X86_64 5535 CPU_BASED_CR8_LOAD_EXITING | CPU_BASED_CR8_STORE_EXITING | 5536 #endif 5537 CPU_BASED_MOV_DR_EXITING | CPU_BASED_UNCOND_IO_EXITING | 5538 CPU_BASED_USE_IO_BITMAPS | CPU_BASED_MONITOR_TRAP_FLAG | 5539 CPU_BASED_MONITOR_EXITING | CPU_BASED_RDPMC_EXITING | 5540 CPU_BASED_RDTSC_EXITING | CPU_BASED_PAUSE_EXITING | 5541 CPU_BASED_TPR_SHADOW | CPU_BASED_ACTIVATE_SECONDARY_CONTROLS; 5542 /* 5543 * We can allow some features even when not supported by the 5544 * hardware. For example, L1 can specify an MSR bitmap - and we 5545 * can use it to avoid exits to L1 - even when L0 runs L2 5546 * without MSR bitmaps. 5547 */ 5548 msrs->procbased_ctls_high |= 5549 CPU_BASED_ALWAYSON_WITHOUT_TRUE_MSR | 5550 CPU_BASED_USE_MSR_BITMAPS; 5551 5552 /* We support free control of CR3 access interception. */ 5553 msrs->procbased_ctls_low &= 5554 ~(CPU_BASED_CR3_LOAD_EXITING | CPU_BASED_CR3_STORE_EXITING); 5555 5556 /* 5557 * secondary cpu-based controls. Do not include those that 5558 * depend on CPUID bits, they are added later by vmx_cpuid_update. 5559 */ 5560 rdmsr(MSR_IA32_VMX_PROCBASED_CTLS2, 5561 msrs->secondary_ctls_low, 5562 msrs->secondary_ctls_high); 5563 msrs->secondary_ctls_low = 0; 5564 msrs->secondary_ctls_high &= 5565 SECONDARY_EXEC_DESC | 5566 SECONDARY_EXEC_VIRTUALIZE_X2APIC_MODE | 5567 SECONDARY_EXEC_APIC_REGISTER_VIRT | 5568 SECONDARY_EXEC_VIRTUAL_INTR_DELIVERY | 5569 SECONDARY_EXEC_WBINVD_EXITING; 5570 5571 /* 5572 * We can emulate "VMCS shadowing," even if the hardware 5573 * doesn't support it. 5574 */ 5575 msrs->secondary_ctls_high |= 5576 SECONDARY_EXEC_SHADOW_VMCS; 5577 5578 if (enable_ept) { 5579 /* nested EPT: emulate EPT also to L1 */ 5580 msrs->secondary_ctls_high |= 5581 SECONDARY_EXEC_ENABLE_EPT; 5582 msrs->ept_caps = VMX_EPT_PAGE_WALK_4_BIT | 5583 VMX_EPTP_WB_BIT | VMX_EPT_INVEPT_BIT; 5584 if (cpu_has_vmx_ept_execute_only()) 5585 msrs->ept_caps |= 5586 VMX_EPT_EXECUTE_ONLY_BIT; 5587 msrs->ept_caps &= ept_caps; 5588 msrs->ept_caps |= VMX_EPT_EXTENT_GLOBAL_BIT | 5589 VMX_EPT_EXTENT_CONTEXT_BIT | VMX_EPT_2MB_PAGE_BIT | 5590 VMX_EPT_1GB_PAGE_BIT; 5591 if (enable_ept_ad_bits) { 5592 msrs->secondary_ctls_high |= 5593 SECONDARY_EXEC_ENABLE_PML; 5594 msrs->ept_caps |= VMX_EPT_AD_BIT; 5595 } 5596 } 5597 5598 if (cpu_has_vmx_vmfunc()) { 5599 msrs->secondary_ctls_high |= 5600 SECONDARY_EXEC_ENABLE_VMFUNC; 5601 /* 5602 * Advertise EPTP switching unconditionally 5603 * since we emulate it 5604 */ 5605 if (enable_ept) 5606 msrs->vmfunc_controls = 5607 VMX_VMFUNC_EPTP_SWITCHING; 5608 } 5609 5610 /* 5611 * Old versions of KVM use the single-context version without 5612 * checking for support, so declare that it is supported even 5613 * though it is treated as global context. The alternative is 5614 * not failing the single-context invvpid, and it is worse. 5615 */ 5616 if (enable_vpid) { 5617 msrs->secondary_ctls_high |= 5618 SECONDARY_EXEC_ENABLE_VPID; 5619 msrs->vpid_caps = VMX_VPID_INVVPID_BIT | 5620 VMX_VPID_EXTENT_SUPPORTED_MASK; 5621 } 5622 5623 if (enable_unrestricted_guest) 5624 msrs->secondary_ctls_high |= 5625 SECONDARY_EXEC_UNRESTRICTED_GUEST; 5626 5627 if (flexpriority_enabled) 5628 msrs->secondary_ctls_high |= 5629 SECONDARY_EXEC_VIRTUALIZE_APIC_ACCESSES; 5630 5631 /* miscellaneous data */ 5632 rdmsr(MSR_IA32_VMX_MISC, 5633 msrs->misc_low, 5634 msrs->misc_high); 5635 msrs->misc_low &= VMX_MISC_SAVE_EFER_LMA; 5636 msrs->misc_low |= 5637 MSR_IA32_VMX_MISC_VMWRITE_SHADOW_RO_FIELDS | 5638 VMX_MISC_EMULATED_PREEMPTION_TIMER_RATE | 5639 VMX_MISC_ACTIVITY_HLT; 5640 msrs->misc_high = 0; 5641 5642 /* 5643 * This MSR reports some information about VMX support. We 5644 * should return information about the VMX we emulate for the 5645 * guest, and the VMCS structure we give it - not about the 5646 * VMX support of the underlying hardware. 5647 */ 5648 msrs->basic = 5649 VMCS12_REVISION | 5650 VMX_BASIC_TRUE_CTLS | 5651 ((u64)VMCS12_SIZE << VMX_BASIC_VMCS_SIZE_SHIFT) | 5652 (VMX_BASIC_MEM_TYPE_WB << VMX_BASIC_MEM_TYPE_SHIFT); 5653 5654 if (cpu_has_vmx_basic_inout()) 5655 msrs->basic |= VMX_BASIC_INOUT; 5656 5657 /* 5658 * These MSRs specify bits which the guest must keep fixed on 5659 * while L1 is in VMXON mode (in L1's root mode, or running an L2). 5660 * We picked the standard core2 setting. 5661 */ 5662 #define VMXON_CR0_ALWAYSON (X86_CR0_PE | X86_CR0_PG | X86_CR0_NE) 5663 #define VMXON_CR4_ALWAYSON X86_CR4_VMXE 5664 msrs->cr0_fixed0 = VMXON_CR0_ALWAYSON; 5665 msrs->cr4_fixed0 = VMXON_CR4_ALWAYSON; 5666 5667 /* These MSRs specify bits which the guest must keep fixed off. */ 5668 rdmsrl(MSR_IA32_VMX_CR0_FIXED1, msrs->cr0_fixed1); 5669 rdmsrl(MSR_IA32_VMX_CR4_FIXED1, msrs->cr4_fixed1); 5670 5671 /* highest index: VMX_PREEMPTION_TIMER_VALUE */ 5672 msrs->vmcs_enum = VMCS12_MAX_FIELD_INDEX << 1; 5673 } 5674 5675 void nested_vmx_hardware_unsetup(void) 5676 { 5677 int i; 5678 5679 if (enable_shadow_vmcs) { 5680 for (i = 0; i < VMX_BITMAP_NR; i++) 5681 free_page((unsigned long)vmx_bitmap[i]); 5682 } 5683 } 5684 5685 __init int nested_vmx_hardware_setup(int (*exit_handlers[])(struct kvm_vcpu *)) 5686 { 5687 int i; 5688 5689 if (!cpu_has_vmx_shadow_vmcs()) 5690 enable_shadow_vmcs = 0; 5691 if (enable_shadow_vmcs) { 5692 for (i = 0; i < VMX_BITMAP_NR; i++) { 5693 vmx_bitmap[i] = (unsigned long *) 5694 __get_free_page(GFP_KERNEL); 5695 if (!vmx_bitmap[i]) { 5696 nested_vmx_hardware_unsetup(); 5697 return -ENOMEM; 5698 } 5699 } 5700 5701 init_vmcs_shadow_fields(); 5702 } 5703 5704 exit_handlers[EXIT_REASON_VMCLEAR] = handle_vmclear, 5705 exit_handlers[EXIT_REASON_VMLAUNCH] = handle_vmlaunch, 5706 exit_handlers[EXIT_REASON_VMPTRLD] = handle_vmptrld, 5707 exit_handlers[EXIT_REASON_VMPTRST] = handle_vmptrst, 5708 exit_handlers[EXIT_REASON_VMREAD] = handle_vmread, 5709 exit_handlers[EXIT_REASON_VMRESUME] = handle_vmresume, 5710 exit_handlers[EXIT_REASON_VMWRITE] = handle_vmwrite, 5711 exit_handlers[EXIT_REASON_VMOFF] = handle_vmoff, 5712 exit_handlers[EXIT_REASON_VMON] = handle_vmon, 5713 exit_handlers[EXIT_REASON_INVEPT] = handle_invept, 5714 exit_handlers[EXIT_REASON_INVVPID] = handle_invvpid, 5715 exit_handlers[EXIT_REASON_VMFUNC] = handle_vmfunc, 5716 5717 kvm_x86_ops->check_nested_events = vmx_check_nested_events; 5718 kvm_x86_ops->get_nested_state = vmx_get_nested_state; 5719 kvm_x86_ops->set_nested_state = vmx_set_nested_state; 5720 kvm_x86_ops->get_vmcs12_pages = nested_get_vmcs12_pages, 5721 kvm_x86_ops->nested_enable_evmcs = nested_enable_evmcs; 5722 kvm_x86_ops->nested_get_evmcs_version = nested_get_evmcs_version; 5723 5724 return 0; 5725 } 5726