1 // SPDX-License-Identifier: GPL-2.0 2 3 #include <linux/frame.h> 4 #include <linux/percpu.h> 5 6 #include <asm/debugreg.h> 7 #include <asm/mmu_context.h> 8 9 #include "cpuid.h" 10 #include "hyperv.h" 11 #include "mmu.h" 12 #include "nested.h" 13 #include "trace.h" 14 #include "x86.h" 15 16 static bool __read_mostly enable_shadow_vmcs = 1; 17 module_param_named(enable_shadow_vmcs, enable_shadow_vmcs, bool, S_IRUGO); 18 19 static bool __read_mostly nested_early_check = 0; 20 module_param(nested_early_check, bool, S_IRUGO); 21 22 /* 23 * Hyper-V requires all of these, so mark them as supported even though 24 * they are just treated the same as all-context. 25 */ 26 #define VMX_VPID_EXTENT_SUPPORTED_MASK \ 27 (VMX_VPID_EXTENT_INDIVIDUAL_ADDR_BIT | \ 28 VMX_VPID_EXTENT_SINGLE_CONTEXT_BIT | \ 29 VMX_VPID_EXTENT_GLOBAL_CONTEXT_BIT | \ 30 VMX_VPID_EXTENT_SINGLE_NON_GLOBAL_BIT) 31 32 #define VMX_MISC_EMULATED_PREEMPTION_TIMER_RATE 5 33 34 enum { 35 VMX_VMREAD_BITMAP, 36 VMX_VMWRITE_BITMAP, 37 VMX_BITMAP_NR 38 }; 39 static unsigned long *vmx_bitmap[VMX_BITMAP_NR]; 40 41 #define vmx_vmread_bitmap (vmx_bitmap[VMX_VMREAD_BITMAP]) 42 #define vmx_vmwrite_bitmap (vmx_bitmap[VMX_VMWRITE_BITMAP]) 43 44 static u16 shadow_read_only_fields[] = { 45 #define SHADOW_FIELD_RO(x) x, 46 #include "vmcs_shadow_fields.h" 47 }; 48 static int max_shadow_read_only_fields = 49 ARRAY_SIZE(shadow_read_only_fields); 50 51 static u16 shadow_read_write_fields[] = { 52 #define SHADOW_FIELD_RW(x) x, 53 #include "vmcs_shadow_fields.h" 54 }; 55 static int max_shadow_read_write_fields = 56 ARRAY_SIZE(shadow_read_write_fields); 57 58 static void init_vmcs_shadow_fields(void) 59 { 60 int i, j; 61 62 memset(vmx_vmread_bitmap, 0xff, PAGE_SIZE); 63 memset(vmx_vmwrite_bitmap, 0xff, PAGE_SIZE); 64 65 for (i = j = 0; i < max_shadow_read_only_fields; i++) { 66 u16 field = shadow_read_only_fields[i]; 67 68 if (vmcs_field_width(field) == VMCS_FIELD_WIDTH_U64 && 69 (i + 1 == max_shadow_read_only_fields || 70 shadow_read_only_fields[i + 1] != field + 1)) 71 pr_err("Missing field from shadow_read_only_field %x\n", 72 field + 1); 73 74 clear_bit(field, vmx_vmread_bitmap); 75 #ifdef CONFIG_X86_64 76 if (field & 1) 77 continue; 78 #endif 79 if (j < i) 80 shadow_read_only_fields[j] = field; 81 j++; 82 } 83 max_shadow_read_only_fields = j; 84 85 for (i = j = 0; i < max_shadow_read_write_fields; i++) { 86 u16 field = shadow_read_write_fields[i]; 87 88 if (vmcs_field_width(field) == VMCS_FIELD_WIDTH_U64 && 89 (i + 1 == max_shadow_read_write_fields || 90 shadow_read_write_fields[i + 1] != field + 1)) 91 pr_err("Missing field from shadow_read_write_field %x\n", 92 field + 1); 93 94 /* 95 * PML and the preemption timer can be emulated, but the 96 * processor cannot vmwrite to fields that don't exist 97 * on bare metal. 98 */ 99 switch (field) { 100 case GUEST_PML_INDEX: 101 if (!cpu_has_vmx_pml()) 102 continue; 103 break; 104 case VMX_PREEMPTION_TIMER_VALUE: 105 if (!cpu_has_vmx_preemption_timer()) 106 continue; 107 break; 108 case GUEST_INTR_STATUS: 109 if (!cpu_has_vmx_apicv()) 110 continue; 111 break; 112 default: 113 break; 114 } 115 116 clear_bit(field, vmx_vmwrite_bitmap); 117 clear_bit(field, vmx_vmread_bitmap); 118 #ifdef CONFIG_X86_64 119 if (field & 1) 120 continue; 121 #endif 122 if (j < i) 123 shadow_read_write_fields[j] = field; 124 j++; 125 } 126 max_shadow_read_write_fields = j; 127 } 128 129 /* 130 * The following 3 functions, nested_vmx_succeed()/failValid()/failInvalid(), 131 * set the success or error code of an emulated VMX instruction (as specified 132 * by Vol 2B, VMX Instruction Reference, "Conventions"), and skip the emulated 133 * instruction. 134 */ 135 static int nested_vmx_succeed(struct kvm_vcpu *vcpu) 136 { 137 vmx_set_rflags(vcpu, vmx_get_rflags(vcpu) 138 & ~(X86_EFLAGS_CF | X86_EFLAGS_PF | X86_EFLAGS_AF | 139 X86_EFLAGS_ZF | X86_EFLAGS_SF | X86_EFLAGS_OF)); 140 return kvm_skip_emulated_instruction(vcpu); 141 } 142 143 static int nested_vmx_failInvalid(struct kvm_vcpu *vcpu) 144 { 145 vmx_set_rflags(vcpu, (vmx_get_rflags(vcpu) 146 & ~(X86_EFLAGS_PF | X86_EFLAGS_AF | X86_EFLAGS_ZF | 147 X86_EFLAGS_SF | X86_EFLAGS_OF)) 148 | X86_EFLAGS_CF); 149 return kvm_skip_emulated_instruction(vcpu); 150 } 151 152 static int nested_vmx_failValid(struct kvm_vcpu *vcpu, 153 u32 vm_instruction_error) 154 { 155 struct vcpu_vmx *vmx = to_vmx(vcpu); 156 157 /* 158 * failValid writes the error number to the current VMCS, which 159 * can't be done if there isn't a current VMCS. 160 */ 161 if (vmx->nested.current_vmptr == -1ull && !vmx->nested.hv_evmcs) 162 return nested_vmx_failInvalid(vcpu); 163 164 vmx_set_rflags(vcpu, (vmx_get_rflags(vcpu) 165 & ~(X86_EFLAGS_CF | X86_EFLAGS_PF | X86_EFLAGS_AF | 166 X86_EFLAGS_SF | X86_EFLAGS_OF)) 167 | X86_EFLAGS_ZF); 168 get_vmcs12(vcpu)->vm_instruction_error = vm_instruction_error; 169 /* 170 * We don't need to force a shadow sync because 171 * VM_INSTRUCTION_ERROR is not shadowed 172 */ 173 return kvm_skip_emulated_instruction(vcpu); 174 } 175 176 static void nested_vmx_abort(struct kvm_vcpu *vcpu, u32 indicator) 177 { 178 /* TODO: not to reset guest simply here. */ 179 kvm_make_request(KVM_REQ_TRIPLE_FAULT, vcpu); 180 pr_debug_ratelimited("kvm: nested vmx abort, indicator %d\n", indicator); 181 } 182 183 static void vmx_disable_shadow_vmcs(struct vcpu_vmx *vmx) 184 { 185 vmcs_clear_bits(SECONDARY_VM_EXEC_CONTROL, SECONDARY_EXEC_SHADOW_VMCS); 186 vmcs_write64(VMCS_LINK_POINTER, -1ull); 187 } 188 189 static inline void nested_release_evmcs(struct kvm_vcpu *vcpu) 190 { 191 struct vcpu_vmx *vmx = to_vmx(vcpu); 192 193 if (!vmx->nested.hv_evmcs) 194 return; 195 196 kunmap(vmx->nested.hv_evmcs_page); 197 kvm_release_page_dirty(vmx->nested.hv_evmcs_page); 198 vmx->nested.hv_evmcs_vmptr = -1ull; 199 vmx->nested.hv_evmcs_page = NULL; 200 vmx->nested.hv_evmcs = NULL; 201 } 202 203 /* 204 * Free whatever needs to be freed from vmx->nested when L1 goes down, or 205 * just stops using VMX. 206 */ 207 static void free_nested(struct kvm_vcpu *vcpu) 208 { 209 struct vcpu_vmx *vmx = to_vmx(vcpu); 210 211 if (!vmx->nested.vmxon && !vmx->nested.smm.vmxon) 212 return; 213 214 hrtimer_cancel(&vmx->nested.preemption_timer); 215 vmx->nested.vmxon = false; 216 vmx->nested.smm.vmxon = false; 217 free_vpid(vmx->nested.vpid02); 218 vmx->nested.posted_intr_nv = -1; 219 vmx->nested.current_vmptr = -1ull; 220 if (enable_shadow_vmcs) { 221 vmx_disable_shadow_vmcs(vmx); 222 vmcs_clear(vmx->vmcs01.shadow_vmcs); 223 free_vmcs(vmx->vmcs01.shadow_vmcs); 224 vmx->vmcs01.shadow_vmcs = NULL; 225 } 226 kfree(vmx->nested.cached_vmcs12); 227 kfree(vmx->nested.cached_shadow_vmcs12); 228 /* Unpin physical memory we referred to in the vmcs02 */ 229 if (vmx->nested.apic_access_page) { 230 kvm_release_page_dirty(vmx->nested.apic_access_page); 231 vmx->nested.apic_access_page = NULL; 232 } 233 if (vmx->nested.virtual_apic_page) { 234 kvm_release_page_dirty(vmx->nested.virtual_apic_page); 235 vmx->nested.virtual_apic_page = NULL; 236 } 237 if (vmx->nested.pi_desc_page) { 238 kunmap(vmx->nested.pi_desc_page); 239 kvm_release_page_dirty(vmx->nested.pi_desc_page); 240 vmx->nested.pi_desc_page = NULL; 241 vmx->nested.pi_desc = NULL; 242 } 243 244 kvm_mmu_free_roots(vcpu, &vcpu->arch.guest_mmu, KVM_MMU_ROOTS_ALL); 245 246 nested_release_evmcs(vcpu); 247 248 free_loaded_vmcs(&vmx->nested.vmcs02); 249 } 250 251 static void vmx_switch_vmcs(struct kvm_vcpu *vcpu, struct loaded_vmcs *vmcs) 252 { 253 struct vcpu_vmx *vmx = to_vmx(vcpu); 254 int cpu; 255 256 if (vmx->loaded_vmcs == vmcs) 257 return; 258 259 cpu = get_cpu(); 260 vmx_vcpu_put(vcpu); 261 vmx->loaded_vmcs = vmcs; 262 vmx_vcpu_load(vcpu, cpu); 263 put_cpu(); 264 265 vm_entry_controls_reset_shadow(vmx); 266 vm_exit_controls_reset_shadow(vmx); 267 vmx_segment_cache_clear(vmx); 268 } 269 270 /* 271 * Ensure that the current vmcs of the logical processor is the 272 * vmcs01 of the vcpu before calling free_nested(). 273 */ 274 void nested_vmx_free_vcpu(struct kvm_vcpu *vcpu) 275 { 276 vcpu_load(vcpu); 277 vmx_switch_vmcs(vcpu, &to_vmx(vcpu)->vmcs01); 278 free_nested(vcpu); 279 vcpu_put(vcpu); 280 } 281 282 static void nested_ept_inject_page_fault(struct kvm_vcpu *vcpu, 283 struct x86_exception *fault) 284 { 285 struct vmcs12 *vmcs12 = get_vmcs12(vcpu); 286 struct vcpu_vmx *vmx = to_vmx(vcpu); 287 u32 exit_reason; 288 unsigned long exit_qualification = vcpu->arch.exit_qualification; 289 290 if (vmx->nested.pml_full) { 291 exit_reason = EXIT_REASON_PML_FULL; 292 vmx->nested.pml_full = false; 293 exit_qualification &= INTR_INFO_UNBLOCK_NMI; 294 } else if (fault->error_code & PFERR_RSVD_MASK) 295 exit_reason = EXIT_REASON_EPT_MISCONFIG; 296 else 297 exit_reason = EXIT_REASON_EPT_VIOLATION; 298 299 nested_vmx_vmexit(vcpu, exit_reason, 0, exit_qualification); 300 vmcs12->guest_physical_address = fault->address; 301 } 302 303 static void nested_ept_init_mmu_context(struct kvm_vcpu *vcpu) 304 { 305 WARN_ON(mmu_is_nested(vcpu)); 306 307 vcpu->arch.mmu = &vcpu->arch.guest_mmu; 308 kvm_init_shadow_ept_mmu(vcpu, 309 to_vmx(vcpu)->nested.msrs.ept_caps & 310 VMX_EPT_EXECUTE_ONLY_BIT, 311 nested_ept_ad_enabled(vcpu), 312 nested_ept_get_cr3(vcpu)); 313 vcpu->arch.mmu->set_cr3 = vmx_set_cr3; 314 vcpu->arch.mmu->get_cr3 = nested_ept_get_cr3; 315 vcpu->arch.mmu->inject_page_fault = nested_ept_inject_page_fault; 316 vcpu->arch.mmu->get_pdptr = kvm_pdptr_read; 317 318 vcpu->arch.walk_mmu = &vcpu->arch.nested_mmu; 319 } 320 321 static void nested_ept_uninit_mmu_context(struct kvm_vcpu *vcpu) 322 { 323 vcpu->arch.mmu = &vcpu->arch.root_mmu; 324 vcpu->arch.walk_mmu = &vcpu->arch.root_mmu; 325 } 326 327 static bool nested_vmx_is_page_fault_vmexit(struct vmcs12 *vmcs12, 328 u16 error_code) 329 { 330 bool inequality, bit; 331 332 bit = (vmcs12->exception_bitmap & (1u << PF_VECTOR)) != 0; 333 inequality = 334 (error_code & vmcs12->page_fault_error_code_mask) != 335 vmcs12->page_fault_error_code_match; 336 return inequality ^ bit; 337 } 338 339 340 /* 341 * KVM wants to inject page-faults which it got to the guest. This function 342 * checks whether in a nested guest, we need to inject them to L1 or L2. 343 */ 344 static int nested_vmx_check_exception(struct kvm_vcpu *vcpu, unsigned long *exit_qual) 345 { 346 struct vmcs12 *vmcs12 = get_vmcs12(vcpu); 347 unsigned int nr = vcpu->arch.exception.nr; 348 bool has_payload = vcpu->arch.exception.has_payload; 349 unsigned long payload = vcpu->arch.exception.payload; 350 351 if (nr == PF_VECTOR) { 352 if (vcpu->arch.exception.nested_apf) { 353 *exit_qual = vcpu->arch.apf.nested_apf_token; 354 return 1; 355 } 356 if (nested_vmx_is_page_fault_vmexit(vmcs12, 357 vcpu->arch.exception.error_code)) { 358 *exit_qual = has_payload ? payload : vcpu->arch.cr2; 359 return 1; 360 } 361 } else if (vmcs12->exception_bitmap & (1u << nr)) { 362 if (nr == DB_VECTOR) { 363 if (!has_payload) { 364 payload = vcpu->arch.dr6; 365 payload &= ~(DR6_FIXED_1 | DR6_BT); 366 payload ^= DR6_RTM; 367 } 368 *exit_qual = payload; 369 } else 370 *exit_qual = 0; 371 return 1; 372 } 373 374 return 0; 375 } 376 377 378 static void vmx_inject_page_fault_nested(struct kvm_vcpu *vcpu, 379 struct x86_exception *fault) 380 { 381 struct vmcs12 *vmcs12 = get_vmcs12(vcpu); 382 383 WARN_ON(!is_guest_mode(vcpu)); 384 385 if (nested_vmx_is_page_fault_vmexit(vmcs12, fault->error_code) && 386 !to_vmx(vcpu)->nested.nested_run_pending) { 387 vmcs12->vm_exit_intr_error_code = fault->error_code; 388 nested_vmx_vmexit(vcpu, EXIT_REASON_EXCEPTION_NMI, 389 PF_VECTOR | INTR_TYPE_HARD_EXCEPTION | 390 INTR_INFO_DELIVER_CODE_MASK | INTR_INFO_VALID_MASK, 391 fault->address); 392 } else { 393 kvm_inject_page_fault(vcpu, fault); 394 } 395 } 396 397 static bool page_address_valid(struct kvm_vcpu *vcpu, gpa_t gpa) 398 { 399 return PAGE_ALIGNED(gpa) && !(gpa >> cpuid_maxphyaddr(vcpu)); 400 } 401 402 static int nested_vmx_check_io_bitmap_controls(struct kvm_vcpu *vcpu, 403 struct vmcs12 *vmcs12) 404 { 405 if (!nested_cpu_has(vmcs12, CPU_BASED_USE_IO_BITMAPS)) 406 return 0; 407 408 if (!page_address_valid(vcpu, vmcs12->io_bitmap_a) || 409 !page_address_valid(vcpu, vmcs12->io_bitmap_b)) 410 return -EINVAL; 411 412 return 0; 413 } 414 415 static int nested_vmx_check_msr_bitmap_controls(struct kvm_vcpu *vcpu, 416 struct vmcs12 *vmcs12) 417 { 418 if (!nested_cpu_has(vmcs12, CPU_BASED_USE_MSR_BITMAPS)) 419 return 0; 420 421 if (!page_address_valid(vcpu, vmcs12->msr_bitmap)) 422 return -EINVAL; 423 424 return 0; 425 } 426 427 static int nested_vmx_check_tpr_shadow_controls(struct kvm_vcpu *vcpu, 428 struct vmcs12 *vmcs12) 429 { 430 if (!nested_cpu_has(vmcs12, CPU_BASED_TPR_SHADOW)) 431 return 0; 432 433 if (!page_address_valid(vcpu, vmcs12->virtual_apic_page_addr)) 434 return -EINVAL; 435 436 return 0; 437 } 438 439 /* 440 * Check if MSR is intercepted for L01 MSR bitmap. 441 */ 442 static bool msr_write_intercepted_l01(struct kvm_vcpu *vcpu, u32 msr) 443 { 444 unsigned long *msr_bitmap; 445 int f = sizeof(unsigned long); 446 447 if (!cpu_has_vmx_msr_bitmap()) 448 return true; 449 450 msr_bitmap = to_vmx(vcpu)->vmcs01.msr_bitmap; 451 452 if (msr <= 0x1fff) { 453 return !!test_bit(msr, msr_bitmap + 0x800 / f); 454 } else if ((msr >= 0xc0000000) && (msr <= 0xc0001fff)) { 455 msr &= 0x1fff; 456 return !!test_bit(msr, msr_bitmap + 0xc00 / f); 457 } 458 459 return true; 460 } 461 462 /* 463 * If a msr is allowed by L0, we should check whether it is allowed by L1. 464 * The corresponding bit will be cleared unless both of L0 and L1 allow it. 465 */ 466 static void nested_vmx_disable_intercept_for_msr(unsigned long *msr_bitmap_l1, 467 unsigned long *msr_bitmap_nested, 468 u32 msr, int type) 469 { 470 int f = sizeof(unsigned long); 471 472 /* 473 * See Intel PRM Vol. 3, 20.6.9 (MSR-Bitmap Address). Early manuals 474 * have the write-low and read-high bitmap offsets the wrong way round. 475 * We can control MSRs 0x00000000-0x00001fff and 0xc0000000-0xc0001fff. 476 */ 477 if (msr <= 0x1fff) { 478 if (type & MSR_TYPE_R && 479 !test_bit(msr, msr_bitmap_l1 + 0x000 / f)) 480 /* read-low */ 481 __clear_bit(msr, msr_bitmap_nested + 0x000 / f); 482 483 if (type & MSR_TYPE_W && 484 !test_bit(msr, msr_bitmap_l1 + 0x800 / f)) 485 /* write-low */ 486 __clear_bit(msr, msr_bitmap_nested + 0x800 / f); 487 488 } else if ((msr >= 0xc0000000) && (msr <= 0xc0001fff)) { 489 msr &= 0x1fff; 490 if (type & MSR_TYPE_R && 491 !test_bit(msr, msr_bitmap_l1 + 0x400 / f)) 492 /* read-high */ 493 __clear_bit(msr, msr_bitmap_nested + 0x400 / f); 494 495 if (type & MSR_TYPE_W && 496 !test_bit(msr, msr_bitmap_l1 + 0xc00 / f)) 497 /* write-high */ 498 __clear_bit(msr, msr_bitmap_nested + 0xc00 / f); 499 500 } 501 } 502 503 /* 504 * Merge L0's and L1's MSR bitmap, return false to indicate that 505 * we do not use the hardware. 506 */ 507 static inline bool nested_vmx_prepare_msr_bitmap(struct kvm_vcpu *vcpu, 508 struct vmcs12 *vmcs12) 509 { 510 int msr; 511 struct page *page; 512 unsigned long *msr_bitmap_l1; 513 unsigned long *msr_bitmap_l0 = to_vmx(vcpu)->nested.vmcs02.msr_bitmap; 514 /* 515 * pred_cmd & spec_ctrl are trying to verify two things: 516 * 517 * 1. L0 gave a permission to L1 to actually passthrough the MSR. This 518 * ensures that we do not accidentally generate an L02 MSR bitmap 519 * from the L12 MSR bitmap that is too permissive. 520 * 2. That L1 or L2s have actually used the MSR. This avoids 521 * unnecessarily merging of the bitmap if the MSR is unused. This 522 * works properly because we only update the L01 MSR bitmap lazily. 523 * So even if L0 should pass L1 these MSRs, the L01 bitmap is only 524 * updated to reflect this when L1 (or its L2s) actually write to 525 * the MSR. 526 */ 527 bool pred_cmd = !msr_write_intercepted_l01(vcpu, MSR_IA32_PRED_CMD); 528 bool spec_ctrl = !msr_write_intercepted_l01(vcpu, MSR_IA32_SPEC_CTRL); 529 530 /* Nothing to do if the MSR bitmap is not in use. */ 531 if (!cpu_has_vmx_msr_bitmap() || 532 !nested_cpu_has(vmcs12, CPU_BASED_USE_MSR_BITMAPS)) 533 return false; 534 535 if (!nested_cpu_has_virt_x2apic_mode(vmcs12) && 536 !pred_cmd && !spec_ctrl) 537 return false; 538 539 page = kvm_vcpu_gpa_to_page(vcpu, vmcs12->msr_bitmap); 540 if (is_error_page(page)) 541 return false; 542 543 msr_bitmap_l1 = (unsigned long *)kmap(page); 544 if (nested_cpu_has_apic_reg_virt(vmcs12)) { 545 /* 546 * L0 need not intercept reads for MSRs between 0x800 and 0x8ff, it 547 * just lets the processor take the value from the virtual-APIC page; 548 * take those 256 bits directly from the L1 bitmap. 549 */ 550 for (msr = 0x800; msr <= 0x8ff; msr += BITS_PER_LONG) { 551 unsigned word = msr / BITS_PER_LONG; 552 msr_bitmap_l0[word] = msr_bitmap_l1[word]; 553 msr_bitmap_l0[word + (0x800 / sizeof(long))] = ~0; 554 } 555 } else { 556 for (msr = 0x800; msr <= 0x8ff; msr += BITS_PER_LONG) { 557 unsigned word = msr / BITS_PER_LONG; 558 msr_bitmap_l0[word] = ~0; 559 msr_bitmap_l0[word + (0x800 / sizeof(long))] = ~0; 560 } 561 } 562 563 nested_vmx_disable_intercept_for_msr( 564 msr_bitmap_l1, msr_bitmap_l0, 565 X2APIC_MSR(APIC_TASKPRI), 566 MSR_TYPE_W); 567 568 if (nested_cpu_has_vid(vmcs12)) { 569 nested_vmx_disable_intercept_for_msr( 570 msr_bitmap_l1, msr_bitmap_l0, 571 X2APIC_MSR(APIC_EOI), 572 MSR_TYPE_W); 573 nested_vmx_disable_intercept_for_msr( 574 msr_bitmap_l1, msr_bitmap_l0, 575 X2APIC_MSR(APIC_SELF_IPI), 576 MSR_TYPE_W); 577 } 578 579 if (spec_ctrl) 580 nested_vmx_disable_intercept_for_msr( 581 msr_bitmap_l1, msr_bitmap_l0, 582 MSR_IA32_SPEC_CTRL, 583 MSR_TYPE_R | MSR_TYPE_W); 584 585 if (pred_cmd) 586 nested_vmx_disable_intercept_for_msr( 587 msr_bitmap_l1, msr_bitmap_l0, 588 MSR_IA32_PRED_CMD, 589 MSR_TYPE_W); 590 591 kunmap(page); 592 kvm_release_page_clean(page); 593 594 return true; 595 } 596 597 static void nested_cache_shadow_vmcs12(struct kvm_vcpu *vcpu, 598 struct vmcs12 *vmcs12) 599 { 600 struct vmcs12 *shadow; 601 struct page *page; 602 603 if (!nested_cpu_has_shadow_vmcs(vmcs12) || 604 vmcs12->vmcs_link_pointer == -1ull) 605 return; 606 607 shadow = get_shadow_vmcs12(vcpu); 608 page = kvm_vcpu_gpa_to_page(vcpu, vmcs12->vmcs_link_pointer); 609 610 memcpy(shadow, kmap(page), VMCS12_SIZE); 611 612 kunmap(page); 613 kvm_release_page_clean(page); 614 } 615 616 static void nested_flush_cached_shadow_vmcs12(struct kvm_vcpu *vcpu, 617 struct vmcs12 *vmcs12) 618 { 619 struct vcpu_vmx *vmx = to_vmx(vcpu); 620 621 if (!nested_cpu_has_shadow_vmcs(vmcs12) || 622 vmcs12->vmcs_link_pointer == -1ull) 623 return; 624 625 kvm_write_guest(vmx->vcpu.kvm, vmcs12->vmcs_link_pointer, 626 get_shadow_vmcs12(vcpu), VMCS12_SIZE); 627 } 628 629 /* 630 * In nested virtualization, check if L1 has set 631 * VM_EXIT_ACK_INTR_ON_EXIT 632 */ 633 static bool nested_exit_intr_ack_set(struct kvm_vcpu *vcpu) 634 { 635 return get_vmcs12(vcpu)->vm_exit_controls & 636 VM_EXIT_ACK_INTR_ON_EXIT; 637 } 638 639 static bool nested_exit_on_nmi(struct kvm_vcpu *vcpu) 640 { 641 return nested_cpu_has_nmi_exiting(get_vmcs12(vcpu)); 642 } 643 644 static int nested_vmx_check_apic_access_controls(struct kvm_vcpu *vcpu, 645 struct vmcs12 *vmcs12) 646 { 647 if (nested_cpu_has2(vmcs12, SECONDARY_EXEC_VIRTUALIZE_APIC_ACCESSES) && 648 !page_address_valid(vcpu, vmcs12->apic_access_addr)) 649 return -EINVAL; 650 else 651 return 0; 652 } 653 654 static int nested_vmx_check_apicv_controls(struct kvm_vcpu *vcpu, 655 struct vmcs12 *vmcs12) 656 { 657 if (!nested_cpu_has_virt_x2apic_mode(vmcs12) && 658 !nested_cpu_has_apic_reg_virt(vmcs12) && 659 !nested_cpu_has_vid(vmcs12) && 660 !nested_cpu_has_posted_intr(vmcs12)) 661 return 0; 662 663 /* 664 * If virtualize x2apic mode is enabled, 665 * virtualize apic access must be disabled. 666 */ 667 if (nested_cpu_has_virt_x2apic_mode(vmcs12) && 668 nested_cpu_has2(vmcs12, SECONDARY_EXEC_VIRTUALIZE_APIC_ACCESSES)) 669 return -EINVAL; 670 671 /* 672 * If virtual interrupt delivery is enabled, 673 * we must exit on external interrupts. 674 */ 675 if (nested_cpu_has_vid(vmcs12) && 676 !nested_exit_on_intr(vcpu)) 677 return -EINVAL; 678 679 /* 680 * bits 15:8 should be zero in posted_intr_nv, 681 * the descriptor address has been already checked 682 * in nested_get_vmcs12_pages. 683 * 684 * bits 5:0 of posted_intr_desc_addr should be zero. 685 */ 686 if (nested_cpu_has_posted_intr(vmcs12) && 687 (!nested_cpu_has_vid(vmcs12) || 688 !nested_exit_intr_ack_set(vcpu) || 689 (vmcs12->posted_intr_nv & 0xff00) || 690 (vmcs12->posted_intr_desc_addr & 0x3f) || 691 (vmcs12->posted_intr_desc_addr >> cpuid_maxphyaddr(vcpu)))) 692 return -EINVAL; 693 694 /* tpr shadow is needed by all apicv features. */ 695 if (!nested_cpu_has(vmcs12, CPU_BASED_TPR_SHADOW)) 696 return -EINVAL; 697 698 return 0; 699 } 700 701 static int nested_vmx_check_msr_switch(struct kvm_vcpu *vcpu, 702 u32 count, u64 addr) 703 { 704 int maxphyaddr; 705 706 if (count == 0) 707 return 0; 708 maxphyaddr = cpuid_maxphyaddr(vcpu); 709 if (!IS_ALIGNED(addr, 16) || addr >> maxphyaddr || 710 (addr + count * sizeof(struct vmx_msr_entry) - 1) >> maxphyaddr) 711 return -EINVAL; 712 713 return 0; 714 } 715 716 static int nested_vmx_check_exit_msr_switch_controls(struct kvm_vcpu *vcpu, 717 struct vmcs12 *vmcs12) 718 { 719 if (nested_vmx_check_msr_switch(vcpu, vmcs12->vm_exit_msr_load_count, 720 vmcs12->vm_exit_msr_load_addr) || 721 nested_vmx_check_msr_switch(vcpu, vmcs12->vm_exit_msr_store_count, 722 vmcs12->vm_exit_msr_store_addr)) 723 return -EINVAL; 724 725 return 0; 726 } 727 728 static int nested_vmx_check_entry_msr_switch_controls(struct kvm_vcpu *vcpu, 729 struct vmcs12 *vmcs12) 730 { 731 if (nested_vmx_check_msr_switch(vcpu, vmcs12->vm_entry_msr_load_count, 732 vmcs12->vm_entry_msr_load_addr)) 733 return -EINVAL; 734 735 return 0; 736 } 737 738 static int nested_vmx_check_pml_controls(struct kvm_vcpu *vcpu, 739 struct vmcs12 *vmcs12) 740 { 741 if (!nested_cpu_has_pml(vmcs12)) 742 return 0; 743 744 if (!nested_cpu_has_ept(vmcs12) || 745 !page_address_valid(vcpu, vmcs12->pml_address)) 746 return -EINVAL; 747 748 return 0; 749 } 750 751 static int nested_vmx_check_unrestricted_guest_controls(struct kvm_vcpu *vcpu, 752 struct vmcs12 *vmcs12) 753 { 754 if (nested_cpu_has2(vmcs12, SECONDARY_EXEC_UNRESTRICTED_GUEST) && 755 !nested_cpu_has_ept(vmcs12)) 756 return -EINVAL; 757 return 0; 758 } 759 760 static int nested_vmx_check_mode_based_ept_exec_controls(struct kvm_vcpu *vcpu, 761 struct vmcs12 *vmcs12) 762 { 763 if (nested_cpu_has2(vmcs12, SECONDARY_EXEC_MODE_BASED_EPT_EXEC) && 764 !nested_cpu_has_ept(vmcs12)) 765 return -EINVAL; 766 return 0; 767 } 768 769 static int nested_vmx_check_shadow_vmcs_controls(struct kvm_vcpu *vcpu, 770 struct vmcs12 *vmcs12) 771 { 772 if (!nested_cpu_has_shadow_vmcs(vmcs12)) 773 return 0; 774 775 if (!page_address_valid(vcpu, vmcs12->vmread_bitmap) || 776 !page_address_valid(vcpu, vmcs12->vmwrite_bitmap)) 777 return -EINVAL; 778 779 return 0; 780 } 781 782 static int nested_vmx_msr_check_common(struct kvm_vcpu *vcpu, 783 struct vmx_msr_entry *e) 784 { 785 /* x2APIC MSR accesses are not allowed */ 786 if (vcpu->arch.apic_base & X2APIC_ENABLE && e->index >> 8 == 0x8) 787 return -EINVAL; 788 if (e->index == MSR_IA32_UCODE_WRITE || /* SDM Table 35-2 */ 789 e->index == MSR_IA32_UCODE_REV) 790 return -EINVAL; 791 if (e->reserved != 0) 792 return -EINVAL; 793 return 0; 794 } 795 796 static int nested_vmx_load_msr_check(struct kvm_vcpu *vcpu, 797 struct vmx_msr_entry *e) 798 { 799 if (e->index == MSR_FS_BASE || 800 e->index == MSR_GS_BASE || 801 e->index == MSR_IA32_SMM_MONITOR_CTL || /* SMM is not supported */ 802 nested_vmx_msr_check_common(vcpu, e)) 803 return -EINVAL; 804 return 0; 805 } 806 807 static int nested_vmx_store_msr_check(struct kvm_vcpu *vcpu, 808 struct vmx_msr_entry *e) 809 { 810 if (e->index == MSR_IA32_SMBASE || /* SMM is not supported */ 811 nested_vmx_msr_check_common(vcpu, e)) 812 return -EINVAL; 813 return 0; 814 } 815 816 /* 817 * Load guest's/host's msr at nested entry/exit. 818 * return 0 for success, entry index for failure. 819 */ 820 static u32 nested_vmx_load_msr(struct kvm_vcpu *vcpu, u64 gpa, u32 count) 821 { 822 u32 i; 823 struct vmx_msr_entry e; 824 struct msr_data msr; 825 826 msr.host_initiated = false; 827 for (i = 0; i < count; i++) { 828 if (kvm_vcpu_read_guest(vcpu, gpa + i * sizeof(e), 829 &e, sizeof(e))) { 830 pr_debug_ratelimited( 831 "%s cannot read MSR entry (%u, 0x%08llx)\n", 832 __func__, i, gpa + i * sizeof(e)); 833 goto fail; 834 } 835 if (nested_vmx_load_msr_check(vcpu, &e)) { 836 pr_debug_ratelimited( 837 "%s check failed (%u, 0x%x, 0x%x)\n", 838 __func__, i, e.index, e.reserved); 839 goto fail; 840 } 841 msr.index = e.index; 842 msr.data = e.value; 843 if (kvm_set_msr(vcpu, &msr)) { 844 pr_debug_ratelimited( 845 "%s cannot write MSR (%u, 0x%x, 0x%llx)\n", 846 __func__, i, e.index, e.value); 847 goto fail; 848 } 849 } 850 return 0; 851 fail: 852 return i + 1; 853 } 854 855 static int nested_vmx_store_msr(struct kvm_vcpu *vcpu, u64 gpa, u32 count) 856 { 857 u32 i; 858 struct vmx_msr_entry e; 859 860 for (i = 0; i < count; i++) { 861 struct msr_data msr_info; 862 if (kvm_vcpu_read_guest(vcpu, 863 gpa + i * sizeof(e), 864 &e, 2 * sizeof(u32))) { 865 pr_debug_ratelimited( 866 "%s cannot read MSR entry (%u, 0x%08llx)\n", 867 __func__, i, gpa + i * sizeof(e)); 868 return -EINVAL; 869 } 870 if (nested_vmx_store_msr_check(vcpu, &e)) { 871 pr_debug_ratelimited( 872 "%s check failed (%u, 0x%x, 0x%x)\n", 873 __func__, i, e.index, e.reserved); 874 return -EINVAL; 875 } 876 msr_info.host_initiated = false; 877 msr_info.index = e.index; 878 if (kvm_get_msr(vcpu, &msr_info)) { 879 pr_debug_ratelimited( 880 "%s cannot read MSR (%u, 0x%x)\n", 881 __func__, i, e.index); 882 return -EINVAL; 883 } 884 if (kvm_vcpu_write_guest(vcpu, 885 gpa + i * sizeof(e) + 886 offsetof(struct vmx_msr_entry, value), 887 &msr_info.data, sizeof(msr_info.data))) { 888 pr_debug_ratelimited( 889 "%s cannot write MSR (%u, 0x%x, 0x%llx)\n", 890 __func__, i, e.index, msr_info.data); 891 return -EINVAL; 892 } 893 } 894 return 0; 895 } 896 897 static bool nested_cr3_valid(struct kvm_vcpu *vcpu, unsigned long val) 898 { 899 unsigned long invalid_mask; 900 901 invalid_mask = (~0ULL) << cpuid_maxphyaddr(vcpu); 902 return (val & invalid_mask) == 0; 903 } 904 905 /* 906 * Load guest's/host's cr3 at nested entry/exit. nested_ept is true if we are 907 * emulating VM entry into a guest with EPT enabled. 908 * Returns 0 on success, 1 on failure. Invalid state exit qualification code 909 * is assigned to entry_failure_code on failure. 910 */ 911 static int nested_vmx_load_cr3(struct kvm_vcpu *vcpu, unsigned long cr3, bool nested_ept, 912 u32 *entry_failure_code) 913 { 914 if (cr3 != kvm_read_cr3(vcpu) || (!nested_ept && pdptrs_changed(vcpu))) { 915 if (!nested_cr3_valid(vcpu, cr3)) { 916 *entry_failure_code = ENTRY_FAIL_DEFAULT; 917 return 1; 918 } 919 920 /* 921 * If PAE paging and EPT are both on, CR3 is not used by the CPU and 922 * must not be dereferenced. 923 */ 924 if (!is_long_mode(vcpu) && is_pae(vcpu) && is_paging(vcpu) && 925 !nested_ept) { 926 if (!load_pdptrs(vcpu, vcpu->arch.walk_mmu, cr3)) { 927 *entry_failure_code = ENTRY_FAIL_PDPTE; 928 return 1; 929 } 930 } 931 } 932 933 if (!nested_ept) 934 kvm_mmu_new_cr3(vcpu, cr3, false); 935 936 vcpu->arch.cr3 = cr3; 937 __set_bit(VCPU_EXREG_CR3, (ulong *)&vcpu->arch.regs_avail); 938 939 kvm_init_mmu(vcpu, false); 940 941 return 0; 942 } 943 944 /* 945 * Returns if KVM is able to config CPU to tag TLB entries 946 * populated by L2 differently than TLB entries populated 947 * by L1. 948 * 949 * If L1 uses EPT, then TLB entries are tagged with different EPTP. 950 * 951 * If L1 uses VPID and we allocated a vpid02, TLB entries are tagged 952 * with different VPID (L1 entries are tagged with vmx->vpid 953 * while L2 entries are tagged with vmx->nested.vpid02). 954 */ 955 static bool nested_has_guest_tlb_tag(struct kvm_vcpu *vcpu) 956 { 957 struct vmcs12 *vmcs12 = get_vmcs12(vcpu); 958 959 return nested_cpu_has_ept(vmcs12) || 960 (nested_cpu_has_vpid(vmcs12) && to_vmx(vcpu)->nested.vpid02); 961 } 962 963 static u16 nested_get_vpid02(struct kvm_vcpu *vcpu) 964 { 965 struct vcpu_vmx *vmx = to_vmx(vcpu); 966 967 return vmx->nested.vpid02 ? vmx->nested.vpid02 : vmx->vpid; 968 } 969 970 971 static inline bool vmx_control_verify(u32 control, u32 low, u32 high) 972 { 973 return fixed_bits_valid(control, low, high); 974 } 975 976 static inline u64 vmx_control_msr(u32 low, u32 high) 977 { 978 return low | ((u64)high << 32); 979 } 980 981 static bool is_bitwise_subset(u64 superset, u64 subset, u64 mask) 982 { 983 superset &= mask; 984 subset &= mask; 985 986 return (superset | subset) == superset; 987 } 988 989 static int vmx_restore_vmx_basic(struct vcpu_vmx *vmx, u64 data) 990 { 991 const u64 feature_and_reserved = 992 /* feature (except bit 48; see below) */ 993 BIT_ULL(49) | BIT_ULL(54) | BIT_ULL(55) | 994 /* reserved */ 995 BIT_ULL(31) | GENMASK_ULL(47, 45) | GENMASK_ULL(63, 56); 996 u64 vmx_basic = vmx->nested.msrs.basic; 997 998 if (!is_bitwise_subset(vmx_basic, data, feature_and_reserved)) 999 return -EINVAL; 1000 1001 /* 1002 * KVM does not emulate a version of VMX that constrains physical 1003 * addresses of VMX structures (e.g. VMCS) to 32-bits. 1004 */ 1005 if (data & BIT_ULL(48)) 1006 return -EINVAL; 1007 1008 if (vmx_basic_vmcs_revision_id(vmx_basic) != 1009 vmx_basic_vmcs_revision_id(data)) 1010 return -EINVAL; 1011 1012 if (vmx_basic_vmcs_size(vmx_basic) > vmx_basic_vmcs_size(data)) 1013 return -EINVAL; 1014 1015 vmx->nested.msrs.basic = data; 1016 return 0; 1017 } 1018 1019 static int 1020 vmx_restore_control_msr(struct vcpu_vmx *vmx, u32 msr_index, u64 data) 1021 { 1022 u64 supported; 1023 u32 *lowp, *highp; 1024 1025 switch (msr_index) { 1026 case MSR_IA32_VMX_TRUE_PINBASED_CTLS: 1027 lowp = &vmx->nested.msrs.pinbased_ctls_low; 1028 highp = &vmx->nested.msrs.pinbased_ctls_high; 1029 break; 1030 case MSR_IA32_VMX_TRUE_PROCBASED_CTLS: 1031 lowp = &vmx->nested.msrs.procbased_ctls_low; 1032 highp = &vmx->nested.msrs.procbased_ctls_high; 1033 break; 1034 case MSR_IA32_VMX_TRUE_EXIT_CTLS: 1035 lowp = &vmx->nested.msrs.exit_ctls_low; 1036 highp = &vmx->nested.msrs.exit_ctls_high; 1037 break; 1038 case MSR_IA32_VMX_TRUE_ENTRY_CTLS: 1039 lowp = &vmx->nested.msrs.entry_ctls_low; 1040 highp = &vmx->nested.msrs.entry_ctls_high; 1041 break; 1042 case MSR_IA32_VMX_PROCBASED_CTLS2: 1043 lowp = &vmx->nested.msrs.secondary_ctls_low; 1044 highp = &vmx->nested.msrs.secondary_ctls_high; 1045 break; 1046 default: 1047 BUG(); 1048 } 1049 1050 supported = vmx_control_msr(*lowp, *highp); 1051 1052 /* Check must-be-1 bits are still 1. */ 1053 if (!is_bitwise_subset(data, supported, GENMASK_ULL(31, 0))) 1054 return -EINVAL; 1055 1056 /* Check must-be-0 bits are still 0. */ 1057 if (!is_bitwise_subset(supported, data, GENMASK_ULL(63, 32))) 1058 return -EINVAL; 1059 1060 *lowp = data; 1061 *highp = data >> 32; 1062 return 0; 1063 } 1064 1065 static int vmx_restore_vmx_misc(struct vcpu_vmx *vmx, u64 data) 1066 { 1067 const u64 feature_and_reserved_bits = 1068 /* feature */ 1069 BIT_ULL(5) | GENMASK_ULL(8, 6) | BIT_ULL(14) | BIT_ULL(15) | 1070 BIT_ULL(28) | BIT_ULL(29) | BIT_ULL(30) | 1071 /* reserved */ 1072 GENMASK_ULL(13, 9) | BIT_ULL(31); 1073 u64 vmx_misc; 1074 1075 vmx_misc = vmx_control_msr(vmx->nested.msrs.misc_low, 1076 vmx->nested.msrs.misc_high); 1077 1078 if (!is_bitwise_subset(vmx_misc, data, feature_and_reserved_bits)) 1079 return -EINVAL; 1080 1081 if ((vmx->nested.msrs.pinbased_ctls_high & 1082 PIN_BASED_VMX_PREEMPTION_TIMER) && 1083 vmx_misc_preemption_timer_rate(data) != 1084 vmx_misc_preemption_timer_rate(vmx_misc)) 1085 return -EINVAL; 1086 1087 if (vmx_misc_cr3_count(data) > vmx_misc_cr3_count(vmx_misc)) 1088 return -EINVAL; 1089 1090 if (vmx_misc_max_msr(data) > vmx_misc_max_msr(vmx_misc)) 1091 return -EINVAL; 1092 1093 if (vmx_misc_mseg_revid(data) != vmx_misc_mseg_revid(vmx_misc)) 1094 return -EINVAL; 1095 1096 vmx->nested.msrs.misc_low = data; 1097 vmx->nested.msrs.misc_high = data >> 32; 1098 1099 /* 1100 * If L1 has read-only VM-exit information fields, use the 1101 * less permissive vmx_vmwrite_bitmap to specify write 1102 * permissions for the shadow VMCS. 1103 */ 1104 if (enable_shadow_vmcs && !nested_cpu_has_vmwrite_any_field(&vmx->vcpu)) 1105 vmcs_write64(VMWRITE_BITMAP, __pa(vmx_vmwrite_bitmap)); 1106 1107 return 0; 1108 } 1109 1110 static int vmx_restore_vmx_ept_vpid_cap(struct vcpu_vmx *vmx, u64 data) 1111 { 1112 u64 vmx_ept_vpid_cap; 1113 1114 vmx_ept_vpid_cap = vmx_control_msr(vmx->nested.msrs.ept_caps, 1115 vmx->nested.msrs.vpid_caps); 1116 1117 /* Every bit is either reserved or a feature bit. */ 1118 if (!is_bitwise_subset(vmx_ept_vpid_cap, data, -1ULL)) 1119 return -EINVAL; 1120 1121 vmx->nested.msrs.ept_caps = data; 1122 vmx->nested.msrs.vpid_caps = data >> 32; 1123 return 0; 1124 } 1125 1126 static int vmx_restore_fixed0_msr(struct vcpu_vmx *vmx, u32 msr_index, u64 data) 1127 { 1128 u64 *msr; 1129 1130 switch (msr_index) { 1131 case MSR_IA32_VMX_CR0_FIXED0: 1132 msr = &vmx->nested.msrs.cr0_fixed0; 1133 break; 1134 case MSR_IA32_VMX_CR4_FIXED0: 1135 msr = &vmx->nested.msrs.cr4_fixed0; 1136 break; 1137 default: 1138 BUG(); 1139 } 1140 1141 /* 1142 * 1 bits (which indicates bits which "must-be-1" during VMX operation) 1143 * must be 1 in the restored value. 1144 */ 1145 if (!is_bitwise_subset(data, *msr, -1ULL)) 1146 return -EINVAL; 1147 1148 *msr = data; 1149 return 0; 1150 } 1151 1152 /* 1153 * Called when userspace is restoring VMX MSRs. 1154 * 1155 * Returns 0 on success, non-0 otherwise. 1156 */ 1157 int vmx_set_vmx_msr(struct kvm_vcpu *vcpu, u32 msr_index, u64 data) 1158 { 1159 struct vcpu_vmx *vmx = to_vmx(vcpu); 1160 1161 /* 1162 * Don't allow changes to the VMX capability MSRs while the vCPU 1163 * is in VMX operation. 1164 */ 1165 if (vmx->nested.vmxon) 1166 return -EBUSY; 1167 1168 switch (msr_index) { 1169 case MSR_IA32_VMX_BASIC: 1170 return vmx_restore_vmx_basic(vmx, data); 1171 case MSR_IA32_VMX_PINBASED_CTLS: 1172 case MSR_IA32_VMX_PROCBASED_CTLS: 1173 case MSR_IA32_VMX_EXIT_CTLS: 1174 case MSR_IA32_VMX_ENTRY_CTLS: 1175 /* 1176 * The "non-true" VMX capability MSRs are generated from the 1177 * "true" MSRs, so we do not support restoring them directly. 1178 * 1179 * If userspace wants to emulate VMX_BASIC[55]=0, userspace 1180 * should restore the "true" MSRs with the must-be-1 bits 1181 * set according to the SDM Vol 3. A.2 "RESERVED CONTROLS AND 1182 * DEFAULT SETTINGS". 1183 */ 1184 return -EINVAL; 1185 case MSR_IA32_VMX_TRUE_PINBASED_CTLS: 1186 case MSR_IA32_VMX_TRUE_PROCBASED_CTLS: 1187 case MSR_IA32_VMX_TRUE_EXIT_CTLS: 1188 case MSR_IA32_VMX_TRUE_ENTRY_CTLS: 1189 case MSR_IA32_VMX_PROCBASED_CTLS2: 1190 return vmx_restore_control_msr(vmx, msr_index, data); 1191 case MSR_IA32_VMX_MISC: 1192 return vmx_restore_vmx_misc(vmx, data); 1193 case MSR_IA32_VMX_CR0_FIXED0: 1194 case MSR_IA32_VMX_CR4_FIXED0: 1195 return vmx_restore_fixed0_msr(vmx, msr_index, data); 1196 case MSR_IA32_VMX_CR0_FIXED1: 1197 case MSR_IA32_VMX_CR4_FIXED1: 1198 /* 1199 * These MSRs are generated based on the vCPU's CPUID, so we 1200 * do not support restoring them directly. 1201 */ 1202 return -EINVAL; 1203 case MSR_IA32_VMX_EPT_VPID_CAP: 1204 return vmx_restore_vmx_ept_vpid_cap(vmx, data); 1205 case MSR_IA32_VMX_VMCS_ENUM: 1206 vmx->nested.msrs.vmcs_enum = data; 1207 return 0; 1208 default: 1209 /* 1210 * The rest of the VMX capability MSRs do not support restore. 1211 */ 1212 return -EINVAL; 1213 } 1214 } 1215 1216 /* Returns 0 on success, non-0 otherwise. */ 1217 int vmx_get_vmx_msr(struct nested_vmx_msrs *msrs, u32 msr_index, u64 *pdata) 1218 { 1219 switch (msr_index) { 1220 case MSR_IA32_VMX_BASIC: 1221 *pdata = msrs->basic; 1222 break; 1223 case MSR_IA32_VMX_TRUE_PINBASED_CTLS: 1224 case MSR_IA32_VMX_PINBASED_CTLS: 1225 *pdata = vmx_control_msr( 1226 msrs->pinbased_ctls_low, 1227 msrs->pinbased_ctls_high); 1228 if (msr_index == MSR_IA32_VMX_PINBASED_CTLS) 1229 *pdata |= PIN_BASED_ALWAYSON_WITHOUT_TRUE_MSR; 1230 break; 1231 case MSR_IA32_VMX_TRUE_PROCBASED_CTLS: 1232 case MSR_IA32_VMX_PROCBASED_CTLS: 1233 *pdata = vmx_control_msr( 1234 msrs->procbased_ctls_low, 1235 msrs->procbased_ctls_high); 1236 if (msr_index == MSR_IA32_VMX_PROCBASED_CTLS) 1237 *pdata |= CPU_BASED_ALWAYSON_WITHOUT_TRUE_MSR; 1238 break; 1239 case MSR_IA32_VMX_TRUE_EXIT_CTLS: 1240 case MSR_IA32_VMX_EXIT_CTLS: 1241 *pdata = vmx_control_msr( 1242 msrs->exit_ctls_low, 1243 msrs->exit_ctls_high); 1244 if (msr_index == MSR_IA32_VMX_EXIT_CTLS) 1245 *pdata |= VM_EXIT_ALWAYSON_WITHOUT_TRUE_MSR; 1246 break; 1247 case MSR_IA32_VMX_TRUE_ENTRY_CTLS: 1248 case MSR_IA32_VMX_ENTRY_CTLS: 1249 *pdata = vmx_control_msr( 1250 msrs->entry_ctls_low, 1251 msrs->entry_ctls_high); 1252 if (msr_index == MSR_IA32_VMX_ENTRY_CTLS) 1253 *pdata |= VM_ENTRY_ALWAYSON_WITHOUT_TRUE_MSR; 1254 break; 1255 case MSR_IA32_VMX_MISC: 1256 *pdata = vmx_control_msr( 1257 msrs->misc_low, 1258 msrs->misc_high); 1259 break; 1260 case MSR_IA32_VMX_CR0_FIXED0: 1261 *pdata = msrs->cr0_fixed0; 1262 break; 1263 case MSR_IA32_VMX_CR0_FIXED1: 1264 *pdata = msrs->cr0_fixed1; 1265 break; 1266 case MSR_IA32_VMX_CR4_FIXED0: 1267 *pdata = msrs->cr4_fixed0; 1268 break; 1269 case MSR_IA32_VMX_CR4_FIXED1: 1270 *pdata = msrs->cr4_fixed1; 1271 break; 1272 case MSR_IA32_VMX_VMCS_ENUM: 1273 *pdata = msrs->vmcs_enum; 1274 break; 1275 case MSR_IA32_VMX_PROCBASED_CTLS2: 1276 *pdata = vmx_control_msr( 1277 msrs->secondary_ctls_low, 1278 msrs->secondary_ctls_high); 1279 break; 1280 case MSR_IA32_VMX_EPT_VPID_CAP: 1281 *pdata = msrs->ept_caps | 1282 ((u64)msrs->vpid_caps << 32); 1283 break; 1284 case MSR_IA32_VMX_VMFUNC: 1285 *pdata = msrs->vmfunc_controls; 1286 break; 1287 default: 1288 return 1; 1289 } 1290 1291 return 0; 1292 } 1293 1294 /* 1295 * Copy the writable VMCS shadow fields back to the VMCS12, in case 1296 * they have been modified by the L1 guest. Note that the "read-only" 1297 * VM-exit information fields are actually writable if the vCPU is 1298 * configured to support "VMWRITE to any supported field in the VMCS." 1299 */ 1300 static void copy_shadow_to_vmcs12(struct vcpu_vmx *vmx) 1301 { 1302 const u16 *fields[] = { 1303 shadow_read_write_fields, 1304 shadow_read_only_fields 1305 }; 1306 const int max_fields[] = { 1307 max_shadow_read_write_fields, 1308 max_shadow_read_only_fields 1309 }; 1310 int i, q; 1311 unsigned long field; 1312 u64 field_value; 1313 struct vmcs *shadow_vmcs = vmx->vmcs01.shadow_vmcs; 1314 1315 preempt_disable(); 1316 1317 vmcs_load(shadow_vmcs); 1318 1319 for (q = 0; q < ARRAY_SIZE(fields); q++) { 1320 for (i = 0; i < max_fields[q]; i++) { 1321 field = fields[q][i]; 1322 field_value = __vmcs_readl(field); 1323 vmcs12_write_any(get_vmcs12(&vmx->vcpu), field, field_value); 1324 } 1325 /* 1326 * Skip the VM-exit information fields if they are read-only. 1327 */ 1328 if (!nested_cpu_has_vmwrite_any_field(&vmx->vcpu)) 1329 break; 1330 } 1331 1332 vmcs_clear(shadow_vmcs); 1333 vmcs_load(vmx->loaded_vmcs->vmcs); 1334 1335 preempt_enable(); 1336 } 1337 1338 static void copy_vmcs12_to_shadow(struct vcpu_vmx *vmx) 1339 { 1340 const u16 *fields[] = { 1341 shadow_read_write_fields, 1342 shadow_read_only_fields 1343 }; 1344 const int max_fields[] = { 1345 max_shadow_read_write_fields, 1346 max_shadow_read_only_fields 1347 }; 1348 int i, q; 1349 unsigned long field; 1350 u64 field_value = 0; 1351 struct vmcs *shadow_vmcs = vmx->vmcs01.shadow_vmcs; 1352 1353 vmcs_load(shadow_vmcs); 1354 1355 for (q = 0; q < ARRAY_SIZE(fields); q++) { 1356 for (i = 0; i < max_fields[q]; i++) { 1357 field = fields[q][i]; 1358 vmcs12_read_any(get_vmcs12(&vmx->vcpu), field, &field_value); 1359 __vmcs_writel(field, field_value); 1360 } 1361 } 1362 1363 vmcs_clear(shadow_vmcs); 1364 vmcs_load(vmx->loaded_vmcs->vmcs); 1365 } 1366 1367 static int copy_enlightened_to_vmcs12(struct vcpu_vmx *vmx) 1368 { 1369 struct vmcs12 *vmcs12 = vmx->nested.cached_vmcs12; 1370 struct hv_enlightened_vmcs *evmcs = vmx->nested.hv_evmcs; 1371 1372 /* HV_VMX_ENLIGHTENED_CLEAN_FIELD_NONE */ 1373 vmcs12->tpr_threshold = evmcs->tpr_threshold; 1374 vmcs12->guest_rip = evmcs->guest_rip; 1375 1376 if (unlikely(!(evmcs->hv_clean_fields & 1377 HV_VMX_ENLIGHTENED_CLEAN_FIELD_GUEST_BASIC))) { 1378 vmcs12->guest_rsp = evmcs->guest_rsp; 1379 vmcs12->guest_rflags = evmcs->guest_rflags; 1380 vmcs12->guest_interruptibility_info = 1381 evmcs->guest_interruptibility_info; 1382 } 1383 1384 if (unlikely(!(evmcs->hv_clean_fields & 1385 HV_VMX_ENLIGHTENED_CLEAN_FIELD_CONTROL_PROC))) { 1386 vmcs12->cpu_based_vm_exec_control = 1387 evmcs->cpu_based_vm_exec_control; 1388 } 1389 1390 if (unlikely(!(evmcs->hv_clean_fields & 1391 HV_VMX_ENLIGHTENED_CLEAN_FIELD_CONTROL_PROC))) { 1392 vmcs12->exception_bitmap = evmcs->exception_bitmap; 1393 } 1394 1395 if (unlikely(!(evmcs->hv_clean_fields & 1396 HV_VMX_ENLIGHTENED_CLEAN_FIELD_CONTROL_ENTRY))) { 1397 vmcs12->vm_entry_controls = evmcs->vm_entry_controls; 1398 } 1399 1400 if (unlikely(!(evmcs->hv_clean_fields & 1401 HV_VMX_ENLIGHTENED_CLEAN_FIELD_CONTROL_EVENT))) { 1402 vmcs12->vm_entry_intr_info_field = 1403 evmcs->vm_entry_intr_info_field; 1404 vmcs12->vm_entry_exception_error_code = 1405 evmcs->vm_entry_exception_error_code; 1406 vmcs12->vm_entry_instruction_len = 1407 evmcs->vm_entry_instruction_len; 1408 } 1409 1410 if (unlikely(!(evmcs->hv_clean_fields & 1411 HV_VMX_ENLIGHTENED_CLEAN_FIELD_HOST_GRP1))) { 1412 vmcs12->host_ia32_pat = evmcs->host_ia32_pat; 1413 vmcs12->host_ia32_efer = evmcs->host_ia32_efer; 1414 vmcs12->host_cr0 = evmcs->host_cr0; 1415 vmcs12->host_cr3 = evmcs->host_cr3; 1416 vmcs12->host_cr4 = evmcs->host_cr4; 1417 vmcs12->host_ia32_sysenter_esp = evmcs->host_ia32_sysenter_esp; 1418 vmcs12->host_ia32_sysenter_eip = evmcs->host_ia32_sysenter_eip; 1419 vmcs12->host_rip = evmcs->host_rip; 1420 vmcs12->host_ia32_sysenter_cs = evmcs->host_ia32_sysenter_cs; 1421 vmcs12->host_es_selector = evmcs->host_es_selector; 1422 vmcs12->host_cs_selector = evmcs->host_cs_selector; 1423 vmcs12->host_ss_selector = evmcs->host_ss_selector; 1424 vmcs12->host_ds_selector = evmcs->host_ds_selector; 1425 vmcs12->host_fs_selector = evmcs->host_fs_selector; 1426 vmcs12->host_gs_selector = evmcs->host_gs_selector; 1427 vmcs12->host_tr_selector = evmcs->host_tr_selector; 1428 } 1429 1430 if (unlikely(!(evmcs->hv_clean_fields & 1431 HV_VMX_ENLIGHTENED_CLEAN_FIELD_HOST_GRP1))) { 1432 vmcs12->pin_based_vm_exec_control = 1433 evmcs->pin_based_vm_exec_control; 1434 vmcs12->vm_exit_controls = evmcs->vm_exit_controls; 1435 vmcs12->secondary_vm_exec_control = 1436 evmcs->secondary_vm_exec_control; 1437 } 1438 1439 if (unlikely(!(evmcs->hv_clean_fields & 1440 HV_VMX_ENLIGHTENED_CLEAN_FIELD_IO_BITMAP))) { 1441 vmcs12->io_bitmap_a = evmcs->io_bitmap_a; 1442 vmcs12->io_bitmap_b = evmcs->io_bitmap_b; 1443 } 1444 1445 if (unlikely(!(evmcs->hv_clean_fields & 1446 HV_VMX_ENLIGHTENED_CLEAN_FIELD_MSR_BITMAP))) { 1447 vmcs12->msr_bitmap = evmcs->msr_bitmap; 1448 } 1449 1450 if (unlikely(!(evmcs->hv_clean_fields & 1451 HV_VMX_ENLIGHTENED_CLEAN_FIELD_GUEST_GRP2))) { 1452 vmcs12->guest_es_base = evmcs->guest_es_base; 1453 vmcs12->guest_cs_base = evmcs->guest_cs_base; 1454 vmcs12->guest_ss_base = evmcs->guest_ss_base; 1455 vmcs12->guest_ds_base = evmcs->guest_ds_base; 1456 vmcs12->guest_fs_base = evmcs->guest_fs_base; 1457 vmcs12->guest_gs_base = evmcs->guest_gs_base; 1458 vmcs12->guest_ldtr_base = evmcs->guest_ldtr_base; 1459 vmcs12->guest_tr_base = evmcs->guest_tr_base; 1460 vmcs12->guest_gdtr_base = evmcs->guest_gdtr_base; 1461 vmcs12->guest_idtr_base = evmcs->guest_idtr_base; 1462 vmcs12->guest_es_limit = evmcs->guest_es_limit; 1463 vmcs12->guest_cs_limit = evmcs->guest_cs_limit; 1464 vmcs12->guest_ss_limit = evmcs->guest_ss_limit; 1465 vmcs12->guest_ds_limit = evmcs->guest_ds_limit; 1466 vmcs12->guest_fs_limit = evmcs->guest_fs_limit; 1467 vmcs12->guest_gs_limit = evmcs->guest_gs_limit; 1468 vmcs12->guest_ldtr_limit = evmcs->guest_ldtr_limit; 1469 vmcs12->guest_tr_limit = evmcs->guest_tr_limit; 1470 vmcs12->guest_gdtr_limit = evmcs->guest_gdtr_limit; 1471 vmcs12->guest_idtr_limit = evmcs->guest_idtr_limit; 1472 vmcs12->guest_es_ar_bytes = evmcs->guest_es_ar_bytes; 1473 vmcs12->guest_cs_ar_bytes = evmcs->guest_cs_ar_bytes; 1474 vmcs12->guest_ss_ar_bytes = evmcs->guest_ss_ar_bytes; 1475 vmcs12->guest_ds_ar_bytes = evmcs->guest_ds_ar_bytes; 1476 vmcs12->guest_fs_ar_bytes = evmcs->guest_fs_ar_bytes; 1477 vmcs12->guest_gs_ar_bytes = evmcs->guest_gs_ar_bytes; 1478 vmcs12->guest_ldtr_ar_bytes = evmcs->guest_ldtr_ar_bytes; 1479 vmcs12->guest_tr_ar_bytes = evmcs->guest_tr_ar_bytes; 1480 vmcs12->guest_es_selector = evmcs->guest_es_selector; 1481 vmcs12->guest_cs_selector = evmcs->guest_cs_selector; 1482 vmcs12->guest_ss_selector = evmcs->guest_ss_selector; 1483 vmcs12->guest_ds_selector = evmcs->guest_ds_selector; 1484 vmcs12->guest_fs_selector = evmcs->guest_fs_selector; 1485 vmcs12->guest_gs_selector = evmcs->guest_gs_selector; 1486 vmcs12->guest_ldtr_selector = evmcs->guest_ldtr_selector; 1487 vmcs12->guest_tr_selector = evmcs->guest_tr_selector; 1488 } 1489 1490 if (unlikely(!(evmcs->hv_clean_fields & 1491 HV_VMX_ENLIGHTENED_CLEAN_FIELD_CONTROL_GRP2))) { 1492 vmcs12->tsc_offset = evmcs->tsc_offset; 1493 vmcs12->virtual_apic_page_addr = evmcs->virtual_apic_page_addr; 1494 vmcs12->xss_exit_bitmap = evmcs->xss_exit_bitmap; 1495 } 1496 1497 if (unlikely(!(evmcs->hv_clean_fields & 1498 HV_VMX_ENLIGHTENED_CLEAN_FIELD_CRDR))) { 1499 vmcs12->cr0_guest_host_mask = evmcs->cr0_guest_host_mask; 1500 vmcs12->cr4_guest_host_mask = evmcs->cr4_guest_host_mask; 1501 vmcs12->cr0_read_shadow = evmcs->cr0_read_shadow; 1502 vmcs12->cr4_read_shadow = evmcs->cr4_read_shadow; 1503 vmcs12->guest_cr0 = evmcs->guest_cr0; 1504 vmcs12->guest_cr3 = evmcs->guest_cr3; 1505 vmcs12->guest_cr4 = evmcs->guest_cr4; 1506 vmcs12->guest_dr7 = evmcs->guest_dr7; 1507 } 1508 1509 if (unlikely(!(evmcs->hv_clean_fields & 1510 HV_VMX_ENLIGHTENED_CLEAN_FIELD_HOST_POINTER))) { 1511 vmcs12->host_fs_base = evmcs->host_fs_base; 1512 vmcs12->host_gs_base = evmcs->host_gs_base; 1513 vmcs12->host_tr_base = evmcs->host_tr_base; 1514 vmcs12->host_gdtr_base = evmcs->host_gdtr_base; 1515 vmcs12->host_idtr_base = evmcs->host_idtr_base; 1516 vmcs12->host_rsp = evmcs->host_rsp; 1517 } 1518 1519 if (unlikely(!(evmcs->hv_clean_fields & 1520 HV_VMX_ENLIGHTENED_CLEAN_FIELD_CONTROL_XLAT))) { 1521 vmcs12->ept_pointer = evmcs->ept_pointer; 1522 vmcs12->virtual_processor_id = evmcs->virtual_processor_id; 1523 } 1524 1525 if (unlikely(!(evmcs->hv_clean_fields & 1526 HV_VMX_ENLIGHTENED_CLEAN_FIELD_GUEST_GRP1))) { 1527 vmcs12->vmcs_link_pointer = evmcs->vmcs_link_pointer; 1528 vmcs12->guest_ia32_debugctl = evmcs->guest_ia32_debugctl; 1529 vmcs12->guest_ia32_pat = evmcs->guest_ia32_pat; 1530 vmcs12->guest_ia32_efer = evmcs->guest_ia32_efer; 1531 vmcs12->guest_pdptr0 = evmcs->guest_pdptr0; 1532 vmcs12->guest_pdptr1 = evmcs->guest_pdptr1; 1533 vmcs12->guest_pdptr2 = evmcs->guest_pdptr2; 1534 vmcs12->guest_pdptr3 = evmcs->guest_pdptr3; 1535 vmcs12->guest_pending_dbg_exceptions = 1536 evmcs->guest_pending_dbg_exceptions; 1537 vmcs12->guest_sysenter_esp = evmcs->guest_sysenter_esp; 1538 vmcs12->guest_sysenter_eip = evmcs->guest_sysenter_eip; 1539 vmcs12->guest_bndcfgs = evmcs->guest_bndcfgs; 1540 vmcs12->guest_activity_state = evmcs->guest_activity_state; 1541 vmcs12->guest_sysenter_cs = evmcs->guest_sysenter_cs; 1542 } 1543 1544 /* 1545 * Not used? 1546 * vmcs12->vm_exit_msr_store_addr = evmcs->vm_exit_msr_store_addr; 1547 * vmcs12->vm_exit_msr_load_addr = evmcs->vm_exit_msr_load_addr; 1548 * vmcs12->vm_entry_msr_load_addr = evmcs->vm_entry_msr_load_addr; 1549 * vmcs12->cr3_target_value0 = evmcs->cr3_target_value0; 1550 * vmcs12->cr3_target_value1 = evmcs->cr3_target_value1; 1551 * vmcs12->cr3_target_value2 = evmcs->cr3_target_value2; 1552 * vmcs12->cr3_target_value3 = evmcs->cr3_target_value3; 1553 * vmcs12->page_fault_error_code_mask = 1554 * evmcs->page_fault_error_code_mask; 1555 * vmcs12->page_fault_error_code_match = 1556 * evmcs->page_fault_error_code_match; 1557 * vmcs12->cr3_target_count = evmcs->cr3_target_count; 1558 * vmcs12->vm_exit_msr_store_count = evmcs->vm_exit_msr_store_count; 1559 * vmcs12->vm_exit_msr_load_count = evmcs->vm_exit_msr_load_count; 1560 * vmcs12->vm_entry_msr_load_count = evmcs->vm_entry_msr_load_count; 1561 */ 1562 1563 /* 1564 * Read only fields: 1565 * vmcs12->guest_physical_address = evmcs->guest_physical_address; 1566 * vmcs12->vm_instruction_error = evmcs->vm_instruction_error; 1567 * vmcs12->vm_exit_reason = evmcs->vm_exit_reason; 1568 * vmcs12->vm_exit_intr_info = evmcs->vm_exit_intr_info; 1569 * vmcs12->vm_exit_intr_error_code = evmcs->vm_exit_intr_error_code; 1570 * vmcs12->idt_vectoring_info_field = evmcs->idt_vectoring_info_field; 1571 * vmcs12->idt_vectoring_error_code = evmcs->idt_vectoring_error_code; 1572 * vmcs12->vm_exit_instruction_len = evmcs->vm_exit_instruction_len; 1573 * vmcs12->vmx_instruction_info = evmcs->vmx_instruction_info; 1574 * vmcs12->exit_qualification = evmcs->exit_qualification; 1575 * vmcs12->guest_linear_address = evmcs->guest_linear_address; 1576 * 1577 * Not present in struct vmcs12: 1578 * vmcs12->exit_io_instruction_ecx = evmcs->exit_io_instruction_ecx; 1579 * vmcs12->exit_io_instruction_esi = evmcs->exit_io_instruction_esi; 1580 * vmcs12->exit_io_instruction_edi = evmcs->exit_io_instruction_edi; 1581 * vmcs12->exit_io_instruction_eip = evmcs->exit_io_instruction_eip; 1582 */ 1583 1584 return 0; 1585 } 1586 1587 static int copy_vmcs12_to_enlightened(struct vcpu_vmx *vmx) 1588 { 1589 struct vmcs12 *vmcs12 = vmx->nested.cached_vmcs12; 1590 struct hv_enlightened_vmcs *evmcs = vmx->nested.hv_evmcs; 1591 1592 /* 1593 * Should not be changed by KVM: 1594 * 1595 * evmcs->host_es_selector = vmcs12->host_es_selector; 1596 * evmcs->host_cs_selector = vmcs12->host_cs_selector; 1597 * evmcs->host_ss_selector = vmcs12->host_ss_selector; 1598 * evmcs->host_ds_selector = vmcs12->host_ds_selector; 1599 * evmcs->host_fs_selector = vmcs12->host_fs_selector; 1600 * evmcs->host_gs_selector = vmcs12->host_gs_selector; 1601 * evmcs->host_tr_selector = vmcs12->host_tr_selector; 1602 * evmcs->host_ia32_pat = vmcs12->host_ia32_pat; 1603 * evmcs->host_ia32_efer = vmcs12->host_ia32_efer; 1604 * evmcs->host_cr0 = vmcs12->host_cr0; 1605 * evmcs->host_cr3 = vmcs12->host_cr3; 1606 * evmcs->host_cr4 = vmcs12->host_cr4; 1607 * evmcs->host_ia32_sysenter_esp = vmcs12->host_ia32_sysenter_esp; 1608 * evmcs->host_ia32_sysenter_eip = vmcs12->host_ia32_sysenter_eip; 1609 * evmcs->host_rip = vmcs12->host_rip; 1610 * evmcs->host_ia32_sysenter_cs = vmcs12->host_ia32_sysenter_cs; 1611 * evmcs->host_fs_base = vmcs12->host_fs_base; 1612 * evmcs->host_gs_base = vmcs12->host_gs_base; 1613 * evmcs->host_tr_base = vmcs12->host_tr_base; 1614 * evmcs->host_gdtr_base = vmcs12->host_gdtr_base; 1615 * evmcs->host_idtr_base = vmcs12->host_idtr_base; 1616 * evmcs->host_rsp = vmcs12->host_rsp; 1617 * sync_vmcs12() doesn't read these: 1618 * evmcs->io_bitmap_a = vmcs12->io_bitmap_a; 1619 * evmcs->io_bitmap_b = vmcs12->io_bitmap_b; 1620 * evmcs->msr_bitmap = vmcs12->msr_bitmap; 1621 * evmcs->ept_pointer = vmcs12->ept_pointer; 1622 * evmcs->xss_exit_bitmap = vmcs12->xss_exit_bitmap; 1623 * evmcs->vm_exit_msr_store_addr = vmcs12->vm_exit_msr_store_addr; 1624 * evmcs->vm_exit_msr_load_addr = vmcs12->vm_exit_msr_load_addr; 1625 * evmcs->vm_entry_msr_load_addr = vmcs12->vm_entry_msr_load_addr; 1626 * evmcs->cr3_target_value0 = vmcs12->cr3_target_value0; 1627 * evmcs->cr3_target_value1 = vmcs12->cr3_target_value1; 1628 * evmcs->cr3_target_value2 = vmcs12->cr3_target_value2; 1629 * evmcs->cr3_target_value3 = vmcs12->cr3_target_value3; 1630 * evmcs->tpr_threshold = vmcs12->tpr_threshold; 1631 * evmcs->virtual_processor_id = vmcs12->virtual_processor_id; 1632 * evmcs->exception_bitmap = vmcs12->exception_bitmap; 1633 * evmcs->vmcs_link_pointer = vmcs12->vmcs_link_pointer; 1634 * evmcs->pin_based_vm_exec_control = vmcs12->pin_based_vm_exec_control; 1635 * evmcs->vm_exit_controls = vmcs12->vm_exit_controls; 1636 * evmcs->secondary_vm_exec_control = vmcs12->secondary_vm_exec_control; 1637 * evmcs->page_fault_error_code_mask = 1638 * vmcs12->page_fault_error_code_mask; 1639 * evmcs->page_fault_error_code_match = 1640 * vmcs12->page_fault_error_code_match; 1641 * evmcs->cr3_target_count = vmcs12->cr3_target_count; 1642 * evmcs->virtual_apic_page_addr = vmcs12->virtual_apic_page_addr; 1643 * evmcs->tsc_offset = vmcs12->tsc_offset; 1644 * evmcs->guest_ia32_debugctl = vmcs12->guest_ia32_debugctl; 1645 * evmcs->cr0_guest_host_mask = vmcs12->cr0_guest_host_mask; 1646 * evmcs->cr4_guest_host_mask = vmcs12->cr4_guest_host_mask; 1647 * evmcs->cr0_read_shadow = vmcs12->cr0_read_shadow; 1648 * evmcs->cr4_read_shadow = vmcs12->cr4_read_shadow; 1649 * evmcs->vm_exit_msr_store_count = vmcs12->vm_exit_msr_store_count; 1650 * evmcs->vm_exit_msr_load_count = vmcs12->vm_exit_msr_load_count; 1651 * evmcs->vm_entry_msr_load_count = vmcs12->vm_entry_msr_load_count; 1652 * 1653 * Not present in struct vmcs12: 1654 * evmcs->exit_io_instruction_ecx = vmcs12->exit_io_instruction_ecx; 1655 * evmcs->exit_io_instruction_esi = vmcs12->exit_io_instruction_esi; 1656 * evmcs->exit_io_instruction_edi = vmcs12->exit_io_instruction_edi; 1657 * evmcs->exit_io_instruction_eip = vmcs12->exit_io_instruction_eip; 1658 */ 1659 1660 evmcs->guest_es_selector = vmcs12->guest_es_selector; 1661 evmcs->guest_cs_selector = vmcs12->guest_cs_selector; 1662 evmcs->guest_ss_selector = vmcs12->guest_ss_selector; 1663 evmcs->guest_ds_selector = vmcs12->guest_ds_selector; 1664 evmcs->guest_fs_selector = vmcs12->guest_fs_selector; 1665 evmcs->guest_gs_selector = vmcs12->guest_gs_selector; 1666 evmcs->guest_ldtr_selector = vmcs12->guest_ldtr_selector; 1667 evmcs->guest_tr_selector = vmcs12->guest_tr_selector; 1668 1669 evmcs->guest_es_limit = vmcs12->guest_es_limit; 1670 evmcs->guest_cs_limit = vmcs12->guest_cs_limit; 1671 evmcs->guest_ss_limit = vmcs12->guest_ss_limit; 1672 evmcs->guest_ds_limit = vmcs12->guest_ds_limit; 1673 evmcs->guest_fs_limit = vmcs12->guest_fs_limit; 1674 evmcs->guest_gs_limit = vmcs12->guest_gs_limit; 1675 evmcs->guest_ldtr_limit = vmcs12->guest_ldtr_limit; 1676 evmcs->guest_tr_limit = vmcs12->guest_tr_limit; 1677 evmcs->guest_gdtr_limit = vmcs12->guest_gdtr_limit; 1678 evmcs->guest_idtr_limit = vmcs12->guest_idtr_limit; 1679 1680 evmcs->guest_es_ar_bytes = vmcs12->guest_es_ar_bytes; 1681 evmcs->guest_cs_ar_bytes = vmcs12->guest_cs_ar_bytes; 1682 evmcs->guest_ss_ar_bytes = vmcs12->guest_ss_ar_bytes; 1683 evmcs->guest_ds_ar_bytes = vmcs12->guest_ds_ar_bytes; 1684 evmcs->guest_fs_ar_bytes = vmcs12->guest_fs_ar_bytes; 1685 evmcs->guest_gs_ar_bytes = vmcs12->guest_gs_ar_bytes; 1686 evmcs->guest_ldtr_ar_bytes = vmcs12->guest_ldtr_ar_bytes; 1687 evmcs->guest_tr_ar_bytes = vmcs12->guest_tr_ar_bytes; 1688 1689 evmcs->guest_es_base = vmcs12->guest_es_base; 1690 evmcs->guest_cs_base = vmcs12->guest_cs_base; 1691 evmcs->guest_ss_base = vmcs12->guest_ss_base; 1692 evmcs->guest_ds_base = vmcs12->guest_ds_base; 1693 evmcs->guest_fs_base = vmcs12->guest_fs_base; 1694 evmcs->guest_gs_base = vmcs12->guest_gs_base; 1695 evmcs->guest_ldtr_base = vmcs12->guest_ldtr_base; 1696 evmcs->guest_tr_base = vmcs12->guest_tr_base; 1697 evmcs->guest_gdtr_base = vmcs12->guest_gdtr_base; 1698 evmcs->guest_idtr_base = vmcs12->guest_idtr_base; 1699 1700 evmcs->guest_ia32_pat = vmcs12->guest_ia32_pat; 1701 evmcs->guest_ia32_efer = vmcs12->guest_ia32_efer; 1702 1703 evmcs->guest_pdptr0 = vmcs12->guest_pdptr0; 1704 evmcs->guest_pdptr1 = vmcs12->guest_pdptr1; 1705 evmcs->guest_pdptr2 = vmcs12->guest_pdptr2; 1706 evmcs->guest_pdptr3 = vmcs12->guest_pdptr3; 1707 1708 evmcs->guest_pending_dbg_exceptions = 1709 vmcs12->guest_pending_dbg_exceptions; 1710 evmcs->guest_sysenter_esp = vmcs12->guest_sysenter_esp; 1711 evmcs->guest_sysenter_eip = vmcs12->guest_sysenter_eip; 1712 1713 evmcs->guest_activity_state = vmcs12->guest_activity_state; 1714 evmcs->guest_sysenter_cs = vmcs12->guest_sysenter_cs; 1715 1716 evmcs->guest_cr0 = vmcs12->guest_cr0; 1717 evmcs->guest_cr3 = vmcs12->guest_cr3; 1718 evmcs->guest_cr4 = vmcs12->guest_cr4; 1719 evmcs->guest_dr7 = vmcs12->guest_dr7; 1720 1721 evmcs->guest_physical_address = vmcs12->guest_physical_address; 1722 1723 evmcs->vm_instruction_error = vmcs12->vm_instruction_error; 1724 evmcs->vm_exit_reason = vmcs12->vm_exit_reason; 1725 evmcs->vm_exit_intr_info = vmcs12->vm_exit_intr_info; 1726 evmcs->vm_exit_intr_error_code = vmcs12->vm_exit_intr_error_code; 1727 evmcs->idt_vectoring_info_field = vmcs12->idt_vectoring_info_field; 1728 evmcs->idt_vectoring_error_code = vmcs12->idt_vectoring_error_code; 1729 evmcs->vm_exit_instruction_len = vmcs12->vm_exit_instruction_len; 1730 evmcs->vmx_instruction_info = vmcs12->vmx_instruction_info; 1731 1732 evmcs->exit_qualification = vmcs12->exit_qualification; 1733 1734 evmcs->guest_linear_address = vmcs12->guest_linear_address; 1735 evmcs->guest_rsp = vmcs12->guest_rsp; 1736 evmcs->guest_rflags = vmcs12->guest_rflags; 1737 1738 evmcs->guest_interruptibility_info = 1739 vmcs12->guest_interruptibility_info; 1740 evmcs->cpu_based_vm_exec_control = vmcs12->cpu_based_vm_exec_control; 1741 evmcs->vm_entry_controls = vmcs12->vm_entry_controls; 1742 evmcs->vm_entry_intr_info_field = vmcs12->vm_entry_intr_info_field; 1743 evmcs->vm_entry_exception_error_code = 1744 vmcs12->vm_entry_exception_error_code; 1745 evmcs->vm_entry_instruction_len = vmcs12->vm_entry_instruction_len; 1746 1747 evmcs->guest_rip = vmcs12->guest_rip; 1748 1749 evmcs->guest_bndcfgs = vmcs12->guest_bndcfgs; 1750 1751 return 0; 1752 } 1753 1754 /* 1755 * This is an equivalent of the nested hypervisor executing the vmptrld 1756 * instruction. 1757 */ 1758 static int nested_vmx_handle_enlightened_vmptrld(struct kvm_vcpu *vcpu, 1759 bool from_launch) 1760 { 1761 struct vcpu_vmx *vmx = to_vmx(vcpu); 1762 struct hv_vp_assist_page assist_page; 1763 1764 if (likely(!vmx->nested.enlightened_vmcs_enabled)) 1765 return 1; 1766 1767 if (unlikely(!kvm_hv_get_assist_page(vcpu, &assist_page))) 1768 return 1; 1769 1770 if (unlikely(!assist_page.enlighten_vmentry)) 1771 return 1; 1772 1773 if (unlikely(assist_page.current_nested_vmcs != 1774 vmx->nested.hv_evmcs_vmptr)) { 1775 1776 if (!vmx->nested.hv_evmcs) 1777 vmx->nested.current_vmptr = -1ull; 1778 1779 nested_release_evmcs(vcpu); 1780 1781 vmx->nested.hv_evmcs_page = kvm_vcpu_gpa_to_page( 1782 vcpu, assist_page.current_nested_vmcs); 1783 1784 if (unlikely(is_error_page(vmx->nested.hv_evmcs_page))) 1785 return 0; 1786 1787 vmx->nested.hv_evmcs = kmap(vmx->nested.hv_evmcs_page); 1788 1789 /* 1790 * Currently, KVM only supports eVMCS version 1 1791 * (== KVM_EVMCS_VERSION) and thus we expect guest to set this 1792 * value to first u32 field of eVMCS which should specify eVMCS 1793 * VersionNumber. 1794 * 1795 * Guest should be aware of supported eVMCS versions by host by 1796 * examining CPUID.0x4000000A.EAX[0:15]. Host userspace VMM is 1797 * expected to set this CPUID leaf according to the value 1798 * returned in vmcs_version from nested_enable_evmcs(). 1799 * 1800 * However, it turns out that Microsoft Hyper-V fails to comply 1801 * to their own invented interface: When Hyper-V use eVMCS, it 1802 * just sets first u32 field of eVMCS to revision_id specified 1803 * in MSR_IA32_VMX_BASIC. Instead of used eVMCS version number 1804 * which is one of the supported versions specified in 1805 * CPUID.0x4000000A.EAX[0:15]. 1806 * 1807 * To overcome Hyper-V bug, we accept here either a supported 1808 * eVMCS version or VMCS12 revision_id as valid values for first 1809 * u32 field of eVMCS. 1810 */ 1811 if ((vmx->nested.hv_evmcs->revision_id != KVM_EVMCS_VERSION) && 1812 (vmx->nested.hv_evmcs->revision_id != VMCS12_REVISION)) { 1813 nested_release_evmcs(vcpu); 1814 return 0; 1815 } 1816 1817 vmx->nested.dirty_vmcs12 = true; 1818 /* 1819 * As we keep L2 state for one guest only 'hv_clean_fields' mask 1820 * can't be used when we switch between them. Reset it here for 1821 * simplicity. 1822 */ 1823 vmx->nested.hv_evmcs->hv_clean_fields &= 1824 ~HV_VMX_ENLIGHTENED_CLEAN_FIELD_ALL; 1825 vmx->nested.hv_evmcs_vmptr = assist_page.current_nested_vmcs; 1826 1827 /* 1828 * Unlike normal vmcs12, enlightened vmcs12 is not fully 1829 * reloaded from guest's memory (read only fields, fields not 1830 * present in struct hv_enlightened_vmcs, ...). Make sure there 1831 * are no leftovers. 1832 */ 1833 if (from_launch) { 1834 struct vmcs12 *vmcs12 = get_vmcs12(vcpu); 1835 memset(vmcs12, 0, sizeof(*vmcs12)); 1836 vmcs12->hdr.revision_id = VMCS12_REVISION; 1837 } 1838 1839 } 1840 return 1; 1841 } 1842 1843 void nested_sync_from_vmcs12(struct kvm_vcpu *vcpu) 1844 { 1845 struct vcpu_vmx *vmx = to_vmx(vcpu); 1846 1847 /* 1848 * hv_evmcs may end up being not mapped after migration (when 1849 * L2 was running), map it here to make sure vmcs12 changes are 1850 * properly reflected. 1851 */ 1852 if (vmx->nested.enlightened_vmcs_enabled && !vmx->nested.hv_evmcs) 1853 nested_vmx_handle_enlightened_vmptrld(vcpu, false); 1854 1855 if (vmx->nested.hv_evmcs) { 1856 copy_vmcs12_to_enlightened(vmx); 1857 /* All fields are clean */ 1858 vmx->nested.hv_evmcs->hv_clean_fields |= 1859 HV_VMX_ENLIGHTENED_CLEAN_FIELD_ALL; 1860 } else { 1861 copy_vmcs12_to_shadow(vmx); 1862 } 1863 1864 vmx->nested.need_vmcs12_sync = false; 1865 } 1866 1867 static enum hrtimer_restart vmx_preemption_timer_fn(struct hrtimer *timer) 1868 { 1869 struct vcpu_vmx *vmx = 1870 container_of(timer, struct vcpu_vmx, nested.preemption_timer); 1871 1872 vmx->nested.preemption_timer_expired = true; 1873 kvm_make_request(KVM_REQ_EVENT, &vmx->vcpu); 1874 kvm_vcpu_kick(&vmx->vcpu); 1875 1876 return HRTIMER_NORESTART; 1877 } 1878 1879 static void vmx_start_preemption_timer(struct kvm_vcpu *vcpu) 1880 { 1881 u64 preemption_timeout = get_vmcs12(vcpu)->vmx_preemption_timer_value; 1882 struct vcpu_vmx *vmx = to_vmx(vcpu); 1883 1884 /* 1885 * A timer value of zero is architecturally guaranteed to cause 1886 * a VMExit prior to executing any instructions in the guest. 1887 */ 1888 if (preemption_timeout == 0) { 1889 vmx_preemption_timer_fn(&vmx->nested.preemption_timer); 1890 return; 1891 } 1892 1893 if (vcpu->arch.virtual_tsc_khz == 0) 1894 return; 1895 1896 preemption_timeout <<= VMX_MISC_EMULATED_PREEMPTION_TIMER_RATE; 1897 preemption_timeout *= 1000000; 1898 do_div(preemption_timeout, vcpu->arch.virtual_tsc_khz); 1899 hrtimer_start(&vmx->nested.preemption_timer, 1900 ns_to_ktime(preemption_timeout), HRTIMER_MODE_REL); 1901 } 1902 1903 static u64 nested_vmx_calc_efer(struct vcpu_vmx *vmx, struct vmcs12 *vmcs12) 1904 { 1905 if (vmx->nested.nested_run_pending && 1906 (vmcs12->vm_entry_controls & VM_ENTRY_LOAD_IA32_EFER)) 1907 return vmcs12->guest_ia32_efer; 1908 else if (vmcs12->vm_entry_controls & VM_ENTRY_IA32E_MODE) 1909 return vmx->vcpu.arch.efer | (EFER_LMA | EFER_LME); 1910 else 1911 return vmx->vcpu.arch.efer & ~(EFER_LMA | EFER_LME); 1912 } 1913 1914 static void prepare_vmcs02_constant_state(struct vcpu_vmx *vmx) 1915 { 1916 /* 1917 * If vmcs02 hasn't been initialized, set the constant vmcs02 state 1918 * according to L0's settings (vmcs12 is irrelevant here). Host 1919 * fields that come from L0 and are not constant, e.g. HOST_CR3, 1920 * will be set as needed prior to VMLAUNCH/VMRESUME. 1921 */ 1922 if (vmx->nested.vmcs02_initialized) 1923 return; 1924 vmx->nested.vmcs02_initialized = true; 1925 1926 /* 1927 * We don't care what the EPTP value is we just need to guarantee 1928 * it's valid so we don't get a false positive when doing early 1929 * consistency checks. 1930 */ 1931 if (enable_ept && nested_early_check) 1932 vmcs_write64(EPT_POINTER, construct_eptp(&vmx->vcpu, 0)); 1933 1934 /* All VMFUNCs are currently emulated through L0 vmexits. */ 1935 if (cpu_has_vmx_vmfunc()) 1936 vmcs_write64(VM_FUNCTION_CONTROL, 0); 1937 1938 if (cpu_has_vmx_posted_intr()) 1939 vmcs_write16(POSTED_INTR_NV, POSTED_INTR_NESTED_VECTOR); 1940 1941 if (cpu_has_vmx_msr_bitmap()) 1942 vmcs_write64(MSR_BITMAP, __pa(vmx->nested.vmcs02.msr_bitmap)); 1943 1944 if (enable_pml) 1945 vmcs_write64(PML_ADDRESS, page_to_phys(vmx->pml_pg)); 1946 1947 /* 1948 * Set the MSR load/store lists to match L0's settings. Only the 1949 * addresses are constant (for vmcs02), the counts can change based 1950 * on L2's behavior, e.g. switching to/from long mode. 1951 */ 1952 vmcs_write32(VM_EXIT_MSR_STORE_COUNT, 0); 1953 vmcs_write64(VM_EXIT_MSR_LOAD_ADDR, __pa(vmx->msr_autoload.host.val)); 1954 vmcs_write64(VM_ENTRY_MSR_LOAD_ADDR, __pa(vmx->msr_autoload.guest.val)); 1955 1956 vmx_set_constant_host_state(vmx); 1957 } 1958 1959 static void prepare_vmcs02_early_full(struct vcpu_vmx *vmx, 1960 struct vmcs12 *vmcs12) 1961 { 1962 prepare_vmcs02_constant_state(vmx); 1963 1964 vmcs_write64(VMCS_LINK_POINTER, -1ull); 1965 1966 if (enable_vpid) { 1967 if (nested_cpu_has_vpid(vmcs12) && vmx->nested.vpid02) 1968 vmcs_write16(VIRTUAL_PROCESSOR_ID, vmx->nested.vpid02); 1969 else 1970 vmcs_write16(VIRTUAL_PROCESSOR_ID, vmx->vpid); 1971 } 1972 } 1973 1974 static void prepare_vmcs02_early(struct vcpu_vmx *vmx, struct vmcs12 *vmcs12) 1975 { 1976 u32 exec_control, vmcs12_exec_ctrl; 1977 u64 guest_efer = nested_vmx_calc_efer(vmx, vmcs12); 1978 1979 if (vmx->nested.dirty_vmcs12 || vmx->nested.hv_evmcs) 1980 prepare_vmcs02_early_full(vmx, vmcs12); 1981 1982 /* 1983 * HOST_RSP is normally set correctly in vmx_vcpu_run() just before 1984 * entry, but only if the current (host) sp changed from the value 1985 * we wrote last (vmx->host_rsp). This cache is no longer relevant 1986 * if we switch vmcs, and rather than hold a separate cache per vmcs, 1987 * here we just force the write to happen on entry. host_rsp will 1988 * also be written unconditionally by nested_vmx_check_vmentry_hw() 1989 * if we are doing early consistency checks via hardware. 1990 */ 1991 vmx->host_rsp = 0; 1992 1993 /* 1994 * PIN CONTROLS 1995 */ 1996 exec_control = vmcs12->pin_based_vm_exec_control; 1997 1998 /* Preemption timer setting is computed directly in vmx_vcpu_run. */ 1999 exec_control |= vmcs_config.pin_based_exec_ctrl; 2000 exec_control &= ~PIN_BASED_VMX_PREEMPTION_TIMER; 2001 vmx->loaded_vmcs->hv_timer_armed = false; 2002 2003 /* Posted interrupts setting is only taken from vmcs12. */ 2004 if (nested_cpu_has_posted_intr(vmcs12)) { 2005 vmx->nested.posted_intr_nv = vmcs12->posted_intr_nv; 2006 vmx->nested.pi_pending = false; 2007 } else { 2008 exec_control &= ~PIN_BASED_POSTED_INTR; 2009 } 2010 vmcs_write32(PIN_BASED_VM_EXEC_CONTROL, exec_control); 2011 2012 /* 2013 * EXEC CONTROLS 2014 */ 2015 exec_control = vmx_exec_control(vmx); /* L0's desires */ 2016 exec_control &= ~CPU_BASED_VIRTUAL_INTR_PENDING; 2017 exec_control &= ~CPU_BASED_VIRTUAL_NMI_PENDING; 2018 exec_control &= ~CPU_BASED_TPR_SHADOW; 2019 exec_control |= vmcs12->cpu_based_vm_exec_control; 2020 2021 /* 2022 * Write an illegal value to VIRTUAL_APIC_PAGE_ADDR. Later, if 2023 * nested_get_vmcs12_pages can't fix it up, the illegal value 2024 * will result in a VM entry failure. 2025 */ 2026 if (exec_control & CPU_BASED_TPR_SHADOW) { 2027 vmcs_write64(VIRTUAL_APIC_PAGE_ADDR, -1ull); 2028 vmcs_write32(TPR_THRESHOLD, vmcs12->tpr_threshold); 2029 } else { 2030 #ifdef CONFIG_X86_64 2031 exec_control |= CPU_BASED_CR8_LOAD_EXITING | 2032 CPU_BASED_CR8_STORE_EXITING; 2033 #endif 2034 } 2035 2036 /* 2037 * A vmexit (to either L1 hypervisor or L0 userspace) is always needed 2038 * for I/O port accesses. 2039 */ 2040 exec_control &= ~CPU_BASED_USE_IO_BITMAPS; 2041 exec_control |= CPU_BASED_UNCOND_IO_EXITING; 2042 vmcs_write32(CPU_BASED_VM_EXEC_CONTROL, exec_control); 2043 2044 /* 2045 * SECONDARY EXEC CONTROLS 2046 */ 2047 if (cpu_has_secondary_exec_ctrls()) { 2048 exec_control = vmx->secondary_exec_control; 2049 2050 /* Take the following fields only from vmcs12 */ 2051 exec_control &= ~(SECONDARY_EXEC_VIRTUALIZE_APIC_ACCESSES | 2052 SECONDARY_EXEC_ENABLE_INVPCID | 2053 SECONDARY_EXEC_RDTSCP | 2054 SECONDARY_EXEC_XSAVES | 2055 SECONDARY_EXEC_VIRTUAL_INTR_DELIVERY | 2056 SECONDARY_EXEC_APIC_REGISTER_VIRT | 2057 SECONDARY_EXEC_ENABLE_VMFUNC); 2058 if (nested_cpu_has(vmcs12, 2059 CPU_BASED_ACTIVATE_SECONDARY_CONTROLS)) { 2060 vmcs12_exec_ctrl = vmcs12->secondary_vm_exec_control & 2061 ~SECONDARY_EXEC_ENABLE_PML; 2062 exec_control |= vmcs12_exec_ctrl; 2063 } 2064 2065 /* VMCS shadowing for L2 is emulated for now */ 2066 exec_control &= ~SECONDARY_EXEC_SHADOW_VMCS; 2067 2068 if (exec_control & SECONDARY_EXEC_VIRTUAL_INTR_DELIVERY) 2069 vmcs_write16(GUEST_INTR_STATUS, 2070 vmcs12->guest_intr_status); 2071 2072 /* 2073 * Write an illegal value to APIC_ACCESS_ADDR. Later, 2074 * nested_get_vmcs12_pages will either fix it up or 2075 * remove the VM execution control. 2076 */ 2077 if (exec_control & SECONDARY_EXEC_VIRTUALIZE_APIC_ACCESSES) 2078 vmcs_write64(APIC_ACCESS_ADDR, -1ull); 2079 2080 if (exec_control & SECONDARY_EXEC_ENCLS_EXITING) 2081 vmcs_write64(ENCLS_EXITING_BITMAP, -1ull); 2082 2083 vmcs_write32(SECONDARY_VM_EXEC_CONTROL, exec_control); 2084 } 2085 2086 /* 2087 * ENTRY CONTROLS 2088 * 2089 * vmcs12's VM_{ENTRY,EXIT}_LOAD_IA32_EFER and VM_ENTRY_IA32E_MODE 2090 * are emulated by vmx_set_efer() in prepare_vmcs02(), but speculate 2091 * on the related bits (if supported by the CPU) in the hope that 2092 * we can avoid VMWrites during vmx_set_efer(). 2093 */ 2094 exec_control = (vmcs12->vm_entry_controls | vmx_vmentry_ctrl()) & 2095 ~VM_ENTRY_IA32E_MODE & ~VM_ENTRY_LOAD_IA32_EFER; 2096 if (cpu_has_load_ia32_efer()) { 2097 if (guest_efer & EFER_LMA) 2098 exec_control |= VM_ENTRY_IA32E_MODE; 2099 if (guest_efer != host_efer) 2100 exec_control |= VM_ENTRY_LOAD_IA32_EFER; 2101 } 2102 vm_entry_controls_init(vmx, exec_control); 2103 2104 /* 2105 * EXIT CONTROLS 2106 * 2107 * L2->L1 exit controls are emulated - the hardware exit is to L0 so 2108 * we should use its exit controls. Note that VM_EXIT_LOAD_IA32_EFER 2109 * bits may be modified by vmx_set_efer() in prepare_vmcs02(). 2110 */ 2111 exec_control = vmx_vmexit_ctrl(); 2112 if (cpu_has_load_ia32_efer() && guest_efer != host_efer) 2113 exec_control |= VM_EXIT_LOAD_IA32_EFER; 2114 vm_exit_controls_init(vmx, exec_control); 2115 2116 /* 2117 * Conceptually we want to copy the PML address and index from 2118 * vmcs01 here, and then back to vmcs01 on nested vmexit. But, 2119 * since we always flush the log on each vmexit and never change 2120 * the PML address (once set), this happens to be equivalent to 2121 * simply resetting the index in vmcs02. 2122 */ 2123 if (enable_pml) 2124 vmcs_write16(GUEST_PML_INDEX, PML_ENTITY_NUM - 1); 2125 2126 /* 2127 * Interrupt/Exception Fields 2128 */ 2129 if (vmx->nested.nested_run_pending) { 2130 vmcs_write32(VM_ENTRY_INTR_INFO_FIELD, 2131 vmcs12->vm_entry_intr_info_field); 2132 vmcs_write32(VM_ENTRY_EXCEPTION_ERROR_CODE, 2133 vmcs12->vm_entry_exception_error_code); 2134 vmcs_write32(VM_ENTRY_INSTRUCTION_LEN, 2135 vmcs12->vm_entry_instruction_len); 2136 vmcs_write32(GUEST_INTERRUPTIBILITY_INFO, 2137 vmcs12->guest_interruptibility_info); 2138 vmx->loaded_vmcs->nmi_known_unmasked = 2139 !(vmcs12->guest_interruptibility_info & GUEST_INTR_STATE_NMI); 2140 } else { 2141 vmcs_write32(VM_ENTRY_INTR_INFO_FIELD, 0); 2142 } 2143 } 2144 2145 static void prepare_vmcs02_full(struct vcpu_vmx *vmx, struct vmcs12 *vmcs12) 2146 { 2147 struct hv_enlightened_vmcs *hv_evmcs = vmx->nested.hv_evmcs; 2148 2149 if (!hv_evmcs || !(hv_evmcs->hv_clean_fields & 2150 HV_VMX_ENLIGHTENED_CLEAN_FIELD_GUEST_GRP2)) { 2151 vmcs_write16(GUEST_ES_SELECTOR, vmcs12->guest_es_selector); 2152 vmcs_write16(GUEST_CS_SELECTOR, vmcs12->guest_cs_selector); 2153 vmcs_write16(GUEST_SS_SELECTOR, vmcs12->guest_ss_selector); 2154 vmcs_write16(GUEST_DS_SELECTOR, vmcs12->guest_ds_selector); 2155 vmcs_write16(GUEST_FS_SELECTOR, vmcs12->guest_fs_selector); 2156 vmcs_write16(GUEST_GS_SELECTOR, vmcs12->guest_gs_selector); 2157 vmcs_write16(GUEST_LDTR_SELECTOR, vmcs12->guest_ldtr_selector); 2158 vmcs_write16(GUEST_TR_SELECTOR, vmcs12->guest_tr_selector); 2159 vmcs_write32(GUEST_ES_LIMIT, vmcs12->guest_es_limit); 2160 vmcs_write32(GUEST_CS_LIMIT, vmcs12->guest_cs_limit); 2161 vmcs_write32(GUEST_SS_LIMIT, vmcs12->guest_ss_limit); 2162 vmcs_write32(GUEST_DS_LIMIT, vmcs12->guest_ds_limit); 2163 vmcs_write32(GUEST_FS_LIMIT, vmcs12->guest_fs_limit); 2164 vmcs_write32(GUEST_GS_LIMIT, vmcs12->guest_gs_limit); 2165 vmcs_write32(GUEST_LDTR_LIMIT, vmcs12->guest_ldtr_limit); 2166 vmcs_write32(GUEST_TR_LIMIT, vmcs12->guest_tr_limit); 2167 vmcs_write32(GUEST_GDTR_LIMIT, vmcs12->guest_gdtr_limit); 2168 vmcs_write32(GUEST_IDTR_LIMIT, vmcs12->guest_idtr_limit); 2169 vmcs_write32(GUEST_ES_AR_BYTES, vmcs12->guest_es_ar_bytes); 2170 vmcs_write32(GUEST_DS_AR_BYTES, vmcs12->guest_ds_ar_bytes); 2171 vmcs_write32(GUEST_FS_AR_BYTES, vmcs12->guest_fs_ar_bytes); 2172 vmcs_write32(GUEST_GS_AR_BYTES, vmcs12->guest_gs_ar_bytes); 2173 vmcs_write32(GUEST_LDTR_AR_BYTES, vmcs12->guest_ldtr_ar_bytes); 2174 vmcs_write32(GUEST_TR_AR_BYTES, vmcs12->guest_tr_ar_bytes); 2175 vmcs_writel(GUEST_ES_BASE, vmcs12->guest_es_base); 2176 vmcs_writel(GUEST_CS_BASE, vmcs12->guest_cs_base); 2177 vmcs_writel(GUEST_SS_BASE, vmcs12->guest_ss_base); 2178 vmcs_writel(GUEST_DS_BASE, vmcs12->guest_ds_base); 2179 vmcs_writel(GUEST_FS_BASE, vmcs12->guest_fs_base); 2180 vmcs_writel(GUEST_GS_BASE, vmcs12->guest_gs_base); 2181 vmcs_writel(GUEST_LDTR_BASE, vmcs12->guest_ldtr_base); 2182 vmcs_writel(GUEST_TR_BASE, vmcs12->guest_tr_base); 2183 vmcs_writel(GUEST_GDTR_BASE, vmcs12->guest_gdtr_base); 2184 vmcs_writel(GUEST_IDTR_BASE, vmcs12->guest_idtr_base); 2185 } 2186 2187 if (!hv_evmcs || !(hv_evmcs->hv_clean_fields & 2188 HV_VMX_ENLIGHTENED_CLEAN_FIELD_GUEST_GRP1)) { 2189 vmcs_write32(GUEST_SYSENTER_CS, vmcs12->guest_sysenter_cs); 2190 vmcs_writel(GUEST_PENDING_DBG_EXCEPTIONS, 2191 vmcs12->guest_pending_dbg_exceptions); 2192 vmcs_writel(GUEST_SYSENTER_ESP, vmcs12->guest_sysenter_esp); 2193 vmcs_writel(GUEST_SYSENTER_EIP, vmcs12->guest_sysenter_eip); 2194 2195 /* 2196 * L1 may access the L2's PDPTR, so save them to construct 2197 * vmcs12 2198 */ 2199 if (enable_ept) { 2200 vmcs_write64(GUEST_PDPTR0, vmcs12->guest_pdptr0); 2201 vmcs_write64(GUEST_PDPTR1, vmcs12->guest_pdptr1); 2202 vmcs_write64(GUEST_PDPTR2, vmcs12->guest_pdptr2); 2203 vmcs_write64(GUEST_PDPTR3, vmcs12->guest_pdptr3); 2204 } 2205 } 2206 2207 if (nested_cpu_has_xsaves(vmcs12)) 2208 vmcs_write64(XSS_EXIT_BITMAP, vmcs12->xss_exit_bitmap); 2209 2210 /* 2211 * Whether page-faults are trapped is determined by a combination of 2212 * 3 settings: PFEC_MASK, PFEC_MATCH and EXCEPTION_BITMAP.PF. 2213 * If enable_ept, L0 doesn't care about page faults and we should 2214 * set all of these to L1's desires. However, if !enable_ept, L0 does 2215 * care about (at least some) page faults, and because it is not easy 2216 * (if at all possible?) to merge L0 and L1's desires, we simply ask 2217 * to exit on each and every L2 page fault. This is done by setting 2218 * MASK=MATCH=0 and (see below) EB.PF=1. 2219 * Note that below we don't need special code to set EB.PF beyond the 2220 * "or"ing of the EB of vmcs01 and vmcs12, because when enable_ept, 2221 * vmcs01's EB.PF is 0 so the "or" will take vmcs12's value, and when 2222 * !enable_ept, EB.PF is 1, so the "or" will always be 1. 2223 */ 2224 vmcs_write32(PAGE_FAULT_ERROR_CODE_MASK, 2225 enable_ept ? vmcs12->page_fault_error_code_mask : 0); 2226 vmcs_write32(PAGE_FAULT_ERROR_CODE_MATCH, 2227 enable_ept ? vmcs12->page_fault_error_code_match : 0); 2228 2229 if (cpu_has_vmx_apicv()) { 2230 vmcs_write64(EOI_EXIT_BITMAP0, vmcs12->eoi_exit_bitmap0); 2231 vmcs_write64(EOI_EXIT_BITMAP1, vmcs12->eoi_exit_bitmap1); 2232 vmcs_write64(EOI_EXIT_BITMAP2, vmcs12->eoi_exit_bitmap2); 2233 vmcs_write64(EOI_EXIT_BITMAP3, vmcs12->eoi_exit_bitmap3); 2234 } 2235 2236 vmcs_write32(VM_EXIT_MSR_LOAD_COUNT, vmx->msr_autoload.host.nr); 2237 vmcs_write32(VM_ENTRY_MSR_LOAD_COUNT, vmx->msr_autoload.guest.nr); 2238 2239 set_cr4_guest_host_mask(vmx); 2240 2241 if (kvm_mpx_supported()) { 2242 if (vmx->nested.nested_run_pending && 2243 (vmcs12->vm_entry_controls & VM_ENTRY_LOAD_BNDCFGS)) 2244 vmcs_write64(GUEST_BNDCFGS, vmcs12->guest_bndcfgs); 2245 else 2246 vmcs_write64(GUEST_BNDCFGS, vmx->nested.vmcs01_guest_bndcfgs); 2247 } 2248 } 2249 2250 /* 2251 * prepare_vmcs02 is called when the L1 guest hypervisor runs its nested 2252 * L2 guest. L1 has a vmcs for L2 (vmcs12), and this function "merges" it 2253 * with L0's requirements for its guest (a.k.a. vmcs01), so we can run the L2 2254 * guest in a way that will both be appropriate to L1's requests, and our 2255 * needs. In addition to modifying the active vmcs (which is vmcs02), this 2256 * function also has additional necessary side-effects, like setting various 2257 * vcpu->arch fields. 2258 * Returns 0 on success, 1 on failure. Invalid state exit qualification code 2259 * is assigned to entry_failure_code on failure. 2260 */ 2261 static int prepare_vmcs02(struct kvm_vcpu *vcpu, struct vmcs12 *vmcs12, 2262 u32 *entry_failure_code) 2263 { 2264 struct vcpu_vmx *vmx = to_vmx(vcpu); 2265 struct hv_enlightened_vmcs *hv_evmcs = vmx->nested.hv_evmcs; 2266 2267 if (vmx->nested.dirty_vmcs12 || vmx->nested.hv_evmcs) { 2268 prepare_vmcs02_full(vmx, vmcs12); 2269 vmx->nested.dirty_vmcs12 = false; 2270 } 2271 2272 /* 2273 * First, the fields that are shadowed. This must be kept in sync 2274 * with vmcs_shadow_fields.h. 2275 */ 2276 if (!hv_evmcs || !(hv_evmcs->hv_clean_fields & 2277 HV_VMX_ENLIGHTENED_CLEAN_FIELD_GUEST_GRP2)) { 2278 vmcs_write32(GUEST_CS_AR_BYTES, vmcs12->guest_cs_ar_bytes); 2279 vmcs_write32(GUEST_SS_AR_BYTES, vmcs12->guest_ss_ar_bytes); 2280 } 2281 2282 if (vmx->nested.nested_run_pending && 2283 (vmcs12->vm_entry_controls & VM_ENTRY_LOAD_DEBUG_CONTROLS)) { 2284 kvm_set_dr(vcpu, 7, vmcs12->guest_dr7); 2285 vmcs_write64(GUEST_IA32_DEBUGCTL, vmcs12->guest_ia32_debugctl); 2286 } else { 2287 kvm_set_dr(vcpu, 7, vcpu->arch.dr7); 2288 vmcs_write64(GUEST_IA32_DEBUGCTL, vmx->nested.vmcs01_debugctl); 2289 } 2290 vmx_set_rflags(vcpu, vmcs12->guest_rflags); 2291 2292 vmx->nested.preemption_timer_expired = false; 2293 if (nested_cpu_has_preemption_timer(vmcs12)) 2294 vmx_start_preemption_timer(vcpu); 2295 2296 /* EXCEPTION_BITMAP and CR0_GUEST_HOST_MASK should basically be the 2297 * bitwise-or of what L1 wants to trap for L2, and what we want to 2298 * trap. Note that CR0.TS also needs updating - we do this later. 2299 */ 2300 update_exception_bitmap(vcpu); 2301 vcpu->arch.cr0_guest_owned_bits &= ~vmcs12->cr0_guest_host_mask; 2302 vmcs_writel(CR0_GUEST_HOST_MASK, ~vcpu->arch.cr0_guest_owned_bits); 2303 2304 if (vmx->nested.nested_run_pending && 2305 (vmcs12->vm_entry_controls & VM_ENTRY_LOAD_IA32_PAT)) { 2306 vmcs_write64(GUEST_IA32_PAT, vmcs12->guest_ia32_pat); 2307 vcpu->arch.pat = vmcs12->guest_ia32_pat; 2308 } else if (vmcs_config.vmentry_ctrl & VM_ENTRY_LOAD_IA32_PAT) { 2309 vmcs_write64(GUEST_IA32_PAT, vmx->vcpu.arch.pat); 2310 } 2311 2312 vmcs_write64(TSC_OFFSET, vcpu->arch.tsc_offset); 2313 2314 if (kvm_has_tsc_control) 2315 decache_tsc_multiplier(vmx); 2316 2317 if (enable_vpid) { 2318 /* 2319 * There is no direct mapping between vpid02 and vpid12, the 2320 * vpid02 is per-vCPU for L0 and reused while the value of 2321 * vpid12 is changed w/ one invvpid during nested vmentry. 2322 * The vpid12 is allocated by L1 for L2, so it will not 2323 * influence global bitmap(for vpid01 and vpid02 allocation) 2324 * even if spawn a lot of nested vCPUs. 2325 */ 2326 if (nested_cpu_has_vpid(vmcs12) && nested_has_guest_tlb_tag(vcpu)) { 2327 if (vmcs12->virtual_processor_id != vmx->nested.last_vpid) { 2328 vmx->nested.last_vpid = vmcs12->virtual_processor_id; 2329 __vmx_flush_tlb(vcpu, nested_get_vpid02(vcpu), false); 2330 } 2331 } else { 2332 /* 2333 * If L1 use EPT, then L0 needs to execute INVEPT on 2334 * EPTP02 instead of EPTP01. Therefore, delay TLB 2335 * flush until vmcs02->eptp is fully updated by 2336 * KVM_REQ_LOAD_CR3. Note that this assumes 2337 * KVM_REQ_TLB_FLUSH is evaluated after 2338 * KVM_REQ_LOAD_CR3 in vcpu_enter_guest(). 2339 */ 2340 kvm_make_request(KVM_REQ_TLB_FLUSH, vcpu); 2341 } 2342 } 2343 2344 if (nested_cpu_has_ept(vmcs12)) 2345 nested_ept_init_mmu_context(vcpu); 2346 else if (nested_cpu_has2(vmcs12, 2347 SECONDARY_EXEC_VIRTUALIZE_APIC_ACCESSES)) 2348 vmx_flush_tlb(vcpu, true); 2349 2350 /* 2351 * This sets GUEST_CR0 to vmcs12->guest_cr0, possibly modifying those 2352 * bits which we consider mandatory enabled. 2353 * The CR0_READ_SHADOW is what L2 should have expected to read given 2354 * the specifications by L1; It's not enough to take 2355 * vmcs12->cr0_read_shadow because on our cr0_guest_host_mask we we 2356 * have more bits than L1 expected. 2357 */ 2358 vmx_set_cr0(vcpu, vmcs12->guest_cr0); 2359 vmcs_writel(CR0_READ_SHADOW, nested_read_cr0(vmcs12)); 2360 2361 vmx_set_cr4(vcpu, vmcs12->guest_cr4); 2362 vmcs_writel(CR4_READ_SHADOW, nested_read_cr4(vmcs12)); 2363 2364 vcpu->arch.efer = nested_vmx_calc_efer(vmx, vmcs12); 2365 /* Note: may modify VM_ENTRY/EXIT_CONTROLS and GUEST/HOST_IA32_EFER */ 2366 vmx_set_efer(vcpu, vcpu->arch.efer); 2367 2368 /* 2369 * Guest state is invalid and unrestricted guest is disabled, 2370 * which means L1 attempted VMEntry to L2 with invalid state. 2371 * Fail the VMEntry. 2372 */ 2373 if (vmx->emulation_required) { 2374 *entry_failure_code = ENTRY_FAIL_DEFAULT; 2375 return 1; 2376 } 2377 2378 /* Shadow page tables on either EPT or shadow page tables. */ 2379 if (nested_vmx_load_cr3(vcpu, vmcs12->guest_cr3, nested_cpu_has_ept(vmcs12), 2380 entry_failure_code)) 2381 return 1; 2382 2383 if (!enable_ept) 2384 vcpu->arch.walk_mmu->inject_page_fault = vmx_inject_page_fault_nested; 2385 2386 kvm_register_write(vcpu, VCPU_REGS_RSP, vmcs12->guest_rsp); 2387 kvm_register_write(vcpu, VCPU_REGS_RIP, vmcs12->guest_rip); 2388 return 0; 2389 } 2390 2391 static int nested_vmx_check_nmi_controls(struct vmcs12 *vmcs12) 2392 { 2393 if (!nested_cpu_has_nmi_exiting(vmcs12) && 2394 nested_cpu_has_virtual_nmis(vmcs12)) 2395 return -EINVAL; 2396 2397 if (!nested_cpu_has_virtual_nmis(vmcs12) && 2398 nested_cpu_has(vmcs12, CPU_BASED_VIRTUAL_NMI_PENDING)) 2399 return -EINVAL; 2400 2401 return 0; 2402 } 2403 2404 static bool valid_ept_address(struct kvm_vcpu *vcpu, u64 address) 2405 { 2406 struct vcpu_vmx *vmx = to_vmx(vcpu); 2407 int maxphyaddr = cpuid_maxphyaddr(vcpu); 2408 2409 /* Check for memory type validity */ 2410 switch (address & VMX_EPTP_MT_MASK) { 2411 case VMX_EPTP_MT_UC: 2412 if (!(vmx->nested.msrs.ept_caps & VMX_EPTP_UC_BIT)) 2413 return false; 2414 break; 2415 case VMX_EPTP_MT_WB: 2416 if (!(vmx->nested.msrs.ept_caps & VMX_EPTP_WB_BIT)) 2417 return false; 2418 break; 2419 default: 2420 return false; 2421 } 2422 2423 /* only 4 levels page-walk length are valid */ 2424 if ((address & VMX_EPTP_PWL_MASK) != VMX_EPTP_PWL_4) 2425 return false; 2426 2427 /* Reserved bits should not be set */ 2428 if (address >> maxphyaddr || ((address >> 7) & 0x1f)) 2429 return false; 2430 2431 /* AD, if set, should be supported */ 2432 if (address & VMX_EPTP_AD_ENABLE_BIT) { 2433 if (!(vmx->nested.msrs.ept_caps & VMX_EPT_AD_BIT)) 2434 return false; 2435 } 2436 2437 return true; 2438 } 2439 2440 /* 2441 * Checks related to VM-Execution Control Fields 2442 */ 2443 static int nested_check_vm_execution_controls(struct kvm_vcpu *vcpu, 2444 struct vmcs12 *vmcs12) 2445 { 2446 struct vcpu_vmx *vmx = to_vmx(vcpu); 2447 2448 if (!vmx_control_verify(vmcs12->pin_based_vm_exec_control, 2449 vmx->nested.msrs.pinbased_ctls_low, 2450 vmx->nested.msrs.pinbased_ctls_high) || 2451 !vmx_control_verify(vmcs12->cpu_based_vm_exec_control, 2452 vmx->nested.msrs.procbased_ctls_low, 2453 vmx->nested.msrs.procbased_ctls_high)) 2454 return -EINVAL; 2455 2456 if (nested_cpu_has(vmcs12, CPU_BASED_ACTIVATE_SECONDARY_CONTROLS) && 2457 !vmx_control_verify(vmcs12->secondary_vm_exec_control, 2458 vmx->nested.msrs.secondary_ctls_low, 2459 vmx->nested.msrs.secondary_ctls_high)) 2460 return -EINVAL; 2461 2462 if (vmcs12->cr3_target_count > nested_cpu_vmx_misc_cr3_count(vcpu) || 2463 nested_vmx_check_io_bitmap_controls(vcpu, vmcs12) || 2464 nested_vmx_check_msr_bitmap_controls(vcpu, vmcs12) || 2465 nested_vmx_check_tpr_shadow_controls(vcpu, vmcs12) || 2466 nested_vmx_check_apic_access_controls(vcpu, vmcs12) || 2467 nested_vmx_check_apicv_controls(vcpu, vmcs12) || 2468 nested_vmx_check_nmi_controls(vmcs12) || 2469 nested_vmx_check_pml_controls(vcpu, vmcs12) || 2470 nested_vmx_check_unrestricted_guest_controls(vcpu, vmcs12) || 2471 nested_vmx_check_mode_based_ept_exec_controls(vcpu, vmcs12) || 2472 nested_vmx_check_shadow_vmcs_controls(vcpu, vmcs12) || 2473 (nested_cpu_has_vpid(vmcs12) && !vmcs12->virtual_processor_id)) 2474 return -EINVAL; 2475 2476 if (!nested_cpu_has_preemption_timer(vmcs12) && 2477 nested_cpu_has_save_preemption_timer(vmcs12)) 2478 return -EINVAL; 2479 2480 if (nested_cpu_has_ept(vmcs12) && 2481 !valid_ept_address(vcpu, vmcs12->ept_pointer)) 2482 return -EINVAL; 2483 2484 if (nested_cpu_has_vmfunc(vmcs12)) { 2485 if (vmcs12->vm_function_control & 2486 ~vmx->nested.msrs.vmfunc_controls) 2487 return -EINVAL; 2488 2489 if (nested_cpu_has_eptp_switching(vmcs12)) { 2490 if (!nested_cpu_has_ept(vmcs12) || 2491 !page_address_valid(vcpu, vmcs12->eptp_list_address)) 2492 return -EINVAL; 2493 } 2494 } 2495 2496 return 0; 2497 } 2498 2499 /* 2500 * Checks related to VM-Exit Control Fields 2501 */ 2502 static int nested_check_vm_exit_controls(struct kvm_vcpu *vcpu, 2503 struct vmcs12 *vmcs12) 2504 { 2505 struct vcpu_vmx *vmx = to_vmx(vcpu); 2506 2507 if (!vmx_control_verify(vmcs12->vm_exit_controls, 2508 vmx->nested.msrs.exit_ctls_low, 2509 vmx->nested.msrs.exit_ctls_high) || 2510 nested_vmx_check_exit_msr_switch_controls(vcpu, vmcs12)) 2511 return -EINVAL; 2512 2513 return 0; 2514 } 2515 2516 /* 2517 * Checks related to VM-Entry Control Fields 2518 */ 2519 static int nested_check_vm_entry_controls(struct kvm_vcpu *vcpu, 2520 struct vmcs12 *vmcs12) 2521 { 2522 struct vcpu_vmx *vmx = to_vmx(vcpu); 2523 2524 if (!vmx_control_verify(vmcs12->vm_entry_controls, 2525 vmx->nested.msrs.entry_ctls_low, 2526 vmx->nested.msrs.entry_ctls_high)) 2527 return -EINVAL; 2528 2529 /* 2530 * From the Intel SDM, volume 3: 2531 * Fields relevant to VM-entry event injection must be set properly. 2532 * These fields are the VM-entry interruption-information field, the 2533 * VM-entry exception error code, and the VM-entry instruction length. 2534 */ 2535 if (vmcs12->vm_entry_intr_info_field & INTR_INFO_VALID_MASK) { 2536 u32 intr_info = vmcs12->vm_entry_intr_info_field; 2537 u8 vector = intr_info & INTR_INFO_VECTOR_MASK; 2538 u32 intr_type = intr_info & INTR_INFO_INTR_TYPE_MASK; 2539 bool has_error_code = intr_info & INTR_INFO_DELIVER_CODE_MASK; 2540 bool should_have_error_code; 2541 bool urg = nested_cpu_has2(vmcs12, 2542 SECONDARY_EXEC_UNRESTRICTED_GUEST); 2543 bool prot_mode = !urg || vmcs12->guest_cr0 & X86_CR0_PE; 2544 2545 /* VM-entry interruption-info field: interruption type */ 2546 if (intr_type == INTR_TYPE_RESERVED || 2547 (intr_type == INTR_TYPE_OTHER_EVENT && 2548 !nested_cpu_supports_monitor_trap_flag(vcpu))) 2549 return -EINVAL; 2550 2551 /* VM-entry interruption-info field: vector */ 2552 if ((intr_type == INTR_TYPE_NMI_INTR && vector != NMI_VECTOR) || 2553 (intr_type == INTR_TYPE_HARD_EXCEPTION && vector > 31) || 2554 (intr_type == INTR_TYPE_OTHER_EVENT && vector != 0)) 2555 return -EINVAL; 2556 2557 /* VM-entry interruption-info field: deliver error code */ 2558 should_have_error_code = 2559 intr_type == INTR_TYPE_HARD_EXCEPTION && prot_mode && 2560 x86_exception_has_error_code(vector); 2561 if (has_error_code != should_have_error_code) 2562 return -EINVAL; 2563 2564 /* VM-entry exception error code */ 2565 if (has_error_code && 2566 vmcs12->vm_entry_exception_error_code & GENMASK(31, 15)) 2567 return -EINVAL; 2568 2569 /* VM-entry interruption-info field: reserved bits */ 2570 if (intr_info & INTR_INFO_RESVD_BITS_MASK) 2571 return -EINVAL; 2572 2573 /* VM-entry instruction length */ 2574 switch (intr_type) { 2575 case INTR_TYPE_SOFT_EXCEPTION: 2576 case INTR_TYPE_SOFT_INTR: 2577 case INTR_TYPE_PRIV_SW_EXCEPTION: 2578 if ((vmcs12->vm_entry_instruction_len > 15) || 2579 (vmcs12->vm_entry_instruction_len == 0 && 2580 !nested_cpu_has_zero_length_injection(vcpu))) 2581 return -EINVAL; 2582 } 2583 } 2584 2585 if (nested_vmx_check_entry_msr_switch_controls(vcpu, vmcs12)) 2586 return -EINVAL; 2587 2588 return 0; 2589 } 2590 2591 /* 2592 * Checks related to Host Control Registers and MSRs 2593 */ 2594 static int nested_check_host_control_regs(struct kvm_vcpu *vcpu, 2595 struct vmcs12 *vmcs12) 2596 { 2597 bool ia32e; 2598 2599 if (!nested_host_cr0_valid(vcpu, vmcs12->host_cr0) || 2600 !nested_host_cr4_valid(vcpu, vmcs12->host_cr4) || 2601 !nested_cr3_valid(vcpu, vmcs12->host_cr3)) 2602 return -EINVAL; 2603 /* 2604 * If the load IA32_EFER VM-exit control is 1, bits reserved in the 2605 * IA32_EFER MSR must be 0 in the field for that register. In addition, 2606 * the values of the LMA and LME bits in the field must each be that of 2607 * the host address-space size VM-exit control. 2608 */ 2609 if (vmcs12->vm_exit_controls & VM_EXIT_LOAD_IA32_EFER) { 2610 ia32e = (vmcs12->vm_exit_controls & 2611 VM_EXIT_HOST_ADDR_SPACE_SIZE) != 0; 2612 if (!kvm_valid_efer(vcpu, vmcs12->host_ia32_efer) || 2613 ia32e != !!(vmcs12->host_ia32_efer & EFER_LMA) || 2614 ia32e != !!(vmcs12->host_ia32_efer & EFER_LME)) 2615 return -EINVAL; 2616 } 2617 2618 return 0; 2619 } 2620 2621 /* 2622 * Checks related to Guest Non-register State 2623 */ 2624 static int nested_check_guest_non_reg_state(struct vmcs12 *vmcs12) 2625 { 2626 if (vmcs12->guest_activity_state != GUEST_ACTIVITY_ACTIVE && 2627 vmcs12->guest_activity_state != GUEST_ACTIVITY_HLT) 2628 return -EINVAL; 2629 2630 return 0; 2631 } 2632 2633 static int nested_vmx_check_vmentry_prereqs(struct kvm_vcpu *vcpu, 2634 struct vmcs12 *vmcs12) 2635 { 2636 if (nested_check_vm_execution_controls(vcpu, vmcs12) || 2637 nested_check_vm_exit_controls(vcpu, vmcs12) || 2638 nested_check_vm_entry_controls(vcpu, vmcs12)) 2639 return VMXERR_ENTRY_INVALID_CONTROL_FIELD; 2640 2641 if (nested_check_host_control_regs(vcpu, vmcs12)) 2642 return VMXERR_ENTRY_INVALID_HOST_STATE_FIELD; 2643 2644 if (nested_check_guest_non_reg_state(vmcs12)) 2645 return VMXERR_ENTRY_INVALID_CONTROL_FIELD; 2646 2647 return 0; 2648 } 2649 2650 static int nested_vmx_check_vmcs_link_ptr(struct kvm_vcpu *vcpu, 2651 struct vmcs12 *vmcs12) 2652 { 2653 int r; 2654 struct page *page; 2655 struct vmcs12 *shadow; 2656 2657 if (vmcs12->vmcs_link_pointer == -1ull) 2658 return 0; 2659 2660 if (!page_address_valid(vcpu, vmcs12->vmcs_link_pointer)) 2661 return -EINVAL; 2662 2663 page = kvm_vcpu_gpa_to_page(vcpu, vmcs12->vmcs_link_pointer); 2664 if (is_error_page(page)) 2665 return -EINVAL; 2666 2667 r = 0; 2668 shadow = kmap(page); 2669 if (shadow->hdr.revision_id != VMCS12_REVISION || 2670 shadow->hdr.shadow_vmcs != nested_cpu_has_shadow_vmcs(vmcs12)) 2671 r = -EINVAL; 2672 kunmap(page); 2673 kvm_release_page_clean(page); 2674 return r; 2675 } 2676 2677 static int nested_vmx_check_vmentry_postreqs(struct kvm_vcpu *vcpu, 2678 struct vmcs12 *vmcs12, 2679 u32 *exit_qual) 2680 { 2681 bool ia32e; 2682 2683 *exit_qual = ENTRY_FAIL_DEFAULT; 2684 2685 if (!nested_guest_cr0_valid(vcpu, vmcs12->guest_cr0) || 2686 !nested_guest_cr4_valid(vcpu, vmcs12->guest_cr4)) 2687 return 1; 2688 2689 if (nested_vmx_check_vmcs_link_ptr(vcpu, vmcs12)) { 2690 *exit_qual = ENTRY_FAIL_VMCS_LINK_PTR; 2691 return 1; 2692 } 2693 2694 /* 2695 * If the load IA32_EFER VM-entry control is 1, the following checks 2696 * are performed on the field for the IA32_EFER MSR: 2697 * - Bits reserved in the IA32_EFER MSR must be 0. 2698 * - Bit 10 (corresponding to IA32_EFER.LMA) must equal the value of 2699 * the IA-32e mode guest VM-exit control. It must also be identical 2700 * to bit 8 (LME) if bit 31 in the CR0 field (corresponding to 2701 * CR0.PG) is 1. 2702 */ 2703 if (to_vmx(vcpu)->nested.nested_run_pending && 2704 (vmcs12->vm_entry_controls & VM_ENTRY_LOAD_IA32_EFER)) { 2705 ia32e = (vmcs12->vm_entry_controls & VM_ENTRY_IA32E_MODE) != 0; 2706 if (!kvm_valid_efer(vcpu, vmcs12->guest_ia32_efer) || 2707 ia32e != !!(vmcs12->guest_ia32_efer & EFER_LMA) || 2708 ((vmcs12->guest_cr0 & X86_CR0_PG) && 2709 ia32e != !!(vmcs12->guest_ia32_efer & EFER_LME))) 2710 return 1; 2711 } 2712 2713 if ((vmcs12->vm_entry_controls & VM_ENTRY_LOAD_BNDCFGS) && 2714 (is_noncanonical_address(vmcs12->guest_bndcfgs & PAGE_MASK, vcpu) || 2715 (vmcs12->guest_bndcfgs & MSR_IA32_BNDCFGS_RSVD))) 2716 return 1; 2717 2718 return 0; 2719 } 2720 2721 static int nested_vmx_check_vmentry_hw(struct kvm_vcpu *vcpu) 2722 { 2723 struct vcpu_vmx *vmx = to_vmx(vcpu); 2724 unsigned long cr3, cr4; 2725 2726 if (!nested_early_check) 2727 return 0; 2728 2729 if (vmx->msr_autoload.host.nr) 2730 vmcs_write32(VM_EXIT_MSR_LOAD_COUNT, 0); 2731 if (vmx->msr_autoload.guest.nr) 2732 vmcs_write32(VM_ENTRY_MSR_LOAD_COUNT, 0); 2733 2734 preempt_disable(); 2735 2736 vmx_prepare_switch_to_guest(vcpu); 2737 2738 /* 2739 * Induce a consistency check VMExit by clearing bit 1 in GUEST_RFLAGS, 2740 * which is reserved to '1' by hardware. GUEST_RFLAGS is guaranteed to 2741 * be written (by preparve_vmcs02()) before the "real" VMEnter, i.e. 2742 * there is no need to preserve other bits or save/restore the field. 2743 */ 2744 vmcs_writel(GUEST_RFLAGS, 0); 2745 2746 cr3 = __get_current_cr3_fast(); 2747 if (unlikely(cr3 != vmx->loaded_vmcs->host_state.cr3)) { 2748 vmcs_writel(HOST_CR3, cr3); 2749 vmx->loaded_vmcs->host_state.cr3 = cr3; 2750 } 2751 2752 cr4 = cr4_read_shadow(); 2753 if (unlikely(cr4 != vmx->loaded_vmcs->host_state.cr4)) { 2754 vmcs_writel(HOST_CR4, cr4); 2755 vmx->loaded_vmcs->host_state.cr4 = cr4; 2756 } 2757 2758 vmx->__launched = vmx->loaded_vmcs->launched; 2759 2760 asm( 2761 /* Set HOST_RSP */ 2762 "sub $%c[wordsize], %%" _ASM_SP "\n\t" /* temporarily adjust RSP for CALL */ 2763 __ex("vmwrite %%" _ASM_SP ", %%" _ASM_DX) "\n\t" 2764 "mov %%" _ASM_SP ", %c[host_rsp](%1)\n\t" 2765 "add $%c[wordsize], %%" _ASM_SP "\n\t" /* un-adjust RSP */ 2766 2767 /* Check if vmlaunch or vmresume is needed */ 2768 "cmpl $0, %c[launched](%% " _ASM_CX")\n\t" 2769 2770 "call vmx_vmenter\n\t" 2771 2772 /* Set vmx->fail accordingly */ 2773 "setbe %c[fail](%% " _ASM_CX")\n\t" 2774 : ASM_CALL_CONSTRAINT 2775 : "c"(vmx), "d"((unsigned long)HOST_RSP), 2776 [launched]"i"(offsetof(struct vcpu_vmx, __launched)), 2777 [fail]"i"(offsetof(struct vcpu_vmx, fail)), 2778 [host_rsp]"i"(offsetof(struct vcpu_vmx, host_rsp)), 2779 [wordsize]"i"(sizeof(ulong)) 2780 : "rax", "cc", "memory" 2781 ); 2782 2783 preempt_enable(); 2784 2785 if (vmx->msr_autoload.host.nr) 2786 vmcs_write32(VM_EXIT_MSR_LOAD_COUNT, vmx->msr_autoload.host.nr); 2787 if (vmx->msr_autoload.guest.nr) 2788 vmcs_write32(VM_ENTRY_MSR_LOAD_COUNT, vmx->msr_autoload.guest.nr); 2789 2790 if (vmx->fail) { 2791 WARN_ON_ONCE(vmcs_read32(VM_INSTRUCTION_ERROR) != 2792 VMXERR_ENTRY_INVALID_CONTROL_FIELD); 2793 vmx->fail = 0; 2794 return 1; 2795 } 2796 2797 /* 2798 * VMExit clears RFLAGS.IF and DR7, even on a consistency check. 2799 */ 2800 local_irq_enable(); 2801 if (hw_breakpoint_active()) 2802 set_debugreg(__this_cpu_read(cpu_dr7), 7); 2803 2804 /* 2805 * A non-failing VMEntry means we somehow entered guest mode with 2806 * an illegal RIP, and that's just the tip of the iceberg. There 2807 * is no telling what memory has been modified or what state has 2808 * been exposed to unknown code. Hitting this all but guarantees 2809 * a (very critical) hardware issue. 2810 */ 2811 WARN_ON(!(vmcs_read32(VM_EXIT_REASON) & 2812 VMX_EXIT_REASONS_FAILED_VMENTRY)); 2813 2814 return 0; 2815 } 2816 STACK_FRAME_NON_STANDARD(nested_vmx_check_vmentry_hw); 2817 2818 2819 static inline bool nested_vmx_prepare_msr_bitmap(struct kvm_vcpu *vcpu, 2820 struct vmcs12 *vmcs12); 2821 2822 static void nested_get_vmcs12_pages(struct kvm_vcpu *vcpu) 2823 { 2824 struct vmcs12 *vmcs12 = get_vmcs12(vcpu); 2825 struct vcpu_vmx *vmx = to_vmx(vcpu); 2826 struct page *page; 2827 u64 hpa; 2828 2829 if (nested_cpu_has2(vmcs12, SECONDARY_EXEC_VIRTUALIZE_APIC_ACCESSES)) { 2830 /* 2831 * Translate L1 physical address to host physical 2832 * address for vmcs02. Keep the page pinned, so this 2833 * physical address remains valid. We keep a reference 2834 * to it so we can release it later. 2835 */ 2836 if (vmx->nested.apic_access_page) { /* shouldn't happen */ 2837 kvm_release_page_dirty(vmx->nested.apic_access_page); 2838 vmx->nested.apic_access_page = NULL; 2839 } 2840 page = kvm_vcpu_gpa_to_page(vcpu, vmcs12->apic_access_addr); 2841 /* 2842 * If translation failed, no matter: This feature asks 2843 * to exit when accessing the given address, and if it 2844 * can never be accessed, this feature won't do 2845 * anything anyway. 2846 */ 2847 if (!is_error_page(page)) { 2848 vmx->nested.apic_access_page = page; 2849 hpa = page_to_phys(vmx->nested.apic_access_page); 2850 vmcs_write64(APIC_ACCESS_ADDR, hpa); 2851 } else { 2852 vmcs_clear_bits(SECONDARY_VM_EXEC_CONTROL, 2853 SECONDARY_EXEC_VIRTUALIZE_APIC_ACCESSES); 2854 } 2855 } 2856 2857 if (nested_cpu_has(vmcs12, CPU_BASED_TPR_SHADOW)) { 2858 if (vmx->nested.virtual_apic_page) { /* shouldn't happen */ 2859 kvm_release_page_dirty(vmx->nested.virtual_apic_page); 2860 vmx->nested.virtual_apic_page = NULL; 2861 } 2862 page = kvm_vcpu_gpa_to_page(vcpu, vmcs12->virtual_apic_page_addr); 2863 2864 /* 2865 * If translation failed, VM entry will fail because 2866 * prepare_vmcs02 set VIRTUAL_APIC_PAGE_ADDR to -1ull. 2867 * Failing the vm entry is _not_ what the processor 2868 * does but it's basically the only possibility we 2869 * have. We could still enter the guest if CR8 load 2870 * exits are enabled, CR8 store exits are enabled, and 2871 * virtualize APIC access is disabled; in this case 2872 * the processor would never use the TPR shadow and we 2873 * could simply clear the bit from the execution 2874 * control. But such a configuration is useless, so 2875 * let's keep the code simple. 2876 */ 2877 if (!is_error_page(page)) { 2878 vmx->nested.virtual_apic_page = page; 2879 hpa = page_to_phys(vmx->nested.virtual_apic_page); 2880 vmcs_write64(VIRTUAL_APIC_PAGE_ADDR, hpa); 2881 } 2882 } 2883 2884 if (nested_cpu_has_posted_intr(vmcs12)) { 2885 if (vmx->nested.pi_desc_page) { /* shouldn't happen */ 2886 kunmap(vmx->nested.pi_desc_page); 2887 kvm_release_page_dirty(vmx->nested.pi_desc_page); 2888 vmx->nested.pi_desc_page = NULL; 2889 vmx->nested.pi_desc = NULL; 2890 vmcs_write64(POSTED_INTR_DESC_ADDR, -1ull); 2891 } 2892 page = kvm_vcpu_gpa_to_page(vcpu, vmcs12->posted_intr_desc_addr); 2893 if (is_error_page(page)) 2894 return; 2895 vmx->nested.pi_desc_page = page; 2896 vmx->nested.pi_desc = kmap(vmx->nested.pi_desc_page); 2897 vmx->nested.pi_desc = 2898 (struct pi_desc *)((void *)vmx->nested.pi_desc + 2899 (unsigned long)(vmcs12->posted_intr_desc_addr & 2900 (PAGE_SIZE - 1))); 2901 vmcs_write64(POSTED_INTR_DESC_ADDR, 2902 page_to_phys(vmx->nested.pi_desc_page) + 2903 (unsigned long)(vmcs12->posted_intr_desc_addr & 2904 (PAGE_SIZE - 1))); 2905 } 2906 if (nested_vmx_prepare_msr_bitmap(vcpu, vmcs12)) 2907 vmcs_set_bits(CPU_BASED_VM_EXEC_CONTROL, 2908 CPU_BASED_USE_MSR_BITMAPS); 2909 else 2910 vmcs_clear_bits(CPU_BASED_VM_EXEC_CONTROL, 2911 CPU_BASED_USE_MSR_BITMAPS); 2912 } 2913 2914 /* 2915 * Intel's VMX Instruction Reference specifies a common set of prerequisites 2916 * for running VMX instructions (except VMXON, whose prerequisites are 2917 * slightly different). It also specifies what exception to inject otherwise. 2918 * Note that many of these exceptions have priority over VM exits, so they 2919 * don't have to be checked again here. 2920 */ 2921 static int nested_vmx_check_permission(struct kvm_vcpu *vcpu) 2922 { 2923 if (!to_vmx(vcpu)->nested.vmxon) { 2924 kvm_queue_exception(vcpu, UD_VECTOR); 2925 return 0; 2926 } 2927 2928 if (vmx_get_cpl(vcpu)) { 2929 kvm_inject_gp(vcpu, 0); 2930 return 0; 2931 } 2932 2933 return 1; 2934 } 2935 2936 static u8 vmx_has_apicv_interrupt(struct kvm_vcpu *vcpu) 2937 { 2938 u8 rvi = vmx_get_rvi(); 2939 u8 vppr = kvm_lapic_get_reg(vcpu->arch.apic, APIC_PROCPRI); 2940 2941 return ((rvi & 0xf0) > (vppr & 0xf0)); 2942 } 2943 2944 static void load_vmcs12_host_state(struct kvm_vcpu *vcpu, 2945 struct vmcs12 *vmcs12); 2946 2947 /* 2948 * If from_vmentry is false, this is being called from state restore (either RSM 2949 * or KVM_SET_NESTED_STATE). Otherwise it's called from vmlaunch/vmresume. 2950 + * 2951 + * Returns: 2952 + * 0 - success, i.e. proceed with actual VMEnter 2953 + * 1 - consistency check VMExit 2954 + * -1 - consistency check VMFail 2955 */ 2956 int nested_vmx_enter_non_root_mode(struct kvm_vcpu *vcpu, bool from_vmentry) 2957 { 2958 struct vcpu_vmx *vmx = to_vmx(vcpu); 2959 struct vmcs12 *vmcs12 = get_vmcs12(vcpu); 2960 bool evaluate_pending_interrupts; 2961 u32 exit_reason = EXIT_REASON_INVALID_STATE; 2962 u32 exit_qual; 2963 2964 evaluate_pending_interrupts = vmcs_read32(CPU_BASED_VM_EXEC_CONTROL) & 2965 (CPU_BASED_VIRTUAL_INTR_PENDING | CPU_BASED_VIRTUAL_NMI_PENDING); 2966 if (likely(!evaluate_pending_interrupts) && kvm_vcpu_apicv_active(vcpu)) 2967 evaluate_pending_interrupts |= vmx_has_apicv_interrupt(vcpu); 2968 2969 if (!(vmcs12->vm_entry_controls & VM_ENTRY_LOAD_DEBUG_CONTROLS)) 2970 vmx->nested.vmcs01_debugctl = vmcs_read64(GUEST_IA32_DEBUGCTL); 2971 if (kvm_mpx_supported() && 2972 !(vmcs12->vm_entry_controls & VM_ENTRY_LOAD_BNDCFGS)) 2973 vmx->nested.vmcs01_guest_bndcfgs = vmcs_read64(GUEST_BNDCFGS); 2974 2975 vmx_switch_vmcs(vcpu, &vmx->nested.vmcs02); 2976 2977 prepare_vmcs02_early(vmx, vmcs12); 2978 2979 if (from_vmentry) { 2980 nested_get_vmcs12_pages(vcpu); 2981 2982 if (nested_vmx_check_vmentry_hw(vcpu)) { 2983 vmx_switch_vmcs(vcpu, &vmx->vmcs01); 2984 return -1; 2985 } 2986 2987 if (nested_vmx_check_vmentry_postreqs(vcpu, vmcs12, &exit_qual)) 2988 goto vmentry_fail_vmexit; 2989 } 2990 2991 enter_guest_mode(vcpu); 2992 if (vmcs12->cpu_based_vm_exec_control & CPU_BASED_USE_TSC_OFFSETING) 2993 vcpu->arch.tsc_offset += vmcs12->tsc_offset; 2994 2995 if (prepare_vmcs02(vcpu, vmcs12, &exit_qual)) 2996 goto vmentry_fail_vmexit_guest_mode; 2997 2998 if (from_vmentry) { 2999 exit_reason = EXIT_REASON_MSR_LOAD_FAIL; 3000 exit_qual = nested_vmx_load_msr(vcpu, 3001 vmcs12->vm_entry_msr_load_addr, 3002 vmcs12->vm_entry_msr_load_count); 3003 if (exit_qual) 3004 goto vmentry_fail_vmexit_guest_mode; 3005 } else { 3006 /* 3007 * The MMU is not initialized to point at the right entities yet and 3008 * "get pages" would need to read data from the guest (i.e. we will 3009 * need to perform gpa to hpa translation). Request a call 3010 * to nested_get_vmcs12_pages before the next VM-entry. The MSRs 3011 * have already been set at vmentry time and should not be reset. 3012 */ 3013 kvm_make_request(KVM_REQ_GET_VMCS12_PAGES, vcpu); 3014 } 3015 3016 /* 3017 * If L1 had a pending IRQ/NMI until it executed 3018 * VMLAUNCH/VMRESUME which wasn't delivered because it was 3019 * disallowed (e.g. interrupts disabled), L0 needs to 3020 * evaluate if this pending event should cause an exit from L2 3021 * to L1 or delivered directly to L2 (e.g. In case L1 don't 3022 * intercept EXTERNAL_INTERRUPT). 3023 * 3024 * Usually this would be handled by the processor noticing an 3025 * IRQ/NMI window request, or checking RVI during evaluation of 3026 * pending virtual interrupts. However, this setting was done 3027 * on VMCS01 and now VMCS02 is active instead. Thus, we force L0 3028 * to perform pending event evaluation by requesting a KVM_REQ_EVENT. 3029 */ 3030 if (unlikely(evaluate_pending_interrupts)) 3031 kvm_make_request(KVM_REQ_EVENT, vcpu); 3032 3033 /* 3034 * Note no nested_vmx_succeed or nested_vmx_fail here. At this point 3035 * we are no longer running L1, and VMLAUNCH/VMRESUME has not yet 3036 * returned as far as L1 is concerned. It will only return (and set 3037 * the success flag) when L2 exits (see nested_vmx_vmexit()). 3038 */ 3039 return 0; 3040 3041 /* 3042 * A failed consistency check that leads to a VMExit during L1's 3043 * VMEnter to L2 is a variation of a normal VMexit, as explained in 3044 * 26.7 "VM-entry failures during or after loading guest state". 3045 */ 3046 vmentry_fail_vmexit_guest_mode: 3047 if (vmcs12->cpu_based_vm_exec_control & CPU_BASED_USE_TSC_OFFSETING) 3048 vcpu->arch.tsc_offset -= vmcs12->tsc_offset; 3049 leave_guest_mode(vcpu); 3050 3051 vmentry_fail_vmexit: 3052 vmx_switch_vmcs(vcpu, &vmx->vmcs01); 3053 3054 if (!from_vmentry) 3055 return 1; 3056 3057 load_vmcs12_host_state(vcpu, vmcs12); 3058 vmcs12->vm_exit_reason = exit_reason | VMX_EXIT_REASONS_FAILED_VMENTRY; 3059 vmcs12->exit_qualification = exit_qual; 3060 if (enable_shadow_vmcs || vmx->nested.hv_evmcs) 3061 vmx->nested.need_vmcs12_sync = true; 3062 return 1; 3063 } 3064 3065 /* 3066 * nested_vmx_run() handles a nested entry, i.e., a VMLAUNCH or VMRESUME on L1 3067 * for running an L2 nested guest. 3068 */ 3069 static int nested_vmx_run(struct kvm_vcpu *vcpu, bool launch) 3070 { 3071 struct vmcs12 *vmcs12; 3072 struct vcpu_vmx *vmx = to_vmx(vcpu); 3073 u32 interrupt_shadow = vmx_get_interrupt_shadow(vcpu); 3074 int ret; 3075 3076 if (!nested_vmx_check_permission(vcpu)) 3077 return 1; 3078 3079 if (!nested_vmx_handle_enlightened_vmptrld(vcpu, true)) 3080 return 1; 3081 3082 if (!vmx->nested.hv_evmcs && vmx->nested.current_vmptr == -1ull) 3083 return nested_vmx_failInvalid(vcpu); 3084 3085 vmcs12 = get_vmcs12(vcpu); 3086 3087 /* 3088 * Can't VMLAUNCH or VMRESUME a shadow VMCS. Despite the fact 3089 * that there *is* a valid VMCS pointer, RFLAGS.CF is set 3090 * rather than RFLAGS.ZF, and no error number is stored to the 3091 * VM-instruction error field. 3092 */ 3093 if (vmcs12->hdr.shadow_vmcs) 3094 return nested_vmx_failInvalid(vcpu); 3095 3096 if (vmx->nested.hv_evmcs) { 3097 copy_enlightened_to_vmcs12(vmx); 3098 /* Enlightened VMCS doesn't have launch state */ 3099 vmcs12->launch_state = !launch; 3100 } else if (enable_shadow_vmcs) { 3101 copy_shadow_to_vmcs12(vmx); 3102 } 3103 3104 /* 3105 * The nested entry process starts with enforcing various prerequisites 3106 * on vmcs12 as required by the Intel SDM, and act appropriately when 3107 * they fail: As the SDM explains, some conditions should cause the 3108 * instruction to fail, while others will cause the instruction to seem 3109 * to succeed, but return an EXIT_REASON_INVALID_STATE. 3110 * To speed up the normal (success) code path, we should avoid checking 3111 * for misconfigurations which will anyway be caught by the processor 3112 * when using the merged vmcs02. 3113 */ 3114 if (interrupt_shadow & KVM_X86_SHADOW_INT_MOV_SS) 3115 return nested_vmx_failValid(vcpu, 3116 VMXERR_ENTRY_EVENTS_BLOCKED_BY_MOV_SS); 3117 3118 if (vmcs12->launch_state == launch) 3119 return nested_vmx_failValid(vcpu, 3120 launch ? VMXERR_VMLAUNCH_NONCLEAR_VMCS 3121 : VMXERR_VMRESUME_NONLAUNCHED_VMCS); 3122 3123 ret = nested_vmx_check_vmentry_prereqs(vcpu, vmcs12); 3124 if (ret) 3125 return nested_vmx_failValid(vcpu, ret); 3126 3127 /* 3128 * We're finally done with prerequisite checking, and can start with 3129 * the nested entry. 3130 */ 3131 vmx->nested.nested_run_pending = 1; 3132 ret = nested_vmx_enter_non_root_mode(vcpu, true); 3133 vmx->nested.nested_run_pending = !ret; 3134 if (ret > 0) 3135 return 1; 3136 else if (ret) 3137 return nested_vmx_failValid(vcpu, 3138 VMXERR_ENTRY_INVALID_CONTROL_FIELD); 3139 3140 /* Hide L1D cache contents from the nested guest. */ 3141 vmx->vcpu.arch.l1tf_flush_l1d = true; 3142 3143 /* 3144 * Must happen outside of nested_vmx_enter_non_root_mode() as it will 3145 * also be used as part of restoring nVMX state for 3146 * snapshot restore (migration). 3147 * 3148 * In this flow, it is assumed that vmcs12 cache was 3149 * trasferred as part of captured nVMX state and should 3150 * therefore not be read from guest memory (which may not 3151 * exist on destination host yet). 3152 */ 3153 nested_cache_shadow_vmcs12(vcpu, vmcs12); 3154 3155 /* 3156 * If we're entering a halted L2 vcpu and the L2 vcpu won't be 3157 * awakened by event injection or by an NMI-window VM-exit or 3158 * by an interrupt-window VM-exit, halt the vcpu. 3159 */ 3160 if ((vmcs12->guest_activity_state == GUEST_ACTIVITY_HLT) && 3161 !(vmcs12->vm_entry_intr_info_field & INTR_INFO_VALID_MASK) && 3162 !(vmcs12->cpu_based_vm_exec_control & CPU_BASED_VIRTUAL_NMI_PENDING) && 3163 !((vmcs12->cpu_based_vm_exec_control & CPU_BASED_VIRTUAL_INTR_PENDING) && 3164 (vmcs12->guest_rflags & X86_EFLAGS_IF))) { 3165 vmx->nested.nested_run_pending = 0; 3166 return kvm_vcpu_halt(vcpu); 3167 } 3168 return 1; 3169 } 3170 3171 /* 3172 * On a nested exit from L2 to L1, vmcs12.guest_cr0 might not be up-to-date 3173 * because L2 may have changed some cr0 bits directly (CRO_GUEST_HOST_MASK). 3174 * This function returns the new value we should put in vmcs12.guest_cr0. 3175 * It's not enough to just return the vmcs02 GUEST_CR0. Rather, 3176 * 1. Bits that neither L0 nor L1 trapped, were set directly by L2 and are now 3177 * available in vmcs02 GUEST_CR0. (Note: It's enough to check that L0 3178 * didn't trap the bit, because if L1 did, so would L0). 3179 * 2. Bits that L1 asked to trap (and therefore L0 also did) could not have 3180 * been modified by L2, and L1 knows it. So just leave the old value of 3181 * the bit from vmcs12.guest_cr0. Note that the bit from vmcs02 GUEST_CR0 3182 * isn't relevant, because if L0 traps this bit it can set it to anything. 3183 * 3. Bits that L1 didn't trap, but L0 did. L1 believes the guest could have 3184 * changed these bits, and therefore they need to be updated, but L0 3185 * didn't necessarily allow them to be changed in GUEST_CR0 - and rather 3186 * put them in vmcs02 CR0_READ_SHADOW. So take these bits from there. 3187 */ 3188 static inline unsigned long 3189 vmcs12_guest_cr0(struct kvm_vcpu *vcpu, struct vmcs12 *vmcs12) 3190 { 3191 return 3192 /*1*/ (vmcs_readl(GUEST_CR0) & vcpu->arch.cr0_guest_owned_bits) | 3193 /*2*/ (vmcs12->guest_cr0 & vmcs12->cr0_guest_host_mask) | 3194 /*3*/ (vmcs_readl(CR0_READ_SHADOW) & ~(vmcs12->cr0_guest_host_mask | 3195 vcpu->arch.cr0_guest_owned_bits)); 3196 } 3197 3198 static inline unsigned long 3199 vmcs12_guest_cr4(struct kvm_vcpu *vcpu, struct vmcs12 *vmcs12) 3200 { 3201 return 3202 /*1*/ (vmcs_readl(GUEST_CR4) & vcpu->arch.cr4_guest_owned_bits) | 3203 /*2*/ (vmcs12->guest_cr4 & vmcs12->cr4_guest_host_mask) | 3204 /*3*/ (vmcs_readl(CR4_READ_SHADOW) & ~(vmcs12->cr4_guest_host_mask | 3205 vcpu->arch.cr4_guest_owned_bits)); 3206 } 3207 3208 static void vmcs12_save_pending_event(struct kvm_vcpu *vcpu, 3209 struct vmcs12 *vmcs12) 3210 { 3211 u32 idt_vectoring; 3212 unsigned int nr; 3213 3214 if (vcpu->arch.exception.injected) { 3215 nr = vcpu->arch.exception.nr; 3216 idt_vectoring = nr | VECTORING_INFO_VALID_MASK; 3217 3218 if (kvm_exception_is_soft(nr)) { 3219 vmcs12->vm_exit_instruction_len = 3220 vcpu->arch.event_exit_inst_len; 3221 idt_vectoring |= INTR_TYPE_SOFT_EXCEPTION; 3222 } else 3223 idt_vectoring |= INTR_TYPE_HARD_EXCEPTION; 3224 3225 if (vcpu->arch.exception.has_error_code) { 3226 idt_vectoring |= VECTORING_INFO_DELIVER_CODE_MASK; 3227 vmcs12->idt_vectoring_error_code = 3228 vcpu->arch.exception.error_code; 3229 } 3230 3231 vmcs12->idt_vectoring_info_field = idt_vectoring; 3232 } else if (vcpu->arch.nmi_injected) { 3233 vmcs12->idt_vectoring_info_field = 3234 INTR_TYPE_NMI_INTR | INTR_INFO_VALID_MASK | NMI_VECTOR; 3235 } else if (vcpu->arch.interrupt.injected) { 3236 nr = vcpu->arch.interrupt.nr; 3237 idt_vectoring = nr | VECTORING_INFO_VALID_MASK; 3238 3239 if (vcpu->arch.interrupt.soft) { 3240 idt_vectoring |= INTR_TYPE_SOFT_INTR; 3241 vmcs12->vm_entry_instruction_len = 3242 vcpu->arch.event_exit_inst_len; 3243 } else 3244 idt_vectoring |= INTR_TYPE_EXT_INTR; 3245 3246 vmcs12->idt_vectoring_info_field = idt_vectoring; 3247 } 3248 } 3249 3250 3251 static void nested_mark_vmcs12_pages_dirty(struct kvm_vcpu *vcpu) 3252 { 3253 struct vmcs12 *vmcs12 = get_vmcs12(vcpu); 3254 gfn_t gfn; 3255 3256 /* 3257 * Don't need to mark the APIC access page dirty; it is never 3258 * written to by the CPU during APIC virtualization. 3259 */ 3260 3261 if (nested_cpu_has(vmcs12, CPU_BASED_TPR_SHADOW)) { 3262 gfn = vmcs12->virtual_apic_page_addr >> PAGE_SHIFT; 3263 kvm_vcpu_mark_page_dirty(vcpu, gfn); 3264 } 3265 3266 if (nested_cpu_has_posted_intr(vmcs12)) { 3267 gfn = vmcs12->posted_intr_desc_addr >> PAGE_SHIFT; 3268 kvm_vcpu_mark_page_dirty(vcpu, gfn); 3269 } 3270 } 3271 3272 static void vmx_complete_nested_posted_interrupt(struct kvm_vcpu *vcpu) 3273 { 3274 struct vcpu_vmx *vmx = to_vmx(vcpu); 3275 int max_irr; 3276 void *vapic_page; 3277 u16 status; 3278 3279 if (!vmx->nested.pi_desc || !vmx->nested.pi_pending) 3280 return; 3281 3282 vmx->nested.pi_pending = false; 3283 if (!pi_test_and_clear_on(vmx->nested.pi_desc)) 3284 return; 3285 3286 max_irr = find_last_bit((unsigned long *)vmx->nested.pi_desc->pir, 256); 3287 if (max_irr != 256) { 3288 vapic_page = kmap(vmx->nested.virtual_apic_page); 3289 __kvm_apic_update_irr(vmx->nested.pi_desc->pir, 3290 vapic_page, &max_irr); 3291 kunmap(vmx->nested.virtual_apic_page); 3292 3293 status = vmcs_read16(GUEST_INTR_STATUS); 3294 if ((u8)max_irr > ((u8)status & 0xff)) { 3295 status &= ~0xff; 3296 status |= (u8)max_irr; 3297 vmcs_write16(GUEST_INTR_STATUS, status); 3298 } 3299 } 3300 3301 nested_mark_vmcs12_pages_dirty(vcpu); 3302 } 3303 3304 static void nested_vmx_inject_exception_vmexit(struct kvm_vcpu *vcpu, 3305 unsigned long exit_qual) 3306 { 3307 struct vmcs12 *vmcs12 = get_vmcs12(vcpu); 3308 unsigned int nr = vcpu->arch.exception.nr; 3309 u32 intr_info = nr | INTR_INFO_VALID_MASK; 3310 3311 if (vcpu->arch.exception.has_error_code) { 3312 vmcs12->vm_exit_intr_error_code = vcpu->arch.exception.error_code; 3313 intr_info |= INTR_INFO_DELIVER_CODE_MASK; 3314 } 3315 3316 if (kvm_exception_is_soft(nr)) 3317 intr_info |= INTR_TYPE_SOFT_EXCEPTION; 3318 else 3319 intr_info |= INTR_TYPE_HARD_EXCEPTION; 3320 3321 if (!(vmcs12->idt_vectoring_info_field & VECTORING_INFO_VALID_MASK) && 3322 vmx_get_nmi_mask(vcpu)) 3323 intr_info |= INTR_INFO_UNBLOCK_NMI; 3324 3325 nested_vmx_vmexit(vcpu, EXIT_REASON_EXCEPTION_NMI, intr_info, exit_qual); 3326 } 3327 3328 static int vmx_check_nested_events(struct kvm_vcpu *vcpu, bool external_intr) 3329 { 3330 struct vcpu_vmx *vmx = to_vmx(vcpu); 3331 unsigned long exit_qual; 3332 bool block_nested_events = 3333 vmx->nested.nested_run_pending || kvm_event_needs_reinjection(vcpu); 3334 3335 if (vcpu->arch.exception.pending && 3336 nested_vmx_check_exception(vcpu, &exit_qual)) { 3337 if (block_nested_events) 3338 return -EBUSY; 3339 nested_vmx_inject_exception_vmexit(vcpu, exit_qual); 3340 return 0; 3341 } 3342 3343 if (nested_cpu_has_preemption_timer(get_vmcs12(vcpu)) && 3344 vmx->nested.preemption_timer_expired) { 3345 if (block_nested_events) 3346 return -EBUSY; 3347 nested_vmx_vmexit(vcpu, EXIT_REASON_PREEMPTION_TIMER, 0, 0); 3348 return 0; 3349 } 3350 3351 if (vcpu->arch.nmi_pending && nested_exit_on_nmi(vcpu)) { 3352 if (block_nested_events) 3353 return -EBUSY; 3354 nested_vmx_vmexit(vcpu, EXIT_REASON_EXCEPTION_NMI, 3355 NMI_VECTOR | INTR_TYPE_NMI_INTR | 3356 INTR_INFO_VALID_MASK, 0); 3357 /* 3358 * The NMI-triggered VM exit counts as injection: 3359 * clear this one and block further NMIs. 3360 */ 3361 vcpu->arch.nmi_pending = 0; 3362 vmx_set_nmi_mask(vcpu, true); 3363 return 0; 3364 } 3365 3366 if ((kvm_cpu_has_interrupt(vcpu) || external_intr) && 3367 nested_exit_on_intr(vcpu)) { 3368 if (block_nested_events) 3369 return -EBUSY; 3370 nested_vmx_vmexit(vcpu, EXIT_REASON_EXTERNAL_INTERRUPT, 0, 0); 3371 return 0; 3372 } 3373 3374 vmx_complete_nested_posted_interrupt(vcpu); 3375 return 0; 3376 } 3377 3378 static u32 vmx_get_preemption_timer_value(struct kvm_vcpu *vcpu) 3379 { 3380 ktime_t remaining = 3381 hrtimer_get_remaining(&to_vmx(vcpu)->nested.preemption_timer); 3382 u64 value; 3383 3384 if (ktime_to_ns(remaining) <= 0) 3385 return 0; 3386 3387 value = ktime_to_ns(remaining) * vcpu->arch.virtual_tsc_khz; 3388 do_div(value, 1000000); 3389 return value >> VMX_MISC_EMULATED_PREEMPTION_TIMER_RATE; 3390 } 3391 3392 /* 3393 * Update the guest state fields of vmcs12 to reflect changes that 3394 * occurred while L2 was running. (The "IA-32e mode guest" bit of the 3395 * VM-entry controls is also updated, since this is really a guest 3396 * state bit.) 3397 */ 3398 static void sync_vmcs12(struct kvm_vcpu *vcpu, struct vmcs12 *vmcs12) 3399 { 3400 vmcs12->guest_cr0 = vmcs12_guest_cr0(vcpu, vmcs12); 3401 vmcs12->guest_cr4 = vmcs12_guest_cr4(vcpu, vmcs12); 3402 3403 vmcs12->guest_rsp = kvm_register_read(vcpu, VCPU_REGS_RSP); 3404 vmcs12->guest_rip = kvm_register_read(vcpu, VCPU_REGS_RIP); 3405 vmcs12->guest_rflags = vmcs_readl(GUEST_RFLAGS); 3406 3407 vmcs12->guest_es_selector = vmcs_read16(GUEST_ES_SELECTOR); 3408 vmcs12->guest_cs_selector = vmcs_read16(GUEST_CS_SELECTOR); 3409 vmcs12->guest_ss_selector = vmcs_read16(GUEST_SS_SELECTOR); 3410 vmcs12->guest_ds_selector = vmcs_read16(GUEST_DS_SELECTOR); 3411 vmcs12->guest_fs_selector = vmcs_read16(GUEST_FS_SELECTOR); 3412 vmcs12->guest_gs_selector = vmcs_read16(GUEST_GS_SELECTOR); 3413 vmcs12->guest_ldtr_selector = vmcs_read16(GUEST_LDTR_SELECTOR); 3414 vmcs12->guest_tr_selector = vmcs_read16(GUEST_TR_SELECTOR); 3415 vmcs12->guest_es_limit = vmcs_read32(GUEST_ES_LIMIT); 3416 vmcs12->guest_cs_limit = vmcs_read32(GUEST_CS_LIMIT); 3417 vmcs12->guest_ss_limit = vmcs_read32(GUEST_SS_LIMIT); 3418 vmcs12->guest_ds_limit = vmcs_read32(GUEST_DS_LIMIT); 3419 vmcs12->guest_fs_limit = vmcs_read32(GUEST_FS_LIMIT); 3420 vmcs12->guest_gs_limit = vmcs_read32(GUEST_GS_LIMIT); 3421 vmcs12->guest_ldtr_limit = vmcs_read32(GUEST_LDTR_LIMIT); 3422 vmcs12->guest_tr_limit = vmcs_read32(GUEST_TR_LIMIT); 3423 vmcs12->guest_gdtr_limit = vmcs_read32(GUEST_GDTR_LIMIT); 3424 vmcs12->guest_idtr_limit = vmcs_read32(GUEST_IDTR_LIMIT); 3425 vmcs12->guest_es_ar_bytes = vmcs_read32(GUEST_ES_AR_BYTES); 3426 vmcs12->guest_cs_ar_bytes = vmcs_read32(GUEST_CS_AR_BYTES); 3427 vmcs12->guest_ss_ar_bytes = vmcs_read32(GUEST_SS_AR_BYTES); 3428 vmcs12->guest_ds_ar_bytes = vmcs_read32(GUEST_DS_AR_BYTES); 3429 vmcs12->guest_fs_ar_bytes = vmcs_read32(GUEST_FS_AR_BYTES); 3430 vmcs12->guest_gs_ar_bytes = vmcs_read32(GUEST_GS_AR_BYTES); 3431 vmcs12->guest_ldtr_ar_bytes = vmcs_read32(GUEST_LDTR_AR_BYTES); 3432 vmcs12->guest_tr_ar_bytes = vmcs_read32(GUEST_TR_AR_BYTES); 3433 vmcs12->guest_es_base = vmcs_readl(GUEST_ES_BASE); 3434 vmcs12->guest_cs_base = vmcs_readl(GUEST_CS_BASE); 3435 vmcs12->guest_ss_base = vmcs_readl(GUEST_SS_BASE); 3436 vmcs12->guest_ds_base = vmcs_readl(GUEST_DS_BASE); 3437 vmcs12->guest_fs_base = vmcs_readl(GUEST_FS_BASE); 3438 vmcs12->guest_gs_base = vmcs_readl(GUEST_GS_BASE); 3439 vmcs12->guest_ldtr_base = vmcs_readl(GUEST_LDTR_BASE); 3440 vmcs12->guest_tr_base = vmcs_readl(GUEST_TR_BASE); 3441 vmcs12->guest_gdtr_base = vmcs_readl(GUEST_GDTR_BASE); 3442 vmcs12->guest_idtr_base = vmcs_readl(GUEST_IDTR_BASE); 3443 3444 vmcs12->guest_interruptibility_info = 3445 vmcs_read32(GUEST_INTERRUPTIBILITY_INFO); 3446 vmcs12->guest_pending_dbg_exceptions = 3447 vmcs_readl(GUEST_PENDING_DBG_EXCEPTIONS); 3448 if (vcpu->arch.mp_state == KVM_MP_STATE_HALTED) 3449 vmcs12->guest_activity_state = GUEST_ACTIVITY_HLT; 3450 else 3451 vmcs12->guest_activity_state = GUEST_ACTIVITY_ACTIVE; 3452 3453 if (nested_cpu_has_preemption_timer(vmcs12)) { 3454 if (vmcs12->vm_exit_controls & 3455 VM_EXIT_SAVE_VMX_PREEMPTION_TIMER) 3456 vmcs12->vmx_preemption_timer_value = 3457 vmx_get_preemption_timer_value(vcpu); 3458 hrtimer_cancel(&to_vmx(vcpu)->nested.preemption_timer); 3459 } 3460 3461 /* 3462 * In some cases (usually, nested EPT), L2 is allowed to change its 3463 * own CR3 without exiting. If it has changed it, we must keep it. 3464 * Of course, if L0 is using shadow page tables, GUEST_CR3 was defined 3465 * by L0, not L1 or L2, so we mustn't unconditionally copy it to vmcs12. 3466 * 3467 * Additionally, restore L2's PDPTR to vmcs12. 3468 */ 3469 if (enable_ept) { 3470 vmcs12->guest_cr3 = vmcs_readl(GUEST_CR3); 3471 vmcs12->guest_pdptr0 = vmcs_read64(GUEST_PDPTR0); 3472 vmcs12->guest_pdptr1 = vmcs_read64(GUEST_PDPTR1); 3473 vmcs12->guest_pdptr2 = vmcs_read64(GUEST_PDPTR2); 3474 vmcs12->guest_pdptr3 = vmcs_read64(GUEST_PDPTR3); 3475 } 3476 3477 vmcs12->guest_linear_address = vmcs_readl(GUEST_LINEAR_ADDRESS); 3478 3479 if (nested_cpu_has_vid(vmcs12)) 3480 vmcs12->guest_intr_status = vmcs_read16(GUEST_INTR_STATUS); 3481 3482 vmcs12->vm_entry_controls = 3483 (vmcs12->vm_entry_controls & ~VM_ENTRY_IA32E_MODE) | 3484 (vm_entry_controls_get(to_vmx(vcpu)) & VM_ENTRY_IA32E_MODE); 3485 3486 if (vmcs12->vm_exit_controls & VM_EXIT_SAVE_DEBUG_CONTROLS) { 3487 kvm_get_dr(vcpu, 7, (unsigned long *)&vmcs12->guest_dr7); 3488 vmcs12->guest_ia32_debugctl = vmcs_read64(GUEST_IA32_DEBUGCTL); 3489 } 3490 3491 /* TODO: These cannot have changed unless we have MSR bitmaps and 3492 * the relevant bit asks not to trap the change */ 3493 if (vmcs12->vm_exit_controls & VM_EXIT_SAVE_IA32_PAT) 3494 vmcs12->guest_ia32_pat = vmcs_read64(GUEST_IA32_PAT); 3495 if (vmcs12->vm_exit_controls & VM_EXIT_SAVE_IA32_EFER) 3496 vmcs12->guest_ia32_efer = vcpu->arch.efer; 3497 vmcs12->guest_sysenter_cs = vmcs_read32(GUEST_SYSENTER_CS); 3498 vmcs12->guest_sysenter_esp = vmcs_readl(GUEST_SYSENTER_ESP); 3499 vmcs12->guest_sysenter_eip = vmcs_readl(GUEST_SYSENTER_EIP); 3500 if (kvm_mpx_supported()) 3501 vmcs12->guest_bndcfgs = vmcs_read64(GUEST_BNDCFGS); 3502 } 3503 3504 /* 3505 * prepare_vmcs12 is part of what we need to do when the nested L2 guest exits 3506 * and we want to prepare to run its L1 parent. L1 keeps a vmcs for L2 (vmcs12), 3507 * and this function updates it to reflect the changes to the guest state while 3508 * L2 was running (and perhaps made some exits which were handled directly by L0 3509 * without going back to L1), and to reflect the exit reason. 3510 * Note that we do not have to copy here all VMCS fields, just those that 3511 * could have changed by the L2 guest or the exit - i.e., the guest-state and 3512 * exit-information fields only. Other fields are modified by L1 with VMWRITE, 3513 * which already writes to vmcs12 directly. 3514 */ 3515 static void prepare_vmcs12(struct kvm_vcpu *vcpu, struct vmcs12 *vmcs12, 3516 u32 exit_reason, u32 exit_intr_info, 3517 unsigned long exit_qualification) 3518 { 3519 /* update guest state fields: */ 3520 sync_vmcs12(vcpu, vmcs12); 3521 3522 /* update exit information fields: */ 3523 3524 vmcs12->vm_exit_reason = exit_reason; 3525 vmcs12->exit_qualification = exit_qualification; 3526 vmcs12->vm_exit_intr_info = exit_intr_info; 3527 3528 vmcs12->idt_vectoring_info_field = 0; 3529 vmcs12->vm_exit_instruction_len = vmcs_read32(VM_EXIT_INSTRUCTION_LEN); 3530 vmcs12->vmx_instruction_info = vmcs_read32(VMX_INSTRUCTION_INFO); 3531 3532 if (!(vmcs12->vm_exit_reason & VMX_EXIT_REASONS_FAILED_VMENTRY)) { 3533 vmcs12->launch_state = 1; 3534 3535 /* vm_entry_intr_info_field is cleared on exit. Emulate this 3536 * instead of reading the real value. */ 3537 vmcs12->vm_entry_intr_info_field &= ~INTR_INFO_VALID_MASK; 3538 3539 /* 3540 * Transfer the event that L0 or L1 may wanted to inject into 3541 * L2 to IDT_VECTORING_INFO_FIELD. 3542 */ 3543 vmcs12_save_pending_event(vcpu, vmcs12); 3544 3545 /* 3546 * According to spec, there's no need to store the guest's 3547 * MSRs if the exit is due to a VM-entry failure that occurs 3548 * during or after loading the guest state. Since this exit 3549 * does not fall in that category, we need to save the MSRs. 3550 */ 3551 if (nested_vmx_store_msr(vcpu, 3552 vmcs12->vm_exit_msr_store_addr, 3553 vmcs12->vm_exit_msr_store_count)) 3554 nested_vmx_abort(vcpu, 3555 VMX_ABORT_SAVE_GUEST_MSR_FAIL); 3556 } 3557 3558 /* 3559 * Drop what we picked up for L2 via vmx_complete_interrupts. It is 3560 * preserved above and would only end up incorrectly in L1. 3561 */ 3562 vcpu->arch.nmi_injected = false; 3563 kvm_clear_exception_queue(vcpu); 3564 kvm_clear_interrupt_queue(vcpu); 3565 } 3566 3567 /* 3568 * A part of what we need to when the nested L2 guest exits and we want to 3569 * run its L1 parent, is to reset L1's guest state to the host state specified 3570 * in vmcs12. 3571 * This function is to be called not only on normal nested exit, but also on 3572 * a nested entry failure, as explained in Intel's spec, 3B.23.7 ("VM-Entry 3573 * Failures During or After Loading Guest State"). 3574 * This function should be called when the active VMCS is L1's (vmcs01). 3575 */ 3576 static void load_vmcs12_host_state(struct kvm_vcpu *vcpu, 3577 struct vmcs12 *vmcs12) 3578 { 3579 struct kvm_segment seg; 3580 u32 entry_failure_code; 3581 3582 if (vmcs12->vm_exit_controls & VM_EXIT_LOAD_IA32_EFER) 3583 vcpu->arch.efer = vmcs12->host_ia32_efer; 3584 else if (vmcs12->vm_exit_controls & VM_EXIT_HOST_ADDR_SPACE_SIZE) 3585 vcpu->arch.efer |= (EFER_LMA | EFER_LME); 3586 else 3587 vcpu->arch.efer &= ~(EFER_LMA | EFER_LME); 3588 vmx_set_efer(vcpu, vcpu->arch.efer); 3589 3590 kvm_register_write(vcpu, VCPU_REGS_RSP, vmcs12->host_rsp); 3591 kvm_register_write(vcpu, VCPU_REGS_RIP, vmcs12->host_rip); 3592 vmx_set_rflags(vcpu, X86_EFLAGS_FIXED); 3593 vmx_set_interrupt_shadow(vcpu, 0); 3594 3595 /* 3596 * Note that calling vmx_set_cr0 is important, even if cr0 hasn't 3597 * actually changed, because vmx_set_cr0 refers to efer set above. 3598 * 3599 * CR0_GUEST_HOST_MASK is already set in the original vmcs01 3600 * (KVM doesn't change it); 3601 */ 3602 vcpu->arch.cr0_guest_owned_bits = X86_CR0_TS; 3603 vmx_set_cr0(vcpu, vmcs12->host_cr0); 3604 3605 /* Same as above - no reason to call set_cr4_guest_host_mask(). */ 3606 vcpu->arch.cr4_guest_owned_bits = ~vmcs_readl(CR4_GUEST_HOST_MASK); 3607 vmx_set_cr4(vcpu, vmcs12->host_cr4); 3608 3609 nested_ept_uninit_mmu_context(vcpu); 3610 3611 /* 3612 * Only PDPTE load can fail as the value of cr3 was checked on entry and 3613 * couldn't have changed. 3614 */ 3615 if (nested_vmx_load_cr3(vcpu, vmcs12->host_cr3, false, &entry_failure_code)) 3616 nested_vmx_abort(vcpu, VMX_ABORT_LOAD_HOST_PDPTE_FAIL); 3617 3618 if (!enable_ept) 3619 vcpu->arch.walk_mmu->inject_page_fault = kvm_inject_page_fault; 3620 3621 /* 3622 * If vmcs01 doesn't use VPID, CPU flushes TLB on every 3623 * VMEntry/VMExit. Thus, no need to flush TLB. 3624 * 3625 * If vmcs12 doesn't use VPID, L1 expects TLB to be 3626 * flushed on every VMEntry/VMExit. 3627 * 3628 * Otherwise, we can preserve TLB entries as long as we are 3629 * able to tag L1 TLB entries differently than L2 TLB entries. 3630 * 3631 * If vmcs12 uses EPT, we need to execute this flush on EPTP01 3632 * and therefore we request the TLB flush to happen only after VMCS EPTP 3633 * has been set by KVM_REQ_LOAD_CR3. 3634 */ 3635 if (enable_vpid && 3636 (!nested_cpu_has_vpid(vmcs12) || !nested_has_guest_tlb_tag(vcpu))) { 3637 kvm_make_request(KVM_REQ_TLB_FLUSH, vcpu); 3638 } 3639 3640 vmcs_write32(GUEST_SYSENTER_CS, vmcs12->host_ia32_sysenter_cs); 3641 vmcs_writel(GUEST_SYSENTER_ESP, vmcs12->host_ia32_sysenter_esp); 3642 vmcs_writel(GUEST_SYSENTER_EIP, vmcs12->host_ia32_sysenter_eip); 3643 vmcs_writel(GUEST_IDTR_BASE, vmcs12->host_idtr_base); 3644 vmcs_writel(GUEST_GDTR_BASE, vmcs12->host_gdtr_base); 3645 vmcs_write32(GUEST_IDTR_LIMIT, 0xFFFF); 3646 vmcs_write32(GUEST_GDTR_LIMIT, 0xFFFF); 3647 3648 /* If not VM_EXIT_CLEAR_BNDCFGS, the L2 value propagates to L1. */ 3649 if (vmcs12->vm_exit_controls & VM_EXIT_CLEAR_BNDCFGS) 3650 vmcs_write64(GUEST_BNDCFGS, 0); 3651 3652 if (vmcs12->vm_exit_controls & VM_EXIT_LOAD_IA32_PAT) { 3653 vmcs_write64(GUEST_IA32_PAT, vmcs12->host_ia32_pat); 3654 vcpu->arch.pat = vmcs12->host_ia32_pat; 3655 } 3656 if (vmcs12->vm_exit_controls & VM_EXIT_LOAD_IA32_PERF_GLOBAL_CTRL) 3657 vmcs_write64(GUEST_IA32_PERF_GLOBAL_CTRL, 3658 vmcs12->host_ia32_perf_global_ctrl); 3659 3660 /* Set L1 segment info according to Intel SDM 3661 27.5.2 Loading Host Segment and Descriptor-Table Registers */ 3662 seg = (struct kvm_segment) { 3663 .base = 0, 3664 .limit = 0xFFFFFFFF, 3665 .selector = vmcs12->host_cs_selector, 3666 .type = 11, 3667 .present = 1, 3668 .s = 1, 3669 .g = 1 3670 }; 3671 if (vmcs12->vm_exit_controls & VM_EXIT_HOST_ADDR_SPACE_SIZE) 3672 seg.l = 1; 3673 else 3674 seg.db = 1; 3675 vmx_set_segment(vcpu, &seg, VCPU_SREG_CS); 3676 seg = (struct kvm_segment) { 3677 .base = 0, 3678 .limit = 0xFFFFFFFF, 3679 .type = 3, 3680 .present = 1, 3681 .s = 1, 3682 .db = 1, 3683 .g = 1 3684 }; 3685 seg.selector = vmcs12->host_ds_selector; 3686 vmx_set_segment(vcpu, &seg, VCPU_SREG_DS); 3687 seg.selector = vmcs12->host_es_selector; 3688 vmx_set_segment(vcpu, &seg, VCPU_SREG_ES); 3689 seg.selector = vmcs12->host_ss_selector; 3690 vmx_set_segment(vcpu, &seg, VCPU_SREG_SS); 3691 seg.selector = vmcs12->host_fs_selector; 3692 seg.base = vmcs12->host_fs_base; 3693 vmx_set_segment(vcpu, &seg, VCPU_SREG_FS); 3694 seg.selector = vmcs12->host_gs_selector; 3695 seg.base = vmcs12->host_gs_base; 3696 vmx_set_segment(vcpu, &seg, VCPU_SREG_GS); 3697 seg = (struct kvm_segment) { 3698 .base = vmcs12->host_tr_base, 3699 .limit = 0x67, 3700 .selector = vmcs12->host_tr_selector, 3701 .type = 11, 3702 .present = 1 3703 }; 3704 vmx_set_segment(vcpu, &seg, VCPU_SREG_TR); 3705 3706 kvm_set_dr(vcpu, 7, 0x400); 3707 vmcs_write64(GUEST_IA32_DEBUGCTL, 0); 3708 3709 if (cpu_has_vmx_msr_bitmap()) 3710 vmx_update_msr_bitmap(vcpu); 3711 3712 if (nested_vmx_load_msr(vcpu, vmcs12->vm_exit_msr_load_addr, 3713 vmcs12->vm_exit_msr_load_count)) 3714 nested_vmx_abort(vcpu, VMX_ABORT_LOAD_HOST_MSR_FAIL); 3715 } 3716 3717 static inline u64 nested_vmx_get_vmcs01_guest_efer(struct vcpu_vmx *vmx) 3718 { 3719 struct shared_msr_entry *efer_msr; 3720 unsigned int i; 3721 3722 if (vm_entry_controls_get(vmx) & VM_ENTRY_LOAD_IA32_EFER) 3723 return vmcs_read64(GUEST_IA32_EFER); 3724 3725 if (cpu_has_load_ia32_efer()) 3726 return host_efer; 3727 3728 for (i = 0; i < vmx->msr_autoload.guest.nr; ++i) { 3729 if (vmx->msr_autoload.guest.val[i].index == MSR_EFER) 3730 return vmx->msr_autoload.guest.val[i].value; 3731 } 3732 3733 efer_msr = find_msr_entry(vmx, MSR_EFER); 3734 if (efer_msr) 3735 return efer_msr->data; 3736 3737 return host_efer; 3738 } 3739 3740 static void nested_vmx_restore_host_state(struct kvm_vcpu *vcpu) 3741 { 3742 struct vmcs12 *vmcs12 = get_vmcs12(vcpu); 3743 struct vcpu_vmx *vmx = to_vmx(vcpu); 3744 struct vmx_msr_entry g, h; 3745 struct msr_data msr; 3746 gpa_t gpa; 3747 u32 i, j; 3748 3749 vcpu->arch.pat = vmcs_read64(GUEST_IA32_PAT); 3750 3751 if (vmcs12->vm_entry_controls & VM_ENTRY_LOAD_DEBUG_CONTROLS) { 3752 /* 3753 * L1's host DR7 is lost if KVM_GUESTDBG_USE_HW_BP is set 3754 * as vmcs01.GUEST_DR7 contains a userspace defined value 3755 * and vcpu->arch.dr7 is not squirreled away before the 3756 * nested VMENTER (not worth adding a variable in nested_vmx). 3757 */ 3758 if (vcpu->guest_debug & KVM_GUESTDBG_USE_HW_BP) 3759 kvm_set_dr(vcpu, 7, DR7_FIXED_1); 3760 else 3761 WARN_ON(kvm_set_dr(vcpu, 7, vmcs_readl(GUEST_DR7))); 3762 } 3763 3764 /* 3765 * Note that calling vmx_set_{efer,cr0,cr4} is important as they 3766 * handle a variety of side effects to KVM's software model. 3767 */ 3768 vmx_set_efer(vcpu, nested_vmx_get_vmcs01_guest_efer(vmx)); 3769 3770 vcpu->arch.cr0_guest_owned_bits = X86_CR0_TS; 3771 vmx_set_cr0(vcpu, vmcs_readl(CR0_READ_SHADOW)); 3772 3773 vcpu->arch.cr4_guest_owned_bits = ~vmcs_readl(CR4_GUEST_HOST_MASK); 3774 vmx_set_cr4(vcpu, vmcs_readl(CR4_READ_SHADOW)); 3775 3776 nested_ept_uninit_mmu_context(vcpu); 3777 vcpu->arch.cr3 = vmcs_readl(GUEST_CR3); 3778 __set_bit(VCPU_EXREG_CR3, (ulong *)&vcpu->arch.regs_avail); 3779 3780 /* 3781 * Use ept_save_pdptrs(vcpu) to load the MMU's cached PDPTRs 3782 * from vmcs01 (if necessary). The PDPTRs are not loaded on 3783 * VMFail, like everything else we just need to ensure our 3784 * software model is up-to-date. 3785 */ 3786 ept_save_pdptrs(vcpu); 3787 3788 kvm_mmu_reset_context(vcpu); 3789 3790 if (cpu_has_vmx_msr_bitmap()) 3791 vmx_update_msr_bitmap(vcpu); 3792 3793 /* 3794 * This nasty bit of open coding is a compromise between blindly 3795 * loading L1's MSRs using the exit load lists (incorrect emulation 3796 * of VMFail), leaving the nested VM's MSRs in the software model 3797 * (incorrect behavior) and snapshotting the modified MSRs (too 3798 * expensive since the lists are unbound by hardware). For each 3799 * MSR that was (prematurely) loaded from the nested VMEntry load 3800 * list, reload it from the exit load list if it exists and differs 3801 * from the guest value. The intent is to stuff host state as 3802 * silently as possible, not to fully process the exit load list. 3803 */ 3804 msr.host_initiated = false; 3805 for (i = 0; i < vmcs12->vm_entry_msr_load_count; i++) { 3806 gpa = vmcs12->vm_entry_msr_load_addr + (i * sizeof(g)); 3807 if (kvm_vcpu_read_guest(vcpu, gpa, &g, sizeof(g))) { 3808 pr_debug_ratelimited( 3809 "%s read MSR index failed (%u, 0x%08llx)\n", 3810 __func__, i, gpa); 3811 goto vmabort; 3812 } 3813 3814 for (j = 0; j < vmcs12->vm_exit_msr_load_count; j++) { 3815 gpa = vmcs12->vm_exit_msr_load_addr + (j * sizeof(h)); 3816 if (kvm_vcpu_read_guest(vcpu, gpa, &h, sizeof(h))) { 3817 pr_debug_ratelimited( 3818 "%s read MSR failed (%u, 0x%08llx)\n", 3819 __func__, j, gpa); 3820 goto vmabort; 3821 } 3822 if (h.index != g.index) 3823 continue; 3824 if (h.value == g.value) 3825 break; 3826 3827 if (nested_vmx_load_msr_check(vcpu, &h)) { 3828 pr_debug_ratelimited( 3829 "%s check failed (%u, 0x%x, 0x%x)\n", 3830 __func__, j, h.index, h.reserved); 3831 goto vmabort; 3832 } 3833 3834 msr.index = h.index; 3835 msr.data = h.value; 3836 if (kvm_set_msr(vcpu, &msr)) { 3837 pr_debug_ratelimited( 3838 "%s WRMSR failed (%u, 0x%x, 0x%llx)\n", 3839 __func__, j, h.index, h.value); 3840 goto vmabort; 3841 } 3842 } 3843 } 3844 3845 return; 3846 3847 vmabort: 3848 nested_vmx_abort(vcpu, VMX_ABORT_LOAD_HOST_MSR_FAIL); 3849 } 3850 3851 /* 3852 * Emulate an exit from nested guest (L2) to L1, i.e., prepare to run L1 3853 * and modify vmcs12 to make it see what it would expect to see there if 3854 * L2 was its real guest. Must only be called when in L2 (is_guest_mode()) 3855 */ 3856 void nested_vmx_vmexit(struct kvm_vcpu *vcpu, u32 exit_reason, 3857 u32 exit_intr_info, unsigned long exit_qualification) 3858 { 3859 struct vcpu_vmx *vmx = to_vmx(vcpu); 3860 struct vmcs12 *vmcs12 = get_vmcs12(vcpu); 3861 3862 /* trying to cancel vmlaunch/vmresume is a bug */ 3863 WARN_ON_ONCE(vmx->nested.nested_run_pending); 3864 3865 leave_guest_mode(vcpu); 3866 3867 if (vmcs12->cpu_based_vm_exec_control & CPU_BASED_USE_TSC_OFFSETING) 3868 vcpu->arch.tsc_offset -= vmcs12->tsc_offset; 3869 3870 if (likely(!vmx->fail)) { 3871 if (exit_reason == -1) 3872 sync_vmcs12(vcpu, vmcs12); 3873 else 3874 prepare_vmcs12(vcpu, vmcs12, exit_reason, exit_intr_info, 3875 exit_qualification); 3876 3877 /* 3878 * Must happen outside of sync_vmcs12() as it will 3879 * also be used to capture vmcs12 cache as part of 3880 * capturing nVMX state for snapshot (migration). 3881 * 3882 * Otherwise, this flush will dirty guest memory at a 3883 * point it is already assumed by user-space to be 3884 * immutable. 3885 */ 3886 nested_flush_cached_shadow_vmcs12(vcpu, vmcs12); 3887 } else { 3888 /* 3889 * The only expected VM-instruction error is "VM entry with 3890 * invalid control field(s)." Anything else indicates a 3891 * problem with L0. And we should never get here with a 3892 * VMFail of any type if early consistency checks are enabled. 3893 */ 3894 WARN_ON_ONCE(vmcs_read32(VM_INSTRUCTION_ERROR) != 3895 VMXERR_ENTRY_INVALID_CONTROL_FIELD); 3896 WARN_ON_ONCE(nested_early_check); 3897 } 3898 3899 vmx_switch_vmcs(vcpu, &vmx->vmcs01); 3900 3901 /* Update any VMCS fields that might have changed while L2 ran */ 3902 vmcs_write32(VM_EXIT_MSR_LOAD_COUNT, vmx->msr_autoload.host.nr); 3903 vmcs_write32(VM_ENTRY_MSR_LOAD_COUNT, vmx->msr_autoload.guest.nr); 3904 vmcs_write64(TSC_OFFSET, vcpu->arch.tsc_offset); 3905 3906 if (kvm_has_tsc_control) 3907 decache_tsc_multiplier(vmx); 3908 3909 if (vmx->nested.change_vmcs01_virtual_apic_mode) { 3910 vmx->nested.change_vmcs01_virtual_apic_mode = false; 3911 vmx_set_virtual_apic_mode(vcpu); 3912 } else if (!nested_cpu_has_ept(vmcs12) && 3913 nested_cpu_has2(vmcs12, 3914 SECONDARY_EXEC_VIRTUALIZE_APIC_ACCESSES)) { 3915 vmx_flush_tlb(vcpu, true); 3916 } 3917 3918 /* This is needed for same reason as it was needed in prepare_vmcs02 */ 3919 vmx->host_rsp = 0; 3920 3921 /* Unpin physical memory we referred to in vmcs02 */ 3922 if (vmx->nested.apic_access_page) { 3923 kvm_release_page_dirty(vmx->nested.apic_access_page); 3924 vmx->nested.apic_access_page = NULL; 3925 } 3926 if (vmx->nested.virtual_apic_page) { 3927 kvm_release_page_dirty(vmx->nested.virtual_apic_page); 3928 vmx->nested.virtual_apic_page = NULL; 3929 } 3930 if (vmx->nested.pi_desc_page) { 3931 kunmap(vmx->nested.pi_desc_page); 3932 kvm_release_page_dirty(vmx->nested.pi_desc_page); 3933 vmx->nested.pi_desc_page = NULL; 3934 vmx->nested.pi_desc = NULL; 3935 } 3936 3937 /* 3938 * We are now running in L2, mmu_notifier will force to reload the 3939 * page's hpa for L2 vmcs. Need to reload it for L1 before entering L1. 3940 */ 3941 kvm_make_request(KVM_REQ_APIC_PAGE_RELOAD, vcpu); 3942 3943 if ((exit_reason != -1) && (enable_shadow_vmcs || vmx->nested.hv_evmcs)) 3944 vmx->nested.need_vmcs12_sync = true; 3945 3946 /* in case we halted in L2 */ 3947 vcpu->arch.mp_state = KVM_MP_STATE_RUNNABLE; 3948 3949 if (likely(!vmx->fail)) { 3950 /* 3951 * TODO: SDM says that with acknowledge interrupt on 3952 * exit, bit 31 of the VM-exit interrupt information 3953 * (valid interrupt) is always set to 1 on 3954 * EXIT_REASON_EXTERNAL_INTERRUPT, so we shouldn't 3955 * need kvm_cpu_has_interrupt(). See the commit 3956 * message for details. 3957 */ 3958 if (nested_exit_intr_ack_set(vcpu) && 3959 exit_reason == EXIT_REASON_EXTERNAL_INTERRUPT && 3960 kvm_cpu_has_interrupt(vcpu)) { 3961 int irq = kvm_cpu_get_interrupt(vcpu); 3962 WARN_ON(irq < 0); 3963 vmcs12->vm_exit_intr_info = irq | 3964 INTR_INFO_VALID_MASK | INTR_TYPE_EXT_INTR; 3965 } 3966 3967 if (exit_reason != -1) 3968 trace_kvm_nested_vmexit_inject(vmcs12->vm_exit_reason, 3969 vmcs12->exit_qualification, 3970 vmcs12->idt_vectoring_info_field, 3971 vmcs12->vm_exit_intr_info, 3972 vmcs12->vm_exit_intr_error_code, 3973 KVM_ISA_VMX); 3974 3975 load_vmcs12_host_state(vcpu, vmcs12); 3976 3977 return; 3978 } 3979 3980 /* 3981 * After an early L2 VM-entry failure, we're now back 3982 * in L1 which thinks it just finished a VMLAUNCH or 3983 * VMRESUME instruction, so we need to set the failure 3984 * flag and the VM-instruction error field of the VMCS 3985 * accordingly, and skip the emulated instruction. 3986 */ 3987 (void)nested_vmx_failValid(vcpu, VMXERR_ENTRY_INVALID_CONTROL_FIELD); 3988 3989 /* 3990 * Restore L1's host state to KVM's software model. We're here 3991 * because a consistency check was caught by hardware, which 3992 * means some amount of guest state has been propagated to KVM's 3993 * model and needs to be unwound to the host's state. 3994 */ 3995 nested_vmx_restore_host_state(vcpu); 3996 3997 vmx->fail = 0; 3998 } 3999 4000 /* 4001 * Decode the memory-address operand of a vmx instruction, as recorded on an 4002 * exit caused by such an instruction (run by a guest hypervisor). 4003 * On success, returns 0. When the operand is invalid, returns 1 and throws 4004 * #UD or #GP. 4005 */ 4006 int get_vmx_mem_address(struct kvm_vcpu *vcpu, unsigned long exit_qualification, 4007 u32 vmx_instruction_info, bool wr, gva_t *ret) 4008 { 4009 gva_t off; 4010 bool exn; 4011 struct kvm_segment s; 4012 4013 /* 4014 * According to Vol. 3B, "Information for VM Exits Due to Instruction 4015 * Execution", on an exit, vmx_instruction_info holds most of the 4016 * addressing components of the operand. Only the displacement part 4017 * is put in exit_qualification (see 3B, "Basic VM-Exit Information"). 4018 * For how an actual address is calculated from all these components, 4019 * refer to Vol. 1, "Operand Addressing". 4020 */ 4021 int scaling = vmx_instruction_info & 3; 4022 int addr_size = (vmx_instruction_info >> 7) & 7; 4023 bool is_reg = vmx_instruction_info & (1u << 10); 4024 int seg_reg = (vmx_instruction_info >> 15) & 7; 4025 int index_reg = (vmx_instruction_info >> 18) & 0xf; 4026 bool index_is_valid = !(vmx_instruction_info & (1u << 22)); 4027 int base_reg = (vmx_instruction_info >> 23) & 0xf; 4028 bool base_is_valid = !(vmx_instruction_info & (1u << 27)); 4029 4030 if (is_reg) { 4031 kvm_queue_exception(vcpu, UD_VECTOR); 4032 return 1; 4033 } 4034 4035 /* Addr = segment_base + offset */ 4036 /* offset = base + [index * scale] + displacement */ 4037 off = exit_qualification; /* holds the displacement */ 4038 if (base_is_valid) 4039 off += kvm_register_read(vcpu, base_reg); 4040 if (index_is_valid) 4041 off += kvm_register_read(vcpu, index_reg)<<scaling; 4042 vmx_get_segment(vcpu, &s, seg_reg); 4043 *ret = s.base + off; 4044 4045 if (addr_size == 1) /* 32 bit */ 4046 *ret &= 0xffffffff; 4047 4048 /* Checks for #GP/#SS exceptions. */ 4049 exn = false; 4050 if (is_long_mode(vcpu)) { 4051 /* Long mode: #GP(0)/#SS(0) if the memory address is in a 4052 * non-canonical form. This is the only check on the memory 4053 * destination for long mode! 4054 */ 4055 exn = is_noncanonical_address(*ret, vcpu); 4056 } else if (is_protmode(vcpu)) { 4057 /* Protected mode: apply checks for segment validity in the 4058 * following order: 4059 * - segment type check (#GP(0) may be thrown) 4060 * - usability check (#GP(0)/#SS(0)) 4061 * - limit check (#GP(0)/#SS(0)) 4062 */ 4063 if (wr) 4064 /* #GP(0) if the destination operand is located in a 4065 * read-only data segment or any code segment. 4066 */ 4067 exn = ((s.type & 0xa) == 0 || (s.type & 8)); 4068 else 4069 /* #GP(0) if the source operand is located in an 4070 * execute-only code segment 4071 */ 4072 exn = ((s.type & 0xa) == 8); 4073 if (exn) { 4074 kvm_queue_exception_e(vcpu, GP_VECTOR, 0); 4075 return 1; 4076 } 4077 /* Protected mode: #GP(0)/#SS(0) if the segment is unusable. 4078 */ 4079 exn = (s.unusable != 0); 4080 /* Protected mode: #GP(0)/#SS(0) if the memory 4081 * operand is outside the segment limit. 4082 */ 4083 exn = exn || (off + sizeof(u64) > s.limit); 4084 } 4085 if (exn) { 4086 kvm_queue_exception_e(vcpu, 4087 seg_reg == VCPU_SREG_SS ? 4088 SS_VECTOR : GP_VECTOR, 4089 0); 4090 return 1; 4091 } 4092 4093 return 0; 4094 } 4095 4096 static int nested_vmx_get_vmptr(struct kvm_vcpu *vcpu, gpa_t *vmpointer) 4097 { 4098 gva_t gva; 4099 struct x86_exception e; 4100 4101 if (get_vmx_mem_address(vcpu, vmcs_readl(EXIT_QUALIFICATION), 4102 vmcs_read32(VMX_INSTRUCTION_INFO), false, &gva)) 4103 return 1; 4104 4105 if (kvm_read_guest_virt(vcpu, gva, vmpointer, sizeof(*vmpointer), &e)) { 4106 kvm_inject_page_fault(vcpu, &e); 4107 return 1; 4108 } 4109 4110 return 0; 4111 } 4112 4113 /* 4114 * Allocate a shadow VMCS and associate it with the currently loaded 4115 * VMCS, unless such a shadow VMCS already exists. The newly allocated 4116 * VMCS is also VMCLEARed, so that it is ready for use. 4117 */ 4118 static struct vmcs *alloc_shadow_vmcs(struct kvm_vcpu *vcpu) 4119 { 4120 struct vcpu_vmx *vmx = to_vmx(vcpu); 4121 struct loaded_vmcs *loaded_vmcs = vmx->loaded_vmcs; 4122 4123 /* 4124 * We should allocate a shadow vmcs for vmcs01 only when L1 4125 * executes VMXON and free it when L1 executes VMXOFF. 4126 * As it is invalid to execute VMXON twice, we shouldn't reach 4127 * here when vmcs01 already have an allocated shadow vmcs. 4128 */ 4129 WARN_ON(loaded_vmcs == &vmx->vmcs01 && loaded_vmcs->shadow_vmcs); 4130 4131 if (!loaded_vmcs->shadow_vmcs) { 4132 loaded_vmcs->shadow_vmcs = alloc_vmcs(true); 4133 if (loaded_vmcs->shadow_vmcs) 4134 vmcs_clear(loaded_vmcs->shadow_vmcs); 4135 } 4136 return loaded_vmcs->shadow_vmcs; 4137 } 4138 4139 static int enter_vmx_operation(struct kvm_vcpu *vcpu) 4140 { 4141 struct vcpu_vmx *vmx = to_vmx(vcpu); 4142 int r; 4143 4144 r = alloc_loaded_vmcs(&vmx->nested.vmcs02); 4145 if (r < 0) 4146 goto out_vmcs02; 4147 4148 vmx->nested.cached_vmcs12 = kzalloc(VMCS12_SIZE, GFP_KERNEL); 4149 if (!vmx->nested.cached_vmcs12) 4150 goto out_cached_vmcs12; 4151 4152 vmx->nested.cached_shadow_vmcs12 = kzalloc(VMCS12_SIZE, GFP_KERNEL); 4153 if (!vmx->nested.cached_shadow_vmcs12) 4154 goto out_cached_shadow_vmcs12; 4155 4156 if (enable_shadow_vmcs && !alloc_shadow_vmcs(vcpu)) 4157 goto out_shadow_vmcs; 4158 4159 hrtimer_init(&vmx->nested.preemption_timer, CLOCK_MONOTONIC, 4160 HRTIMER_MODE_REL_PINNED); 4161 vmx->nested.preemption_timer.function = vmx_preemption_timer_fn; 4162 4163 vmx->nested.vpid02 = allocate_vpid(); 4164 4165 vmx->nested.vmcs02_initialized = false; 4166 vmx->nested.vmxon = true; 4167 4168 if (pt_mode == PT_MODE_HOST_GUEST) { 4169 vmx->pt_desc.guest.ctl = 0; 4170 pt_update_intercept_for_msr(vmx); 4171 } 4172 4173 return 0; 4174 4175 out_shadow_vmcs: 4176 kfree(vmx->nested.cached_shadow_vmcs12); 4177 4178 out_cached_shadow_vmcs12: 4179 kfree(vmx->nested.cached_vmcs12); 4180 4181 out_cached_vmcs12: 4182 free_loaded_vmcs(&vmx->nested.vmcs02); 4183 4184 out_vmcs02: 4185 return -ENOMEM; 4186 } 4187 4188 /* 4189 * Emulate the VMXON instruction. 4190 * Currently, we just remember that VMX is active, and do not save or even 4191 * inspect the argument to VMXON (the so-called "VMXON pointer") because we 4192 * do not currently need to store anything in that guest-allocated memory 4193 * region. Consequently, VMCLEAR and VMPTRLD also do not verify that the their 4194 * argument is different from the VMXON pointer (which the spec says they do). 4195 */ 4196 static int handle_vmon(struct kvm_vcpu *vcpu) 4197 { 4198 int ret; 4199 gpa_t vmptr; 4200 struct page *page; 4201 struct vcpu_vmx *vmx = to_vmx(vcpu); 4202 const u64 VMXON_NEEDED_FEATURES = FEATURE_CONTROL_LOCKED 4203 | FEATURE_CONTROL_VMXON_ENABLED_OUTSIDE_SMX; 4204 4205 /* 4206 * The Intel VMX Instruction Reference lists a bunch of bits that are 4207 * prerequisite to running VMXON, most notably cr4.VMXE must be set to 4208 * 1 (see vmx_set_cr4() for when we allow the guest to set this). 4209 * Otherwise, we should fail with #UD. But most faulting conditions 4210 * have already been checked by hardware, prior to the VM-exit for 4211 * VMXON. We do test guest cr4.VMXE because processor CR4 always has 4212 * that bit set to 1 in non-root mode. 4213 */ 4214 if (!kvm_read_cr4_bits(vcpu, X86_CR4_VMXE)) { 4215 kvm_queue_exception(vcpu, UD_VECTOR); 4216 return 1; 4217 } 4218 4219 /* CPL=0 must be checked manually. */ 4220 if (vmx_get_cpl(vcpu)) { 4221 kvm_inject_gp(vcpu, 0); 4222 return 1; 4223 } 4224 4225 if (vmx->nested.vmxon) 4226 return nested_vmx_failValid(vcpu, 4227 VMXERR_VMXON_IN_VMX_ROOT_OPERATION); 4228 4229 if ((vmx->msr_ia32_feature_control & VMXON_NEEDED_FEATURES) 4230 != VMXON_NEEDED_FEATURES) { 4231 kvm_inject_gp(vcpu, 0); 4232 return 1; 4233 } 4234 4235 if (nested_vmx_get_vmptr(vcpu, &vmptr)) 4236 return 1; 4237 4238 /* 4239 * SDM 3: 24.11.5 4240 * The first 4 bytes of VMXON region contain the supported 4241 * VMCS revision identifier 4242 * 4243 * Note - IA32_VMX_BASIC[48] will never be 1 for the nested case; 4244 * which replaces physical address width with 32 4245 */ 4246 if (!PAGE_ALIGNED(vmptr) || (vmptr >> cpuid_maxphyaddr(vcpu))) 4247 return nested_vmx_failInvalid(vcpu); 4248 4249 page = kvm_vcpu_gpa_to_page(vcpu, vmptr); 4250 if (is_error_page(page)) 4251 return nested_vmx_failInvalid(vcpu); 4252 4253 if (*(u32 *)kmap(page) != VMCS12_REVISION) { 4254 kunmap(page); 4255 kvm_release_page_clean(page); 4256 return nested_vmx_failInvalid(vcpu); 4257 } 4258 kunmap(page); 4259 kvm_release_page_clean(page); 4260 4261 vmx->nested.vmxon_ptr = vmptr; 4262 ret = enter_vmx_operation(vcpu); 4263 if (ret) 4264 return ret; 4265 4266 return nested_vmx_succeed(vcpu); 4267 } 4268 4269 static inline void nested_release_vmcs12(struct kvm_vcpu *vcpu) 4270 { 4271 struct vcpu_vmx *vmx = to_vmx(vcpu); 4272 4273 if (vmx->nested.current_vmptr == -1ull) 4274 return; 4275 4276 if (enable_shadow_vmcs) { 4277 /* copy to memory all shadowed fields in case 4278 they were modified */ 4279 copy_shadow_to_vmcs12(vmx); 4280 vmx->nested.need_vmcs12_sync = false; 4281 vmx_disable_shadow_vmcs(vmx); 4282 } 4283 vmx->nested.posted_intr_nv = -1; 4284 4285 /* Flush VMCS12 to guest memory */ 4286 kvm_vcpu_write_guest_page(vcpu, 4287 vmx->nested.current_vmptr >> PAGE_SHIFT, 4288 vmx->nested.cached_vmcs12, 0, VMCS12_SIZE); 4289 4290 kvm_mmu_free_roots(vcpu, &vcpu->arch.guest_mmu, KVM_MMU_ROOTS_ALL); 4291 4292 vmx->nested.current_vmptr = -1ull; 4293 } 4294 4295 /* Emulate the VMXOFF instruction */ 4296 static int handle_vmoff(struct kvm_vcpu *vcpu) 4297 { 4298 if (!nested_vmx_check_permission(vcpu)) 4299 return 1; 4300 free_nested(vcpu); 4301 return nested_vmx_succeed(vcpu); 4302 } 4303 4304 /* Emulate the VMCLEAR instruction */ 4305 static int handle_vmclear(struct kvm_vcpu *vcpu) 4306 { 4307 struct vcpu_vmx *vmx = to_vmx(vcpu); 4308 u32 zero = 0; 4309 gpa_t vmptr; 4310 4311 if (!nested_vmx_check_permission(vcpu)) 4312 return 1; 4313 4314 if (nested_vmx_get_vmptr(vcpu, &vmptr)) 4315 return 1; 4316 4317 if (!PAGE_ALIGNED(vmptr) || (vmptr >> cpuid_maxphyaddr(vcpu))) 4318 return nested_vmx_failValid(vcpu, 4319 VMXERR_VMCLEAR_INVALID_ADDRESS); 4320 4321 if (vmptr == vmx->nested.vmxon_ptr) 4322 return nested_vmx_failValid(vcpu, 4323 VMXERR_VMCLEAR_VMXON_POINTER); 4324 4325 if (vmx->nested.hv_evmcs_page) { 4326 if (vmptr == vmx->nested.hv_evmcs_vmptr) 4327 nested_release_evmcs(vcpu); 4328 } else { 4329 if (vmptr == vmx->nested.current_vmptr) 4330 nested_release_vmcs12(vcpu); 4331 4332 kvm_vcpu_write_guest(vcpu, 4333 vmptr + offsetof(struct vmcs12, 4334 launch_state), 4335 &zero, sizeof(zero)); 4336 } 4337 4338 return nested_vmx_succeed(vcpu); 4339 } 4340 4341 static int nested_vmx_run(struct kvm_vcpu *vcpu, bool launch); 4342 4343 /* Emulate the VMLAUNCH instruction */ 4344 static int handle_vmlaunch(struct kvm_vcpu *vcpu) 4345 { 4346 return nested_vmx_run(vcpu, true); 4347 } 4348 4349 /* Emulate the VMRESUME instruction */ 4350 static int handle_vmresume(struct kvm_vcpu *vcpu) 4351 { 4352 4353 return nested_vmx_run(vcpu, false); 4354 } 4355 4356 static int handle_vmread(struct kvm_vcpu *vcpu) 4357 { 4358 unsigned long field; 4359 u64 field_value; 4360 unsigned long exit_qualification = vmcs_readl(EXIT_QUALIFICATION); 4361 u32 vmx_instruction_info = vmcs_read32(VMX_INSTRUCTION_INFO); 4362 gva_t gva = 0; 4363 struct vmcs12 *vmcs12; 4364 4365 if (!nested_vmx_check_permission(vcpu)) 4366 return 1; 4367 4368 if (to_vmx(vcpu)->nested.current_vmptr == -1ull) 4369 return nested_vmx_failInvalid(vcpu); 4370 4371 if (!is_guest_mode(vcpu)) 4372 vmcs12 = get_vmcs12(vcpu); 4373 else { 4374 /* 4375 * When vmcs->vmcs_link_pointer is -1ull, any VMREAD 4376 * to shadowed-field sets the ALU flags for VMfailInvalid. 4377 */ 4378 if (get_vmcs12(vcpu)->vmcs_link_pointer == -1ull) 4379 return nested_vmx_failInvalid(vcpu); 4380 vmcs12 = get_shadow_vmcs12(vcpu); 4381 } 4382 4383 /* Decode instruction info and find the field to read */ 4384 field = kvm_register_readl(vcpu, (((vmx_instruction_info) >> 28) & 0xf)); 4385 /* Read the field, zero-extended to a u64 field_value */ 4386 if (vmcs12_read_any(vmcs12, field, &field_value) < 0) 4387 return nested_vmx_failValid(vcpu, 4388 VMXERR_UNSUPPORTED_VMCS_COMPONENT); 4389 4390 /* 4391 * Now copy part of this value to register or memory, as requested. 4392 * Note that the number of bits actually copied is 32 or 64 depending 4393 * on the guest's mode (32 or 64 bit), not on the given field's length. 4394 */ 4395 if (vmx_instruction_info & (1u << 10)) { 4396 kvm_register_writel(vcpu, (((vmx_instruction_info) >> 3) & 0xf), 4397 field_value); 4398 } else { 4399 if (get_vmx_mem_address(vcpu, exit_qualification, 4400 vmx_instruction_info, true, &gva)) 4401 return 1; 4402 /* _system ok, nested_vmx_check_permission has verified cpl=0 */ 4403 kvm_write_guest_virt_system(vcpu, gva, &field_value, 4404 (is_long_mode(vcpu) ? 8 : 4), NULL); 4405 } 4406 4407 return nested_vmx_succeed(vcpu); 4408 } 4409 4410 4411 static int handle_vmwrite(struct kvm_vcpu *vcpu) 4412 { 4413 unsigned long field; 4414 gva_t gva; 4415 struct vcpu_vmx *vmx = to_vmx(vcpu); 4416 unsigned long exit_qualification = vmcs_readl(EXIT_QUALIFICATION); 4417 u32 vmx_instruction_info = vmcs_read32(VMX_INSTRUCTION_INFO); 4418 4419 /* The value to write might be 32 or 64 bits, depending on L1's long 4420 * mode, and eventually we need to write that into a field of several 4421 * possible lengths. The code below first zero-extends the value to 64 4422 * bit (field_value), and then copies only the appropriate number of 4423 * bits into the vmcs12 field. 4424 */ 4425 u64 field_value = 0; 4426 struct x86_exception e; 4427 struct vmcs12 *vmcs12; 4428 4429 if (!nested_vmx_check_permission(vcpu)) 4430 return 1; 4431 4432 if (vmx->nested.current_vmptr == -1ull) 4433 return nested_vmx_failInvalid(vcpu); 4434 4435 if (vmx_instruction_info & (1u << 10)) 4436 field_value = kvm_register_readl(vcpu, 4437 (((vmx_instruction_info) >> 3) & 0xf)); 4438 else { 4439 if (get_vmx_mem_address(vcpu, exit_qualification, 4440 vmx_instruction_info, false, &gva)) 4441 return 1; 4442 if (kvm_read_guest_virt(vcpu, gva, &field_value, 4443 (is_64_bit_mode(vcpu) ? 8 : 4), &e)) { 4444 kvm_inject_page_fault(vcpu, &e); 4445 return 1; 4446 } 4447 } 4448 4449 4450 field = kvm_register_readl(vcpu, (((vmx_instruction_info) >> 28) & 0xf)); 4451 /* 4452 * If the vCPU supports "VMWRITE to any supported field in the 4453 * VMCS," then the "read-only" fields are actually read/write. 4454 */ 4455 if (vmcs_field_readonly(field) && 4456 !nested_cpu_has_vmwrite_any_field(vcpu)) 4457 return nested_vmx_failValid(vcpu, 4458 VMXERR_VMWRITE_READ_ONLY_VMCS_COMPONENT); 4459 4460 if (!is_guest_mode(vcpu)) 4461 vmcs12 = get_vmcs12(vcpu); 4462 else { 4463 /* 4464 * When vmcs->vmcs_link_pointer is -1ull, any VMWRITE 4465 * to shadowed-field sets the ALU flags for VMfailInvalid. 4466 */ 4467 if (get_vmcs12(vcpu)->vmcs_link_pointer == -1ull) 4468 return nested_vmx_failInvalid(vcpu); 4469 vmcs12 = get_shadow_vmcs12(vcpu); 4470 } 4471 4472 if (vmcs12_write_any(vmcs12, field, field_value) < 0) 4473 return nested_vmx_failValid(vcpu, 4474 VMXERR_UNSUPPORTED_VMCS_COMPONENT); 4475 4476 /* 4477 * Do not track vmcs12 dirty-state if in guest-mode 4478 * as we actually dirty shadow vmcs12 instead of vmcs12. 4479 */ 4480 if (!is_guest_mode(vcpu)) { 4481 switch (field) { 4482 #define SHADOW_FIELD_RW(x) case x: 4483 #include "vmcs_shadow_fields.h" 4484 /* 4485 * The fields that can be updated by L1 without a vmexit are 4486 * always updated in the vmcs02, the others go down the slow 4487 * path of prepare_vmcs02. 4488 */ 4489 break; 4490 default: 4491 vmx->nested.dirty_vmcs12 = true; 4492 break; 4493 } 4494 } 4495 4496 return nested_vmx_succeed(vcpu); 4497 } 4498 4499 static void set_current_vmptr(struct vcpu_vmx *vmx, gpa_t vmptr) 4500 { 4501 vmx->nested.current_vmptr = vmptr; 4502 if (enable_shadow_vmcs) { 4503 vmcs_set_bits(SECONDARY_VM_EXEC_CONTROL, 4504 SECONDARY_EXEC_SHADOW_VMCS); 4505 vmcs_write64(VMCS_LINK_POINTER, 4506 __pa(vmx->vmcs01.shadow_vmcs)); 4507 vmx->nested.need_vmcs12_sync = true; 4508 } 4509 vmx->nested.dirty_vmcs12 = true; 4510 } 4511 4512 /* Emulate the VMPTRLD instruction */ 4513 static int handle_vmptrld(struct kvm_vcpu *vcpu) 4514 { 4515 struct vcpu_vmx *vmx = to_vmx(vcpu); 4516 gpa_t vmptr; 4517 4518 if (!nested_vmx_check_permission(vcpu)) 4519 return 1; 4520 4521 if (nested_vmx_get_vmptr(vcpu, &vmptr)) 4522 return 1; 4523 4524 if (!PAGE_ALIGNED(vmptr) || (vmptr >> cpuid_maxphyaddr(vcpu))) 4525 return nested_vmx_failValid(vcpu, 4526 VMXERR_VMPTRLD_INVALID_ADDRESS); 4527 4528 if (vmptr == vmx->nested.vmxon_ptr) 4529 return nested_vmx_failValid(vcpu, 4530 VMXERR_VMPTRLD_VMXON_POINTER); 4531 4532 /* Forbid normal VMPTRLD if Enlightened version was used */ 4533 if (vmx->nested.hv_evmcs) 4534 return 1; 4535 4536 if (vmx->nested.current_vmptr != vmptr) { 4537 struct vmcs12 *new_vmcs12; 4538 struct page *page; 4539 4540 page = kvm_vcpu_gpa_to_page(vcpu, vmptr); 4541 if (is_error_page(page)) { 4542 /* 4543 * Reads from an unbacked page return all 1s, 4544 * which means that the 32 bits located at the 4545 * given physical address won't match the required 4546 * VMCS12_REVISION identifier. 4547 */ 4548 return nested_vmx_failValid(vcpu, 4549 VMXERR_VMPTRLD_INCORRECT_VMCS_REVISION_ID); 4550 } 4551 new_vmcs12 = kmap(page); 4552 if (new_vmcs12->hdr.revision_id != VMCS12_REVISION || 4553 (new_vmcs12->hdr.shadow_vmcs && 4554 !nested_cpu_has_vmx_shadow_vmcs(vcpu))) { 4555 kunmap(page); 4556 kvm_release_page_clean(page); 4557 return nested_vmx_failValid(vcpu, 4558 VMXERR_VMPTRLD_INCORRECT_VMCS_REVISION_ID); 4559 } 4560 4561 nested_release_vmcs12(vcpu); 4562 4563 /* 4564 * Load VMCS12 from guest memory since it is not already 4565 * cached. 4566 */ 4567 memcpy(vmx->nested.cached_vmcs12, new_vmcs12, VMCS12_SIZE); 4568 kunmap(page); 4569 kvm_release_page_clean(page); 4570 4571 set_current_vmptr(vmx, vmptr); 4572 } 4573 4574 return nested_vmx_succeed(vcpu); 4575 } 4576 4577 /* Emulate the VMPTRST instruction */ 4578 static int handle_vmptrst(struct kvm_vcpu *vcpu) 4579 { 4580 unsigned long exit_qual = vmcs_readl(EXIT_QUALIFICATION); 4581 u32 instr_info = vmcs_read32(VMX_INSTRUCTION_INFO); 4582 gpa_t current_vmptr = to_vmx(vcpu)->nested.current_vmptr; 4583 struct x86_exception e; 4584 gva_t gva; 4585 4586 if (!nested_vmx_check_permission(vcpu)) 4587 return 1; 4588 4589 if (unlikely(to_vmx(vcpu)->nested.hv_evmcs)) 4590 return 1; 4591 4592 if (get_vmx_mem_address(vcpu, exit_qual, instr_info, true, &gva)) 4593 return 1; 4594 /* *_system ok, nested_vmx_check_permission has verified cpl=0 */ 4595 if (kvm_write_guest_virt_system(vcpu, gva, (void *)¤t_vmptr, 4596 sizeof(gpa_t), &e)) { 4597 kvm_inject_page_fault(vcpu, &e); 4598 return 1; 4599 } 4600 return nested_vmx_succeed(vcpu); 4601 } 4602 4603 /* Emulate the INVEPT instruction */ 4604 static int handle_invept(struct kvm_vcpu *vcpu) 4605 { 4606 struct vcpu_vmx *vmx = to_vmx(vcpu); 4607 u32 vmx_instruction_info, types; 4608 unsigned long type; 4609 gva_t gva; 4610 struct x86_exception e; 4611 struct { 4612 u64 eptp, gpa; 4613 } operand; 4614 4615 if (!(vmx->nested.msrs.secondary_ctls_high & 4616 SECONDARY_EXEC_ENABLE_EPT) || 4617 !(vmx->nested.msrs.ept_caps & VMX_EPT_INVEPT_BIT)) { 4618 kvm_queue_exception(vcpu, UD_VECTOR); 4619 return 1; 4620 } 4621 4622 if (!nested_vmx_check_permission(vcpu)) 4623 return 1; 4624 4625 vmx_instruction_info = vmcs_read32(VMX_INSTRUCTION_INFO); 4626 type = kvm_register_readl(vcpu, (vmx_instruction_info >> 28) & 0xf); 4627 4628 types = (vmx->nested.msrs.ept_caps >> VMX_EPT_EXTENT_SHIFT) & 6; 4629 4630 if (type >= 32 || !(types & (1 << type))) 4631 return nested_vmx_failValid(vcpu, 4632 VMXERR_INVALID_OPERAND_TO_INVEPT_INVVPID); 4633 4634 /* According to the Intel VMX instruction reference, the memory 4635 * operand is read even if it isn't needed (e.g., for type==global) 4636 */ 4637 if (get_vmx_mem_address(vcpu, vmcs_readl(EXIT_QUALIFICATION), 4638 vmx_instruction_info, false, &gva)) 4639 return 1; 4640 if (kvm_read_guest_virt(vcpu, gva, &operand, sizeof(operand), &e)) { 4641 kvm_inject_page_fault(vcpu, &e); 4642 return 1; 4643 } 4644 4645 switch (type) { 4646 case VMX_EPT_EXTENT_GLOBAL: 4647 /* 4648 * TODO: track mappings and invalidate 4649 * single context requests appropriately 4650 */ 4651 case VMX_EPT_EXTENT_CONTEXT: 4652 kvm_mmu_sync_roots(vcpu); 4653 kvm_make_request(KVM_REQ_TLB_FLUSH, vcpu); 4654 break; 4655 default: 4656 BUG_ON(1); 4657 break; 4658 } 4659 4660 return nested_vmx_succeed(vcpu); 4661 } 4662 4663 static int handle_invvpid(struct kvm_vcpu *vcpu) 4664 { 4665 struct vcpu_vmx *vmx = to_vmx(vcpu); 4666 u32 vmx_instruction_info; 4667 unsigned long type, types; 4668 gva_t gva; 4669 struct x86_exception e; 4670 struct { 4671 u64 vpid; 4672 u64 gla; 4673 } operand; 4674 u16 vpid02; 4675 4676 if (!(vmx->nested.msrs.secondary_ctls_high & 4677 SECONDARY_EXEC_ENABLE_VPID) || 4678 !(vmx->nested.msrs.vpid_caps & VMX_VPID_INVVPID_BIT)) { 4679 kvm_queue_exception(vcpu, UD_VECTOR); 4680 return 1; 4681 } 4682 4683 if (!nested_vmx_check_permission(vcpu)) 4684 return 1; 4685 4686 vmx_instruction_info = vmcs_read32(VMX_INSTRUCTION_INFO); 4687 type = kvm_register_readl(vcpu, (vmx_instruction_info >> 28) & 0xf); 4688 4689 types = (vmx->nested.msrs.vpid_caps & 4690 VMX_VPID_EXTENT_SUPPORTED_MASK) >> 8; 4691 4692 if (type >= 32 || !(types & (1 << type))) 4693 return nested_vmx_failValid(vcpu, 4694 VMXERR_INVALID_OPERAND_TO_INVEPT_INVVPID); 4695 4696 /* according to the intel vmx instruction reference, the memory 4697 * operand is read even if it isn't needed (e.g., for type==global) 4698 */ 4699 if (get_vmx_mem_address(vcpu, vmcs_readl(EXIT_QUALIFICATION), 4700 vmx_instruction_info, false, &gva)) 4701 return 1; 4702 if (kvm_read_guest_virt(vcpu, gva, &operand, sizeof(operand), &e)) { 4703 kvm_inject_page_fault(vcpu, &e); 4704 return 1; 4705 } 4706 if (operand.vpid >> 16) 4707 return nested_vmx_failValid(vcpu, 4708 VMXERR_INVALID_OPERAND_TO_INVEPT_INVVPID); 4709 4710 vpid02 = nested_get_vpid02(vcpu); 4711 switch (type) { 4712 case VMX_VPID_EXTENT_INDIVIDUAL_ADDR: 4713 if (!operand.vpid || 4714 is_noncanonical_address(operand.gla, vcpu)) 4715 return nested_vmx_failValid(vcpu, 4716 VMXERR_INVALID_OPERAND_TO_INVEPT_INVVPID); 4717 if (cpu_has_vmx_invvpid_individual_addr()) { 4718 __invvpid(VMX_VPID_EXTENT_INDIVIDUAL_ADDR, 4719 vpid02, operand.gla); 4720 } else 4721 __vmx_flush_tlb(vcpu, vpid02, false); 4722 break; 4723 case VMX_VPID_EXTENT_SINGLE_CONTEXT: 4724 case VMX_VPID_EXTENT_SINGLE_NON_GLOBAL: 4725 if (!operand.vpid) 4726 return nested_vmx_failValid(vcpu, 4727 VMXERR_INVALID_OPERAND_TO_INVEPT_INVVPID); 4728 __vmx_flush_tlb(vcpu, vpid02, false); 4729 break; 4730 case VMX_VPID_EXTENT_ALL_CONTEXT: 4731 __vmx_flush_tlb(vcpu, vpid02, false); 4732 break; 4733 default: 4734 WARN_ON_ONCE(1); 4735 return kvm_skip_emulated_instruction(vcpu); 4736 } 4737 4738 return nested_vmx_succeed(vcpu); 4739 } 4740 4741 static int nested_vmx_eptp_switching(struct kvm_vcpu *vcpu, 4742 struct vmcs12 *vmcs12) 4743 { 4744 u32 index = vcpu->arch.regs[VCPU_REGS_RCX]; 4745 u64 address; 4746 bool accessed_dirty; 4747 struct kvm_mmu *mmu = vcpu->arch.walk_mmu; 4748 4749 if (!nested_cpu_has_eptp_switching(vmcs12) || 4750 !nested_cpu_has_ept(vmcs12)) 4751 return 1; 4752 4753 if (index >= VMFUNC_EPTP_ENTRIES) 4754 return 1; 4755 4756 4757 if (kvm_vcpu_read_guest_page(vcpu, vmcs12->eptp_list_address >> PAGE_SHIFT, 4758 &address, index * 8, 8)) 4759 return 1; 4760 4761 accessed_dirty = !!(address & VMX_EPTP_AD_ENABLE_BIT); 4762 4763 /* 4764 * If the (L2) guest does a vmfunc to the currently 4765 * active ept pointer, we don't have to do anything else 4766 */ 4767 if (vmcs12->ept_pointer != address) { 4768 if (!valid_ept_address(vcpu, address)) 4769 return 1; 4770 4771 kvm_mmu_unload(vcpu); 4772 mmu->ept_ad = accessed_dirty; 4773 mmu->mmu_role.base.ad_disabled = !accessed_dirty; 4774 vmcs12->ept_pointer = address; 4775 /* 4776 * TODO: Check what's the correct approach in case 4777 * mmu reload fails. Currently, we just let the next 4778 * reload potentially fail 4779 */ 4780 kvm_mmu_reload(vcpu); 4781 } 4782 4783 return 0; 4784 } 4785 4786 static int handle_vmfunc(struct kvm_vcpu *vcpu) 4787 { 4788 struct vcpu_vmx *vmx = to_vmx(vcpu); 4789 struct vmcs12 *vmcs12; 4790 u32 function = vcpu->arch.regs[VCPU_REGS_RAX]; 4791 4792 /* 4793 * VMFUNC is only supported for nested guests, but we always enable the 4794 * secondary control for simplicity; for non-nested mode, fake that we 4795 * didn't by injecting #UD. 4796 */ 4797 if (!is_guest_mode(vcpu)) { 4798 kvm_queue_exception(vcpu, UD_VECTOR); 4799 return 1; 4800 } 4801 4802 vmcs12 = get_vmcs12(vcpu); 4803 if ((vmcs12->vm_function_control & (1 << function)) == 0) 4804 goto fail; 4805 4806 switch (function) { 4807 case 0: 4808 if (nested_vmx_eptp_switching(vcpu, vmcs12)) 4809 goto fail; 4810 break; 4811 default: 4812 goto fail; 4813 } 4814 return kvm_skip_emulated_instruction(vcpu); 4815 4816 fail: 4817 nested_vmx_vmexit(vcpu, vmx->exit_reason, 4818 vmcs_read32(VM_EXIT_INTR_INFO), 4819 vmcs_readl(EXIT_QUALIFICATION)); 4820 return 1; 4821 } 4822 4823 4824 static bool nested_vmx_exit_handled_io(struct kvm_vcpu *vcpu, 4825 struct vmcs12 *vmcs12) 4826 { 4827 unsigned long exit_qualification; 4828 gpa_t bitmap, last_bitmap; 4829 unsigned int port; 4830 int size; 4831 u8 b; 4832 4833 if (!nested_cpu_has(vmcs12, CPU_BASED_USE_IO_BITMAPS)) 4834 return nested_cpu_has(vmcs12, CPU_BASED_UNCOND_IO_EXITING); 4835 4836 exit_qualification = vmcs_readl(EXIT_QUALIFICATION); 4837 4838 port = exit_qualification >> 16; 4839 size = (exit_qualification & 7) + 1; 4840 4841 last_bitmap = (gpa_t)-1; 4842 b = -1; 4843 4844 while (size > 0) { 4845 if (port < 0x8000) 4846 bitmap = vmcs12->io_bitmap_a; 4847 else if (port < 0x10000) 4848 bitmap = vmcs12->io_bitmap_b; 4849 else 4850 return true; 4851 bitmap += (port & 0x7fff) / 8; 4852 4853 if (last_bitmap != bitmap) 4854 if (kvm_vcpu_read_guest(vcpu, bitmap, &b, 1)) 4855 return true; 4856 if (b & (1 << (port & 7))) 4857 return true; 4858 4859 port++; 4860 size--; 4861 last_bitmap = bitmap; 4862 } 4863 4864 return false; 4865 } 4866 4867 /* 4868 * Return 1 if we should exit from L2 to L1 to handle an MSR access access, 4869 * rather than handle it ourselves in L0. I.e., check whether L1 expressed 4870 * disinterest in the current event (read or write a specific MSR) by using an 4871 * MSR bitmap. This may be the case even when L0 doesn't use MSR bitmaps. 4872 */ 4873 static bool nested_vmx_exit_handled_msr(struct kvm_vcpu *vcpu, 4874 struct vmcs12 *vmcs12, u32 exit_reason) 4875 { 4876 u32 msr_index = vcpu->arch.regs[VCPU_REGS_RCX]; 4877 gpa_t bitmap; 4878 4879 if (!nested_cpu_has(vmcs12, CPU_BASED_USE_MSR_BITMAPS)) 4880 return true; 4881 4882 /* 4883 * The MSR_BITMAP page is divided into four 1024-byte bitmaps, 4884 * for the four combinations of read/write and low/high MSR numbers. 4885 * First we need to figure out which of the four to use: 4886 */ 4887 bitmap = vmcs12->msr_bitmap; 4888 if (exit_reason == EXIT_REASON_MSR_WRITE) 4889 bitmap += 2048; 4890 if (msr_index >= 0xc0000000) { 4891 msr_index -= 0xc0000000; 4892 bitmap += 1024; 4893 } 4894 4895 /* Then read the msr_index'th bit from this bitmap: */ 4896 if (msr_index < 1024*8) { 4897 unsigned char b; 4898 if (kvm_vcpu_read_guest(vcpu, bitmap + msr_index/8, &b, 1)) 4899 return true; 4900 return 1 & (b >> (msr_index & 7)); 4901 } else 4902 return true; /* let L1 handle the wrong parameter */ 4903 } 4904 4905 /* 4906 * Return 1 if we should exit from L2 to L1 to handle a CR access exit, 4907 * rather than handle it ourselves in L0. I.e., check if L1 wanted to 4908 * intercept (via guest_host_mask etc.) the current event. 4909 */ 4910 static bool nested_vmx_exit_handled_cr(struct kvm_vcpu *vcpu, 4911 struct vmcs12 *vmcs12) 4912 { 4913 unsigned long exit_qualification = vmcs_readl(EXIT_QUALIFICATION); 4914 int cr = exit_qualification & 15; 4915 int reg; 4916 unsigned long val; 4917 4918 switch ((exit_qualification >> 4) & 3) { 4919 case 0: /* mov to cr */ 4920 reg = (exit_qualification >> 8) & 15; 4921 val = kvm_register_readl(vcpu, reg); 4922 switch (cr) { 4923 case 0: 4924 if (vmcs12->cr0_guest_host_mask & 4925 (val ^ vmcs12->cr0_read_shadow)) 4926 return true; 4927 break; 4928 case 3: 4929 if ((vmcs12->cr3_target_count >= 1 && 4930 vmcs12->cr3_target_value0 == val) || 4931 (vmcs12->cr3_target_count >= 2 && 4932 vmcs12->cr3_target_value1 == val) || 4933 (vmcs12->cr3_target_count >= 3 && 4934 vmcs12->cr3_target_value2 == val) || 4935 (vmcs12->cr3_target_count >= 4 && 4936 vmcs12->cr3_target_value3 == val)) 4937 return false; 4938 if (nested_cpu_has(vmcs12, CPU_BASED_CR3_LOAD_EXITING)) 4939 return true; 4940 break; 4941 case 4: 4942 if (vmcs12->cr4_guest_host_mask & 4943 (vmcs12->cr4_read_shadow ^ val)) 4944 return true; 4945 break; 4946 case 8: 4947 if (nested_cpu_has(vmcs12, CPU_BASED_CR8_LOAD_EXITING)) 4948 return true; 4949 break; 4950 } 4951 break; 4952 case 2: /* clts */ 4953 if ((vmcs12->cr0_guest_host_mask & X86_CR0_TS) && 4954 (vmcs12->cr0_read_shadow & X86_CR0_TS)) 4955 return true; 4956 break; 4957 case 1: /* mov from cr */ 4958 switch (cr) { 4959 case 3: 4960 if (vmcs12->cpu_based_vm_exec_control & 4961 CPU_BASED_CR3_STORE_EXITING) 4962 return true; 4963 break; 4964 case 8: 4965 if (vmcs12->cpu_based_vm_exec_control & 4966 CPU_BASED_CR8_STORE_EXITING) 4967 return true; 4968 break; 4969 } 4970 break; 4971 case 3: /* lmsw */ 4972 /* 4973 * lmsw can change bits 1..3 of cr0, and only set bit 0 of 4974 * cr0. Other attempted changes are ignored, with no exit. 4975 */ 4976 val = (exit_qualification >> LMSW_SOURCE_DATA_SHIFT) & 0x0f; 4977 if (vmcs12->cr0_guest_host_mask & 0xe & 4978 (val ^ vmcs12->cr0_read_shadow)) 4979 return true; 4980 if ((vmcs12->cr0_guest_host_mask & 0x1) && 4981 !(vmcs12->cr0_read_shadow & 0x1) && 4982 (val & 0x1)) 4983 return true; 4984 break; 4985 } 4986 return false; 4987 } 4988 4989 static bool nested_vmx_exit_handled_vmcs_access(struct kvm_vcpu *vcpu, 4990 struct vmcs12 *vmcs12, gpa_t bitmap) 4991 { 4992 u32 vmx_instruction_info; 4993 unsigned long field; 4994 u8 b; 4995 4996 if (!nested_cpu_has_shadow_vmcs(vmcs12)) 4997 return true; 4998 4999 /* Decode instruction info and find the field to access */ 5000 vmx_instruction_info = vmcs_read32(VMX_INSTRUCTION_INFO); 5001 field = kvm_register_read(vcpu, (((vmx_instruction_info) >> 28) & 0xf)); 5002 5003 /* Out-of-range fields always cause a VM exit from L2 to L1 */ 5004 if (field >> 15) 5005 return true; 5006 5007 if (kvm_vcpu_read_guest(vcpu, bitmap + field/8, &b, 1)) 5008 return true; 5009 5010 return 1 & (b >> (field & 7)); 5011 } 5012 5013 /* 5014 * Return 1 if we should exit from L2 to L1 to handle an exit, or 0 if we 5015 * should handle it ourselves in L0 (and then continue L2). Only call this 5016 * when in is_guest_mode (L2). 5017 */ 5018 bool nested_vmx_exit_reflected(struct kvm_vcpu *vcpu, u32 exit_reason) 5019 { 5020 u32 intr_info = vmcs_read32(VM_EXIT_INTR_INFO); 5021 struct vcpu_vmx *vmx = to_vmx(vcpu); 5022 struct vmcs12 *vmcs12 = get_vmcs12(vcpu); 5023 5024 if (vmx->nested.nested_run_pending) 5025 return false; 5026 5027 if (unlikely(vmx->fail)) { 5028 pr_info_ratelimited("%s failed vm entry %x\n", __func__, 5029 vmcs_read32(VM_INSTRUCTION_ERROR)); 5030 return true; 5031 } 5032 5033 /* 5034 * The host physical addresses of some pages of guest memory 5035 * are loaded into the vmcs02 (e.g. vmcs12's Virtual APIC 5036 * Page). The CPU may write to these pages via their host 5037 * physical address while L2 is running, bypassing any 5038 * address-translation-based dirty tracking (e.g. EPT write 5039 * protection). 5040 * 5041 * Mark them dirty on every exit from L2 to prevent them from 5042 * getting out of sync with dirty tracking. 5043 */ 5044 nested_mark_vmcs12_pages_dirty(vcpu); 5045 5046 trace_kvm_nested_vmexit(kvm_rip_read(vcpu), exit_reason, 5047 vmcs_readl(EXIT_QUALIFICATION), 5048 vmx->idt_vectoring_info, 5049 intr_info, 5050 vmcs_read32(VM_EXIT_INTR_ERROR_CODE), 5051 KVM_ISA_VMX); 5052 5053 switch (exit_reason) { 5054 case EXIT_REASON_EXCEPTION_NMI: 5055 if (is_nmi(intr_info)) 5056 return false; 5057 else if (is_page_fault(intr_info)) 5058 return !vmx->vcpu.arch.apf.host_apf_reason && enable_ept; 5059 else if (is_debug(intr_info) && 5060 vcpu->guest_debug & 5061 (KVM_GUESTDBG_SINGLESTEP | KVM_GUESTDBG_USE_HW_BP)) 5062 return false; 5063 else if (is_breakpoint(intr_info) && 5064 vcpu->guest_debug & KVM_GUESTDBG_USE_SW_BP) 5065 return false; 5066 return vmcs12->exception_bitmap & 5067 (1u << (intr_info & INTR_INFO_VECTOR_MASK)); 5068 case EXIT_REASON_EXTERNAL_INTERRUPT: 5069 return false; 5070 case EXIT_REASON_TRIPLE_FAULT: 5071 return true; 5072 case EXIT_REASON_PENDING_INTERRUPT: 5073 return nested_cpu_has(vmcs12, CPU_BASED_VIRTUAL_INTR_PENDING); 5074 case EXIT_REASON_NMI_WINDOW: 5075 return nested_cpu_has(vmcs12, CPU_BASED_VIRTUAL_NMI_PENDING); 5076 case EXIT_REASON_TASK_SWITCH: 5077 return true; 5078 case EXIT_REASON_CPUID: 5079 return true; 5080 case EXIT_REASON_HLT: 5081 return nested_cpu_has(vmcs12, CPU_BASED_HLT_EXITING); 5082 case EXIT_REASON_INVD: 5083 return true; 5084 case EXIT_REASON_INVLPG: 5085 return nested_cpu_has(vmcs12, CPU_BASED_INVLPG_EXITING); 5086 case EXIT_REASON_RDPMC: 5087 return nested_cpu_has(vmcs12, CPU_BASED_RDPMC_EXITING); 5088 case EXIT_REASON_RDRAND: 5089 return nested_cpu_has2(vmcs12, SECONDARY_EXEC_RDRAND_EXITING); 5090 case EXIT_REASON_RDSEED: 5091 return nested_cpu_has2(vmcs12, SECONDARY_EXEC_RDSEED_EXITING); 5092 case EXIT_REASON_RDTSC: case EXIT_REASON_RDTSCP: 5093 return nested_cpu_has(vmcs12, CPU_BASED_RDTSC_EXITING); 5094 case EXIT_REASON_VMREAD: 5095 return nested_vmx_exit_handled_vmcs_access(vcpu, vmcs12, 5096 vmcs12->vmread_bitmap); 5097 case EXIT_REASON_VMWRITE: 5098 return nested_vmx_exit_handled_vmcs_access(vcpu, vmcs12, 5099 vmcs12->vmwrite_bitmap); 5100 case EXIT_REASON_VMCALL: case EXIT_REASON_VMCLEAR: 5101 case EXIT_REASON_VMLAUNCH: case EXIT_REASON_VMPTRLD: 5102 case EXIT_REASON_VMPTRST: case EXIT_REASON_VMRESUME: 5103 case EXIT_REASON_VMOFF: case EXIT_REASON_VMON: 5104 case EXIT_REASON_INVEPT: case EXIT_REASON_INVVPID: 5105 /* 5106 * VMX instructions trap unconditionally. This allows L1 to 5107 * emulate them for its L2 guest, i.e., allows 3-level nesting! 5108 */ 5109 return true; 5110 case EXIT_REASON_CR_ACCESS: 5111 return nested_vmx_exit_handled_cr(vcpu, vmcs12); 5112 case EXIT_REASON_DR_ACCESS: 5113 return nested_cpu_has(vmcs12, CPU_BASED_MOV_DR_EXITING); 5114 case EXIT_REASON_IO_INSTRUCTION: 5115 return nested_vmx_exit_handled_io(vcpu, vmcs12); 5116 case EXIT_REASON_GDTR_IDTR: case EXIT_REASON_LDTR_TR: 5117 return nested_cpu_has2(vmcs12, SECONDARY_EXEC_DESC); 5118 case EXIT_REASON_MSR_READ: 5119 case EXIT_REASON_MSR_WRITE: 5120 return nested_vmx_exit_handled_msr(vcpu, vmcs12, exit_reason); 5121 case EXIT_REASON_INVALID_STATE: 5122 return true; 5123 case EXIT_REASON_MWAIT_INSTRUCTION: 5124 return nested_cpu_has(vmcs12, CPU_BASED_MWAIT_EXITING); 5125 case EXIT_REASON_MONITOR_TRAP_FLAG: 5126 return nested_cpu_has(vmcs12, CPU_BASED_MONITOR_TRAP_FLAG); 5127 case EXIT_REASON_MONITOR_INSTRUCTION: 5128 return nested_cpu_has(vmcs12, CPU_BASED_MONITOR_EXITING); 5129 case EXIT_REASON_PAUSE_INSTRUCTION: 5130 return nested_cpu_has(vmcs12, CPU_BASED_PAUSE_EXITING) || 5131 nested_cpu_has2(vmcs12, 5132 SECONDARY_EXEC_PAUSE_LOOP_EXITING); 5133 case EXIT_REASON_MCE_DURING_VMENTRY: 5134 return false; 5135 case EXIT_REASON_TPR_BELOW_THRESHOLD: 5136 return nested_cpu_has(vmcs12, CPU_BASED_TPR_SHADOW); 5137 case EXIT_REASON_APIC_ACCESS: 5138 case EXIT_REASON_APIC_WRITE: 5139 case EXIT_REASON_EOI_INDUCED: 5140 /* 5141 * The controls for "virtualize APIC accesses," "APIC- 5142 * register virtualization," and "virtual-interrupt 5143 * delivery" only come from vmcs12. 5144 */ 5145 return true; 5146 case EXIT_REASON_EPT_VIOLATION: 5147 /* 5148 * L0 always deals with the EPT violation. If nested EPT is 5149 * used, and the nested mmu code discovers that the address is 5150 * missing in the guest EPT table (EPT12), the EPT violation 5151 * will be injected with nested_ept_inject_page_fault() 5152 */ 5153 return false; 5154 case EXIT_REASON_EPT_MISCONFIG: 5155 /* 5156 * L2 never uses directly L1's EPT, but rather L0's own EPT 5157 * table (shadow on EPT) or a merged EPT table that L0 built 5158 * (EPT on EPT). So any problems with the structure of the 5159 * table is L0's fault. 5160 */ 5161 return false; 5162 case EXIT_REASON_INVPCID: 5163 return 5164 nested_cpu_has2(vmcs12, SECONDARY_EXEC_ENABLE_INVPCID) && 5165 nested_cpu_has(vmcs12, CPU_BASED_INVLPG_EXITING); 5166 case EXIT_REASON_WBINVD: 5167 return nested_cpu_has2(vmcs12, SECONDARY_EXEC_WBINVD_EXITING); 5168 case EXIT_REASON_XSETBV: 5169 return true; 5170 case EXIT_REASON_XSAVES: case EXIT_REASON_XRSTORS: 5171 /* 5172 * This should never happen, since it is not possible to 5173 * set XSS to a non-zero value---neither in L1 nor in L2. 5174 * If if it were, XSS would have to be checked against 5175 * the XSS exit bitmap in vmcs12. 5176 */ 5177 return nested_cpu_has2(vmcs12, SECONDARY_EXEC_XSAVES); 5178 case EXIT_REASON_PREEMPTION_TIMER: 5179 return false; 5180 case EXIT_REASON_PML_FULL: 5181 /* We emulate PML support to L1. */ 5182 return false; 5183 case EXIT_REASON_VMFUNC: 5184 /* VM functions are emulated through L2->L0 vmexits. */ 5185 return false; 5186 case EXIT_REASON_ENCLS: 5187 /* SGX is never exposed to L1 */ 5188 return false; 5189 default: 5190 return true; 5191 } 5192 } 5193 5194 5195 static int vmx_get_nested_state(struct kvm_vcpu *vcpu, 5196 struct kvm_nested_state __user *user_kvm_nested_state, 5197 u32 user_data_size) 5198 { 5199 struct vcpu_vmx *vmx; 5200 struct vmcs12 *vmcs12; 5201 struct kvm_nested_state kvm_state = { 5202 .flags = 0, 5203 .format = 0, 5204 .size = sizeof(kvm_state), 5205 .vmx.vmxon_pa = -1ull, 5206 .vmx.vmcs_pa = -1ull, 5207 }; 5208 5209 if (!vcpu) 5210 return kvm_state.size + 2 * VMCS12_SIZE; 5211 5212 vmx = to_vmx(vcpu); 5213 vmcs12 = get_vmcs12(vcpu); 5214 5215 if (nested_vmx_allowed(vcpu) && vmx->nested.enlightened_vmcs_enabled) 5216 kvm_state.flags |= KVM_STATE_NESTED_EVMCS; 5217 5218 if (nested_vmx_allowed(vcpu) && 5219 (vmx->nested.vmxon || vmx->nested.smm.vmxon)) { 5220 kvm_state.vmx.vmxon_pa = vmx->nested.vmxon_ptr; 5221 kvm_state.vmx.vmcs_pa = vmx->nested.current_vmptr; 5222 5223 if (vmx_has_valid_vmcs12(vcpu)) { 5224 kvm_state.size += VMCS12_SIZE; 5225 5226 if (is_guest_mode(vcpu) && 5227 nested_cpu_has_shadow_vmcs(vmcs12) && 5228 vmcs12->vmcs_link_pointer != -1ull) 5229 kvm_state.size += VMCS12_SIZE; 5230 } 5231 5232 if (vmx->nested.smm.vmxon) 5233 kvm_state.vmx.smm.flags |= KVM_STATE_NESTED_SMM_VMXON; 5234 5235 if (vmx->nested.smm.guest_mode) 5236 kvm_state.vmx.smm.flags |= KVM_STATE_NESTED_SMM_GUEST_MODE; 5237 5238 if (is_guest_mode(vcpu)) { 5239 kvm_state.flags |= KVM_STATE_NESTED_GUEST_MODE; 5240 5241 if (vmx->nested.nested_run_pending) 5242 kvm_state.flags |= KVM_STATE_NESTED_RUN_PENDING; 5243 } 5244 } 5245 5246 if (user_data_size < kvm_state.size) 5247 goto out; 5248 5249 if (copy_to_user(user_kvm_nested_state, &kvm_state, sizeof(kvm_state))) 5250 return -EFAULT; 5251 5252 if (!vmx_has_valid_vmcs12(vcpu)) 5253 goto out; 5254 5255 /* 5256 * When running L2, the authoritative vmcs12 state is in the 5257 * vmcs02. When running L1, the authoritative vmcs12 state is 5258 * in the shadow or enlightened vmcs linked to vmcs01, unless 5259 * need_vmcs12_sync is set, in which case, the authoritative 5260 * vmcs12 state is in the vmcs12 already. 5261 */ 5262 if (is_guest_mode(vcpu)) { 5263 sync_vmcs12(vcpu, vmcs12); 5264 } else if (!vmx->nested.need_vmcs12_sync) { 5265 if (vmx->nested.hv_evmcs) 5266 copy_enlightened_to_vmcs12(vmx); 5267 else if (enable_shadow_vmcs) 5268 copy_shadow_to_vmcs12(vmx); 5269 } 5270 5271 /* 5272 * Copy over the full allocated size of vmcs12 rather than just the size 5273 * of the struct. 5274 */ 5275 if (copy_to_user(user_kvm_nested_state->data, vmcs12, VMCS12_SIZE)) 5276 return -EFAULT; 5277 5278 if (nested_cpu_has_shadow_vmcs(vmcs12) && 5279 vmcs12->vmcs_link_pointer != -1ull) { 5280 if (copy_to_user(user_kvm_nested_state->data + VMCS12_SIZE, 5281 get_shadow_vmcs12(vcpu), VMCS12_SIZE)) 5282 return -EFAULT; 5283 } 5284 5285 out: 5286 return kvm_state.size; 5287 } 5288 5289 /* 5290 * Forcibly leave nested mode in order to be able to reset the VCPU later on. 5291 */ 5292 void vmx_leave_nested(struct kvm_vcpu *vcpu) 5293 { 5294 if (is_guest_mode(vcpu)) { 5295 to_vmx(vcpu)->nested.nested_run_pending = 0; 5296 nested_vmx_vmexit(vcpu, -1, 0, 0); 5297 } 5298 free_nested(vcpu); 5299 } 5300 5301 static int vmx_set_nested_state(struct kvm_vcpu *vcpu, 5302 struct kvm_nested_state __user *user_kvm_nested_state, 5303 struct kvm_nested_state *kvm_state) 5304 { 5305 struct vcpu_vmx *vmx = to_vmx(vcpu); 5306 struct vmcs12 *vmcs12; 5307 u32 exit_qual; 5308 int ret; 5309 5310 if (kvm_state->format != 0) 5311 return -EINVAL; 5312 5313 if (kvm_state->flags & KVM_STATE_NESTED_EVMCS) 5314 nested_enable_evmcs(vcpu, NULL); 5315 5316 if (!nested_vmx_allowed(vcpu)) 5317 return kvm_state->vmx.vmxon_pa == -1ull ? 0 : -EINVAL; 5318 5319 if (kvm_state->vmx.vmxon_pa == -1ull) { 5320 if (kvm_state->vmx.smm.flags) 5321 return -EINVAL; 5322 5323 if (kvm_state->vmx.vmcs_pa != -1ull) 5324 return -EINVAL; 5325 5326 vmx_leave_nested(vcpu); 5327 return 0; 5328 } 5329 5330 if (!page_address_valid(vcpu, kvm_state->vmx.vmxon_pa)) 5331 return -EINVAL; 5332 5333 if ((kvm_state->vmx.smm.flags & KVM_STATE_NESTED_SMM_GUEST_MODE) && 5334 (kvm_state->flags & KVM_STATE_NESTED_GUEST_MODE)) 5335 return -EINVAL; 5336 5337 if (kvm_state->vmx.smm.flags & 5338 ~(KVM_STATE_NESTED_SMM_GUEST_MODE | KVM_STATE_NESTED_SMM_VMXON)) 5339 return -EINVAL; 5340 5341 /* 5342 * SMM temporarily disables VMX, so we cannot be in guest mode, 5343 * nor can VMLAUNCH/VMRESUME be pending. Outside SMM, SMM flags 5344 * must be zero. 5345 */ 5346 if (is_smm(vcpu) ? kvm_state->flags : kvm_state->vmx.smm.flags) 5347 return -EINVAL; 5348 5349 if ((kvm_state->vmx.smm.flags & KVM_STATE_NESTED_SMM_GUEST_MODE) && 5350 !(kvm_state->vmx.smm.flags & KVM_STATE_NESTED_SMM_VMXON)) 5351 return -EINVAL; 5352 5353 vmx_leave_nested(vcpu); 5354 if (kvm_state->vmx.vmxon_pa == -1ull) 5355 return 0; 5356 5357 vmx->nested.vmxon_ptr = kvm_state->vmx.vmxon_pa; 5358 ret = enter_vmx_operation(vcpu); 5359 if (ret) 5360 return ret; 5361 5362 /* Empty 'VMXON' state is permitted */ 5363 if (kvm_state->size < sizeof(kvm_state) + sizeof(*vmcs12)) 5364 return 0; 5365 5366 if (kvm_state->vmx.vmcs_pa != -1ull) { 5367 if (kvm_state->vmx.vmcs_pa == kvm_state->vmx.vmxon_pa || 5368 !page_address_valid(vcpu, kvm_state->vmx.vmcs_pa)) 5369 return -EINVAL; 5370 5371 set_current_vmptr(vmx, kvm_state->vmx.vmcs_pa); 5372 } else if (kvm_state->flags & KVM_STATE_NESTED_EVMCS) { 5373 /* 5374 * Sync eVMCS upon entry as we may not have 5375 * HV_X64_MSR_VP_ASSIST_PAGE set up yet. 5376 */ 5377 vmx->nested.need_vmcs12_sync = true; 5378 } else { 5379 return -EINVAL; 5380 } 5381 5382 if (kvm_state->vmx.smm.flags & KVM_STATE_NESTED_SMM_VMXON) { 5383 vmx->nested.smm.vmxon = true; 5384 vmx->nested.vmxon = false; 5385 5386 if (kvm_state->vmx.smm.flags & KVM_STATE_NESTED_SMM_GUEST_MODE) 5387 vmx->nested.smm.guest_mode = true; 5388 } 5389 5390 vmcs12 = get_vmcs12(vcpu); 5391 if (copy_from_user(vmcs12, user_kvm_nested_state->data, sizeof(*vmcs12))) 5392 return -EFAULT; 5393 5394 if (vmcs12->hdr.revision_id != VMCS12_REVISION) 5395 return -EINVAL; 5396 5397 if (!(kvm_state->flags & KVM_STATE_NESTED_GUEST_MODE)) 5398 return 0; 5399 5400 vmx->nested.nested_run_pending = 5401 !!(kvm_state->flags & KVM_STATE_NESTED_RUN_PENDING); 5402 5403 if (nested_cpu_has_shadow_vmcs(vmcs12) && 5404 vmcs12->vmcs_link_pointer != -1ull) { 5405 struct vmcs12 *shadow_vmcs12 = get_shadow_vmcs12(vcpu); 5406 5407 if (kvm_state->size < sizeof(kvm_state) + 2 * sizeof(*vmcs12)) 5408 return -EINVAL; 5409 5410 if (copy_from_user(shadow_vmcs12, 5411 user_kvm_nested_state->data + VMCS12_SIZE, 5412 sizeof(*vmcs12))) 5413 return -EFAULT; 5414 5415 if (shadow_vmcs12->hdr.revision_id != VMCS12_REVISION || 5416 !shadow_vmcs12->hdr.shadow_vmcs) 5417 return -EINVAL; 5418 } 5419 5420 if (nested_vmx_check_vmentry_prereqs(vcpu, vmcs12) || 5421 nested_vmx_check_vmentry_postreqs(vcpu, vmcs12, &exit_qual)) 5422 return -EINVAL; 5423 5424 vmx->nested.dirty_vmcs12 = true; 5425 ret = nested_vmx_enter_non_root_mode(vcpu, false); 5426 if (ret) 5427 return -EINVAL; 5428 5429 return 0; 5430 } 5431 5432 void nested_vmx_vcpu_setup(void) 5433 { 5434 if (enable_shadow_vmcs) { 5435 /* 5436 * At vCPU creation, "VMWRITE to any supported field 5437 * in the VMCS" is supported, so use the more 5438 * permissive vmx_vmread_bitmap to specify both read 5439 * and write permissions for the shadow VMCS. 5440 */ 5441 vmcs_write64(VMREAD_BITMAP, __pa(vmx_vmread_bitmap)); 5442 vmcs_write64(VMWRITE_BITMAP, __pa(vmx_vmread_bitmap)); 5443 } 5444 } 5445 5446 /* 5447 * nested_vmx_setup_ctls_msrs() sets up variables containing the values to be 5448 * returned for the various VMX controls MSRs when nested VMX is enabled. 5449 * The same values should also be used to verify that vmcs12 control fields are 5450 * valid during nested entry from L1 to L2. 5451 * Each of these control msrs has a low and high 32-bit half: A low bit is on 5452 * if the corresponding bit in the (32-bit) control field *must* be on, and a 5453 * bit in the high half is on if the corresponding bit in the control field 5454 * may be on. See also vmx_control_verify(). 5455 */ 5456 void nested_vmx_setup_ctls_msrs(struct nested_vmx_msrs *msrs, u32 ept_caps, 5457 bool apicv) 5458 { 5459 /* 5460 * Note that as a general rule, the high half of the MSRs (bits in 5461 * the control fields which may be 1) should be initialized by the 5462 * intersection of the underlying hardware's MSR (i.e., features which 5463 * can be supported) and the list of features we want to expose - 5464 * because they are known to be properly supported in our code. 5465 * Also, usually, the low half of the MSRs (bits which must be 1) can 5466 * be set to 0, meaning that L1 may turn off any of these bits. The 5467 * reason is that if one of these bits is necessary, it will appear 5468 * in vmcs01 and prepare_vmcs02, when it bitwise-or's the control 5469 * fields of vmcs01 and vmcs02, will turn these bits off - and 5470 * nested_vmx_exit_reflected() will not pass related exits to L1. 5471 * These rules have exceptions below. 5472 */ 5473 5474 /* pin-based controls */ 5475 rdmsr(MSR_IA32_VMX_PINBASED_CTLS, 5476 msrs->pinbased_ctls_low, 5477 msrs->pinbased_ctls_high); 5478 msrs->pinbased_ctls_low |= 5479 PIN_BASED_ALWAYSON_WITHOUT_TRUE_MSR; 5480 msrs->pinbased_ctls_high &= 5481 PIN_BASED_EXT_INTR_MASK | 5482 PIN_BASED_NMI_EXITING | 5483 PIN_BASED_VIRTUAL_NMIS | 5484 (apicv ? PIN_BASED_POSTED_INTR : 0); 5485 msrs->pinbased_ctls_high |= 5486 PIN_BASED_ALWAYSON_WITHOUT_TRUE_MSR | 5487 PIN_BASED_VMX_PREEMPTION_TIMER; 5488 5489 /* exit controls */ 5490 rdmsr(MSR_IA32_VMX_EXIT_CTLS, 5491 msrs->exit_ctls_low, 5492 msrs->exit_ctls_high); 5493 msrs->exit_ctls_low = 5494 VM_EXIT_ALWAYSON_WITHOUT_TRUE_MSR; 5495 5496 msrs->exit_ctls_high &= 5497 #ifdef CONFIG_X86_64 5498 VM_EXIT_HOST_ADDR_SPACE_SIZE | 5499 #endif 5500 VM_EXIT_LOAD_IA32_PAT | VM_EXIT_SAVE_IA32_PAT; 5501 msrs->exit_ctls_high |= 5502 VM_EXIT_ALWAYSON_WITHOUT_TRUE_MSR | 5503 VM_EXIT_LOAD_IA32_EFER | VM_EXIT_SAVE_IA32_EFER | 5504 VM_EXIT_SAVE_VMX_PREEMPTION_TIMER | VM_EXIT_ACK_INTR_ON_EXIT; 5505 5506 /* We support free control of debug control saving. */ 5507 msrs->exit_ctls_low &= ~VM_EXIT_SAVE_DEBUG_CONTROLS; 5508 5509 /* entry controls */ 5510 rdmsr(MSR_IA32_VMX_ENTRY_CTLS, 5511 msrs->entry_ctls_low, 5512 msrs->entry_ctls_high); 5513 msrs->entry_ctls_low = 5514 VM_ENTRY_ALWAYSON_WITHOUT_TRUE_MSR; 5515 msrs->entry_ctls_high &= 5516 #ifdef CONFIG_X86_64 5517 VM_ENTRY_IA32E_MODE | 5518 #endif 5519 VM_ENTRY_LOAD_IA32_PAT; 5520 msrs->entry_ctls_high |= 5521 (VM_ENTRY_ALWAYSON_WITHOUT_TRUE_MSR | VM_ENTRY_LOAD_IA32_EFER); 5522 5523 /* We support free control of debug control loading. */ 5524 msrs->entry_ctls_low &= ~VM_ENTRY_LOAD_DEBUG_CONTROLS; 5525 5526 /* cpu-based controls */ 5527 rdmsr(MSR_IA32_VMX_PROCBASED_CTLS, 5528 msrs->procbased_ctls_low, 5529 msrs->procbased_ctls_high); 5530 msrs->procbased_ctls_low = 5531 CPU_BASED_ALWAYSON_WITHOUT_TRUE_MSR; 5532 msrs->procbased_ctls_high &= 5533 CPU_BASED_VIRTUAL_INTR_PENDING | 5534 CPU_BASED_VIRTUAL_NMI_PENDING | CPU_BASED_USE_TSC_OFFSETING | 5535 CPU_BASED_HLT_EXITING | CPU_BASED_INVLPG_EXITING | 5536 CPU_BASED_MWAIT_EXITING | CPU_BASED_CR3_LOAD_EXITING | 5537 CPU_BASED_CR3_STORE_EXITING | 5538 #ifdef CONFIG_X86_64 5539 CPU_BASED_CR8_LOAD_EXITING | CPU_BASED_CR8_STORE_EXITING | 5540 #endif 5541 CPU_BASED_MOV_DR_EXITING | CPU_BASED_UNCOND_IO_EXITING | 5542 CPU_BASED_USE_IO_BITMAPS | CPU_BASED_MONITOR_TRAP_FLAG | 5543 CPU_BASED_MONITOR_EXITING | CPU_BASED_RDPMC_EXITING | 5544 CPU_BASED_RDTSC_EXITING | CPU_BASED_PAUSE_EXITING | 5545 CPU_BASED_TPR_SHADOW | CPU_BASED_ACTIVATE_SECONDARY_CONTROLS; 5546 /* 5547 * We can allow some features even when not supported by the 5548 * hardware. For example, L1 can specify an MSR bitmap - and we 5549 * can use it to avoid exits to L1 - even when L0 runs L2 5550 * without MSR bitmaps. 5551 */ 5552 msrs->procbased_ctls_high |= 5553 CPU_BASED_ALWAYSON_WITHOUT_TRUE_MSR | 5554 CPU_BASED_USE_MSR_BITMAPS; 5555 5556 /* We support free control of CR3 access interception. */ 5557 msrs->procbased_ctls_low &= 5558 ~(CPU_BASED_CR3_LOAD_EXITING | CPU_BASED_CR3_STORE_EXITING); 5559 5560 /* 5561 * secondary cpu-based controls. Do not include those that 5562 * depend on CPUID bits, they are added later by vmx_cpuid_update. 5563 */ 5564 if (msrs->procbased_ctls_high & CPU_BASED_ACTIVATE_SECONDARY_CONTROLS) 5565 rdmsr(MSR_IA32_VMX_PROCBASED_CTLS2, 5566 msrs->secondary_ctls_low, 5567 msrs->secondary_ctls_high); 5568 5569 msrs->secondary_ctls_low = 0; 5570 msrs->secondary_ctls_high &= 5571 SECONDARY_EXEC_DESC | 5572 SECONDARY_EXEC_VIRTUALIZE_X2APIC_MODE | 5573 SECONDARY_EXEC_APIC_REGISTER_VIRT | 5574 SECONDARY_EXEC_VIRTUAL_INTR_DELIVERY | 5575 SECONDARY_EXEC_WBINVD_EXITING; 5576 5577 /* 5578 * We can emulate "VMCS shadowing," even if the hardware 5579 * doesn't support it. 5580 */ 5581 msrs->secondary_ctls_high |= 5582 SECONDARY_EXEC_SHADOW_VMCS; 5583 5584 if (enable_ept) { 5585 /* nested EPT: emulate EPT also to L1 */ 5586 msrs->secondary_ctls_high |= 5587 SECONDARY_EXEC_ENABLE_EPT; 5588 msrs->ept_caps = VMX_EPT_PAGE_WALK_4_BIT | 5589 VMX_EPTP_WB_BIT | VMX_EPT_INVEPT_BIT; 5590 if (cpu_has_vmx_ept_execute_only()) 5591 msrs->ept_caps |= 5592 VMX_EPT_EXECUTE_ONLY_BIT; 5593 msrs->ept_caps &= ept_caps; 5594 msrs->ept_caps |= VMX_EPT_EXTENT_GLOBAL_BIT | 5595 VMX_EPT_EXTENT_CONTEXT_BIT | VMX_EPT_2MB_PAGE_BIT | 5596 VMX_EPT_1GB_PAGE_BIT; 5597 if (enable_ept_ad_bits) { 5598 msrs->secondary_ctls_high |= 5599 SECONDARY_EXEC_ENABLE_PML; 5600 msrs->ept_caps |= VMX_EPT_AD_BIT; 5601 } 5602 } 5603 5604 if (cpu_has_vmx_vmfunc()) { 5605 msrs->secondary_ctls_high |= 5606 SECONDARY_EXEC_ENABLE_VMFUNC; 5607 /* 5608 * Advertise EPTP switching unconditionally 5609 * since we emulate it 5610 */ 5611 if (enable_ept) 5612 msrs->vmfunc_controls = 5613 VMX_VMFUNC_EPTP_SWITCHING; 5614 } 5615 5616 /* 5617 * Old versions of KVM use the single-context version without 5618 * checking for support, so declare that it is supported even 5619 * though it is treated as global context. The alternative is 5620 * not failing the single-context invvpid, and it is worse. 5621 */ 5622 if (enable_vpid) { 5623 msrs->secondary_ctls_high |= 5624 SECONDARY_EXEC_ENABLE_VPID; 5625 msrs->vpid_caps = VMX_VPID_INVVPID_BIT | 5626 VMX_VPID_EXTENT_SUPPORTED_MASK; 5627 } 5628 5629 if (enable_unrestricted_guest) 5630 msrs->secondary_ctls_high |= 5631 SECONDARY_EXEC_UNRESTRICTED_GUEST; 5632 5633 if (flexpriority_enabled) 5634 msrs->secondary_ctls_high |= 5635 SECONDARY_EXEC_VIRTUALIZE_APIC_ACCESSES; 5636 5637 /* miscellaneous data */ 5638 rdmsr(MSR_IA32_VMX_MISC, 5639 msrs->misc_low, 5640 msrs->misc_high); 5641 msrs->misc_low &= VMX_MISC_SAVE_EFER_LMA; 5642 msrs->misc_low |= 5643 MSR_IA32_VMX_MISC_VMWRITE_SHADOW_RO_FIELDS | 5644 VMX_MISC_EMULATED_PREEMPTION_TIMER_RATE | 5645 VMX_MISC_ACTIVITY_HLT; 5646 msrs->misc_high = 0; 5647 5648 /* 5649 * This MSR reports some information about VMX support. We 5650 * should return information about the VMX we emulate for the 5651 * guest, and the VMCS structure we give it - not about the 5652 * VMX support of the underlying hardware. 5653 */ 5654 msrs->basic = 5655 VMCS12_REVISION | 5656 VMX_BASIC_TRUE_CTLS | 5657 ((u64)VMCS12_SIZE << VMX_BASIC_VMCS_SIZE_SHIFT) | 5658 (VMX_BASIC_MEM_TYPE_WB << VMX_BASIC_MEM_TYPE_SHIFT); 5659 5660 if (cpu_has_vmx_basic_inout()) 5661 msrs->basic |= VMX_BASIC_INOUT; 5662 5663 /* 5664 * These MSRs specify bits which the guest must keep fixed on 5665 * while L1 is in VMXON mode (in L1's root mode, or running an L2). 5666 * We picked the standard core2 setting. 5667 */ 5668 #define VMXON_CR0_ALWAYSON (X86_CR0_PE | X86_CR0_PG | X86_CR0_NE) 5669 #define VMXON_CR4_ALWAYSON X86_CR4_VMXE 5670 msrs->cr0_fixed0 = VMXON_CR0_ALWAYSON; 5671 msrs->cr4_fixed0 = VMXON_CR4_ALWAYSON; 5672 5673 /* These MSRs specify bits which the guest must keep fixed off. */ 5674 rdmsrl(MSR_IA32_VMX_CR0_FIXED1, msrs->cr0_fixed1); 5675 rdmsrl(MSR_IA32_VMX_CR4_FIXED1, msrs->cr4_fixed1); 5676 5677 /* highest index: VMX_PREEMPTION_TIMER_VALUE */ 5678 msrs->vmcs_enum = VMCS12_MAX_FIELD_INDEX << 1; 5679 } 5680 5681 void nested_vmx_hardware_unsetup(void) 5682 { 5683 int i; 5684 5685 if (enable_shadow_vmcs) { 5686 for (i = 0; i < VMX_BITMAP_NR; i++) 5687 free_page((unsigned long)vmx_bitmap[i]); 5688 } 5689 } 5690 5691 __init int nested_vmx_hardware_setup(int (*exit_handlers[])(struct kvm_vcpu *)) 5692 { 5693 int i; 5694 5695 if (!cpu_has_vmx_shadow_vmcs()) 5696 enable_shadow_vmcs = 0; 5697 if (enable_shadow_vmcs) { 5698 for (i = 0; i < VMX_BITMAP_NR; i++) { 5699 vmx_bitmap[i] = (unsigned long *) 5700 __get_free_page(GFP_KERNEL); 5701 if (!vmx_bitmap[i]) { 5702 nested_vmx_hardware_unsetup(); 5703 return -ENOMEM; 5704 } 5705 } 5706 5707 init_vmcs_shadow_fields(); 5708 } 5709 5710 exit_handlers[EXIT_REASON_VMCLEAR] = handle_vmclear, 5711 exit_handlers[EXIT_REASON_VMLAUNCH] = handle_vmlaunch, 5712 exit_handlers[EXIT_REASON_VMPTRLD] = handle_vmptrld, 5713 exit_handlers[EXIT_REASON_VMPTRST] = handle_vmptrst, 5714 exit_handlers[EXIT_REASON_VMREAD] = handle_vmread, 5715 exit_handlers[EXIT_REASON_VMRESUME] = handle_vmresume, 5716 exit_handlers[EXIT_REASON_VMWRITE] = handle_vmwrite, 5717 exit_handlers[EXIT_REASON_VMOFF] = handle_vmoff, 5718 exit_handlers[EXIT_REASON_VMON] = handle_vmon, 5719 exit_handlers[EXIT_REASON_INVEPT] = handle_invept, 5720 exit_handlers[EXIT_REASON_INVVPID] = handle_invvpid, 5721 exit_handlers[EXIT_REASON_VMFUNC] = handle_vmfunc, 5722 5723 kvm_x86_ops->check_nested_events = vmx_check_nested_events; 5724 kvm_x86_ops->get_nested_state = vmx_get_nested_state; 5725 kvm_x86_ops->set_nested_state = vmx_set_nested_state; 5726 kvm_x86_ops->get_vmcs12_pages = nested_get_vmcs12_pages, 5727 kvm_x86_ops->nested_enable_evmcs = nested_enable_evmcs; 5728 kvm_x86_ops->nested_get_evmcs_version = nested_get_evmcs_version; 5729 5730 return 0; 5731 } 5732