1 // SPDX-License-Identifier: GPL-2.0 2 3 #include <linux/frame.h> 4 #include <linux/percpu.h> 5 6 #include <asm/debugreg.h> 7 #include <asm/mmu_context.h> 8 9 #include "cpuid.h" 10 #include "hyperv.h" 11 #include "mmu.h" 12 #include "nested.h" 13 #include "trace.h" 14 #include "x86.h" 15 16 static bool __read_mostly enable_shadow_vmcs = 1; 17 module_param_named(enable_shadow_vmcs, enable_shadow_vmcs, bool, S_IRUGO); 18 19 static bool __read_mostly nested_early_check = 0; 20 module_param(nested_early_check, bool, S_IRUGO); 21 22 /* 23 * Hyper-V requires all of these, so mark them as supported even though 24 * they are just treated the same as all-context. 25 */ 26 #define VMX_VPID_EXTENT_SUPPORTED_MASK \ 27 (VMX_VPID_EXTENT_INDIVIDUAL_ADDR_BIT | \ 28 VMX_VPID_EXTENT_SINGLE_CONTEXT_BIT | \ 29 VMX_VPID_EXTENT_GLOBAL_CONTEXT_BIT | \ 30 VMX_VPID_EXTENT_SINGLE_NON_GLOBAL_BIT) 31 32 #define VMX_MISC_EMULATED_PREEMPTION_TIMER_RATE 5 33 34 enum { 35 VMX_VMREAD_BITMAP, 36 VMX_VMWRITE_BITMAP, 37 VMX_BITMAP_NR 38 }; 39 static unsigned long *vmx_bitmap[VMX_BITMAP_NR]; 40 41 #define vmx_vmread_bitmap (vmx_bitmap[VMX_VMREAD_BITMAP]) 42 #define vmx_vmwrite_bitmap (vmx_bitmap[VMX_VMWRITE_BITMAP]) 43 44 static u16 shadow_read_only_fields[] = { 45 #define SHADOW_FIELD_RO(x) x, 46 #include "vmcs_shadow_fields.h" 47 }; 48 static int max_shadow_read_only_fields = 49 ARRAY_SIZE(shadow_read_only_fields); 50 51 static u16 shadow_read_write_fields[] = { 52 #define SHADOW_FIELD_RW(x) x, 53 #include "vmcs_shadow_fields.h" 54 }; 55 static int max_shadow_read_write_fields = 56 ARRAY_SIZE(shadow_read_write_fields); 57 58 static void init_vmcs_shadow_fields(void) 59 { 60 int i, j; 61 62 memset(vmx_vmread_bitmap, 0xff, PAGE_SIZE); 63 memset(vmx_vmwrite_bitmap, 0xff, PAGE_SIZE); 64 65 for (i = j = 0; i < max_shadow_read_only_fields; i++) { 66 u16 field = shadow_read_only_fields[i]; 67 68 if (vmcs_field_width(field) == VMCS_FIELD_WIDTH_U64 && 69 (i + 1 == max_shadow_read_only_fields || 70 shadow_read_only_fields[i + 1] != field + 1)) 71 pr_err("Missing field from shadow_read_only_field %x\n", 72 field + 1); 73 74 clear_bit(field, vmx_vmread_bitmap); 75 #ifdef CONFIG_X86_64 76 if (field & 1) 77 continue; 78 #endif 79 if (j < i) 80 shadow_read_only_fields[j] = field; 81 j++; 82 } 83 max_shadow_read_only_fields = j; 84 85 for (i = j = 0; i < max_shadow_read_write_fields; i++) { 86 u16 field = shadow_read_write_fields[i]; 87 88 if (vmcs_field_width(field) == VMCS_FIELD_WIDTH_U64 && 89 (i + 1 == max_shadow_read_write_fields || 90 shadow_read_write_fields[i + 1] != field + 1)) 91 pr_err("Missing field from shadow_read_write_field %x\n", 92 field + 1); 93 94 /* 95 * PML and the preemption timer can be emulated, but the 96 * processor cannot vmwrite to fields that don't exist 97 * on bare metal. 98 */ 99 switch (field) { 100 case GUEST_PML_INDEX: 101 if (!cpu_has_vmx_pml()) 102 continue; 103 break; 104 case VMX_PREEMPTION_TIMER_VALUE: 105 if (!cpu_has_vmx_preemption_timer()) 106 continue; 107 break; 108 case GUEST_INTR_STATUS: 109 if (!cpu_has_vmx_apicv()) 110 continue; 111 break; 112 default: 113 break; 114 } 115 116 clear_bit(field, vmx_vmwrite_bitmap); 117 clear_bit(field, vmx_vmread_bitmap); 118 #ifdef CONFIG_X86_64 119 if (field & 1) 120 continue; 121 #endif 122 if (j < i) 123 shadow_read_write_fields[j] = field; 124 j++; 125 } 126 max_shadow_read_write_fields = j; 127 } 128 129 /* 130 * The following 3 functions, nested_vmx_succeed()/failValid()/failInvalid(), 131 * set the success or error code of an emulated VMX instruction (as specified 132 * by Vol 2B, VMX Instruction Reference, "Conventions"), and skip the emulated 133 * instruction. 134 */ 135 static int nested_vmx_succeed(struct kvm_vcpu *vcpu) 136 { 137 vmx_set_rflags(vcpu, vmx_get_rflags(vcpu) 138 & ~(X86_EFLAGS_CF | X86_EFLAGS_PF | X86_EFLAGS_AF | 139 X86_EFLAGS_ZF | X86_EFLAGS_SF | X86_EFLAGS_OF)); 140 return kvm_skip_emulated_instruction(vcpu); 141 } 142 143 static int nested_vmx_failInvalid(struct kvm_vcpu *vcpu) 144 { 145 vmx_set_rflags(vcpu, (vmx_get_rflags(vcpu) 146 & ~(X86_EFLAGS_PF | X86_EFLAGS_AF | X86_EFLAGS_ZF | 147 X86_EFLAGS_SF | X86_EFLAGS_OF)) 148 | X86_EFLAGS_CF); 149 return kvm_skip_emulated_instruction(vcpu); 150 } 151 152 static int nested_vmx_failValid(struct kvm_vcpu *vcpu, 153 u32 vm_instruction_error) 154 { 155 struct vcpu_vmx *vmx = to_vmx(vcpu); 156 157 /* 158 * failValid writes the error number to the current VMCS, which 159 * can't be done if there isn't a current VMCS. 160 */ 161 if (vmx->nested.current_vmptr == -1ull && !vmx->nested.hv_evmcs) 162 return nested_vmx_failInvalid(vcpu); 163 164 vmx_set_rflags(vcpu, (vmx_get_rflags(vcpu) 165 & ~(X86_EFLAGS_CF | X86_EFLAGS_PF | X86_EFLAGS_AF | 166 X86_EFLAGS_SF | X86_EFLAGS_OF)) 167 | X86_EFLAGS_ZF); 168 get_vmcs12(vcpu)->vm_instruction_error = vm_instruction_error; 169 /* 170 * We don't need to force a shadow sync because 171 * VM_INSTRUCTION_ERROR is not shadowed 172 */ 173 return kvm_skip_emulated_instruction(vcpu); 174 } 175 176 static void nested_vmx_abort(struct kvm_vcpu *vcpu, u32 indicator) 177 { 178 /* TODO: not to reset guest simply here. */ 179 kvm_make_request(KVM_REQ_TRIPLE_FAULT, vcpu); 180 pr_debug_ratelimited("kvm: nested vmx abort, indicator %d\n", indicator); 181 } 182 183 static void vmx_disable_shadow_vmcs(struct vcpu_vmx *vmx) 184 { 185 vmcs_clear_bits(SECONDARY_VM_EXEC_CONTROL, SECONDARY_EXEC_SHADOW_VMCS); 186 vmcs_write64(VMCS_LINK_POINTER, -1ull); 187 } 188 189 static inline void nested_release_evmcs(struct kvm_vcpu *vcpu) 190 { 191 struct vcpu_vmx *vmx = to_vmx(vcpu); 192 193 if (!vmx->nested.hv_evmcs) 194 return; 195 196 kunmap(vmx->nested.hv_evmcs_page); 197 kvm_release_page_dirty(vmx->nested.hv_evmcs_page); 198 vmx->nested.hv_evmcs_vmptr = -1ull; 199 vmx->nested.hv_evmcs_page = NULL; 200 vmx->nested.hv_evmcs = NULL; 201 } 202 203 /* 204 * Free whatever needs to be freed from vmx->nested when L1 goes down, or 205 * just stops using VMX. 206 */ 207 static void free_nested(struct kvm_vcpu *vcpu) 208 { 209 struct vcpu_vmx *vmx = to_vmx(vcpu); 210 211 if (!vmx->nested.vmxon && !vmx->nested.smm.vmxon) 212 return; 213 214 vmx->nested.vmxon = false; 215 vmx->nested.smm.vmxon = false; 216 free_vpid(vmx->nested.vpid02); 217 vmx->nested.posted_intr_nv = -1; 218 vmx->nested.current_vmptr = -1ull; 219 if (enable_shadow_vmcs) { 220 vmx_disable_shadow_vmcs(vmx); 221 vmcs_clear(vmx->vmcs01.shadow_vmcs); 222 free_vmcs(vmx->vmcs01.shadow_vmcs); 223 vmx->vmcs01.shadow_vmcs = NULL; 224 } 225 kfree(vmx->nested.cached_vmcs12); 226 kfree(vmx->nested.cached_shadow_vmcs12); 227 /* Unpin physical memory we referred to in the vmcs02 */ 228 if (vmx->nested.apic_access_page) { 229 kvm_release_page_dirty(vmx->nested.apic_access_page); 230 vmx->nested.apic_access_page = NULL; 231 } 232 if (vmx->nested.virtual_apic_page) { 233 kvm_release_page_dirty(vmx->nested.virtual_apic_page); 234 vmx->nested.virtual_apic_page = NULL; 235 } 236 if (vmx->nested.pi_desc_page) { 237 kunmap(vmx->nested.pi_desc_page); 238 kvm_release_page_dirty(vmx->nested.pi_desc_page); 239 vmx->nested.pi_desc_page = NULL; 240 vmx->nested.pi_desc = NULL; 241 } 242 243 kvm_mmu_free_roots(vcpu, &vcpu->arch.guest_mmu, KVM_MMU_ROOTS_ALL); 244 245 nested_release_evmcs(vcpu); 246 247 free_loaded_vmcs(&vmx->nested.vmcs02); 248 } 249 250 static void vmx_switch_vmcs(struct kvm_vcpu *vcpu, struct loaded_vmcs *vmcs) 251 { 252 struct vcpu_vmx *vmx = to_vmx(vcpu); 253 int cpu; 254 255 if (vmx->loaded_vmcs == vmcs) 256 return; 257 258 cpu = get_cpu(); 259 vmx_vcpu_put(vcpu); 260 vmx->loaded_vmcs = vmcs; 261 vmx_vcpu_load(vcpu, cpu); 262 put_cpu(); 263 264 vm_entry_controls_reset_shadow(vmx); 265 vm_exit_controls_reset_shadow(vmx); 266 vmx_segment_cache_clear(vmx); 267 } 268 269 /* 270 * Ensure that the current vmcs of the logical processor is the 271 * vmcs01 of the vcpu before calling free_nested(). 272 */ 273 void nested_vmx_free_vcpu(struct kvm_vcpu *vcpu) 274 { 275 vcpu_load(vcpu); 276 vmx_leave_nested(vcpu); 277 vmx_switch_vmcs(vcpu, &to_vmx(vcpu)->vmcs01); 278 free_nested(vcpu); 279 vcpu_put(vcpu); 280 } 281 282 static void nested_ept_inject_page_fault(struct kvm_vcpu *vcpu, 283 struct x86_exception *fault) 284 { 285 struct vmcs12 *vmcs12 = get_vmcs12(vcpu); 286 struct vcpu_vmx *vmx = to_vmx(vcpu); 287 u32 exit_reason; 288 unsigned long exit_qualification = vcpu->arch.exit_qualification; 289 290 if (vmx->nested.pml_full) { 291 exit_reason = EXIT_REASON_PML_FULL; 292 vmx->nested.pml_full = false; 293 exit_qualification &= INTR_INFO_UNBLOCK_NMI; 294 } else if (fault->error_code & PFERR_RSVD_MASK) 295 exit_reason = EXIT_REASON_EPT_MISCONFIG; 296 else 297 exit_reason = EXIT_REASON_EPT_VIOLATION; 298 299 nested_vmx_vmexit(vcpu, exit_reason, 0, exit_qualification); 300 vmcs12->guest_physical_address = fault->address; 301 } 302 303 static void nested_ept_init_mmu_context(struct kvm_vcpu *vcpu) 304 { 305 WARN_ON(mmu_is_nested(vcpu)); 306 307 vcpu->arch.mmu = &vcpu->arch.guest_mmu; 308 kvm_init_shadow_ept_mmu(vcpu, 309 to_vmx(vcpu)->nested.msrs.ept_caps & 310 VMX_EPT_EXECUTE_ONLY_BIT, 311 nested_ept_ad_enabled(vcpu), 312 nested_ept_get_cr3(vcpu)); 313 vcpu->arch.mmu->set_cr3 = vmx_set_cr3; 314 vcpu->arch.mmu->get_cr3 = nested_ept_get_cr3; 315 vcpu->arch.mmu->inject_page_fault = nested_ept_inject_page_fault; 316 vcpu->arch.mmu->get_pdptr = kvm_pdptr_read; 317 318 vcpu->arch.walk_mmu = &vcpu->arch.nested_mmu; 319 } 320 321 static void nested_ept_uninit_mmu_context(struct kvm_vcpu *vcpu) 322 { 323 vcpu->arch.mmu = &vcpu->arch.root_mmu; 324 vcpu->arch.walk_mmu = &vcpu->arch.root_mmu; 325 } 326 327 static bool nested_vmx_is_page_fault_vmexit(struct vmcs12 *vmcs12, 328 u16 error_code) 329 { 330 bool inequality, bit; 331 332 bit = (vmcs12->exception_bitmap & (1u << PF_VECTOR)) != 0; 333 inequality = 334 (error_code & vmcs12->page_fault_error_code_mask) != 335 vmcs12->page_fault_error_code_match; 336 return inequality ^ bit; 337 } 338 339 340 /* 341 * KVM wants to inject page-faults which it got to the guest. This function 342 * checks whether in a nested guest, we need to inject them to L1 or L2. 343 */ 344 static int nested_vmx_check_exception(struct kvm_vcpu *vcpu, unsigned long *exit_qual) 345 { 346 struct vmcs12 *vmcs12 = get_vmcs12(vcpu); 347 unsigned int nr = vcpu->arch.exception.nr; 348 bool has_payload = vcpu->arch.exception.has_payload; 349 unsigned long payload = vcpu->arch.exception.payload; 350 351 if (nr == PF_VECTOR) { 352 if (vcpu->arch.exception.nested_apf) { 353 *exit_qual = vcpu->arch.apf.nested_apf_token; 354 return 1; 355 } 356 if (nested_vmx_is_page_fault_vmexit(vmcs12, 357 vcpu->arch.exception.error_code)) { 358 *exit_qual = has_payload ? payload : vcpu->arch.cr2; 359 return 1; 360 } 361 } else if (vmcs12->exception_bitmap & (1u << nr)) { 362 if (nr == DB_VECTOR) { 363 if (!has_payload) { 364 payload = vcpu->arch.dr6; 365 payload &= ~(DR6_FIXED_1 | DR6_BT); 366 payload ^= DR6_RTM; 367 } 368 *exit_qual = payload; 369 } else 370 *exit_qual = 0; 371 return 1; 372 } 373 374 return 0; 375 } 376 377 378 static void vmx_inject_page_fault_nested(struct kvm_vcpu *vcpu, 379 struct x86_exception *fault) 380 { 381 struct vmcs12 *vmcs12 = get_vmcs12(vcpu); 382 383 WARN_ON(!is_guest_mode(vcpu)); 384 385 if (nested_vmx_is_page_fault_vmexit(vmcs12, fault->error_code) && 386 !to_vmx(vcpu)->nested.nested_run_pending) { 387 vmcs12->vm_exit_intr_error_code = fault->error_code; 388 nested_vmx_vmexit(vcpu, EXIT_REASON_EXCEPTION_NMI, 389 PF_VECTOR | INTR_TYPE_HARD_EXCEPTION | 390 INTR_INFO_DELIVER_CODE_MASK | INTR_INFO_VALID_MASK, 391 fault->address); 392 } else { 393 kvm_inject_page_fault(vcpu, fault); 394 } 395 } 396 397 static bool page_address_valid(struct kvm_vcpu *vcpu, gpa_t gpa) 398 { 399 return PAGE_ALIGNED(gpa) && !(gpa >> cpuid_maxphyaddr(vcpu)); 400 } 401 402 static int nested_vmx_check_io_bitmap_controls(struct kvm_vcpu *vcpu, 403 struct vmcs12 *vmcs12) 404 { 405 if (!nested_cpu_has(vmcs12, CPU_BASED_USE_IO_BITMAPS)) 406 return 0; 407 408 if (!page_address_valid(vcpu, vmcs12->io_bitmap_a) || 409 !page_address_valid(vcpu, vmcs12->io_bitmap_b)) 410 return -EINVAL; 411 412 return 0; 413 } 414 415 static int nested_vmx_check_msr_bitmap_controls(struct kvm_vcpu *vcpu, 416 struct vmcs12 *vmcs12) 417 { 418 if (!nested_cpu_has(vmcs12, CPU_BASED_USE_MSR_BITMAPS)) 419 return 0; 420 421 if (!page_address_valid(vcpu, vmcs12->msr_bitmap)) 422 return -EINVAL; 423 424 return 0; 425 } 426 427 static int nested_vmx_check_tpr_shadow_controls(struct kvm_vcpu *vcpu, 428 struct vmcs12 *vmcs12) 429 { 430 if (!nested_cpu_has(vmcs12, CPU_BASED_TPR_SHADOW)) 431 return 0; 432 433 if (!page_address_valid(vcpu, vmcs12->virtual_apic_page_addr)) 434 return -EINVAL; 435 436 return 0; 437 } 438 439 /* 440 * Check if MSR is intercepted for L01 MSR bitmap. 441 */ 442 static bool msr_write_intercepted_l01(struct kvm_vcpu *vcpu, u32 msr) 443 { 444 unsigned long *msr_bitmap; 445 int f = sizeof(unsigned long); 446 447 if (!cpu_has_vmx_msr_bitmap()) 448 return true; 449 450 msr_bitmap = to_vmx(vcpu)->vmcs01.msr_bitmap; 451 452 if (msr <= 0x1fff) { 453 return !!test_bit(msr, msr_bitmap + 0x800 / f); 454 } else if ((msr >= 0xc0000000) && (msr <= 0xc0001fff)) { 455 msr &= 0x1fff; 456 return !!test_bit(msr, msr_bitmap + 0xc00 / f); 457 } 458 459 return true; 460 } 461 462 /* 463 * If a msr is allowed by L0, we should check whether it is allowed by L1. 464 * The corresponding bit will be cleared unless both of L0 and L1 allow it. 465 */ 466 static void nested_vmx_disable_intercept_for_msr(unsigned long *msr_bitmap_l1, 467 unsigned long *msr_bitmap_nested, 468 u32 msr, int type) 469 { 470 int f = sizeof(unsigned long); 471 472 /* 473 * See Intel PRM Vol. 3, 20.6.9 (MSR-Bitmap Address). Early manuals 474 * have the write-low and read-high bitmap offsets the wrong way round. 475 * We can control MSRs 0x00000000-0x00001fff and 0xc0000000-0xc0001fff. 476 */ 477 if (msr <= 0x1fff) { 478 if (type & MSR_TYPE_R && 479 !test_bit(msr, msr_bitmap_l1 + 0x000 / f)) 480 /* read-low */ 481 __clear_bit(msr, msr_bitmap_nested + 0x000 / f); 482 483 if (type & MSR_TYPE_W && 484 !test_bit(msr, msr_bitmap_l1 + 0x800 / f)) 485 /* write-low */ 486 __clear_bit(msr, msr_bitmap_nested + 0x800 / f); 487 488 } else if ((msr >= 0xc0000000) && (msr <= 0xc0001fff)) { 489 msr &= 0x1fff; 490 if (type & MSR_TYPE_R && 491 !test_bit(msr, msr_bitmap_l1 + 0x400 / f)) 492 /* read-high */ 493 __clear_bit(msr, msr_bitmap_nested + 0x400 / f); 494 495 if (type & MSR_TYPE_W && 496 !test_bit(msr, msr_bitmap_l1 + 0xc00 / f)) 497 /* write-high */ 498 __clear_bit(msr, msr_bitmap_nested + 0xc00 / f); 499 500 } 501 } 502 503 /* 504 * Merge L0's and L1's MSR bitmap, return false to indicate that 505 * we do not use the hardware. 506 */ 507 static inline bool nested_vmx_prepare_msr_bitmap(struct kvm_vcpu *vcpu, 508 struct vmcs12 *vmcs12) 509 { 510 int msr; 511 struct page *page; 512 unsigned long *msr_bitmap_l1; 513 unsigned long *msr_bitmap_l0 = to_vmx(vcpu)->nested.vmcs02.msr_bitmap; 514 /* 515 * pred_cmd & spec_ctrl are trying to verify two things: 516 * 517 * 1. L0 gave a permission to L1 to actually passthrough the MSR. This 518 * ensures that we do not accidentally generate an L02 MSR bitmap 519 * from the L12 MSR bitmap that is too permissive. 520 * 2. That L1 or L2s have actually used the MSR. This avoids 521 * unnecessarily merging of the bitmap if the MSR is unused. This 522 * works properly because we only update the L01 MSR bitmap lazily. 523 * So even if L0 should pass L1 these MSRs, the L01 bitmap is only 524 * updated to reflect this when L1 (or its L2s) actually write to 525 * the MSR. 526 */ 527 bool pred_cmd = !msr_write_intercepted_l01(vcpu, MSR_IA32_PRED_CMD); 528 bool spec_ctrl = !msr_write_intercepted_l01(vcpu, MSR_IA32_SPEC_CTRL); 529 530 /* Nothing to do if the MSR bitmap is not in use. */ 531 if (!cpu_has_vmx_msr_bitmap() || 532 !nested_cpu_has(vmcs12, CPU_BASED_USE_MSR_BITMAPS)) 533 return false; 534 535 if (!nested_cpu_has_virt_x2apic_mode(vmcs12) && 536 !pred_cmd && !spec_ctrl) 537 return false; 538 539 page = kvm_vcpu_gpa_to_page(vcpu, vmcs12->msr_bitmap); 540 if (is_error_page(page)) 541 return false; 542 543 msr_bitmap_l1 = (unsigned long *)kmap(page); 544 if (nested_cpu_has_apic_reg_virt(vmcs12)) { 545 /* 546 * L0 need not intercept reads for MSRs between 0x800 and 0x8ff, it 547 * just lets the processor take the value from the virtual-APIC page; 548 * take those 256 bits directly from the L1 bitmap. 549 */ 550 for (msr = 0x800; msr <= 0x8ff; msr += BITS_PER_LONG) { 551 unsigned word = msr / BITS_PER_LONG; 552 msr_bitmap_l0[word] = msr_bitmap_l1[word]; 553 msr_bitmap_l0[word + (0x800 / sizeof(long))] = ~0; 554 } 555 } else { 556 for (msr = 0x800; msr <= 0x8ff; msr += BITS_PER_LONG) { 557 unsigned word = msr / BITS_PER_LONG; 558 msr_bitmap_l0[word] = ~0; 559 msr_bitmap_l0[word + (0x800 / sizeof(long))] = ~0; 560 } 561 } 562 563 nested_vmx_disable_intercept_for_msr( 564 msr_bitmap_l1, msr_bitmap_l0, 565 X2APIC_MSR(APIC_TASKPRI), 566 MSR_TYPE_W); 567 568 if (nested_cpu_has_vid(vmcs12)) { 569 nested_vmx_disable_intercept_for_msr( 570 msr_bitmap_l1, msr_bitmap_l0, 571 X2APIC_MSR(APIC_EOI), 572 MSR_TYPE_W); 573 nested_vmx_disable_intercept_for_msr( 574 msr_bitmap_l1, msr_bitmap_l0, 575 X2APIC_MSR(APIC_SELF_IPI), 576 MSR_TYPE_W); 577 } 578 579 if (spec_ctrl) 580 nested_vmx_disable_intercept_for_msr( 581 msr_bitmap_l1, msr_bitmap_l0, 582 MSR_IA32_SPEC_CTRL, 583 MSR_TYPE_R | MSR_TYPE_W); 584 585 if (pred_cmd) 586 nested_vmx_disable_intercept_for_msr( 587 msr_bitmap_l1, msr_bitmap_l0, 588 MSR_IA32_PRED_CMD, 589 MSR_TYPE_W); 590 591 kunmap(page); 592 kvm_release_page_clean(page); 593 594 return true; 595 } 596 597 static void nested_cache_shadow_vmcs12(struct kvm_vcpu *vcpu, 598 struct vmcs12 *vmcs12) 599 { 600 struct vmcs12 *shadow; 601 struct page *page; 602 603 if (!nested_cpu_has_shadow_vmcs(vmcs12) || 604 vmcs12->vmcs_link_pointer == -1ull) 605 return; 606 607 shadow = get_shadow_vmcs12(vcpu); 608 page = kvm_vcpu_gpa_to_page(vcpu, vmcs12->vmcs_link_pointer); 609 610 memcpy(shadow, kmap(page), VMCS12_SIZE); 611 612 kunmap(page); 613 kvm_release_page_clean(page); 614 } 615 616 static void nested_flush_cached_shadow_vmcs12(struct kvm_vcpu *vcpu, 617 struct vmcs12 *vmcs12) 618 { 619 struct vcpu_vmx *vmx = to_vmx(vcpu); 620 621 if (!nested_cpu_has_shadow_vmcs(vmcs12) || 622 vmcs12->vmcs_link_pointer == -1ull) 623 return; 624 625 kvm_write_guest(vmx->vcpu.kvm, vmcs12->vmcs_link_pointer, 626 get_shadow_vmcs12(vcpu), VMCS12_SIZE); 627 } 628 629 /* 630 * In nested virtualization, check if L1 has set 631 * VM_EXIT_ACK_INTR_ON_EXIT 632 */ 633 static bool nested_exit_intr_ack_set(struct kvm_vcpu *vcpu) 634 { 635 return get_vmcs12(vcpu)->vm_exit_controls & 636 VM_EXIT_ACK_INTR_ON_EXIT; 637 } 638 639 static bool nested_exit_on_nmi(struct kvm_vcpu *vcpu) 640 { 641 return nested_cpu_has_nmi_exiting(get_vmcs12(vcpu)); 642 } 643 644 static int nested_vmx_check_apic_access_controls(struct kvm_vcpu *vcpu, 645 struct vmcs12 *vmcs12) 646 { 647 if (nested_cpu_has2(vmcs12, SECONDARY_EXEC_VIRTUALIZE_APIC_ACCESSES) && 648 !page_address_valid(vcpu, vmcs12->apic_access_addr)) 649 return -EINVAL; 650 else 651 return 0; 652 } 653 654 static int nested_vmx_check_apicv_controls(struct kvm_vcpu *vcpu, 655 struct vmcs12 *vmcs12) 656 { 657 if (!nested_cpu_has_virt_x2apic_mode(vmcs12) && 658 !nested_cpu_has_apic_reg_virt(vmcs12) && 659 !nested_cpu_has_vid(vmcs12) && 660 !nested_cpu_has_posted_intr(vmcs12)) 661 return 0; 662 663 /* 664 * If virtualize x2apic mode is enabled, 665 * virtualize apic access must be disabled. 666 */ 667 if (nested_cpu_has_virt_x2apic_mode(vmcs12) && 668 nested_cpu_has2(vmcs12, SECONDARY_EXEC_VIRTUALIZE_APIC_ACCESSES)) 669 return -EINVAL; 670 671 /* 672 * If virtual interrupt delivery is enabled, 673 * we must exit on external interrupts. 674 */ 675 if (nested_cpu_has_vid(vmcs12) && 676 !nested_exit_on_intr(vcpu)) 677 return -EINVAL; 678 679 /* 680 * bits 15:8 should be zero in posted_intr_nv, 681 * the descriptor address has been already checked 682 * in nested_get_vmcs12_pages. 683 * 684 * bits 5:0 of posted_intr_desc_addr should be zero. 685 */ 686 if (nested_cpu_has_posted_intr(vmcs12) && 687 (!nested_cpu_has_vid(vmcs12) || 688 !nested_exit_intr_ack_set(vcpu) || 689 (vmcs12->posted_intr_nv & 0xff00) || 690 (vmcs12->posted_intr_desc_addr & 0x3f) || 691 (vmcs12->posted_intr_desc_addr >> cpuid_maxphyaddr(vcpu)))) 692 return -EINVAL; 693 694 /* tpr shadow is needed by all apicv features. */ 695 if (!nested_cpu_has(vmcs12, CPU_BASED_TPR_SHADOW)) 696 return -EINVAL; 697 698 return 0; 699 } 700 701 static int nested_vmx_check_msr_switch(struct kvm_vcpu *vcpu, 702 u32 count, u64 addr) 703 { 704 int maxphyaddr; 705 706 if (count == 0) 707 return 0; 708 maxphyaddr = cpuid_maxphyaddr(vcpu); 709 if (!IS_ALIGNED(addr, 16) || addr >> maxphyaddr || 710 (addr + count * sizeof(struct vmx_msr_entry) - 1) >> maxphyaddr) 711 return -EINVAL; 712 713 return 0; 714 } 715 716 static int nested_vmx_check_exit_msr_switch_controls(struct kvm_vcpu *vcpu, 717 struct vmcs12 *vmcs12) 718 { 719 if (nested_vmx_check_msr_switch(vcpu, vmcs12->vm_exit_msr_load_count, 720 vmcs12->vm_exit_msr_load_addr) || 721 nested_vmx_check_msr_switch(vcpu, vmcs12->vm_exit_msr_store_count, 722 vmcs12->vm_exit_msr_store_addr)) 723 return -EINVAL; 724 725 return 0; 726 } 727 728 static int nested_vmx_check_entry_msr_switch_controls(struct kvm_vcpu *vcpu, 729 struct vmcs12 *vmcs12) 730 { 731 if (nested_vmx_check_msr_switch(vcpu, vmcs12->vm_entry_msr_load_count, 732 vmcs12->vm_entry_msr_load_addr)) 733 return -EINVAL; 734 735 return 0; 736 } 737 738 static int nested_vmx_check_pml_controls(struct kvm_vcpu *vcpu, 739 struct vmcs12 *vmcs12) 740 { 741 if (!nested_cpu_has_pml(vmcs12)) 742 return 0; 743 744 if (!nested_cpu_has_ept(vmcs12) || 745 !page_address_valid(vcpu, vmcs12->pml_address)) 746 return -EINVAL; 747 748 return 0; 749 } 750 751 static int nested_vmx_check_unrestricted_guest_controls(struct kvm_vcpu *vcpu, 752 struct vmcs12 *vmcs12) 753 { 754 if (nested_cpu_has2(vmcs12, SECONDARY_EXEC_UNRESTRICTED_GUEST) && 755 !nested_cpu_has_ept(vmcs12)) 756 return -EINVAL; 757 return 0; 758 } 759 760 static int nested_vmx_check_mode_based_ept_exec_controls(struct kvm_vcpu *vcpu, 761 struct vmcs12 *vmcs12) 762 { 763 if (nested_cpu_has2(vmcs12, SECONDARY_EXEC_MODE_BASED_EPT_EXEC) && 764 !nested_cpu_has_ept(vmcs12)) 765 return -EINVAL; 766 return 0; 767 } 768 769 static int nested_vmx_check_shadow_vmcs_controls(struct kvm_vcpu *vcpu, 770 struct vmcs12 *vmcs12) 771 { 772 if (!nested_cpu_has_shadow_vmcs(vmcs12)) 773 return 0; 774 775 if (!page_address_valid(vcpu, vmcs12->vmread_bitmap) || 776 !page_address_valid(vcpu, vmcs12->vmwrite_bitmap)) 777 return -EINVAL; 778 779 return 0; 780 } 781 782 static int nested_vmx_msr_check_common(struct kvm_vcpu *vcpu, 783 struct vmx_msr_entry *e) 784 { 785 /* x2APIC MSR accesses are not allowed */ 786 if (vcpu->arch.apic_base & X2APIC_ENABLE && e->index >> 8 == 0x8) 787 return -EINVAL; 788 if (e->index == MSR_IA32_UCODE_WRITE || /* SDM Table 35-2 */ 789 e->index == MSR_IA32_UCODE_REV) 790 return -EINVAL; 791 if (e->reserved != 0) 792 return -EINVAL; 793 return 0; 794 } 795 796 static int nested_vmx_load_msr_check(struct kvm_vcpu *vcpu, 797 struct vmx_msr_entry *e) 798 { 799 if (e->index == MSR_FS_BASE || 800 e->index == MSR_GS_BASE || 801 e->index == MSR_IA32_SMM_MONITOR_CTL || /* SMM is not supported */ 802 nested_vmx_msr_check_common(vcpu, e)) 803 return -EINVAL; 804 return 0; 805 } 806 807 static int nested_vmx_store_msr_check(struct kvm_vcpu *vcpu, 808 struct vmx_msr_entry *e) 809 { 810 if (e->index == MSR_IA32_SMBASE || /* SMM is not supported */ 811 nested_vmx_msr_check_common(vcpu, e)) 812 return -EINVAL; 813 return 0; 814 } 815 816 /* 817 * Load guest's/host's msr at nested entry/exit. 818 * return 0 for success, entry index for failure. 819 */ 820 static u32 nested_vmx_load_msr(struct kvm_vcpu *vcpu, u64 gpa, u32 count) 821 { 822 u32 i; 823 struct vmx_msr_entry e; 824 struct msr_data msr; 825 826 msr.host_initiated = false; 827 for (i = 0; i < count; i++) { 828 if (kvm_vcpu_read_guest(vcpu, gpa + i * sizeof(e), 829 &e, sizeof(e))) { 830 pr_debug_ratelimited( 831 "%s cannot read MSR entry (%u, 0x%08llx)\n", 832 __func__, i, gpa + i * sizeof(e)); 833 goto fail; 834 } 835 if (nested_vmx_load_msr_check(vcpu, &e)) { 836 pr_debug_ratelimited( 837 "%s check failed (%u, 0x%x, 0x%x)\n", 838 __func__, i, e.index, e.reserved); 839 goto fail; 840 } 841 msr.index = e.index; 842 msr.data = e.value; 843 if (kvm_set_msr(vcpu, &msr)) { 844 pr_debug_ratelimited( 845 "%s cannot write MSR (%u, 0x%x, 0x%llx)\n", 846 __func__, i, e.index, e.value); 847 goto fail; 848 } 849 } 850 return 0; 851 fail: 852 return i + 1; 853 } 854 855 static int nested_vmx_store_msr(struct kvm_vcpu *vcpu, u64 gpa, u32 count) 856 { 857 u32 i; 858 struct vmx_msr_entry e; 859 860 for (i = 0; i < count; i++) { 861 struct msr_data msr_info; 862 if (kvm_vcpu_read_guest(vcpu, 863 gpa + i * sizeof(e), 864 &e, 2 * sizeof(u32))) { 865 pr_debug_ratelimited( 866 "%s cannot read MSR entry (%u, 0x%08llx)\n", 867 __func__, i, gpa + i * sizeof(e)); 868 return -EINVAL; 869 } 870 if (nested_vmx_store_msr_check(vcpu, &e)) { 871 pr_debug_ratelimited( 872 "%s check failed (%u, 0x%x, 0x%x)\n", 873 __func__, i, e.index, e.reserved); 874 return -EINVAL; 875 } 876 msr_info.host_initiated = false; 877 msr_info.index = e.index; 878 if (kvm_get_msr(vcpu, &msr_info)) { 879 pr_debug_ratelimited( 880 "%s cannot read MSR (%u, 0x%x)\n", 881 __func__, i, e.index); 882 return -EINVAL; 883 } 884 if (kvm_vcpu_write_guest(vcpu, 885 gpa + i * sizeof(e) + 886 offsetof(struct vmx_msr_entry, value), 887 &msr_info.data, sizeof(msr_info.data))) { 888 pr_debug_ratelimited( 889 "%s cannot write MSR (%u, 0x%x, 0x%llx)\n", 890 __func__, i, e.index, msr_info.data); 891 return -EINVAL; 892 } 893 } 894 return 0; 895 } 896 897 static bool nested_cr3_valid(struct kvm_vcpu *vcpu, unsigned long val) 898 { 899 unsigned long invalid_mask; 900 901 invalid_mask = (~0ULL) << cpuid_maxphyaddr(vcpu); 902 return (val & invalid_mask) == 0; 903 } 904 905 /* 906 * Load guest's/host's cr3 at nested entry/exit. nested_ept is true if we are 907 * emulating VM entry into a guest with EPT enabled. 908 * Returns 0 on success, 1 on failure. Invalid state exit qualification code 909 * is assigned to entry_failure_code on failure. 910 */ 911 static int nested_vmx_load_cr3(struct kvm_vcpu *vcpu, unsigned long cr3, bool nested_ept, 912 u32 *entry_failure_code) 913 { 914 if (cr3 != kvm_read_cr3(vcpu) || (!nested_ept && pdptrs_changed(vcpu))) { 915 if (!nested_cr3_valid(vcpu, cr3)) { 916 *entry_failure_code = ENTRY_FAIL_DEFAULT; 917 return 1; 918 } 919 920 /* 921 * If PAE paging and EPT are both on, CR3 is not used by the CPU and 922 * must not be dereferenced. 923 */ 924 if (!is_long_mode(vcpu) && is_pae(vcpu) && is_paging(vcpu) && 925 !nested_ept) { 926 if (!load_pdptrs(vcpu, vcpu->arch.walk_mmu, cr3)) { 927 *entry_failure_code = ENTRY_FAIL_PDPTE; 928 return 1; 929 } 930 } 931 } 932 933 if (!nested_ept) 934 kvm_mmu_new_cr3(vcpu, cr3, false); 935 936 vcpu->arch.cr3 = cr3; 937 __set_bit(VCPU_EXREG_CR3, (ulong *)&vcpu->arch.regs_avail); 938 939 kvm_init_mmu(vcpu, false); 940 941 return 0; 942 } 943 944 /* 945 * Returns if KVM is able to config CPU to tag TLB entries 946 * populated by L2 differently than TLB entries populated 947 * by L1. 948 * 949 * If L1 uses EPT, then TLB entries are tagged with different EPTP. 950 * 951 * If L1 uses VPID and we allocated a vpid02, TLB entries are tagged 952 * with different VPID (L1 entries are tagged with vmx->vpid 953 * while L2 entries are tagged with vmx->nested.vpid02). 954 */ 955 static bool nested_has_guest_tlb_tag(struct kvm_vcpu *vcpu) 956 { 957 struct vmcs12 *vmcs12 = get_vmcs12(vcpu); 958 959 return nested_cpu_has_ept(vmcs12) || 960 (nested_cpu_has_vpid(vmcs12) && to_vmx(vcpu)->nested.vpid02); 961 } 962 963 static u16 nested_get_vpid02(struct kvm_vcpu *vcpu) 964 { 965 struct vcpu_vmx *vmx = to_vmx(vcpu); 966 967 return vmx->nested.vpid02 ? vmx->nested.vpid02 : vmx->vpid; 968 } 969 970 971 static inline bool vmx_control_verify(u32 control, u32 low, u32 high) 972 { 973 return fixed_bits_valid(control, low, high); 974 } 975 976 static inline u64 vmx_control_msr(u32 low, u32 high) 977 { 978 return low | ((u64)high << 32); 979 } 980 981 static bool is_bitwise_subset(u64 superset, u64 subset, u64 mask) 982 { 983 superset &= mask; 984 subset &= mask; 985 986 return (superset | subset) == superset; 987 } 988 989 static int vmx_restore_vmx_basic(struct vcpu_vmx *vmx, u64 data) 990 { 991 const u64 feature_and_reserved = 992 /* feature (except bit 48; see below) */ 993 BIT_ULL(49) | BIT_ULL(54) | BIT_ULL(55) | 994 /* reserved */ 995 BIT_ULL(31) | GENMASK_ULL(47, 45) | GENMASK_ULL(63, 56); 996 u64 vmx_basic = vmx->nested.msrs.basic; 997 998 if (!is_bitwise_subset(vmx_basic, data, feature_and_reserved)) 999 return -EINVAL; 1000 1001 /* 1002 * KVM does not emulate a version of VMX that constrains physical 1003 * addresses of VMX structures (e.g. VMCS) to 32-bits. 1004 */ 1005 if (data & BIT_ULL(48)) 1006 return -EINVAL; 1007 1008 if (vmx_basic_vmcs_revision_id(vmx_basic) != 1009 vmx_basic_vmcs_revision_id(data)) 1010 return -EINVAL; 1011 1012 if (vmx_basic_vmcs_size(vmx_basic) > vmx_basic_vmcs_size(data)) 1013 return -EINVAL; 1014 1015 vmx->nested.msrs.basic = data; 1016 return 0; 1017 } 1018 1019 static int 1020 vmx_restore_control_msr(struct vcpu_vmx *vmx, u32 msr_index, u64 data) 1021 { 1022 u64 supported; 1023 u32 *lowp, *highp; 1024 1025 switch (msr_index) { 1026 case MSR_IA32_VMX_TRUE_PINBASED_CTLS: 1027 lowp = &vmx->nested.msrs.pinbased_ctls_low; 1028 highp = &vmx->nested.msrs.pinbased_ctls_high; 1029 break; 1030 case MSR_IA32_VMX_TRUE_PROCBASED_CTLS: 1031 lowp = &vmx->nested.msrs.procbased_ctls_low; 1032 highp = &vmx->nested.msrs.procbased_ctls_high; 1033 break; 1034 case MSR_IA32_VMX_TRUE_EXIT_CTLS: 1035 lowp = &vmx->nested.msrs.exit_ctls_low; 1036 highp = &vmx->nested.msrs.exit_ctls_high; 1037 break; 1038 case MSR_IA32_VMX_TRUE_ENTRY_CTLS: 1039 lowp = &vmx->nested.msrs.entry_ctls_low; 1040 highp = &vmx->nested.msrs.entry_ctls_high; 1041 break; 1042 case MSR_IA32_VMX_PROCBASED_CTLS2: 1043 lowp = &vmx->nested.msrs.secondary_ctls_low; 1044 highp = &vmx->nested.msrs.secondary_ctls_high; 1045 break; 1046 default: 1047 BUG(); 1048 } 1049 1050 supported = vmx_control_msr(*lowp, *highp); 1051 1052 /* Check must-be-1 bits are still 1. */ 1053 if (!is_bitwise_subset(data, supported, GENMASK_ULL(31, 0))) 1054 return -EINVAL; 1055 1056 /* Check must-be-0 bits are still 0. */ 1057 if (!is_bitwise_subset(supported, data, GENMASK_ULL(63, 32))) 1058 return -EINVAL; 1059 1060 *lowp = data; 1061 *highp = data >> 32; 1062 return 0; 1063 } 1064 1065 static int vmx_restore_vmx_misc(struct vcpu_vmx *vmx, u64 data) 1066 { 1067 const u64 feature_and_reserved_bits = 1068 /* feature */ 1069 BIT_ULL(5) | GENMASK_ULL(8, 6) | BIT_ULL(14) | BIT_ULL(15) | 1070 BIT_ULL(28) | BIT_ULL(29) | BIT_ULL(30) | 1071 /* reserved */ 1072 GENMASK_ULL(13, 9) | BIT_ULL(31); 1073 u64 vmx_misc; 1074 1075 vmx_misc = vmx_control_msr(vmx->nested.msrs.misc_low, 1076 vmx->nested.msrs.misc_high); 1077 1078 if (!is_bitwise_subset(vmx_misc, data, feature_and_reserved_bits)) 1079 return -EINVAL; 1080 1081 if ((vmx->nested.msrs.pinbased_ctls_high & 1082 PIN_BASED_VMX_PREEMPTION_TIMER) && 1083 vmx_misc_preemption_timer_rate(data) != 1084 vmx_misc_preemption_timer_rate(vmx_misc)) 1085 return -EINVAL; 1086 1087 if (vmx_misc_cr3_count(data) > vmx_misc_cr3_count(vmx_misc)) 1088 return -EINVAL; 1089 1090 if (vmx_misc_max_msr(data) > vmx_misc_max_msr(vmx_misc)) 1091 return -EINVAL; 1092 1093 if (vmx_misc_mseg_revid(data) != vmx_misc_mseg_revid(vmx_misc)) 1094 return -EINVAL; 1095 1096 vmx->nested.msrs.misc_low = data; 1097 vmx->nested.msrs.misc_high = data >> 32; 1098 1099 /* 1100 * If L1 has read-only VM-exit information fields, use the 1101 * less permissive vmx_vmwrite_bitmap to specify write 1102 * permissions for the shadow VMCS. 1103 */ 1104 if (enable_shadow_vmcs && !nested_cpu_has_vmwrite_any_field(&vmx->vcpu)) 1105 vmcs_write64(VMWRITE_BITMAP, __pa(vmx_vmwrite_bitmap)); 1106 1107 return 0; 1108 } 1109 1110 static int vmx_restore_vmx_ept_vpid_cap(struct vcpu_vmx *vmx, u64 data) 1111 { 1112 u64 vmx_ept_vpid_cap; 1113 1114 vmx_ept_vpid_cap = vmx_control_msr(vmx->nested.msrs.ept_caps, 1115 vmx->nested.msrs.vpid_caps); 1116 1117 /* Every bit is either reserved or a feature bit. */ 1118 if (!is_bitwise_subset(vmx_ept_vpid_cap, data, -1ULL)) 1119 return -EINVAL; 1120 1121 vmx->nested.msrs.ept_caps = data; 1122 vmx->nested.msrs.vpid_caps = data >> 32; 1123 return 0; 1124 } 1125 1126 static int vmx_restore_fixed0_msr(struct vcpu_vmx *vmx, u32 msr_index, u64 data) 1127 { 1128 u64 *msr; 1129 1130 switch (msr_index) { 1131 case MSR_IA32_VMX_CR0_FIXED0: 1132 msr = &vmx->nested.msrs.cr0_fixed0; 1133 break; 1134 case MSR_IA32_VMX_CR4_FIXED0: 1135 msr = &vmx->nested.msrs.cr4_fixed0; 1136 break; 1137 default: 1138 BUG(); 1139 } 1140 1141 /* 1142 * 1 bits (which indicates bits which "must-be-1" during VMX operation) 1143 * must be 1 in the restored value. 1144 */ 1145 if (!is_bitwise_subset(data, *msr, -1ULL)) 1146 return -EINVAL; 1147 1148 *msr = data; 1149 return 0; 1150 } 1151 1152 /* 1153 * Called when userspace is restoring VMX MSRs. 1154 * 1155 * Returns 0 on success, non-0 otherwise. 1156 */ 1157 int vmx_set_vmx_msr(struct kvm_vcpu *vcpu, u32 msr_index, u64 data) 1158 { 1159 struct vcpu_vmx *vmx = to_vmx(vcpu); 1160 1161 /* 1162 * Don't allow changes to the VMX capability MSRs while the vCPU 1163 * is in VMX operation. 1164 */ 1165 if (vmx->nested.vmxon) 1166 return -EBUSY; 1167 1168 switch (msr_index) { 1169 case MSR_IA32_VMX_BASIC: 1170 return vmx_restore_vmx_basic(vmx, data); 1171 case MSR_IA32_VMX_PINBASED_CTLS: 1172 case MSR_IA32_VMX_PROCBASED_CTLS: 1173 case MSR_IA32_VMX_EXIT_CTLS: 1174 case MSR_IA32_VMX_ENTRY_CTLS: 1175 /* 1176 * The "non-true" VMX capability MSRs are generated from the 1177 * "true" MSRs, so we do not support restoring them directly. 1178 * 1179 * If userspace wants to emulate VMX_BASIC[55]=0, userspace 1180 * should restore the "true" MSRs with the must-be-1 bits 1181 * set according to the SDM Vol 3. A.2 "RESERVED CONTROLS AND 1182 * DEFAULT SETTINGS". 1183 */ 1184 return -EINVAL; 1185 case MSR_IA32_VMX_TRUE_PINBASED_CTLS: 1186 case MSR_IA32_VMX_TRUE_PROCBASED_CTLS: 1187 case MSR_IA32_VMX_TRUE_EXIT_CTLS: 1188 case MSR_IA32_VMX_TRUE_ENTRY_CTLS: 1189 case MSR_IA32_VMX_PROCBASED_CTLS2: 1190 return vmx_restore_control_msr(vmx, msr_index, data); 1191 case MSR_IA32_VMX_MISC: 1192 return vmx_restore_vmx_misc(vmx, data); 1193 case MSR_IA32_VMX_CR0_FIXED0: 1194 case MSR_IA32_VMX_CR4_FIXED0: 1195 return vmx_restore_fixed0_msr(vmx, msr_index, data); 1196 case MSR_IA32_VMX_CR0_FIXED1: 1197 case MSR_IA32_VMX_CR4_FIXED1: 1198 /* 1199 * These MSRs are generated based on the vCPU's CPUID, so we 1200 * do not support restoring them directly. 1201 */ 1202 return -EINVAL; 1203 case MSR_IA32_VMX_EPT_VPID_CAP: 1204 return vmx_restore_vmx_ept_vpid_cap(vmx, data); 1205 case MSR_IA32_VMX_VMCS_ENUM: 1206 vmx->nested.msrs.vmcs_enum = data; 1207 return 0; 1208 default: 1209 /* 1210 * The rest of the VMX capability MSRs do not support restore. 1211 */ 1212 return -EINVAL; 1213 } 1214 } 1215 1216 /* Returns 0 on success, non-0 otherwise. */ 1217 int vmx_get_vmx_msr(struct nested_vmx_msrs *msrs, u32 msr_index, u64 *pdata) 1218 { 1219 switch (msr_index) { 1220 case MSR_IA32_VMX_BASIC: 1221 *pdata = msrs->basic; 1222 break; 1223 case MSR_IA32_VMX_TRUE_PINBASED_CTLS: 1224 case MSR_IA32_VMX_PINBASED_CTLS: 1225 *pdata = vmx_control_msr( 1226 msrs->pinbased_ctls_low, 1227 msrs->pinbased_ctls_high); 1228 if (msr_index == MSR_IA32_VMX_PINBASED_CTLS) 1229 *pdata |= PIN_BASED_ALWAYSON_WITHOUT_TRUE_MSR; 1230 break; 1231 case MSR_IA32_VMX_TRUE_PROCBASED_CTLS: 1232 case MSR_IA32_VMX_PROCBASED_CTLS: 1233 *pdata = vmx_control_msr( 1234 msrs->procbased_ctls_low, 1235 msrs->procbased_ctls_high); 1236 if (msr_index == MSR_IA32_VMX_PROCBASED_CTLS) 1237 *pdata |= CPU_BASED_ALWAYSON_WITHOUT_TRUE_MSR; 1238 break; 1239 case MSR_IA32_VMX_TRUE_EXIT_CTLS: 1240 case MSR_IA32_VMX_EXIT_CTLS: 1241 *pdata = vmx_control_msr( 1242 msrs->exit_ctls_low, 1243 msrs->exit_ctls_high); 1244 if (msr_index == MSR_IA32_VMX_EXIT_CTLS) 1245 *pdata |= VM_EXIT_ALWAYSON_WITHOUT_TRUE_MSR; 1246 break; 1247 case MSR_IA32_VMX_TRUE_ENTRY_CTLS: 1248 case MSR_IA32_VMX_ENTRY_CTLS: 1249 *pdata = vmx_control_msr( 1250 msrs->entry_ctls_low, 1251 msrs->entry_ctls_high); 1252 if (msr_index == MSR_IA32_VMX_ENTRY_CTLS) 1253 *pdata |= VM_ENTRY_ALWAYSON_WITHOUT_TRUE_MSR; 1254 break; 1255 case MSR_IA32_VMX_MISC: 1256 *pdata = vmx_control_msr( 1257 msrs->misc_low, 1258 msrs->misc_high); 1259 break; 1260 case MSR_IA32_VMX_CR0_FIXED0: 1261 *pdata = msrs->cr0_fixed0; 1262 break; 1263 case MSR_IA32_VMX_CR0_FIXED1: 1264 *pdata = msrs->cr0_fixed1; 1265 break; 1266 case MSR_IA32_VMX_CR4_FIXED0: 1267 *pdata = msrs->cr4_fixed0; 1268 break; 1269 case MSR_IA32_VMX_CR4_FIXED1: 1270 *pdata = msrs->cr4_fixed1; 1271 break; 1272 case MSR_IA32_VMX_VMCS_ENUM: 1273 *pdata = msrs->vmcs_enum; 1274 break; 1275 case MSR_IA32_VMX_PROCBASED_CTLS2: 1276 *pdata = vmx_control_msr( 1277 msrs->secondary_ctls_low, 1278 msrs->secondary_ctls_high); 1279 break; 1280 case MSR_IA32_VMX_EPT_VPID_CAP: 1281 *pdata = msrs->ept_caps | 1282 ((u64)msrs->vpid_caps << 32); 1283 break; 1284 case MSR_IA32_VMX_VMFUNC: 1285 *pdata = msrs->vmfunc_controls; 1286 break; 1287 default: 1288 return 1; 1289 } 1290 1291 return 0; 1292 } 1293 1294 /* 1295 * Copy the writable VMCS shadow fields back to the VMCS12, in case 1296 * they have been modified by the L1 guest. Note that the "read-only" 1297 * VM-exit information fields are actually writable if the vCPU is 1298 * configured to support "VMWRITE to any supported field in the VMCS." 1299 */ 1300 static void copy_shadow_to_vmcs12(struct vcpu_vmx *vmx) 1301 { 1302 const u16 *fields[] = { 1303 shadow_read_write_fields, 1304 shadow_read_only_fields 1305 }; 1306 const int max_fields[] = { 1307 max_shadow_read_write_fields, 1308 max_shadow_read_only_fields 1309 }; 1310 int i, q; 1311 unsigned long field; 1312 u64 field_value; 1313 struct vmcs *shadow_vmcs = vmx->vmcs01.shadow_vmcs; 1314 1315 preempt_disable(); 1316 1317 vmcs_load(shadow_vmcs); 1318 1319 for (q = 0; q < ARRAY_SIZE(fields); q++) { 1320 for (i = 0; i < max_fields[q]; i++) { 1321 field = fields[q][i]; 1322 field_value = __vmcs_readl(field); 1323 vmcs12_write_any(get_vmcs12(&vmx->vcpu), field, field_value); 1324 } 1325 /* 1326 * Skip the VM-exit information fields if they are read-only. 1327 */ 1328 if (!nested_cpu_has_vmwrite_any_field(&vmx->vcpu)) 1329 break; 1330 } 1331 1332 vmcs_clear(shadow_vmcs); 1333 vmcs_load(vmx->loaded_vmcs->vmcs); 1334 1335 preempt_enable(); 1336 } 1337 1338 static void copy_vmcs12_to_shadow(struct vcpu_vmx *vmx) 1339 { 1340 const u16 *fields[] = { 1341 shadow_read_write_fields, 1342 shadow_read_only_fields 1343 }; 1344 const int max_fields[] = { 1345 max_shadow_read_write_fields, 1346 max_shadow_read_only_fields 1347 }; 1348 int i, q; 1349 unsigned long field; 1350 u64 field_value = 0; 1351 struct vmcs *shadow_vmcs = vmx->vmcs01.shadow_vmcs; 1352 1353 vmcs_load(shadow_vmcs); 1354 1355 for (q = 0; q < ARRAY_SIZE(fields); q++) { 1356 for (i = 0; i < max_fields[q]; i++) { 1357 field = fields[q][i]; 1358 vmcs12_read_any(get_vmcs12(&vmx->vcpu), field, &field_value); 1359 __vmcs_writel(field, field_value); 1360 } 1361 } 1362 1363 vmcs_clear(shadow_vmcs); 1364 vmcs_load(vmx->loaded_vmcs->vmcs); 1365 } 1366 1367 static int copy_enlightened_to_vmcs12(struct vcpu_vmx *vmx) 1368 { 1369 struct vmcs12 *vmcs12 = vmx->nested.cached_vmcs12; 1370 struct hv_enlightened_vmcs *evmcs = vmx->nested.hv_evmcs; 1371 1372 /* HV_VMX_ENLIGHTENED_CLEAN_FIELD_NONE */ 1373 vmcs12->tpr_threshold = evmcs->tpr_threshold; 1374 vmcs12->guest_rip = evmcs->guest_rip; 1375 1376 if (unlikely(!(evmcs->hv_clean_fields & 1377 HV_VMX_ENLIGHTENED_CLEAN_FIELD_GUEST_BASIC))) { 1378 vmcs12->guest_rsp = evmcs->guest_rsp; 1379 vmcs12->guest_rflags = evmcs->guest_rflags; 1380 vmcs12->guest_interruptibility_info = 1381 evmcs->guest_interruptibility_info; 1382 } 1383 1384 if (unlikely(!(evmcs->hv_clean_fields & 1385 HV_VMX_ENLIGHTENED_CLEAN_FIELD_CONTROL_PROC))) { 1386 vmcs12->cpu_based_vm_exec_control = 1387 evmcs->cpu_based_vm_exec_control; 1388 } 1389 1390 if (unlikely(!(evmcs->hv_clean_fields & 1391 HV_VMX_ENLIGHTENED_CLEAN_FIELD_CONTROL_PROC))) { 1392 vmcs12->exception_bitmap = evmcs->exception_bitmap; 1393 } 1394 1395 if (unlikely(!(evmcs->hv_clean_fields & 1396 HV_VMX_ENLIGHTENED_CLEAN_FIELD_CONTROL_ENTRY))) { 1397 vmcs12->vm_entry_controls = evmcs->vm_entry_controls; 1398 } 1399 1400 if (unlikely(!(evmcs->hv_clean_fields & 1401 HV_VMX_ENLIGHTENED_CLEAN_FIELD_CONTROL_EVENT))) { 1402 vmcs12->vm_entry_intr_info_field = 1403 evmcs->vm_entry_intr_info_field; 1404 vmcs12->vm_entry_exception_error_code = 1405 evmcs->vm_entry_exception_error_code; 1406 vmcs12->vm_entry_instruction_len = 1407 evmcs->vm_entry_instruction_len; 1408 } 1409 1410 if (unlikely(!(evmcs->hv_clean_fields & 1411 HV_VMX_ENLIGHTENED_CLEAN_FIELD_HOST_GRP1))) { 1412 vmcs12->host_ia32_pat = evmcs->host_ia32_pat; 1413 vmcs12->host_ia32_efer = evmcs->host_ia32_efer; 1414 vmcs12->host_cr0 = evmcs->host_cr0; 1415 vmcs12->host_cr3 = evmcs->host_cr3; 1416 vmcs12->host_cr4 = evmcs->host_cr4; 1417 vmcs12->host_ia32_sysenter_esp = evmcs->host_ia32_sysenter_esp; 1418 vmcs12->host_ia32_sysenter_eip = evmcs->host_ia32_sysenter_eip; 1419 vmcs12->host_rip = evmcs->host_rip; 1420 vmcs12->host_ia32_sysenter_cs = evmcs->host_ia32_sysenter_cs; 1421 vmcs12->host_es_selector = evmcs->host_es_selector; 1422 vmcs12->host_cs_selector = evmcs->host_cs_selector; 1423 vmcs12->host_ss_selector = evmcs->host_ss_selector; 1424 vmcs12->host_ds_selector = evmcs->host_ds_selector; 1425 vmcs12->host_fs_selector = evmcs->host_fs_selector; 1426 vmcs12->host_gs_selector = evmcs->host_gs_selector; 1427 vmcs12->host_tr_selector = evmcs->host_tr_selector; 1428 } 1429 1430 if (unlikely(!(evmcs->hv_clean_fields & 1431 HV_VMX_ENLIGHTENED_CLEAN_FIELD_HOST_GRP1))) { 1432 vmcs12->pin_based_vm_exec_control = 1433 evmcs->pin_based_vm_exec_control; 1434 vmcs12->vm_exit_controls = evmcs->vm_exit_controls; 1435 vmcs12->secondary_vm_exec_control = 1436 evmcs->secondary_vm_exec_control; 1437 } 1438 1439 if (unlikely(!(evmcs->hv_clean_fields & 1440 HV_VMX_ENLIGHTENED_CLEAN_FIELD_IO_BITMAP))) { 1441 vmcs12->io_bitmap_a = evmcs->io_bitmap_a; 1442 vmcs12->io_bitmap_b = evmcs->io_bitmap_b; 1443 } 1444 1445 if (unlikely(!(evmcs->hv_clean_fields & 1446 HV_VMX_ENLIGHTENED_CLEAN_FIELD_MSR_BITMAP))) { 1447 vmcs12->msr_bitmap = evmcs->msr_bitmap; 1448 } 1449 1450 if (unlikely(!(evmcs->hv_clean_fields & 1451 HV_VMX_ENLIGHTENED_CLEAN_FIELD_GUEST_GRP2))) { 1452 vmcs12->guest_es_base = evmcs->guest_es_base; 1453 vmcs12->guest_cs_base = evmcs->guest_cs_base; 1454 vmcs12->guest_ss_base = evmcs->guest_ss_base; 1455 vmcs12->guest_ds_base = evmcs->guest_ds_base; 1456 vmcs12->guest_fs_base = evmcs->guest_fs_base; 1457 vmcs12->guest_gs_base = evmcs->guest_gs_base; 1458 vmcs12->guest_ldtr_base = evmcs->guest_ldtr_base; 1459 vmcs12->guest_tr_base = evmcs->guest_tr_base; 1460 vmcs12->guest_gdtr_base = evmcs->guest_gdtr_base; 1461 vmcs12->guest_idtr_base = evmcs->guest_idtr_base; 1462 vmcs12->guest_es_limit = evmcs->guest_es_limit; 1463 vmcs12->guest_cs_limit = evmcs->guest_cs_limit; 1464 vmcs12->guest_ss_limit = evmcs->guest_ss_limit; 1465 vmcs12->guest_ds_limit = evmcs->guest_ds_limit; 1466 vmcs12->guest_fs_limit = evmcs->guest_fs_limit; 1467 vmcs12->guest_gs_limit = evmcs->guest_gs_limit; 1468 vmcs12->guest_ldtr_limit = evmcs->guest_ldtr_limit; 1469 vmcs12->guest_tr_limit = evmcs->guest_tr_limit; 1470 vmcs12->guest_gdtr_limit = evmcs->guest_gdtr_limit; 1471 vmcs12->guest_idtr_limit = evmcs->guest_idtr_limit; 1472 vmcs12->guest_es_ar_bytes = evmcs->guest_es_ar_bytes; 1473 vmcs12->guest_cs_ar_bytes = evmcs->guest_cs_ar_bytes; 1474 vmcs12->guest_ss_ar_bytes = evmcs->guest_ss_ar_bytes; 1475 vmcs12->guest_ds_ar_bytes = evmcs->guest_ds_ar_bytes; 1476 vmcs12->guest_fs_ar_bytes = evmcs->guest_fs_ar_bytes; 1477 vmcs12->guest_gs_ar_bytes = evmcs->guest_gs_ar_bytes; 1478 vmcs12->guest_ldtr_ar_bytes = evmcs->guest_ldtr_ar_bytes; 1479 vmcs12->guest_tr_ar_bytes = evmcs->guest_tr_ar_bytes; 1480 vmcs12->guest_es_selector = evmcs->guest_es_selector; 1481 vmcs12->guest_cs_selector = evmcs->guest_cs_selector; 1482 vmcs12->guest_ss_selector = evmcs->guest_ss_selector; 1483 vmcs12->guest_ds_selector = evmcs->guest_ds_selector; 1484 vmcs12->guest_fs_selector = evmcs->guest_fs_selector; 1485 vmcs12->guest_gs_selector = evmcs->guest_gs_selector; 1486 vmcs12->guest_ldtr_selector = evmcs->guest_ldtr_selector; 1487 vmcs12->guest_tr_selector = evmcs->guest_tr_selector; 1488 } 1489 1490 if (unlikely(!(evmcs->hv_clean_fields & 1491 HV_VMX_ENLIGHTENED_CLEAN_FIELD_CONTROL_GRP2))) { 1492 vmcs12->tsc_offset = evmcs->tsc_offset; 1493 vmcs12->virtual_apic_page_addr = evmcs->virtual_apic_page_addr; 1494 vmcs12->xss_exit_bitmap = evmcs->xss_exit_bitmap; 1495 } 1496 1497 if (unlikely(!(evmcs->hv_clean_fields & 1498 HV_VMX_ENLIGHTENED_CLEAN_FIELD_CRDR))) { 1499 vmcs12->cr0_guest_host_mask = evmcs->cr0_guest_host_mask; 1500 vmcs12->cr4_guest_host_mask = evmcs->cr4_guest_host_mask; 1501 vmcs12->cr0_read_shadow = evmcs->cr0_read_shadow; 1502 vmcs12->cr4_read_shadow = evmcs->cr4_read_shadow; 1503 vmcs12->guest_cr0 = evmcs->guest_cr0; 1504 vmcs12->guest_cr3 = evmcs->guest_cr3; 1505 vmcs12->guest_cr4 = evmcs->guest_cr4; 1506 vmcs12->guest_dr7 = evmcs->guest_dr7; 1507 } 1508 1509 if (unlikely(!(evmcs->hv_clean_fields & 1510 HV_VMX_ENLIGHTENED_CLEAN_FIELD_HOST_POINTER))) { 1511 vmcs12->host_fs_base = evmcs->host_fs_base; 1512 vmcs12->host_gs_base = evmcs->host_gs_base; 1513 vmcs12->host_tr_base = evmcs->host_tr_base; 1514 vmcs12->host_gdtr_base = evmcs->host_gdtr_base; 1515 vmcs12->host_idtr_base = evmcs->host_idtr_base; 1516 vmcs12->host_rsp = evmcs->host_rsp; 1517 } 1518 1519 if (unlikely(!(evmcs->hv_clean_fields & 1520 HV_VMX_ENLIGHTENED_CLEAN_FIELD_CONTROL_XLAT))) { 1521 vmcs12->ept_pointer = evmcs->ept_pointer; 1522 vmcs12->virtual_processor_id = evmcs->virtual_processor_id; 1523 } 1524 1525 if (unlikely(!(evmcs->hv_clean_fields & 1526 HV_VMX_ENLIGHTENED_CLEAN_FIELD_GUEST_GRP1))) { 1527 vmcs12->vmcs_link_pointer = evmcs->vmcs_link_pointer; 1528 vmcs12->guest_ia32_debugctl = evmcs->guest_ia32_debugctl; 1529 vmcs12->guest_ia32_pat = evmcs->guest_ia32_pat; 1530 vmcs12->guest_ia32_efer = evmcs->guest_ia32_efer; 1531 vmcs12->guest_pdptr0 = evmcs->guest_pdptr0; 1532 vmcs12->guest_pdptr1 = evmcs->guest_pdptr1; 1533 vmcs12->guest_pdptr2 = evmcs->guest_pdptr2; 1534 vmcs12->guest_pdptr3 = evmcs->guest_pdptr3; 1535 vmcs12->guest_pending_dbg_exceptions = 1536 evmcs->guest_pending_dbg_exceptions; 1537 vmcs12->guest_sysenter_esp = evmcs->guest_sysenter_esp; 1538 vmcs12->guest_sysenter_eip = evmcs->guest_sysenter_eip; 1539 vmcs12->guest_bndcfgs = evmcs->guest_bndcfgs; 1540 vmcs12->guest_activity_state = evmcs->guest_activity_state; 1541 vmcs12->guest_sysenter_cs = evmcs->guest_sysenter_cs; 1542 } 1543 1544 /* 1545 * Not used? 1546 * vmcs12->vm_exit_msr_store_addr = evmcs->vm_exit_msr_store_addr; 1547 * vmcs12->vm_exit_msr_load_addr = evmcs->vm_exit_msr_load_addr; 1548 * vmcs12->vm_entry_msr_load_addr = evmcs->vm_entry_msr_load_addr; 1549 * vmcs12->cr3_target_value0 = evmcs->cr3_target_value0; 1550 * vmcs12->cr3_target_value1 = evmcs->cr3_target_value1; 1551 * vmcs12->cr3_target_value2 = evmcs->cr3_target_value2; 1552 * vmcs12->cr3_target_value3 = evmcs->cr3_target_value3; 1553 * vmcs12->page_fault_error_code_mask = 1554 * evmcs->page_fault_error_code_mask; 1555 * vmcs12->page_fault_error_code_match = 1556 * evmcs->page_fault_error_code_match; 1557 * vmcs12->cr3_target_count = evmcs->cr3_target_count; 1558 * vmcs12->vm_exit_msr_store_count = evmcs->vm_exit_msr_store_count; 1559 * vmcs12->vm_exit_msr_load_count = evmcs->vm_exit_msr_load_count; 1560 * vmcs12->vm_entry_msr_load_count = evmcs->vm_entry_msr_load_count; 1561 */ 1562 1563 /* 1564 * Read only fields: 1565 * vmcs12->guest_physical_address = evmcs->guest_physical_address; 1566 * vmcs12->vm_instruction_error = evmcs->vm_instruction_error; 1567 * vmcs12->vm_exit_reason = evmcs->vm_exit_reason; 1568 * vmcs12->vm_exit_intr_info = evmcs->vm_exit_intr_info; 1569 * vmcs12->vm_exit_intr_error_code = evmcs->vm_exit_intr_error_code; 1570 * vmcs12->idt_vectoring_info_field = evmcs->idt_vectoring_info_field; 1571 * vmcs12->idt_vectoring_error_code = evmcs->idt_vectoring_error_code; 1572 * vmcs12->vm_exit_instruction_len = evmcs->vm_exit_instruction_len; 1573 * vmcs12->vmx_instruction_info = evmcs->vmx_instruction_info; 1574 * vmcs12->exit_qualification = evmcs->exit_qualification; 1575 * vmcs12->guest_linear_address = evmcs->guest_linear_address; 1576 * 1577 * Not present in struct vmcs12: 1578 * vmcs12->exit_io_instruction_ecx = evmcs->exit_io_instruction_ecx; 1579 * vmcs12->exit_io_instruction_esi = evmcs->exit_io_instruction_esi; 1580 * vmcs12->exit_io_instruction_edi = evmcs->exit_io_instruction_edi; 1581 * vmcs12->exit_io_instruction_eip = evmcs->exit_io_instruction_eip; 1582 */ 1583 1584 return 0; 1585 } 1586 1587 static int copy_vmcs12_to_enlightened(struct vcpu_vmx *vmx) 1588 { 1589 struct vmcs12 *vmcs12 = vmx->nested.cached_vmcs12; 1590 struct hv_enlightened_vmcs *evmcs = vmx->nested.hv_evmcs; 1591 1592 /* 1593 * Should not be changed by KVM: 1594 * 1595 * evmcs->host_es_selector = vmcs12->host_es_selector; 1596 * evmcs->host_cs_selector = vmcs12->host_cs_selector; 1597 * evmcs->host_ss_selector = vmcs12->host_ss_selector; 1598 * evmcs->host_ds_selector = vmcs12->host_ds_selector; 1599 * evmcs->host_fs_selector = vmcs12->host_fs_selector; 1600 * evmcs->host_gs_selector = vmcs12->host_gs_selector; 1601 * evmcs->host_tr_selector = vmcs12->host_tr_selector; 1602 * evmcs->host_ia32_pat = vmcs12->host_ia32_pat; 1603 * evmcs->host_ia32_efer = vmcs12->host_ia32_efer; 1604 * evmcs->host_cr0 = vmcs12->host_cr0; 1605 * evmcs->host_cr3 = vmcs12->host_cr3; 1606 * evmcs->host_cr4 = vmcs12->host_cr4; 1607 * evmcs->host_ia32_sysenter_esp = vmcs12->host_ia32_sysenter_esp; 1608 * evmcs->host_ia32_sysenter_eip = vmcs12->host_ia32_sysenter_eip; 1609 * evmcs->host_rip = vmcs12->host_rip; 1610 * evmcs->host_ia32_sysenter_cs = vmcs12->host_ia32_sysenter_cs; 1611 * evmcs->host_fs_base = vmcs12->host_fs_base; 1612 * evmcs->host_gs_base = vmcs12->host_gs_base; 1613 * evmcs->host_tr_base = vmcs12->host_tr_base; 1614 * evmcs->host_gdtr_base = vmcs12->host_gdtr_base; 1615 * evmcs->host_idtr_base = vmcs12->host_idtr_base; 1616 * evmcs->host_rsp = vmcs12->host_rsp; 1617 * sync_vmcs12() doesn't read these: 1618 * evmcs->io_bitmap_a = vmcs12->io_bitmap_a; 1619 * evmcs->io_bitmap_b = vmcs12->io_bitmap_b; 1620 * evmcs->msr_bitmap = vmcs12->msr_bitmap; 1621 * evmcs->ept_pointer = vmcs12->ept_pointer; 1622 * evmcs->xss_exit_bitmap = vmcs12->xss_exit_bitmap; 1623 * evmcs->vm_exit_msr_store_addr = vmcs12->vm_exit_msr_store_addr; 1624 * evmcs->vm_exit_msr_load_addr = vmcs12->vm_exit_msr_load_addr; 1625 * evmcs->vm_entry_msr_load_addr = vmcs12->vm_entry_msr_load_addr; 1626 * evmcs->cr3_target_value0 = vmcs12->cr3_target_value0; 1627 * evmcs->cr3_target_value1 = vmcs12->cr3_target_value1; 1628 * evmcs->cr3_target_value2 = vmcs12->cr3_target_value2; 1629 * evmcs->cr3_target_value3 = vmcs12->cr3_target_value3; 1630 * evmcs->tpr_threshold = vmcs12->tpr_threshold; 1631 * evmcs->virtual_processor_id = vmcs12->virtual_processor_id; 1632 * evmcs->exception_bitmap = vmcs12->exception_bitmap; 1633 * evmcs->vmcs_link_pointer = vmcs12->vmcs_link_pointer; 1634 * evmcs->pin_based_vm_exec_control = vmcs12->pin_based_vm_exec_control; 1635 * evmcs->vm_exit_controls = vmcs12->vm_exit_controls; 1636 * evmcs->secondary_vm_exec_control = vmcs12->secondary_vm_exec_control; 1637 * evmcs->page_fault_error_code_mask = 1638 * vmcs12->page_fault_error_code_mask; 1639 * evmcs->page_fault_error_code_match = 1640 * vmcs12->page_fault_error_code_match; 1641 * evmcs->cr3_target_count = vmcs12->cr3_target_count; 1642 * evmcs->virtual_apic_page_addr = vmcs12->virtual_apic_page_addr; 1643 * evmcs->tsc_offset = vmcs12->tsc_offset; 1644 * evmcs->guest_ia32_debugctl = vmcs12->guest_ia32_debugctl; 1645 * evmcs->cr0_guest_host_mask = vmcs12->cr0_guest_host_mask; 1646 * evmcs->cr4_guest_host_mask = vmcs12->cr4_guest_host_mask; 1647 * evmcs->cr0_read_shadow = vmcs12->cr0_read_shadow; 1648 * evmcs->cr4_read_shadow = vmcs12->cr4_read_shadow; 1649 * evmcs->vm_exit_msr_store_count = vmcs12->vm_exit_msr_store_count; 1650 * evmcs->vm_exit_msr_load_count = vmcs12->vm_exit_msr_load_count; 1651 * evmcs->vm_entry_msr_load_count = vmcs12->vm_entry_msr_load_count; 1652 * 1653 * Not present in struct vmcs12: 1654 * evmcs->exit_io_instruction_ecx = vmcs12->exit_io_instruction_ecx; 1655 * evmcs->exit_io_instruction_esi = vmcs12->exit_io_instruction_esi; 1656 * evmcs->exit_io_instruction_edi = vmcs12->exit_io_instruction_edi; 1657 * evmcs->exit_io_instruction_eip = vmcs12->exit_io_instruction_eip; 1658 */ 1659 1660 evmcs->guest_es_selector = vmcs12->guest_es_selector; 1661 evmcs->guest_cs_selector = vmcs12->guest_cs_selector; 1662 evmcs->guest_ss_selector = vmcs12->guest_ss_selector; 1663 evmcs->guest_ds_selector = vmcs12->guest_ds_selector; 1664 evmcs->guest_fs_selector = vmcs12->guest_fs_selector; 1665 evmcs->guest_gs_selector = vmcs12->guest_gs_selector; 1666 evmcs->guest_ldtr_selector = vmcs12->guest_ldtr_selector; 1667 evmcs->guest_tr_selector = vmcs12->guest_tr_selector; 1668 1669 evmcs->guest_es_limit = vmcs12->guest_es_limit; 1670 evmcs->guest_cs_limit = vmcs12->guest_cs_limit; 1671 evmcs->guest_ss_limit = vmcs12->guest_ss_limit; 1672 evmcs->guest_ds_limit = vmcs12->guest_ds_limit; 1673 evmcs->guest_fs_limit = vmcs12->guest_fs_limit; 1674 evmcs->guest_gs_limit = vmcs12->guest_gs_limit; 1675 evmcs->guest_ldtr_limit = vmcs12->guest_ldtr_limit; 1676 evmcs->guest_tr_limit = vmcs12->guest_tr_limit; 1677 evmcs->guest_gdtr_limit = vmcs12->guest_gdtr_limit; 1678 evmcs->guest_idtr_limit = vmcs12->guest_idtr_limit; 1679 1680 evmcs->guest_es_ar_bytes = vmcs12->guest_es_ar_bytes; 1681 evmcs->guest_cs_ar_bytes = vmcs12->guest_cs_ar_bytes; 1682 evmcs->guest_ss_ar_bytes = vmcs12->guest_ss_ar_bytes; 1683 evmcs->guest_ds_ar_bytes = vmcs12->guest_ds_ar_bytes; 1684 evmcs->guest_fs_ar_bytes = vmcs12->guest_fs_ar_bytes; 1685 evmcs->guest_gs_ar_bytes = vmcs12->guest_gs_ar_bytes; 1686 evmcs->guest_ldtr_ar_bytes = vmcs12->guest_ldtr_ar_bytes; 1687 evmcs->guest_tr_ar_bytes = vmcs12->guest_tr_ar_bytes; 1688 1689 evmcs->guest_es_base = vmcs12->guest_es_base; 1690 evmcs->guest_cs_base = vmcs12->guest_cs_base; 1691 evmcs->guest_ss_base = vmcs12->guest_ss_base; 1692 evmcs->guest_ds_base = vmcs12->guest_ds_base; 1693 evmcs->guest_fs_base = vmcs12->guest_fs_base; 1694 evmcs->guest_gs_base = vmcs12->guest_gs_base; 1695 evmcs->guest_ldtr_base = vmcs12->guest_ldtr_base; 1696 evmcs->guest_tr_base = vmcs12->guest_tr_base; 1697 evmcs->guest_gdtr_base = vmcs12->guest_gdtr_base; 1698 evmcs->guest_idtr_base = vmcs12->guest_idtr_base; 1699 1700 evmcs->guest_ia32_pat = vmcs12->guest_ia32_pat; 1701 evmcs->guest_ia32_efer = vmcs12->guest_ia32_efer; 1702 1703 evmcs->guest_pdptr0 = vmcs12->guest_pdptr0; 1704 evmcs->guest_pdptr1 = vmcs12->guest_pdptr1; 1705 evmcs->guest_pdptr2 = vmcs12->guest_pdptr2; 1706 evmcs->guest_pdptr3 = vmcs12->guest_pdptr3; 1707 1708 evmcs->guest_pending_dbg_exceptions = 1709 vmcs12->guest_pending_dbg_exceptions; 1710 evmcs->guest_sysenter_esp = vmcs12->guest_sysenter_esp; 1711 evmcs->guest_sysenter_eip = vmcs12->guest_sysenter_eip; 1712 1713 evmcs->guest_activity_state = vmcs12->guest_activity_state; 1714 evmcs->guest_sysenter_cs = vmcs12->guest_sysenter_cs; 1715 1716 evmcs->guest_cr0 = vmcs12->guest_cr0; 1717 evmcs->guest_cr3 = vmcs12->guest_cr3; 1718 evmcs->guest_cr4 = vmcs12->guest_cr4; 1719 evmcs->guest_dr7 = vmcs12->guest_dr7; 1720 1721 evmcs->guest_physical_address = vmcs12->guest_physical_address; 1722 1723 evmcs->vm_instruction_error = vmcs12->vm_instruction_error; 1724 evmcs->vm_exit_reason = vmcs12->vm_exit_reason; 1725 evmcs->vm_exit_intr_info = vmcs12->vm_exit_intr_info; 1726 evmcs->vm_exit_intr_error_code = vmcs12->vm_exit_intr_error_code; 1727 evmcs->idt_vectoring_info_field = vmcs12->idt_vectoring_info_field; 1728 evmcs->idt_vectoring_error_code = vmcs12->idt_vectoring_error_code; 1729 evmcs->vm_exit_instruction_len = vmcs12->vm_exit_instruction_len; 1730 evmcs->vmx_instruction_info = vmcs12->vmx_instruction_info; 1731 1732 evmcs->exit_qualification = vmcs12->exit_qualification; 1733 1734 evmcs->guest_linear_address = vmcs12->guest_linear_address; 1735 evmcs->guest_rsp = vmcs12->guest_rsp; 1736 evmcs->guest_rflags = vmcs12->guest_rflags; 1737 1738 evmcs->guest_interruptibility_info = 1739 vmcs12->guest_interruptibility_info; 1740 evmcs->cpu_based_vm_exec_control = vmcs12->cpu_based_vm_exec_control; 1741 evmcs->vm_entry_controls = vmcs12->vm_entry_controls; 1742 evmcs->vm_entry_intr_info_field = vmcs12->vm_entry_intr_info_field; 1743 evmcs->vm_entry_exception_error_code = 1744 vmcs12->vm_entry_exception_error_code; 1745 evmcs->vm_entry_instruction_len = vmcs12->vm_entry_instruction_len; 1746 1747 evmcs->guest_rip = vmcs12->guest_rip; 1748 1749 evmcs->guest_bndcfgs = vmcs12->guest_bndcfgs; 1750 1751 return 0; 1752 } 1753 1754 /* 1755 * This is an equivalent of the nested hypervisor executing the vmptrld 1756 * instruction. 1757 */ 1758 static int nested_vmx_handle_enlightened_vmptrld(struct kvm_vcpu *vcpu, 1759 bool from_launch) 1760 { 1761 struct vcpu_vmx *vmx = to_vmx(vcpu); 1762 struct hv_vp_assist_page assist_page; 1763 1764 if (likely(!vmx->nested.enlightened_vmcs_enabled)) 1765 return 1; 1766 1767 if (unlikely(!kvm_hv_get_assist_page(vcpu, &assist_page))) 1768 return 1; 1769 1770 if (unlikely(!assist_page.enlighten_vmentry)) 1771 return 1; 1772 1773 if (unlikely(assist_page.current_nested_vmcs != 1774 vmx->nested.hv_evmcs_vmptr)) { 1775 1776 if (!vmx->nested.hv_evmcs) 1777 vmx->nested.current_vmptr = -1ull; 1778 1779 nested_release_evmcs(vcpu); 1780 1781 vmx->nested.hv_evmcs_page = kvm_vcpu_gpa_to_page( 1782 vcpu, assist_page.current_nested_vmcs); 1783 1784 if (unlikely(is_error_page(vmx->nested.hv_evmcs_page))) 1785 return 0; 1786 1787 vmx->nested.hv_evmcs = kmap(vmx->nested.hv_evmcs_page); 1788 1789 /* 1790 * Currently, KVM only supports eVMCS version 1 1791 * (== KVM_EVMCS_VERSION) and thus we expect guest to set this 1792 * value to first u32 field of eVMCS which should specify eVMCS 1793 * VersionNumber. 1794 * 1795 * Guest should be aware of supported eVMCS versions by host by 1796 * examining CPUID.0x4000000A.EAX[0:15]. Host userspace VMM is 1797 * expected to set this CPUID leaf according to the value 1798 * returned in vmcs_version from nested_enable_evmcs(). 1799 * 1800 * However, it turns out that Microsoft Hyper-V fails to comply 1801 * to their own invented interface: When Hyper-V use eVMCS, it 1802 * just sets first u32 field of eVMCS to revision_id specified 1803 * in MSR_IA32_VMX_BASIC. Instead of used eVMCS version number 1804 * which is one of the supported versions specified in 1805 * CPUID.0x4000000A.EAX[0:15]. 1806 * 1807 * To overcome Hyper-V bug, we accept here either a supported 1808 * eVMCS version or VMCS12 revision_id as valid values for first 1809 * u32 field of eVMCS. 1810 */ 1811 if ((vmx->nested.hv_evmcs->revision_id != KVM_EVMCS_VERSION) && 1812 (vmx->nested.hv_evmcs->revision_id != VMCS12_REVISION)) { 1813 nested_release_evmcs(vcpu); 1814 return 0; 1815 } 1816 1817 vmx->nested.dirty_vmcs12 = true; 1818 /* 1819 * As we keep L2 state for one guest only 'hv_clean_fields' mask 1820 * can't be used when we switch between them. Reset it here for 1821 * simplicity. 1822 */ 1823 vmx->nested.hv_evmcs->hv_clean_fields &= 1824 ~HV_VMX_ENLIGHTENED_CLEAN_FIELD_ALL; 1825 vmx->nested.hv_evmcs_vmptr = assist_page.current_nested_vmcs; 1826 1827 /* 1828 * Unlike normal vmcs12, enlightened vmcs12 is not fully 1829 * reloaded from guest's memory (read only fields, fields not 1830 * present in struct hv_enlightened_vmcs, ...). Make sure there 1831 * are no leftovers. 1832 */ 1833 if (from_launch) { 1834 struct vmcs12 *vmcs12 = get_vmcs12(vcpu); 1835 memset(vmcs12, 0, sizeof(*vmcs12)); 1836 vmcs12->hdr.revision_id = VMCS12_REVISION; 1837 } 1838 1839 } 1840 return 1; 1841 } 1842 1843 void nested_sync_from_vmcs12(struct kvm_vcpu *vcpu) 1844 { 1845 struct vcpu_vmx *vmx = to_vmx(vcpu); 1846 1847 /* 1848 * hv_evmcs may end up being not mapped after migration (when 1849 * L2 was running), map it here to make sure vmcs12 changes are 1850 * properly reflected. 1851 */ 1852 if (vmx->nested.enlightened_vmcs_enabled && !vmx->nested.hv_evmcs) 1853 nested_vmx_handle_enlightened_vmptrld(vcpu, false); 1854 1855 if (vmx->nested.hv_evmcs) { 1856 copy_vmcs12_to_enlightened(vmx); 1857 /* All fields are clean */ 1858 vmx->nested.hv_evmcs->hv_clean_fields |= 1859 HV_VMX_ENLIGHTENED_CLEAN_FIELD_ALL; 1860 } else { 1861 copy_vmcs12_to_shadow(vmx); 1862 } 1863 1864 vmx->nested.need_vmcs12_sync = false; 1865 } 1866 1867 static enum hrtimer_restart vmx_preemption_timer_fn(struct hrtimer *timer) 1868 { 1869 struct vcpu_vmx *vmx = 1870 container_of(timer, struct vcpu_vmx, nested.preemption_timer); 1871 1872 vmx->nested.preemption_timer_expired = true; 1873 kvm_make_request(KVM_REQ_EVENT, &vmx->vcpu); 1874 kvm_vcpu_kick(&vmx->vcpu); 1875 1876 return HRTIMER_NORESTART; 1877 } 1878 1879 static void vmx_start_preemption_timer(struct kvm_vcpu *vcpu) 1880 { 1881 u64 preemption_timeout = get_vmcs12(vcpu)->vmx_preemption_timer_value; 1882 struct vcpu_vmx *vmx = to_vmx(vcpu); 1883 1884 /* 1885 * A timer value of zero is architecturally guaranteed to cause 1886 * a VMExit prior to executing any instructions in the guest. 1887 */ 1888 if (preemption_timeout == 0) { 1889 vmx_preemption_timer_fn(&vmx->nested.preemption_timer); 1890 return; 1891 } 1892 1893 if (vcpu->arch.virtual_tsc_khz == 0) 1894 return; 1895 1896 preemption_timeout <<= VMX_MISC_EMULATED_PREEMPTION_TIMER_RATE; 1897 preemption_timeout *= 1000000; 1898 do_div(preemption_timeout, vcpu->arch.virtual_tsc_khz); 1899 hrtimer_start(&vmx->nested.preemption_timer, 1900 ns_to_ktime(preemption_timeout), HRTIMER_MODE_REL); 1901 } 1902 1903 static u64 nested_vmx_calc_efer(struct vcpu_vmx *vmx, struct vmcs12 *vmcs12) 1904 { 1905 if (vmx->nested.nested_run_pending && 1906 (vmcs12->vm_entry_controls & VM_ENTRY_LOAD_IA32_EFER)) 1907 return vmcs12->guest_ia32_efer; 1908 else if (vmcs12->vm_entry_controls & VM_ENTRY_IA32E_MODE) 1909 return vmx->vcpu.arch.efer | (EFER_LMA | EFER_LME); 1910 else 1911 return vmx->vcpu.arch.efer & ~(EFER_LMA | EFER_LME); 1912 } 1913 1914 static void prepare_vmcs02_constant_state(struct vcpu_vmx *vmx) 1915 { 1916 /* 1917 * If vmcs02 hasn't been initialized, set the constant vmcs02 state 1918 * according to L0's settings (vmcs12 is irrelevant here). Host 1919 * fields that come from L0 and are not constant, e.g. HOST_CR3, 1920 * will be set as needed prior to VMLAUNCH/VMRESUME. 1921 */ 1922 if (vmx->nested.vmcs02_initialized) 1923 return; 1924 vmx->nested.vmcs02_initialized = true; 1925 1926 /* 1927 * We don't care what the EPTP value is we just need to guarantee 1928 * it's valid so we don't get a false positive when doing early 1929 * consistency checks. 1930 */ 1931 if (enable_ept && nested_early_check) 1932 vmcs_write64(EPT_POINTER, construct_eptp(&vmx->vcpu, 0)); 1933 1934 /* All VMFUNCs are currently emulated through L0 vmexits. */ 1935 if (cpu_has_vmx_vmfunc()) 1936 vmcs_write64(VM_FUNCTION_CONTROL, 0); 1937 1938 if (cpu_has_vmx_posted_intr()) 1939 vmcs_write16(POSTED_INTR_NV, POSTED_INTR_NESTED_VECTOR); 1940 1941 if (cpu_has_vmx_msr_bitmap()) 1942 vmcs_write64(MSR_BITMAP, __pa(vmx->nested.vmcs02.msr_bitmap)); 1943 1944 if (enable_pml) 1945 vmcs_write64(PML_ADDRESS, page_to_phys(vmx->pml_pg)); 1946 1947 /* 1948 * Set the MSR load/store lists to match L0's settings. Only the 1949 * addresses are constant (for vmcs02), the counts can change based 1950 * on L2's behavior, e.g. switching to/from long mode. 1951 */ 1952 vmcs_write32(VM_EXIT_MSR_STORE_COUNT, 0); 1953 vmcs_write64(VM_EXIT_MSR_LOAD_ADDR, __pa(vmx->msr_autoload.host.val)); 1954 vmcs_write64(VM_ENTRY_MSR_LOAD_ADDR, __pa(vmx->msr_autoload.guest.val)); 1955 1956 vmx_set_constant_host_state(vmx); 1957 } 1958 1959 static void prepare_vmcs02_early_full(struct vcpu_vmx *vmx, 1960 struct vmcs12 *vmcs12) 1961 { 1962 prepare_vmcs02_constant_state(vmx); 1963 1964 vmcs_write64(VMCS_LINK_POINTER, -1ull); 1965 1966 if (enable_vpid) { 1967 if (nested_cpu_has_vpid(vmcs12) && vmx->nested.vpid02) 1968 vmcs_write16(VIRTUAL_PROCESSOR_ID, vmx->nested.vpid02); 1969 else 1970 vmcs_write16(VIRTUAL_PROCESSOR_ID, vmx->vpid); 1971 } 1972 } 1973 1974 static void prepare_vmcs02_early(struct vcpu_vmx *vmx, struct vmcs12 *vmcs12) 1975 { 1976 u32 exec_control, vmcs12_exec_ctrl; 1977 u64 guest_efer = nested_vmx_calc_efer(vmx, vmcs12); 1978 1979 if (vmx->nested.dirty_vmcs12 || vmx->nested.hv_evmcs) 1980 prepare_vmcs02_early_full(vmx, vmcs12); 1981 1982 /* 1983 * PIN CONTROLS 1984 */ 1985 exec_control = vmcs12->pin_based_vm_exec_control; 1986 1987 /* Preemption timer setting is computed directly in vmx_vcpu_run. */ 1988 exec_control |= vmcs_config.pin_based_exec_ctrl; 1989 exec_control &= ~PIN_BASED_VMX_PREEMPTION_TIMER; 1990 vmx->loaded_vmcs->hv_timer_armed = false; 1991 1992 /* Posted interrupts setting is only taken from vmcs12. */ 1993 if (nested_cpu_has_posted_intr(vmcs12)) { 1994 vmx->nested.posted_intr_nv = vmcs12->posted_intr_nv; 1995 vmx->nested.pi_pending = false; 1996 } else { 1997 exec_control &= ~PIN_BASED_POSTED_INTR; 1998 } 1999 vmcs_write32(PIN_BASED_VM_EXEC_CONTROL, exec_control); 2000 2001 /* 2002 * EXEC CONTROLS 2003 */ 2004 exec_control = vmx_exec_control(vmx); /* L0's desires */ 2005 exec_control &= ~CPU_BASED_VIRTUAL_INTR_PENDING; 2006 exec_control &= ~CPU_BASED_VIRTUAL_NMI_PENDING; 2007 exec_control &= ~CPU_BASED_TPR_SHADOW; 2008 exec_control |= vmcs12->cpu_based_vm_exec_control; 2009 2010 /* 2011 * Write an illegal value to VIRTUAL_APIC_PAGE_ADDR. Later, if 2012 * nested_get_vmcs12_pages can't fix it up, the illegal value 2013 * will result in a VM entry failure. 2014 */ 2015 if (exec_control & CPU_BASED_TPR_SHADOW) { 2016 vmcs_write64(VIRTUAL_APIC_PAGE_ADDR, -1ull); 2017 vmcs_write32(TPR_THRESHOLD, vmcs12->tpr_threshold); 2018 } else { 2019 #ifdef CONFIG_X86_64 2020 exec_control |= CPU_BASED_CR8_LOAD_EXITING | 2021 CPU_BASED_CR8_STORE_EXITING; 2022 #endif 2023 } 2024 2025 /* 2026 * A vmexit (to either L1 hypervisor or L0 userspace) is always needed 2027 * for I/O port accesses. 2028 */ 2029 exec_control &= ~CPU_BASED_USE_IO_BITMAPS; 2030 exec_control |= CPU_BASED_UNCOND_IO_EXITING; 2031 vmcs_write32(CPU_BASED_VM_EXEC_CONTROL, exec_control); 2032 2033 /* 2034 * SECONDARY EXEC CONTROLS 2035 */ 2036 if (cpu_has_secondary_exec_ctrls()) { 2037 exec_control = vmx->secondary_exec_control; 2038 2039 /* Take the following fields only from vmcs12 */ 2040 exec_control &= ~(SECONDARY_EXEC_VIRTUALIZE_APIC_ACCESSES | 2041 SECONDARY_EXEC_ENABLE_INVPCID | 2042 SECONDARY_EXEC_RDTSCP | 2043 SECONDARY_EXEC_XSAVES | 2044 SECONDARY_EXEC_VIRTUAL_INTR_DELIVERY | 2045 SECONDARY_EXEC_APIC_REGISTER_VIRT | 2046 SECONDARY_EXEC_ENABLE_VMFUNC); 2047 if (nested_cpu_has(vmcs12, 2048 CPU_BASED_ACTIVATE_SECONDARY_CONTROLS)) { 2049 vmcs12_exec_ctrl = vmcs12->secondary_vm_exec_control & 2050 ~SECONDARY_EXEC_ENABLE_PML; 2051 exec_control |= vmcs12_exec_ctrl; 2052 } 2053 2054 /* VMCS shadowing for L2 is emulated for now */ 2055 exec_control &= ~SECONDARY_EXEC_SHADOW_VMCS; 2056 2057 if (exec_control & SECONDARY_EXEC_VIRTUAL_INTR_DELIVERY) 2058 vmcs_write16(GUEST_INTR_STATUS, 2059 vmcs12->guest_intr_status); 2060 2061 /* 2062 * Write an illegal value to APIC_ACCESS_ADDR. Later, 2063 * nested_get_vmcs12_pages will either fix it up or 2064 * remove the VM execution control. 2065 */ 2066 if (exec_control & SECONDARY_EXEC_VIRTUALIZE_APIC_ACCESSES) 2067 vmcs_write64(APIC_ACCESS_ADDR, -1ull); 2068 2069 if (exec_control & SECONDARY_EXEC_ENCLS_EXITING) 2070 vmcs_write64(ENCLS_EXITING_BITMAP, -1ull); 2071 2072 vmcs_write32(SECONDARY_VM_EXEC_CONTROL, exec_control); 2073 } 2074 2075 /* 2076 * ENTRY CONTROLS 2077 * 2078 * vmcs12's VM_{ENTRY,EXIT}_LOAD_IA32_EFER and VM_ENTRY_IA32E_MODE 2079 * are emulated by vmx_set_efer() in prepare_vmcs02(), but speculate 2080 * on the related bits (if supported by the CPU) in the hope that 2081 * we can avoid VMWrites during vmx_set_efer(). 2082 */ 2083 exec_control = (vmcs12->vm_entry_controls | vmx_vmentry_ctrl()) & 2084 ~VM_ENTRY_IA32E_MODE & ~VM_ENTRY_LOAD_IA32_EFER; 2085 if (cpu_has_load_ia32_efer()) { 2086 if (guest_efer & EFER_LMA) 2087 exec_control |= VM_ENTRY_IA32E_MODE; 2088 if (guest_efer != host_efer) 2089 exec_control |= VM_ENTRY_LOAD_IA32_EFER; 2090 } 2091 vm_entry_controls_init(vmx, exec_control); 2092 2093 /* 2094 * EXIT CONTROLS 2095 * 2096 * L2->L1 exit controls are emulated - the hardware exit is to L0 so 2097 * we should use its exit controls. Note that VM_EXIT_LOAD_IA32_EFER 2098 * bits may be modified by vmx_set_efer() in prepare_vmcs02(). 2099 */ 2100 exec_control = vmx_vmexit_ctrl(); 2101 if (cpu_has_load_ia32_efer() && guest_efer != host_efer) 2102 exec_control |= VM_EXIT_LOAD_IA32_EFER; 2103 vm_exit_controls_init(vmx, exec_control); 2104 2105 /* 2106 * Conceptually we want to copy the PML address and index from 2107 * vmcs01 here, and then back to vmcs01 on nested vmexit. But, 2108 * since we always flush the log on each vmexit and never change 2109 * the PML address (once set), this happens to be equivalent to 2110 * simply resetting the index in vmcs02. 2111 */ 2112 if (enable_pml) 2113 vmcs_write16(GUEST_PML_INDEX, PML_ENTITY_NUM - 1); 2114 2115 /* 2116 * Interrupt/Exception Fields 2117 */ 2118 if (vmx->nested.nested_run_pending) { 2119 vmcs_write32(VM_ENTRY_INTR_INFO_FIELD, 2120 vmcs12->vm_entry_intr_info_field); 2121 vmcs_write32(VM_ENTRY_EXCEPTION_ERROR_CODE, 2122 vmcs12->vm_entry_exception_error_code); 2123 vmcs_write32(VM_ENTRY_INSTRUCTION_LEN, 2124 vmcs12->vm_entry_instruction_len); 2125 vmcs_write32(GUEST_INTERRUPTIBILITY_INFO, 2126 vmcs12->guest_interruptibility_info); 2127 vmx->loaded_vmcs->nmi_known_unmasked = 2128 !(vmcs12->guest_interruptibility_info & GUEST_INTR_STATE_NMI); 2129 } else { 2130 vmcs_write32(VM_ENTRY_INTR_INFO_FIELD, 0); 2131 } 2132 } 2133 2134 static void prepare_vmcs02_full(struct vcpu_vmx *vmx, struct vmcs12 *vmcs12) 2135 { 2136 struct hv_enlightened_vmcs *hv_evmcs = vmx->nested.hv_evmcs; 2137 2138 if (!hv_evmcs || !(hv_evmcs->hv_clean_fields & 2139 HV_VMX_ENLIGHTENED_CLEAN_FIELD_GUEST_GRP2)) { 2140 vmcs_write16(GUEST_ES_SELECTOR, vmcs12->guest_es_selector); 2141 vmcs_write16(GUEST_CS_SELECTOR, vmcs12->guest_cs_selector); 2142 vmcs_write16(GUEST_SS_SELECTOR, vmcs12->guest_ss_selector); 2143 vmcs_write16(GUEST_DS_SELECTOR, vmcs12->guest_ds_selector); 2144 vmcs_write16(GUEST_FS_SELECTOR, vmcs12->guest_fs_selector); 2145 vmcs_write16(GUEST_GS_SELECTOR, vmcs12->guest_gs_selector); 2146 vmcs_write16(GUEST_LDTR_SELECTOR, vmcs12->guest_ldtr_selector); 2147 vmcs_write16(GUEST_TR_SELECTOR, vmcs12->guest_tr_selector); 2148 vmcs_write32(GUEST_ES_LIMIT, vmcs12->guest_es_limit); 2149 vmcs_write32(GUEST_CS_LIMIT, vmcs12->guest_cs_limit); 2150 vmcs_write32(GUEST_SS_LIMIT, vmcs12->guest_ss_limit); 2151 vmcs_write32(GUEST_DS_LIMIT, vmcs12->guest_ds_limit); 2152 vmcs_write32(GUEST_FS_LIMIT, vmcs12->guest_fs_limit); 2153 vmcs_write32(GUEST_GS_LIMIT, vmcs12->guest_gs_limit); 2154 vmcs_write32(GUEST_LDTR_LIMIT, vmcs12->guest_ldtr_limit); 2155 vmcs_write32(GUEST_TR_LIMIT, vmcs12->guest_tr_limit); 2156 vmcs_write32(GUEST_GDTR_LIMIT, vmcs12->guest_gdtr_limit); 2157 vmcs_write32(GUEST_IDTR_LIMIT, vmcs12->guest_idtr_limit); 2158 vmcs_write32(GUEST_ES_AR_BYTES, vmcs12->guest_es_ar_bytes); 2159 vmcs_write32(GUEST_DS_AR_BYTES, vmcs12->guest_ds_ar_bytes); 2160 vmcs_write32(GUEST_FS_AR_BYTES, vmcs12->guest_fs_ar_bytes); 2161 vmcs_write32(GUEST_GS_AR_BYTES, vmcs12->guest_gs_ar_bytes); 2162 vmcs_write32(GUEST_LDTR_AR_BYTES, vmcs12->guest_ldtr_ar_bytes); 2163 vmcs_write32(GUEST_TR_AR_BYTES, vmcs12->guest_tr_ar_bytes); 2164 vmcs_writel(GUEST_ES_BASE, vmcs12->guest_es_base); 2165 vmcs_writel(GUEST_CS_BASE, vmcs12->guest_cs_base); 2166 vmcs_writel(GUEST_SS_BASE, vmcs12->guest_ss_base); 2167 vmcs_writel(GUEST_DS_BASE, vmcs12->guest_ds_base); 2168 vmcs_writel(GUEST_FS_BASE, vmcs12->guest_fs_base); 2169 vmcs_writel(GUEST_GS_BASE, vmcs12->guest_gs_base); 2170 vmcs_writel(GUEST_LDTR_BASE, vmcs12->guest_ldtr_base); 2171 vmcs_writel(GUEST_TR_BASE, vmcs12->guest_tr_base); 2172 vmcs_writel(GUEST_GDTR_BASE, vmcs12->guest_gdtr_base); 2173 vmcs_writel(GUEST_IDTR_BASE, vmcs12->guest_idtr_base); 2174 } 2175 2176 if (!hv_evmcs || !(hv_evmcs->hv_clean_fields & 2177 HV_VMX_ENLIGHTENED_CLEAN_FIELD_GUEST_GRP1)) { 2178 vmcs_write32(GUEST_SYSENTER_CS, vmcs12->guest_sysenter_cs); 2179 vmcs_writel(GUEST_PENDING_DBG_EXCEPTIONS, 2180 vmcs12->guest_pending_dbg_exceptions); 2181 vmcs_writel(GUEST_SYSENTER_ESP, vmcs12->guest_sysenter_esp); 2182 vmcs_writel(GUEST_SYSENTER_EIP, vmcs12->guest_sysenter_eip); 2183 2184 /* 2185 * L1 may access the L2's PDPTR, so save them to construct 2186 * vmcs12 2187 */ 2188 if (enable_ept) { 2189 vmcs_write64(GUEST_PDPTR0, vmcs12->guest_pdptr0); 2190 vmcs_write64(GUEST_PDPTR1, vmcs12->guest_pdptr1); 2191 vmcs_write64(GUEST_PDPTR2, vmcs12->guest_pdptr2); 2192 vmcs_write64(GUEST_PDPTR3, vmcs12->guest_pdptr3); 2193 } 2194 } 2195 2196 if (nested_cpu_has_xsaves(vmcs12)) 2197 vmcs_write64(XSS_EXIT_BITMAP, vmcs12->xss_exit_bitmap); 2198 2199 /* 2200 * Whether page-faults are trapped is determined by a combination of 2201 * 3 settings: PFEC_MASK, PFEC_MATCH and EXCEPTION_BITMAP.PF. 2202 * If enable_ept, L0 doesn't care about page faults and we should 2203 * set all of these to L1's desires. However, if !enable_ept, L0 does 2204 * care about (at least some) page faults, and because it is not easy 2205 * (if at all possible?) to merge L0 and L1's desires, we simply ask 2206 * to exit on each and every L2 page fault. This is done by setting 2207 * MASK=MATCH=0 and (see below) EB.PF=1. 2208 * Note that below we don't need special code to set EB.PF beyond the 2209 * "or"ing of the EB of vmcs01 and vmcs12, because when enable_ept, 2210 * vmcs01's EB.PF is 0 so the "or" will take vmcs12's value, and when 2211 * !enable_ept, EB.PF is 1, so the "or" will always be 1. 2212 */ 2213 vmcs_write32(PAGE_FAULT_ERROR_CODE_MASK, 2214 enable_ept ? vmcs12->page_fault_error_code_mask : 0); 2215 vmcs_write32(PAGE_FAULT_ERROR_CODE_MATCH, 2216 enable_ept ? vmcs12->page_fault_error_code_match : 0); 2217 2218 if (cpu_has_vmx_apicv()) { 2219 vmcs_write64(EOI_EXIT_BITMAP0, vmcs12->eoi_exit_bitmap0); 2220 vmcs_write64(EOI_EXIT_BITMAP1, vmcs12->eoi_exit_bitmap1); 2221 vmcs_write64(EOI_EXIT_BITMAP2, vmcs12->eoi_exit_bitmap2); 2222 vmcs_write64(EOI_EXIT_BITMAP3, vmcs12->eoi_exit_bitmap3); 2223 } 2224 2225 vmcs_write32(VM_EXIT_MSR_LOAD_COUNT, vmx->msr_autoload.host.nr); 2226 vmcs_write32(VM_ENTRY_MSR_LOAD_COUNT, vmx->msr_autoload.guest.nr); 2227 2228 set_cr4_guest_host_mask(vmx); 2229 2230 if (kvm_mpx_supported()) { 2231 if (vmx->nested.nested_run_pending && 2232 (vmcs12->vm_entry_controls & VM_ENTRY_LOAD_BNDCFGS)) 2233 vmcs_write64(GUEST_BNDCFGS, vmcs12->guest_bndcfgs); 2234 else 2235 vmcs_write64(GUEST_BNDCFGS, vmx->nested.vmcs01_guest_bndcfgs); 2236 } 2237 } 2238 2239 /* 2240 * prepare_vmcs02 is called when the L1 guest hypervisor runs its nested 2241 * L2 guest. L1 has a vmcs for L2 (vmcs12), and this function "merges" it 2242 * with L0's requirements for its guest (a.k.a. vmcs01), so we can run the L2 2243 * guest in a way that will both be appropriate to L1's requests, and our 2244 * needs. In addition to modifying the active vmcs (which is vmcs02), this 2245 * function also has additional necessary side-effects, like setting various 2246 * vcpu->arch fields. 2247 * Returns 0 on success, 1 on failure. Invalid state exit qualification code 2248 * is assigned to entry_failure_code on failure. 2249 */ 2250 static int prepare_vmcs02(struct kvm_vcpu *vcpu, struct vmcs12 *vmcs12, 2251 u32 *entry_failure_code) 2252 { 2253 struct vcpu_vmx *vmx = to_vmx(vcpu); 2254 struct hv_enlightened_vmcs *hv_evmcs = vmx->nested.hv_evmcs; 2255 2256 if (vmx->nested.dirty_vmcs12 || vmx->nested.hv_evmcs) { 2257 prepare_vmcs02_full(vmx, vmcs12); 2258 vmx->nested.dirty_vmcs12 = false; 2259 } 2260 2261 /* 2262 * First, the fields that are shadowed. This must be kept in sync 2263 * with vmcs_shadow_fields.h. 2264 */ 2265 if (!hv_evmcs || !(hv_evmcs->hv_clean_fields & 2266 HV_VMX_ENLIGHTENED_CLEAN_FIELD_GUEST_GRP2)) { 2267 vmcs_write32(GUEST_CS_AR_BYTES, vmcs12->guest_cs_ar_bytes); 2268 vmcs_write32(GUEST_SS_AR_BYTES, vmcs12->guest_ss_ar_bytes); 2269 } 2270 2271 if (vmx->nested.nested_run_pending && 2272 (vmcs12->vm_entry_controls & VM_ENTRY_LOAD_DEBUG_CONTROLS)) { 2273 kvm_set_dr(vcpu, 7, vmcs12->guest_dr7); 2274 vmcs_write64(GUEST_IA32_DEBUGCTL, vmcs12->guest_ia32_debugctl); 2275 } else { 2276 kvm_set_dr(vcpu, 7, vcpu->arch.dr7); 2277 vmcs_write64(GUEST_IA32_DEBUGCTL, vmx->nested.vmcs01_debugctl); 2278 } 2279 vmx_set_rflags(vcpu, vmcs12->guest_rflags); 2280 2281 /* EXCEPTION_BITMAP and CR0_GUEST_HOST_MASK should basically be the 2282 * bitwise-or of what L1 wants to trap for L2, and what we want to 2283 * trap. Note that CR0.TS also needs updating - we do this later. 2284 */ 2285 update_exception_bitmap(vcpu); 2286 vcpu->arch.cr0_guest_owned_bits &= ~vmcs12->cr0_guest_host_mask; 2287 vmcs_writel(CR0_GUEST_HOST_MASK, ~vcpu->arch.cr0_guest_owned_bits); 2288 2289 if (vmx->nested.nested_run_pending && 2290 (vmcs12->vm_entry_controls & VM_ENTRY_LOAD_IA32_PAT)) { 2291 vmcs_write64(GUEST_IA32_PAT, vmcs12->guest_ia32_pat); 2292 vcpu->arch.pat = vmcs12->guest_ia32_pat; 2293 } else if (vmcs_config.vmentry_ctrl & VM_ENTRY_LOAD_IA32_PAT) { 2294 vmcs_write64(GUEST_IA32_PAT, vmx->vcpu.arch.pat); 2295 } 2296 2297 vmcs_write64(TSC_OFFSET, vcpu->arch.tsc_offset); 2298 2299 if (kvm_has_tsc_control) 2300 decache_tsc_multiplier(vmx); 2301 2302 if (enable_vpid) { 2303 /* 2304 * There is no direct mapping between vpid02 and vpid12, the 2305 * vpid02 is per-vCPU for L0 and reused while the value of 2306 * vpid12 is changed w/ one invvpid during nested vmentry. 2307 * The vpid12 is allocated by L1 for L2, so it will not 2308 * influence global bitmap(for vpid01 and vpid02 allocation) 2309 * even if spawn a lot of nested vCPUs. 2310 */ 2311 if (nested_cpu_has_vpid(vmcs12) && nested_has_guest_tlb_tag(vcpu)) { 2312 if (vmcs12->virtual_processor_id != vmx->nested.last_vpid) { 2313 vmx->nested.last_vpid = vmcs12->virtual_processor_id; 2314 __vmx_flush_tlb(vcpu, nested_get_vpid02(vcpu), false); 2315 } 2316 } else { 2317 /* 2318 * If L1 use EPT, then L0 needs to execute INVEPT on 2319 * EPTP02 instead of EPTP01. Therefore, delay TLB 2320 * flush until vmcs02->eptp is fully updated by 2321 * KVM_REQ_LOAD_CR3. Note that this assumes 2322 * KVM_REQ_TLB_FLUSH is evaluated after 2323 * KVM_REQ_LOAD_CR3 in vcpu_enter_guest(). 2324 */ 2325 kvm_make_request(KVM_REQ_TLB_FLUSH, vcpu); 2326 } 2327 } 2328 2329 if (nested_cpu_has_ept(vmcs12)) 2330 nested_ept_init_mmu_context(vcpu); 2331 else if (nested_cpu_has2(vmcs12, 2332 SECONDARY_EXEC_VIRTUALIZE_APIC_ACCESSES)) 2333 vmx_flush_tlb(vcpu, true); 2334 2335 /* 2336 * This sets GUEST_CR0 to vmcs12->guest_cr0, possibly modifying those 2337 * bits which we consider mandatory enabled. 2338 * The CR0_READ_SHADOW is what L2 should have expected to read given 2339 * the specifications by L1; It's not enough to take 2340 * vmcs12->cr0_read_shadow because on our cr0_guest_host_mask we we 2341 * have more bits than L1 expected. 2342 */ 2343 vmx_set_cr0(vcpu, vmcs12->guest_cr0); 2344 vmcs_writel(CR0_READ_SHADOW, nested_read_cr0(vmcs12)); 2345 2346 vmx_set_cr4(vcpu, vmcs12->guest_cr4); 2347 vmcs_writel(CR4_READ_SHADOW, nested_read_cr4(vmcs12)); 2348 2349 vcpu->arch.efer = nested_vmx_calc_efer(vmx, vmcs12); 2350 /* Note: may modify VM_ENTRY/EXIT_CONTROLS and GUEST/HOST_IA32_EFER */ 2351 vmx_set_efer(vcpu, vcpu->arch.efer); 2352 2353 /* 2354 * Guest state is invalid and unrestricted guest is disabled, 2355 * which means L1 attempted VMEntry to L2 with invalid state. 2356 * Fail the VMEntry. 2357 */ 2358 if (vmx->emulation_required) { 2359 *entry_failure_code = ENTRY_FAIL_DEFAULT; 2360 return 1; 2361 } 2362 2363 /* Shadow page tables on either EPT or shadow page tables. */ 2364 if (nested_vmx_load_cr3(vcpu, vmcs12->guest_cr3, nested_cpu_has_ept(vmcs12), 2365 entry_failure_code)) 2366 return 1; 2367 2368 if (!enable_ept) 2369 vcpu->arch.walk_mmu->inject_page_fault = vmx_inject_page_fault_nested; 2370 2371 kvm_register_write(vcpu, VCPU_REGS_RSP, vmcs12->guest_rsp); 2372 kvm_register_write(vcpu, VCPU_REGS_RIP, vmcs12->guest_rip); 2373 return 0; 2374 } 2375 2376 static int nested_vmx_check_nmi_controls(struct vmcs12 *vmcs12) 2377 { 2378 if (!nested_cpu_has_nmi_exiting(vmcs12) && 2379 nested_cpu_has_virtual_nmis(vmcs12)) 2380 return -EINVAL; 2381 2382 if (!nested_cpu_has_virtual_nmis(vmcs12) && 2383 nested_cpu_has(vmcs12, CPU_BASED_VIRTUAL_NMI_PENDING)) 2384 return -EINVAL; 2385 2386 return 0; 2387 } 2388 2389 static bool valid_ept_address(struct kvm_vcpu *vcpu, u64 address) 2390 { 2391 struct vcpu_vmx *vmx = to_vmx(vcpu); 2392 int maxphyaddr = cpuid_maxphyaddr(vcpu); 2393 2394 /* Check for memory type validity */ 2395 switch (address & VMX_EPTP_MT_MASK) { 2396 case VMX_EPTP_MT_UC: 2397 if (!(vmx->nested.msrs.ept_caps & VMX_EPTP_UC_BIT)) 2398 return false; 2399 break; 2400 case VMX_EPTP_MT_WB: 2401 if (!(vmx->nested.msrs.ept_caps & VMX_EPTP_WB_BIT)) 2402 return false; 2403 break; 2404 default: 2405 return false; 2406 } 2407 2408 /* only 4 levels page-walk length are valid */ 2409 if ((address & VMX_EPTP_PWL_MASK) != VMX_EPTP_PWL_4) 2410 return false; 2411 2412 /* Reserved bits should not be set */ 2413 if (address >> maxphyaddr || ((address >> 7) & 0x1f)) 2414 return false; 2415 2416 /* AD, if set, should be supported */ 2417 if (address & VMX_EPTP_AD_ENABLE_BIT) { 2418 if (!(vmx->nested.msrs.ept_caps & VMX_EPT_AD_BIT)) 2419 return false; 2420 } 2421 2422 return true; 2423 } 2424 2425 /* 2426 * Checks related to VM-Execution Control Fields 2427 */ 2428 static int nested_check_vm_execution_controls(struct kvm_vcpu *vcpu, 2429 struct vmcs12 *vmcs12) 2430 { 2431 struct vcpu_vmx *vmx = to_vmx(vcpu); 2432 2433 if (!vmx_control_verify(vmcs12->pin_based_vm_exec_control, 2434 vmx->nested.msrs.pinbased_ctls_low, 2435 vmx->nested.msrs.pinbased_ctls_high) || 2436 !vmx_control_verify(vmcs12->cpu_based_vm_exec_control, 2437 vmx->nested.msrs.procbased_ctls_low, 2438 vmx->nested.msrs.procbased_ctls_high)) 2439 return -EINVAL; 2440 2441 if (nested_cpu_has(vmcs12, CPU_BASED_ACTIVATE_SECONDARY_CONTROLS) && 2442 !vmx_control_verify(vmcs12->secondary_vm_exec_control, 2443 vmx->nested.msrs.secondary_ctls_low, 2444 vmx->nested.msrs.secondary_ctls_high)) 2445 return -EINVAL; 2446 2447 if (vmcs12->cr3_target_count > nested_cpu_vmx_misc_cr3_count(vcpu) || 2448 nested_vmx_check_io_bitmap_controls(vcpu, vmcs12) || 2449 nested_vmx_check_msr_bitmap_controls(vcpu, vmcs12) || 2450 nested_vmx_check_tpr_shadow_controls(vcpu, vmcs12) || 2451 nested_vmx_check_apic_access_controls(vcpu, vmcs12) || 2452 nested_vmx_check_apicv_controls(vcpu, vmcs12) || 2453 nested_vmx_check_nmi_controls(vmcs12) || 2454 nested_vmx_check_pml_controls(vcpu, vmcs12) || 2455 nested_vmx_check_unrestricted_guest_controls(vcpu, vmcs12) || 2456 nested_vmx_check_mode_based_ept_exec_controls(vcpu, vmcs12) || 2457 nested_vmx_check_shadow_vmcs_controls(vcpu, vmcs12) || 2458 (nested_cpu_has_vpid(vmcs12) && !vmcs12->virtual_processor_id)) 2459 return -EINVAL; 2460 2461 if (!nested_cpu_has_preemption_timer(vmcs12) && 2462 nested_cpu_has_save_preemption_timer(vmcs12)) 2463 return -EINVAL; 2464 2465 if (nested_cpu_has_ept(vmcs12) && 2466 !valid_ept_address(vcpu, vmcs12->ept_pointer)) 2467 return -EINVAL; 2468 2469 if (nested_cpu_has_vmfunc(vmcs12)) { 2470 if (vmcs12->vm_function_control & 2471 ~vmx->nested.msrs.vmfunc_controls) 2472 return -EINVAL; 2473 2474 if (nested_cpu_has_eptp_switching(vmcs12)) { 2475 if (!nested_cpu_has_ept(vmcs12) || 2476 !page_address_valid(vcpu, vmcs12->eptp_list_address)) 2477 return -EINVAL; 2478 } 2479 } 2480 2481 return 0; 2482 } 2483 2484 /* 2485 * Checks related to VM-Exit Control Fields 2486 */ 2487 static int nested_check_vm_exit_controls(struct kvm_vcpu *vcpu, 2488 struct vmcs12 *vmcs12) 2489 { 2490 struct vcpu_vmx *vmx = to_vmx(vcpu); 2491 2492 if (!vmx_control_verify(vmcs12->vm_exit_controls, 2493 vmx->nested.msrs.exit_ctls_low, 2494 vmx->nested.msrs.exit_ctls_high) || 2495 nested_vmx_check_exit_msr_switch_controls(vcpu, vmcs12)) 2496 return -EINVAL; 2497 2498 return 0; 2499 } 2500 2501 /* 2502 * Checks related to VM-Entry Control Fields 2503 */ 2504 static int nested_check_vm_entry_controls(struct kvm_vcpu *vcpu, 2505 struct vmcs12 *vmcs12) 2506 { 2507 struct vcpu_vmx *vmx = to_vmx(vcpu); 2508 2509 if (!vmx_control_verify(vmcs12->vm_entry_controls, 2510 vmx->nested.msrs.entry_ctls_low, 2511 vmx->nested.msrs.entry_ctls_high)) 2512 return -EINVAL; 2513 2514 /* 2515 * From the Intel SDM, volume 3: 2516 * Fields relevant to VM-entry event injection must be set properly. 2517 * These fields are the VM-entry interruption-information field, the 2518 * VM-entry exception error code, and the VM-entry instruction length. 2519 */ 2520 if (vmcs12->vm_entry_intr_info_field & INTR_INFO_VALID_MASK) { 2521 u32 intr_info = vmcs12->vm_entry_intr_info_field; 2522 u8 vector = intr_info & INTR_INFO_VECTOR_MASK; 2523 u32 intr_type = intr_info & INTR_INFO_INTR_TYPE_MASK; 2524 bool has_error_code = intr_info & INTR_INFO_DELIVER_CODE_MASK; 2525 bool should_have_error_code; 2526 bool urg = nested_cpu_has2(vmcs12, 2527 SECONDARY_EXEC_UNRESTRICTED_GUEST); 2528 bool prot_mode = !urg || vmcs12->guest_cr0 & X86_CR0_PE; 2529 2530 /* VM-entry interruption-info field: interruption type */ 2531 if (intr_type == INTR_TYPE_RESERVED || 2532 (intr_type == INTR_TYPE_OTHER_EVENT && 2533 !nested_cpu_supports_monitor_trap_flag(vcpu))) 2534 return -EINVAL; 2535 2536 /* VM-entry interruption-info field: vector */ 2537 if ((intr_type == INTR_TYPE_NMI_INTR && vector != NMI_VECTOR) || 2538 (intr_type == INTR_TYPE_HARD_EXCEPTION && vector > 31) || 2539 (intr_type == INTR_TYPE_OTHER_EVENT && vector != 0)) 2540 return -EINVAL; 2541 2542 /* VM-entry interruption-info field: deliver error code */ 2543 should_have_error_code = 2544 intr_type == INTR_TYPE_HARD_EXCEPTION && prot_mode && 2545 x86_exception_has_error_code(vector); 2546 if (has_error_code != should_have_error_code) 2547 return -EINVAL; 2548 2549 /* VM-entry exception error code */ 2550 if (has_error_code && 2551 vmcs12->vm_entry_exception_error_code & GENMASK(31, 15)) 2552 return -EINVAL; 2553 2554 /* VM-entry interruption-info field: reserved bits */ 2555 if (intr_info & INTR_INFO_RESVD_BITS_MASK) 2556 return -EINVAL; 2557 2558 /* VM-entry instruction length */ 2559 switch (intr_type) { 2560 case INTR_TYPE_SOFT_EXCEPTION: 2561 case INTR_TYPE_SOFT_INTR: 2562 case INTR_TYPE_PRIV_SW_EXCEPTION: 2563 if ((vmcs12->vm_entry_instruction_len > 15) || 2564 (vmcs12->vm_entry_instruction_len == 0 && 2565 !nested_cpu_has_zero_length_injection(vcpu))) 2566 return -EINVAL; 2567 } 2568 } 2569 2570 if (nested_vmx_check_entry_msr_switch_controls(vcpu, vmcs12)) 2571 return -EINVAL; 2572 2573 return 0; 2574 } 2575 2576 /* 2577 * Checks related to Host Control Registers and MSRs 2578 */ 2579 static int nested_check_host_control_regs(struct kvm_vcpu *vcpu, 2580 struct vmcs12 *vmcs12) 2581 { 2582 bool ia32e; 2583 2584 if (!nested_host_cr0_valid(vcpu, vmcs12->host_cr0) || 2585 !nested_host_cr4_valid(vcpu, vmcs12->host_cr4) || 2586 !nested_cr3_valid(vcpu, vmcs12->host_cr3)) 2587 return -EINVAL; 2588 2589 if (is_noncanonical_address(vmcs12->host_ia32_sysenter_esp, vcpu) || 2590 is_noncanonical_address(vmcs12->host_ia32_sysenter_eip, vcpu)) 2591 return -EINVAL; 2592 2593 /* 2594 * If the load IA32_EFER VM-exit control is 1, bits reserved in the 2595 * IA32_EFER MSR must be 0 in the field for that register. In addition, 2596 * the values of the LMA and LME bits in the field must each be that of 2597 * the host address-space size VM-exit control. 2598 */ 2599 if (vmcs12->vm_exit_controls & VM_EXIT_LOAD_IA32_EFER) { 2600 ia32e = (vmcs12->vm_exit_controls & 2601 VM_EXIT_HOST_ADDR_SPACE_SIZE) != 0; 2602 if (!kvm_valid_efer(vcpu, vmcs12->host_ia32_efer) || 2603 ia32e != !!(vmcs12->host_ia32_efer & EFER_LMA) || 2604 ia32e != !!(vmcs12->host_ia32_efer & EFER_LME)) 2605 return -EINVAL; 2606 } 2607 2608 return 0; 2609 } 2610 2611 /* 2612 * Checks related to Guest Non-register State 2613 */ 2614 static int nested_check_guest_non_reg_state(struct vmcs12 *vmcs12) 2615 { 2616 if (vmcs12->guest_activity_state != GUEST_ACTIVITY_ACTIVE && 2617 vmcs12->guest_activity_state != GUEST_ACTIVITY_HLT) 2618 return -EINVAL; 2619 2620 return 0; 2621 } 2622 2623 static int nested_vmx_check_vmentry_prereqs(struct kvm_vcpu *vcpu, 2624 struct vmcs12 *vmcs12) 2625 { 2626 if (nested_check_vm_execution_controls(vcpu, vmcs12) || 2627 nested_check_vm_exit_controls(vcpu, vmcs12) || 2628 nested_check_vm_entry_controls(vcpu, vmcs12)) 2629 return VMXERR_ENTRY_INVALID_CONTROL_FIELD; 2630 2631 if (nested_check_host_control_regs(vcpu, vmcs12)) 2632 return VMXERR_ENTRY_INVALID_HOST_STATE_FIELD; 2633 2634 if (nested_check_guest_non_reg_state(vmcs12)) 2635 return VMXERR_ENTRY_INVALID_CONTROL_FIELD; 2636 2637 return 0; 2638 } 2639 2640 static int nested_vmx_check_vmcs_link_ptr(struct kvm_vcpu *vcpu, 2641 struct vmcs12 *vmcs12) 2642 { 2643 int r; 2644 struct page *page; 2645 struct vmcs12 *shadow; 2646 2647 if (vmcs12->vmcs_link_pointer == -1ull) 2648 return 0; 2649 2650 if (!page_address_valid(vcpu, vmcs12->vmcs_link_pointer)) 2651 return -EINVAL; 2652 2653 page = kvm_vcpu_gpa_to_page(vcpu, vmcs12->vmcs_link_pointer); 2654 if (is_error_page(page)) 2655 return -EINVAL; 2656 2657 r = 0; 2658 shadow = kmap(page); 2659 if (shadow->hdr.revision_id != VMCS12_REVISION || 2660 shadow->hdr.shadow_vmcs != nested_cpu_has_shadow_vmcs(vmcs12)) 2661 r = -EINVAL; 2662 kunmap(page); 2663 kvm_release_page_clean(page); 2664 return r; 2665 } 2666 2667 static int nested_vmx_check_vmentry_postreqs(struct kvm_vcpu *vcpu, 2668 struct vmcs12 *vmcs12, 2669 u32 *exit_qual) 2670 { 2671 bool ia32e; 2672 2673 *exit_qual = ENTRY_FAIL_DEFAULT; 2674 2675 if (!nested_guest_cr0_valid(vcpu, vmcs12->guest_cr0) || 2676 !nested_guest_cr4_valid(vcpu, vmcs12->guest_cr4)) 2677 return 1; 2678 2679 if (nested_vmx_check_vmcs_link_ptr(vcpu, vmcs12)) { 2680 *exit_qual = ENTRY_FAIL_VMCS_LINK_PTR; 2681 return 1; 2682 } 2683 2684 /* 2685 * If the load IA32_EFER VM-entry control is 1, the following checks 2686 * are performed on the field for the IA32_EFER MSR: 2687 * - Bits reserved in the IA32_EFER MSR must be 0. 2688 * - Bit 10 (corresponding to IA32_EFER.LMA) must equal the value of 2689 * the IA-32e mode guest VM-exit control. It must also be identical 2690 * to bit 8 (LME) if bit 31 in the CR0 field (corresponding to 2691 * CR0.PG) is 1. 2692 */ 2693 if (to_vmx(vcpu)->nested.nested_run_pending && 2694 (vmcs12->vm_entry_controls & VM_ENTRY_LOAD_IA32_EFER)) { 2695 ia32e = (vmcs12->vm_entry_controls & VM_ENTRY_IA32E_MODE) != 0; 2696 if (!kvm_valid_efer(vcpu, vmcs12->guest_ia32_efer) || 2697 ia32e != !!(vmcs12->guest_ia32_efer & EFER_LMA) || 2698 ((vmcs12->guest_cr0 & X86_CR0_PG) && 2699 ia32e != !!(vmcs12->guest_ia32_efer & EFER_LME))) 2700 return 1; 2701 } 2702 2703 if ((vmcs12->vm_entry_controls & VM_ENTRY_LOAD_BNDCFGS) && 2704 (is_noncanonical_address(vmcs12->guest_bndcfgs & PAGE_MASK, vcpu) || 2705 (vmcs12->guest_bndcfgs & MSR_IA32_BNDCFGS_RSVD))) 2706 return 1; 2707 2708 return 0; 2709 } 2710 2711 static int nested_vmx_check_vmentry_hw(struct kvm_vcpu *vcpu) 2712 { 2713 struct vcpu_vmx *vmx = to_vmx(vcpu); 2714 unsigned long cr3, cr4; 2715 bool vm_fail; 2716 2717 if (!nested_early_check) 2718 return 0; 2719 2720 if (vmx->msr_autoload.host.nr) 2721 vmcs_write32(VM_EXIT_MSR_LOAD_COUNT, 0); 2722 if (vmx->msr_autoload.guest.nr) 2723 vmcs_write32(VM_ENTRY_MSR_LOAD_COUNT, 0); 2724 2725 preempt_disable(); 2726 2727 vmx_prepare_switch_to_guest(vcpu); 2728 2729 /* 2730 * Induce a consistency check VMExit by clearing bit 1 in GUEST_RFLAGS, 2731 * which is reserved to '1' by hardware. GUEST_RFLAGS is guaranteed to 2732 * be written (by preparve_vmcs02()) before the "real" VMEnter, i.e. 2733 * there is no need to preserve other bits or save/restore the field. 2734 */ 2735 vmcs_writel(GUEST_RFLAGS, 0); 2736 2737 cr3 = __get_current_cr3_fast(); 2738 if (unlikely(cr3 != vmx->loaded_vmcs->host_state.cr3)) { 2739 vmcs_writel(HOST_CR3, cr3); 2740 vmx->loaded_vmcs->host_state.cr3 = cr3; 2741 } 2742 2743 cr4 = cr4_read_shadow(); 2744 if (unlikely(cr4 != vmx->loaded_vmcs->host_state.cr4)) { 2745 vmcs_writel(HOST_CR4, cr4); 2746 vmx->loaded_vmcs->host_state.cr4 = cr4; 2747 } 2748 2749 asm( 2750 "sub $%c[wordsize], %%" _ASM_SP "\n\t" /* temporarily adjust RSP for CALL */ 2751 "cmp %%" _ASM_SP ", %c[host_state_rsp](%[loaded_vmcs]) \n\t" 2752 "je 1f \n\t" 2753 __ex("vmwrite %%" _ASM_SP ", %[HOST_RSP]") "\n\t" 2754 "mov %%" _ASM_SP ", %c[host_state_rsp](%[loaded_vmcs]) \n\t" 2755 "1: \n\t" 2756 "add $%c[wordsize], %%" _ASM_SP "\n\t" /* un-adjust RSP */ 2757 2758 /* Check if vmlaunch or vmresume is needed */ 2759 "cmpb $0, %c[launched](%[loaded_vmcs])\n\t" 2760 2761 /* 2762 * VMLAUNCH and VMRESUME clear RFLAGS.{CF,ZF} on VM-Exit, set 2763 * RFLAGS.CF on VM-Fail Invalid and set RFLAGS.ZF on VM-Fail 2764 * Valid. vmx_vmenter() directly "returns" RFLAGS, and so the 2765 * results of VM-Enter is captured via CC_{SET,OUT} to vm_fail. 2766 */ 2767 "call vmx_vmenter\n\t" 2768 2769 CC_SET(be) 2770 : ASM_CALL_CONSTRAINT, CC_OUT(be) (vm_fail) 2771 : [HOST_RSP]"r"((unsigned long)HOST_RSP), 2772 [loaded_vmcs]"r"(vmx->loaded_vmcs), 2773 [launched]"i"(offsetof(struct loaded_vmcs, launched)), 2774 [host_state_rsp]"i"(offsetof(struct loaded_vmcs, host_state.rsp)), 2775 [wordsize]"i"(sizeof(ulong)) 2776 : "cc", "memory" 2777 ); 2778 2779 preempt_enable(); 2780 2781 if (vmx->msr_autoload.host.nr) 2782 vmcs_write32(VM_EXIT_MSR_LOAD_COUNT, vmx->msr_autoload.host.nr); 2783 if (vmx->msr_autoload.guest.nr) 2784 vmcs_write32(VM_ENTRY_MSR_LOAD_COUNT, vmx->msr_autoload.guest.nr); 2785 2786 if (vm_fail) { 2787 WARN_ON_ONCE(vmcs_read32(VM_INSTRUCTION_ERROR) != 2788 VMXERR_ENTRY_INVALID_CONTROL_FIELD); 2789 return 1; 2790 } 2791 2792 /* 2793 * VMExit clears RFLAGS.IF and DR7, even on a consistency check. 2794 */ 2795 local_irq_enable(); 2796 if (hw_breakpoint_active()) 2797 set_debugreg(__this_cpu_read(cpu_dr7), 7); 2798 2799 /* 2800 * A non-failing VMEntry means we somehow entered guest mode with 2801 * an illegal RIP, and that's just the tip of the iceberg. There 2802 * is no telling what memory has been modified or what state has 2803 * been exposed to unknown code. Hitting this all but guarantees 2804 * a (very critical) hardware issue. 2805 */ 2806 WARN_ON(!(vmcs_read32(VM_EXIT_REASON) & 2807 VMX_EXIT_REASONS_FAILED_VMENTRY)); 2808 2809 return 0; 2810 } 2811 2812 static inline bool nested_vmx_prepare_msr_bitmap(struct kvm_vcpu *vcpu, 2813 struct vmcs12 *vmcs12); 2814 2815 static void nested_get_vmcs12_pages(struct kvm_vcpu *vcpu) 2816 { 2817 struct vmcs12 *vmcs12 = get_vmcs12(vcpu); 2818 struct vcpu_vmx *vmx = to_vmx(vcpu); 2819 struct page *page; 2820 u64 hpa; 2821 2822 if (nested_cpu_has2(vmcs12, SECONDARY_EXEC_VIRTUALIZE_APIC_ACCESSES)) { 2823 /* 2824 * Translate L1 physical address to host physical 2825 * address for vmcs02. Keep the page pinned, so this 2826 * physical address remains valid. We keep a reference 2827 * to it so we can release it later. 2828 */ 2829 if (vmx->nested.apic_access_page) { /* shouldn't happen */ 2830 kvm_release_page_dirty(vmx->nested.apic_access_page); 2831 vmx->nested.apic_access_page = NULL; 2832 } 2833 page = kvm_vcpu_gpa_to_page(vcpu, vmcs12->apic_access_addr); 2834 /* 2835 * If translation failed, no matter: This feature asks 2836 * to exit when accessing the given address, and if it 2837 * can never be accessed, this feature won't do 2838 * anything anyway. 2839 */ 2840 if (!is_error_page(page)) { 2841 vmx->nested.apic_access_page = page; 2842 hpa = page_to_phys(vmx->nested.apic_access_page); 2843 vmcs_write64(APIC_ACCESS_ADDR, hpa); 2844 } else { 2845 vmcs_clear_bits(SECONDARY_VM_EXEC_CONTROL, 2846 SECONDARY_EXEC_VIRTUALIZE_APIC_ACCESSES); 2847 } 2848 } 2849 2850 if (nested_cpu_has(vmcs12, CPU_BASED_TPR_SHADOW)) { 2851 if (vmx->nested.virtual_apic_page) { /* shouldn't happen */ 2852 kvm_release_page_dirty(vmx->nested.virtual_apic_page); 2853 vmx->nested.virtual_apic_page = NULL; 2854 } 2855 page = kvm_vcpu_gpa_to_page(vcpu, vmcs12->virtual_apic_page_addr); 2856 2857 /* 2858 * If translation failed, VM entry will fail because 2859 * prepare_vmcs02 set VIRTUAL_APIC_PAGE_ADDR to -1ull. 2860 * Failing the vm entry is _not_ what the processor 2861 * does but it's basically the only possibility we 2862 * have. We could still enter the guest if CR8 load 2863 * exits are enabled, CR8 store exits are enabled, and 2864 * virtualize APIC access is disabled; in this case 2865 * the processor would never use the TPR shadow and we 2866 * could simply clear the bit from the execution 2867 * control. But such a configuration is useless, so 2868 * let's keep the code simple. 2869 */ 2870 if (!is_error_page(page)) { 2871 vmx->nested.virtual_apic_page = page; 2872 hpa = page_to_phys(vmx->nested.virtual_apic_page); 2873 vmcs_write64(VIRTUAL_APIC_PAGE_ADDR, hpa); 2874 } 2875 } 2876 2877 if (nested_cpu_has_posted_intr(vmcs12)) { 2878 if (vmx->nested.pi_desc_page) { /* shouldn't happen */ 2879 kunmap(vmx->nested.pi_desc_page); 2880 kvm_release_page_dirty(vmx->nested.pi_desc_page); 2881 vmx->nested.pi_desc_page = NULL; 2882 vmx->nested.pi_desc = NULL; 2883 vmcs_write64(POSTED_INTR_DESC_ADDR, -1ull); 2884 } 2885 page = kvm_vcpu_gpa_to_page(vcpu, vmcs12->posted_intr_desc_addr); 2886 if (is_error_page(page)) 2887 return; 2888 vmx->nested.pi_desc_page = page; 2889 vmx->nested.pi_desc = kmap(vmx->nested.pi_desc_page); 2890 vmx->nested.pi_desc = 2891 (struct pi_desc *)((void *)vmx->nested.pi_desc + 2892 (unsigned long)(vmcs12->posted_intr_desc_addr & 2893 (PAGE_SIZE - 1))); 2894 vmcs_write64(POSTED_INTR_DESC_ADDR, 2895 page_to_phys(vmx->nested.pi_desc_page) + 2896 (unsigned long)(vmcs12->posted_intr_desc_addr & 2897 (PAGE_SIZE - 1))); 2898 } 2899 if (nested_vmx_prepare_msr_bitmap(vcpu, vmcs12)) 2900 vmcs_set_bits(CPU_BASED_VM_EXEC_CONTROL, 2901 CPU_BASED_USE_MSR_BITMAPS); 2902 else 2903 vmcs_clear_bits(CPU_BASED_VM_EXEC_CONTROL, 2904 CPU_BASED_USE_MSR_BITMAPS); 2905 } 2906 2907 /* 2908 * Intel's VMX Instruction Reference specifies a common set of prerequisites 2909 * for running VMX instructions (except VMXON, whose prerequisites are 2910 * slightly different). It also specifies what exception to inject otherwise. 2911 * Note that many of these exceptions have priority over VM exits, so they 2912 * don't have to be checked again here. 2913 */ 2914 static int nested_vmx_check_permission(struct kvm_vcpu *vcpu) 2915 { 2916 if (!to_vmx(vcpu)->nested.vmxon) { 2917 kvm_queue_exception(vcpu, UD_VECTOR); 2918 return 0; 2919 } 2920 2921 if (vmx_get_cpl(vcpu)) { 2922 kvm_inject_gp(vcpu, 0); 2923 return 0; 2924 } 2925 2926 return 1; 2927 } 2928 2929 static u8 vmx_has_apicv_interrupt(struct kvm_vcpu *vcpu) 2930 { 2931 u8 rvi = vmx_get_rvi(); 2932 u8 vppr = kvm_lapic_get_reg(vcpu->arch.apic, APIC_PROCPRI); 2933 2934 return ((rvi & 0xf0) > (vppr & 0xf0)); 2935 } 2936 2937 static void load_vmcs12_host_state(struct kvm_vcpu *vcpu, 2938 struct vmcs12 *vmcs12); 2939 2940 /* 2941 * If from_vmentry is false, this is being called from state restore (either RSM 2942 * or KVM_SET_NESTED_STATE). Otherwise it's called from vmlaunch/vmresume. 2943 + * 2944 + * Returns: 2945 + * 0 - success, i.e. proceed with actual VMEnter 2946 + * 1 - consistency check VMExit 2947 + * -1 - consistency check VMFail 2948 */ 2949 int nested_vmx_enter_non_root_mode(struct kvm_vcpu *vcpu, bool from_vmentry) 2950 { 2951 struct vcpu_vmx *vmx = to_vmx(vcpu); 2952 struct vmcs12 *vmcs12 = get_vmcs12(vcpu); 2953 bool evaluate_pending_interrupts; 2954 u32 exit_reason = EXIT_REASON_INVALID_STATE; 2955 u32 exit_qual; 2956 2957 evaluate_pending_interrupts = vmcs_read32(CPU_BASED_VM_EXEC_CONTROL) & 2958 (CPU_BASED_VIRTUAL_INTR_PENDING | CPU_BASED_VIRTUAL_NMI_PENDING); 2959 if (likely(!evaluate_pending_interrupts) && kvm_vcpu_apicv_active(vcpu)) 2960 evaluate_pending_interrupts |= vmx_has_apicv_interrupt(vcpu); 2961 2962 if (!(vmcs12->vm_entry_controls & VM_ENTRY_LOAD_DEBUG_CONTROLS)) 2963 vmx->nested.vmcs01_debugctl = vmcs_read64(GUEST_IA32_DEBUGCTL); 2964 if (kvm_mpx_supported() && 2965 !(vmcs12->vm_entry_controls & VM_ENTRY_LOAD_BNDCFGS)) 2966 vmx->nested.vmcs01_guest_bndcfgs = vmcs_read64(GUEST_BNDCFGS); 2967 2968 vmx_switch_vmcs(vcpu, &vmx->nested.vmcs02); 2969 2970 prepare_vmcs02_early(vmx, vmcs12); 2971 2972 if (from_vmentry) { 2973 nested_get_vmcs12_pages(vcpu); 2974 2975 if (nested_vmx_check_vmentry_hw(vcpu)) { 2976 vmx_switch_vmcs(vcpu, &vmx->vmcs01); 2977 return -1; 2978 } 2979 2980 if (nested_vmx_check_vmentry_postreqs(vcpu, vmcs12, &exit_qual)) 2981 goto vmentry_fail_vmexit; 2982 } 2983 2984 enter_guest_mode(vcpu); 2985 if (vmcs12->cpu_based_vm_exec_control & CPU_BASED_USE_TSC_OFFSETING) 2986 vcpu->arch.tsc_offset += vmcs12->tsc_offset; 2987 2988 if (prepare_vmcs02(vcpu, vmcs12, &exit_qual)) 2989 goto vmentry_fail_vmexit_guest_mode; 2990 2991 if (from_vmentry) { 2992 exit_reason = EXIT_REASON_MSR_LOAD_FAIL; 2993 exit_qual = nested_vmx_load_msr(vcpu, 2994 vmcs12->vm_entry_msr_load_addr, 2995 vmcs12->vm_entry_msr_load_count); 2996 if (exit_qual) 2997 goto vmentry_fail_vmexit_guest_mode; 2998 } else { 2999 /* 3000 * The MMU is not initialized to point at the right entities yet and 3001 * "get pages" would need to read data from the guest (i.e. we will 3002 * need to perform gpa to hpa translation). Request a call 3003 * to nested_get_vmcs12_pages before the next VM-entry. The MSRs 3004 * have already been set at vmentry time and should not be reset. 3005 */ 3006 kvm_make_request(KVM_REQ_GET_VMCS12_PAGES, vcpu); 3007 } 3008 3009 /* 3010 * If L1 had a pending IRQ/NMI until it executed 3011 * VMLAUNCH/VMRESUME which wasn't delivered because it was 3012 * disallowed (e.g. interrupts disabled), L0 needs to 3013 * evaluate if this pending event should cause an exit from L2 3014 * to L1 or delivered directly to L2 (e.g. In case L1 don't 3015 * intercept EXTERNAL_INTERRUPT). 3016 * 3017 * Usually this would be handled by the processor noticing an 3018 * IRQ/NMI window request, or checking RVI during evaluation of 3019 * pending virtual interrupts. However, this setting was done 3020 * on VMCS01 and now VMCS02 is active instead. Thus, we force L0 3021 * to perform pending event evaluation by requesting a KVM_REQ_EVENT. 3022 */ 3023 if (unlikely(evaluate_pending_interrupts)) 3024 kvm_make_request(KVM_REQ_EVENT, vcpu); 3025 3026 /* 3027 * Do not start the preemption timer hrtimer until after we know 3028 * we are successful, so that only nested_vmx_vmexit needs to cancel 3029 * the timer. 3030 */ 3031 vmx->nested.preemption_timer_expired = false; 3032 if (nested_cpu_has_preemption_timer(vmcs12)) 3033 vmx_start_preemption_timer(vcpu); 3034 3035 /* 3036 * Note no nested_vmx_succeed or nested_vmx_fail here. At this point 3037 * we are no longer running L1, and VMLAUNCH/VMRESUME has not yet 3038 * returned as far as L1 is concerned. It will only return (and set 3039 * the success flag) when L2 exits (see nested_vmx_vmexit()). 3040 */ 3041 return 0; 3042 3043 /* 3044 * A failed consistency check that leads to a VMExit during L1's 3045 * VMEnter to L2 is a variation of a normal VMexit, as explained in 3046 * 26.7 "VM-entry failures during or after loading guest state". 3047 */ 3048 vmentry_fail_vmexit_guest_mode: 3049 if (vmcs12->cpu_based_vm_exec_control & CPU_BASED_USE_TSC_OFFSETING) 3050 vcpu->arch.tsc_offset -= vmcs12->tsc_offset; 3051 leave_guest_mode(vcpu); 3052 3053 vmentry_fail_vmexit: 3054 vmx_switch_vmcs(vcpu, &vmx->vmcs01); 3055 3056 if (!from_vmentry) 3057 return 1; 3058 3059 load_vmcs12_host_state(vcpu, vmcs12); 3060 vmcs12->vm_exit_reason = exit_reason | VMX_EXIT_REASONS_FAILED_VMENTRY; 3061 vmcs12->exit_qualification = exit_qual; 3062 if (enable_shadow_vmcs || vmx->nested.hv_evmcs) 3063 vmx->nested.need_vmcs12_sync = true; 3064 return 1; 3065 } 3066 3067 /* 3068 * nested_vmx_run() handles a nested entry, i.e., a VMLAUNCH or VMRESUME on L1 3069 * for running an L2 nested guest. 3070 */ 3071 static int nested_vmx_run(struct kvm_vcpu *vcpu, bool launch) 3072 { 3073 struct vmcs12 *vmcs12; 3074 struct vcpu_vmx *vmx = to_vmx(vcpu); 3075 u32 interrupt_shadow = vmx_get_interrupt_shadow(vcpu); 3076 int ret; 3077 3078 if (!nested_vmx_check_permission(vcpu)) 3079 return 1; 3080 3081 if (!nested_vmx_handle_enlightened_vmptrld(vcpu, true)) 3082 return 1; 3083 3084 if (!vmx->nested.hv_evmcs && vmx->nested.current_vmptr == -1ull) 3085 return nested_vmx_failInvalid(vcpu); 3086 3087 vmcs12 = get_vmcs12(vcpu); 3088 3089 /* 3090 * Can't VMLAUNCH or VMRESUME a shadow VMCS. Despite the fact 3091 * that there *is* a valid VMCS pointer, RFLAGS.CF is set 3092 * rather than RFLAGS.ZF, and no error number is stored to the 3093 * VM-instruction error field. 3094 */ 3095 if (vmcs12->hdr.shadow_vmcs) 3096 return nested_vmx_failInvalid(vcpu); 3097 3098 if (vmx->nested.hv_evmcs) { 3099 copy_enlightened_to_vmcs12(vmx); 3100 /* Enlightened VMCS doesn't have launch state */ 3101 vmcs12->launch_state = !launch; 3102 } else if (enable_shadow_vmcs) { 3103 copy_shadow_to_vmcs12(vmx); 3104 } 3105 3106 /* 3107 * The nested entry process starts with enforcing various prerequisites 3108 * on vmcs12 as required by the Intel SDM, and act appropriately when 3109 * they fail: As the SDM explains, some conditions should cause the 3110 * instruction to fail, while others will cause the instruction to seem 3111 * to succeed, but return an EXIT_REASON_INVALID_STATE. 3112 * To speed up the normal (success) code path, we should avoid checking 3113 * for misconfigurations which will anyway be caught by the processor 3114 * when using the merged vmcs02. 3115 */ 3116 if (interrupt_shadow & KVM_X86_SHADOW_INT_MOV_SS) 3117 return nested_vmx_failValid(vcpu, 3118 VMXERR_ENTRY_EVENTS_BLOCKED_BY_MOV_SS); 3119 3120 if (vmcs12->launch_state == launch) 3121 return nested_vmx_failValid(vcpu, 3122 launch ? VMXERR_VMLAUNCH_NONCLEAR_VMCS 3123 : VMXERR_VMRESUME_NONLAUNCHED_VMCS); 3124 3125 ret = nested_vmx_check_vmentry_prereqs(vcpu, vmcs12); 3126 if (ret) 3127 return nested_vmx_failValid(vcpu, ret); 3128 3129 /* 3130 * We're finally done with prerequisite checking, and can start with 3131 * the nested entry. 3132 */ 3133 vmx->nested.nested_run_pending = 1; 3134 ret = nested_vmx_enter_non_root_mode(vcpu, true); 3135 vmx->nested.nested_run_pending = !ret; 3136 if (ret > 0) 3137 return 1; 3138 else if (ret) 3139 return nested_vmx_failValid(vcpu, 3140 VMXERR_ENTRY_INVALID_CONTROL_FIELD); 3141 3142 /* Hide L1D cache contents from the nested guest. */ 3143 vmx->vcpu.arch.l1tf_flush_l1d = true; 3144 3145 /* 3146 * Must happen outside of nested_vmx_enter_non_root_mode() as it will 3147 * also be used as part of restoring nVMX state for 3148 * snapshot restore (migration). 3149 * 3150 * In this flow, it is assumed that vmcs12 cache was 3151 * trasferred as part of captured nVMX state and should 3152 * therefore not be read from guest memory (which may not 3153 * exist on destination host yet). 3154 */ 3155 nested_cache_shadow_vmcs12(vcpu, vmcs12); 3156 3157 /* 3158 * If we're entering a halted L2 vcpu and the L2 vcpu won't be 3159 * awakened by event injection or by an NMI-window VM-exit or 3160 * by an interrupt-window VM-exit, halt the vcpu. 3161 */ 3162 if ((vmcs12->guest_activity_state == GUEST_ACTIVITY_HLT) && 3163 !(vmcs12->vm_entry_intr_info_field & INTR_INFO_VALID_MASK) && 3164 !(vmcs12->cpu_based_vm_exec_control & CPU_BASED_VIRTUAL_NMI_PENDING) && 3165 !((vmcs12->cpu_based_vm_exec_control & CPU_BASED_VIRTUAL_INTR_PENDING) && 3166 (vmcs12->guest_rflags & X86_EFLAGS_IF))) { 3167 vmx->nested.nested_run_pending = 0; 3168 return kvm_vcpu_halt(vcpu); 3169 } 3170 return 1; 3171 } 3172 3173 /* 3174 * On a nested exit from L2 to L1, vmcs12.guest_cr0 might not be up-to-date 3175 * because L2 may have changed some cr0 bits directly (CRO_GUEST_HOST_MASK). 3176 * This function returns the new value we should put in vmcs12.guest_cr0. 3177 * It's not enough to just return the vmcs02 GUEST_CR0. Rather, 3178 * 1. Bits that neither L0 nor L1 trapped, were set directly by L2 and are now 3179 * available in vmcs02 GUEST_CR0. (Note: It's enough to check that L0 3180 * didn't trap the bit, because if L1 did, so would L0). 3181 * 2. Bits that L1 asked to trap (and therefore L0 also did) could not have 3182 * been modified by L2, and L1 knows it. So just leave the old value of 3183 * the bit from vmcs12.guest_cr0. Note that the bit from vmcs02 GUEST_CR0 3184 * isn't relevant, because if L0 traps this bit it can set it to anything. 3185 * 3. Bits that L1 didn't trap, but L0 did. L1 believes the guest could have 3186 * changed these bits, and therefore they need to be updated, but L0 3187 * didn't necessarily allow them to be changed in GUEST_CR0 - and rather 3188 * put them in vmcs02 CR0_READ_SHADOW. So take these bits from there. 3189 */ 3190 static inline unsigned long 3191 vmcs12_guest_cr0(struct kvm_vcpu *vcpu, struct vmcs12 *vmcs12) 3192 { 3193 return 3194 /*1*/ (vmcs_readl(GUEST_CR0) & vcpu->arch.cr0_guest_owned_bits) | 3195 /*2*/ (vmcs12->guest_cr0 & vmcs12->cr0_guest_host_mask) | 3196 /*3*/ (vmcs_readl(CR0_READ_SHADOW) & ~(vmcs12->cr0_guest_host_mask | 3197 vcpu->arch.cr0_guest_owned_bits)); 3198 } 3199 3200 static inline unsigned long 3201 vmcs12_guest_cr4(struct kvm_vcpu *vcpu, struct vmcs12 *vmcs12) 3202 { 3203 return 3204 /*1*/ (vmcs_readl(GUEST_CR4) & vcpu->arch.cr4_guest_owned_bits) | 3205 /*2*/ (vmcs12->guest_cr4 & vmcs12->cr4_guest_host_mask) | 3206 /*3*/ (vmcs_readl(CR4_READ_SHADOW) & ~(vmcs12->cr4_guest_host_mask | 3207 vcpu->arch.cr4_guest_owned_bits)); 3208 } 3209 3210 static void vmcs12_save_pending_event(struct kvm_vcpu *vcpu, 3211 struct vmcs12 *vmcs12) 3212 { 3213 u32 idt_vectoring; 3214 unsigned int nr; 3215 3216 if (vcpu->arch.exception.injected) { 3217 nr = vcpu->arch.exception.nr; 3218 idt_vectoring = nr | VECTORING_INFO_VALID_MASK; 3219 3220 if (kvm_exception_is_soft(nr)) { 3221 vmcs12->vm_exit_instruction_len = 3222 vcpu->arch.event_exit_inst_len; 3223 idt_vectoring |= INTR_TYPE_SOFT_EXCEPTION; 3224 } else 3225 idt_vectoring |= INTR_TYPE_HARD_EXCEPTION; 3226 3227 if (vcpu->arch.exception.has_error_code) { 3228 idt_vectoring |= VECTORING_INFO_DELIVER_CODE_MASK; 3229 vmcs12->idt_vectoring_error_code = 3230 vcpu->arch.exception.error_code; 3231 } 3232 3233 vmcs12->idt_vectoring_info_field = idt_vectoring; 3234 } else if (vcpu->arch.nmi_injected) { 3235 vmcs12->idt_vectoring_info_field = 3236 INTR_TYPE_NMI_INTR | INTR_INFO_VALID_MASK | NMI_VECTOR; 3237 } else if (vcpu->arch.interrupt.injected) { 3238 nr = vcpu->arch.interrupt.nr; 3239 idt_vectoring = nr | VECTORING_INFO_VALID_MASK; 3240 3241 if (vcpu->arch.interrupt.soft) { 3242 idt_vectoring |= INTR_TYPE_SOFT_INTR; 3243 vmcs12->vm_entry_instruction_len = 3244 vcpu->arch.event_exit_inst_len; 3245 } else 3246 idt_vectoring |= INTR_TYPE_EXT_INTR; 3247 3248 vmcs12->idt_vectoring_info_field = idt_vectoring; 3249 } 3250 } 3251 3252 3253 static void nested_mark_vmcs12_pages_dirty(struct kvm_vcpu *vcpu) 3254 { 3255 struct vmcs12 *vmcs12 = get_vmcs12(vcpu); 3256 gfn_t gfn; 3257 3258 /* 3259 * Don't need to mark the APIC access page dirty; it is never 3260 * written to by the CPU during APIC virtualization. 3261 */ 3262 3263 if (nested_cpu_has(vmcs12, CPU_BASED_TPR_SHADOW)) { 3264 gfn = vmcs12->virtual_apic_page_addr >> PAGE_SHIFT; 3265 kvm_vcpu_mark_page_dirty(vcpu, gfn); 3266 } 3267 3268 if (nested_cpu_has_posted_intr(vmcs12)) { 3269 gfn = vmcs12->posted_intr_desc_addr >> PAGE_SHIFT; 3270 kvm_vcpu_mark_page_dirty(vcpu, gfn); 3271 } 3272 } 3273 3274 static void vmx_complete_nested_posted_interrupt(struct kvm_vcpu *vcpu) 3275 { 3276 struct vcpu_vmx *vmx = to_vmx(vcpu); 3277 int max_irr; 3278 void *vapic_page; 3279 u16 status; 3280 3281 if (!vmx->nested.pi_desc || !vmx->nested.pi_pending) 3282 return; 3283 3284 vmx->nested.pi_pending = false; 3285 if (!pi_test_and_clear_on(vmx->nested.pi_desc)) 3286 return; 3287 3288 max_irr = find_last_bit((unsigned long *)vmx->nested.pi_desc->pir, 256); 3289 if (max_irr != 256) { 3290 vapic_page = kmap(vmx->nested.virtual_apic_page); 3291 __kvm_apic_update_irr(vmx->nested.pi_desc->pir, 3292 vapic_page, &max_irr); 3293 kunmap(vmx->nested.virtual_apic_page); 3294 3295 status = vmcs_read16(GUEST_INTR_STATUS); 3296 if ((u8)max_irr > ((u8)status & 0xff)) { 3297 status &= ~0xff; 3298 status |= (u8)max_irr; 3299 vmcs_write16(GUEST_INTR_STATUS, status); 3300 } 3301 } 3302 3303 nested_mark_vmcs12_pages_dirty(vcpu); 3304 } 3305 3306 static void nested_vmx_inject_exception_vmexit(struct kvm_vcpu *vcpu, 3307 unsigned long exit_qual) 3308 { 3309 struct vmcs12 *vmcs12 = get_vmcs12(vcpu); 3310 unsigned int nr = vcpu->arch.exception.nr; 3311 u32 intr_info = nr | INTR_INFO_VALID_MASK; 3312 3313 if (vcpu->arch.exception.has_error_code) { 3314 vmcs12->vm_exit_intr_error_code = vcpu->arch.exception.error_code; 3315 intr_info |= INTR_INFO_DELIVER_CODE_MASK; 3316 } 3317 3318 if (kvm_exception_is_soft(nr)) 3319 intr_info |= INTR_TYPE_SOFT_EXCEPTION; 3320 else 3321 intr_info |= INTR_TYPE_HARD_EXCEPTION; 3322 3323 if (!(vmcs12->idt_vectoring_info_field & VECTORING_INFO_VALID_MASK) && 3324 vmx_get_nmi_mask(vcpu)) 3325 intr_info |= INTR_INFO_UNBLOCK_NMI; 3326 3327 nested_vmx_vmexit(vcpu, EXIT_REASON_EXCEPTION_NMI, intr_info, exit_qual); 3328 } 3329 3330 static int vmx_check_nested_events(struct kvm_vcpu *vcpu, bool external_intr) 3331 { 3332 struct vcpu_vmx *vmx = to_vmx(vcpu); 3333 unsigned long exit_qual; 3334 bool block_nested_events = 3335 vmx->nested.nested_run_pending || kvm_event_needs_reinjection(vcpu); 3336 3337 if (vcpu->arch.exception.pending && 3338 nested_vmx_check_exception(vcpu, &exit_qual)) { 3339 if (block_nested_events) 3340 return -EBUSY; 3341 nested_vmx_inject_exception_vmexit(vcpu, exit_qual); 3342 return 0; 3343 } 3344 3345 if (nested_cpu_has_preemption_timer(get_vmcs12(vcpu)) && 3346 vmx->nested.preemption_timer_expired) { 3347 if (block_nested_events) 3348 return -EBUSY; 3349 nested_vmx_vmexit(vcpu, EXIT_REASON_PREEMPTION_TIMER, 0, 0); 3350 return 0; 3351 } 3352 3353 if (vcpu->arch.nmi_pending && nested_exit_on_nmi(vcpu)) { 3354 if (block_nested_events) 3355 return -EBUSY; 3356 nested_vmx_vmexit(vcpu, EXIT_REASON_EXCEPTION_NMI, 3357 NMI_VECTOR | INTR_TYPE_NMI_INTR | 3358 INTR_INFO_VALID_MASK, 0); 3359 /* 3360 * The NMI-triggered VM exit counts as injection: 3361 * clear this one and block further NMIs. 3362 */ 3363 vcpu->arch.nmi_pending = 0; 3364 vmx_set_nmi_mask(vcpu, true); 3365 return 0; 3366 } 3367 3368 if ((kvm_cpu_has_interrupt(vcpu) || external_intr) && 3369 nested_exit_on_intr(vcpu)) { 3370 if (block_nested_events) 3371 return -EBUSY; 3372 nested_vmx_vmexit(vcpu, EXIT_REASON_EXTERNAL_INTERRUPT, 0, 0); 3373 return 0; 3374 } 3375 3376 vmx_complete_nested_posted_interrupt(vcpu); 3377 return 0; 3378 } 3379 3380 static u32 vmx_get_preemption_timer_value(struct kvm_vcpu *vcpu) 3381 { 3382 ktime_t remaining = 3383 hrtimer_get_remaining(&to_vmx(vcpu)->nested.preemption_timer); 3384 u64 value; 3385 3386 if (ktime_to_ns(remaining) <= 0) 3387 return 0; 3388 3389 value = ktime_to_ns(remaining) * vcpu->arch.virtual_tsc_khz; 3390 do_div(value, 1000000); 3391 return value >> VMX_MISC_EMULATED_PREEMPTION_TIMER_RATE; 3392 } 3393 3394 /* 3395 * Update the guest state fields of vmcs12 to reflect changes that 3396 * occurred while L2 was running. (The "IA-32e mode guest" bit of the 3397 * VM-entry controls is also updated, since this is really a guest 3398 * state bit.) 3399 */ 3400 static void sync_vmcs12(struct kvm_vcpu *vcpu, struct vmcs12 *vmcs12) 3401 { 3402 vmcs12->guest_cr0 = vmcs12_guest_cr0(vcpu, vmcs12); 3403 vmcs12->guest_cr4 = vmcs12_guest_cr4(vcpu, vmcs12); 3404 3405 vmcs12->guest_rsp = kvm_register_read(vcpu, VCPU_REGS_RSP); 3406 vmcs12->guest_rip = kvm_register_read(vcpu, VCPU_REGS_RIP); 3407 vmcs12->guest_rflags = vmcs_readl(GUEST_RFLAGS); 3408 3409 vmcs12->guest_es_selector = vmcs_read16(GUEST_ES_SELECTOR); 3410 vmcs12->guest_cs_selector = vmcs_read16(GUEST_CS_SELECTOR); 3411 vmcs12->guest_ss_selector = vmcs_read16(GUEST_SS_SELECTOR); 3412 vmcs12->guest_ds_selector = vmcs_read16(GUEST_DS_SELECTOR); 3413 vmcs12->guest_fs_selector = vmcs_read16(GUEST_FS_SELECTOR); 3414 vmcs12->guest_gs_selector = vmcs_read16(GUEST_GS_SELECTOR); 3415 vmcs12->guest_ldtr_selector = vmcs_read16(GUEST_LDTR_SELECTOR); 3416 vmcs12->guest_tr_selector = vmcs_read16(GUEST_TR_SELECTOR); 3417 vmcs12->guest_es_limit = vmcs_read32(GUEST_ES_LIMIT); 3418 vmcs12->guest_cs_limit = vmcs_read32(GUEST_CS_LIMIT); 3419 vmcs12->guest_ss_limit = vmcs_read32(GUEST_SS_LIMIT); 3420 vmcs12->guest_ds_limit = vmcs_read32(GUEST_DS_LIMIT); 3421 vmcs12->guest_fs_limit = vmcs_read32(GUEST_FS_LIMIT); 3422 vmcs12->guest_gs_limit = vmcs_read32(GUEST_GS_LIMIT); 3423 vmcs12->guest_ldtr_limit = vmcs_read32(GUEST_LDTR_LIMIT); 3424 vmcs12->guest_tr_limit = vmcs_read32(GUEST_TR_LIMIT); 3425 vmcs12->guest_gdtr_limit = vmcs_read32(GUEST_GDTR_LIMIT); 3426 vmcs12->guest_idtr_limit = vmcs_read32(GUEST_IDTR_LIMIT); 3427 vmcs12->guest_es_ar_bytes = vmcs_read32(GUEST_ES_AR_BYTES); 3428 vmcs12->guest_cs_ar_bytes = vmcs_read32(GUEST_CS_AR_BYTES); 3429 vmcs12->guest_ss_ar_bytes = vmcs_read32(GUEST_SS_AR_BYTES); 3430 vmcs12->guest_ds_ar_bytes = vmcs_read32(GUEST_DS_AR_BYTES); 3431 vmcs12->guest_fs_ar_bytes = vmcs_read32(GUEST_FS_AR_BYTES); 3432 vmcs12->guest_gs_ar_bytes = vmcs_read32(GUEST_GS_AR_BYTES); 3433 vmcs12->guest_ldtr_ar_bytes = vmcs_read32(GUEST_LDTR_AR_BYTES); 3434 vmcs12->guest_tr_ar_bytes = vmcs_read32(GUEST_TR_AR_BYTES); 3435 vmcs12->guest_es_base = vmcs_readl(GUEST_ES_BASE); 3436 vmcs12->guest_cs_base = vmcs_readl(GUEST_CS_BASE); 3437 vmcs12->guest_ss_base = vmcs_readl(GUEST_SS_BASE); 3438 vmcs12->guest_ds_base = vmcs_readl(GUEST_DS_BASE); 3439 vmcs12->guest_fs_base = vmcs_readl(GUEST_FS_BASE); 3440 vmcs12->guest_gs_base = vmcs_readl(GUEST_GS_BASE); 3441 vmcs12->guest_ldtr_base = vmcs_readl(GUEST_LDTR_BASE); 3442 vmcs12->guest_tr_base = vmcs_readl(GUEST_TR_BASE); 3443 vmcs12->guest_gdtr_base = vmcs_readl(GUEST_GDTR_BASE); 3444 vmcs12->guest_idtr_base = vmcs_readl(GUEST_IDTR_BASE); 3445 3446 vmcs12->guest_interruptibility_info = 3447 vmcs_read32(GUEST_INTERRUPTIBILITY_INFO); 3448 vmcs12->guest_pending_dbg_exceptions = 3449 vmcs_readl(GUEST_PENDING_DBG_EXCEPTIONS); 3450 if (vcpu->arch.mp_state == KVM_MP_STATE_HALTED) 3451 vmcs12->guest_activity_state = GUEST_ACTIVITY_HLT; 3452 else 3453 vmcs12->guest_activity_state = GUEST_ACTIVITY_ACTIVE; 3454 3455 if (nested_cpu_has_preemption_timer(vmcs12) && 3456 vmcs12->vm_exit_controls & VM_EXIT_SAVE_VMX_PREEMPTION_TIMER) 3457 vmcs12->vmx_preemption_timer_value = 3458 vmx_get_preemption_timer_value(vcpu); 3459 3460 /* 3461 * In some cases (usually, nested EPT), L2 is allowed to change its 3462 * own CR3 without exiting. If it has changed it, we must keep it. 3463 * Of course, if L0 is using shadow page tables, GUEST_CR3 was defined 3464 * by L0, not L1 or L2, so we mustn't unconditionally copy it to vmcs12. 3465 * 3466 * Additionally, restore L2's PDPTR to vmcs12. 3467 */ 3468 if (enable_ept) { 3469 vmcs12->guest_cr3 = vmcs_readl(GUEST_CR3); 3470 vmcs12->guest_pdptr0 = vmcs_read64(GUEST_PDPTR0); 3471 vmcs12->guest_pdptr1 = vmcs_read64(GUEST_PDPTR1); 3472 vmcs12->guest_pdptr2 = vmcs_read64(GUEST_PDPTR2); 3473 vmcs12->guest_pdptr3 = vmcs_read64(GUEST_PDPTR3); 3474 } 3475 3476 vmcs12->guest_linear_address = vmcs_readl(GUEST_LINEAR_ADDRESS); 3477 3478 if (nested_cpu_has_vid(vmcs12)) 3479 vmcs12->guest_intr_status = vmcs_read16(GUEST_INTR_STATUS); 3480 3481 vmcs12->vm_entry_controls = 3482 (vmcs12->vm_entry_controls & ~VM_ENTRY_IA32E_MODE) | 3483 (vm_entry_controls_get(to_vmx(vcpu)) & VM_ENTRY_IA32E_MODE); 3484 3485 if (vmcs12->vm_exit_controls & VM_EXIT_SAVE_DEBUG_CONTROLS) { 3486 kvm_get_dr(vcpu, 7, (unsigned long *)&vmcs12->guest_dr7); 3487 vmcs12->guest_ia32_debugctl = vmcs_read64(GUEST_IA32_DEBUGCTL); 3488 } 3489 3490 /* TODO: These cannot have changed unless we have MSR bitmaps and 3491 * the relevant bit asks not to trap the change */ 3492 if (vmcs12->vm_exit_controls & VM_EXIT_SAVE_IA32_PAT) 3493 vmcs12->guest_ia32_pat = vmcs_read64(GUEST_IA32_PAT); 3494 if (vmcs12->vm_exit_controls & VM_EXIT_SAVE_IA32_EFER) 3495 vmcs12->guest_ia32_efer = vcpu->arch.efer; 3496 vmcs12->guest_sysenter_cs = vmcs_read32(GUEST_SYSENTER_CS); 3497 vmcs12->guest_sysenter_esp = vmcs_readl(GUEST_SYSENTER_ESP); 3498 vmcs12->guest_sysenter_eip = vmcs_readl(GUEST_SYSENTER_EIP); 3499 if (kvm_mpx_supported()) 3500 vmcs12->guest_bndcfgs = vmcs_read64(GUEST_BNDCFGS); 3501 } 3502 3503 /* 3504 * prepare_vmcs12 is part of what we need to do when the nested L2 guest exits 3505 * and we want to prepare to run its L1 parent. L1 keeps a vmcs for L2 (vmcs12), 3506 * and this function updates it to reflect the changes to the guest state while 3507 * L2 was running (and perhaps made some exits which were handled directly by L0 3508 * without going back to L1), and to reflect the exit reason. 3509 * Note that we do not have to copy here all VMCS fields, just those that 3510 * could have changed by the L2 guest or the exit - i.e., the guest-state and 3511 * exit-information fields only. Other fields are modified by L1 with VMWRITE, 3512 * which already writes to vmcs12 directly. 3513 */ 3514 static void prepare_vmcs12(struct kvm_vcpu *vcpu, struct vmcs12 *vmcs12, 3515 u32 exit_reason, u32 exit_intr_info, 3516 unsigned long exit_qualification) 3517 { 3518 /* update guest state fields: */ 3519 sync_vmcs12(vcpu, vmcs12); 3520 3521 /* update exit information fields: */ 3522 3523 vmcs12->vm_exit_reason = exit_reason; 3524 vmcs12->exit_qualification = exit_qualification; 3525 vmcs12->vm_exit_intr_info = exit_intr_info; 3526 3527 vmcs12->idt_vectoring_info_field = 0; 3528 vmcs12->vm_exit_instruction_len = vmcs_read32(VM_EXIT_INSTRUCTION_LEN); 3529 vmcs12->vmx_instruction_info = vmcs_read32(VMX_INSTRUCTION_INFO); 3530 3531 if (!(vmcs12->vm_exit_reason & VMX_EXIT_REASONS_FAILED_VMENTRY)) { 3532 vmcs12->launch_state = 1; 3533 3534 /* vm_entry_intr_info_field is cleared on exit. Emulate this 3535 * instead of reading the real value. */ 3536 vmcs12->vm_entry_intr_info_field &= ~INTR_INFO_VALID_MASK; 3537 3538 /* 3539 * Transfer the event that L0 or L1 may wanted to inject into 3540 * L2 to IDT_VECTORING_INFO_FIELD. 3541 */ 3542 vmcs12_save_pending_event(vcpu, vmcs12); 3543 3544 /* 3545 * According to spec, there's no need to store the guest's 3546 * MSRs if the exit is due to a VM-entry failure that occurs 3547 * during or after loading the guest state. Since this exit 3548 * does not fall in that category, we need to save the MSRs. 3549 */ 3550 if (nested_vmx_store_msr(vcpu, 3551 vmcs12->vm_exit_msr_store_addr, 3552 vmcs12->vm_exit_msr_store_count)) 3553 nested_vmx_abort(vcpu, 3554 VMX_ABORT_SAVE_GUEST_MSR_FAIL); 3555 } 3556 3557 /* 3558 * Drop what we picked up for L2 via vmx_complete_interrupts. It is 3559 * preserved above and would only end up incorrectly in L1. 3560 */ 3561 vcpu->arch.nmi_injected = false; 3562 kvm_clear_exception_queue(vcpu); 3563 kvm_clear_interrupt_queue(vcpu); 3564 } 3565 3566 /* 3567 * A part of what we need to when the nested L2 guest exits and we want to 3568 * run its L1 parent, is to reset L1's guest state to the host state specified 3569 * in vmcs12. 3570 * This function is to be called not only on normal nested exit, but also on 3571 * a nested entry failure, as explained in Intel's spec, 3B.23.7 ("VM-Entry 3572 * Failures During or After Loading Guest State"). 3573 * This function should be called when the active VMCS is L1's (vmcs01). 3574 */ 3575 static void load_vmcs12_host_state(struct kvm_vcpu *vcpu, 3576 struct vmcs12 *vmcs12) 3577 { 3578 struct kvm_segment seg; 3579 u32 entry_failure_code; 3580 3581 if (vmcs12->vm_exit_controls & VM_EXIT_LOAD_IA32_EFER) 3582 vcpu->arch.efer = vmcs12->host_ia32_efer; 3583 else if (vmcs12->vm_exit_controls & VM_EXIT_HOST_ADDR_SPACE_SIZE) 3584 vcpu->arch.efer |= (EFER_LMA | EFER_LME); 3585 else 3586 vcpu->arch.efer &= ~(EFER_LMA | EFER_LME); 3587 vmx_set_efer(vcpu, vcpu->arch.efer); 3588 3589 kvm_register_write(vcpu, VCPU_REGS_RSP, vmcs12->host_rsp); 3590 kvm_register_write(vcpu, VCPU_REGS_RIP, vmcs12->host_rip); 3591 vmx_set_rflags(vcpu, X86_EFLAGS_FIXED); 3592 vmx_set_interrupt_shadow(vcpu, 0); 3593 3594 /* 3595 * Note that calling vmx_set_cr0 is important, even if cr0 hasn't 3596 * actually changed, because vmx_set_cr0 refers to efer set above. 3597 * 3598 * CR0_GUEST_HOST_MASK is already set in the original vmcs01 3599 * (KVM doesn't change it); 3600 */ 3601 vcpu->arch.cr0_guest_owned_bits = X86_CR0_TS; 3602 vmx_set_cr0(vcpu, vmcs12->host_cr0); 3603 3604 /* Same as above - no reason to call set_cr4_guest_host_mask(). */ 3605 vcpu->arch.cr4_guest_owned_bits = ~vmcs_readl(CR4_GUEST_HOST_MASK); 3606 vmx_set_cr4(vcpu, vmcs12->host_cr4); 3607 3608 nested_ept_uninit_mmu_context(vcpu); 3609 3610 /* 3611 * Only PDPTE load can fail as the value of cr3 was checked on entry and 3612 * couldn't have changed. 3613 */ 3614 if (nested_vmx_load_cr3(vcpu, vmcs12->host_cr3, false, &entry_failure_code)) 3615 nested_vmx_abort(vcpu, VMX_ABORT_LOAD_HOST_PDPTE_FAIL); 3616 3617 if (!enable_ept) 3618 vcpu->arch.walk_mmu->inject_page_fault = kvm_inject_page_fault; 3619 3620 /* 3621 * If vmcs01 doesn't use VPID, CPU flushes TLB on every 3622 * VMEntry/VMExit. Thus, no need to flush TLB. 3623 * 3624 * If vmcs12 doesn't use VPID, L1 expects TLB to be 3625 * flushed on every VMEntry/VMExit. 3626 * 3627 * Otherwise, we can preserve TLB entries as long as we are 3628 * able to tag L1 TLB entries differently than L2 TLB entries. 3629 * 3630 * If vmcs12 uses EPT, we need to execute this flush on EPTP01 3631 * and therefore we request the TLB flush to happen only after VMCS EPTP 3632 * has been set by KVM_REQ_LOAD_CR3. 3633 */ 3634 if (enable_vpid && 3635 (!nested_cpu_has_vpid(vmcs12) || !nested_has_guest_tlb_tag(vcpu))) { 3636 kvm_make_request(KVM_REQ_TLB_FLUSH, vcpu); 3637 } 3638 3639 vmcs_write32(GUEST_SYSENTER_CS, vmcs12->host_ia32_sysenter_cs); 3640 vmcs_writel(GUEST_SYSENTER_ESP, vmcs12->host_ia32_sysenter_esp); 3641 vmcs_writel(GUEST_SYSENTER_EIP, vmcs12->host_ia32_sysenter_eip); 3642 vmcs_writel(GUEST_IDTR_BASE, vmcs12->host_idtr_base); 3643 vmcs_writel(GUEST_GDTR_BASE, vmcs12->host_gdtr_base); 3644 vmcs_write32(GUEST_IDTR_LIMIT, 0xFFFF); 3645 vmcs_write32(GUEST_GDTR_LIMIT, 0xFFFF); 3646 3647 /* If not VM_EXIT_CLEAR_BNDCFGS, the L2 value propagates to L1. */ 3648 if (vmcs12->vm_exit_controls & VM_EXIT_CLEAR_BNDCFGS) 3649 vmcs_write64(GUEST_BNDCFGS, 0); 3650 3651 if (vmcs12->vm_exit_controls & VM_EXIT_LOAD_IA32_PAT) { 3652 vmcs_write64(GUEST_IA32_PAT, vmcs12->host_ia32_pat); 3653 vcpu->arch.pat = vmcs12->host_ia32_pat; 3654 } 3655 if (vmcs12->vm_exit_controls & VM_EXIT_LOAD_IA32_PERF_GLOBAL_CTRL) 3656 vmcs_write64(GUEST_IA32_PERF_GLOBAL_CTRL, 3657 vmcs12->host_ia32_perf_global_ctrl); 3658 3659 /* Set L1 segment info according to Intel SDM 3660 27.5.2 Loading Host Segment and Descriptor-Table Registers */ 3661 seg = (struct kvm_segment) { 3662 .base = 0, 3663 .limit = 0xFFFFFFFF, 3664 .selector = vmcs12->host_cs_selector, 3665 .type = 11, 3666 .present = 1, 3667 .s = 1, 3668 .g = 1 3669 }; 3670 if (vmcs12->vm_exit_controls & VM_EXIT_HOST_ADDR_SPACE_SIZE) 3671 seg.l = 1; 3672 else 3673 seg.db = 1; 3674 vmx_set_segment(vcpu, &seg, VCPU_SREG_CS); 3675 seg = (struct kvm_segment) { 3676 .base = 0, 3677 .limit = 0xFFFFFFFF, 3678 .type = 3, 3679 .present = 1, 3680 .s = 1, 3681 .db = 1, 3682 .g = 1 3683 }; 3684 seg.selector = vmcs12->host_ds_selector; 3685 vmx_set_segment(vcpu, &seg, VCPU_SREG_DS); 3686 seg.selector = vmcs12->host_es_selector; 3687 vmx_set_segment(vcpu, &seg, VCPU_SREG_ES); 3688 seg.selector = vmcs12->host_ss_selector; 3689 vmx_set_segment(vcpu, &seg, VCPU_SREG_SS); 3690 seg.selector = vmcs12->host_fs_selector; 3691 seg.base = vmcs12->host_fs_base; 3692 vmx_set_segment(vcpu, &seg, VCPU_SREG_FS); 3693 seg.selector = vmcs12->host_gs_selector; 3694 seg.base = vmcs12->host_gs_base; 3695 vmx_set_segment(vcpu, &seg, VCPU_SREG_GS); 3696 seg = (struct kvm_segment) { 3697 .base = vmcs12->host_tr_base, 3698 .limit = 0x67, 3699 .selector = vmcs12->host_tr_selector, 3700 .type = 11, 3701 .present = 1 3702 }; 3703 vmx_set_segment(vcpu, &seg, VCPU_SREG_TR); 3704 3705 kvm_set_dr(vcpu, 7, 0x400); 3706 vmcs_write64(GUEST_IA32_DEBUGCTL, 0); 3707 3708 if (cpu_has_vmx_msr_bitmap()) 3709 vmx_update_msr_bitmap(vcpu); 3710 3711 if (nested_vmx_load_msr(vcpu, vmcs12->vm_exit_msr_load_addr, 3712 vmcs12->vm_exit_msr_load_count)) 3713 nested_vmx_abort(vcpu, VMX_ABORT_LOAD_HOST_MSR_FAIL); 3714 } 3715 3716 static inline u64 nested_vmx_get_vmcs01_guest_efer(struct vcpu_vmx *vmx) 3717 { 3718 struct shared_msr_entry *efer_msr; 3719 unsigned int i; 3720 3721 if (vm_entry_controls_get(vmx) & VM_ENTRY_LOAD_IA32_EFER) 3722 return vmcs_read64(GUEST_IA32_EFER); 3723 3724 if (cpu_has_load_ia32_efer()) 3725 return host_efer; 3726 3727 for (i = 0; i < vmx->msr_autoload.guest.nr; ++i) { 3728 if (vmx->msr_autoload.guest.val[i].index == MSR_EFER) 3729 return vmx->msr_autoload.guest.val[i].value; 3730 } 3731 3732 efer_msr = find_msr_entry(vmx, MSR_EFER); 3733 if (efer_msr) 3734 return efer_msr->data; 3735 3736 return host_efer; 3737 } 3738 3739 static void nested_vmx_restore_host_state(struct kvm_vcpu *vcpu) 3740 { 3741 struct vmcs12 *vmcs12 = get_vmcs12(vcpu); 3742 struct vcpu_vmx *vmx = to_vmx(vcpu); 3743 struct vmx_msr_entry g, h; 3744 struct msr_data msr; 3745 gpa_t gpa; 3746 u32 i, j; 3747 3748 vcpu->arch.pat = vmcs_read64(GUEST_IA32_PAT); 3749 3750 if (vmcs12->vm_entry_controls & VM_ENTRY_LOAD_DEBUG_CONTROLS) { 3751 /* 3752 * L1's host DR7 is lost if KVM_GUESTDBG_USE_HW_BP is set 3753 * as vmcs01.GUEST_DR7 contains a userspace defined value 3754 * and vcpu->arch.dr7 is not squirreled away before the 3755 * nested VMENTER (not worth adding a variable in nested_vmx). 3756 */ 3757 if (vcpu->guest_debug & KVM_GUESTDBG_USE_HW_BP) 3758 kvm_set_dr(vcpu, 7, DR7_FIXED_1); 3759 else 3760 WARN_ON(kvm_set_dr(vcpu, 7, vmcs_readl(GUEST_DR7))); 3761 } 3762 3763 /* 3764 * Note that calling vmx_set_{efer,cr0,cr4} is important as they 3765 * handle a variety of side effects to KVM's software model. 3766 */ 3767 vmx_set_efer(vcpu, nested_vmx_get_vmcs01_guest_efer(vmx)); 3768 3769 vcpu->arch.cr0_guest_owned_bits = X86_CR0_TS; 3770 vmx_set_cr0(vcpu, vmcs_readl(CR0_READ_SHADOW)); 3771 3772 vcpu->arch.cr4_guest_owned_bits = ~vmcs_readl(CR4_GUEST_HOST_MASK); 3773 vmx_set_cr4(vcpu, vmcs_readl(CR4_READ_SHADOW)); 3774 3775 nested_ept_uninit_mmu_context(vcpu); 3776 vcpu->arch.cr3 = vmcs_readl(GUEST_CR3); 3777 __set_bit(VCPU_EXREG_CR3, (ulong *)&vcpu->arch.regs_avail); 3778 3779 /* 3780 * Use ept_save_pdptrs(vcpu) to load the MMU's cached PDPTRs 3781 * from vmcs01 (if necessary). The PDPTRs are not loaded on 3782 * VMFail, like everything else we just need to ensure our 3783 * software model is up-to-date. 3784 */ 3785 ept_save_pdptrs(vcpu); 3786 3787 kvm_mmu_reset_context(vcpu); 3788 3789 if (cpu_has_vmx_msr_bitmap()) 3790 vmx_update_msr_bitmap(vcpu); 3791 3792 /* 3793 * This nasty bit of open coding is a compromise between blindly 3794 * loading L1's MSRs using the exit load lists (incorrect emulation 3795 * of VMFail), leaving the nested VM's MSRs in the software model 3796 * (incorrect behavior) and snapshotting the modified MSRs (too 3797 * expensive since the lists are unbound by hardware). For each 3798 * MSR that was (prematurely) loaded from the nested VMEntry load 3799 * list, reload it from the exit load list if it exists and differs 3800 * from the guest value. The intent is to stuff host state as 3801 * silently as possible, not to fully process the exit load list. 3802 */ 3803 msr.host_initiated = false; 3804 for (i = 0; i < vmcs12->vm_entry_msr_load_count; i++) { 3805 gpa = vmcs12->vm_entry_msr_load_addr + (i * sizeof(g)); 3806 if (kvm_vcpu_read_guest(vcpu, gpa, &g, sizeof(g))) { 3807 pr_debug_ratelimited( 3808 "%s read MSR index failed (%u, 0x%08llx)\n", 3809 __func__, i, gpa); 3810 goto vmabort; 3811 } 3812 3813 for (j = 0; j < vmcs12->vm_exit_msr_load_count; j++) { 3814 gpa = vmcs12->vm_exit_msr_load_addr + (j * sizeof(h)); 3815 if (kvm_vcpu_read_guest(vcpu, gpa, &h, sizeof(h))) { 3816 pr_debug_ratelimited( 3817 "%s read MSR failed (%u, 0x%08llx)\n", 3818 __func__, j, gpa); 3819 goto vmabort; 3820 } 3821 if (h.index != g.index) 3822 continue; 3823 if (h.value == g.value) 3824 break; 3825 3826 if (nested_vmx_load_msr_check(vcpu, &h)) { 3827 pr_debug_ratelimited( 3828 "%s check failed (%u, 0x%x, 0x%x)\n", 3829 __func__, j, h.index, h.reserved); 3830 goto vmabort; 3831 } 3832 3833 msr.index = h.index; 3834 msr.data = h.value; 3835 if (kvm_set_msr(vcpu, &msr)) { 3836 pr_debug_ratelimited( 3837 "%s WRMSR failed (%u, 0x%x, 0x%llx)\n", 3838 __func__, j, h.index, h.value); 3839 goto vmabort; 3840 } 3841 } 3842 } 3843 3844 return; 3845 3846 vmabort: 3847 nested_vmx_abort(vcpu, VMX_ABORT_LOAD_HOST_MSR_FAIL); 3848 } 3849 3850 /* 3851 * Emulate an exit from nested guest (L2) to L1, i.e., prepare to run L1 3852 * and modify vmcs12 to make it see what it would expect to see there if 3853 * L2 was its real guest. Must only be called when in L2 (is_guest_mode()) 3854 */ 3855 void nested_vmx_vmexit(struct kvm_vcpu *vcpu, u32 exit_reason, 3856 u32 exit_intr_info, unsigned long exit_qualification) 3857 { 3858 struct vcpu_vmx *vmx = to_vmx(vcpu); 3859 struct vmcs12 *vmcs12 = get_vmcs12(vcpu); 3860 3861 /* trying to cancel vmlaunch/vmresume is a bug */ 3862 WARN_ON_ONCE(vmx->nested.nested_run_pending); 3863 3864 leave_guest_mode(vcpu); 3865 3866 if (nested_cpu_has_preemption_timer(vmcs12)) 3867 hrtimer_cancel(&to_vmx(vcpu)->nested.preemption_timer); 3868 3869 if (vmcs12->cpu_based_vm_exec_control & CPU_BASED_USE_TSC_OFFSETING) 3870 vcpu->arch.tsc_offset -= vmcs12->tsc_offset; 3871 3872 if (likely(!vmx->fail)) { 3873 if (exit_reason == -1) 3874 sync_vmcs12(vcpu, vmcs12); 3875 else 3876 prepare_vmcs12(vcpu, vmcs12, exit_reason, exit_intr_info, 3877 exit_qualification); 3878 3879 /* 3880 * Must happen outside of sync_vmcs12() as it will 3881 * also be used to capture vmcs12 cache as part of 3882 * capturing nVMX state for snapshot (migration). 3883 * 3884 * Otherwise, this flush will dirty guest memory at a 3885 * point it is already assumed by user-space to be 3886 * immutable. 3887 */ 3888 nested_flush_cached_shadow_vmcs12(vcpu, vmcs12); 3889 } else { 3890 /* 3891 * The only expected VM-instruction error is "VM entry with 3892 * invalid control field(s)." Anything else indicates a 3893 * problem with L0. And we should never get here with a 3894 * VMFail of any type if early consistency checks are enabled. 3895 */ 3896 WARN_ON_ONCE(vmcs_read32(VM_INSTRUCTION_ERROR) != 3897 VMXERR_ENTRY_INVALID_CONTROL_FIELD); 3898 WARN_ON_ONCE(nested_early_check); 3899 } 3900 3901 vmx_switch_vmcs(vcpu, &vmx->vmcs01); 3902 3903 /* Update any VMCS fields that might have changed while L2 ran */ 3904 vmcs_write32(VM_EXIT_MSR_LOAD_COUNT, vmx->msr_autoload.host.nr); 3905 vmcs_write32(VM_ENTRY_MSR_LOAD_COUNT, vmx->msr_autoload.guest.nr); 3906 vmcs_write64(TSC_OFFSET, vcpu->arch.tsc_offset); 3907 3908 if (kvm_has_tsc_control) 3909 decache_tsc_multiplier(vmx); 3910 3911 if (vmx->nested.change_vmcs01_virtual_apic_mode) { 3912 vmx->nested.change_vmcs01_virtual_apic_mode = false; 3913 vmx_set_virtual_apic_mode(vcpu); 3914 } else if (!nested_cpu_has_ept(vmcs12) && 3915 nested_cpu_has2(vmcs12, 3916 SECONDARY_EXEC_VIRTUALIZE_APIC_ACCESSES)) { 3917 vmx_flush_tlb(vcpu, true); 3918 } 3919 3920 /* Unpin physical memory we referred to in vmcs02 */ 3921 if (vmx->nested.apic_access_page) { 3922 kvm_release_page_dirty(vmx->nested.apic_access_page); 3923 vmx->nested.apic_access_page = NULL; 3924 } 3925 if (vmx->nested.virtual_apic_page) { 3926 kvm_release_page_dirty(vmx->nested.virtual_apic_page); 3927 vmx->nested.virtual_apic_page = NULL; 3928 } 3929 if (vmx->nested.pi_desc_page) { 3930 kunmap(vmx->nested.pi_desc_page); 3931 kvm_release_page_dirty(vmx->nested.pi_desc_page); 3932 vmx->nested.pi_desc_page = NULL; 3933 vmx->nested.pi_desc = NULL; 3934 } 3935 3936 /* 3937 * We are now running in L2, mmu_notifier will force to reload the 3938 * page's hpa for L2 vmcs. Need to reload it for L1 before entering L1. 3939 */ 3940 kvm_make_request(KVM_REQ_APIC_PAGE_RELOAD, vcpu); 3941 3942 if ((exit_reason != -1) && (enable_shadow_vmcs || vmx->nested.hv_evmcs)) 3943 vmx->nested.need_vmcs12_sync = true; 3944 3945 /* in case we halted in L2 */ 3946 vcpu->arch.mp_state = KVM_MP_STATE_RUNNABLE; 3947 3948 if (likely(!vmx->fail)) { 3949 /* 3950 * TODO: SDM says that with acknowledge interrupt on 3951 * exit, bit 31 of the VM-exit interrupt information 3952 * (valid interrupt) is always set to 1 on 3953 * EXIT_REASON_EXTERNAL_INTERRUPT, so we shouldn't 3954 * need kvm_cpu_has_interrupt(). See the commit 3955 * message for details. 3956 */ 3957 if (nested_exit_intr_ack_set(vcpu) && 3958 exit_reason == EXIT_REASON_EXTERNAL_INTERRUPT && 3959 kvm_cpu_has_interrupt(vcpu)) { 3960 int irq = kvm_cpu_get_interrupt(vcpu); 3961 WARN_ON(irq < 0); 3962 vmcs12->vm_exit_intr_info = irq | 3963 INTR_INFO_VALID_MASK | INTR_TYPE_EXT_INTR; 3964 } 3965 3966 if (exit_reason != -1) 3967 trace_kvm_nested_vmexit_inject(vmcs12->vm_exit_reason, 3968 vmcs12->exit_qualification, 3969 vmcs12->idt_vectoring_info_field, 3970 vmcs12->vm_exit_intr_info, 3971 vmcs12->vm_exit_intr_error_code, 3972 KVM_ISA_VMX); 3973 3974 load_vmcs12_host_state(vcpu, vmcs12); 3975 3976 return; 3977 } 3978 3979 /* 3980 * After an early L2 VM-entry failure, we're now back 3981 * in L1 which thinks it just finished a VMLAUNCH or 3982 * VMRESUME instruction, so we need to set the failure 3983 * flag and the VM-instruction error field of the VMCS 3984 * accordingly, and skip the emulated instruction. 3985 */ 3986 (void)nested_vmx_failValid(vcpu, VMXERR_ENTRY_INVALID_CONTROL_FIELD); 3987 3988 /* 3989 * Restore L1's host state to KVM's software model. We're here 3990 * because a consistency check was caught by hardware, which 3991 * means some amount of guest state has been propagated to KVM's 3992 * model and needs to be unwound to the host's state. 3993 */ 3994 nested_vmx_restore_host_state(vcpu); 3995 3996 vmx->fail = 0; 3997 } 3998 3999 /* 4000 * Decode the memory-address operand of a vmx instruction, as recorded on an 4001 * exit caused by such an instruction (run by a guest hypervisor). 4002 * On success, returns 0. When the operand is invalid, returns 1 and throws 4003 * #UD or #GP. 4004 */ 4005 int get_vmx_mem_address(struct kvm_vcpu *vcpu, unsigned long exit_qualification, 4006 u32 vmx_instruction_info, bool wr, gva_t *ret) 4007 { 4008 gva_t off; 4009 bool exn; 4010 struct kvm_segment s; 4011 4012 /* 4013 * According to Vol. 3B, "Information for VM Exits Due to Instruction 4014 * Execution", on an exit, vmx_instruction_info holds most of the 4015 * addressing components of the operand. Only the displacement part 4016 * is put in exit_qualification (see 3B, "Basic VM-Exit Information"). 4017 * For how an actual address is calculated from all these components, 4018 * refer to Vol. 1, "Operand Addressing". 4019 */ 4020 int scaling = vmx_instruction_info & 3; 4021 int addr_size = (vmx_instruction_info >> 7) & 7; 4022 bool is_reg = vmx_instruction_info & (1u << 10); 4023 int seg_reg = (vmx_instruction_info >> 15) & 7; 4024 int index_reg = (vmx_instruction_info >> 18) & 0xf; 4025 bool index_is_valid = !(vmx_instruction_info & (1u << 22)); 4026 int base_reg = (vmx_instruction_info >> 23) & 0xf; 4027 bool base_is_valid = !(vmx_instruction_info & (1u << 27)); 4028 4029 if (is_reg) { 4030 kvm_queue_exception(vcpu, UD_VECTOR); 4031 return 1; 4032 } 4033 4034 /* Addr = segment_base + offset */ 4035 /* offset = base + [index * scale] + displacement */ 4036 off = exit_qualification; /* holds the displacement */ 4037 if (addr_size == 1) 4038 off = (gva_t)sign_extend64(off, 31); 4039 else if (addr_size == 0) 4040 off = (gva_t)sign_extend64(off, 15); 4041 if (base_is_valid) 4042 off += kvm_register_read(vcpu, base_reg); 4043 if (index_is_valid) 4044 off += kvm_register_read(vcpu, index_reg)<<scaling; 4045 vmx_get_segment(vcpu, &s, seg_reg); 4046 4047 /* 4048 * The effective address, i.e. @off, of a memory operand is truncated 4049 * based on the address size of the instruction. Note that this is 4050 * the *effective address*, i.e. the address prior to accounting for 4051 * the segment's base. 4052 */ 4053 if (addr_size == 1) /* 32 bit */ 4054 off &= 0xffffffff; 4055 else if (addr_size == 0) /* 16 bit */ 4056 off &= 0xffff; 4057 4058 /* Checks for #GP/#SS exceptions. */ 4059 exn = false; 4060 if (is_long_mode(vcpu)) { 4061 /* 4062 * The virtual/linear address is never truncated in 64-bit 4063 * mode, e.g. a 32-bit address size can yield a 64-bit virtual 4064 * address when using FS/GS with a non-zero base. 4065 */ 4066 *ret = s.base + off; 4067 4068 /* Long mode: #GP(0)/#SS(0) if the memory address is in a 4069 * non-canonical form. This is the only check on the memory 4070 * destination for long mode! 4071 */ 4072 exn = is_noncanonical_address(*ret, vcpu); 4073 } else { 4074 /* 4075 * When not in long mode, the virtual/linear address is 4076 * unconditionally truncated to 32 bits regardless of the 4077 * address size. 4078 */ 4079 *ret = (s.base + off) & 0xffffffff; 4080 4081 /* Protected mode: apply checks for segment validity in the 4082 * following order: 4083 * - segment type check (#GP(0) may be thrown) 4084 * - usability check (#GP(0)/#SS(0)) 4085 * - limit check (#GP(0)/#SS(0)) 4086 */ 4087 if (wr) 4088 /* #GP(0) if the destination operand is located in a 4089 * read-only data segment or any code segment. 4090 */ 4091 exn = ((s.type & 0xa) == 0 || (s.type & 8)); 4092 else 4093 /* #GP(0) if the source operand is located in an 4094 * execute-only code segment 4095 */ 4096 exn = ((s.type & 0xa) == 8); 4097 if (exn) { 4098 kvm_queue_exception_e(vcpu, GP_VECTOR, 0); 4099 return 1; 4100 } 4101 /* Protected mode: #GP(0)/#SS(0) if the segment is unusable. 4102 */ 4103 exn = (s.unusable != 0); 4104 4105 /* 4106 * Protected mode: #GP(0)/#SS(0) if the memory operand is 4107 * outside the segment limit. All CPUs that support VMX ignore 4108 * limit checks for flat segments, i.e. segments with base==0, 4109 * limit==0xffffffff and of type expand-up data or code. 4110 */ 4111 if (!(s.base == 0 && s.limit == 0xffffffff && 4112 ((s.type & 8) || !(s.type & 4)))) 4113 exn = exn || (off + sizeof(u64) > s.limit); 4114 } 4115 if (exn) { 4116 kvm_queue_exception_e(vcpu, 4117 seg_reg == VCPU_SREG_SS ? 4118 SS_VECTOR : GP_VECTOR, 4119 0); 4120 return 1; 4121 } 4122 4123 return 0; 4124 } 4125 4126 static int nested_vmx_get_vmptr(struct kvm_vcpu *vcpu, gpa_t *vmpointer) 4127 { 4128 gva_t gva; 4129 struct x86_exception e; 4130 4131 if (get_vmx_mem_address(vcpu, vmcs_readl(EXIT_QUALIFICATION), 4132 vmcs_read32(VMX_INSTRUCTION_INFO), false, &gva)) 4133 return 1; 4134 4135 if (kvm_read_guest_virt(vcpu, gva, vmpointer, sizeof(*vmpointer), &e)) { 4136 kvm_inject_page_fault(vcpu, &e); 4137 return 1; 4138 } 4139 4140 return 0; 4141 } 4142 4143 /* 4144 * Allocate a shadow VMCS and associate it with the currently loaded 4145 * VMCS, unless such a shadow VMCS already exists. The newly allocated 4146 * VMCS is also VMCLEARed, so that it is ready for use. 4147 */ 4148 static struct vmcs *alloc_shadow_vmcs(struct kvm_vcpu *vcpu) 4149 { 4150 struct vcpu_vmx *vmx = to_vmx(vcpu); 4151 struct loaded_vmcs *loaded_vmcs = vmx->loaded_vmcs; 4152 4153 /* 4154 * We should allocate a shadow vmcs for vmcs01 only when L1 4155 * executes VMXON and free it when L1 executes VMXOFF. 4156 * As it is invalid to execute VMXON twice, we shouldn't reach 4157 * here when vmcs01 already have an allocated shadow vmcs. 4158 */ 4159 WARN_ON(loaded_vmcs == &vmx->vmcs01 && loaded_vmcs->shadow_vmcs); 4160 4161 if (!loaded_vmcs->shadow_vmcs) { 4162 loaded_vmcs->shadow_vmcs = alloc_vmcs(true); 4163 if (loaded_vmcs->shadow_vmcs) 4164 vmcs_clear(loaded_vmcs->shadow_vmcs); 4165 } 4166 return loaded_vmcs->shadow_vmcs; 4167 } 4168 4169 static int enter_vmx_operation(struct kvm_vcpu *vcpu) 4170 { 4171 struct vcpu_vmx *vmx = to_vmx(vcpu); 4172 int r; 4173 4174 r = alloc_loaded_vmcs(&vmx->nested.vmcs02); 4175 if (r < 0) 4176 goto out_vmcs02; 4177 4178 vmx->nested.cached_vmcs12 = kzalloc(VMCS12_SIZE, GFP_KERNEL_ACCOUNT); 4179 if (!vmx->nested.cached_vmcs12) 4180 goto out_cached_vmcs12; 4181 4182 vmx->nested.cached_shadow_vmcs12 = kzalloc(VMCS12_SIZE, GFP_KERNEL_ACCOUNT); 4183 if (!vmx->nested.cached_shadow_vmcs12) 4184 goto out_cached_shadow_vmcs12; 4185 4186 if (enable_shadow_vmcs && !alloc_shadow_vmcs(vcpu)) 4187 goto out_shadow_vmcs; 4188 4189 hrtimer_init(&vmx->nested.preemption_timer, CLOCK_MONOTONIC, 4190 HRTIMER_MODE_REL_PINNED); 4191 vmx->nested.preemption_timer.function = vmx_preemption_timer_fn; 4192 4193 vmx->nested.vpid02 = allocate_vpid(); 4194 4195 vmx->nested.vmcs02_initialized = false; 4196 vmx->nested.vmxon = true; 4197 4198 if (pt_mode == PT_MODE_HOST_GUEST) { 4199 vmx->pt_desc.guest.ctl = 0; 4200 pt_update_intercept_for_msr(vmx); 4201 } 4202 4203 return 0; 4204 4205 out_shadow_vmcs: 4206 kfree(vmx->nested.cached_shadow_vmcs12); 4207 4208 out_cached_shadow_vmcs12: 4209 kfree(vmx->nested.cached_vmcs12); 4210 4211 out_cached_vmcs12: 4212 free_loaded_vmcs(&vmx->nested.vmcs02); 4213 4214 out_vmcs02: 4215 return -ENOMEM; 4216 } 4217 4218 /* 4219 * Emulate the VMXON instruction. 4220 * Currently, we just remember that VMX is active, and do not save or even 4221 * inspect the argument to VMXON (the so-called "VMXON pointer") because we 4222 * do not currently need to store anything in that guest-allocated memory 4223 * region. Consequently, VMCLEAR and VMPTRLD also do not verify that the their 4224 * argument is different from the VMXON pointer (which the spec says they do). 4225 */ 4226 static int handle_vmon(struct kvm_vcpu *vcpu) 4227 { 4228 int ret; 4229 gpa_t vmptr; 4230 struct page *page; 4231 struct vcpu_vmx *vmx = to_vmx(vcpu); 4232 const u64 VMXON_NEEDED_FEATURES = FEATURE_CONTROL_LOCKED 4233 | FEATURE_CONTROL_VMXON_ENABLED_OUTSIDE_SMX; 4234 4235 /* 4236 * The Intel VMX Instruction Reference lists a bunch of bits that are 4237 * prerequisite to running VMXON, most notably cr4.VMXE must be set to 4238 * 1 (see vmx_set_cr4() for when we allow the guest to set this). 4239 * Otherwise, we should fail with #UD. But most faulting conditions 4240 * have already been checked by hardware, prior to the VM-exit for 4241 * VMXON. We do test guest cr4.VMXE because processor CR4 always has 4242 * that bit set to 1 in non-root mode. 4243 */ 4244 if (!kvm_read_cr4_bits(vcpu, X86_CR4_VMXE)) { 4245 kvm_queue_exception(vcpu, UD_VECTOR); 4246 return 1; 4247 } 4248 4249 /* CPL=0 must be checked manually. */ 4250 if (vmx_get_cpl(vcpu)) { 4251 kvm_inject_gp(vcpu, 0); 4252 return 1; 4253 } 4254 4255 if (vmx->nested.vmxon) 4256 return nested_vmx_failValid(vcpu, 4257 VMXERR_VMXON_IN_VMX_ROOT_OPERATION); 4258 4259 if ((vmx->msr_ia32_feature_control & VMXON_NEEDED_FEATURES) 4260 != VMXON_NEEDED_FEATURES) { 4261 kvm_inject_gp(vcpu, 0); 4262 return 1; 4263 } 4264 4265 if (nested_vmx_get_vmptr(vcpu, &vmptr)) 4266 return 1; 4267 4268 /* 4269 * SDM 3: 24.11.5 4270 * The first 4 bytes of VMXON region contain the supported 4271 * VMCS revision identifier 4272 * 4273 * Note - IA32_VMX_BASIC[48] will never be 1 for the nested case; 4274 * which replaces physical address width with 32 4275 */ 4276 if (!PAGE_ALIGNED(vmptr) || (vmptr >> cpuid_maxphyaddr(vcpu))) 4277 return nested_vmx_failInvalid(vcpu); 4278 4279 page = kvm_vcpu_gpa_to_page(vcpu, vmptr); 4280 if (is_error_page(page)) 4281 return nested_vmx_failInvalid(vcpu); 4282 4283 if (*(u32 *)kmap(page) != VMCS12_REVISION) { 4284 kunmap(page); 4285 kvm_release_page_clean(page); 4286 return nested_vmx_failInvalid(vcpu); 4287 } 4288 kunmap(page); 4289 kvm_release_page_clean(page); 4290 4291 vmx->nested.vmxon_ptr = vmptr; 4292 ret = enter_vmx_operation(vcpu); 4293 if (ret) 4294 return ret; 4295 4296 return nested_vmx_succeed(vcpu); 4297 } 4298 4299 static inline void nested_release_vmcs12(struct kvm_vcpu *vcpu) 4300 { 4301 struct vcpu_vmx *vmx = to_vmx(vcpu); 4302 4303 if (vmx->nested.current_vmptr == -1ull) 4304 return; 4305 4306 if (enable_shadow_vmcs) { 4307 /* copy to memory all shadowed fields in case 4308 they were modified */ 4309 copy_shadow_to_vmcs12(vmx); 4310 vmx->nested.need_vmcs12_sync = false; 4311 vmx_disable_shadow_vmcs(vmx); 4312 } 4313 vmx->nested.posted_intr_nv = -1; 4314 4315 /* Flush VMCS12 to guest memory */ 4316 kvm_vcpu_write_guest_page(vcpu, 4317 vmx->nested.current_vmptr >> PAGE_SHIFT, 4318 vmx->nested.cached_vmcs12, 0, VMCS12_SIZE); 4319 4320 kvm_mmu_free_roots(vcpu, &vcpu->arch.guest_mmu, KVM_MMU_ROOTS_ALL); 4321 4322 vmx->nested.current_vmptr = -1ull; 4323 } 4324 4325 /* Emulate the VMXOFF instruction */ 4326 static int handle_vmoff(struct kvm_vcpu *vcpu) 4327 { 4328 if (!nested_vmx_check_permission(vcpu)) 4329 return 1; 4330 free_nested(vcpu); 4331 return nested_vmx_succeed(vcpu); 4332 } 4333 4334 /* Emulate the VMCLEAR instruction */ 4335 static int handle_vmclear(struct kvm_vcpu *vcpu) 4336 { 4337 struct vcpu_vmx *vmx = to_vmx(vcpu); 4338 u32 zero = 0; 4339 gpa_t vmptr; 4340 4341 if (!nested_vmx_check_permission(vcpu)) 4342 return 1; 4343 4344 if (nested_vmx_get_vmptr(vcpu, &vmptr)) 4345 return 1; 4346 4347 if (!PAGE_ALIGNED(vmptr) || (vmptr >> cpuid_maxphyaddr(vcpu))) 4348 return nested_vmx_failValid(vcpu, 4349 VMXERR_VMCLEAR_INVALID_ADDRESS); 4350 4351 if (vmptr == vmx->nested.vmxon_ptr) 4352 return nested_vmx_failValid(vcpu, 4353 VMXERR_VMCLEAR_VMXON_POINTER); 4354 4355 if (vmx->nested.hv_evmcs_page) { 4356 if (vmptr == vmx->nested.hv_evmcs_vmptr) 4357 nested_release_evmcs(vcpu); 4358 } else { 4359 if (vmptr == vmx->nested.current_vmptr) 4360 nested_release_vmcs12(vcpu); 4361 4362 kvm_vcpu_write_guest(vcpu, 4363 vmptr + offsetof(struct vmcs12, 4364 launch_state), 4365 &zero, sizeof(zero)); 4366 } 4367 4368 return nested_vmx_succeed(vcpu); 4369 } 4370 4371 static int nested_vmx_run(struct kvm_vcpu *vcpu, bool launch); 4372 4373 /* Emulate the VMLAUNCH instruction */ 4374 static int handle_vmlaunch(struct kvm_vcpu *vcpu) 4375 { 4376 return nested_vmx_run(vcpu, true); 4377 } 4378 4379 /* Emulate the VMRESUME instruction */ 4380 static int handle_vmresume(struct kvm_vcpu *vcpu) 4381 { 4382 4383 return nested_vmx_run(vcpu, false); 4384 } 4385 4386 static int handle_vmread(struct kvm_vcpu *vcpu) 4387 { 4388 unsigned long field; 4389 u64 field_value; 4390 unsigned long exit_qualification = vmcs_readl(EXIT_QUALIFICATION); 4391 u32 vmx_instruction_info = vmcs_read32(VMX_INSTRUCTION_INFO); 4392 gva_t gva = 0; 4393 struct vmcs12 *vmcs12; 4394 4395 if (!nested_vmx_check_permission(vcpu)) 4396 return 1; 4397 4398 if (to_vmx(vcpu)->nested.current_vmptr == -1ull) 4399 return nested_vmx_failInvalid(vcpu); 4400 4401 if (!is_guest_mode(vcpu)) 4402 vmcs12 = get_vmcs12(vcpu); 4403 else { 4404 /* 4405 * When vmcs->vmcs_link_pointer is -1ull, any VMREAD 4406 * to shadowed-field sets the ALU flags for VMfailInvalid. 4407 */ 4408 if (get_vmcs12(vcpu)->vmcs_link_pointer == -1ull) 4409 return nested_vmx_failInvalid(vcpu); 4410 vmcs12 = get_shadow_vmcs12(vcpu); 4411 } 4412 4413 /* Decode instruction info and find the field to read */ 4414 field = kvm_register_readl(vcpu, (((vmx_instruction_info) >> 28) & 0xf)); 4415 /* Read the field, zero-extended to a u64 field_value */ 4416 if (vmcs12_read_any(vmcs12, field, &field_value) < 0) 4417 return nested_vmx_failValid(vcpu, 4418 VMXERR_UNSUPPORTED_VMCS_COMPONENT); 4419 4420 /* 4421 * Now copy part of this value to register or memory, as requested. 4422 * Note that the number of bits actually copied is 32 or 64 depending 4423 * on the guest's mode (32 or 64 bit), not on the given field's length. 4424 */ 4425 if (vmx_instruction_info & (1u << 10)) { 4426 kvm_register_writel(vcpu, (((vmx_instruction_info) >> 3) & 0xf), 4427 field_value); 4428 } else { 4429 if (get_vmx_mem_address(vcpu, exit_qualification, 4430 vmx_instruction_info, true, &gva)) 4431 return 1; 4432 /* _system ok, nested_vmx_check_permission has verified cpl=0 */ 4433 kvm_write_guest_virt_system(vcpu, gva, &field_value, 4434 (is_long_mode(vcpu) ? 8 : 4), NULL); 4435 } 4436 4437 return nested_vmx_succeed(vcpu); 4438 } 4439 4440 4441 static int handle_vmwrite(struct kvm_vcpu *vcpu) 4442 { 4443 unsigned long field; 4444 gva_t gva; 4445 struct vcpu_vmx *vmx = to_vmx(vcpu); 4446 unsigned long exit_qualification = vmcs_readl(EXIT_QUALIFICATION); 4447 u32 vmx_instruction_info = vmcs_read32(VMX_INSTRUCTION_INFO); 4448 4449 /* The value to write might be 32 or 64 bits, depending on L1's long 4450 * mode, and eventually we need to write that into a field of several 4451 * possible lengths. The code below first zero-extends the value to 64 4452 * bit (field_value), and then copies only the appropriate number of 4453 * bits into the vmcs12 field. 4454 */ 4455 u64 field_value = 0; 4456 struct x86_exception e; 4457 struct vmcs12 *vmcs12; 4458 4459 if (!nested_vmx_check_permission(vcpu)) 4460 return 1; 4461 4462 if (vmx->nested.current_vmptr == -1ull) 4463 return nested_vmx_failInvalid(vcpu); 4464 4465 if (vmx_instruction_info & (1u << 10)) 4466 field_value = kvm_register_readl(vcpu, 4467 (((vmx_instruction_info) >> 3) & 0xf)); 4468 else { 4469 if (get_vmx_mem_address(vcpu, exit_qualification, 4470 vmx_instruction_info, false, &gva)) 4471 return 1; 4472 if (kvm_read_guest_virt(vcpu, gva, &field_value, 4473 (is_64_bit_mode(vcpu) ? 8 : 4), &e)) { 4474 kvm_inject_page_fault(vcpu, &e); 4475 return 1; 4476 } 4477 } 4478 4479 4480 field = kvm_register_readl(vcpu, (((vmx_instruction_info) >> 28) & 0xf)); 4481 /* 4482 * If the vCPU supports "VMWRITE to any supported field in the 4483 * VMCS," then the "read-only" fields are actually read/write. 4484 */ 4485 if (vmcs_field_readonly(field) && 4486 !nested_cpu_has_vmwrite_any_field(vcpu)) 4487 return nested_vmx_failValid(vcpu, 4488 VMXERR_VMWRITE_READ_ONLY_VMCS_COMPONENT); 4489 4490 if (!is_guest_mode(vcpu)) 4491 vmcs12 = get_vmcs12(vcpu); 4492 else { 4493 /* 4494 * When vmcs->vmcs_link_pointer is -1ull, any VMWRITE 4495 * to shadowed-field sets the ALU flags for VMfailInvalid. 4496 */ 4497 if (get_vmcs12(vcpu)->vmcs_link_pointer == -1ull) 4498 return nested_vmx_failInvalid(vcpu); 4499 vmcs12 = get_shadow_vmcs12(vcpu); 4500 } 4501 4502 if (vmcs12_write_any(vmcs12, field, field_value) < 0) 4503 return nested_vmx_failValid(vcpu, 4504 VMXERR_UNSUPPORTED_VMCS_COMPONENT); 4505 4506 /* 4507 * Do not track vmcs12 dirty-state if in guest-mode 4508 * as we actually dirty shadow vmcs12 instead of vmcs12. 4509 */ 4510 if (!is_guest_mode(vcpu)) { 4511 switch (field) { 4512 #define SHADOW_FIELD_RW(x) case x: 4513 #include "vmcs_shadow_fields.h" 4514 /* 4515 * The fields that can be updated by L1 without a vmexit are 4516 * always updated in the vmcs02, the others go down the slow 4517 * path of prepare_vmcs02. 4518 */ 4519 break; 4520 default: 4521 vmx->nested.dirty_vmcs12 = true; 4522 break; 4523 } 4524 } 4525 4526 return nested_vmx_succeed(vcpu); 4527 } 4528 4529 static void set_current_vmptr(struct vcpu_vmx *vmx, gpa_t vmptr) 4530 { 4531 vmx->nested.current_vmptr = vmptr; 4532 if (enable_shadow_vmcs) { 4533 vmcs_set_bits(SECONDARY_VM_EXEC_CONTROL, 4534 SECONDARY_EXEC_SHADOW_VMCS); 4535 vmcs_write64(VMCS_LINK_POINTER, 4536 __pa(vmx->vmcs01.shadow_vmcs)); 4537 vmx->nested.need_vmcs12_sync = true; 4538 } 4539 vmx->nested.dirty_vmcs12 = true; 4540 } 4541 4542 /* Emulate the VMPTRLD instruction */ 4543 static int handle_vmptrld(struct kvm_vcpu *vcpu) 4544 { 4545 struct vcpu_vmx *vmx = to_vmx(vcpu); 4546 gpa_t vmptr; 4547 4548 if (!nested_vmx_check_permission(vcpu)) 4549 return 1; 4550 4551 if (nested_vmx_get_vmptr(vcpu, &vmptr)) 4552 return 1; 4553 4554 if (!PAGE_ALIGNED(vmptr) || (vmptr >> cpuid_maxphyaddr(vcpu))) 4555 return nested_vmx_failValid(vcpu, 4556 VMXERR_VMPTRLD_INVALID_ADDRESS); 4557 4558 if (vmptr == vmx->nested.vmxon_ptr) 4559 return nested_vmx_failValid(vcpu, 4560 VMXERR_VMPTRLD_VMXON_POINTER); 4561 4562 /* Forbid normal VMPTRLD if Enlightened version was used */ 4563 if (vmx->nested.hv_evmcs) 4564 return 1; 4565 4566 if (vmx->nested.current_vmptr != vmptr) { 4567 struct vmcs12 *new_vmcs12; 4568 struct page *page; 4569 4570 page = kvm_vcpu_gpa_to_page(vcpu, vmptr); 4571 if (is_error_page(page)) { 4572 /* 4573 * Reads from an unbacked page return all 1s, 4574 * which means that the 32 bits located at the 4575 * given physical address won't match the required 4576 * VMCS12_REVISION identifier. 4577 */ 4578 return nested_vmx_failValid(vcpu, 4579 VMXERR_VMPTRLD_INCORRECT_VMCS_REVISION_ID); 4580 } 4581 new_vmcs12 = kmap(page); 4582 if (new_vmcs12->hdr.revision_id != VMCS12_REVISION || 4583 (new_vmcs12->hdr.shadow_vmcs && 4584 !nested_cpu_has_vmx_shadow_vmcs(vcpu))) { 4585 kunmap(page); 4586 kvm_release_page_clean(page); 4587 return nested_vmx_failValid(vcpu, 4588 VMXERR_VMPTRLD_INCORRECT_VMCS_REVISION_ID); 4589 } 4590 4591 nested_release_vmcs12(vcpu); 4592 4593 /* 4594 * Load VMCS12 from guest memory since it is not already 4595 * cached. 4596 */ 4597 memcpy(vmx->nested.cached_vmcs12, new_vmcs12, VMCS12_SIZE); 4598 kunmap(page); 4599 kvm_release_page_clean(page); 4600 4601 set_current_vmptr(vmx, vmptr); 4602 } 4603 4604 return nested_vmx_succeed(vcpu); 4605 } 4606 4607 /* Emulate the VMPTRST instruction */ 4608 static int handle_vmptrst(struct kvm_vcpu *vcpu) 4609 { 4610 unsigned long exit_qual = vmcs_readl(EXIT_QUALIFICATION); 4611 u32 instr_info = vmcs_read32(VMX_INSTRUCTION_INFO); 4612 gpa_t current_vmptr = to_vmx(vcpu)->nested.current_vmptr; 4613 struct x86_exception e; 4614 gva_t gva; 4615 4616 if (!nested_vmx_check_permission(vcpu)) 4617 return 1; 4618 4619 if (unlikely(to_vmx(vcpu)->nested.hv_evmcs)) 4620 return 1; 4621 4622 if (get_vmx_mem_address(vcpu, exit_qual, instr_info, true, &gva)) 4623 return 1; 4624 /* *_system ok, nested_vmx_check_permission has verified cpl=0 */ 4625 if (kvm_write_guest_virt_system(vcpu, gva, (void *)¤t_vmptr, 4626 sizeof(gpa_t), &e)) { 4627 kvm_inject_page_fault(vcpu, &e); 4628 return 1; 4629 } 4630 return nested_vmx_succeed(vcpu); 4631 } 4632 4633 /* Emulate the INVEPT instruction */ 4634 static int handle_invept(struct kvm_vcpu *vcpu) 4635 { 4636 struct vcpu_vmx *vmx = to_vmx(vcpu); 4637 u32 vmx_instruction_info, types; 4638 unsigned long type; 4639 gva_t gva; 4640 struct x86_exception e; 4641 struct { 4642 u64 eptp, gpa; 4643 } operand; 4644 4645 if (!(vmx->nested.msrs.secondary_ctls_high & 4646 SECONDARY_EXEC_ENABLE_EPT) || 4647 !(vmx->nested.msrs.ept_caps & VMX_EPT_INVEPT_BIT)) { 4648 kvm_queue_exception(vcpu, UD_VECTOR); 4649 return 1; 4650 } 4651 4652 if (!nested_vmx_check_permission(vcpu)) 4653 return 1; 4654 4655 vmx_instruction_info = vmcs_read32(VMX_INSTRUCTION_INFO); 4656 type = kvm_register_readl(vcpu, (vmx_instruction_info >> 28) & 0xf); 4657 4658 types = (vmx->nested.msrs.ept_caps >> VMX_EPT_EXTENT_SHIFT) & 6; 4659 4660 if (type >= 32 || !(types & (1 << type))) 4661 return nested_vmx_failValid(vcpu, 4662 VMXERR_INVALID_OPERAND_TO_INVEPT_INVVPID); 4663 4664 /* According to the Intel VMX instruction reference, the memory 4665 * operand is read even if it isn't needed (e.g., for type==global) 4666 */ 4667 if (get_vmx_mem_address(vcpu, vmcs_readl(EXIT_QUALIFICATION), 4668 vmx_instruction_info, false, &gva)) 4669 return 1; 4670 if (kvm_read_guest_virt(vcpu, gva, &operand, sizeof(operand), &e)) { 4671 kvm_inject_page_fault(vcpu, &e); 4672 return 1; 4673 } 4674 4675 switch (type) { 4676 case VMX_EPT_EXTENT_GLOBAL: 4677 /* 4678 * TODO: track mappings and invalidate 4679 * single context requests appropriately 4680 */ 4681 case VMX_EPT_EXTENT_CONTEXT: 4682 kvm_mmu_sync_roots(vcpu); 4683 kvm_make_request(KVM_REQ_TLB_FLUSH, vcpu); 4684 break; 4685 default: 4686 BUG_ON(1); 4687 break; 4688 } 4689 4690 return nested_vmx_succeed(vcpu); 4691 } 4692 4693 static int handle_invvpid(struct kvm_vcpu *vcpu) 4694 { 4695 struct vcpu_vmx *vmx = to_vmx(vcpu); 4696 u32 vmx_instruction_info; 4697 unsigned long type, types; 4698 gva_t gva; 4699 struct x86_exception e; 4700 struct { 4701 u64 vpid; 4702 u64 gla; 4703 } operand; 4704 u16 vpid02; 4705 4706 if (!(vmx->nested.msrs.secondary_ctls_high & 4707 SECONDARY_EXEC_ENABLE_VPID) || 4708 !(vmx->nested.msrs.vpid_caps & VMX_VPID_INVVPID_BIT)) { 4709 kvm_queue_exception(vcpu, UD_VECTOR); 4710 return 1; 4711 } 4712 4713 if (!nested_vmx_check_permission(vcpu)) 4714 return 1; 4715 4716 vmx_instruction_info = vmcs_read32(VMX_INSTRUCTION_INFO); 4717 type = kvm_register_readl(vcpu, (vmx_instruction_info >> 28) & 0xf); 4718 4719 types = (vmx->nested.msrs.vpid_caps & 4720 VMX_VPID_EXTENT_SUPPORTED_MASK) >> 8; 4721 4722 if (type >= 32 || !(types & (1 << type))) 4723 return nested_vmx_failValid(vcpu, 4724 VMXERR_INVALID_OPERAND_TO_INVEPT_INVVPID); 4725 4726 /* according to the intel vmx instruction reference, the memory 4727 * operand is read even if it isn't needed (e.g., for type==global) 4728 */ 4729 if (get_vmx_mem_address(vcpu, vmcs_readl(EXIT_QUALIFICATION), 4730 vmx_instruction_info, false, &gva)) 4731 return 1; 4732 if (kvm_read_guest_virt(vcpu, gva, &operand, sizeof(operand), &e)) { 4733 kvm_inject_page_fault(vcpu, &e); 4734 return 1; 4735 } 4736 if (operand.vpid >> 16) 4737 return nested_vmx_failValid(vcpu, 4738 VMXERR_INVALID_OPERAND_TO_INVEPT_INVVPID); 4739 4740 vpid02 = nested_get_vpid02(vcpu); 4741 switch (type) { 4742 case VMX_VPID_EXTENT_INDIVIDUAL_ADDR: 4743 if (!operand.vpid || 4744 is_noncanonical_address(operand.gla, vcpu)) 4745 return nested_vmx_failValid(vcpu, 4746 VMXERR_INVALID_OPERAND_TO_INVEPT_INVVPID); 4747 if (cpu_has_vmx_invvpid_individual_addr()) { 4748 __invvpid(VMX_VPID_EXTENT_INDIVIDUAL_ADDR, 4749 vpid02, operand.gla); 4750 } else 4751 __vmx_flush_tlb(vcpu, vpid02, false); 4752 break; 4753 case VMX_VPID_EXTENT_SINGLE_CONTEXT: 4754 case VMX_VPID_EXTENT_SINGLE_NON_GLOBAL: 4755 if (!operand.vpid) 4756 return nested_vmx_failValid(vcpu, 4757 VMXERR_INVALID_OPERAND_TO_INVEPT_INVVPID); 4758 __vmx_flush_tlb(vcpu, vpid02, false); 4759 break; 4760 case VMX_VPID_EXTENT_ALL_CONTEXT: 4761 __vmx_flush_tlb(vcpu, vpid02, false); 4762 break; 4763 default: 4764 WARN_ON_ONCE(1); 4765 return kvm_skip_emulated_instruction(vcpu); 4766 } 4767 4768 return nested_vmx_succeed(vcpu); 4769 } 4770 4771 static int nested_vmx_eptp_switching(struct kvm_vcpu *vcpu, 4772 struct vmcs12 *vmcs12) 4773 { 4774 u32 index = vcpu->arch.regs[VCPU_REGS_RCX]; 4775 u64 address; 4776 bool accessed_dirty; 4777 struct kvm_mmu *mmu = vcpu->arch.walk_mmu; 4778 4779 if (!nested_cpu_has_eptp_switching(vmcs12) || 4780 !nested_cpu_has_ept(vmcs12)) 4781 return 1; 4782 4783 if (index >= VMFUNC_EPTP_ENTRIES) 4784 return 1; 4785 4786 4787 if (kvm_vcpu_read_guest_page(vcpu, vmcs12->eptp_list_address >> PAGE_SHIFT, 4788 &address, index * 8, 8)) 4789 return 1; 4790 4791 accessed_dirty = !!(address & VMX_EPTP_AD_ENABLE_BIT); 4792 4793 /* 4794 * If the (L2) guest does a vmfunc to the currently 4795 * active ept pointer, we don't have to do anything else 4796 */ 4797 if (vmcs12->ept_pointer != address) { 4798 if (!valid_ept_address(vcpu, address)) 4799 return 1; 4800 4801 kvm_mmu_unload(vcpu); 4802 mmu->ept_ad = accessed_dirty; 4803 mmu->mmu_role.base.ad_disabled = !accessed_dirty; 4804 vmcs12->ept_pointer = address; 4805 /* 4806 * TODO: Check what's the correct approach in case 4807 * mmu reload fails. Currently, we just let the next 4808 * reload potentially fail 4809 */ 4810 kvm_mmu_reload(vcpu); 4811 } 4812 4813 return 0; 4814 } 4815 4816 static int handle_vmfunc(struct kvm_vcpu *vcpu) 4817 { 4818 struct vcpu_vmx *vmx = to_vmx(vcpu); 4819 struct vmcs12 *vmcs12; 4820 u32 function = vcpu->arch.regs[VCPU_REGS_RAX]; 4821 4822 /* 4823 * VMFUNC is only supported for nested guests, but we always enable the 4824 * secondary control for simplicity; for non-nested mode, fake that we 4825 * didn't by injecting #UD. 4826 */ 4827 if (!is_guest_mode(vcpu)) { 4828 kvm_queue_exception(vcpu, UD_VECTOR); 4829 return 1; 4830 } 4831 4832 vmcs12 = get_vmcs12(vcpu); 4833 if ((vmcs12->vm_function_control & (1 << function)) == 0) 4834 goto fail; 4835 4836 switch (function) { 4837 case 0: 4838 if (nested_vmx_eptp_switching(vcpu, vmcs12)) 4839 goto fail; 4840 break; 4841 default: 4842 goto fail; 4843 } 4844 return kvm_skip_emulated_instruction(vcpu); 4845 4846 fail: 4847 nested_vmx_vmexit(vcpu, vmx->exit_reason, 4848 vmcs_read32(VM_EXIT_INTR_INFO), 4849 vmcs_readl(EXIT_QUALIFICATION)); 4850 return 1; 4851 } 4852 4853 4854 static bool nested_vmx_exit_handled_io(struct kvm_vcpu *vcpu, 4855 struct vmcs12 *vmcs12) 4856 { 4857 unsigned long exit_qualification; 4858 gpa_t bitmap, last_bitmap; 4859 unsigned int port; 4860 int size; 4861 u8 b; 4862 4863 if (!nested_cpu_has(vmcs12, CPU_BASED_USE_IO_BITMAPS)) 4864 return nested_cpu_has(vmcs12, CPU_BASED_UNCOND_IO_EXITING); 4865 4866 exit_qualification = vmcs_readl(EXIT_QUALIFICATION); 4867 4868 port = exit_qualification >> 16; 4869 size = (exit_qualification & 7) + 1; 4870 4871 last_bitmap = (gpa_t)-1; 4872 b = -1; 4873 4874 while (size > 0) { 4875 if (port < 0x8000) 4876 bitmap = vmcs12->io_bitmap_a; 4877 else if (port < 0x10000) 4878 bitmap = vmcs12->io_bitmap_b; 4879 else 4880 return true; 4881 bitmap += (port & 0x7fff) / 8; 4882 4883 if (last_bitmap != bitmap) 4884 if (kvm_vcpu_read_guest(vcpu, bitmap, &b, 1)) 4885 return true; 4886 if (b & (1 << (port & 7))) 4887 return true; 4888 4889 port++; 4890 size--; 4891 last_bitmap = bitmap; 4892 } 4893 4894 return false; 4895 } 4896 4897 /* 4898 * Return 1 if we should exit from L2 to L1 to handle an MSR access access, 4899 * rather than handle it ourselves in L0. I.e., check whether L1 expressed 4900 * disinterest in the current event (read or write a specific MSR) by using an 4901 * MSR bitmap. This may be the case even when L0 doesn't use MSR bitmaps. 4902 */ 4903 static bool nested_vmx_exit_handled_msr(struct kvm_vcpu *vcpu, 4904 struct vmcs12 *vmcs12, u32 exit_reason) 4905 { 4906 u32 msr_index = vcpu->arch.regs[VCPU_REGS_RCX]; 4907 gpa_t bitmap; 4908 4909 if (!nested_cpu_has(vmcs12, CPU_BASED_USE_MSR_BITMAPS)) 4910 return true; 4911 4912 /* 4913 * The MSR_BITMAP page is divided into four 1024-byte bitmaps, 4914 * for the four combinations of read/write and low/high MSR numbers. 4915 * First we need to figure out which of the four to use: 4916 */ 4917 bitmap = vmcs12->msr_bitmap; 4918 if (exit_reason == EXIT_REASON_MSR_WRITE) 4919 bitmap += 2048; 4920 if (msr_index >= 0xc0000000) { 4921 msr_index -= 0xc0000000; 4922 bitmap += 1024; 4923 } 4924 4925 /* Then read the msr_index'th bit from this bitmap: */ 4926 if (msr_index < 1024*8) { 4927 unsigned char b; 4928 if (kvm_vcpu_read_guest(vcpu, bitmap + msr_index/8, &b, 1)) 4929 return true; 4930 return 1 & (b >> (msr_index & 7)); 4931 } else 4932 return true; /* let L1 handle the wrong parameter */ 4933 } 4934 4935 /* 4936 * Return 1 if we should exit from L2 to L1 to handle a CR access exit, 4937 * rather than handle it ourselves in L0. I.e., check if L1 wanted to 4938 * intercept (via guest_host_mask etc.) the current event. 4939 */ 4940 static bool nested_vmx_exit_handled_cr(struct kvm_vcpu *vcpu, 4941 struct vmcs12 *vmcs12) 4942 { 4943 unsigned long exit_qualification = vmcs_readl(EXIT_QUALIFICATION); 4944 int cr = exit_qualification & 15; 4945 int reg; 4946 unsigned long val; 4947 4948 switch ((exit_qualification >> 4) & 3) { 4949 case 0: /* mov to cr */ 4950 reg = (exit_qualification >> 8) & 15; 4951 val = kvm_register_readl(vcpu, reg); 4952 switch (cr) { 4953 case 0: 4954 if (vmcs12->cr0_guest_host_mask & 4955 (val ^ vmcs12->cr0_read_shadow)) 4956 return true; 4957 break; 4958 case 3: 4959 if ((vmcs12->cr3_target_count >= 1 && 4960 vmcs12->cr3_target_value0 == val) || 4961 (vmcs12->cr3_target_count >= 2 && 4962 vmcs12->cr3_target_value1 == val) || 4963 (vmcs12->cr3_target_count >= 3 && 4964 vmcs12->cr3_target_value2 == val) || 4965 (vmcs12->cr3_target_count >= 4 && 4966 vmcs12->cr3_target_value3 == val)) 4967 return false; 4968 if (nested_cpu_has(vmcs12, CPU_BASED_CR3_LOAD_EXITING)) 4969 return true; 4970 break; 4971 case 4: 4972 if (vmcs12->cr4_guest_host_mask & 4973 (vmcs12->cr4_read_shadow ^ val)) 4974 return true; 4975 break; 4976 case 8: 4977 if (nested_cpu_has(vmcs12, CPU_BASED_CR8_LOAD_EXITING)) 4978 return true; 4979 break; 4980 } 4981 break; 4982 case 2: /* clts */ 4983 if ((vmcs12->cr0_guest_host_mask & X86_CR0_TS) && 4984 (vmcs12->cr0_read_shadow & X86_CR0_TS)) 4985 return true; 4986 break; 4987 case 1: /* mov from cr */ 4988 switch (cr) { 4989 case 3: 4990 if (vmcs12->cpu_based_vm_exec_control & 4991 CPU_BASED_CR3_STORE_EXITING) 4992 return true; 4993 break; 4994 case 8: 4995 if (vmcs12->cpu_based_vm_exec_control & 4996 CPU_BASED_CR8_STORE_EXITING) 4997 return true; 4998 break; 4999 } 5000 break; 5001 case 3: /* lmsw */ 5002 /* 5003 * lmsw can change bits 1..3 of cr0, and only set bit 0 of 5004 * cr0. Other attempted changes are ignored, with no exit. 5005 */ 5006 val = (exit_qualification >> LMSW_SOURCE_DATA_SHIFT) & 0x0f; 5007 if (vmcs12->cr0_guest_host_mask & 0xe & 5008 (val ^ vmcs12->cr0_read_shadow)) 5009 return true; 5010 if ((vmcs12->cr0_guest_host_mask & 0x1) && 5011 !(vmcs12->cr0_read_shadow & 0x1) && 5012 (val & 0x1)) 5013 return true; 5014 break; 5015 } 5016 return false; 5017 } 5018 5019 static bool nested_vmx_exit_handled_vmcs_access(struct kvm_vcpu *vcpu, 5020 struct vmcs12 *vmcs12, gpa_t bitmap) 5021 { 5022 u32 vmx_instruction_info; 5023 unsigned long field; 5024 u8 b; 5025 5026 if (!nested_cpu_has_shadow_vmcs(vmcs12)) 5027 return true; 5028 5029 /* Decode instruction info and find the field to access */ 5030 vmx_instruction_info = vmcs_read32(VMX_INSTRUCTION_INFO); 5031 field = kvm_register_read(vcpu, (((vmx_instruction_info) >> 28) & 0xf)); 5032 5033 /* Out-of-range fields always cause a VM exit from L2 to L1 */ 5034 if (field >> 15) 5035 return true; 5036 5037 if (kvm_vcpu_read_guest(vcpu, bitmap + field/8, &b, 1)) 5038 return true; 5039 5040 return 1 & (b >> (field & 7)); 5041 } 5042 5043 /* 5044 * Return 1 if we should exit from L2 to L1 to handle an exit, or 0 if we 5045 * should handle it ourselves in L0 (and then continue L2). Only call this 5046 * when in is_guest_mode (L2). 5047 */ 5048 bool nested_vmx_exit_reflected(struct kvm_vcpu *vcpu, u32 exit_reason) 5049 { 5050 u32 intr_info = vmcs_read32(VM_EXIT_INTR_INFO); 5051 struct vcpu_vmx *vmx = to_vmx(vcpu); 5052 struct vmcs12 *vmcs12 = get_vmcs12(vcpu); 5053 5054 if (vmx->nested.nested_run_pending) 5055 return false; 5056 5057 if (unlikely(vmx->fail)) { 5058 pr_info_ratelimited("%s failed vm entry %x\n", __func__, 5059 vmcs_read32(VM_INSTRUCTION_ERROR)); 5060 return true; 5061 } 5062 5063 /* 5064 * The host physical addresses of some pages of guest memory 5065 * are loaded into the vmcs02 (e.g. vmcs12's Virtual APIC 5066 * Page). The CPU may write to these pages via their host 5067 * physical address while L2 is running, bypassing any 5068 * address-translation-based dirty tracking (e.g. EPT write 5069 * protection). 5070 * 5071 * Mark them dirty on every exit from L2 to prevent them from 5072 * getting out of sync with dirty tracking. 5073 */ 5074 nested_mark_vmcs12_pages_dirty(vcpu); 5075 5076 trace_kvm_nested_vmexit(kvm_rip_read(vcpu), exit_reason, 5077 vmcs_readl(EXIT_QUALIFICATION), 5078 vmx->idt_vectoring_info, 5079 intr_info, 5080 vmcs_read32(VM_EXIT_INTR_ERROR_CODE), 5081 KVM_ISA_VMX); 5082 5083 switch (exit_reason) { 5084 case EXIT_REASON_EXCEPTION_NMI: 5085 if (is_nmi(intr_info)) 5086 return false; 5087 else if (is_page_fault(intr_info)) 5088 return !vmx->vcpu.arch.apf.host_apf_reason && enable_ept; 5089 else if (is_debug(intr_info) && 5090 vcpu->guest_debug & 5091 (KVM_GUESTDBG_SINGLESTEP | KVM_GUESTDBG_USE_HW_BP)) 5092 return false; 5093 else if (is_breakpoint(intr_info) && 5094 vcpu->guest_debug & KVM_GUESTDBG_USE_SW_BP) 5095 return false; 5096 return vmcs12->exception_bitmap & 5097 (1u << (intr_info & INTR_INFO_VECTOR_MASK)); 5098 case EXIT_REASON_EXTERNAL_INTERRUPT: 5099 return false; 5100 case EXIT_REASON_TRIPLE_FAULT: 5101 return true; 5102 case EXIT_REASON_PENDING_INTERRUPT: 5103 return nested_cpu_has(vmcs12, CPU_BASED_VIRTUAL_INTR_PENDING); 5104 case EXIT_REASON_NMI_WINDOW: 5105 return nested_cpu_has(vmcs12, CPU_BASED_VIRTUAL_NMI_PENDING); 5106 case EXIT_REASON_TASK_SWITCH: 5107 return true; 5108 case EXIT_REASON_CPUID: 5109 return true; 5110 case EXIT_REASON_HLT: 5111 return nested_cpu_has(vmcs12, CPU_BASED_HLT_EXITING); 5112 case EXIT_REASON_INVD: 5113 return true; 5114 case EXIT_REASON_INVLPG: 5115 return nested_cpu_has(vmcs12, CPU_BASED_INVLPG_EXITING); 5116 case EXIT_REASON_RDPMC: 5117 return nested_cpu_has(vmcs12, CPU_BASED_RDPMC_EXITING); 5118 case EXIT_REASON_RDRAND: 5119 return nested_cpu_has2(vmcs12, SECONDARY_EXEC_RDRAND_EXITING); 5120 case EXIT_REASON_RDSEED: 5121 return nested_cpu_has2(vmcs12, SECONDARY_EXEC_RDSEED_EXITING); 5122 case EXIT_REASON_RDTSC: case EXIT_REASON_RDTSCP: 5123 return nested_cpu_has(vmcs12, CPU_BASED_RDTSC_EXITING); 5124 case EXIT_REASON_VMREAD: 5125 return nested_vmx_exit_handled_vmcs_access(vcpu, vmcs12, 5126 vmcs12->vmread_bitmap); 5127 case EXIT_REASON_VMWRITE: 5128 return nested_vmx_exit_handled_vmcs_access(vcpu, vmcs12, 5129 vmcs12->vmwrite_bitmap); 5130 case EXIT_REASON_VMCALL: case EXIT_REASON_VMCLEAR: 5131 case EXIT_REASON_VMLAUNCH: case EXIT_REASON_VMPTRLD: 5132 case EXIT_REASON_VMPTRST: case EXIT_REASON_VMRESUME: 5133 case EXIT_REASON_VMOFF: case EXIT_REASON_VMON: 5134 case EXIT_REASON_INVEPT: case EXIT_REASON_INVVPID: 5135 /* 5136 * VMX instructions trap unconditionally. This allows L1 to 5137 * emulate them for its L2 guest, i.e., allows 3-level nesting! 5138 */ 5139 return true; 5140 case EXIT_REASON_CR_ACCESS: 5141 return nested_vmx_exit_handled_cr(vcpu, vmcs12); 5142 case EXIT_REASON_DR_ACCESS: 5143 return nested_cpu_has(vmcs12, CPU_BASED_MOV_DR_EXITING); 5144 case EXIT_REASON_IO_INSTRUCTION: 5145 return nested_vmx_exit_handled_io(vcpu, vmcs12); 5146 case EXIT_REASON_GDTR_IDTR: case EXIT_REASON_LDTR_TR: 5147 return nested_cpu_has2(vmcs12, SECONDARY_EXEC_DESC); 5148 case EXIT_REASON_MSR_READ: 5149 case EXIT_REASON_MSR_WRITE: 5150 return nested_vmx_exit_handled_msr(vcpu, vmcs12, exit_reason); 5151 case EXIT_REASON_INVALID_STATE: 5152 return true; 5153 case EXIT_REASON_MWAIT_INSTRUCTION: 5154 return nested_cpu_has(vmcs12, CPU_BASED_MWAIT_EXITING); 5155 case EXIT_REASON_MONITOR_TRAP_FLAG: 5156 return nested_cpu_has(vmcs12, CPU_BASED_MONITOR_TRAP_FLAG); 5157 case EXIT_REASON_MONITOR_INSTRUCTION: 5158 return nested_cpu_has(vmcs12, CPU_BASED_MONITOR_EXITING); 5159 case EXIT_REASON_PAUSE_INSTRUCTION: 5160 return nested_cpu_has(vmcs12, CPU_BASED_PAUSE_EXITING) || 5161 nested_cpu_has2(vmcs12, 5162 SECONDARY_EXEC_PAUSE_LOOP_EXITING); 5163 case EXIT_REASON_MCE_DURING_VMENTRY: 5164 return false; 5165 case EXIT_REASON_TPR_BELOW_THRESHOLD: 5166 return nested_cpu_has(vmcs12, CPU_BASED_TPR_SHADOW); 5167 case EXIT_REASON_APIC_ACCESS: 5168 case EXIT_REASON_APIC_WRITE: 5169 case EXIT_REASON_EOI_INDUCED: 5170 /* 5171 * The controls for "virtualize APIC accesses," "APIC- 5172 * register virtualization," and "virtual-interrupt 5173 * delivery" only come from vmcs12. 5174 */ 5175 return true; 5176 case EXIT_REASON_EPT_VIOLATION: 5177 /* 5178 * L0 always deals with the EPT violation. If nested EPT is 5179 * used, and the nested mmu code discovers that the address is 5180 * missing in the guest EPT table (EPT12), the EPT violation 5181 * will be injected with nested_ept_inject_page_fault() 5182 */ 5183 return false; 5184 case EXIT_REASON_EPT_MISCONFIG: 5185 /* 5186 * L2 never uses directly L1's EPT, but rather L0's own EPT 5187 * table (shadow on EPT) or a merged EPT table that L0 built 5188 * (EPT on EPT). So any problems with the structure of the 5189 * table is L0's fault. 5190 */ 5191 return false; 5192 case EXIT_REASON_INVPCID: 5193 return 5194 nested_cpu_has2(vmcs12, SECONDARY_EXEC_ENABLE_INVPCID) && 5195 nested_cpu_has(vmcs12, CPU_BASED_INVLPG_EXITING); 5196 case EXIT_REASON_WBINVD: 5197 return nested_cpu_has2(vmcs12, SECONDARY_EXEC_WBINVD_EXITING); 5198 case EXIT_REASON_XSETBV: 5199 return true; 5200 case EXIT_REASON_XSAVES: case EXIT_REASON_XRSTORS: 5201 /* 5202 * This should never happen, since it is not possible to 5203 * set XSS to a non-zero value---neither in L1 nor in L2. 5204 * If if it were, XSS would have to be checked against 5205 * the XSS exit bitmap in vmcs12. 5206 */ 5207 return nested_cpu_has2(vmcs12, SECONDARY_EXEC_XSAVES); 5208 case EXIT_REASON_PREEMPTION_TIMER: 5209 return false; 5210 case EXIT_REASON_PML_FULL: 5211 /* We emulate PML support to L1. */ 5212 return false; 5213 case EXIT_REASON_VMFUNC: 5214 /* VM functions are emulated through L2->L0 vmexits. */ 5215 return false; 5216 case EXIT_REASON_ENCLS: 5217 /* SGX is never exposed to L1 */ 5218 return false; 5219 default: 5220 return true; 5221 } 5222 } 5223 5224 5225 static int vmx_get_nested_state(struct kvm_vcpu *vcpu, 5226 struct kvm_nested_state __user *user_kvm_nested_state, 5227 u32 user_data_size) 5228 { 5229 struct vcpu_vmx *vmx; 5230 struct vmcs12 *vmcs12; 5231 struct kvm_nested_state kvm_state = { 5232 .flags = 0, 5233 .format = 0, 5234 .size = sizeof(kvm_state), 5235 .vmx.vmxon_pa = -1ull, 5236 .vmx.vmcs_pa = -1ull, 5237 }; 5238 5239 if (!vcpu) 5240 return kvm_state.size + 2 * VMCS12_SIZE; 5241 5242 vmx = to_vmx(vcpu); 5243 vmcs12 = get_vmcs12(vcpu); 5244 5245 if (nested_vmx_allowed(vcpu) && vmx->nested.enlightened_vmcs_enabled) 5246 kvm_state.flags |= KVM_STATE_NESTED_EVMCS; 5247 5248 if (nested_vmx_allowed(vcpu) && 5249 (vmx->nested.vmxon || vmx->nested.smm.vmxon)) { 5250 kvm_state.vmx.vmxon_pa = vmx->nested.vmxon_ptr; 5251 kvm_state.vmx.vmcs_pa = vmx->nested.current_vmptr; 5252 5253 if (vmx_has_valid_vmcs12(vcpu)) { 5254 kvm_state.size += VMCS12_SIZE; 5255 5256 if (is_guest_mode(vcpu) && 5257 nested_cpu_has_shadow_vmcs(vmcs12) && 5258 vmcs12->vmcs_link_pointer != -1ull) 5259 kvm_state.size += VMCS12_SIZE; 5260 } 5261 5262 if (vmx->nested.smm.vmxon) 5263 kvm_state.vmx.smm.flags |= KVM_STATE_NESTED_SMM_VMXON; 5264 5265 if (vmx->nested.smm.guest_mode) 5266 kvm_state.vmx.smm.flags |= KVM_STATE_NESTED_SMM_GUEST_MODE; 5267 5268 if (is_guest_mode(vcpu)) { 5269 kvm_state.flags |= KVM_STATE_NESTED_GUEST_MODE; 5270 5271 if (vmx->nested.nested_run_pending) 5272 kvm_state.flags |= KVM_STATE_NESTED_RUN_PENDING; 5273 } 5274 } 5275 5276 if (user_data_size < kvm_state.size) 5277 goto out; 5278 5279 if (copy_to_user(user_kvm_nested_state, &kvm_state, sizeof(kvm_state))) 5280 return -EFAULT; 5281 5282 if (!vmx_has_valid_vmcs12(vcpu)) 5283 goto out; 5284 5285 /* 5286 * When running L2, the authoritative vmcs12 state is in the 5287 * vmcs02. When running L1, the authoritative vmcs12 state is 5288 * in the shadow or enlightened vmcs linked to vmcs01, unless 5289 * need_vmcs12_sync is set, in which case, the authoritative 5290 * vmcs12 state is in the vmcs12 already. 5291 */ 5292 if (is_guest_mode(vcpu)) { 5293 sync_vmcs12(vcpu, vmcs12); 5294 } else if (!vmx->nested.need_vmcs12_sync) { 5295 if (vmx->nested.hv_evmcs) 5296 copy_enlightened_to_vmcs12(vmx); 5297 else if (enable_shadow_vmcs) 5298 copy_shadow_to_vmcs12(vmx); 5299 } 5300 5301 /* 5302 * Copy over the full allocated size of vmcs12 rather than just the size 5303 * of the struct. 5304 */ 5305 if (copy_to_user(user_kvm_nested_state->data, vmcs12, VMCS12_SIZE)) 5306 return -EFAULT; 5307 5308 if (nested_cpu_has_shadow_vmcs(vmcs12) && 5309 vmcs12->vmcs_link_pointer != -1ull) { 5310 if (copy_to_user(user_kvm_nested_state->data + VMCS12_SIZE, 5311 get_shadow_vmcs12(vcpu), VMCS12_SIZE)) 5312 return -EFAULT; 5313 } 5314 5315 out: 5316 return kvm_state.size; 5317 } 5318 5319 /* 5320 * Forcibly leave nested mode in order to be able to reset the VCPU later on. 5321 */ 5322 void vmx_leave_nested(struct kvm_vcpu *vcpu) 5323 { 5324 if (is_guest_mode(vcpu)) { 5325 to_vmx(vcpu)->nested.nested_run_pending = 0; 5326 nested_vmx_vmexit(vcpu, -1, 0, 0); 5327 } 5328 free_nested(vcpu); 5329 } 5330 5331 static int vmx_set_nested_state(struct kvm_vcpu *vcpu, 5332 struct kvm_nested_state __user *user_kvm_nested_state, 5333 struct kvm_nested_state *kvm_state) 5334 { 5335 struct vcpu_vmx *vmx = to_vmx(vcpu); 5336 struct vmcs12 *vmcs12; 5337 u32 exit_qual; 5338 int ret; 5339 5340 if (kvm_state->format != 0) 5341 return -EINVAL; 5342 5343 if (kvm_state->flags & KVM_STATE_NESTED_EVMCS) 5344 nested_enable_evmcs(vcpu, NULL); 5345 5346 if (!nested_vmx_allowed(vcpu)) 5347 return kvm_state->vmx.vmxon_pa == -1ull ? 0 : -EINVAL; 5348 5349 if (kvm_state->vmx.vmxon_pa == -1ull) { 5350 if (kvm_state->vmx.smm.flags) 5351 return -EINVAL; 5352 5353 if (kvm_state->vmx.vmcs_pa != -1ull) 5354 return -EINVAL; 5355 5356 vmx_leave_nested(vcpu); 5357 return 0; 5358 } 5359 5360 if (!page_address_valid(vcpu, kvm_state->vmx.vmxon_pa)) 5361 return -EINVAL; 5362 5363 if ((kvm_state->vmx.smm.flags & KVM_STATE_NESTED_SMM_GUEST_MODE) && 5364 (kvm_state->flags & KVM_STATE_NESTED_GUEST_MODE)) 5365 return -EINVAL; 5366 5367 if (kvm_state->vmx.smm.flags & 5368 ~(KVM_STATE_NESTED_SMM_GUEST_MODE | KVM_STATE_NESTED_SMM_VMXON)) 5369 return -EINVAL; 5370 5371 /* 5372 * SMM temporarily disables VMX, so we cannot be in guest mode, 5373 * nor can VMLAUNCH/VMRESUME be pending. Outside SMM, SMM flags 5374 * must be zero. 5375 */ 5376 if (is_smm(vcpu) ? kvm_state->flags : kvm_state->vmx.smm.flags) 5377 return -EINVAL; 5378 5379 if ((kvm_state->vmx.smm.flags & KVM_STATE_NESTED_SMM_GUEST_MODE) && 5380 !(kvm_state->vmx.smm.flags & KVM_STATE_NESTED_SMM_VMXON)) 5381 return -EINVAL; 5382 5383 vmx_leave_nested(vcpu); 5384 if (kvm_state->vmx.vmxon_pa == -1ull) 5385 return 0; 5386 5387 vmx->nested.vmxon_ptr = kvm_state->vmx.vmxon_pa; 5388 ret = enter_vmx_operation(vcpu); 5389 if (ret) 5390 return ret; 5391 5392 /* Empty 'VMXON' state is permitted */ 5393 if (kvm_state->size < sizeof(kvm_state) + sizeof(*vmcs12)) 5394 return 0; 5395 5396 if (kvm_state->vmx.vmcs_pa != -1ull) { 5397 if (kvm_state->vmx.vmcs_pa == kvm_state->vmx.vmxon_pa || 5398 !page_address_valid(vcpu, kvm_state->vmx.vmcs_pa)) 5399 return -EINVAL; 5400 5401 set_current_vmptr(vmx, kvm_state->vmx.vmcs_pa); 5402 } else if (kvm_state->flags & KVM_STATE_NESTED_EVMCS) { 5403 /* 5404 * Sync eVMCS upon entry as we may not have 5405 * HV_X64_MSR_VP_ASSIST_PAGE set up yet. 5406 */ 5407 vmx->nested.need_vmcs12_sync = true; 5408 } else { 5409 return -EINVAL; 5410 } 5411 5412 if (kvm_state->vmx.smm.flags & KVM_STATE_NESTED_SMM_VMXON) { 5413 vmx->nested.smm.vmxon = true; 5414 vmx->nested.vmxon = false; 5415 5416 if (kvm_state->vmx.smm.flags & KVM_STATE_NESTED_SMM_GUEST_MODE) 5417 vmx->nested.smm.guest_mode = true; 5418 } 5419 5420 vmcs12 = get_vmcs12(vcpu); 5421 if (copy_from_user(vmcs12, user_kvm_nested_state->data, sizeof(*vmcs12))) 5422 return -EFAULT; 5423 5424 if (vmcs12->hdr.revision_id != VMCS12_REVISION) 5425 return -EINVAL; 5426 5427 if (!(kvm_state->flags & KVM_STATE_NESTED_GUEST_MODE)) 5428 return 0; 5429 5430 vmx->nested.nested_run_pending = 5431 !!(kvm_state->flags & KVM_STATE_NESTED_RUN_PENDING); 5432 5433 if (nested_cpu_has_shadow_vmcs(vmcs12) && 5434 vmcs12->vmcs_link_pointer != -1ull) { 5435 struct vmcs12 *shadow_vmcs12 = get_shadow_vmcs12(vcpu); 5436 5437 if (kvm_state->size < sizeof(kvm_state) + 2 * sizeof(*vmcs12)) 5438 return -EINVAL; 5439 5440 if (copy_from_user(shadow_vmcs12, 5441 user_kvm_nested_state->data + VMCS12_SIZE, 5442 sizeof(*vmcs12))) 5443 return -EFAULT; 5444 5445 if (shadow_vmcs12->hdr.revision_id != VMCS12_REVISION || 5446 !shadow_vmcs12->hdr.shadow_vmcs) 5447 return -EINVAL; 5448 } 5449 5450 if (nested_vmx_check_vmentry_prereqs(vcpu, vmcs12) || 5451 nested_vmx_check_vmentry_postreqs(vcpu, vmcs12, &exit_qual)) 5452 return -EINVAL; 5453 5454 vmx->nested.dirty_vmcs12 = true; 5455 ret = nested_vmx_enter_non_root_mode(vcpu, false); 5456 if (ret) 5457 return -EINVAL; 5458 5459 return 0; 5460 } 5461 5462 void nested_vmx_vcpu_setup(void) 5463 { 5464 if (enable_shadow_vmcs) { 5465 /* 5466 * At vCPU creation, "VMWRITE to any supported field 5467 * in the VMCS" is supported, so use the more 5468 * permissive vmx_vmread_bitmap to specify both read 5469 * and write permissions for the shadow VMCS. 5470 */ 5471 vmcs_write64(VMREAD_BITMAP, __pa(vmx_vmread_bitmap)); 5472 vmcs_write64(VMWRITE_BITMAP, __pa(vmx_vmread_bitmap)); 5473 } 5474 } 5475 5476 /* 5477 * nested_vmx_setup_ctls_msrs() sets up variables containing the values to be 5478 * returned for the various VMX controls MSRs when nested VMX is enabled. 5479 * The same values should also be used to verify that vmcs12 control fields are 5480 * valid during nested entry from L1 to L2. 5481 * Each of these control msrs has a low and high 32-bit half: A low bit is on 5482 * if the corresponding bit in the (32-bit) control field *must* be on, and a 5483 * bit in the high half is on if the corresponding bit in the control field 5484 * may be on. See also vmx_control_verify(). 5485 */ 5486 void nested_vmx_setup_ctls_msrs(struct nested_vmx_msrs *msrs, u32 ept_caps, 5487 bool apicv) 5488 { 5489 /* 5490 * Note that as a general rule, the high half of the MSRs (bits in 5491 * the control fields which may be 1) should be initialized by the 5492 * intersection of the underlying hardware's MSR (i.e., features which 5493 * can be supported) and the list of features we want to expose - 5494 * because they are known to be properly supported in our code. 5495 * Also, usually, the low half of the MSRs (bits which must be 1) can 5496 * be set to 0, meaning that L1 may turn off any of these bits. The 5497 * reason is that if one of these bits is necessary, it will appear 5498 * in vmcs01 and prepare_vmcs02, when it bitwise-or's the control 5499 * fields of vmcs01 and vmcs02, will turn these bits off - and 5500 * nested_vmx_exit_reflected() will not pass related exits to L1. 5501 * These rules have exceptions below. 5502 */ 5503 5504 /* pin-based controls */ 5505 rdmsr(MSR_IA32_VMX_PINBASED_CTLS, 5506 msrs->pinbased_ctls_low, 5507 msrs->pinbased_ctls_high); 5508 msrs->pinbased_ctls_low |= 5509 PIN_BASED_ALWAYSON_WITHOUT_TRUE_MSR; 5510 msrs->pinbased_ctls_high &= 5511 PIN_BASED_EXT_INTR_MASK | 5512 PIN_BASED_NMI_EXITING | 5513 PIN_BASED_VIRTUAL_NMIS | 5514 (apicv ? PIN_BASED_POSTED_INTR : 0); 5515 msrs->pinbased_ctls_high |= 5516 PIN_BASED_ALWAYSON_WITHOUT_TRUE_MSR | 5517 PIN_BASED_VMX_PREEMPTION_TIMER; 5518 5519 /* exit controls */ 5520 rdmsr(MSR_IA32_VMX_EXIT_CTLS, 5521 msrs->exit_ctls_low, 5522 msrs->exit_ctls_high); 5523 msrs->exit_ctls_low = 5524 VM_EXIT_ALWAYSON_WITHOUT_TRUE_MSR; 5525 5526 msrs->exit_ctls_high &= 5527 #ifdef CONFIG_X86_64 5528 VM_EXIT_HOST_ADDR_SPACE_SIZE | 5529 #endif 5530 VM_EXIT_LOAD_IA32_PAT | VM_EXIT_SAVE_IA32_PAT; 5531 msrs->exit_ctls_high |= 5532 VM_EXIT_ALWAYSON_WITHOUT_TRUE_MSR | 5533 VM_EXIT_LOAD_IA32_EFER | VM_EXIT_SAVE_IA32_EFER | 5534 VM_EXIT_SAVE_VMX_PREEMPTION_TIMER | VM_EXIT_ACK_INTR_ON_EXIT; 5535 5536 /* We support free control of debug control saving. */ 5537 msrs->exit_ctls_low &= ~VM_EXIT_SAVE_DEBUG_CONTROLS; 5538 5539 /* entry controls */ 5540 rdmsr(MSR_IA32_VMX_ENTRY_CTLS, 5541 msrs->entry_ctls_low, 5542 msrs->entry_ctls_high); 5543 msrs->entry_ctls_low = 5544 VM_ENTRY_ALWAYSON_WITHOUT_TRUE_MSR; 5545 msrs->entry_ctls_high &= 5546 #ifdef CONFIG_X86_64 5547 VM_ENTRY_IA32E_MODE | 5548 #endif 5549 VM_ENTRY_LOAD_IA32_PAT; 5550 msrs->entry_ctls_high |= 5551 (VM_ENTRY_ALWAYSON_WITHOUT_TRUE_MSR | VM_ENTRY_LOAD_IA32_EFER); 5552 5553 /* We support free control of debug control loading. */ 5554 msrs->entry_ctls_low &= ~VM_ENTRY_LOAD_DEBUG_CONTROLS; 5555 5556 /* cpu-based controls */ 5557 rdmsr(MSR_IA32_VMX_PROCBASED_CTLS, 5558 msrs->procbased_ctls_low, 5559 msrs->procbased_ctls_high); 5560 msrs->procbased_ctls_low = 5561 CPU_BASED_ALWAYSON_WITHOUT_TRUE_MSR; 5562 msrs->procbased_ctls_high &= 5563 CPU_BASED_VIRTUAL_INTR_PENDING | 5564 CPU_BASED_VIRTUAL_NMI_PENDING | CPU_BASED_USE_TSC_OFFSETING | 5565 CPU_BASED_HLT_EXITING | CPU_BASED_INVLPG_EXITING | 5566 CPU_BASED_MWAIT_EXITING | CPU_BASED_CR3_LOAD_EXITING | 5567 CPU_BASED_CR3_STORE_EXITING | 5568 #ifdef CONFIG_X86_64 5569 CPU_BASED_CR8_LOAD_EXITING | CPU_BASED_CR8_STORE_EXITING | 5570 #endif 5571 CPU_BASED_MOV_DR_EXITING | CPU_BASED_UNCOND_IO_EXITING | 5572 CPU_BASED_USE_IO_BITMAPS | CPU_BASED_MONITOR_TRAP_FLAG | 5573 CPU_BASED_MONITOR_EXITING | CPU_BASED_RDPMC_EXITING | 5574 CPU_BASED_RDTSC_EXITING | CPU_BASED_PAUSE_EXITING | 5575 CPU_BASED_TPR_SHADOW | CPU_BASED_ACTIVATE_SECONDARY_CONTROLS; 5576 /* 5577 * We can allow some features even when not supported by the 5578 * hardware. For example, L1 can specify an MSR bitmap - and we 5579 * can use it to avoid exits to L1 - even when L0 runs L2 5580 * without MSR bitmaps. 5581 */ 5582 msrs->procbased_ctls_high |= 5583 CPU_BASED_ALWAYSON_WITHOUT_TRUE_MSR | 5584 CPU_BASED_USE_MSR_BITMAPS; 5585 5586 /* We support free control of CR3 access interception. */ 5587 msrs->procbased_ctls_low &= 5588 ~(CPU_BASED_CR3_LOAD_EXITING | CPU_BASED_CR3_STORE_EXITING); 5589 5590 /* 5591 * secondary cpu-based controls. Do not include those that 5592 * depend on CPUID bits, they are added later by vmx_cpuid_update. 5593 */ 5594 if (msrs->procbased_ctls_high & CPU_BASED_ACTIVATE_SECONDARY_CONTROLS) 5595 rdmsr(MSR_IA32_VMX_PROCBASED_CTLS2, 5596 msrs->secondary_ctls_low, 5597 msrs->secondary_ctls_high); 5598 5599 msrs->secondary_ctls_low = 0; 5600 msrs->secondary_ctls_high &= 5601 SECONDARY_EXEC_DESC | 5602 SECONDARY_EXEC_VIRTUALIZE_X2APIC_MODE | 5603 SECONDARY_EXEC_APIC_REGISTER_VIRT | 5604 SECONDARY_EXEC_VIRTUAL_INTR_DELIVERY | 5605 SECONDARY_EXEC_WBINVD_EXITING; 5606 5607 /* 5608 * We can emulate "VMCS shadowing," even if the hardware 5609 * doesn't support it. 5610 */ 5611 msrs->secondary_ctls_high |= 5612 SECONDARY_EXEC_SHADOW_VMCS; 5613 5614 if (enable_ept) { 5615 /* nested EPT: emulate EPT also to L1 */ 5616 msrs->secondary_ctls_high |= 5617 SECONDARY_EXEC_ENABLE_EPT; 5618 msrs->ept_caps = VMX_EPT_PAGE_WALK_4_BIT | 5619 VMX_EPTP_WB_BIT | VMX_EPT_INVEPT_BIT; 5620 if (cpu_has_vmx_ept_execute_only()) 5621 msrs->ept_caps |= 5622 VMX_EPT_EXECUTE_ONLY_BIT; 5623 msrs->ept_caps &= ept_caps; 5624 msrs->ept_caps |= VMX_EPT_EXTENT_GLOBAL_BIT | 5625 VMX_EPT_EXTENT_CONTEXT_BIT | VMX_EPT_2MB_PAGE_BIT | 5626 VMX_EPT_1GB_PAGE_BIT; 5627 if (enable_ept_ad_bits) { 5628 msrs->secondary_ctls_high |= 5629 SECONDARY_EXEC_ENABLE_PML; 5630 msrs->ept_caps |= VMX_EPT_AD_BIT; 5631 } 5632 } 5633 5634 if (cpu_has_vmx_vmfunc()) { 5635 msrs->secondary_ctls_high |= 5636 SECONDARY_EXEC_ENABLE_VMFUNC; 5637 /* 5638 * Advertise EPTP switching unconditionally 5639 * since we emulate it 5640 */ 5641 if (enable_ept) 5642 msrs->vmfunc_controls = 5643 VMX_VMFUNC_EPTP_SWITCHING; 5644 } 5645 5646 /* 5647 * Old versions of KVM use the single-context version without 5648 * checking for support, so declare that it is supported even 5649 * though it is treated as global context. The alternative is 5650 * not failing the single-context invvpid, and it is worse. 5651 */ 5652 if (enable_vpid) { 5653 msrs->secondary_ctls_high |= 5654 SECONDARY_EXEC_ENABLE_VPID; 5655 msrs->vpid_caps = VMX_VPID_INVVPID_BIT | 5656 VMX_VPID_EXTENT_SUPPORTED_MASK; 5657 } 5658 5659 if (enable_unrestricted_guest) 5660 msrs->secondary_ctls_high |= 5661 SECONDARY_EXEC_UNRESTRICTED_GUEST; 5662 5663 if (flexpriority_enabled) 5664 msrs->secondary_ctls_high |= 5665 SECONDARY_EXEC_VIRTUALIZE_APIC_ACCESSES; 5666 5667 /* miscellaneous data */ 5668 rdmsr(MSR_IA32_VMX_MISC, 5669 msrs->misc_low, 5670 msrs->misc_high); 5671 msrs->misc_low &= VMX_MISC_SAVE_EFER_LMA; 5672 msrs->misc_low |= 5673 MSR_IA32_VMX_MISC_VMWRITE_SHADOW_RO_FIELDS | 5674 VMX_MISC_EMULATED_PREEMPTION_TIMER_RATE | 5675 VMX_MISC_ACTIVITY_HLT; 5676 msrs->misc_high = 0; 5677 5678 /* 5679 * This MSR reports some information about VMX support. We 5680 * should return information about the VMX we emulate for the 5681 * guest, and the VMCS structure we give it - not about the 5682 * VMX support of the underlying hardware. 5683 */ 5684 msrs->basic = 5685 VMCS12_REVISION | 5686 VMX_BASIC_TRUE_CTLS | 5687 ((u64)VMCS12_SIZE << VMX_BASIC_VMCS_SIZE_SHIFT) | 5688 (VMX_BASIC_MEM_TYPE_WB << VMX_BASIC_MEM_TYPE_SHIFT); 5689 5690 if (cpu_has_vmx_basic_inout()) 5691 msrs->basic |= VMX_BASIC_INOUT; 5692 5693 /* 5694 * These MSRs specify bits which the guest must keep fixed on 5695 * while L1 is in VMXON mode (in L1's root mode, or running an L2). 5696 * We picked the standard core2 setting. 5697 */ 5698 #define VMXON_CR0_ALWAYSON (X86_CR0_PE | X86_CR0_PG | X86_CR0_NE) 5699 #define VMXON_CR4_ALWAYSON X86_CR4_VMXE 5700 msrs->cr0_fixed0 = VMXON_CR0_ALWAYSON; 5701 msrs->cr4_fixed0 = VMXON_CR4_ALWAYSON; 5702 5703 /* These MSRs specify bits which the guest must keep fixed off. */ 5704 rdmsrl(MSR_IA32_VMX_CR0_FIXED1, msrs->cr0_fixed1); 5705 rdmsrl(MSR_IA32_VMX_CR4_FIXED1, msrs->cr4_fixed1); 5706 5707 /* highest index: VMX_PREEMPTION_TIMER_VALUE */ 5708 msrs->vmcs_enum = VMCS12_MAX_FIELD_INDEX << 1; 5709 } 5710 5711 void nested_vmx_hardware_unsetup(void) 5712 { 5713 int i; 5714 5715 if (enable_shadow_vmcs) { 5716 for (i = 0; i < VMX_BITMAP_NR; i++) 5717 free_page((unsigned long)vmx_bitmap[i]); 5718 } 5719 } 5720 5721 __init int nested_vmx_hardware_setup(int (*exit_handlers[])(struct kvm_vcpu *)) 5722 { 5723 int i; 5724 5725 if (!cpu_has_vmx_shadow_vmcs()) 5726 enable_shadow_vmcs = 0; 5727 if (enable_shadow_vmcs) { 5728 for (i = 0; i < VMX_BITMAP_NR; i++) { 5729 /* 5730 * The vmx_bitmap is not tied to a VM and so should 5731 * not be charged to a memcg. 5732 */ 5733 vmx_bitmap[i] = (unsigned long *) 5734 __get_free_page(GFP_KERNEL); 5735 if (!vmx_bitmap[i]) { 5736 nested_vmx_hardware_unsetup(); 5737 return -ENOMEM; 5738 } 5739 } 5740 5741 init_vmcs_shadow_fields(); 5742 } 5743 5744 exit_handlers[EXIT_REASON_VMCLEAR] = handle_vmclear, 5745 exit_handlers[EXIT_REASON_VMLAUNCH] = handle_vmlaunch, 5746 exit_handlers[EXIT_REASON_VMPTRLD] = handle_vmptrld, 5747 exit_handlers[EXIT_REASON_VMPTRST] = handle_vmptrst, 5748 exit_handlers[EXIT_REASON_VMREAD] = handle_vmread, 5749 exit_handlers[EXIT_REASON_VMRESUME] = handle_vmresume, 5750 exit_handlers[EXIT_REASON_VMWRITE] = handle_vmwrite, 5751 exit_handlers[EXIT_REASON_VMOFF] = handle_vmoff, 5752 exit_handlers[EXIT_REASON_VMON] = handle_vmon, 5753 exit_handlers[EXIT_REASON_INVEPT] = handle_invept, 5754 exit_handlers[EXIT_REASON_INVVPID] = handle_invvpid, 5755 exit_handlers[EXIT_REASON_VMFUNC] = handle_vmfunc, 5756 5757 kvm_x86_ops->check_nested_events = vmx_check_nested_events; 5758 kvm_x86_ops->get_nested_state = vmx_get_nested_state; 5759 kvm_x86_ops->set_nested_state = vmx_set_nested_state; 5760 kvm_x86_ops->get_vmcs12_pages = nested_get_vmcs12_pages, 5761 kvm_x86_ops->nested_enable_evmcs = nested_enable_evmcs; 5762 kvm_x86_ops->nested_get_evmcs_version = nested_get_evmcs_version; 5763 5764 return 0; 5765 } 5766